2024-09-21T09:11:59.831 INFO:root:teuthology version: 1.2.2.dev2+g3752d38 2024-09-21T09:11:59.856 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-09-21T09:11:59.942 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2024-09-21_03:08:26-upgrade-main-distro-default-smithi/7913638 branch: main description: upgrade/quincy-x/parallel/{0-random-distro$/{centos_9.stream_runc} 0-start 1-tasks mon_election/classic upgrade-sequence workload/{ec-rados-default rados_api rados_loadgenbig rbd_import_export test_rbd_api test_rbd_python}} email: ceph-qa@ceph.com first_in_suite: false job_id: '7913638' kernel: kdb: 1 sha1: distro last_in_suite: false machine_type: smithi meta: - desc: 'run run randomized correctness test for rados operations on an erasure-coded pool ' - desc: 'object class functional tests ' - desc: 'generate read/write load with rados objects ranging from 1MB to 25MB ' - desc: 'run basic import/export cli tests for rbd ' - desc: 'librbd C and C++ api tests ' - desc: 'librbd python api tests ' name: teuthology-2024-09-21_03:08:26-upgrade-main-distro-default-smithi no_nested_subset: false openstack: - volumes: count: 4 size: 10 os_type: centos os_version: 9.stream overrides: admin_socket: branch: main ceph: conf: global: mon election default strategy: 1 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true create_rbd_pool: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) - \(POOL_APP_NOT_ENABLED\) - OSD_DOWN - mons down - mon down - MON_DOWN - out of quorum - PG_DEGRADED - Reduced data availability - Degraded data redundancy - OSDMAP_FLAGS - mons down - mon down - MON_DOWN - out of quorum - PG_AVAILABILITY sha1: 62da54b4b83a96ca9976b7e1d0c8272a564a2208 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: extra_system_packages: - python3-pytest flavor: default sha1: 62da54b4b83a96ca9976b7e1d0c8272a564a2208 selinux: allowlist: - scontext=system_u:system_r:logrotate_t:s0 workunit: branch: main sha1: 62da54b4b83a96ca9976b7e1d0c8272a564a2208 owner: scheduled_teuthology@teuthology priority: 850 repo: https://git.ceph.com/ceph.git roles: - - mon.a - mon.c - mgr.y - mds.a - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mds.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b seed: 5357 sha1: 62da54b4b83a96ca9976b7e1d0c8272a564a2208 sleep_before_teardown: 0 subset: 31/32 suite: upgrade suite_branch: main suite_path: /home/teuthworker/src/git.ceph.com_ceph_62da54b4b83a96ca9976b7e1d0c8272a564a2208/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph.git suite_sha1: 62da54b4b83a96ca9976b7e1d0c8272a564a2208 targets: smithi022.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBKYcUCbbyEaKnOwl/+nwGd4iArcVn4/CqDDpnZwvsY+QwCE832B6p56gNwTmpFaREPVAj92nSIOI69GjIE8b6Jo= smithi064.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBME/Imej8nYEvmCea/ftGbkISzupQcpxBjsMyJZA9KTZEGuQhLbxv3/0MK+rO55r9BKtEx/+VdyvwFwo8aTSoyk= tasks: - pexec: all: - sudo dnf remove nvme-cli -y - sudo dnf install runc nvmetcli nvme-cli -y - sudo sed -i 's/^#runtime = "crun"/runtime = "runc"/g' /usr/share/containers/containers.conf - sudo sed -i 's/runtime = "crun"/#runtime = "crun"/g' /usr/share/containers/containers.conf - install: branch: quincy exclude_packages: - ceph-volume - print: '**** done install task...' - print: '**** done start installing quincy cephadm ...' - cephadm: cephadm_branch: quincy cephadm_git_url: https://github.com/ceph/ceph conf: osd: osd_class_default_list: '*' osd_class_load_list: '*' image: quay.ceph.io/ceph-ci/ceph:quincy - print: '**** done end installing quincy cephadm ...' - print: '**** done start cephadm.shell ceph config set mgr...' - cephadm.shell: mon.a: - ceph config set mgr mgr/cephadm/use_repo_digest true --force - print: '**** done cephadm.shell ceph config set mgr...' - print: '**** done start telemetry quincy...' - workunit: clients: client.0: - test_telemetry_quincy.sh - print: '**** done end telemetry quincy...' - print: '**** done start parallel' - parallel: - workload - upgrade-sequence - print: '**** done end parallel' - print: '**** done start telemetry x...' - workunit: clients: client.0: - test_telemetry_quincy_x.sh - print: '**** done end telemetry x...' teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a timestamp: 2024-09-21_03:08:26 tube: smithi upgrade-sequence: sequential: - print: '**** done start upgrade, wait' - cephadm.shell: env: - sha1 mon.a: - ceph config set global log_to_journald false --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 - print: '**** done end upgrade, wait...' user: teuthology verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2902 workload: full_sequential: - print: '**** done start ec-rados-default.yaml' - rados: clients: - client.0 ec_pool: true objects: 50 op_weights: append: 100 copy_from: 50 delete: 50 read: 100 rmattr: 25 rollback: 50 setattr: 25 snap_create: 50 snap_remove: 50 write: 0 ops: 4000 write_append_excl: false - print: '**** done end ec-rados-default.yaml' - print: '**** done start rados_api.yaml' - workunit: branch: quincy clients: client.0: - cls - print: '**** done end rados_api.yaml' - print: '**** done start rados_loadgenbig.yaml' - workunit: branch: quincy clients: client.0: - rados/load-gen-big.sh - print: '**** done end rados_loadgenbig.yaml' - print: '**** done start rbd_import_export.yaml' - workunit: branch: quincy clients: client.1: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done end rbd_import_export.yaml' - print: '**** done start test_rbd_api.yaml' - workunit: branch: quincy clients: client.0: - rbd/test_librbd.sh env: RBD_FEATURES: '61' - print: '**** done end test_rbd_api.yaml' - print: '**** done start test_rbd_python.yaml' - workunit: branch: quincy clients: client.0: - rbd/test_librbd_python.sh env: RBD_FEATURES: '61' - print: '**** done end test_rbd_python.yaml' 2024-09-21T09:11:59.943 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph_62da54b4b83a96ca9976b7e1d0c8272a564a2208/qa; will attempt to use it 2024-09-21T09:11:59.943 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph_62da54b4b83a96ca9976b7e1d0c8272a564a2208/qa/tasks 2024-09-21T09:11:59.943 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-09-21T09:11:59.945 INFO:teuthology.task.internal:Checking packages... 2024-09-21T09:11:59.965 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash '62da54b4b83a96ca9976b7e1d0c8272a564a2208' 2024-09-21T09:11:59.966 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-21T09:11:59.966 INFO:teuthology.packaging:ref: None 2024-09-21T09:11:59.966 INFO:teuthology.packaging:tag: None 2024-09-21T09:11:59.966 INFO:teuthology.packaging:branch: main 2024-09-21T09:11:59.966 INFO:teuthology.packaging:sha1: 62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:11:59.966 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=main 2024-09-21T09:12:00.211 INFO:teuthology.task.internal:Found packages for ceph version 19.3.0-5092.g62da54b4 2024-09-21T09:12:00.212 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-09-21T09:12:00.218 INFO:teuthology.task.internal:no buildpackages task found 2024-09-21T09:12:00.218 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-09-21T09:12:00.223 INFO:teuthology.task.internal:Saving configuration 2024-09-21T09:12:00.241 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-09-21T09:12:00.247 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-09-21T09:12:00.271 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi022.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-09-21_03:08:26-upgrade-main-distro-default-smithi/7913638', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-09-21 09:07:03.188466', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBKYcUCbbyEaKnOwl/+nwGd4iArcVn4/CqDDpnZwvsY+QwCE832B6p56gNwTmpFaREPVAj92nSIOI69GjIE8b6Jo='} 2024-09-21T09:12:00.299 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi064.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-09-21_03:08:26-upgrade-main-distro-default-smithi/7913638', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-09-21 09:07:03.190411', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBME/Imej8nYEvmCea/ftGbkISzupQcpxBjsMyJZA9KTZEGuQhLbxv3/0MK+rO55r9BKtEx/+VdyvwFwo8aTSoyk='} 2024-09-21T09:12:00.299 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-09-21T09:12:00.305 INFO:teuthology.task.internal:roles: ubuntu@smithi022.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'mds.a', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2024-09-21T09:12:00.305 INFO:teuthology.task.internal:roles: ubuntu@smithi064.front.sepia.ceph.com - ['mon.b', 'mds.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2024-09-21T09:12:00.305 INFO:teuthology.run_tasks:Running task console_log... 2024-09-21T09:12:00.360 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f185cb04160>, signals=[15]) 2024-09-21T09:12:00.360 INFO:teuthology.run_tasks:Running task internal.connect... 2024-09-21T09:12:00.365 INFO:teuthology.task.internal:Opening connections... 2024-09-21T09:12:00.365 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi022.front.sepia.ceph.com 2024-09-21T09:12:00.366 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi022.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-21T09:12:00.442 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi064.front.sepia.ceph.com 2024-09-21T09:12:00.443 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi064.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-21T09:12:00.516 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-09-21T09:12:00.521 DEBUG:teuthology.orchestra.run.smithi022:> uname -m 2024-09-21T09:12:00.540 INFO:teuthology.orchestra.run.smithi022.stdout:x86_64 2024-09-21T09:12:00.540 DEBUG:teuthology.orchestra.run.smithi022:> cat /etc/os-release 2024-09-21T09:12:00.597 INFO:teuthology.orchestra.run.smithi022.stdout:NAME="CentOS Stream" 2024-09-21T09:12:00.597 INFO:teuthology.orchestra.run.smithi022.stdout:VERSION="9" 2024-09-21T09:12:00.597 INFO:teuthology.orchestra.run.smithi022.stdout:ID="centos" 2024-09-21T09:12:00.597 INFO:teuthology.orchestra.run.smithi022.stdout:ID_LIKE="rhel fedora" 2024-09-21T09:12:00.597 INFO:teuthology.orchestra.run.smithi022.stdout:VERSION_ID="9" 2024-09-21T09:12:00.597 INFO:teuthology.orchestra.run.smithi022.stdout:PLATFORM_ID="platform:el9" 2024-09-21T09:12:00.597 INFO:teuthology.orchestra.run.smithi022.stdout:PRETTY_NAME="CentOS Stream 9" 2024-09-21T09:12:00.597 INFO:teuthology.orchestra.run.smithi022.stdout:ANSI_COLOR="0;31" 2024-09-21T09:12:00.597 INFO:teuthology.orchestra.run.smithi022.stdout:LOGO="fedora-logo-icon" 2024-09-21T09:12:00.597 INFO:teuthology.orchestra.run.smithi022.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-09-21T09:12:00.597 INFO:teuthology.orchestra.run.smithi022.stdout:HOME_URL="https://centos.org/" 2024-09-21T09:12:00.598 INFO:teuthology.orchestra.run.smithi022.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2024-09-21T09:12:00.598 INFO:teuthology.orchestra.run.smithi022.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-09-21T09:12:00.598 INFO:teuthology.orchestra.run.smithi022.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-09-21T09:12:00.598 INFO:teuthology.lock.ops:Updating smithi022.front.sepia.ceph.com on lock server 2024-09-21T09:12:00.620 DEBUG:teuthology.orchestra.run.smithi064:> uname -m 2024-09-21T09:12:00.639 INFO:teuthology.orchestra.run.smithi064.stdout:x86_64 2024-09-21T09:12:00.639 DEBUG:teuthology.orchestra.run.smithi064:> cat /etc/os-release 2024-09-21T09:12:00.697 INFO:teuthology.orchestra.run.smithi064.stdout:NAME="CentOS Stream" 2024-09-21T09:12:00.697 INFO:teuthology.orchestra.run.smithi064.stdout:VERSION="9" 2024-09-21T09:12:00.697 INFO:teuthology.orchestra.run.smithi064.stdout:ID="centos" 2024-09-21T09:12:00.698 INFO:teuthology.orchestra.run.smithi064.stdout:ID_LIKE="rhel fedora" 2024-09-21T09:12:00.698 INFO:teuthology.orchestra.run.smithi064.stdout:VERSION_ID="9" 2024-09-21T09:12:00.698 INFO:teuthology.orchestra.run.smithi064.stdout:PLATFORM_ID="platform:el9" 2024-09-21T09:12:00.698 INFO:teuthology.orchestra.run.smithi064.stdout:PRETTY_NAME="CentOS Stream 9" 2024-09-21T09:12:00.698 INFO:teuthology.orchestra.run.smithi064.stdout:ANSI_COLOR="0;31" 2024-09-21T09:12:00.698 INFO:teuthology.orchestra.run.smithi064.stdout:LOGO="fedora-logo-icon" 2024-09-21T09:12:00.698 INFO:teuthology.orchestra.run.smithi064.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-09-21T09:12:00.698 INFO:teuthology.orchestra.run.smithi064.stdout:HOME_URL="https://centos.org/" 2024-09-21T09:12:00.698 INFO:teuthology.orchestra.run.smithi064.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2024-09-21T09:12:00.698 INFO:teuthology.orchestra.run.smithi064.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-09-21T09:12:00.698 INFO:teuthology.orchestra.run.smithi064.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-09-21T09:12:00.699 INFO:teuthology.lock.ops:Updating smithi064.front.sepia.ceph.com on lock server 2024-09-21T09:12:00.719 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-09-21T09:12:00.731 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-09-21T09:12:00.810 INFO:teuthology.task.internal:Checking for old test directory... 2024-09-21T09:12:00.810 DEBUG:teuthology.orchestra.run.smithi022:> test '!' -e /home/ubuntu/cephtest 2024-09-21T09:12:00.813 DEBUG:teuthology.orchestra.run.smithi064:> test '!' -e /home/ubuntu/cephtest 2024-09-21T09:12:00.829 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-09-21T09:12:00.834 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-09-21T09:12:00.834 DEBUG:teuthology.orchestra.run.smithi022:> test -z $(ls -A /var/lib/ceph) 2024-09-21T09:12:00.870 DEBUG:teuthology.orchestra.run.smithi064:> test -z $(ls -A /var/lib/ceph) 2024-09-21T09:12:00.895 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-09-21T09:12:00.948 INFO:teuthology.run_tasks:Running task kernel... 2024-09-21T09:12:00.962 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-09-21T09:12:00.962 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': 1, 'sha1': 'distro'}, 'mon.b': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-09-21T09:12:00.962 DEBUG:teuthology.orchestra.run.smithi022:> test -f /run/.containerenv -o -f /.dockerenv 2024-09-21T09:12:00.963 DEBUG:teuthology.orchestra.run.smithi064:> test -f /run/.containerenv -o -f /.dockerenv 2024-09-21T09:12:00.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:12:00.979 DEBUG:teuthology.orchestra.run.smithi064:> uname -r 2024-09-21T09:12:00.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:12:00.980 DEBUG:teuthology.orchestra.run.smithi022:> uname -r 2024-09-21T09:12:01.037 INFO:teuthology.orchestra.run.smithi064.stdout:5.14.0-503.el9.x86_64 2024-09-21T09:12:01.037 INFO:teuthology.task.kernel:Running kernel on smithi064: 5.14.0-503.el9.x86_64 2024-09-21T09:12:01.038 DEBUG:teuthology.orchestra.run.smithi064:> sudo yum install -y kernel 2024-09-21T09:12:01.039 INFO:teuthology.orchestra.run.smithi022.stdout:5.14.0-503.el9.x86_64 2024-09-21T09:12:01.039 INFO:teuthology.task.kernel:Running kernel on smithi022: 5.14.0-503.el9.x86_64 2024-09-21T09:12:01.039 DEBUG:teuthology.orchestra.run.smithi022:> sudo yum install -y kernel 2024-09-21T09:12:04.711 INFO:teuthology.orchestra.run.smithi064.stdout:CentOS Stream 9 - BaseOS 4.2 MB/s | 8.3 MB 00:01 2024-09-21T09:12:04.971 INFO:teuthology.orchestra.run.smithi022.stdout:CentOS Stream 9 - BaseOS 3.7 MB/s | 8.3 MB 00:02 2024-09-21T09:12:07.753 INFO:teuthology.orchestra.run.smithi064.stdout:CentOS Stream 9 - AppStream 11 MB/s | 20 MB 00:01 2024-09-21T09:12:08.064 INFO:teuthology.orchestra.run.smithi022.stdout:CentOS Stream 9 - AppStream 10 MB/s | 20 MB 00:01 2024-09-21T09:12:12.312 INFO:teuthology.orchestra.run.smithi064.stdout:CentOS Stream 9 - CRB 11 MB/s | 6.5 MB 00:00 2024-09-21T09:12:12.856 INFO:teuthology.orchestra.run.smithi022.stdout:CentOS Stream 9 - CRB 7.4 MB/s | 6.5 MB 00:00 2024-09-21T09:12:14.054 INFO:teuthology.orchestra.run.smithi064.stdout:CentOS Stream 9 - Extras packages 41 kB/s | 19 kB 00:00 2024-09-21T09:12:14.573 INFO:teuthology.orchestra.run.smithi022.stdout:CentOS Stream 9 - Extras packages 43 kB/s | 19 kB 00:00 2024-09-21T09:12:15.266 INFO:teuthology.orchestra.run.smithi064.stdout:Extra Packages for Enterprise Linux 21 MB/s | 23 MB 00:01 2024-09-21T09:12:15.536 INFO:teuthology.orchestra.run.smithi022.stdout:Extra Packages for Enterprise Linux 27 MB/s | 23 MB 00:00 2024-09-21T09:12:20.939 INFO:teuthology.orchestra.run.smithi064.stdout:lab-extras 34 kB/s | 1.7 kB 00:00 2024-09-21T09:12:21.167 INFO:teuthology.orchestra.run.smithi022.stdout:lab-extras 39 kB/s | 1.7 kB 00:00 2024-09-21T09:12:22.887 INFO:teuthology.orchestra.run.smithi064.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-09-21T09:12:22.887 INFO:teuthology.orchestra.run.smithi064.stdout:Package kernel-5.14.0-503.el9.x86_64 is already installed. 2024-09-21T09:12:22.983 INFO:teuthology.orchestra.run.smithi064.stdout:Dependencies resolved. 2024-09-21T09:12:22.991 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2024-09-21T09:12:22.991 INFO:teuthology.orchestra.run.smithi064.stdout: Package Architecture Version Repository Size 2024-09-21T09:12:22.991 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2024-09-21T09:12:22.991 INFO:teuthology.orchestra.run.smithi064.stdout:Installing: 2024-09-21T09:12:22.991 INFO:teuthology.orchestra.run.smithi064.stdout: kernel x86_64 5.14.0-508.el9 baseos 36 k 2024-09-21T09:12:22.991 INFO:teuthology.orchestra.run.smithi064.stdout:Installing dependencies: 2024-09-21T09:12:22.991 INFO:teuthology.orchestra.run.smithi064.stdout: kernel-core x86_64 5.14.0-508.el9 baseos 16 M 2024-09-21T09:12:22.992 INFO:teuthology.orchestra.run.smithi064.stdout: kernel-modules x86_64 5.14.0-508.el9 baseos 35 M 2024-09-21T09:12:22.992 INFO:teuthology.orchestra.run.smithi064.stdout: kernel-modules-core x86_64 5.14.0-508.el9 baseos 29 M 2024-09-21T09:12:22.992 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-09-21T09:12:22.992 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction Summary 2024-09-21T09:12:22.992 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2024-09-21T09:12:22.992 INFO:teuthology.orchestra.run.smithi064.stdout:Install 4 Packages 2024-09-21T09:12:22.992 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-09-21T09:12:22.992 INFO:teuthology.orchestra.run.smithi064.stdout:Total download size: 79 M 2024-09-21T09:12:22.993 INFO:teuthology.orchestra.run.smithi064.stdout:Installed size: 126 M 2024-09-21T09:12:22.993 INFO:teuthology.orchestra.run.smithi064.stdout:Downloading Packages: 2024-09-21T09:12:23.132 INFO:teuthology.orchestra.run.smithi064.stdout:(1/4): kernel-5.14.0-508.el9.x86_64.rpm 410 kB/s | 36 kB 00:00 2024-09-21T09:12:23.265 INFO:teuthology.orchestra.run.smithi022.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-09-21T09:12:23.265 INFO:teuthology.orchestra.run.smithi022.stdout:Package kernel-5.14.0-503.el9.x86_64 is already installed. 2024-09-21T09:12:23.372 INFO:teuthology.orchestra.run.smithi022.stdout:Dependencies resolved. 2024-09-21T09:12:23.382 INFO:teuthology.orchestra.run.smithi022.stdout:================================================================================ 2024-09-21T09:12:23.382 INFO:teuthology.orchestra.run.smithi022.stdout: Package Architecture Version Repository Size 2024-09-21T09:12:23.382 INFO:teuthology.orchestra.run.smithi022.stdout:================================================================================ 2024-09-21T09:12:23.382 INFO:teuthology.orchestra.run.smithi022.stdout:Installing: 2024-09-21T09:12:23.382 INFO:teuthology.orchestra.run.smithi022.stdout: kernel x86_64 5.14.0-508.el9 baseos 36 k 2024-09-21T09:12:23.383 INFO:teuthology.orchestra.run.smithi022.stdout:Installing dependencies: 2024-09-21T09:12:23.383 INFO:teuthology.orchestra.run.smithi022.stdout: kernel-core x86_64 5.14.0-508.el9 baseos 16 M 2024-09-21T09:12:23.383 INFO:teuthology.orchestra.run.smithi022.stdout: kernel-modules x86_64 5.14.0-508.el9 baseos 35 M 2024-09-21T09:12:23.383 INFO:teuthology.orchestra.run.smithi022.stdout: kernel-modules-core x86_64 5.14.0-508.el9 baseos 29 M 2024-09-21T09:12:23.383 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-09-21T09:12:23.383 INFO:teuthology.orchestra.run.smithi022.stdout:Transaction Summary 2024-09-21T09:12:23.383 INFO:teuthology.orchestra.run.smithi022.stdout:================================================================================ 2024-09-21T09:12:23.383 INFO:teuthology.orchestra.run.smithi022.stdout:Install 4 Packages 2024-09-21T09:12:23.383 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-09-21T09:12:23.384 INFO:teuthology.orchestra.run.smithi022.stdout:Total download size: 79 M 2024-09-21T09:12:23.384 INFO:teuthology.orchestra.run.smithi022.stdout:Installed size: 126 M 2024-09-21T09:12:23.384 INFO:teuthology.orchestra.run.smithi022.stdout:Downloading Packages: 2024-09-21T09:12:23.524 INFO:teuthology.orchestra.run.smithi064.stdout:(2/4): kernel-core-5.14.0-508.el9.x86_64.rpm 33 MB/s | 16 MB 00:00 2024-09-21T09:12:23.593 INFO:teuthology.orchestra.run.smithi022.stdout:(1/4): kernel-5.14.0-508.el9.x86_64.rpm 428 kB/s | 36 kB 00:00 2024-09-21T09:12:24.091 INFO:teuthology.orchestra.run.smithi064.stdout:(3/4): kernel-modules-5.14.0-508.el9.x86_64.rpm 33 MB/s | 35 MB 00:01 2024-09-21T09:12:24.375 INFO:teuthology.orchestra.run.smithi064.stdout:(4/4): kernel-modules-core-5.14.0-508.el9.x86_6 23 MB/s | 29 MB 00:01 2024-09-21T09:12:24.375 INFO:teuthology.orchestra.run.smithi064.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:12:24.375 INFO:teuthology.orchestra.run.smithi064.stdout:Total 57 MB/s | 79 MB 00:01 2024-09-21T09:12:25.085 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction check 2024-09-21T09:12:25.458 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction check succeeded. 2024-09-21T09:12:25.459 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction test 2024-09-21T09:12:26.463 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction test succeeded. 2024-09-21T09:12:26.464 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction 2024-09-21T09:12:27.476 INFO:teuthology.orchestra.run.smithi022.stdout:(2/4): kernel-core-5.14.0-508.el9.x86_64.rpm 3.9 MB/s | 16 MB 00:03 2024-09-21T09:12:29.019 INFO:teuthology.orchestra.run.smithi064.stdout: Preparing : 1/1 2024-09-21T09:12:29.678 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : kernel-modules-core-5.14.0-508.el9.x86_64 1/4 2024-09-21T09:12:29.758 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-21T09:12:30.759 INFO:teuthology.orchestra.run.smithi022.stdout:(3/4): kernel-modules-core-5.14.0-508.el9.x86_6 4.0 MB/s | 29 MB 00:07 2024-09-21T09:12:32.260 INFO:teuthology.orchestra.run.smithi022.stdout:(4/4): kernel-modules-5.14.0-508.el9.x86_64.rpm 3.9 MB/s | 35 MB 00:08 2024-09-21T09:12:32.260 INFO:teuthology.orchestra.run.smithi022.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:12:32.260 INFO:teuthology.orchestra.run.smithi022.stdout:Total 8.9 MB/s | 79 MB 00:08 2024-09-21T09:12:32.535 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-21T09:12:32.659 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-21T09:12:32.976 INFO:teuthology.orchestra.run.smithi022.stdout:Running transaction check 2024-09-21T09:12:33.365 INFO:teuthology.orchestra.run.smithi022.stdout:Transaction check succeeded. 2024-09-21T09:12:33.365 INFO:teuthology.orchestra.run.smithi022.stdout:Running transaction test 2024-09-21T09:12:34.358 INFO:teuthology.orchestra.run.smithi022.stdout:Transaction test succeeded. 2024-09-21T09:12:34.358 INFO:teuthology.orchestra.run.smithi022.stdout:Running transaction 2024-09-21T09:12:36.933 INFO:teuthology.orchestra.run.smithi022.stdout: Preparing : 1/1 2024-09-21T09:12:37.637 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : kernel-modules-core-5.14.0-508.el9.x86_64 1/4 2024-09-21T09:12:37.719 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-21T09:12:37.783 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-21T09:12:37.826 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : kernel-5.14.0-508.el9.x86_64 4/4 2024-09-21T09:12:40.485 INFO:teuthology.orchestra.run.smithi022.stdout: Running scriptlet: kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-21T09:12:40.614 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-21T09:12:42.960 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: kernel-modules-core-5.14.0-508.el9.x86_64 4/4 2024-09-21T09:12:45.735 INFO:teuthology.orchestra.run.smithi022.stdout: Running scriptlet: kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-21T09:12:45.773 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : kernel-5.14.0-508.el9.x86_64 4/4 2024-09-21T09:12:50.914 INFO:teuthology.orchestra.run.smithi022.stdout: Running scriptlet: kernel-modules-core-5.14.0-508.el9.x86_64 4/4 2024-09-21T09:13:22.116 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: kernel-core-5.14.0-508.el9.x86_64 4/4 2024-09-21T09:13:22.158 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: kernel-modules-5.14.0-508.el9.x86_64 4/4 2024-09-21T09:13:22.558 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: kernel-5.14.0-508.el9.x86_64 4/4 2024-09-21T09:13:22.558 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : kernel-5.14.0-508.el9.x86_64 1/4 2024-09-21T09:13:22.559 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-21T09:13:22.559 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-21T09:13:23.390 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : kernel-modules-core-5.14.0-508.el9.x86_64 4/4 2024-09-21T09:13:23.390 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-09-21T09:13:23.390 INFO:teuthology.orchestra.run.smithi064.stdout:Installed: 2024-09-21T09:13:23.390 INFO:teuthology.orchestra.run.smithi064.stdout: kernel-5.14.0-508.el9.x86_64 2024-09-21T09:13:23.390 INFO:teuthology.orchestra.run.smithi064.stdout: kernel-core-5.14.0-508.el9.x86_64 2024-09-21T09:13:23.390 INFO:teuthology.orchestra.run.smithi064.stdout: kernel-modules-5.14.0-508.el9.x86_64 2024-09-21T09:13:23.390 INFO:teuthology.orchestra.run.smithi064.stdout: kernel-modules-core-5.14.0-508.el9.x86_64 2024-09-21T09:13:23.390 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-09-21T09:13:23.390 INFO:teuthology.orchestra.run.smithi064.stdout:Complete! 2024-09-21T09:13:23.749 DEBUG:teuthology.orchestra.run.smithi064:> rpm -q kernel | sort -rV | head -n 1 2024-09-21T09:13:24.006 INFO:teuthology.orchestra.run.smithi064.stdout:kernel-5.14.0-508.el9.x86_64 2024-09-21T09:13:24.006 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-508.el9.x86_64 2024-09-21T09:13:24.006 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-503.el9.x86_64 Expected: 5.14.0-508.el9.x86_64 2024-09-21T09:13:24.007 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-09-21T09:13:24.007 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-09-21T09:13:24.007 INFO:teuthology.task.kernel:Installing distro kernel on mon.b... 2024-09-21T09:13:24.007 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi064.front.sepia.ceph.com, path=None, version=distro) 2024-09-21T09:13:24.007 DEBUG:teuthology.orchestra.run.smithi064:> rpm -q kernel | sort -rV | head -n 1 2024-09-21T09:13:24.065 INFO:teuthology.orchestra.run.smithi064.stdout:kernel-5.14.0-508.el9.x86_64 2024-09-21T09:13:24.066 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-508.el9.x86_64 2024-09-21T09:13:24.066 DEBUG:teuthology.orchestra.run.smithi064:> sudo rpm -qi grub2-tools 2024-09-21T09:13:24.105 INFO:teuthology.orchestra.run.smithi064.stdout:Name : grub2-tools 2024-09-21T09:13:24.105 INFO:teuthology.orchestra.run.smithi064.stdout:Epoch : 1 2024-09-21T09:13:24.105 INFO:teuthology.orchestra.run.smithi064.stdout:Version : 2.06 2024-09-21T09:13:24.106 INFO:teuthology.orchestra.run.smithi064.stdout:Release : 82.el9 2024-09-21T09:13:24.106 INFO:teuthology.orchestra.run.smithi064.stdout:Architecture: x86_64 2024-09-21T09:13:24.106 INFO:teuthology.orchestra.run.smithi064.stdout:Install Date: Wed 28 Aug 2024 09:06:27 PM UTC 2024-09-21T09:13:24.106 INFO:teuthology.orchestra.run.smithi064.stdout:Group : Unspecified 2024-09-21T09:13:24.106 INFO:teuthology.orchestra.run.smithi064.stdout:Size : 8274670 2024-09-21T09:13:24.106 INFO:teuthology.orchestra.run.smithi064.stdout:License : GPLv3+ 2024-09-21T09:13:24.106 INFO:teuthology.orchestra.run.smithi064.stdout:Signature : RSA/SHA256, Mon 01 Jul 2024 07:46:42 AM UTC, Key ID 05b555b38483c65d 2024-09-21T09:13:24.106 INFO:teuthology.orchestra.run.smithi064.stdout:Source RPM : grub2-2.06-82.el9.src.rpm 2024-09-21T09:13:24.106 INFO:teuthology.orchestra.run.smithi064.stdout:Build Date : Fri 28 Jun 2024 09:24:44 AM UTC 2024-09-21T09:13:24.106 INFO:teuthology.orchestra.run.smithi064.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-09-21T09:13:24.107 INFO:teuthology.orchestra.run.smithi064.stdout:Packager : builder@centos.org 2024-09-21T09:13:24.107 INFO:teuthology.orchestra.run.smithi064.stdout:Vendor : CentOS 2024-09-21T09:13:24.107 INFO:teuthology.orchestra.run.smithi064.stdout:URL : http://www.gnu.org/software/grub/ 2024-09-21T09:13:24.107 INFO:teuthology.orchestra.run.smithi064.stdout:Summary : Support tools for GRUB. 2024-09-21T09:13:24.107 INFO:teuthology.orchestra.run.smithi064.stdout:Description : 2024-09-21T09:13:24.107 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-09-21T09:13:24.107 INFO:teuthology.orchestra.run.smithi064.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-09-21T09:13:24.107 INFO:teuthology.orchestra.run.smithi064.stdout:customizable bootloader with modular architecture. It supports a rich 2024-09-21T09:13:24.107 INFO:teuthology.orchestra.run.smithi064.stdout:variety of kernel formats, file systems, computer architectures and 2024-09-21T09:13:24.107 INFO:teuthology.orchestra.run.smithi064.stdout:hardware devices. 2024-09-21T09:13:24.107 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-09-21T09:13:24.107 INFO:teuthology.orchestra.run.smithi064.stdout:This subpackage provides tools for support of all platforms. 2024-09-21T09:13:24.108 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-09-21T09:13:24.108 INFO:teuthology.task.kernel:Updating grub on smithi064 to boot 5.14.0-508.el9.x86_64 2024-09-21T09:13:24.109 DEBUG:teuthology.orchestra.run.smithi064:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-09-21T09:13:24.730 INFO:teuthology.orchestra.run.smithi064.stderr:Generating grub configuration file ... 2024-09-21T09:13:26.054 INFO:teuthology.orchestra.run.smithi064.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-09-21T09:13:26.094 INFO:teuthology.orchestra.run.smithi064.stderr:done 2024-09-21T09:13:26.095 DEBUG:teuthology.orchestra.run.smithi064:> mktemp 2024-09-21T09:13:26.111 INFO:teuthology.orchestra.run.smithi064.stdout:/tmp/tmp.4DwIWflNox 2024-09-21T09:13:26.111 DEBUG:teuthology.orchestra.run.smithi064:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.4DwIWflNox 2024-09-21T09:13:26.178 DEBUG:teuthology.orchestra.run.smithi064:> sudo chmod 0666 /tmp/tmp.4DwIWflNox 2024-09-21T09:13:26.311 DEBUG:teuthology.orchestra.remote:smithi064:/tmp/tmp.4DwIWflNox is 6KB 2024-09-21T09:13:26.358 DEBUG:teuthology.orchestra.run.smithi064:> rm -fr /tmp/tmp.4DwIWflNox 2024-09-21T09:13:26.373 DEBUG:teuthology.orchestra.run.smithi064:> sudo /bin/ls /boot/loader/entries || true 2024-09-21T09:13:26.435 INFO:teuthology.orchestra.run.smithi064.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-0-rescue.conf 2024-09-21T09:13:26.435 INFO:teuthology.orchestra.run.smithi064.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-239.el9.x86_64.conf 2024-09-21T09:13:26.435 INFO:teuthology.orchestra.run.smithi064.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-503.el9.x86_64.conf 2024-09-21T09:13:26.436 INFO:teuthology.orchestra.run.smithi064.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-508.el9.x86_64.conf 2024-09-21T09:13:26.437 DEBUG:teuthology.orchestra.run.smithi064:> sudo grub2-set-default fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-508.el9.x86_64 2024-09-21T09:13:26.603 DEBUG:teuthology.orchestra.run.smithi064:> sudo shutdown -r now 2024-09-21T09:13:29.758 INFO:teuthology.orchestra.run.smithi022.stdout: Running scriptlet: kernel-core-5.14.0-508.el9.x86_64 4/4 2024-09-21T09:13:29.801 INFO:teuthology.orchestra.run.smithi022.stdout: Running scriptlet: kernel-modules-5.14.0-508.el9.x86_64 4/4 2024-09-21T09:13:30.116 INFO:teuthology.orchestra.run.smithi022.stdout: Running scriptlet: kernel-5.14.0-508.el9.x86_64 4/4 2024-09-21T09:13:30.116 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : kernel-5.14.0-508.el9.x86_64 1/4 2024-09-21T09:13:30.116 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-21T09:13:30.116 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-21T09:13:30.994 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : kernel-modules-core-5.14.0-508.el9.x86_64 4/4 2024-09-21T09:13:30.995 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-09-21T09:13:30.995 INFO:teuthology.orchestra.run.smithi022.stdout:Installed: 2024-09-21T09:13:30.995 INFO:teuthology.orchestra.run.smithi022.stdout: kernel-5.14.0-508.el9.x86_64 2024-09-21T09:13:30.995 INFO:teuthology.orchestra.run.smithi022.stdout: kernel-core-5.14.0-508.el9.x86_64 2024-09-21T09:13:30.995 INFO:teuthology.orchestra.run.smithi022.stdout: kernel-modules-5.14.0-508.el9.x86_64 2024-09-21T09:13:30.995 INFO:teuthology.orchestra.run.smithi022.stdout: kernel-modules-core-5.14.0-508.el9.x86_64 2024-09-21T09:13:30.995 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-09-21T09:13:30.995 INFO:teuthology.orchestra.run.smithi022.stdout:Complete! 2024-09-21T09:13:31.381 DEBUG:teuthology.orchestra.run.smithi022:> rpm -q kernel | sort -rV | head -n 1 2024-09-21T09:13:31.471 INFO:teuthology.orchestra.run.smithi022.stdout:kernel-5.14.0-508.el9.x86_64 2024-09-21T09:13:31.471 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-508.el9.x86_64 2024-09-21T09:13:31.471 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-503.el9.x86_64 Expected: 5.14.0-508.el9.x86_64 2024-09-21T09:13:31.472 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-09-21T09:13:31.472 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-09-21T09:13:31.472 INFO:teuthology.task.kernel:Installing distro kernel on mon.a... 2024-09-21T09:13:31.472 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi022.front.sepia.ceph.com, path=None, version=distro) 2024-09-21T09:13:31.472 DEBUG:teuthology.orchestra.run.smithi022:> rpm -q kernel | sort -rV | head -n 1 2024-09-21T09:13:31.530 INFO:teuthology.orchestra.run.smithi022.stdout:kernel-5.14.0-508.el9.x86_64 2024-09-21T09:13:31.530 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-508.el9.x86_64 2024-09-21T09:13:31.531 DEBUG:teuthology.orchestra.run.smithi022:> sudo rpm -qi grub2-tools 2024-09-21T09:13:31.562 INFO:teuthology.orchestra.run.smithi022.stdout:Name : grub2-tools 2024-09-21T09:13:31.562 INFO:teuthology.orchestra.run.smithi022.stdout:Epoch : 1 2024-09-21T09:13:31.562 INFO:teuthology.orchestra.run.smithi022.stdout:Version : 2.06 2024-09-21T09:13:31.562 INFO:teuthology.orchestra.run.smithi022.stdout:Release : 82.el9 2024-09-21T09:13:31.562 INFO:teuthology.orchestra.run.smithi022.stdout:Architecture: x86_64 2024-09-21T09:13:31.562 INFO:teuthology.orchestra.run.smithi022.stdout:Install Date: Wed 28 Aug 2024 09:06:27 PM UTC 2024-09-21T09:13:31.562 INFO:teuthology.orchestra.run.smithi022.stdout:Group : Unspecified 2024-09-21T09:13:31.562 INFO:teuthology.orchestra.run.smithi022.stdout:Size : 8274670 2024-09-21T09:13:31.562 INFO:teuthology.orchestra.run.smithi022.stdout:License : GPLv3+ 2024-09-21T09:13:31.562 INFO:teuthology.orchestra.run.smithi022.stdout:Signature : RSA/SHA256, Mon 01 Jul 2024 07:46:42 AM UTC, Key ID 05b555b38483c65d 2024-09-21T09:13:31.563 INFO:teuthology.orchestra.run.smithi022.stdout:Source RPM : grub2-2.06-82.el9.src.rpm 2024-09-21T09:13:31.563 INFO:teuthology.orchestra.run.smithi022.stdout:Build Date : Fri 28 Jun 2024 09:24:44 AM UTC 2024-09-21T09:13:31.563 INFO:teuthology.orchestra.run.smithi022.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-09-21T09:13:31.563 INFO:teuthology.orchestra.run.smithi022.stdout:Packager : builder@centos.org 2024-09-21T09:13:31.563 INFO:teuthology.orchestra.run.smithi022.stdout:Vendor : CentOS 2024-09-21T09:13:31.563 INFO:teuthology.orchestra.run.smithi022.stdout:URL : http://www.gnu.org/software/grub/ 2024-09-21T09:13:31.563 INFO:teuthology.orchestra.run.smithi022.stdout:Summary : Support tools for GRUB. 2024-09-21T09:13:31.563 INFO:teuthology.orchestra.run.smithi022.stdout:Description : 2024-09-21T09:13:31.563 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-09-21T09:13:31.563 INFO:teuthology.orchestra.run.smithi022.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-09-21T09:13:31.563 INFO:teuthology.orchestra.run.smithi022.stdout:customizable bootloader with modular architecture. It supports a rich 2024-09-21T09:13:31.563 INFO:teuthology.orchestra.run.smithi022.stdout:variety of kernel formats, file systems, computer architectures and 2024-09-21T09:13:31.563 INFO:teuthology.orchestra.run.smithi022.stdout:hardware devices. 2024-09-21T09:13:31.563 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-09-21T09:13:31.564 INFO:teuthology.orchestra.run.smithi022.stdout:This subpackage provides tools for support of all platforms. 2024-09-21T09:13:31.564 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-09-21T09:13:31.564 INFO:teuthology.task.kernel:Updating grub on smithi022 to boot 5.14.0-508.el9.x86_64 2024-09-21T09:13:31.565 DEBUG:teuthology.orchestra.run.smithi022:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-09-21T09:13:32.269 INFO:teuthology.orchestra.run.smithi022.stderr:Generating grub configuration file ... 2024-09-21T09:13:33.635 INFO:teuthology.orchestra.run.smithi022.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-09-21T09:13:33.667 INFO:teuthology.orchestra.run.smithi022.stderr:done 2024-09-21T09:13:33.670 DEBUG:teuthology.orchestra.run.smithi022:> mktemp 2024-09-21T09:13:33.685 INFO:teuthology.orchestra.run.smithi022.stdout:/tmp/tmp.SFvO55XvAz 2024-09-21T09:13:33.685 DEBUG:teuthology.orchestra.run.smithi022:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.SFvO55XvAz 2024-09-21T09:13:33.754 DEBUG:teuthology.orchestra.run.smithi022:> sudo chmod 0666 /tmp/tmp.SFvO55XvAz 2024-09-21T09:13:33.904 DEBUG:teuthology.orchestra.remote:smithi022:/tmp/tmp.SFvO55XvAz is 6KB 2024-09-21T09:13:33.953 DEBUG:teuthology.orchestra.run.smithi022:> rm -fr /tmp/tmp.SFvO55XvAz 2024-09-21T09:13:33.968 DEBUG:teuthology.orchestra.run.smithi022:> sudo /bin/ls /boot/loader/entries || true 2024-09-21T09:13:34.031 INFO:teuthology.orchestra.run.smithi022.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-0-rescue.conf 2024-09-21T09:13:34.031 INFO:teuthology.orchestra.run.smithi022.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-239.el9.x86_64.conf 2024-09-21T09:13:34.031 INFO:teuthology.orchestra.run.smithi022.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-503.el9.x86_64.conf 2024-09-21T09:13:34.031 INFO:teuthology.orchestra.run.smithi022.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-508.el9.x86_64.conf 2024-09-21T09:13:34.032 DEBUG:teuthology.orchestra.run.smithi022:> sudo grub2-set-default fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-508.el9.x86_64 2024-09-21T09:13:34.188 DEBUG:teuthology.orchestra.run.smithi022:> sudo shutdown -r now 2024-09-21T09:13:56.630 INFO:teuthology.task.kernel:Checking client mon.b for new kernel version... 2024-09-21T09:13:56.630 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi064.front.sepia.ceph.com' 2024-09-21T09:13:56.631 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi064.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-21T09:14:04.196 INFO:teuthology.task.kernel:Checking client mon.a for new kernel version... 2024-09-21T09:14:04.196 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi022.front.sepia.ceph.com' 2024-09-21T09:14:04.197 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi022.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-21T09:14:15.152 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.64 2024-09-21T09:14:22.575 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.22 2024-09-21T09:14:24.154 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi064.front.sepia.ceph.com' 2024-09-21T09:14:24.154 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi064.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-21T09:14:27.215 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.64 2024-09-21T09:14:31.580 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi022.front.sepia.ceph.com' 2024-09-21T09:14:31.580 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi022.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-21T09:14:39.222 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi064.front.sepia.ceph.com' 2024-09-21T09:14:39.223 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi064.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-21T09:14:57.647 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.64 2024-09-21T09:15:12.658 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi064.front.sepia.ceph.com' 2024-09-21T09:15:12.659 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi064.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-21T09:15:12.968 DEBUG:teuthology.orchestra.run.smithi064:> true 2024-09-21T09:15:13.393 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi064.front.sepia.ceph.com' 2024-09-21T09:15:13.393 INFO:teuthology.task.kernel:Checking kernel version of mon.b, want "5.14.0-508.el9.x86_64"... 2024-09-21T09:15:13.393 DEBUG:teuthology.orchestra.run.smithi064:> uname -r 2024-09-21T09:15:13.408 INFO:teuthology.orchestra.run.smithi064.stdout:5.14.0-508.el9.x86_64 2024-09-21T09:15:13.409 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-508.el9.x86_64 vs 5.14.0-508.el9.x86_64 2024-09-21T09:15:13.409 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-09-21T09:15:13.409 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-09-21T09:15:14.410 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2024-09-21T09:15:14.410 DEBUG:teuthology.orchestra.run.smithi064:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-09-21T09:15:14.521 INFO:teuthology.orchestra.run.smithi064.stdout:ttyS1 2024-09-21T09:15:14.549 DEBUG:teuthology.parallel:result is None 2024-09-21T09:15:31.582 DEBUG:teuthology.orchestra.remote:timed out 2024-09-21T09:15:43.586 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi022.front.sepia.ceph.com' 2024-09-21T09:15:43.587 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi022.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-21T09:15:43.983 DEBUG:teuthology.orchestra.run.smithi022:> true 2024-09-21T09:15:44.347 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi022.front.sepia.ceph.com' 2024-09-21T09:15:44.347 INFO:teuthology.task.kernel:Checking kernel version of mon.a, want "5.14.0-508.el9.x86_64"... 2024-09-21T09:15:44.347 DEBUG:teuthology.orchestra.run.smithi022:> uname -r 2024-09-21T09:15:44.363 INFO:teuthology.orchestra.run.smithi022.stdout:5.14.0-508.el9.x86_64 2024-09-21T09:15:44.363 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-508.el9.x86_64 vs 5.14.0-508.el9.x86_64 2024-09-21T09:15:44.364 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-09-21T09:15:44.364 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-09-21T09:15:45.365 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2024-09-21T09:15:45.365 DEBUG:teuthology.orchestra.run.smithi022:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-09-21T09:15:45.442 INFO:teuthology.orchestra.run.smithi022.stdout:ttyS1 2024-09-21T09:15:45.461 DEBUG:teuthology.parallel:result is None 2024-09-21T09:15:45.461 INFO:teuthology.run_tasks:Running task internal.base... 2024-09-21T09:15:45.479 INFO:teuthology.task.internal:Creating test directory... 2024-09-21T09:15:45.479 DEBUG:teuthology.orchestra.run.smithi022:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-09-21T09:15:45.481 DEBUG:teuthology.orchestra.run.smithi064:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-09-21T09:15:45.501 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-09-21T09:15:45.507 INFO:teuthology.run_tasks:Running task internal.archive... 2024-09-21T09:15:45.515 INFO:teuthology.task.internal:Creating archive directory... 2024-09-21T09:15:45.515 DEBUG:teuthology.orchestra.run.smithi022:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-09-21T09:15:45.540 DEBUG:teuthology.orchestra.run.smithi064:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-09-21T09:15:45.575 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-09-21T09:15:45.582 INFO:teuthology.task.internal:Enabling coredump saving... 2024-09-21T09:15:45.582 DEBUG:teuthology.orchestra.run.smithi022:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-09-21T09:15:45.604 DEBUG:teuthology.orchestra.run.smithi064:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-09-21T09:15:45.640 INFO:teuthology.orchestra.run.smithi022.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-21T09:15:45.645 INFO:teuthology.orchestra.run.smithi064.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-21T09:15:45.653 INFO:teuthology.orchestra.run.smithi022.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-21T09:15:45.657 INFO:teuthology.orchestra.run.smithi064.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-21T09:15:45.659 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-09-21T09:15:45.666 INFO:teuthology.task.internal:Configuring sudo... 2024-09-21T09:15:45.666 DEBUG:teuthology.orchestra.run.smithi022:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-09-21T09:15:45.697 DEBUG:teuthology.orchestra.run.smithi064:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-09-21T09:15:45.728 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-09-21T09:15:45.737 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-09-21T09:15:45.737 DEBUG:teuthology.orchestra.run.smithi022:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-09-21T09:15:45.766 DEBUG:teuthology.orchestra.run.smithi064:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-09-21T09:15:45.785 DEBUG:teuthology.orchestra.run.smithi022:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-21T09:15:45.863 DEBUG:teuthology.orchestra.run.smithi022:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-21T09:15:45.919 DEBUG:teuthology.orchestra.run.smithi022:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-21T09:15:46.000 DEBUG:teuthology.orchestra.run.smithi022:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-21T09:15:46.047 DEBUG:teuthology.orchestra.run.smithi022:> set -ex 2024-09-21T09:15:46.047 DEBUG:teuthology.orchestra.run.smithi022:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-09-21T09:15:46.115 DEBUG:teuthology.orchestra.run.smithi064:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-21T09:15:46.152 DEBUG:teuthology.orchestra.run.smithi064:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-21T09:15:46.219 DEBUG:teuthology.orchestra.run.smithi064:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-21T09:15:46.297 DEBUG:teuthology.orchestra.run.smithi064:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-21T09:15:46.348 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-09-21T09:15:46.348 DEBUG:teuthology.orchestra.run.smithi064:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-09-21T09:15:46.416 DEBUG:teuthology.orchestra.run.smithi022:> sudo service rsyslog restart 2024-09-21T09:15:46.418 DEBUG:teuthology.orchestra.run.smithi064:> sudo service rsyslog restart 2024-09-21T09:15:46.465 INFO:teuthology.orchestra.run.smithi022.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-21T09:15:46.508 INFO:teuthology.orchestra.run.smithi064.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-21T09:15:46.979 INFO:teuthology.run_tasks:Running task internal.timer... 2024-09-21T09:15:46.986 INFO:teuthology.task.internal:Starting timer... 2024-09-21T09:15:46.986 INFO:teuthology.run_tasks:Running task pcp... 2024-09-21T09:15:46.996 INFO:teuthology.run_tasks:Running task selinux... 2024-09-21T09:15:47.004 DEBUG:teuthology.task:Applying overrides for task selinux: {'allowlist': ['scontext=system_u:system_r:logrotate_t:s0']} 2024-09-21T09:15:47.004 DEBUG:teuthology.orchestra.run.smithi022:> sudo service auditd rotate 2024-09-21T09:15:47.049 INFO:teuthology.orchestra.run.smithi022.stdout:Rotating logs: 2024-09-21T09:15:47.051 DEBUG:teuthology.orchestra.run.smithi064:> sudo service auditd rotate 2024-09-21T09:15:47.106 INFO:teuthology.orchestra.run.smithi064.stdout:Rotating logs: 2024-09-21T09:15:47.167 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-09-21T09:15:47.167 DEBUG:teuthology.orchestra.run.smithi022:> /usr/sbin/getenforce 2024-09-21T09:15:47.199 INFO:teuthology.orchestra.run.smithi022.stdout:Permissive 2024-09-21T09:15:47.200 DEBUG:teuthology.orchestra.run.smithi064:> /usr/sbin/getenforce 2024-09-21T09:15:47.235 INFO:teuthology.orchestra.run.smithi064.stdout:Permissive 2024-09-21T09:15:47.236 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi022.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi064.front.sepia.ceph.com': 'permissive'} 2024-09-21T09:15:47.236 DEBUG:teuthology.orchestra.run.smithi022:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-21T09:15:47.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:15:47.279 DEBUG:teuthology.orchestra.run.smithi064:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-21T09:15:47.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:15:47.314 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-09-21T09:15:47.314 DEBUG:teuthology.orchestra.run.smithi022:> sudo /usr/sbin/setenforce permissive 2024-09-21T09:15:47.363 DEBUG:teuthology.orchestra.run.smithi064:> sudo /usr/sbin/setenforce permissive 2024-09-21T09:15:47.418 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-09-21T09:15:47.435 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-09-21T09:15:47.445 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-09-21T09:15:47.446 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi022.front.sepia.ceph.com,smithi064.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-09-21T09:19:35.715 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi022.front.sepia.ceph.com'), Remote(name='ubuntu@smithi064.front.sepia.ceph.com')] 2024-09-21T09:19:35.716 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi022.front.sepia.ceph.com' 2024-09-21T09:19:35.717 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi022.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-21T09:19:35.796 DEBUG:teuthology.orchestra.run.smithi022:> true 2024-09-21T09:19:35.875 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi022.front.sepia.ceph.com' 2024-09-21T09:19:35.876 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi064.front.sepia.ceph.com' 2024-09-21T09:19:35.876 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi064.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-21T09:19:35.950 DEBUG:teuthology.orchestra.run.smithi064:> true 2024-09-21T09:19:36.033 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi064.front.sepia.ceph.com' 2024-09-21T09:19:36.033 INFO:teuthology.run_tasks:Running task clock... 2024-09-21T09:19:36.043 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-09-21T09:19:36.043 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-09-21T09:19:36.044 DEBUG:teuthology.orchestra.run.smithi022:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-21T09:19:36.046 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-09-21T09:19:36.046 DEBUG:teuthology.orchestra.run.smithi064:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-21T09:19:36.079 INFO:teuthology.orchestra.run.smithi022.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-09-21T09:19:36.096 INFO:teuthology.orchestra.run.smithi022.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-09-21T09:19:36.108 INFO:teuthology.orchestra.run.smithi064.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-09-21T09:19:36.123 INFO:teuthology.orchestra.run.smithi022.stderr:sudo: ntpd: command not found 2024-09-21T09:19:36.124 INFO:teuthology.orchestra.run.smithi064.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-09-21T09:19:36.136 INFO:teuthology.orchestra.run.smithi022.stdout:506 Cannot talk to daemon 2024-09-21T09:19:36.152 INFO:teuthology.orchestra.run.smithi022.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-09-21T09:19:36.152 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: ntpd: command not found 2024-09-21T09:19:36.164 INFO:teuthology.orchestra.run.smithi064.stdout:506 Cannot talk to daemon 2024-09-21T09:19:36.167 INFO:teuthology.orchestra.run.smithi022.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-09-21T09:19:36.180 INFO:teuthology.orchestra.run.smithi064.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-09-21T09:19:36.195 INFO:teuthology.orchestra.run.smithi064.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-09-21T09:19:36.224 INFO:teuthology.orchestra.run.smithi022.stderr:bash: line 1: ntpq: command not found 2024-09-21T09:19:36.227 INFO:teuthology.orchestra.run.smithi022.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-21T09:19:36.227 INFO:teuthology.orchestra.run.smithi022.stdout:=============================================================================== 2024-09-21T09:19:36.227 INFO:teuthology.orchestra.run.smithi022.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T09:19:36.227 INFO:teuthology.orchestra.run.smithi022.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T09:19:36.227 INFO:teuthology.orchestra.run.smithi022.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T09:19:36.227 INFO:teuthology.orchestra.run.smithi022.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T09:19:36.256 INFO:teuthology.orchestra.run.smithi064.stderr:bash: line 1: ntpq: command not found 2024-09-21T09:19:36.260 INFO:teuthology.orchestra.run.smithi064.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-21T09:19:36.260 INFO:teuthology.orchestra.run.smithi064.stdout:=============================================================================== 2024-09-21T09:19:36.260 INFO:teuthology.orchestra.run.smithi064.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T09:19:36.261 INFO:teuthology.orchestra.run.smithi064.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T09:19:36.261 INFO:teuthology.orchestra.run.smithi064.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T09:19:36.261 INFO:teuthology.orchestra.run.smithi064.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T09:19:36.261 INFO:teuthology.run_tasks:Running task pexec... 2024-09-21T09:19:36.273 INFO:teuthology.task.pexec:Executing custom commands... 2024-09-21T09:19:36.274 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi022.front.sepia.ceph.com 2024-09-21T09:19:36.274 DEBUG:teuthology.orchestra.run.smithi022:> TESTDIR=/home/ubuntu/cephtest bash -s 2024-09-21T09:19:36.274 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi064.front.sepia.ceph.com 2024-09-21T09:19:36.274 DEBUG:teuthology.orchestra.run.smithi064:> TESTDIR=/home/ubuntu/cephtest bash -s 2024-09-21T09:19:36.640 INFO:teuthology.orchestra.run.smithi022.stdout:Dependencies resolved. 2024-09-21T09:19:36.642 INFO:teuthology.orchestra.run.smithi022.stdout:================================================================================ 2024-09-21T09:19:36.642 INFO:teuthology.orchestra.run.smithi022.stdout: Package Architecture Version Repository Size 2024-09-21T09:19:36.642 INFO:teuthology.orchestra.run.smithi022.stdout:================================================================================ 2024-09-21T09:19:36.642 INFO:teuthology.orchestra.run.smithi022.stdout:Removing: 2024-09-21T09:19:36.642 INFO:teuthology.orchestra.run.smithi022.stdout: nvme-cli x86_64 2.9.1-6.el9 @baseos 5.3 M 2024-09-21T09:19:36.642 INFO:teuthology.orchestra.run.smithi022.stdout:Removing unused dependencies: 2024-09-21T09:19:36.642 INFO:teuthology.orchestra.run.smithi022.stdout: libnvme x86_64 1.9-3.el9 @baseos 273 k 2024-09-21T09:19:36.642 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-09-21T09:19:36.642 INFO:teuthology.orchestra.run.smithi022.stdout:Transaction Summary 2024-09-21T09:19:36.642 INFO:teuthology.orchestra.run.smithi022.stdout:================================================================================ 2024-09-21T09:19:36.642 INFO:teuthology.orchestra.run.smithi022.stdout:Remove 2 Packages 2024-09-21T09:19:36.642 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-09-21T09:19:36.643 INFO:teuthology.orchestra.run.smithi022.stdout:Freed space: 5.5 M 2024-09-21T09:19:36.643 INFO:teuthology.orchestra.run.smithi022.stdout:Running transaction check 2024-09-21T09:19:36.647 INFO:teuthology.orchestra.run.smithi022.stdout:Transaction check succeeded. 2024-09-21T09:19:36.647 INFO:teuthology.orchestra.run.smithi022.stdout:Running transaction test 2024-09-21T09:19:36.668 INFO:teuthology.orchestra.run.smithi064.stdout:Dependencies resolved. 2024-09-21T09:19:36.670 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2024-09-21T09:19:36.670 INFO:teuthology.orchestra.run.smithi064.stdout: Package Architecture Version Repository Size 2024-09-21T09:19:36.670 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2024-09-21T09:19:36.670 INFO:teuthology.orchestra.run.smithi064.stdout:Removing: 2024-09-21T09:19:36.671 INFO:teuthology.orchestra.run.smithi064.stdout: nvme-cli x86_64 2.9.1-6.el9 @baseos 5.3 M 2024-09-21T09:19:36.671 INFO:teuthology.orchestra.run.smithi064.stdout:Removing unused dependencies: 2024-09-21T09:19:36.671 INFO:teuthology.orchestra.run.smithi064.stdout: libnvme x86_64 1.9-3.el9 @baseos 273 k 2024-09-21T09:19:36.671 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-09-21T09:19:36.671 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction Summary 2024-09-21T09:19:36.671 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2024-09-21T09:19:36.671 INFO:teuthology.orchestra.run.smithi064.stdout:Remove 2 Packages 2024-09-21T09:19:36.671 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-09-21T09:19:36.671 INFO:teuthology.orchestra.run.smithi064.stdout:Freed space: 5.5 M 2024-09-21T09:19:36.671 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction check 2024-09-21T09:19:36.675 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction check succeeded. 2024-09-21T09:19:36.675 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction test 2024-09-21T09:19:36.721 INFO:teuthology.orchestra.run.smithi022.stdout:Transaction test succeeded. 2024-09-21T09:19:36.721 INFO:teuthology.orchestra.run.smithi022.stdout:Running transaction 2024-09-21T09:19:36.743 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction test succeeded. 2024-09-21T09:19:36.743 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction 2024-09-21T09:19:36.883 INFO:teuthology.orchestra.run.smithi022.stdout: Preparing : 1/1 2024-09-21T09:19:36.897 INFO:teuthology.orchestra.run.smithi064.stdout: Preparing : 1/1 2024-09-21T09:19:36.955 INFO:teuthology.orchestra.run.smithi022.stdout: Erasing : nvme-cli-2.9.1-6.el9.x86_64 1/2 2024-09-21T09:19:36.967 INFO:teuthology.orchestra.run.smithi064.stdout: Erasing : nvme-cli-2.9.1-6.el9.x86_64 1/2 2024-09-21T09:19:37.020 INFO:teuthology.orchestra.run.smithi022.stdout: Erasing : libnvme-1.9-3.el9.x86_64 2/2 2024-09-21T09:19:37.035 INFO:teuthology.orchestra.run.smithi064.stdout: Erasing : libnvme-1.9-3.el9.x86_64 2/2 2024-09-21T09:19:37.532 INFO:teuthology.orchestra.run.smithi022.stdout: Running scriptlet: libnvme-1.9-3.el9.x86_64 2/2 2024-09-21T09:19:37.532 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : libnvme-1.9-3.el9.x86_64 1/2 2024-09-21T09:19:37.547 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: libnvme-1.9-3.el9.x86_64 2/2 2024-09-21T09:19:37.547 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : libnvme-1.9-3.el9.x86_64 1/2 2024-09-21T09:19:37.769 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : nvme-cli-2.9.1-6.el9.x86_64 2/2 2024-09-21T09:19:37.769 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-09-21T09:19:37.769 INFO:teuthology.orchestra.run.smithi022.stdout:Removed: 2024-09-21T09:19:37.769 INFO:teuthology.orchestra.run.smithi022.stdout: libnvme-1.9-3.el9.x86_64 nvme-cli-2.9.1-6.el9.x86_64 2024-09-21T09:19:37.769 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-09-21T09:19:37.769 INFO:teuthology.orchestra.run.smithi022.stdout:Complete! 2024-09-21T09:19:37.811 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : nvme-cli-2.9.1-6.el9.x86_64 2/2 2024-09-21T09:19:37.811 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-09-21T09:19:37.811 INFO:teuthology.orchestra.run.smithi064.stdout:Removed: 2024-09-21T09:19:37.811 INFO:teuthology.orchestra.run.smithi064.stdout: libnvme-1.9-3.el9.x86_64 nvme-cli-2.9.1-6.el9.x86_64 2024-09-21T09:19:37.811 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-09-21T09:19:37.811 INFO:teuthology.orchestra.run.smithi064.stdout:Complete! 2024-09-21T09:19:38.627 INFO:teuthology.orchestra.run.smithi022.stdout:Last metadata expiration check: 0:00:48 ago on Sat 21 Sep 2024 09:18:50 AM UTC. 2024-09-21T09:19:38.754 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:00:48 ago on Sat 21 Sep 2024 09:18:50 AM UTC. 2024-09-21T09:19:38.764 INFO:teuthology.orchestra.run.smithi022.stdout:Dependencies resolved. 2024-09-21T09:19:38.765 INFO:teuthology.orchestra.run.smithi022.stdout:================================================================================ 2024-09-21T09:19:38.765 INFO:teuthology.orchestra.run.smithi022.stdout: Package Arch Version Repository Size 2024-09-21T09:19:38.765 INFO:teuthology.orchestra.run.smithi022.stdout:================================================================================ 2024-09-21T09:19:38.765 INFO:teuthology.orchestra.run.smithi022.stdout:Installing: 2024-09-21T09:19:38.766 INFO:teuthology.orchestra.run.smithi022.stdout: nvme-cli x86_64 2.9.1-6.el9 baseos 906 k 2024-09-21T09:19:38.766 INFO:teuthology.orchestra.run.smithi022.stdout: nvmetcli noarch 0.7-3.el9 baseos 44 k 2024-09-21T09:19:38.766 INFO:teuthology.orchestra.run.smithi022.stdout: runc x86_64 4:1.1.13-4.el9 appstream 3.2 M 2024-09-21T09:19:38.766 INFO:teuthology.orchestra.run.smithi022.stdout:Installing dependencies: 2024-09-21T09:19:38.766 INFO:teuthology.orchestra.run.smithi022.stdout: libnvme x86_64 1.9-3.el9 baseos 106 k 2024-09-21T09:19:38.766 INFO:teuthology.orchestra.run.smithi022.stdout: python3-configshell noarch 1:1.1.30-1.el9 baseos 72 k 2024-09-21T09:19:38.766 INFO:teuthology.orchestra.run.smithi022.stdout: python3-kmod x86_64 0.9-32.el9 baseos 84 k 2024-09-21T09:19:38.767 INFO:teuthology.orchestra.run.smithi022.stdout: python3-urwid x86_64 2.1.2-4.el9 baseos 837 k 2024-09-21T09:19:38.767 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-09-21T09:19:38.767 INFO:teuthology.orchestra.run.smithi022.stdout:Transaction Summary 2024-09-21T09:19:38.767 INFO:teuthology.orchestra.run.smithi022.stdout:================================================================================ 2024-09-21T09:19:38.767 INFO:teuthology.orchestra.run.smithi022.stdout:Install 7 Packages 2024-09-21T09:19:38.767 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-09-21T09:19:38.780 INFO:teuthology.orchestra.run.smithi022.stdout:Total download size: 5.2 M 2024-09-21T09:19:38.780 INFO:teuthology.orchestra.run.smithi022.stdout:Installed size: 19 M 2024-09-21T09:19:38.781 INFO:teuthology.orchestra.run.smithi022.stdout:Downloading Packages: 2024-09-21T09:19:38.895 INFO:teuthology.orchestra.run.smithi064.stdout:Dependencies resolved. 2024-09-21T09:19:38.895 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2024-09-21T09:19:38.895 INFO:teuthology.orchestra.run.smithi064.stdout: Package Arch Version Repository Size 2024-09-21T09:19:38.895 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2024-09-21T09:19:38.895 INFO:teuthology.orchestra.run.smithi064.stdout:Installing: 2024-09-21T09:19:38.895 INFO:teuthology.orchestra.run.smithi064.stdout: nvme-cli x86_64 2.9.1-6.el9 baseos 906 k 2024-09-21T09:19:38.895 INFO:teuthology.orchestra.run.smithi064.stdout: nvmetcli noarch 0.7-3.el9 baseos 44 k 2024-09-21T09:19:38.895 INFO:teuthology.orchestra.run.smithi064.stdout: runc x86_64 4:1.1.13-4.el9 appstream 3.2 M 2024-09-21T09:19:38.896 INFO:teuthology.orchestra.run.smithi064.stdout:Installing dependencies: 2024-09-21T09:19:38.896 INFO:teuthology.orchestra.run.smithi064.stdout: libnvme x86_64 1.9-3.el9 baseos 106 k 2024-09-21T09:19:38.896 INFO:teuthology.orchestra.run.smithi064.stdout: python3-configshell noarch 1:1.1.30-1.el9 baseos 72 k 2024-09-21T09:19:38.896 INFO:teuthology.orchestra.run.smithi064.stdout: python3-kmod x86_64 0.9-32.el9 baseos 84 k 2024-09-21T09:19:38.896 INFO:teuthology.orchestra.run.smithi064.stdout: python3-urwid x86_64 2.1.2-4.el9 baseos 837 k 2024-09-21T09:19:38.896 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-09-21T09:19:38.896 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction Summary 2024-09-21T09:19:38.897 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2024-09-21T09:19:38.897 INFO:teuthology.orchestra.run.smithi064.stdout:Install 7 Packages 2024-09-21T09:19:38.897 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-09-21T09:19:38.905 INFO:teuthology.orchestra.run.smithi064.stdout:Total download size: 5.2 M 2024-09-21T09:19:38.905 INFO:teuthology.orchestra.run.smithi064.stdout:Installed size: 19 M 2024-09-21T09:19:38.906 INFO:teuthology.orchestra.run.smithi064.stdout:Downloading Packages: 2024-09-21T09:19:38.959 INFO:teuthology.orchestra.run.smithi022.stdout:(1/7): nvmetcli-0.7-3.el9.noarch.rpm 405 kB/s | 44 kB 00:00 2024-09-21T09:19:38.984 INFO:teuthology.orchestra.run.smithi022.stdout:(2/7): libnvme-1.9-3.el9.x86_64.rpm 785 kB/s | 106 kB 00:00 2024-09-21T09:19:39.035 INFO:teuthology.orchestra.run.smithi022.stdout:(3/7): python3-kmod-0.9-32.el9.x86_64.rpm 1.6 MB/s | 84 kB 00:00 2024-09-21T09:19:39.060 INFO:teuthology.orchestra.run.smithi022.stdout:(4/7): python3-configshell-1.1.30-1.el9.noarch. 712 kB/s | 72 kB 00:00 2024-09-21T09:19:39.235 INFO:teuthology.orchestra.run.smithi022.stdout:(5/7): nvme-cli-2.9.1-6.el9.x86_64.rpm 2.3 MB/s | 906 kB 00:00 2024-09-21T09:19:39.248 INFO:teuthology.orchestra.run.smithi064.stdout:(1/7): nvmetcli-0.7-3.el9.noarch.rpm 410 kB/s | 44 kB 00:00 2024-09-21T09:19:39.282 INFO:teuthology.orchestra.run.smithi064.stdout:(2/7): libnvme-1.9-3.el9.x86_64.rpm 746 kB/s | 106 kB 00:00 2024-09-21T09:19:39.285 INFO:teuthology.orchestra.run.smithi022.stdout:(6/7): runc-1.1.13-4.el9.x86_64.rpm 14 MB/s | 3.2 MB 00:00 2024-09-21T09:19:39.307 INFO:teuthology.orchestra.run.smithi064.stdout:(3/7): python3-configshell-1.1.30-1.el9.noarch. 1.2 MB/s | 72 kB 00:00 2024-09-21T09:19:39.311 INFO:teuthology.orchestra.run.smithi022.stdout:(7/7): python3-urwid-2.1.2-4.el9.x86_64.rpm 3.0 MB/s | 837 kB 00:00 2024-09-21T09:19:39.311 INFO:teuthology.orchestra.run.smithi022.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:19:39.311 INFO:teuthology.orchestra.run.smithi022.stdout:Total 9.7 MB/s | 5.2 MB 00:00 2024-09-21T09:19:39.352 INFO:teuthology.orchestra.run.smithi064.stdout:(4/7): python3-kmod-0.9-32.el9.x86_64.rpm 1.2 MB/s | 84 kB 00:00 2024-09-21T09:19:39.393 INFO:teuthology.orchestra.run.smithi064.stdout:(5/7): nvme-cli-2.9.1-6.el9.x86_64.rpm 3.5 MB/s | 906 kB 00:00 2024-09-21T09:19:39.424 INFO:teuthology.orchestra.run.smithi022.stdout:Running transaction check 2024-09-21T09:19:39.435 INFO:teuthology.orchestra.run.smithi064.stdout:(6/7): python3-urwid-2.1.2-4.el9.x86_64.rpm 6.4 MB/s | 837 kB 00:00 2024-09-21T09:19:39.440 INFO:teuthology.orchestra.run.smithi022.stdout:Transaction check succeeded. 2024-09-21T09:19:39.440 INFO:teuthology.orchestra.run.smithi022.stdout:Running transaction test 2024-09-21T09:19:39.737 INFO:teuthology.orchestra.run.smithi022.stdout:Transaction test succeeded. 2024-09-21T09:19:39.737 INFO:teuthology.orchestra.run.smithi022.stdout:Running transaction 2024-09-21T09:19:40.094 INFO:teuthology.orchestra.run.smithi064.stdout:(7/7): runc-1.1.13-4.el9.x86_64.rpm 4.3 MB/s | 3.2 MB 00:00 2024-09-21T09:19:40.094 INFO:teuthology.orchestra.run.smithi064.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:19:40.094 INFO:teuthology.orchestra.run.smithi064.stdout:Total 4.3 MB/s | 5.2 MB 00:01 2024-09-21T09:19:40.215 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction check 2024-09-21T09:19:40.231 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction check succeeded. 2024-09-21T09:19:40.231 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction test 2024-09-21T09:19:40.232 INFO:teuthology.orchestra.run.smithi022.stdout: Preparing : 1/1 2024-09-21T09:19:40.315 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : python3-urwid-2.1.2-4.el9.x86_64 1/7 2024-09-21T09:19:40.348 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : python3-configshell-1:1.1.30-1.el9.noarch 2/7 2024-09-21T09:19:40.377 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : python3-kmod-0.9-32.el9.x86_64 3/7 2024-09-21T09:19:40.553 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction test succeeded. 2024-09-21T09:19:40.554 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction 2024-09-21T09:19:40.598 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : libnvme-1.9-3.el9.x86_64 4/7 2024-09-21T09:19:40.624 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : nvme-cli-2.9.1-6.el9.x86_64 5/7 2024-09-21T09:19:41.059 INFO:teuthology.orchestra.run.smithi064.stdout: Preparing : 1/1 2024-09-21T09:19:41.102 INFO:teuthology.orchestra.run.smithi022.stdout: Running scriptlet: nvme-cli-2.9.1-6.el9.x86_64 5/7 2024-09-21T09:19:41.127 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : nvmetcli-0.7-3.el9.noarch 6/7 2024-09-21T09:19:41.155 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-urwid-2.1.2-4.el9.x86_64 1/7 2024-09-21T09:19:41.189 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-configshell-1:1.1.30-1.el9.noarch 2/7 2024-09-21T09:19:41.226 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-kmod-0.9-32.el9.x86_64 3/7 2024-09-21T09:19:41.267 INFO:teuthology.orchestra.run.smithi022.stdout: Running scriptlet: nvmetcli-0.7-3.el9.noarch 6/7 2024-09-21T09:19:41.335 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : runc-4:1.1.13-4.el9.x86_64 7/7 2024-09-21T09:19:41.448 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : libnvme-1.9-3.el9.x86_64 4/7 2024-09-21T09:19:41.482 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : nvme-cli-2.9.1-6.el9.x86_64 5/7 2024-09-21T09:19:41.963 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: nvme-cli-2.9.1-6.el9.x86_64 5/7 2024-09-21T09:19:41.993 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : nvmetcli-0.7-3.el9.noarch 6/7 2024-09-21T09:19:42.116 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: nvmetcli-0.7-3.el9.noarch 6/7 2024-09-21T09:19:42.183 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : runc-4:1.1.13-4.el9.x86_64 7/7 2024-09-21T09:19:42.481 INFO:teuthology.orchestra.run.smithi022.stdout: Running scriptlet: runc-4:1.1.13-4.el9.x86_64 7/7 2024-09-21T09:19:42.481 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : libnvme-1.9-3.el9.x86_64 1/7 2024-09-21T09:19:42.481 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : nvme-cli-2.9.1-6.el9.x86_64 2/7 2024-09-21T09:19:42.481 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : nvmetcli-0.7-3.el9.noarch 3/7 2024-09-21T09:19:42.481 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : python3-configshell-1:1.1.30-1.el9.noarch 4/7 2024-09-21T09:19:42.481 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : python3-kmod-0.9-32.el9.x86_64 5/7 2024-09-21T09:19:42.481 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : python3-urwid-2.1.2-4.el9.x86_64 6/7 2024-09-21T09:19:42.885 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : runc-4:1.1.13-4.el9.x86_64 7/7 2024-09-21T09:19:42.885 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-09-21T09:19:42.886 INFO:teuthology.orchestra.run.smithi022.stdout:Installed: 2024-09-21T09:19:42.886 INFO:teuthology.orchestra.run.smithi022.stdout: libnvme-1.9-3.el9.x86_64 nvme-cli-2.9.1-6.el9.x86_64 2024-09-21T09:19:42.886 INFO:teuthology.orchestra.run.smithi022.stdout: nvmetcli-0.7-3.el9.noarch python3-configshell-1:1.1.30-1.el9.noarch 2024-09-21T09:19:42.886 INFO:teuthology.orchestra.run.smithi022.stdout: python3-kmod-0.9-32.el9.x86_64 python3-urwid-2.1.2-4.el9.x86_64 2024-09-21T09:19:42.886 INFO:teuthology.orchestra.run.smithi022.stdout: runc-4:1.1.13-4.el9.x86_64 2024-09-21T09:19:42.886 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-09-21T09:19:42.887 INFO:teuthology.orchestra.run.smithi022.stdout:Complete! 2024-09-21T09:19:43.154 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: runc-4:1.1.13-4.el9.x86_64 7/7 2024-09-21T09:19:43.154 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : libnvme-1.9-3.el9.x86_64 1/7 2024-09-21T09:19:43.155 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : nvme-cli-2.9.1-6.el9.x86_64 2/7 2024-09-21T09:19:43.155 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : nvmetcli-0.7-3.el9.noarch 3/7 2024-09-21T09:19:43.155 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-configshell-1:1.1.30-1.el9.noarch 4/7 2024-09-21T09:19:43.155 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-kmod-0.9-32.el9.x86_64 5/7 2024-09-21T09:19:43.155 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-urwid-2.1.2-4.el9.x86_64 6/7 2024-09-21T09:19:43.250 DEBUG:teuthology.parallel:result is None 2024-09-21T09:19:43.547 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : runc-4:1.1.13-4.el9.x86_64 7/7 2024-09-21T09:19:43.547 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-09-21T09:19:43.547 INFO:teuthology.orchestra.run.smithi064.stdout:Installed: 2024-09-21T09:19:43.547 INFO:teuthology.orchestra.run.smithi064.stdout: libnvme-1.9-3.el9.x86_64 nvme-cli-2.9.1-6.el9.x86_64 2024-09-21T09:19:43.547 INFO:teuthology.orchestra.run.smithi064.stdout: nvmetcli-0.7-3.el9.noarch python3-configshell-1:1.1.30-1.el9.noarch 2024-09-21T09:19:43.547 INFO:teuthology.orchestra.run.smithi064.stdout: python3-kmod-0.9-32.el9.x86_64 python3-urwid-2.1.2-4.el9.x86_64 2024-09-21T09:19:43.547 INFO:teuthology.orchestra.run.smithi064.stdout: runc-4:1.1.13-4.el9.x86_64 2024-09-21T09:19:43.548 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-09-21T09:19:43.548 INFO:teuthology.orchestra.run.smithi064.stdout:Complete! 2024-09-21T09:19:43.903 DEBUG:teuthology.parallel:result is None 2024-09-21T09:19:43.903 INFO:teuthology.run_tasks:Running task install... 2024-09-21T09:19:43.911 DEBUG:teuthology.task.install:project ceph 2024-09-21T09:19:43.911 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'extra_system_packages': ['python3-pytest'], 'flavor': 'default', 'sha1': '62da54b4b83a96ca9976b7e1d0c8272a564a2208'}} 2024-09-21T09:19:43.911 DEBUG:teuthology.task.install:config {'branch': 'quincy', 'exclude_packages': ['ceph-volume'], 'extra_system_packages': ['python3-pytest'], 'flavor': 'default', 'sha1': '62da54b4b83a96ca9976b7e1d0c8272a564a2208'} 2024-09-21T09:19:43.911 INFO:teuthology.task.install:Using flavor: default 2024-09-21T09:19:43.917 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2024-09-21T09:19:43.917 INFO:teuthology.task.install:extra packages: [] 2024-09-21T09:19:43.917 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'quincy', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': ['python3-pytest'], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': '62da54b4b83a96ca9976b7e1d0c8272a564a2208', 'tag': None, 'wait_for_package': False} 2024-09-21T09:19:43.917 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-21T09:19:43.917 INFO:teuthology.packaging:ref: None 2024-09-21T09:19:43.917 INFO:teuthology.packaging:tag: None 2024-09-21T09:19:43.917 INFO:teuthology.packaging:branch: quincy 2024-09-21T09:19:43.917 INFO:teuthology.packaging:sha1: 62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:19:43.918 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=quincy 2024-09-21T09:19:43.919 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'quincy', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': ['python3-pytest'], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': '62da54b4b83a96ca9976b7e1d0c8272a564a2208', 'tag': None, 'wait_for_package': False} 2024-09-21T09:19:43.919 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-21T09:19:43.919 INFO:teuthology.packaging:ref: None 2024-09-21T09:19:43.919 INFO:teuthology.packaging:tag: None 2024-09-21T09:19:43.919 INFO:teuthology.packaging:branch: quincy 2024-09-21T09:19:43.919 INFO:teuthology.packaging:sha1: 62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:19:43.920 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=quincy 2024-09-21T09:19:44.137 INFO:teuthology.task.install.rpm:Pulling from https://4.chacra.ceph.com/r/ceph/quincy/fbd1a6ab0448f011c60f6376195d5c078b0cf20b/centos/9/flavors/default/ 2024-09-21T09:19:44.137 INFO:teuthology.task.install.rpm:Package version is 17.2.7-1649.gfbd1a6ab 2024-09-21T09:19:44.156 INFO:teuthology.task.install.rpm:Pulling from https://4.chacra.ceph.com/r/ceph/quincy/fbd1a6ab0448f011c60f6376195d5c078b0cf20b/centos/9/flavors/default/ 2024-09-21T09:19:44.156 INFO:teuthology.task.install.rpm:Package version is 17.2.7-1649.gfbd1a6ab 2024-09-21T09:19:44.246 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://4.chacra.ceph.com/r/ceph/quincy/fbd1a6ab0448f011c60f6376195d5c078b0cf20b/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://4.chacra.ceph.com/r/ceph/quincy/fbd1a6ab0448f011c60f6376195d5c078b0cf20b/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://4.chacra.ceph.com/r/ceph/quincy/fbd1a6ab0448f011c60f6376195d5c078b0cf20b/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2024-09-21T09:19:44.246 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-09-21T09:19:44.246 DEBUG:teuthology.orchestra.run.smithi064:> sudo dd of=/etc/yum.repos.d/ceph.repo 2024-09-21T09:19:44.259 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://4.chacra.ceph.com/r/ceph/quincy/fbd1a6ab0448f011c60f6376195d5c078b0cf20b/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://4.chacra.ceph.com/r/ceph/quincy/fbd1a6ab0448f011c60f6376195d5c078b0cf20b/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://4.chacra.ceph.com/r/ceph/quincy/fbd1a6ab0448f011c60f6376195d5c078b0cf20b/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2024-09-21T09:19:44.259 DEBUG:teuthology.orchestra.run.smithi022:> set -ex 2024-09-21T09:19:44.259 DEBUG:teuthology.orchestra.run.smithi022:> sudo dd of=/etc/yum.repos.d/ceph.repo 2024-09-21T09:19:44.274 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd, python3-pytest, python3-pytest, python3-pytest on remote rpm x86_64 2024-09-21T09:19:44.274 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-21T09:19:44.274 INFO:teuthology.packaging:ref: None 2024-09-21T09:19:44.274 INFO:teuthology.packaging:tag: None 2024-09-21T09:19:44.275 INFO:teuthology.packaging:branch: quincy 2024-09-21T09:19:44.275 INFO:teuthology.packaging:sha1: 62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:19:44.275 DEBUG:teuthology.orchestra.run.smithi064:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/quincy/;g' /etc/yum.repos.d/ceph.repo ; fi 2024-09-21T09:19:44.291 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd, python3-pytest, python3-pytest, python3-pytest, python3-pytest on remote rpm x86_64 2024-09-21T09:19:44.291 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-21T09:19:44.291 INFO:teuthology.packaging:ref: None 2024-09-21T09:19:44.291 INFO:teuthology.packaging:tag: None 2024-09-21T09:19:44.291 INFO:teuthology.packaging:branch: quincy 2024-09-21T09:19:44.291 INFO:teuthology.packaging:sha1: 62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:19:44.292 DEBUG:teuthology.orchestra.run.smithi022:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/quincy/;g' /etc/yum.repos.d/ceph.repo ; fi 2024-09-21T09:19:44.362 DEBUG:teuthology.orchestra.run.smithi022:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2024-09-21T09:19:44.376 DEBUG:teuthology.orchestra.run.smithi064:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2024-09-21T09:19:44.414 DEBUG:teuthology.orchestra.run.smithi064:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2024-09-21T09:19:44.442 DEBUG:teuthology.orchestra.run.smithi022:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2024-09-21T09:19:44.480 INFO:teuthology.orchestra.run.smithi064.stdout:check_obsoletes = 1 2024-09-21T09:19:44.481 DEBUG:teuthology.orchestra.run.smithi064:> sudo yum clean all 2024-09-21T09:19:44.509 INFO:teuthology.orchestra.run.smithi022.stdout:check_obsoletes = 1 2024-09-21T09:19:44.511 DEBUG:teuthology.orchestra.run.smithi022:> sudo yum clean all 2024-09-21T09:19:44.717 INFO:teuthology.orchestra.run.smithi064.stdout:45 files removed 2024-09-21T09:19:44.744 DEBUG:teuthology.orchestra.run.smithi064:> sudo yum -y install ceph-radosgw ceph-test ceph ceph-base cephadm ceph-immutable-object-cache ceph-mgr ceph-mgr-dashboard ceph-mgr-diskprediction-local ceph-mgr-rook ceph-mgr-cephadm ceph-fuse librados-devel libcephfs2 libcephfs-devel librados2 librbd1 python3-rados python3-rgw python3-cephfs python3-rbd rbd-fuse rbd-mirror rbd-nbd python3-pytest python3-pytest python3-pytest python3-pytest 2024-09-21T09:19:44.745 INFO:teuthology.orchestra.run.smithi022.stdout:45 files removed 2024-09-21T09:19:44.772 DEBUG:teuthology.orchestra.run.smithi022:> sudo yum -y install ceph-radosgw ceph-test ceph ceph-base cephadm ceph-immutable-object-cache ceph-mgr ceph-mgr-dashboard ceph-mgr-diskprediction-local ceph-mgr-rook ceph-mgr-cephadm ceph-fuse librados-devel libcephfs2 libcephfs-devel librados2 librbd1 python3-rados python3-rgw python3-cephfs python3-rbd rbd-fuse rbd-mirror rbd-nbd python3-pytest python3-pytest python3-pytest python3-pytest 2024-09-21T09:19:45.327 INFO:teuthology.orchestra.run.smithi064.stdout:ceph packages for x86_64 276 kB/s | 77 kB 00:00 2024-09-21T09:19:45.351 INFO:teuthology.orchestra.run.smithi022.stdout:ceph packages for x86_64 278 kB/s | 77 kB 00:00 2024-09-21T09:19:45.619 INFO:teuthology.orchestra.run.smithi064.stdout:ceph noarch packages 43 kB/s | 11 kB 00:00 2024-09-21T09:19:45.635 INFO:teuthology.orchestra.run.smithi022.stdout:ceph noarch packages 45 kB/s | 11 kB 00:00 2024-09-21T09:19:45.894 INFO:teuthology.orchestra.run.smithi064.stdout:ceph source packages 7.5 kB/s | 1.8 kB 00:00 2024-09-21T09:19:45.919 INFO:teuthology.orchestra.run.smithi022.stdout:ceph source packages 7.2 kB/s | 1.8 kB 00:00 2024-09-21T09:19:46.453 INFO:teuthology.orchestra.run.smithi064.stdout:CentOS Stream 9 - BaseOS 16 MB/s | 8.3 MB 00:00 2024-09-21T09:19:48.053 INFO:teuthology.orchestra.run.smithi022.stdout:CentOS Stream 9 - BaseOS 3.9 MB/s | 8.3 MB 00:02 2024-09-21T09:19:49.812 INFO:teuthology.orchestra.run.smithi022.stdout:CentOS Stream 9 - AppStream 32 MB/s | 20 MB 00:00 2024-09-21T09:19:51.912 INFO:teuthology.orchestra.run.smithi064.stdout:CentOS Stream 9 - AppStream 4.7 MB/s | 20 MB 00:04 2024-09-21T09:19:54.238 INFO:teuthology.orchestra.run.smithi022.stdout:CentOS Stream 9 - CRB 13 MB/s | 6.5 MB 00:00 2024-09-21T09:19:55.981 INFO:teuthology.orchestra.run.smithi022.stdout:CentOS Stream 9 - Extras packages 42 kB/s | 19 kB 00:00 2024-09-21T09:19:56.496 INFO:teuthology.orchestra.run.smithi064.stdout:CentOS Stream 9 - CRB 11 MB/s | 6.5 MB 00:00 2024-09-21T09:19:56.885 INFO:teuthology.orchestra.run.smithi022.stdout:Extra Packages for Enterprise Linux 29 MB/s | 23 MB 00:00 2024-09-21T09:19:58.039 INFO:teuthology.orchestra.run.smithi064.stdout:CentOS Stream 9 - Extras packages 76 kB/s | 19 kB 00:00 2024-09-21T09:19:58.869 INFO:teuthology.orchestra.run.smithi064.stdout:Extra Packages for Enterprise Linux 32 MB/s | 23 MB 00:00 2024-09-21T09:20:02.457 INFO:teuthology.orchestra.run.smithi022.stdout:lab-extras 44 kB/s | 1.7 kB 00:00 2024-09-21T09:20:04.566 INFO:teuthology.orchestra.run.smithi064.stdout:lab-extras 29 kB/s | 1.7 kB 00:00 2024-09-21T09:20:04.752 INFO:teuthology.orchestra.run.smithi022.stdout:Package librados2-2:16.2.4-5.el9.x86_64 is already installed. 2024-09-21T09:20:04.753 INFO:teuthology.orchestra.run.smithi022.stdout:Package librbd1-2:16.2.4-5.el9.x86_64 is already installed. 2024-09-21T09:20:04.822 INFO:teuthology.orchestra.run.smithi022.stdout:Dependencies resolved. 2024-09-21T09:20:04.829 INFO:teuthology.orchestra.run.smithi022.stdout:======================================================================================= 2024-09-21T09:20:04.829 INFO:teuthology.orchestra.run.smithi022.stdout: Package Arch Version Repository Size 2024-09-21T09:20:04.829 INFO:teuthology.orchestra.run.smithi022.stdout:======================================================================================= 2024-09-21T09:20:04.830 INFO:teuthology.orchestra.run.smithi022.stdout:Installing: 2024-09-21T09:20:04.830 INFO:teuthology.orchestra.run.smithi022.stdout: ceph x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 6.5 k 2024-09-21T09:20:04.830 INFO:teuthology.orchestra.run.smithi022.stdout: ceph-base x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 5.0 M 2024-09-21T09:20:04.830 INFO:teuthology.orchestra.run.smithi022.stdout: ceph-fuse x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 813 k 2024-09-21T09:20:04.830 INFO:teuthology.orchestra.run.smithi022.stdout: ceph-immutable-object-cache x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 137 k 2024-09-21T09:20:04.830 INFO:teuthology.orchestra.run.smithi022.stdout: ceph-mgr x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 1.4 M 2024-09-21T09:20:04.830 INFO:teuthology.orchestra.run.smithi022.stdout: ceph-mgr-cephadm noarch 2:17.2.7-1649.gfbd1a6ab.el9 ceph-noarch 124 k 2024-09-21T09:20:04.830 INFO:teuthology.orchestra.run.smithi022.stdout: ceph-mgr-dashboard noarch 2:17.2.7-1649.gfbd1a6ab.el9 ceph-noarch 3.4 M 2024-09-21T09:20:04.830 INFO:teuthology.orchestra.run.smithi022.stdout: ceph-mgr-diskprediction-local noarch 2:17.2.7-1649.gfbd1a6ab.el9 ceph-noarch 7.4 M 2024-09-21T09:20:04.830 INFO:teuthology.orchestra.run.smithi022.stdout: ceph-mgr-rook noarch 2:17.2.7-1649.gfbd1a6ab.el9 ceph-noarch 47 k 2024-09-21T09:20:04.830 INFO:teuthology.orchestra.run.smithi022.stdout: ceph-radosgw x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 14 M 2024-09-21T09:20:04.830 INFO:teuthology.orchestra.run.smithi022.stdout: ceph-test x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 48 M 2024-09-21T09:20:04.830 INFO:teuthology.orchestra.run.smithi022.stdout: cephadm noarch 2:17.2.7-1649.gfbd1a6ab.el9 ceph-noarch 83 k 2024-09-21T09:20:04.831 INFO:teuthology.orchestra.run.smithi022.stdout: libcephfs-devel x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 25 k 2024-09-21T09:20:04.831 INFO:teuthology.orchestra.run.smithi022.stdout: libcephfs2 x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 636 k 2024-09-21T09:20:04.831 INFO:teuthology.orchestra.run.smithi022.stdout: librados-devel x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 126 k 2024-09-21T09:20:04.831 INFO:teuthology.orchestra.run.smithi022.stdout: python3-cephfs x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 156 k 2024-09-21T09:20:04.831 INFO:teuthology.orchestra.run.smithi022.stdout: python3-pytest noarch 6.2.2-7.el9 appstream 519 k 2024-09-21T09:20:04.831 INFO:teuthology.orchestra.run.smithi022.stdout: python3-rados x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 320 k 2024-09-21T09:20:04.831 INFO:teuthology.orchestra.run.smithi022.stdout: python3-rbd x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 295 k 2024-09-21T09:20:04.831 INFO:teuthology.orchestra.run.smithi022.stdout: python3-rgw x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 99 k 2024-09-21T09:20:04.831 INFO:teuthology.orchestra.run.smithi022.stdout: rbd-fuse x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 82 k 2024-09-21T09:20:04.831 INFO:teuthology.orchestra.run.smithi022.stdout: rbd-mirror x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 2.9 M 2024-09-21T09:20:04.831 INFO:teuthology.orchestra.run.smithi022.stdout: rbd-nbd x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 165 k 2024-09-21T09:20:04.831 INFO:teuthology.orchestra.run.smithi022.stdout:Upgrading: 2024-09-21T09:20:04.832 INFO:teuthology.orchestra.run.smithi022.stdout: librados2 x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 3.1 M 2024-09-21T09:20:04.832 INFO:teuthology.orchestra.run.smithi022.stdout: librbd1 x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 2.9 M 2024-09-21T09:20:04.832 INFO:teuthology.orchestra.run.smithi022.stdout:Installing dependencies: 2024-09-21T09:20:04.832 INFO:teuthology.orchestra.run.smithi022.stdout: ceph-common x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 22 M 2024-09-21T09:20:04.832 INFO:teuthology.orchestra.run.smithi022.stdout: ceph-grafana-dashboards noarch 2:17.2.7-1649.gfbd1a6ab.el9 ceph-noarch 24 k 2024-09-21T09:20:04.832 INFO:teuthology.orchestra.run.smithi022.stdout: ceph-mds x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 2.1 M 2024-09-21T09:20:04.832 INFO:teuthology.orchestra.run.smithi022.stdout: ceph-mgr-modules-core noarch 2:17.2.7-1649.gfbd1a6ab.el9 ceph-noarch 240 k 2024-09-21T09:20:04.832 INFO:teuthology.orchestra.run.smithi022.stdout: ceph-mon x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 3.9 M 2024-09-21T09:20:04.832 INFO:teuthology.orchestra.run.smithi022.stdout: ceph-osd x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 15 M 2024-09-21T09:20:04.832 INFO:teuthology.orchestra.run.smithi022.stdout: ceph-prometheus-alerts noarch 2:17.2.7-1649.gfbd1a6ab.el9 ceph-noarch 15 k 2024-09-21T09:20:04.832 INFO:teuthology.orchestra.run.smithi022.stdout: ceph-selinux x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 25 k 2024-09-21T09:20:04.832 INFO:teuthology.orchestra.run.smithi022.stdout: flexiblas x86_64 3.0.4-8.el9 appstream 33 k 2024-09-21T09:20:04.832 INFO:teuthology.orchestra.run.smithi022.stdout: flexiblas-netlib x86_64 3.0.4-8.el9 appstream 3.0 M 2024-09-21T09:20:04.832 INFO:teuthology.orchestra.run.smithi022.stdout: flexiblas-openblas-openmp x86_64 3.0.4-8.el9 appstream 18 k 2024-09-21T09:20:04.833 INFO:teuthology.orchestra.run.smithi022.stdout: fuse x86_64 2.9.9-16.el9 baseos 80 k 2024-09-21T09:20:04.833 INFO:teuthology.orchestra.run.smithi022.stdout: gperftools-libs x86_64 2.9.1-3.el9 epel 308 k 2024-09-21T09:20:04.833 INFO:teuthology.orchestra.run.smithi022.stdout: libcephsqlite x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 163 k 2024-09-21T09:20:04.833 INFO:teuthology.orchestra.run.smithi022.stdout: libgfortran x86_64 11.5.0-2.el9 baseos 801 k 2024-09-21T09:20:04.833 INFO:teuthology.orchestra.run.smithi022.stdout: liboath x86_64 2.6.7-2.el9 epel 49 k 2024-09-21T09:20:04.833 INFO:teuthology.orchestra.run.smithi022.stdout: libpmemobj x86_64 1.12.1-1.el9 appstream 160 k 2024-09-21T09:20:04.833 INFO:teuthology.orchestra.run.smithi022.stdout: libquadmath x86_64 11.5.0-2.el9 baseos 191 k 2024-09-21T09:20:04.833 INFO:teuthology.orchestra.run.smithi022.stdout: librabbitmq x86_64 0.11.0-7.el9 appstream 45 k 2024-09-21T09:20:04.833 INFO:teuthology.orchestra.run.smithi022.stdout: libradosstriper1 x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 457 k 2024-09-21T09:20:04.833 INFO:teuthology.orchestra.run.smithi022.stdout: librdkafka x86_64 1.6.1-102.el9 appstream 662 k 2024-09-21T09:20:04.833 INFO:teuthology.orchestra.run.smithi022.stdout: librgw2 x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 4.9 M 2024-09-21T09:20:04.833 INFO:teuthology.orchestra.run.smithi022.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2024-09-21T09:20:04.833 INFO:teuthology.orchestra.run.smithi022.stdout: lttng-ust x86_64 2.12.0-6.el9 appstream 292 k 2024-09-21T09:20:04.834 INFO:teuthology.orchestra.run.smithi022.stdout: openblas x86_64 0.3.26-2.el9 appstream 38 k 2024-09-21T09:20:04.834 INFO:teuthology.orchestra.run.smithi022.stdout: openblas-openmp x86_64 0.3.26-2.el9 appstream 4.9 M 2024-09-21T09:20:04.834 INFO:teuthology.orchestra.run.smithi022.stdout: python3-asyncssh noarch 2.13.2-5.el9 epel 548 k 2024-09-21T09:20:04.834 INFO:teuthology.orchestra.run.smithi022.stdout: python3-attrs noarch 20.3.0-7.el9 appstream 87 k 2024-09-21T09:20:04.834 INFO:teuthology.orchestra.run.smithi022.stdout: python3-babel noarch 2.9.1-2.el9 appstream 6.0 M 2024-09-21T09:20:04.834 INFO:teuthology.orchestra.run.smithi022.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2024-09-21T09:20:04.834 INFO:teuthology.orchestra.run.smithi022.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2024-09-21T09:20:04.834 INFO:teuthology.orchestra.run.smithi022.stdout: python3-ceph-argparse x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 45 k 2024-09-21T09:20:04.834 INFO:teuthology.orchestra.run.smithi022.stdout: python3-ceph-common x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 120 k 2024-09-21T09:20:04.834 INFO:teuthology.orchestra.run.smithi022.stdout: python3-certifi noarch 2023.05.07-4.el9 epel 14 k 2024-09-21T09:20:04.834 INFO:teuthology.orchestra.run.smithi022.stdout: python3-cffi x86_64 1.14.5-5.el9 baseos 253 k 2024-09-21T09:20:04.835 INFO:teuthology.orchestra.run.smithi022.stdout: python3-cheroot noarch 8.6.0-4.el9 epel 172 k 2024-09-21T09:20:04.835 INFO:teuthology.orchestra.run.smithi022.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2024-09-21T09:20:04.835 INFO:teuthology.orchestra.run.smithi022.stdout: python3-cryptography x86_64 36.0.1-4.el9 baseos 1.2 M 2024-09-21T09:20:04.835 INFO:teuthology.orchestra.run.smithi022.stdout: python3-devel x86_64 3.9.19-8.el9 appstream 245 k 2024-09-21T09:20:04.835 INFO:teuthology.orchestra.run.smithi022.stdout: python3-google-auth noarch 1:2.34.0-1.el9 epel 220 k 2024-09-21T09:20:04.835 INFO:teuthology.orchestra.run.smithi022.stdout: python3-iniconfig noarch 1.1.1-7.el9 appstream 17 k 2024-09-21T09:20:04.835 INFO:teuthology.orchestra.run.smithi022.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2024-09-21T09:20:04.835 INFO:teuthology.orchestra.run.smithi022.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2024-09-21T09:20:04.835 INFO:teuthology.orchestra.run.smithi022.stdout: python3-jaraco-collections noarch 3.0.0-8.el9 epel 23 k 2024-09-21T09:20:04.835 INFO:teuthology.orchestra.run.smithi022.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2024-09-21T09:20:04.835 INFO:teuthology.orchestra.run.smithi022.stdout: python3-jaraco-text noarch 3.2.0-6.el9 epel 20 k 2024-09-21T09:20:04.835 INFO:teuthology.orchestra.run.smithi022.stdout: python3-jinja2 noarch 2.11.3-6.el9 appstream 248 k 2024-09-21T09:20:04.835 INFO:teuthology.orchestra.run.smithi022.stdout: python3-jsonpatch noarch 1.21-16.el9 appstream 26 k 2024-09-21T09:20:04.836 INFO:teuthology.orchestra.run.smithi022.stdout: python3-jsonpointer noarch 2.0-4.el9 appstream 19 k 2024-09-21T09:20:04.836 INFO:teuthology.orchestra.run.smithi022.stdout: python3-jwt noarch 2.4.0-1.el9 epel 41 k 2024-09-21T09:20:04.836 INFO:teuthology.orchestra.run.smithi022.stdout: python3-kubernetes noarch 1:26.1.0-2.el9 epel 1.0 M 2024-09-21T09:20:04.836 INFO:teuthology.orchestra.run.smithi022.stdout: python3-logutils noarch 0.3.5-21.el9 epel 46 k 2024-09-21T09:20:04.836 INFO:teuthology.orchestra.run.smithi022.stdout: python3-mako noarch 1.1.4-6.el9 appstream 172 k 2024-09-21T09:20:04.836 INFO:teuthology.orchestra.run.smithi022.stdout: python3-markupsafe x86_64 1.1.1-12.el9 appstream 35 k 2024-09-21T09:20:04.836 INFO:teuthology.orchestra.run.smithi022.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2024-09-21T09:20:04.836 INFO:teuthology.orchestra.run.smithi022.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2024-09-21T09:20:04.836 INFO:teuthology.orchestra.run.smithi022.stdout: python3-numpy x86_64 1:1.20.1-5.el9 appstream 5.1 M 2024-09-21T09:20:04.836 INFO:teuthology.orchestra.run.smithi022.stdout: python3-numpy-f2py x86_64 1:1.20.1-5.el9 appstream 250 k 2024-09-21T09:20:04.836 INFO:teuthology.orchestra.run.smithi022.stdout: python3-oauthlib noarch 3.1.1-5.el9 appstream 222 k 2024-09-21T09:20:04.836 INFO:teuthology.orchestra.run.smithi022.stdout: python3-packaging noarch 20.9-5.el9 appstream 77 k 2024-09-21T09:20:04.836 INFO:teuthology.orchestra.run.smithi022.stdout: python3-pecan noarch 1.4.2-3.el9 epel 272 k 2024-09-21T09:20:04.836 INFO:teuthology.orchestra.run.smithi022.stdout: python3-pluggy noarch 0.13.1-7.el9 appstream 41 k 2024-09-21T09:20:04.837 INFO:teuthology.orchestra.run.smithi022.stdout: python3-ply noarch 3.11-14.el9 baseos 106 k 2024-09-21T09:20:04.837 INFO:teuthology.orchestra.run.smithi022.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2024-09-21T09:20:04.837 INFO:teuthology.orchestra.run.smithi022.stdout: python3-prettytable noarch 0.7.2-27.el9 appstream 42 k 2024-09-21T09:20:04.837 INFO:teuthology.orchestra.run.smithi022.stdout: python3-py noarch 1.10.0-6.el9 appstream 477 k 2024-09-21T09:20:04.837 INFO:teuthology.orchestra.run.smithi022.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2024-09-21T09:20:04.837 INFO:teuthology.orchestra.run.smithi022.stdout: python3-pyasn1 noarch 0.4.8-6.el9 appstream 159 k 2024-09-21T09:20:04.837 INFO:teuthology.orchestra.run.smithi022.stdout: python3-pyasn1-modules noarch 0.4.8-6.el9 appstream 279 k 2024-09-21T09:20:04.837 INFO:teuthology.orchestra.run.smithi022.stdout: python3-pycparser noarch 2.20-6.el9 baseos 135 k 2024-09-21T09:20:04.837 INFO:teuthology.orchestra.run.smithi022.stdout: python3-pytz noarch 2021.1-5.el9 appstream 51 k 2024-09-21T09:20:04.837 INFO:teuthology.orchestra.run.smithi022.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2024-09-21T09:20:04.837 INFO:teuthology.orchestra.run.smithi022.stdout: python3-requests noarch 2.25.1-8.el9 baseos 125 k 2024-09-21T09:20:04.837 INFO:teuthology.orchestra.run.smithi022.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 appstream 54 k 2024-09-21T09:20:04.837 INFO:teuthology.orchestra.run.smithi022.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2024-09-21T09:20:04.837 INFO:teuthology.orchestra.run.smithi022.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2024-09-21T09:20:04.838 INFO:teuthology.orchestra.run.smithi022.stdout: python3-scipy x86_64 1.6.2-8.el9 appstream 16 M 2024-09-21T09:20:04.838 INFO:teuthology.orchestra.run.smithi022.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2024-09-21T09:20:04.838 INFO:teuthology.orchestra.run.smithi022.stdout: python3-toml noarch 0.10.2-6.el9 appstream 42 k 2024-09-21T09:20:04.838 INFO:teuthology.orchestra.run.smithi022.stdout: python3-typing-extensions noarch 4.12.2-2.el9 epel 74 k 2024-09-21T09:20:04.838 INFO:teuthology.orchestra.run.smithi022.stdout: python3-urllib3 noarch 1.26.5-6.el9 baseos 215 k 2024-09-21T09:20:04.838 INFO:teuthology.orchestra.run.smithi022.stdout: python3-webob noarch 1.8.8-2.el9 epel 230 k 2024-09-21T09:20:04.838 INFO:teuthology.orchestra.run.smithi022.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2024-09-21T09:20:04.838 INFO:teuthology.orchestra.run.smithi022.stdout: python3-werkzeug noarch 2.0.3-3.el9.1 epel 427 k 2024-09-21T09:20:04.838 INFO:teuthology.orchestra.run.smithi022.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2024-09-21T09:20:04.838 INFO:teuthology.orchestra.run.smithi022.stdout: socat x86_64 1.7.4.1-6.el9 appstream 304 k 2024-09-21T09:20:04.838 INFO:teuthology.orchestra.run.smithi022.stdout: thrift x86_64 0.15.0-2.el9 epel 1.6 M 2024-09-21T09:20:04.838 INFO:teuthology.orchestra.run.smithi022.stdout: xmlstarlet x86_64 1.6.1-20.el9 appstream 64 k 2024-09-21T09:20:04.838 INFO:teuthology.orchestra.run.smithi022.stdout:Installing weak dependencies: 2024-09-21T09:20:04.839 INFO:teuthology.orchestra.run.smithi022.stdout: python3-jwt+crypto noarch 2.4.0-1.el9 epel 9.0 k 2024-09-21T09:20:04.839 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-09-21T09:20:04.839 INFO:teuthology.orchestra.run.smithi022.stdout:Transaction Summary 2024-09-21T09:20:04.839 INFO:teuthology.orchestra.run.smithi022.stdout:======================================================================================= 2024-09-21T09:20:04.839 INFO:teuthology.orchestra.run.smithi022.stdout:Install 112 Packages 2024-09-21T09:20:04.839 INFO:teuthology.orchestra.run.smithi022.stdout:Upgrade 2 Packages 2024-09-21T09:20:04.839 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-09-21T09:20:04.840 INFO:teuthology.orchestra.run.smithi022.stdout:Total download size: 190 M 2024-09-21T09:20:04.840 INFO:teuthology.orchestra.run.smithi022.stdout:Downloading Packages: 2024-09-21T09:20:05.258 INFO:teuthology.orchestra.run.smithi022.stdout:(1/114): ceph-17.2.7-1649.gfbd1a6ab.el9.x86_64. 48 kB/s | 6.5 kB 00:00 2024-09-21T09:20:05.567 INFO:teuthology.orchestra.run.smithi022.stdout:(2/114): ceph-base-17.2.7-1649.gfbd1a6ab.el9.x8 11 MB/s | 5.0 MB 00:00 2024-09-21T09:20:05.601 INFO:teuthology.orchestra.run.smithi022.stdout:(3/114): ceph-fuse-17.2.7-1649.gfbd1a6ab.el9.x8 2.3 MB/s | 813 kB 00:00 2024-09-21T09:20:05.626 INFO:teuthology.orchestra.run.smithi022.stdout:(4/114): ceph-immutable-object-cache-17.2.7-164 2.3 MB/s | 137 kB 00:00 2024-09-21T09:20:05.709 INFO:teuthology.orchestra.run.smithi022.stdout:(5/114): ceph-mgr-17.2.7-1649.gfbd1a6ab.el9.x86 17 MB/s | 1.4 MB 00:00 2024-09-21T09:20:05.876 INFO:teuthology.orchestra.run.smithi022.stdout:(6/114): ceph-mds-17.2.7-1649.gfbd1a6ab.el9.x86 7.5 MB/s | 2.1 MB 00:00 2024-09-21T09:20:05.943 INFO:teuthology.orchestra.run.smithi022.stdout:(7/114): ceph-mon-17.2.7-1649.gfbd1a6ab.el9.x86 17 MB/s | 3.9 MB 00:00 2024-09-21T09:20:06.444 INFO:teuthology.orchestra.run.smithi022.stdout:(8/114): ceph-common-17.2.7-1649.gfbd1a6ab.el9. 17 MB/s | 22 MB 00:01 2024-09-21T09:20:06.503 INFO:teuthology.orchestra.run.smithi022.stdout:(9/114): ceph-selinux-17.2.7-1649.gfbd1a6ab.el9 430 kB/s | 25 kB 00:00 2024-09-21T09:20:06.745 INFO:teuthology.orchestra.run.smithi022.stdout:(10/114): ceph-radosgw-17.2.7-1649.gfbd1a6ab.el 18 MB/s | 14 MB 00:00 2024-09-21T09:20:06.795 INFO:teuthology.orchestra.run.smithi022.stdout:(11/114): libcephfs-devel-17.2.7-1649.gfbd1a6ab 503 kB/s | 25 kB 00:00 2024-09-21T09:20:06.860 INFO:teuthology.orchestra.run.smithi064.stdout:Package librados2-2:16.2.4-5.el9.x86_64 is already installed. 2024-09-21T09:20:06.860 INFO:teuthology.orchestra.run.smithi064.stdout:Package librbd1-2:16.2.4-5.el9.x86_64 is already installed. 2024-09-21T09:20:06.870 INFO:teuthology.orchestra.run.smithi022.stdout:(12/114): libcephfs2-17.2.7-1649.gfbd1a6ab.el9. 8.3 MB/s | 636 kB 00:00 2024-09-21T09:20:06.930 INFO:teuthology.orchestra.run.smithi064.stdout:Dependencies resolved. 2024-09-21T09:20:06.937 INFO:teuthology.orchestra.run.smithi022.stdout:(13/114): libcephsqlite-17.2.7-1649.gfbd1a6ab.e 2.4 MB/s | 163 kB 00:00 2024-09-21T09:20:06.937 INFO:teuthology.orchestra.run.smithi064.stdout:======================================================================================= 2024-09-21T09:20:06.937 INFO:teuthology.orchestra.run.smithi064.stdout: Package Arch Version Repository Size 2024-09-21T09:20:06.937 INFO:teuthology.orchestra.run.smithi064.stdout:======================================================================================= 2024-09-21T09:20:06.937 INFO:teuthology.orchestra.run.smithi064.stdout:Installing: 2024-09-21T09:20:06.937 INFO:teuthology.orchestra.run.smithi064.stdout: ceph x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 6.5 k 2024-09-21T09:20:06.937 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-base x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 5.0 M 2024-09-21T09:20:06.938 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-fuse x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 813 k 2024-09-21T09:20:06.938 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-immutable-object-cache x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 137 k 2024-09-21T09:20:06.938 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-mgr x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 1.4 M 2024-09-21T09:20:06.938 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-mgr-cephadm noarch 2:17.2.7-1649.gfbd1a6ab.el9 ceph-noarch 124 k 2024-09-21T09:20:06.938 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-mgr-dashboard noarch 2:17.2.7-1649.gfbd1a6ab.el9 ceph-noarch 3.4 M 2024-09-21T09:20:06.938 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-mgr-diskprediction-local noarch 2:17.2.7-1649.gfbd1a6ab.el9 ceph-noarch 7.4 M 2024-09-21T09:20:06.938 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-mgr-rook noarch 2:17.2.7-1649.gfbd1a6ab.el9 ceph-noarch 47 k 2024-09-21T09:20:06.938 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-radosgw x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 14 M 2024-09-21T09:20:06.938 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-test x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 48 M 2024-09-21T09:20:06.938 INFO:teuthology.orchestra.run.smithi064.stdout: cephadm noarch 2:17.2.7-1649.gfbd1a6ab.el9 ceph-noarch 83 k 2024-09-21T09:20:06.938 INFO:teuthology.orchestra.run.smithi064.stdout: libcephfs-devel x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 25 k 2024-09-21T09:20:06.938 INFO:teuthology.orchestra.run.smithi064.stdout: libcephfs2 x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 636 k 2024-09-21T09:20:06.939 INFO:teuthology.orchestra.run.smithi064.stdout: librados-devel x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 126 k 2024-09-21T09:20:06.939 INFO:teuthology.orchestra.run.smithi064.stdout: python3-cephfs x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 156 k 2024-09-21T09:20:06.939 INFO:teuthology.orchestra.run.smithi064.stdout: python3-pytest noarch 6.2.2-7.el9 appstream 519 k 2024-09-21T09:20:06.939 INFO:teuthology.orchestra.run.smithi064.stdout: python3-rados x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 320 k 2024-09-21T09:20:06.939 INFO:teuthology.orchestra.run.smithi064.stdout: python3-rbd x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 295 k 2024-09-21T09:20:06.939 INFO:teuthology.orchestra.run.smithi064.stdout: python3-rgw x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 99 k 2024-09-21T09:20:06.939 INFO:teuthology.orchestra.run.smithi064.stdout: rbd-fuse x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 82 k 2024-09-21T09:20:06.939 INFO:teuthology.orchestra.run.smithi064.stdout: rbd-mirror x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 2.9 M 2024-09-21T09:20:06.939 INFO:teuthology.orchestra.run.smithi064.stdout: rbd-nbd x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 165 k 2024-09-21T09:20:06.939 INFO:teuthology.orchestra.run.smithi064.stdout:Upgrading: 2024-09-21T09:20:06.939 INFO:teuthology.orchestra.run.smithi064.stdout: librados2 x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 3.1 M 2024-09-21T09:20:06.939 INFO:teuthology.orchestra.run.smithi064.stdout: librbd1 x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 2.9 M 2024-09-21T09:20:06.939 INFO:teuthology.orchestra.run.smithi064.stdout:Installing dependencies: 2024-09-21T09:20:06.939 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-common x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 22 M 2024-09-21T09:20:06.940 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-grafana-dashboards noarch 2:17.2.7-1649.gfbd1a6ab.el9 ceph-noarch 24 k 2024-09-21T09:20:06.940 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-mds x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 2.1 M 2024-09-21T09:20:06.940 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-mgr-modules-core noarch 2:17.2.7-1649.gfbd1a6ab.el9 ceph-noarch 240 k 2024-09-21T09:20:06.940 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-mon x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 3.9 M 2024-09-21T09:20:06.940 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-osd x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 15 M 2024-09-21T09:20:06.940 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-prometheus-alerts noarch 2:17.2.7-1649.gfbd1a6ab.el9 ceph-noarch 15 k 2024-09-21T09:20:06.940 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-selinux x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 25 k 2024-09-21T09:20:06.940 INFO:teuthology.orchestra.run.smithi064.stdout: flexiblas x86_64 3.0.4-8.el9 appstream 33 k 2024-09-21T09:20:06.940 INFO:teuthology.orchestra.run.smithi064.stdout: flexiblas-netlib x86_64 3.0.4-8.el9 appstream 3.0 M 2024-09-21T09:20:06.940 INFO:teuthology.orchestra.run.smithi064.stdout: flexiblas-openblas-openmp x86_64 3.0.4-8.el9 appstream 18 k 2024-09-21T09:20:06.940 INFO:teuthology.orchestra.run.smithi064.stdout: fuse x86_64 2.9.9-16.el9 baseos 80 k 2024-09-21T09:20:06.940 INFO:teuthology.orchestra.run.smithi064.stdout: gperftools-libs x86_64 2.9.1-3.el9 epel 308 k 2024-09-21T09:20:06.940 INFO:teuthology.orchestra.run.smithi064.stdout: libcephsqlite x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 163 k 2024-09-21T09:20:06.940 INFO:teuthology.orchestra.run.smithi064.stdout: libgfortran x86_64 11.5.0-2.el9 baseos 801 k 2024-09-21T09:20:06.940 INFO:teuthology.orchestra.run.smithi064.stdout: liboath x86_64 2.6.7-2.el9 epel 49 k 2024-09-21T09:20:06.941 INFO:teuthology.orchestra.run.smithi064.stdout: libpmemobj x86_64 1.12.1-1.el9 appstream 160 k 2024-09-21T09:20:06.941 INFO:teuthology.orchestra.run.smithi064.stdout: libquadmath x86_64 11.5.0-2.el9 baseos 191 k 2024-09-21T09:20:06.941 INFO:teuthology.orchestra.run.smithi064.stdout: librabbitmq x86_64 0.11.0-7.el9 appstream 45 k 2024-09-21T09:20:06.941 INFO:teuthology.orchestra.run.smithi064.stdout: libradosstriper1 x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 457 k 2024-09-21T09:20:06.941 INFO:teuthology.orchestra.run.smithi064.stdout: librdkafka x86_64 1.6.1-102.el9 appstream 662 k 2024-09-21T09:20:06.941 INFO:teuthology.orchestra.run.smithi064.stdout: librgw2 x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 4.9 M 2024-09-21T09:20:06.941 INFO:teuthology.orchestra.run.smithi064.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2024-09-21T09:20:06.941 INFO:teuthology.orchestra.run.smithi064.stdout: lttng-ust x86_64 2.12.0-6.el9 appstream 292 k 2024-09-21T09:20:06.941 INFO:teuthology.orchestra.run.smithi064.stdout: openblas x86_64 0.3.26-2.el9 appstream 38 k 2024-09-21T09:20:06.941 INFO:teuthology.orchestra.run.smithi064.stdout: openblas-openmp x86_64 0.3.26-2.el9 appstream 4.9 M 2024-09-21T09:20:06.941 INFO:teuthology.orchestra.run.smithi064.stdout: python3-asyncssh noarch 2.13.2-5.el9 epel 548 k 2024-09-21T09:20:06.941 INFO:teuthology.orchestra.run.smithi064.stdout: python3-attrs noarch 20.3.0-7.el9 appstream 87 k 2024-09-21T09:20:06.941 INFO:teuthology.orchestra.run.smithi064.stdout: python3-babel noarch 2.9.1-2.el9 appstream 6.0 M 2024-09-21T09:20:06.942 INFO:teuthology.orchestra.run.smithi064.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2024-09-21T09:20:06.942 INFO:teuthology.orchestra.run.smithi064.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2024-09-21T09:20:06.942 INFO:teuthology.orchestra.run.smithi064.stdout: python3-ceph-argparse x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 45 k 2024-09-21T09:20:06.942 INFO:teuthology.orchestra.run.smithi064.stdout: python3-ceph-common x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 120 k 2024-09-21T09:20:06.942 INFO:teuthology.orchestra.run.smithi064.stdout: python3-certifi noarch 2023.05.07-4.el9 epel 14 k 2024-09-21T09:20:06.942 INFO:teuthology.orchestra.run.smithi064.stdout: python3-cffi x86_64 1.14.5-5.el9 baseos 253 k 2024-09-21T09:20:06.942 INFO:teuthology.orchestra.run.smithi064.stdout: python3-cheroot noarch 8.6.0-4.el9 epel 172 k 2024-09-21T09:20:06.942 INFO:teuthology.orchestra.run.smithi064.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2024-09-21T09:20:06.942 INFO:teuthology.orchestra.run.smithi064.stdout: python3-cryptography x86_64 36.0.1-4.el9 baseos 1.2 M 2024-09-21T09:20:06.942 INFO:teuthology.orchestra.run.smithi064.stdout: python3-devel x86_64 3.9.19-8.el9 appstream 245 k 2024-09-21T09:20:06.942 INFO:teuthology.orchestra.run.smithi064.stdout: python3-google-auth noarch 1:2.34.0-1.el9 epel 220 k 2024-09-21T09:20:06.942 INFO:teuthology.orchestra.run.smithi064.stdout: python3-iniconfig noarch 1.1.1-7.el9 appstream 17 k 2024-09-21T09:20:06.942 INFO:teuthology.orchestra.run.smithi064.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2024-09-21T09:20:06.943 INFO:teuthology.orchestra.run.smithi064.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2024-09-21T09:20:06.943 INFO:teuthology.orchestra.run.smithi064.stdout: python3-jaraco-collections noarch 3.0.0-8.el9 epel 23 k 2024-09-21T09:20:06.943 INFO:teuthology.orchestra.run.smithi064.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2024-09-21T09:20:06.943 INFO:teuthology.orchestra.run.smithi064.stdout: python3-jaraco-text noarch 3.2.0-6.el9 epel 20 k 2024-09-21T09:20:06.943 INFO:teuthology.orchestra.run.smithi064.stdout: python3-jinja2 noarch 2.11.3-6.el9 appstream 248 k 2024-09-21T09:20:06.943 INFO:teuthology.orchestra.run.smithi064.stdout: python3-jsonpatch noarch 1.21-16.el9 appstream 26 k 2024-09-21T09:20:06.943 INFO:teuthology.orchestra.run.smithi064.stdout: python3-jsonpointer noarch 2.0-4.el9 appstream 19 k 2024-09-21T09:20:06.943 INFO:teuthology.orchestra.run.smithi064.stdout: python3-jwt noarch 2.4.0-1.el9 epel 41 k 2024-09-21T09:20:06.943 INFO:teuthology.orchestra.run.smithi064.stdout: python3-kubernetes noarch 1:26.1.0-2.el9 epel 1.0 M 2024-09-21T09:20:06.943 INFO:teuthology.orchestra.run.smithi064.stdout: python3-logutils noarch 0.3.5-21.el9 epel 46 k 2024-09-21T09:20:06.943 INFO:teuthology.orchestra.run.smithi064.stdout: python3-mako noarch 1.1.4-6.el9 appstream 172 k 2024-09-21T09:20:06.943 INFO:teuthology.orchestra.run.smithi064.stdout: python3-markupsafe x86_64 1.1.1-12.el9 appstream 35 k 2024-09-21T09:20:06.943 INFO:teuthology.orchestra.run.smithi064.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2024-09-21T09:20:06.943 INFO:teuthology.orchestra.run.smithi064.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2024-09-21T09:20:06.943 INFO:teuthology.orchestra.run.smithi064.stdout: python3-numpy x86_64 1:1.20.1-5.el9 appstream 5.1 M 2024-09-21T09:20:06.944 INFO:teuthology.orchestra.run.smithi064.stdout: python3-numpy-f2py x86_64 1:1.20.1-5.el9 appstream 250 k 2024-09-21T09:20:06.944 INFO:teuthology.orchestra.run.smithi064.stdout: python3-oauthlib noarch 3.1.1-5.el9 appstream 222 k 2024-09-21T09:20:06.944 INFO:teuthology.orchestra.run.smithi064.stdout: python3-packaging noarch 20.9-5.el9 appstream 77 k 2024-09-21T09:20:06.944 INFO:teuthology.orchestra.run.smithi064.stdout: python3-pecan noarch 1.4.2-3.el9 epel 272 k 2024-09-21T09:20:06.944 INFO:teuthology.orchestra.run.smithi064.stdout: python3-pluggy noarch 0.13.1-7.el9 appstream 41 k 2024-09-21T09:20:06.944 INFO:teuthology.orchestra.run.smithi064.stdout: python3-ply noarch 3.11-14.el9 baseos 106 k 2024-09-21T09:20:06.944 INFO:teuthology.orchestra.run.smithi064.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2024-09-21T09:20:06.944 INFO:teuthology.orchestra.run.smithi064.stdout: python3-prettytable noarch 0.7.2-27.el9 appstream 42 k 2024-09-21T09:20:06.944 INFO:teuthology.orchestra.run.smithi064.stdout: python3-py noarch 1.10.0-6.el9 appstream 477 k 2024-09-21T09:20:06.944 INFO:teuthology.orchestra.run.smithi064.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2024-09-21T09:20:06.944 INFO:teuthology.orchestra.run.smithi064.stdout: python3-pyasn1 noarch 0.4.8-6.el9 appstream 159 k 2024-09-21T09:20:06.944 INFO:teuthology.orchestra.run.smithi064.stdout: python3-pyasn1-modules noarch 0.4.8-6.el9 appstream 279 k 2024-09-21T09:20:06.944 INFO:teuthology.orchestra.run.smithi064.stdout: python3-pycparser noarch 2.20-6.el9 baseos 135 k 2024-09-21T09:20:06.944 INFO:teuthology.orchestra.run.smithi064.stdout: python3-pytz noarch 2021.1-5.el9 appstream 51 k 2024-09-21T09:20:06.945 INFO:teuthology.orchestra.run.smithi064.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2024-09-21T09:20:06.945 INFO:teuthology.orchestra.run.smithi064.stdout: python3-requests noarch 2.25.1-8.el9 baseos 125 k 2024-09-21T09:20:06.945 INFO:teuthology.orchestra.run.smithi064.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 appstream 54 k 2024-09-21T09:20:06.945 INFO:teuthology.orchestra.run.smithi064.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2024-09-21T09:20:06.945 INFO:teuthology.orchestra.run.smithi064.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2024-09-21T09:20:06.945 INFO:teuthology.orchestra.run.smithi064.stdout: python3-scipy x86_64 1.6.2-8.el9 appstream 16 M 2024-09-21T09:20:06.945 INFO:teuthology.orchestra.run.smithi064.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2024-09-21T09:20:06.945 INFO:teuthology.orchestra.run.smithi064.stdout: python3-toml noarch 0.10.2-6.el9 appstream 42 k 2024-09-21T09:20:06.945 INFO:teuthology.orchestra.run.smithi064.stdout: python3-typing-extensions noarch 4.12.2-2.el9 epel 74 k 2024-09-21T09:20:06.945 INFO:teuthology.orchestra.run.smithi064.stdout: python3-urllib3 noarch 1.26.5-6.el9 baseos 215 k 2024-09-21T09:20:06.945 INFO:teuthology.orchestra.run.smithi064.stdout: python3-webob noarch 1.8.8-2.el9 epel 230 k 2024-09-21T09:20:06.945 INFO:teuthology.orchestra.run.smithi064.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2024-09-21T09:20:06.945 INFO:teuthology.orchestra.run.smithi064.stdout: python3-werkzeug noarch 2.0.3-3.el9.1 epel 427 k 2024-09-21T09:20:06.945 INFO:teuthology.orchestra.run.smithi064.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2024-09-21T09:20:06.946 INFO:teuthology.orchestra.run.smithi064.stdout: socat x86_64 1.7.4.1-6.el9 appstream 304 k 2024-09-21T09:20:06.946 INFO:teuthology.orchestra.run.smithi064.stdout: thrift x86_64 0.15.0-2.el9 epel 1.6 M 2024-09-21T09:20:06.946 INFO:teuthology.orchestra.run.smithi064.stdout: xmlstarlet x86_64 1.6.1-20.el9 appstream 64 k 2024-09-21T09:20:06.946 INFO:teuthology.orchestra.run.smithi064.stdout:Installing weak dependencies: 2024-09-21T09:20:06.946 INFO:teuthology.orchestra.run.smithi064.stdout: python3-jwt+crypto noarch 2.4.0-1.el9 epel 9.0 k 2024-09-21T09:20:06.946 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-09-21T09:20:06.946 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction Summary 2024-09-21T09:20:06.946 INFO:teuthology.orchestra.run.smithi064.stdout:======================================================================================= 2024-09-21T09:20:06.946 INFO:teuthology.orchestra.run.smithi064.stdout:Install 112 Packages 2024-09-21T09:20:06.946 INFO:teuthology.orchestra.run.smithi064.stdout:Upgrade 2 Packages 2024-09-21T09:20:06.946 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-09-21T09:20:06.947 INFO:teuthology.orchestra.run.smithi064.stdout:Total download size: 190 M 2024-09-21T09:20:06.947 INFO:teuthology.orchestra.run.smithi064.stdout:Downloading Packages: 2024-09-21T09:20:07.004 INFO:teuthology.orchestra.run.smithi022.stdout:(14/114): librados-devel-17.2.7-1649.gfbd1a6ab. 1.9 MB/s | 126 kB 00:00 2024-09-21T09:20:07.079 INFO:teuthology.orchestra.run.smithi022.stdout:(15/114): libradosstriper1-17.2.7-1649.gfbd1a6a 6.0 MB/s | 457 kB 00:00 2024-09-21T09:20:07.259 INFO:teuthology.orchestra.run.smithi064.stdout:(1/114): ceph-17.2.7-1649.gfbd1a6ab.el9.x86_64. 30 kB/s | 6.5 kB 00:00 2024-09-21T09:20:07.354 INFO:teuthology.orchestra.run.smithi022.stdout:(16/114): librgw2-17.2.7-1649.gfbd1a6ab.el9.x86 18 MB/s | 4.9 MB 00:00 2024-09-21T09:20:07.538 INFO:teuthology.orchestra.run.smithi022.stdout:(17/114): ceph-osd-17.2.7-1649.gfbd1a6ab.el9.x8 9.2 MB/s | 15 MB 00:01 2024-09-21T09:20:07.563 INFO:teuthology.orchestra.run.smithi022.stdout:(18/114): python3-ceph-argparse-17.2.7-1649.gfb 216 kB/s | 45 kB 00:00 2024-09-21T09:20:07.622 INFO:teuthology.orchestra.run.smithi022.stdout:(19/114): python3-ceph-common-17.2.7-1649.gfbd1 1.4 MB/s | 120 kB 00:00 2024-09-21T09:20:07.647 INFO:teuthology.orchestra.run.smithi022.stdout:(20/114): python3-cephfs-17.2.7-1649.gfbd1a6ab. 1.8 MB/s | 156 kB 00:00 2024-09-21T09:20:07.659 INFO:teuthology.orchestra.run.smithi064.stdout:(2/114): ceph-fuse-17.2.7-1649.gfbd1a6ab.el9.x8 2.0 MB/s | 813 kB 00:00 2024-09-21T09:20:07.697 INFO:teuthology.orchestra.run.smithi022.stdout:(21/114): python3-rados-17.2.7-1649.gfbd1a6ab.e 4.2 MB/s | 320 kB 00:00 2024-09-21T09:20:07.726 INFO:teuthology.orchestra.run.smithi064.stdout:(3/114): ceph-base-17.2.7-1649.gfbd1a6ab.el9.x8 7.3 MB/s | 5.0 MB 00:00 2024-09-21T09:20:07.731 INFO:teuthology.orchestra.run.smithi022.stdout:(22/114): python3-rbd-17.2.7-1649.gfbd1a6ab.el9 3.4 MB/s | 295 kB 00:00 2024-09-21T09:20:07.756 INFO:teuthology.orchestra.run.smithi022.stdout:(23/114): python3-rgw-17.2.7-1649.gfbd1a6ab.el9 1.7 MB/s | 99 kB 00:00 2024-09-21T09:20:07.759 INFO:teuthology.orchestra.run.smithi064.stdout:(4/114): ceph-immutable-object-cache-17.2.7-164 1.3 MB/s | 137 kB 00:00 2024-09-21T09:20:07.815 INFO:teuthology.orchestra.run.smithi022.stdout:(24/114): rbd-fuse-17.2.7-1649.gfbd1a6ab.el9.x8 978 kB/s | 82 kB 00:00 2024-09-21T09:20:07.902 INFO:teuthology.orchestra.run.smithi064.stdout:(5/114): ceph-mds-17.2.7-1649.gfbd1a6ab.el9.x86 12 MB/s | 2.1 MB 00:00 2024-09-21T09:20:07.907 INFO:teuthology.orchestra.run.smithi022.stdout:(25/114): rbd-nbd-17.2.7-1649.gfbd1a6ab.el9.x86 1.8 MB/s | 165 kB 00:00 2024-09-21T09:20:08.007 INFO:teuthology.orchestra.run.smithi022.stdout:(26/114): ceph-grafana-dashboards-17.2.7-1649.g 244 kB/s | 24 kB 00:00 2024-09-21T09:20:08.116 INFO:teuthology.orchestra.run.smithi022.stdout:(27/114): ceph-mgr-cephadm-17.2.7-1649.gfbd1a6a 1.1 MB/s | 124 kB 00:00 2024-09-21T09:20:08.283 INFO:teuthology.orchestra.run.smithi022.stdout:(28/114): rbd-mirror-17.2.7-1649.gfbd1a6ab.el9. 5.6 MB/s | 2.9 MB 00:00 2024-09-21T09:20:08.319 INFO:teuthology.orchestra.run.smithi064.stdout:(6/114): ceph-mon-17.2.7-1649.gfbd1a6ab.el9.x86 9.4 MB/s | 3.9 MB 00:00 2024-09-21T09:20:08.661 INFO:teuthology.orchestra.run.smithi064.stdout:(7/114): ceph-common-17.2.7-1649.gfbd1a6ab.el9. 14 MB/s | 22 MB 00:01 2024-09-21T09:20:08.691 INFO:teuthology.orchestra.run.smithi022.stdout:(29/114): ceph-mgr-dashboard-17.2.7-1649.gfbd1a 5.8 MB/s | 3.4 MB 00:00 2024-09-21T09:20:08.702 INFO:teuthology.orchestra.run.smithi064.stdout:(8/114): ceph-mgr-17.2.7-1649.gfbd1a6ab.el9.x86 1.5 MB/s | 1.4 MB 00:00 2024-09-21T09:20:08.750 INFO:teuthology.orchestra.run.smithi022.stdout:(30/114): ceph-mgr-modules-core-17.2.7-1649.gfb 4.0 MB/s | 240 kB 00:00 2024-09-21T09:20:08.786 INFO:teuthology.orchestra.run.smithi064.stdout:(9/114): ceph-selinux-17.2.7-1649.gfbd1a6ab.el9 301 kB/s | 25 kB 00:00 2024-09-21T09:20:08.850 INFO:teuthology.orchestra.run.smithi022.stdout:(31/114): ceph-mgr-rook-17.2.7-1649.gfbd1a6ab.e 475 kB/s | 47 kB 00:00 2024-09-21T09:20:09.309 INFO:teuthology.orchestra.run.smithi022.stdout:(32/114): ceph-test-17.2.7-1649.gfbd1a6ab.el9.x 17 MB/s | 48 MB 00:02 2024-09-21T09:20:09.326 INFO:teuthology.orchestra.run.smithi022.stdout:(33/114): ceph-prometheus-alerts-17.2.7-1649.gf 31 kB/s | 15 kB 00:00 2024-09-21T09:20:09.351 INFO:teuthology.orchestra.run.smithi022.stdout:(34/114): cephadm-17.2.7-1649.gfbd1a6ab.el9.noa 1.9 MB/s | 83 kB 00:00 2024-09-21T09:20:09.420 INFO:teuthology.orchestra.run.smithi064.stdout:(10/114): ceph-osd-17.2.7-1649.gfbd1a6ab.el9.x8 14 MB/s | 15 MB 00:01 2024-09-21T09:20:09.452 INFO:teuthology.orchestra.run.smithi022.stdout:(35/114): ceph-mgr-diskprediction-local-17.2.7- 6.3 MB/s | 7.4 MB 00:01 2024-09-21T09:20:09.477 INFO:teuthology.orchestra.run.smithi022.stdout:(36/114): fuse-2.9.9-16.el9.x86_64.rpm 529 kB/s | 80 kB 00:00 2024-09-21T09:20:09.562 INFO:teuthology.orchestra.run.smithi064.stdout:(11/114): ceph-radosgw-17.2.7-1649.gfbd1a6ab.el 16 MB/s | 14 MB 00:00 2024-09-21T09:20:09.569 INFO:teuthology.orchestra.run.smithi022.stdout:(37/114): python3-cffi-1.14.5-5.el9.x86_64.rpm 2.7 MB/s | 253 kB 00:00 2024-09-21T09:20:09.587 INFO:teuthology.orchestra.run.smithi064.stdout:(12/114): libcephfs-devel-17.2.7-1649.gfbd1a6ab 150 kB/s | 25 kB 00:00 2024-09-21T09:20:09.594 INFO:teuthology.orchestra.run.smithi022.stdout:(38/114): libquadmath-11.5.0-2.el9.x86_64.rpm 1.3 MB/s | 191 kB 00:00 2024-09-21T09:20:09.629 INFO:teuthology.orchestra.run.smithi064.stdout:(13/114): libcephfs2-17.2.7-1649.gfbd1a6ab.el9. 9.3 MB/s | 636 kB 00:00 2024-09-21T09:20:09.636 INFO:teuthology.orchestra.run.smithi022.stdout:(39/114): libgfortran-11.5.0-2.el9.x86_64.rpm 2.7 MB/s | 801 kB 00:00 2024-09-21T09:20:09.654 INFO:teuthology.orchestra.run.smithi064.stdout:(14/114): libcephsqlite-17.2.7-1649.gfbd1a6ab.e 2.4 MB/s | 163 kB 00:00 2024-09-21T09:20:09.662 INFO:teuthology.orchestra.run.smithi022.stdout:(40/114): python3-ply-3.11-14.el9.noarch.rpm 1.6 MB/s | 106 kB 00:00 2024-09-21T09:20:09.703 INFO:teuthology.orchestra.run.smithi022.stdout:(41/114): python3-requests-2.25.1-8.el9.noarch. 2.9 MB/s | 125 kB 00:00 2024-09-21T09:20:09.704 INFO:teuthology.orchestra.run.smithi064.stdout:(15/114): librados-devel-17.2.7-1649.gfbd1a6ab. 1.6 MB/s | 126 kB 00:00 2024-09-21T09:20:09.729 INFO:teuthology.orchestra.run.smithi022.stdout:(42/114): python3-pycparser-2.20-6.el9.noarch.r 1.4 MB/s | 135 kB 00:00 2024-09-21T09:20:09.738 INFO:teuthology.orchestra.run.smithi064.stdout:(16/114): libradosstriper1-17.2.7-1649.gfbd1a6a 5.4 MB/s | 457 kB 00:00 2024-09-21T09:20:09.796 INFO:teuthology.orchestra.run.smithi022.stdout:(43/114): python3-cryptography-36.0.1-4.el9.x86 5.4 MB/s | 1.2 MB 00:00 2024-09-21T09:20:09.796 INFO:teuthology.orchestra.run.smithi064.stdout:(17/114): python3-ceph-argparse-17.2.7-1649.gfb 773 kB/s | 45 kB 00:00 2024-09-21T09:20:09.821 INFO:teuthology.orchestra.run.smithi022.stdout:(44/114): python3-urllib3-1.26.5-6.el9.noarch.r 1.8 MB/s | 215 kB 00:00 2024-09-21T09:20:09.848 INFO:teuthology.orchestra.run.smithi022.stdout:(45/114): flexiblas-3.0.4-8.el9.x86_64.rpm 275 kB/s | 33 kB 00:00 2024-09-21T09:20:09.873 INFO:teuthology.orchestra.run.smithi022.stdout:(46/114): flexiblas-openblas-openmp-3.0.4-8.el9 352 kB/s | 18 kB 00:00 2024-09-21T09:20:09.905 INFO:teuthology.orchestra.run.smithi064.stdout:(18/114): librgw2-17.2.7-1649.gfbd1a6ab.el9.x86 25 MB/s | 4.9 MB 00:00 2024-09-21T09:20:09.930 INFO:teuthology.orchestra.run.smithi064.stdout:(19/114): python3-ceph-common-17.2.7-1649.gfbd1 899 kB/s | 120 kB 00:00 2024-09-21T09:20:09.956 INFO:teuthology.orchestra.run.smithi064.stdout:(20/114): python3-cephfs-17.2.7-1649.gfbd1a6ab. 3.0 MB/s | 156 kB 00:00 2024-09-21T09:20:09.956 INFO:teuthology.orchestra.run.smithi022.stdout:(47/114): librabbitmq-0.11.0-7.el9.x86_64.rpm 544 kB/s | 45 kB 00:00 2024-09-21T09:20:09.981 INFO:teuthology.orchestra.run.smithi064.stdout:(21/114): python3-rados-17.2.7-1649.gfbd1a6ab.e 6.2 MB/s | 320 kB 00:00 2024-09-21T09:20:10.015 INFO:teuthology.orchestra.run.smithi022.stdout:(48/114): libpmemobj-1.12.1-1.el9.x86_64.rpm 959 kB/s | 160 kB 00:00 2024-09-21T09:20:10.016 INFO:teuthology.orchestra.run.smithi064.stdout:(22/114): python3-rbd-17.2.7-1649.gfbd1a6ab.el9 4.8 MB/s | 295 kB 00:00 2024-09-21T09:20:10.050 INFO:teuthology.orchestra.run.smithi064.stdout:(23/114): python3-rgw-17.2.7-1649.gfbd1a6ab.el9 1.4 MB/s | 99 kB 00:00 2024-09-21T09:20:10.075 INFO:teuthology.orchestra.run.smithi064.stdout:(24/114): rbd-fuse-17.2.7-1649.gfbd1a6ab.el9.x8 1.4 MB/s | 82 kB 00:00 2024-09-21T09:20:10.091 INFO:teuthology.orchestra.run.smithi022.stdout:(49/114): flexiblas-netlib-3.0.4-8.el9.x86_64.r 10 MB/s | 3.0 MB 00:00 2024-09-21T09:20:10.125 INFO:teuthology.orchestra.run.smithi064.stdout:(25/114): rbd-nbd-17.2.7-1649.gfbd1a6ab.el9.x86 3.2 MB/s | 165 kB 00:00 2024-09-21T09:20:10.132 INFO:teuthology.orchestra.run.smithi022.stdout:(50/114): lttng-ust-2.12.0-6.el9.x86_64.rpm 2.4 MB/s | 292 kB 00:00 2024-09-21T09:20:10.159 INFO:teuthology.orchestra.run.smithi064.stdout:(26/114): ceph-grafana-dashboards-17.2.7-1649.g 730 kB/s | 24 kB 00:00 2024-09-21T09:20:10.166 INFO:teuthology.orchestra.run.smithi022.stdout:(51/114): librdkafka-1.6.1-102.el9.x86_64.rpm 3.1 MB/s | 662 kB 00:00 2024-09-21T09:20:10.183 INFO:teuthology.orchestra.run.smithi022.stdout:(52/114): openblas-0.3.26-2.el9.x86_64.rpm 418 kB/s | 38 kB 00:00 2024-09-21T09:20:10.218 INFO:teuthology.orchestra.run.smithi064.stdout:(27/114): rbd-mirror-17.2.7-1649.gfbd1a6ab.el9. 18 MB/s | 2.9 MB 00:00 2024-09-21T09:20:10.243 INFO:teuthology.orchestra.run.smithi064.stdout:(28/114): ceph-mgr-cephadm-17.2.7-1649.gfbd1a6a 1.5 MB/s | 124 kB 00:00 2024-09-21T09:20:10.317 INFO:teuthology.orchestra.run.smithi022.stdout:(53/114): python3-babel-2.9.1-2.el9.noarch.rpm 45 MB/s | 6.0 MB 00:00 2024-09-21T09:20:10.342 INFO:teuthology.orchestra.run.smithi022.stdout:(54/114): python3-attrs-20.3.0-7.el9.noarch.rpm 498 kB/s | 87 kB 00:00 2024-09-21T09:20:10.367 INFO:teuthology.orchestra.run.smithi022.stdout:(55/114): python3-devel-3.9.19-8.el9.x86_64.rpm 4.8 MB/s | 245 kB 00:00 2024-09-21T09:20:10.384 INFO:teuthology.orchestra.run.smithi022.stdout:(56/114): python3-iniconfig-1.1.1-7.el9.noarch. 417 kB/s | 17 kB 00:00 2024-09-21T09:20:10.410 INFO:teuthology.orchestra.run.smithi064.stdout:(29/114): ceph-mgr-dashboard-17.2.7-1649.gfbd1a 17 MB/s | 3.4 MB 00:00 2024-09-21T09:20:10.418 INFO:teuthology.orchestra.run.smithi022.stdout:(57/114): python3-jinja2-2.11.3-6.el9.noarch.rp 4.8 MB/s | 248 kB 00:00 2024-09-21T09:20:10.435 INFO:teuthology.orchestra.run.smithi022.stdout:(58/114): python3-jsonpatch-1.21-16.el9.noarch. 509 kB/s | 26 kB 00:00 2024-09-21T09:20:10.468 INFO:teuthology.orchestra.run.smithi064.stdout:(30/114): ceph-mgr-modules-core-17.2.7-1649.gfb 4.0 MB/s | 240 kB 00:00 2024-09-21T09:20:10.502 INFO:teuthology.orchestra.run.smithi022.stdout:(59/114): openblas-openmp-0.3.26-2.el9.x86_64.r 13 MB/s | 4.9 MB 00:00 2024-09-21T09:20:10.510 INFO:teuthology.orchestra.run.smithi064.stdout:(31/114): ceph-mgr-rook-17.2.7-1649.gfbd1a6ab.e 1.1 MB/s | 47 kB 00:00 2024-09-21T09:20:10.519 INFO:teuthology.orchestra.run.smithi022.stdout:(60/114): python3-jsonpointer-2.0-4.el9.noarch. 190 kB/s | 19 kB 00:00 2024-09-21T09:20:10.544 INFO:teuthology.orchestra.run.smithi022.stdout:(61/114): python3-mako-1.1.4-6.el9.noarch.rpm 1.5 MB/s | 172 kB 00:00 2024-09-21T09:20:10.561 INFO:teuthology.orchestra.run.smithi022.stdout:(62/114): python3-markupsafe-1.1.1-12.el9.x86_6 592 kB/s | 35 kB 00:00 2024-09-21T09:20:10.602 INFO:teuthology.orchestra.run.smithi064.stdout:(32/114): ceph-mgr-diskprediction-local-17.2.7- 21 MB/s | 7.4 MB 00:00 2024-09-21T09:20:10.611 INFO:teuthology.orchestra.run.smithi022.stdout:(63/114): python3-oauthlib-3.1.1-5.el9.noarch.r 4.3 MB/s | 222 kB 00:00 2024-09-21T09:20:10.628 INFO:teuthology.orchestra.run.smithi064.stdout:(33/114): ceph-prometheus-alerts-17.2.7-1649.gf 125 kB/s | 15 kB 00:00 2024-09-21T09:20:10.653 INFO:teuthology.orchestra.run.smithi064.stdout:(34/114): cephadm-17.2.7-1649.gfbd1a6ab.el9.noa 1.6 MB/s | 83 kB 00:00 2024-09-21T09:20:10.686 INFO:teuthology.orchestra.run.smithi022.stdout:(64/114): python3-numpy-1.20.1-5.el9.x86_64.rpm 30 MB/s | 5.1 MB 00:00 2024-09-21T09:20:10.720 INFO:teuthology.orchestra.run.smithi022.stdout:(65/114): python3-numpy-f2py-1.20.1-5.el9.x86_6 1.4 MB/s | 250 kB 00:00 2024-09-21T09:20:10.737 INFO:teuthology.orchestra.run.smithi022.stdout:(66/114): python3-pluggy-0.13.1-7.el9.noarch.rp 822 kB/s | 41 kB 00:00 2024-09-21T09:20:10.762 INFO:teuthology.orchestra.run.smithi064.stdout:(35/114): fuse-2.9.9-16.el9.x86_64.rpm 597 kB/s | 80 kB 00:00 2024-09-21T09:20:10.762 INFO:teuthology.orchestra.run.smithi022.stdout:(67/114): python3-packaging-20.9-5.el9.noarch.r 511 kB/s | 77 kB 00:00 2024-09-21T09:20:10.788 INFO:teuthology.orchestra.run.smithi022.stdout:(68/114): python3-prettytable-0.7.2-27.el9.noar 630 kB/s | 42 kB 00:00 2024-09-21T09:20:10.813 INFO:teuthology.orchestra.run.smithi022.stdout:(69/114): python3-pyasn1-0.4.8-6.el9.noarch.rpm 3.1 MB/s | 159 kB 00:00 2024-09-21T09:20:10.838 INFO:teuthology.orchestra.run.smithi022.stdout:(70/114): python3-py-1.10.0-6.el9.noarch.rpm 4.6 MB/s | 477 kB 00:00 2024-09-21T09:20:10.863 INFO:teuthology.orchestra.run.smithi022.stdout:(71/114): python3-pytest-6.2.2-7.el9.noarch.rpm 10 MB/s | 519 kB 00:00 2024-09-21T09:20:10.870 INFO:teuthology.orchestra.run.smithi064.stdout:(36/114): libquadmath-11.5.0-2.el9.x86_64.rpm 1.7 MB/s | 191 kB 00:00 2024-09-21T09:20:10.889 INFO:teuthology.orchestra.run.smithi022.stdout:(72/114): python3-pytz-2021.1-5.el9.noarch.rpm 1.0 MB/s | 51 kB 00:00 2024-09-21T09:20:10.922 INFO:teuthology.orchestra.run.smithi022.stdout:(73/114): python3-pyasn1-modules-0.4.8-6.el9.no 2.0 MB/s | 279 kB 00:00 2024-09-21T09:20:10.948 INFO:teuthology.orchestra.run.smithi022.stdout:(74/114): python3-requests-oauthlib-1.3.0-12.el 640 kB/s | 54 kB 00:00 2024-09-21T09:20:10.973 INFO:teuthology.orchestra.run.smithi022.stdout:(75/114): python3-toml-0.10.2-6.el9.noarch.rpm 829 kB/s | 42 kB 00:00 2024-09-21T09:20:10.979 INFO:teuthology.orchestra.run.smithi064.stdout:(37/114): libgfortran-11.5.0-2.el9.x86_64.rpm 2.4 MB/s | 801 kB 00:00 2024-09-21T09:20:11.004 INFO:teuthology.orchestra.run.smithi064.stdout:(38/114): python3-cffi-1.14.5-5.el9.x86_64.rpm 1.9 MB/s | 253 kB 00:00 2024-09-21T09:20:11.015 INFO:teuthology.orchestra.run.smithi022.stdout:(76/114): xmlstarlet-1.6.1-20.el9.x86_64.rpm 1.5 MB/s | 64 kB 00:00 2024-09-21T09:20:11.040 INFO:teuthology.orchestra.run.smithi022.stdout:(77/114): socat-1.7.4.1-6.el9.x86_64.rpm 3.2 MB/s | 304 kB 00:00 2024-09-21T09:20:11.054 INFO:teuthology.orchestra.run.smithi064.stdout:(39/114): python3-ply-3.11-14.el9.noarch.rpm 2.1 MB/s | 106 kB 00:00 2024-09-21T09:20:11.091 INFO:teuthology.orchestra.run.smithi022.stdout:(78/114): liboath-2.6.7-2.el9.x86_64.rpm 1.1 MB/s | 49 kB 00:00 2024-09-21T09:20:11.116 INFO:teuthology.orchestra.run.smithi022.stdout:(79/114): gperftools-libs-2.9.1-3.el9.x86_64.rp 3.0 MB/s | 308 kB 00:00 2024-09-21T09:20:11.130 INFO:teuthology.orchestra.run.smithi064.stdout:(40/114): python3-pycparser-2.20-6.el9.noarch.r 1.8 MB/s | 135 kB 00:00 2024-09-21T09:20:11.141 INFO:teuthology.orchestra.run.smithi022.stdout:(80/114): libunwind-1.6.2-1.el9.x86_64.rpm 1.3 MB/s | 67 kB 00:00 2024-09-21T09:20:11.166 INFO:teuthology.orchestra.run.smithi022.stdout:(81/114): python3-bcrypt-3.2.2-1.el9.x86_64.rpm 1.7 MB/s | 43 kB 00:00 2024-09-21T09:20:11.192 INFO:teuthology.orchestra.run.smithi022.stdout:(82/114): python3-cachetools-4.2.4-1.el9.noarch 1.3 MB/s | 32 kB 00:00 2024-09-21T09:20:11.213 INFO:teuthology.orchestra.run.smithi064.stdout:(41/114): python3-requests-2.25.1-8.el9.noarch. 1.5 MB/s | 125 kB 00:00 2024-09-21T09:20:11.217 INFO:teuthology.orchestra.run.smithi022.stdout:(83/114): python3-certifi-2023.05.07-4.el9.noar 570 kB/s | 14 kB 00:00 2024-09-21T09:20:11.242 INFO:teuthology.orchestra.run.smithi022.stdout:(84/114): python3-cheroot-8.6.0-4.el9.noarch.rp 6.7 MB/s | 172 kB 00:00 2024-09-21T09:20:11.268 INFO:teuthology.orchestra.run.smithi022.stdout:(85/114): python3-asyncssh-2.13.2-5.el9.noarch. 3.5 MB/s | 548 kB 00:00 2024-09-21T09:20:11.297 INFO:teuthology.orchestra.run.smithi064.stdout:(42/114): python3-urllib3-1.26.5-6.el9.noarch.r 2.5 MB/s | 215 kB 00:00 2024-09-21T09:20:11.301 INFO:teuthology.orchestra.run.smithi022.stdout:(86/114): python3-google-auth-2.34.0-1.el9.noar 6.5 MB/s | 220 kB 00:00 2024-09-21T09:20:11.326 INFO:teuthology.orchestra.run.smithi022.stdout:(87/114): python3-cherrypy-18.6.1-2.el9.noarch. 4.2 MB/s | 358 kB 00:00 2024-09-21T09:20:11.343 INFO:teuthology.orchestra.run.smithi022.stdout:(88/114): python3-jaraco-8.2.1-3.el9.noarch.rpm 255 kB/s | 11 kB 00:00 2024-09-21T09:20:11.360 INFO:teuthology.orchestra.run.smithi022.stdout:(89/114): python3-jaraco-classes-3.2.1-5.el9.no 527 kB/s | 18 kB 00:00 2024-09-21T09:20:11.364 INFO:teuthology.orchestra.run.smithi064.stdout:(43/114): python3-cryptography-36.0.1-4.el9.x86 3.4 MB/s | 1.2 MB 00:00 2024-09-21T09:20:11.377 INFO:teuthology.orchestra.run.smithi022.stdout:(90/114): python3-jaraco-collections-3.0.0-8.el 689 kB/s | 23 kB 00:00 2024-09-21T09:20:11.389 INFO:teuthology.orchestra.run.smithi064.stdout:(44/114): flexiblas-3.0.4-8.el9.x86_64.rpm 356 kB/s | 33 kB 00:00 2024-09-21T09:20:11.394 INFO:teuthology.orchestra.run.smithi022.stdout:(91/114): python3-jaraco-functools-3.5.0-2.el9. 583 kB/s | 19 kB 00:00 2024-09-21T09:20:11.411 INFO:teuthology.orchestra.run.smithi022.stdout:(92/114): python3-jaraco-text-3.2.0-6.el9.noarc 587 kB/s | 20 kB 00:00 2024-09-21T09:20:11.436 INFO:teuthology.orchestra.run.smithi022.stdout:(93/114): python3-jwt+crypto-2.4.0-1.el9.noarch 216 kB/s | 9.0 kB 00:00 2024-09-21T09:20:11.439 INFO:teuthology.orchestra.run.smithi064.stdout:(45/114): flexiblas-openblas-openmp-3.0.4-8.el9 364 kB/s | 18 kB 00:00 2024-09-21T09:20:11.454 INFO:teuthology.orchestra.run.smithi022.stdout:(94/114): python3-jwt-2.4.0-1.el9.noarch.rpm 974 kB/s | 41 kB 00:00 2024-09-21T09:20:11.471 INFO:teuthology.orchestra.run.smithi022.stdout:(95/114): python3-logutils-0.3.5-21.el9.noarch. 2.7 MB/s | 46 kB 00:00 2024-09-21T09:20:11.496 INFO:teuthology.orchestra.run.smithi022.stdout:(96/114): python3-more-itertools-8.12.0-2.el9.n 3.1 MB/s | 79 kB 00:00 2024-09-21T09:20:11.513 INFO:teuthology.orchestra.run.smithi022.stdout:(97/114): python3-natsort-7.1.1-5.el9.noarch.rp 3.4 MB/s | 58 kB 00:00 2024-09-21T09:20:11.540 INFO:teuthology.orchestra.run.smithi064.stdout:(46/114): libpmemobj-1.12.1-1.el9.x86_64.rpm 1.6 MB/s | 160 kB 00:00 2024-09-21T09:20:11.546 INFO:teuthology.orchestra.run.smithi022.stdout:(98/114): python3-pecan-1.4.2-3.el9.noarch.rpm 8.0 MB/s | 272 kB 00:00 2024-09-21T09:20:11.564 INFO:teuthology.orchestra.run.smithi022.stdout:(99/114): python3-portend-3.1.0-2.el9.noarch.rp 995 kB/s | 16 kB 00:00 2024-09-21T09:20:11.581 INFO:teuthology.orchestra.run.smithi064.stdout:(47/114): librabbitmq-0.11.0-7.el9.x86_64.rpm 1.1 MB/s | 45 kB 00:00 2024-09-21T09:20:11.589 INFO:teuthology.orchestra.run.smithi022.stdout:(100/114): python3-pyOpenSSL-21.0.0-1.el9.noarc 3.5 MB/s | 90 kB 00:00 2024-09-21T09:20:11.606 INFO:teuthology.orchestra.run.smithi022.stdout:(101/114): python3-repoze-lru-0.7-16.el9.noarch 1.8 MB/s | 31 kB 00:00 2024-09-21T09:20:11.648 INFO:teuthology.orchestra.run.smithi022.stdout:(102/114): python3-kubernetes-26.1.0-2.el9.noar 4.8 MB/s | 1.0 MB 00:00 2024-09-21T09:20:11.648 INFO:teuthology.orchestra.run.smithi064.stdout:(48/114): flexiblas-netlib-3.0.4-8.el9.x86_64.r 11 MB/s | 3.0 MB 00:00 2024-09-21T09:20:11.682 INFO:teuthology.orchestra.run.smithi022.stdout:(103/114): python3-routes-2.5.1-5.el9.noarch.rp 2.4 MB/s | 188 kB 00:00 2024-09-21T09:20:11.698 INFO:teuthology.orchestra.run.smithi022.stdout:(104/114): python3-rsa-4.9-2.el9.noarch.rpm 1.1 MB/s | 59 kB 00:00 2024-09-21T09:20:11.699 INFO:teuthology.orchestra.run.smithi064.stdout:(49/114): lttng-ust-2.12.0-6.el9.x86_64.rpm 5.7 MB/s | 292 kB 00:00 2024-09-21T09:20:11.715 INFO:teuthology.orchestra.run.smithi022.stdout:(105/114): python3-tempora-5.0.0-2.el9.noarch.r 1.0 MB/s | 36 kB 00:00 2024-09-21T09:20:11.733 INFO:teuthology.orchestra.run.smithi064.stdout:(50/114): librdkafka-1.6.1-102.el9.x86_64.rpm 4.3 MB/s | 662 kB 00:00 2024-09-21T09:20:11.740 INFO:teuthology.orchestra.run.smithi022.stdout:(106/114): python3-typing-extensions-4.12.2-2.e 1.7 MB/s | 74 kB 00:00 2024-09-21T09:20:11.758 INFO:teuthology.orchestra.run.smithi064.stdout:(51/114): openblas-0.3.26-2.el9.x86_64.rpm 654 kB/s | 38 kB 00:00 2024-09-21T09:20:11.766 INFO:teuthology.orchestra.run.smithi022.stdout:(107/114): python3-websocket-client-1.2.3-2.el9 3.5 MB/s | 90 kB 00:00 2024-09-21T09:20:11.791 INFO:teuthology.orchestra.run.smithi022.stdout:(108/114): python3-webob-1.8.8-2.el9.noarch.rpm 3.0 MB/s | 230 kB 00:00 2024-09-21T09:20:11.808 INFO:teuthology.orchestra.run.smithi022.stdout:(109/114): python3-zc-lockfile-2.0-10.el9.noarc 1.2 MB/s | 20 kB 00:00 2024-09-21T09:20:11.809 INFO:teuthology.orchestra.run.smithi064.stdout:(52/114): python3-attrs-20.3.0-7.el9.noarch.rpm 1.7 MB/s | 87 kB 00:00 2024-09-21T09:20:11.842 INFO:teuthology.orchestra.run.smithi022.stdout:(110/114): python3-werkzeug-2.0.3-3.el9.1.noarc 5.5 MB/s | 427 kB 00:00 2024-09-21T09:20:11.901 INFO:teuthology.orchestra.run.smithi022.stdout:(111/114): thrift-0.15.0-2.el9.x86_64.rpm 17 MB/s | 1.6 MB 00:00 2024-09-21T09:20:11.909 INFO:teuthology.orchestra.run.smithi064.stdout:(53/114): openblas-openmp-0.3.26-2.el9.x86_64.r 28 MB/s | 4.9 MB 00:00 2024-09-21T09:20:11.959 INFO:teuthology.orchestra.run.smithi064.stdout:(54/114): python3-devel-3.9.19-8.el9.x86_64.rpm 4.8 MB/s | 245 kB 00:00 2024-09-21T09:20:12.035 INFO:teuthology.orchestra.run.smithi064.stdout:(55/114): python3-babel-2.9.1-2.el9.noarch.rpm 26 MB/s | 6.0 MB 00:00 2024-09-21T09:20:12.060 INFO:teuthology.orchestra.run.smithi064.stdout:(56/114): python3-iniconfig-1.1.1-7.el9.noarch. 175 kB/s | 17 kB 00:00 2024-09-21T09:20:12.085 INFO:teuthology.orchestra.run.smithi064.stdout:(57/114): python3-jinja2-2.11.3-6.el9.noarch.rp 4.8 MB/s | 248 kB 00:00 2024-09-21T09:20:12.109 INFO:teuthology.orchestra.run.smithi022.stdout:(112/114): python3-scipy-1.6.2-8.el9.x86_64.rpm 13 MB/s | 16 MB 00:01 2024-09-21T09:20:12.110 INFO:teuthology.orchestra.run.smithi064.stdout:(58/114): python3-jsonpatch-1.21-16.el9.noarch. 511 kB/s | 26 kB 00:00 2024-09-21T09:20:12.135 INFO:teuthology.orchestra.run.smithi064.stdout:(59/114): python3-jsonpointer-2.0-4.el9.noarch. 383 kB/s | 19 kB 00:00 2024-09-21T09:20:12.161 INFO:teuthology.orchestra.run.smithi064.stdout:(60/114): python3-mako-1.1.4-6.el9.noarch.rpm 3.4 MB/s | 172 kB 00:00 2024-09-21T09:20:12.186 INFO:teuthology.orchestra.run.smithi064.stdout:(61/114): python3-markupsafe-1.1.1-12.el9.x86_6 691 kB/s | 35 kB 00:00 2024-09-21T09:20:12.226 INFO:teuthology.orchestra.run.smithi022.stdout:(113/114): librbd1-17.2.7-1649.gfbd1a6ab.el9.x8 9.0 MB/s | 2.9 MB 00:00 2024-09-21T09:20:12.236 INFO:teuthology.orchestra.run.smithi064.stdout:(62/114): python3-numpy-f2py-1.20.1-5.el9.x86_6 4.9 MB/s | 250 kB 00:00 2024-09-21T09:20:12.284 INFO:teuthology.orchestra.run.smithi022.stdout:(114/114): librados2-17.2.7-1649.gfbd1a6ab.el9. 7.0 MB/s | 3.1 MB 00:00 2024-09-21T09:20:12.289 INFO:teuthology.orchestra.run.smithi022.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:20:12.290 INFO:teuthology.orchestra.run.smithi022.stdout:Total 26 MB/s | 190 MB 00:07 2024-09-21T09:20:12.312 INFO:teuthology.orchestra.run.smithi064.stdout:(63/114): python3-numpy-1.20.1-5.el9.x86_64.rpm 34 MB/s | 5.1 MB 00:00 2024-09-21T09:20:12.337 INFO:teuthology.orchestra.run.smithi064.stdout:(64/114): python3-oauthlib-3.1.1-5.el9.noarch.r 2.2 MB/s | 222 kB 00:00 2024-09-21T09:20:12.362 INFO:teuthology.orchestra.run.smithi064.stdout:(65/114): python3-packaging-20.9-5.el9.noarch.r 1.5 MB/s | 77 kB 00:00 2024-09-21T09:20:12.388 INFO:teuthology.orchestra.run.smithi064.stdout:(66/114): python3-pluggy-0.13.1-7.el9.noarch.rp 825 kB/s | 41 kB 00:00 2024-09-21T09:20:12.413 INFO:teuthology.orchestra.run.smithi064.stdout:(67/114): python3-prettytable-0.7.2-27.el9.noar 840 kB/s | 42 kB 00:00 2024-09-21T09:20:12.447 INFO:teuthology.orchestra.run.smithi064.stdout:(68/114): python3-py-1.10.0-6.el9.noarch.rpm 8.0 MB/s | 477 kB 00:00 2024-09-21T09:20:12.472 INFO:teuthology.orchestra.run.smithi064.stdout:(69/114): python3-pyasn1-0.4.8-6.el9.noarch.rpm 2.7 MB/s | 159 kB 00:00 2024-09-21T09:20:12.497 INFO:teuthology.orchestra.run.smithi064.stdout:(70/114): python3-pyasn1-modules-0.4.8-6.el9.no 5.4 MB/s | 279 kB 00:00 2024-09-21T09:20:12.531 INFO:teuthology.orchestra.run.smithi064.stdout:(71/114): python3-pytest-6.2.2-7.el9.noarch.rpm 8.7 MB/s | 519 kB 00:00 2024-09-21T09:20:12.556 INFO:teuthology.orchestra.run.smithi064.stdout:(72/114): python3-pytz-2021.1-5.el9.noarch.rpm 871 kB/s | 51 kB 00:00 2024-09-21T09:20:12.581 INFO:teuthology.orchestra.run.smithi064.stdout:(73/114): python3-requests-oauthlib-1.3.0-12.el 1.0 MB/s | 54 kB 00:00 2024-09-21T09:20:12.623 INFO:teuthology.orchestra.run.smithi064.stdout:(74/114): python3-toml-0.10.2-6.el9.noarch.rpm 1.0 MB/s | 42 kB 00:00 2024-09-21T09:20:12.682 INFO:teuthology.orchestra.run.smithi064.stdout:(75/114): socat-1.7.4.1-6.el9.x86_64.rpm 5.1 MB/s | 304 kB 00:00 2024-09-21T09:20:12.724 INFO:teuthology.orchestra.run.smithi064.stdout:(76/114): xmlstarlet-1.6.1-20.el9.x86_64.rpm 1.5 MB/s | 64 kB 00:00 2024-09-21T09:20:12.774 INFO:teuthology.orchestra.run.smithi064.stdout:(77/114): gperftools-libs-2.9.1-3.el9.x86_64.rp 6.1 MB/s | 308 kB 00:00 2024-09-21T09:20:12.799 INFO:teuthology.orchestra.run.smithi064.stdout:(78/114): liboath-2.6.7-2.el9.x86_64.rpm 1.9 MB/s | 49 kB 00:00 2024-09-21T09:20:12.825 INFO:teuthology.orchestra.run.smithi064.stdout:(79/114): libunwind-1.6.2-1.el9.x86_64.rpm 2.6 MB/s | 67 kB 00:00 2024-09-21T09:20:12.859 INFO:teuthology.orchestra.run.smithi064.stdout:(80/114): python3-asyncssh-2.13.2-5.el9.noarch. 16 MB/s | 548 kB 00:00 2024-09-21T09:20:12.875 INFO:teuthology.orchestra.run.smithi064.stdout:(81/114): python3-bcrypt-3.2.2-1.el9.x86_64.rpm 2.6 MB/s | 43 kB 00:00 2024-09-21T09:20:13.043 INFO:teuthology.orchestra.run.smithi064.stdout:(82/114): python3-scipy-1.6.2-8.el9.x86_64.rpm 33 MB/s | 16 MB 00:00 2024-09-21T09:20:13.059 INFO:teuthology.orchestra.run.smithi064.stdout:(83/114): python3-cachetools-4.2.4-1.el9.noarch 175 kB/s | 32 kB 00:00 2024-09-21T09:20:13.066 INFO:teuthology.orchestra.run.smithi022.stdout:Running transaction check 2024-09-21T09:20:13.093 INFO:teuthology.orchestra.run.smithi064.stdout:(84/114): python3-cheroot-8.6.0-4.el9.noarch.rp 6.6 MB/s | 172 kB 00:00 2024-09-21T09:20:13.118 INFO:teuthology.orchestra.run.smithi064.stdout:(85/114): python3-certifi-2023.05.07-4.el9.noar 187 kB/s | 14 kB 00:00 2024-09-21T09:20:13.152 INFO:teuthology.orchestra.run.smithi064.stdout:(86/114): python3-cherrypy-18.6.1-2.el9.noarch. 6.2 MB/s | 358 kB 00:00 2024-09-21T09:20:13.170 INFO:teuthology.orchestra.run.smithi022.stdout:Transaction check succeeded. 2024-09-21T09:20:13.170 INFO:teuthology.orchestra.run.smithi022.stdout:Running transaction test 2024-09-21T09:20:13.177 INFO:teuthology.orchestra.run.smithi064.stdout:(87/114): python3-jaraco-8.2.1-3.el9.noarch.rpm 433 kB/s | 11 kB 00:00 2024-09-21T09:20:13.203 INFO:teuthology.orchestra.run.smithi064.stdout:(88/114): python3-jaraco-classes-3.2.1-5.el9.no 706 kB/s | 18 kB 00:00 2024-09-21T09:20:13.228 INFO:teuthology.orchestra.run.smithi064.stdout:(89/114): python3-jaraco-collections-3.0.0-8.el 938 kB/s | 23 kB 00:00 2024-09-21T09:20:13.253 INFO:teuthology.orchestra.run.smithi064.stdout:(90/114): python3-google-auth-2.34.0-1.el9.noar 1.6 MB/s | 220 kB 00:00 2024-09-21T09:20:13.278 INFO:teuthology.orchestra.run.smithi064.stdout:(91/114): python3-jaraco-functools-3.5.0-2.el9. 387 kB/s | 19 kB 00:00 2024-09-21T09:20:13.304 INFO:teuthology.orchestra.run.smithi064.stdout:(92/114): python3-jaraco-text-3.2.0-6.el9.noarc 390 kB/s | 20 kB 00:00 2024-09-21T09:20:13.329 INFO:teuthology.orchestra.run.smithi064.stdout:(93/114): python3-jwt+crypto-2.4.0-1.el9.noarch 179 kB/s | 9.0 kB 00:00 2024-09-21T09:20:13.354 INFO:teuthology.orchestra.run.smithi064.stdout:(94/114): python3-jwt-2.4.0-1.el9.noarch.rpm 817 kB/s | 41 kB 00:00 2024-09-21T09:20:13.380 INFO:teuthology.orchestra.run.smithi064.stdout:(95/114): python3-logutils-0.3.5-21.el9.noarch. 1.8 MB/s | 46 kB 00:00 2024-09-21T09:20:13.405 INFO:teuthology.orchestra.run.smithi064.stdout:(96/114): python3-more-itertools-8.12.0-2.el9.n 3.1 MB/s | 79 kB 00:00 2024-09-21T09:20:13.430 INFO:teuthology.orchestra.run.smithi064.stdout:(97/114): python3-natsort-7.1.1-5.el9.noarch.rp 2.3 MB/s | 58 kB 00:00 2024-09-21T09:20:13.464 INFO:teuthology.orchestra.run.smithi064.stdout:(98/114): python3-pecan-1.4.2-3.el9.noarch.rpm 8.0 MB/s | 272 kB 00:00 2024-09-21T09:20:13.489 INFO:teuthology.orchestra.run.smithi064.stdout:(99/114): python3-portend-3.1.0-2.el9.noarch.rp 655 kB/s | 16 kB 00:00 2024-09-21T09:20:13.514 INFO:teuthology.orchestra.run.smithi064.stdout:(100/114): python3-pyOpenSSL-21.0.0-1.el9.noarc 3.6 MB/s | 90 kB 00:00 2024-09-21T09:20:13.540 INFO:teuthology.orchestra.run.smithi064.stdout:(101/114): python3-repoze-lru-0.7-16.el9.noarch 1.2 MB/s | 31 kB 00:00 2024-09-21T09:20:13.573 INFO:teuthology.orchestra.run.smithi064.stdout:(102/114): python3-kubernetes-26.1.0-2.el9.noar 4.2 MB/s | 1.0 MB 00:00 2024-09-21T09:20:13.615 INFO:teuthology.orchestra.run.smithi064.stdout:(103/114): python3-rsa-4.9-2.el9.noarch.rpm 1.4 MB/s | 59 kB 00:00 2024-09-21T09:20:13.641 INFO:teuthology.orchestra.run.smithi064.stdout:(104/114): python3-tempora-5.0.0-2.el9.noarch.r 1.4 MB/s | 36 kB 00:00 2024-09-21T09:20:13.666 INFO:teuthology.orchestra.run.smithi064.stdout:(105/114): python3-routes-2.5.1-5.el9.noarch.rp 1.5 MB/s | 188 kB 00:00 2024-09-21T09:20:13.692 INFO:teuthology.orchestra.run.smithi064.stdout:(106/114): python3-typing-extensions-4.12.2-2.e 1.4 MB/s | 74 kB 00:00 2024-09-21T09:20:13.717 INFO:teuthology.orchestra.run.smithi064.stdout:(107/114): python3-websocket-client-1.2.3-2.el9 3.6 MB/s | 90 kB 00:00 2024-09-21T09:20:13.742 INFO:teuthology.orchestra.run.smithi064.stdout:(108/114): python3-webob-1.8.8-2.el9.noarch.rpm 3.0 MB/s | 230 kB 00:00 2024-09-21T09:20:13.768 INFO:teuthology.orchestra.run.smithi064.stdout:(109/114): python3-zc-lockfile-2.0-10.el9.noarc 804 kB/s | 20 kB 00:00 2024-09-21T09:20:13.809 INFO:teuthology.orchestra.run.smithi064.stdout:(110/114): python3-werkzeug-2.0.3-3.el9.1.noarc 4.5 MB/s | 427 kB 00:00 2024-09-21T09:20:13.876 INFO:teuthology.orchestra.run.smithi064.stdout:(111/114): thrift-0.15.0-2.el9.x86_64.rpm 15 MB/s | 1.6 MB 00:00 2024-09-21T09:20:14.060 INFO:teuthology.orchestra.run.smithi064.stdout:(112/114): librados2-17.2.7-1649.gfbd1a6ab.el9. 12 MB/s | 3.1 MB 00:00 2024-09-21T09:20:14.160 INFO:teuthology.orchestra.run.smithi064.stdout:(113/114): librbd1-17.2.7-1649.gfbd1a6ab.el9.x8 10 MB/s | 2.9 MB 00:00 2024-09-21T09:20:14.921 INFO:teuthology.orchestra.run.smithi022.stdout:Transaction test succeeded. 2024-09-21T09:20:14.921 INFO:teuthology.orchestra.run.smithi022.stdout:Running transaction 2024-09-21T09:20:15.660 INFO:teuthology.orchestra.run.smithi064.stdout:(114/114): ceph-test-17.2.7-1649.gfbd1a6ab.el9. 7.0 MB/s | 48 MB 00:06 2024-09-21T09:20:15.666 INFO:teuthology.orchestra.run.smithi064.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:20:15.667 INFO:teuthology.orchestra.run.smithi064.stdout:Total 22 MB/s | 190 MB 00:08 2024-09-21T09:20:16.448 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction check 2024-09-21T09:20:16.552 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction check succeeded. 2024-09-21T09:20:16.552 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction test 2024-09-21T09:20:18.318 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction test succeeded. 2024-09-21T09:20:18.318 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction 2024-09-21T09:20:19.618 INFO:teuthology.orchestra.run.smithi022.stdout: Preparing : 1/1 2024-09-21T09:20:19.688 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : thrift-0.15.0-2.el9.x86_64 1/116 2024-09-21T09:20:19.713 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 2/116 2024-09-21T09:20:19.773 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 3/116 2024-09-21T09:20:20.091 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 4/116 2024-09-21T09:20:20.196 INFO:teuthology.orchestra.run.smithi022.stdout: Upgrading : librados2-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 5/116 2024-09-21T09:20:20.296 INFO:teuthology.orchestra.run.smithi022.stdout: Running scriptlet: librados2-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 5/116 2024-09-21T09:20:20.331 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : libcephfs2-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 6/116 2024-09-21T09:20:20.425 INFO:teuthology.orchestra.run.smithi022.stdout: Running scriptlet: libcephfs2-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 6/116 2024-09-21T09:20:20.480 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : python3-rados-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 7/116 2024-09-21T09:20:20.511 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 8/116 2024-09-21T09:20:20.987 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 9/116 2024-09-21T09:20:21.017 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : librgw2-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 10/116 2024-09-21T09:20:21.084 INFO:teuthology.orchestra.run.smithi022.stdout: Running scriptlet: librgw2-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 10/116 2024-09-21T09:20:21.110 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : libcephsqlite-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 11/116 2024-09-21T09:20:21.212 INFO:teuthology.orchestra.run.smithi022.stdout: Running scriptlet: libcephsqlite-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 11/116 2024-09-21T09:20:21.236 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : libradosstriper1-2:17.2.7-1649.gfbd1a6ab.el9.x86 12/116 2024-09-21T09:20:21.380 INFO:teuthology.orchestra.run.smithi022.stdout: Running scriptlet: libradosstriper1-2:17.2.7-1649.gfbd1a6ab.el9.x86 12/116 2024-09-21T09:20:21.416 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : python3-werkzeug-2.0.3-3.el9.1.noarch 13/116 2024-09-21T09:20:21.457 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : liboath-2.6.7-2.el9.x86_64 14/116 2024-09-21T09:20:21.487 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 15/116 2024-09-21T09:20:21.527 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 16/116 2024-09-21T09:20:21.603 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : python3-pytz-2021.1-5.el9.noarch 17/116 2024-09-21T09:20:21.684 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : python3-pyasn1-0.4.8-6.el9.noarch 18/116 2024-09-21T09:20:21.727 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : python3-packaging-20.9-5.el9.noarch 19/116 2024-09-21T09:20:21.754 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 20/116 2024-09-21T09:20:21.826 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : flexiblas-3.0.4-8.el9.x86_64 21/116 2024-09-21T09:20:21.878 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : python3-urllib3-1.26.5-6.el9.noarch 22/116 2024-09-21T09:20:21.911 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : python3-requests-2.25.1-8.el9.noarch 23/116 2024-09-21T09:20:21.951 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : libquadmath-11.5.0-2.el9.x86_64 24/116 2024-09-21T09:20:21.986 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : libgfortran-11.5.0-2.el9.x86_64 25/116 2024-09-21T09:20:22.039 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : python3-ceph-argparse-2:17.2.7-1649.gfbd1a6ab.el 26/116 2024-09-21T09:20:22.116 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : python3-cephfs-2:17.2.7-1649.gfbd1a6ab.el9.x86_6 27/116 2024-09-21T09:20:22.270 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : python3-mako-1.1.4-6.el9.noarch 28/116 2024-09-21T09:20:22.325 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : python3-pyasn1-modules-0.4.8-6.el9.noarch 29/116 2024-09-21T09:20:22.888 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : python3-rsa-4.9-2.el9.noarch 30/116 2024-09-21T09:20:22.971 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 31/116 2024-09-21T09:20:23.016 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : python3-jinja2-2.11.3-6.el9.noarch 32/116 2024-09-21T09:20:23.044 INFO:teuthology.orchestra.run.smithi064.stdout: Preparing : 1/1 2024-09-21T09:20:23.049 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 33/116 2024-09-21T09:20:23.084 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 34/116 2024-09-21T09:20:23.105 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : thrift-0.15.0-2.el9.x86_64 1/116 2024-09-21T09:20:23.124 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : python3-jaraco-text-3.2.0-6.el9.noarch 35/116 2024-09-21T09:20:23.130 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 2/116 2024-09-21T09:20:23.160 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : python3-rgw-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 36/116 2024-09-21T09:20:23.188 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 3/116 2024-09-21T09:20:23.197 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : librados-devel-2:17.2.7-1649.gfbd1a6ab.el9.x86_6 37/116 2024-09-21T09:20:23.236 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 38/116 2024-09-21T09:20:23.273 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 39/116 2024-09-21T09:20:23.330 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 40/116 2024-09-21T09:20:23.403 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 41/116 2024-09-21T09:20:23.440 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : python3-webob-1.8.8-2.el9.noarch 42/116 2024-09-21T09:20:23.476 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : python3-typing-extensions-4.12.2-2.el9.noarch 43/116 2024-09-21T09:20:23.517 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 4/116 2024-09-21T09:20:23.524 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 44/116 2024-09-21T09:20:23.569 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 45/116 2024-09-21T09:20:23.614 INFO:teuthology.orchestra.run.smithi064.stdout: Upgrading : librados2-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 5/116 2024-09-21T09:20:23.620 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 46/116 2024-09-21T09:20:23.741 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: librados2-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 5/116 2024-09-21T09:20:23.763 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : python3-logutils-0.3.5-21.el9.noarch 47/116 2024-09-21T09:20:23.774 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : libcephfs2-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 6/116 2024-09-21T09:20:23.812 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : python3-pecan-1.4.2-3.el9.noarch 48/116 2024-09-21T09:20:23.854 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : python3-certifi-2023.05.07-4.el9.noarch 49/116 2024-09-21T09:20:23.868 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: libcephfs2-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 6/116 2024-09-21T09:20:23.920 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-rados-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 7/116 2024-09-21T09:20:23.954 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 8/116 2024-09-21T09:20:23.961 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 50/116 2024-09-21T09:20:24.004 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : python3-google-auth-1:2.34.0-1.el9.noarch 51/116 2024-09-21T09:20:24.053 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 52/116 2024-09-21T09:20:24.090 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : gperftools-libs-2.9.1-3.el9.x86_64 53/116 2024-09-21T09:20:24.140 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 54/116 2024-09-21T09:20:24.276 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : socat-1.7.4.1-6.el9.x86_64 55/116 2024-09-21T09:20:24.322 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : python3-py-1.10.0-6.el9.noarch 56/116 2024-09-21T09:20:24.361 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : python3-prettytable-0.7.2-27.el9.noarch 57/116 2024-09-21T09:20:24.417 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 9/116 2024-09-21T09:20:24.444 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : librgw2-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 10/116 2024-09-21T09:20:24.480 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : python3-pluggy-0.13.1-7.el9.noarch 58/116 2024-09-21T09:20:24.510 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: librgw2-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 10/116 2024-09-21T09:20:24.533 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : python3-oauthlib-3.1.1-5.el9.noarch 59/116 2024-09-21T09:20:24.536 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : libcephsqlite-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 11/116 2024-09-21T09:20:24.638 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: libcephsqlite-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 11/116 2024-09-21T09:20:24.662 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : libradosstriper1-2:17.2.7-1649.gfbd1a6ab.el9.x86 12/116 2024-09-21T09:20:24.795 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: libradosstriper1-2:17.2.7-1649.gfbd1a6ab.el9.x86 12/116 2024-09-21T09:20:24.834 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-werkzeug-2.0.3-3.el9.1.noarch 13/116 2024-09-21T09:20:24.875 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : liboath-2.6.7-2.el9.x86_64 14/116 2024-09-21T09:20:24.904 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 15/116 2024-09-21T09:20:24.944 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 16/116 2024-09-21T09:20:25.030 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-pytz-2021.1-5.el9.noarch 17/116 2024-09-21T09:20:25.080 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-pyasn1-0.4.8-6.el9.noarch 18/116 2024-09-21T09:20:25.129 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-packaging-20.9-5.el9.noarch 19/116 2024-09-21T09:20:25.155 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 20/116 2024-09-21T09:20:25.228 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : flexiblas-3.0.4-8.el9.x86_64 21/116 2024-09-21T09:20:25.304 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-urllib3-1.26.5-6.el9.noarch 22/116 2024-09-21T09:20:25.315 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 60/116 2024-09-21T09:20:25.351 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-requests-2.25.1-8.el9.noarch 23/116 2024-09-21T09:20:25.361 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : python3-kubernetes-1:26.1.0-2.el9.noarch 61/116 2024-09-21T09:20:25.394 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : libquadmath-11.5.0-2.el9.x86_64 24/116 2024-09-21T09:20:25.395 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : python3-jsonpointer-2.0-4.el9.noarch 62/116 2024-09-21T09:20:25.428 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : libgfortran-11.5.0-2.el9.x86_64 25/116 2024-09-21T09:20:25.438 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : python3-jsonpatch-1.21-16.el9.noarch 63/116 2024-09-21T09:20:25.480 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-ceph-argparse-2:17.2.7-1649.gfbd1a6ab.el 26/116 2024-09-21T09:20:25.542 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : python3-iniconfig-1.1.1-7.el9.noarch 64/116 2024-09-21T09:20:25.559 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-cephfs-2:17.2.7-1649.gfbd1a6ab.el9.x86_6 27/116 2024-09-21T09:20:25.596 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : python3-devel-3.9.19-8.el9.x86_64 65/116 2024-09-21T09:20:25.630 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : python3-attrs-20.3.0-7.el9.noarch 66/116 2024-09-21T09:20:25.716 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-mako-1.1.4-6.el9.noarch 28/116 2024-09-21T09:20:25.776 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-pyasn1-modules-0.4.8-6.el9.noarch 29/116 2024-09-21T09:20:25.820 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : openblas-0.3.26-2.el9.x86_64 67/116 2024-09-21T09:20:25.846 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : openblas-openmp-0.3.26-2.el9.x86_64 68/116 2024-09-21T09:20:25.924 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 69/116 2024-09-21T09:20:26.345 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-rsa-4.9-2.el9.noarch 30/116 2024-09-21T09:20:26.432 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 31/116 2024-09-21T09:20:26.474 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-jinja2-2.11.3-6.el9.noarch 32/116 2024-09-21T09:20:26.508 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 33/116 2024-09-21T09:20:26.543 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 34/116 2024-09-21T09:20:26.582 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-jaraco-text-3.2.0-6.el9.noarch 35/116 2024-09-21T09:20:26.618 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-rgw-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 36/116 2024-09-21T09:20:26.656 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : librados-devel-2:17.2.7-1649.gfbd1a6ab.el9.x86_6 37/116 2024-09-21T09:20:26.686 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 38/116 2024-09-21T09:20:26.723 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 39/116 2024-09-21T09:20:26.780 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 40/116 2024-09-21T09:20:26.855 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 41/116 2024-09-21T09:20:26.883 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : flexiblas-netlib-3.0.4-8.el9.x86_64 70/116 2024-09-21T09:20:26.891 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-webob-1.8.8-2.el9.noarch 42/116 2024-09-21T09:20:26.927 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-typing-extensions-4.12.2-2.el9.noarch 43/116 2024-09-21T09:20:26.976 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 44/116 2024-09-21T09:20:26.999 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : python3-numpy-1:1.20.1-5.el9.x86_64 71/116 2024-09-21T09:20:27.023 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 45/116 2024-09-21T09:20:27.065 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 46/116 2024-09-21T09:20:27.212 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-logutils-0.3.5-21.el9.noarch 47/116 2024-09-21T09:20:27.258 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-pecan-1.4.2-3.el9.noarch 48/116 2024-09-21T09:20:27.300 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-certifi-2023.05.07-4.el9.noarch 49/116 2024-09-21T09:20:27.408 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 50/116 2024-09-21T09:20:27.450 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-google-auth-1:2.34.0-1.el9.noarch 51/116 2024-09-21T09:20:27.490 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 52/116 2024-09-21T09:20:27.528 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : gperftools-libs-2.9.1-3.el9.x86_64 53/116 2024-09-21T09:20:27.588 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 54/116 2024-09-21T09:20:27.722 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : socat-1.7.4.1-6.el9.x86_64 55/116 2024-09-21T09:20:27.754 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-py-1.10.0-6.el9.noarch 56/116 2024-09-21T09:20:27.794 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-prettytable-0.7.2-27.el9.noarch 57/116 2024-09-21T09:20:27.899 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-pluggy-0.13.1-7.el9.noarch 58/116 2024-09-21T09:20:27.950 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-oauthlib-3.1.1-5.el9.noarch 59/116 2024-09-21T09:20:28.384 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 72/116 2024-09-21T09:20:28.470 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : python3-scipy-1.6.2-8.el9.x86_64 73/116 2024-09-21T09:20:28.734 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 60/116 2024-09-21T09:20:28.778 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-kubernetes-1:26.1.0-2.el9.noarch 61/116 2024-09-21T09:20:28.796 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 74/116 2024-09-21T09:20:28.812 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-jsonpointer-2.0-4.el9.noarch 62/116 2024-09-21T09:20:28.846 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-jsonpatch-1.21-16.el9.noarch 63/116 2024-09-21T09:20:28.848 INFO:teuthology.orchestra.run.smithi022.stdout: Upgrading : librbd1-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 75/116 2024-09-21T09:20:28.922 INFO:teuthology.orchestra.run.smithi022.stdout: Running scriptlet: librbd1-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 75/116 2024-09-21T09:20:28.952 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-iniconfig-1.1.1-7.el9.noarch 64/116 2024-09-21T09:20:28.962 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : python3-rbd-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 76/116 2024-09-21T09:20:29.014 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-devel-3.9.19-8.el9.x86_64 65/116 2024-09-21T09:20:29.021 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : python3-ply-3.11-14.el9.noarch 77/116 2024-09-21T09:20:29.056 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-attrs-20.3.0-7.el9.noarch 66/116 2024-09-21T09:20:29.084 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 78/116 2024-09-21T09:20:29.249 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : openblas-0.3.26-2.el9.x86_64 67/116 2024-09-21T09:20:29.279 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : openblas-openmp-0.3.26-2.el9.x86_64 68/116 2024-09-21T09:20:29.281 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 79/116 2024-09-21T09:20:29.333 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : python3-cryptography-36.0.1-4.el9.x86_64 80/116 2024-09-21T09:20:29.359 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 69/116 2024-09-21T09:20:29.422 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 81/116 2024-09-21T09:20:29.530 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : python3-cheroot-8.6.0-4.el9.noarch 82/116 2024-09-21T09:20:29.680 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 83/116 2024-09-21T09:20:29.729 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : python3-asyncssh-2.13.2-5.el9.noarch 84/116 2024-09-21T09:20:29.757 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : python3-jwt-2.4.0-1.el9.noarch 85/116 2024-09-21T09:20:29.792 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : python3-jwt+crypto-2.4.0-1.el9.noarch 86/116 2024-09-21T09:20:29.832 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 87/116 2024-09-21T09:20:29.867 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : fuse-2.9.9-16.el9.x86_64 88/116 2024-09-21T09:20:30.058 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : flexiblas-netlib-3.0.4-8.el9.x86_64 70/116 2024-09-21T09:20:30.174 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-numpy-1:1.20.1-5.el9.x86_64 71/116 2024-09-21T09:20:30.313 INFO:teuthology.orchestra.run.smithi022.stdout: Running scriptlet: cephadm-2:17.2.7-1649.gfbd1a6ab.el9.noarch 89/116 2024-09-21T09:20:30.339 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : cephadm-2:17.2.7-1649.gfbd1a6ab.el9.noarch 89/116 2024-09-21T09:20:30.370 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : ceph-prometheus-alerts-2:17.2.7-1649.gfbd1a6ab.e 90/116 2024-09-21T09:20:30.455 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : ceph-grafana-dashboards-2:17.2.7-1649.gfbd1a6ab. 91/116 2024-09-21T09:20:30.498 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : python3-ceph-common-2:17.2.7-1649.gfbd1a6ab.el9. 92/116 2024-09-21T09:20:32.047 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 72/116 2024-09-21T09:20:32.143 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-scipy-1.6.2-8.el9.x86_64 73/116 2024-09-21T09:20:32.452 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 74/116 2024-09-21T09:20:32.505 INFO:teuthology.orchestra.run.smithi064.stdout: Upgrading : librbd1-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 75/116 2024-09-21T09:20:32.579 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: librbd1-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 75/116 2024-09-21T09:20:32.619 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-rbd-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 76/116 2024-09-21T09:20:32.678 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-ply-3.11-14.el9.noarch 77/116 2024-09-21T09:20:32.742 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 78/116 2024-09-21T09:20:32.940 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 79/116 2024-09-21T09:20:32.962 INFO:teuthology.orchestra.run.smithi022.stdout: Running scriptlet: ceph-common-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 93/116 2024-09-21T09:20:32.990 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-cryptography-36.0.1-4.el9.x86_64 80/116 2024-09-21T09:20:32.994 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : ceph-common-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 93/116 2024-09-21T09:20:33.072 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 81/116 2024-09-21T09:20:33.192 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-cheroot-8.6.0-4.el9.noarch 82/116 2024-09-21T09:20:33.338 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 83/116 2024-09-21T09:20:33.378 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-asyncssh-2.13.2-5.el9.noarch 84/116 2024-09-21T09:20:33.406 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-jwt-2.4.0-1.el9.noarch 85/116 2024-09-21T09:20:33.440 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-jwt+crypto-2.4.0-1.el9.noarch 86/116 2024-09-21T09:20:33.472 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 87/116 2024-09-21T09:20:33.499 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : fuse-2.9.9-16.el9.x86_64 88/116 2024-09-21T09:20:33.542 INFO:teuthology.orchestra.run.smithi022.stdout: Running scriptlet: ceph-common-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 93/116 2024-09-21T09:20:33.581 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : ceph-base-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 94/116 2024-09-21T09:20:33.679 INFO:teuthology.orchestra.run.smithi022.stdout: Running scriptlet: ceph-base-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 94/116 2024-09-21T09:20:33.679 INFO:teuthology.orchestra.run.smithi022.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2024-09-21T09:20:33.680 INFO:teuthology.orchestra.run.smithi022.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2024-09-21T09:20:33.680 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-09-21T09:20:33.740 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : ceph-selinux-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 95/116 2024-09-21T09:20:34.125 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: cephadm-2:17.2.7-1649.gfbd1a6ab.el9.noarch 89/116 2024-09-21T09:20:34.155 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : cephadm-2:17.2.7-1649.gfbd1a6ab.el9.noarch 89/116 2024-09-21T09:20:34.186 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : ceph-prometheus-alerts-2:17.2.7-1649.gfbd1a6ab.e 90/116 2024-09-21T09:20:34.267 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : ceph-grafana-dashboards-2:17.2.7-1649.gfbd1a6ab. 91/116 2024-09-21T09:20:34.296 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-ceph-common-2:17.2.7-1649.gfbd1a6ab.el9. 92/116 2024-09-21T09:20:36.857 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: ceph-common-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 93/116 2024-09-21T09:20:36.894 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : ceph-common-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 93/116 2024-09-21T09:20:37.448 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: ceph-common-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 93/116 2024-09-21T09:20:37.505 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : ceph-base-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 94/116 2024-09-21T09:20:37.608 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: ceph-base-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 94/116 2024-09-21T09:20:37.608 INFO:teuthology.orchestra.run.smithi064.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2024-09-21T09:20:37.608 INFO:teuthology.orchestra.run.smithi064.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2024-09-21T09:20:37.608 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-09-21T09:20:37.631 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : ceph-selinux-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 95/116 2024-09-21T09:20:45.708 INFO:teuthology.orchestra.run.smithi022.stdout: Running scriptlet: ceph-selinux-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 95/116 2024-09-21T09:20:45.708 INFO:teuthology.orchestra.run.smithi022.stdout:skipping the directory /sys 2024-09-21T09:20:45.708 INFO:teuthology.orchestra.run.smithi022.stdout:skipping the directory /proc 2024-09-21T09:20:45.708 INFO:teuthology.orchestra.run.smithi022.stdout:skipping the directory /mnt 2024-09-21T09:20:45.708 INFO:teuthology.orchestra.run.smithi022.stdout:skipping the directory /var/tmp 2024-09-21T09:20:45.708 INFO:teuthology.orchestra.run.smithi022.stdout:skipping the directory /home 2024-09-21T09:20:45.708 INFO:teuthology.orchestra.run.smithi022.stdout:skipping the directory /root 2024-09-21T09:20:45.708 INFO:teuthology.orchestra.run.smithi022.stdout:skipping the directory /tmp 2024-09-21T09:20:45.709 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-09-21T09:20:45.777 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : ceph-mgr-cephadm-2:17.2.7-1649.gfbd1a6ab.el9.noa 96/116 2024-09-21T09:20:46.831 INFO:teuthology.orchestra.run.smithi022.stdout: Running scriptlet: ceph-mgr-cephadm-2:17.2.7-1649.gfbd1a6ab.el9.noa 96/116 2024-09-21T09:20:46.864 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : ceph-mgr-dashboard-2:17.2.7-1649.gfbd1a6ab.el9.n 97/116 2024-09-21T09:20:47.901 INFO:teuthology.orchestra.run.smithi022.stdout: Running scriptlet: ceph-mgr-dashboard-2:17.2.7-1649.gfbd1a6ab.el9.n 97/116 2024-09-21T09:20:47.941 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : ceph-mgr-diskprediction-local-2:17.2.7-1649.gfbd 98/116 2024-09-21T09:20:48.049 INFO:teuthology.orchestra.run.smithi022.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:17.2.7-1649.gfbd 98/116 2024-09-21T09:20:48.228 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : ceph-mgr-modules-core-2:17.2.7-1649.gfbd1a6ab.el 99/116 2024-09-21T09:20:48.260 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : ceph-mgr-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 100/116 2024-09-21T09:20:48.285 INFO:teuthology.orchestra.run.smithi022.stdout: Running scriptlet: ceph-mgr-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 100/116 2024-09-21T09:20:48.285 INFO:teuthology.orchestra.run.smithi022.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-21T09:20:48.286 INFO:teuthology.orchestra.run.smithi022.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2024-09-21T09:20:48.286 INFO:teuthology.orchestra.run.smithi022.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-09-21T09:20:48.286 INFO:teuthology.orchestra.run.smithi022.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-09-21T09:20:48.286 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-09-21T09:20:48.327 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : ceph-mgr-rook-2:17.2.7-1649.gfbd1a6ab.el9.noarch 101/116 2024-09-21T09:20:48.520 INFO:teuthology.orchestra.run.smithi022.stdout: Running scriptlet: ceph-mgr-rook-2:17.2.7-1649.gfbd1a6ab.el9.noarch 101/116 2024-09-21T09:20:48.553 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : ceph-mds-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 102/116 2024-09-21T09:20:48.578 INFO:teuthology.orchestra.run.smithi022.stdout: Running scriptlet: ceph-mds-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 102/116 2024-09-21T09:20:48.578 INFO:teuthology.orchestra.run.smithi022.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-21T09:20:48.579 INFO:teuthology.orchestra.run.smithi022.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2024-09-21T09:20:48.579 INFO:teuthology.orchestra.run.smithi022.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-09-21T09:20:48.579 INFO:teuthology.orchestra.run.smithi022.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-09-21T09:20:48.579 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-09-21T09:20:49.000 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : ceph-mon-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 103/116 2024-09-21T09:20:49.025 INFO:teuthology.orchestra.run.smithi022.stdout: Running scriptlet: ceph-mon-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 103/116 2024-09-21T09:20:49.025 INFO:teuthology.orchestra.run.smithi022.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-21T09:20:49.025 INFO:teuthology.orchestra.run.smithi022.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2024-09-21T09:20:49.026 INFO:teuthology.orchestra.run.smithi022.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-09-21T09:20:49.026 INFO:teuthology.orchestra.run.smithi022.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-09-21T09:20:49.026 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-09-21T09:20:49.574 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: ceph-selinux-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 95/116 2024-09-21T09:20:49.574 INFO:teuthology.orchestra.run.smithi064.stdout:skipping the directory /sys 2024-09-21T09:20:49.574 INFO:teuthology.orchestra.run.smithi064.stdout:skipping the directory /proc 2024-09-21T09:20:49.574 INFO:teuthology.orchestra.run.smithi064.stdout:skipping the directory /mnt 2024-09-21T09:20:49.574 INFO:teuthology.orchestra.run.smithi064.stdout:skipping the directory /var/tmp 2024-09-21T09:20:49.574 INFO:teuthology.orchestra.run.smithi064.stdout:skipping the directory /home 2024-09-21T09:20:49.575 INFO:teuthology.orchestra.run.smithi064.stdout:skipping the directory /root 2024-09-21T09:20:49.575 INFO:teuthology.orchestra.run.smithi064.stdout:skipping the directory /tmp 2024-09-21T09:20:49.575 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-09-21T09:20:49.644 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : ceph-mgr-cephadm-2:17.2.7-1649.gfbd1a6ab.el9.noa 96/116 2024-09-21T09:20:50.421 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : ceph-osd-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 104/116 2024-09-21T09:20:50.450 INFO:teuthology.orchestra.run.smithi022.stdout: Running scriptlet: ceph-osd-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 104/116 2024-09-21T09:20:50.450 INFO:teuthology.orchestra.run.smithi022.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-21T09:20:50.450 INFO:teuthology.orchestra.run.smithi022.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2024-09-21T09:20:50.450 INFO:teuthology.orchestra.run.smithi022.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-09-21T09:20:50.450 INFO:teuthology.orchestra.run.smithi022.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-09-21T09:20:50.450 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-09-21T09:20:50.691 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: ceph-mgr-cephadm-2:17.2.7-1649.gfbd1a6ab.el9.noa 96/116 2024-09-21T09:20:50.723 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : ceph-mgr-dashboard-2:17.2.7-1649.gfbd1a6ab.el9.n 97/116 2024-09-21T09:20:51.737 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: ceph-mgr-dashboard-2:17.2.7-1649.gfbd1a6ab.el9.n 97/116 2024-09-21T09:20:51.766 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : ceph-mgr-diskprediction-local-2:17.2.7-1649.gfbd 98/116 2024-09-21T09:20:51.776 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : ceph-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 105/116 2024-09-21T09:20:51.815 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : ceph-radosgw-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 106/116 2024-09-21T09:20:51.877 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:17.2.7-1649.gfbd 98/116 2024-09-21T09:20:51.894 INFO:teuthology.orchestra.run.smithi022.stdout: Running scriptlet: ceph-radosgw-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 106/116 2024-09-21T09:20:51.894 INFO:teuthology.orchestra.run.smithi022.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-21T09:20:51.894 INFO:teuthology.orchestra.run.smithi022.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2024-09-21T09:20:51.894 INFO:teuthology.orchestra.run.smithi022.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-09-21T09:20:51.894 INFO:teuthology.orchestra.run.smithi022.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-09-21T09:20:51.894 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-09-21T09:20:51.932 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : ceph-immutable-object-cache-2:17.2.7-1649.gfbd1a 107/116 2024-09-21T09:20:51.957 INFO:teuthology.orchestra.run.smithi022.stdout: Running scriptlet: ceph-immutable-object-cache-2:17.2.7-1649.gfbd1a 107/116 2024-09-21T09:20:51.957 INFO:teuthology.orchestra.run.smithi022.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-21T09:20:51.957 INFO:teuthology.orchestra.run.smithi022.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2024-09-21T09:20:51.957 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-09-21T09:20:52.052 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : ceph-mgr-modules-core-2:17.2.7-1649.gfbd1a6ab.el 99/116 2024-09-21T09:20:52.076 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : ceph-mgr-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 100/116 2024-09-21T09:20:52.101 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: ceph-mgr-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 100/116 2024-09-21T09:20:52.101 INFO:teuthology.orchestra.run.smithi064.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-21T09:20:52.101 INFO:teuthology.orchestra.run.smithi064.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2024-09-21T09:20:52.102 INFO:teuthology.orchestra.run.smithi064.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-09-21T09:20:52.102 INFO:teuthology.orchestra.run.smithi064.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-09-21T09:20:52.102 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-09-21T09:20:52.143 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : ceph-mgr-rook-2:17.2.7-1649.gfbd1a6ab.el9.noarch 101/116 2024-09-21T09:20:52.266 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : rbd-mirror-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 108/116 2024-09-21T09:20:52.291 INFO:teuthology.orchestra.run.smithi022.stdout: Running scriptlet: rbd-mirror-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 108/116 2024-09-21T09:20:52.291 INFO:teuthology.orchestra.run.smithi022.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-21T09:20:52.291 INFO:teuthology.orchestra.run.smithi022.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2024-09-21T09:20:52.291 INFO:teuthology.orchestra.run.smithi022.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-09-21T09:20:52.291 INFO:teuthology.orchestra.run.smithi022.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-09-21T09:20:52.291 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-09-21T09:20:52.337 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: ceph-mgr-rook-2:17.2.7-1649.gfbd1a6ab.el9.noarch 101/116 2024-09-21T09:20:52.360 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : ceph-mds-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 102/116 2024-09-21T09:20:52.385 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: ceph-mds-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 102/116 2024-09-21T09:20:52.385 INFO:teuthology.orchestra.run.smithi064.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-21T09:20:52.385 INFO:teuthology.orchestra.run.smithi064.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2024-09-21T09:20:52.385 INFO:teuthology.orchestra.run.smithi064.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-09-21T09:20:52.385 INFO:teuthology.orchestra.run.smithi064.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-09-21T09:20:52.385 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-09-21T09:20:52.795 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : ceph-mon-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 103/116 2024-09-21T09:20:52.820 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: ceph-mon-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 103/116 2024-09-21T09:20:52.820 INFO:teuthology.orchestra.run.smithi064.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-21T09:20:52.820 INFO:teuthology.orchestra.run.smithi064.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2024-09-21T09:20:52.820 INFO:teuthology.orchestra.run.smithi064.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-09-21T09:20:52.821 INFO:teuthology.orchestra.run.smithi064.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-09-21T09:20:52.821 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-09-21T09:20:54.212 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : ceph-osd-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 104/116 2024-09-21T09:20:54.241 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: ceph-osd-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 104/116 2024-09-21T09:20:54.241 INFO:teuthology.orchestra.run.smithi064.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-21T09:20:54.241 INFO:teuthology.orchestra.run.smithi064.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2024-09-21T09:20:54.241 INFO:teuthology.orchestra.run.smithi064.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-09-21T09:20:54.241 INFO:teuthology.orchestra.run.smithi064.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-09-21T09:20:54.241 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-09-21T09:20:55.563 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : ceph-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 105/116 2024-09-21T09:20:55.597 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : ceph-radosgw-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 106/116 2024-09-21T09:20:55.676 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: ceph-radosgw-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 106/116 2024-09-21T09:20:55.676 INFO:teuthology.orchestra.run.smithi064.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-21T09:20:55.677 INFO:teuthology.orchestra.run.smithi064.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2024-09-21T09:20:55.677 INFO:teuthology.orchestra.run.smithi064.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-09-21T09:20:55.677 INFO:teuthology.orchestra.run.smithi064.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-09-21T09:20:55.677 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-09-21T09:20:55.715 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : ceph-immutable-object-cache-2:17.2.7-1649.gfbd1a 107/116 2024-09-21T09:20:55.739 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: ceph-immutable-object-cache-2:17.2.7-1649.gfbd1a 107/116 2024-09-21T09:20:55.740 INFO:teuthology.orchestra.run.smithi064.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-21T09:20:55.740 INFO:teuthology.orchestra.run.smithi064.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2024-09-21T09:20:55.740 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-09-21T09:20:56.057 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : rbd-mirror-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 108/116 2024-09-21T09:20:56.082 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: rbd-mirror-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 108/116 2024-09-21T09:20:56.082 INFO:teuthology.orchestra.run.smithi064.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-21T09:20:56.082 INFO:teuthology.orchestra.run.smithi064.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2024-09-21T09:20:56.082 INFO:teuthology.orchestra.run.smithi064.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-09-21T09:20:56.082 INFO:teuthology.orchestra.run.smithi064.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-09-21T09:20:56.082 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-09-21T09:20:56.992 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : ceph-test-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 109/116 2024-09-21T09:20:57.028 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : ceph-fuse-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 110/116 2024-09-21T09:20:57.069 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : rbd-fuse-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 111/116 2024-09-21T09:20:57.194 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : rbd-nbd-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 112/116 2024-09-21T09:20:57.235 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : python3-pytest-6.2.2-7.el9.noarch 113/116 2024-09-21T09:20:57.258 INFO:teuthology.orchestra.run.smithi022.stdout: Installing : libcephfs-devel-2:17.2.7-1649.gfbd1a6ab.el9.x86_ 114/116 2024-09-21T09:20:57.258 INFO:teuthology.orchestra.run.smithi022.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 115/116 2024-09-21T09:20:57.342 INFO:teuthology.orchestra.run.smithi022.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 115/116 2024-09-21T09:20:57.342 INFO:teuthology.orchestra.run.smithi022.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 116/116 2024-09-21T09:21:00.784 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : ceph-test-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 109/116 2024-09-21T09:21:00.819 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : ceph-fuse-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 110/116 2024-09-21T09:21:00.860 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : rbd-fuse-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 111/116 2024-09-21T09:21:00.986 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : rbd-nbd-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 112/116 2024-09-21T09:21:01.017 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-pytest-6.2.2-7.el9.noarch 113/116 2024-09-21T09:21:01.040 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : libcephfs-devel-2:17.2.7-1649.gfbd1a6ab.el9.x86_ 114/116 2024-09-21T09:21:01.040 INFO:teuthology.orchestra.run.smithi064.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 115/116 2024-09-21T09:21:01.124 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 115/116 2024-09-21T09:21:01.124 INFO:teuthology.orchestra.run.smithi064.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 116/116 2024-09-21T09:21:02.443 INFO:teuthology.orchestra.run.smithi022.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 116/116 2024-09-21T09:21:02.443 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : ceph-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 1/116 2024-09-21T09:21:02.443 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : ceph-base-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2/116 2024-09-21T09:21:02.443 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : ceph-common-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 3/116 2024-09-21T09:21:02.443 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : ceph-fuse-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 4/116 2024-09-21T09:21:02.443 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : ceph-immutable-object-cache-2:17.2.7-1649.gfbd1a 5/116 2024-09-21T09:21:02.444 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : ceph-mds-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 6/116 2024-09-21T09:21:02.444 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : ceph-mgr-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 7/116 2024-09-21T09:21:02.444 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : ceph-mon-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 8/116 2024-09-21T09:21:02.444 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : ceph-osd-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 9/116 2024-09-21T09:21:02.444 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : ceph-radosgw-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 10/116 2024-09-21T09:21:02.445 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : ceph-selinux-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 11/116 2024-09-21T09:21:02.445 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : ceph-test-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 12/116 2024-09-21T09:21:02.445 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : libcephfs-devel-2:17.2.7-1649.gfbd1a6ab.el9.x86_ 13/116 2024-09-21T09:21:02.445 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : libcephfs2-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 14/116 2024-09-21T09:21:02.445 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : libcephsqlite-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 15/116 2024-09-21T09:21:02.446 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : librados-devel-2:17.2.7-1649.gfbd1a6ab.el9.x86_6 16/116 2024-09-21T09:21:02.446 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : libradosstriper1-2:17.2.7-1649.gfbd1a6ab.el9.x86 17/116 2024-09-21T09:21:02.446 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : librgw2-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 18/116 2024-09-21T09:21:02.446 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : python3-ceph-argparse-2:17.2.7-1649.gfbd1a6ab.el 19/116 2024-09-21T09:21:02.446 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : python3-ceph-common-2:17.2.7-1649.gfbd1a6ab.el9. 20/116 2024-09-21T09:21:02.446 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : python3-cephfs-2:17.2.7-1649.gfbd1a6ab.el9.x86_6 21/116 2024-09-21T09:21:02.446 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : python3-rados-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 22/116 2024-09-21T09:21:02.446 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : python3-rbd-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 23/116 2024-09-21T09:21:02.446 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : python3-rgw-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 24/116 2024-09-21T09:21:02.446 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : rbd-fuse-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 25/116 2024-09-21T09:21:02.446 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : rbd-mirror-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 26/116 2024-09-21T09:21:02.446 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : rbd-nbd-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 27/116 2024-09-21T09:21:02.447 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : ceph-grafana-dashboards-2:17.2.7-1649.gfbd1a6ab. 28/116 2024-09-21T09:21:02.447 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : ceph-mgr-cephadm-2:17.2.7-1649.gfbd1a6ab.el9.noa 29/116 2024-09-21T09:21:02.447 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : ceph-mgr-dashboard-2:17.2.7-1649.gfbd1a6ab.el9.n 30/116 2024-09-21T09:21:02.447 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : ceph-mgr-diskprediction-local-2:17.2.7-1649.gfbd 31/116 2024-09-21T09:21:02.447 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : ceph-mgr-modules-core-2:17.2.7-1649.gfbd1a6ab.el 32/116 2024-09-21T09:21:02.447 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : ceph-mgr-rook-2:17.2.7-1649.gfbd1a6ab.el9.noarch 33/116 2024-09-21T09:21:02.447 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : ceph-prometheus-alerts-2:17.2.7-1649.gfbd1a6ab.e 34/116 2024-09-21T09:21:02.447 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : cephadm-2:17.2.7-1649.gfbd1a6ab.el9.noarch 35/116 2024-09-21T09:21:02.447 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : fuse-2.9.9-16.el9.x86_64 36/116 2024-09-21T09:21:02.447 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : libgfortran-11.5.0-2.el9.x86_64 37/116 2024-09-21T09:21:02.447 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : libquadmath-11.5.0-2.el9.x86_64 38/116 2024-09-21T09:21:02.447 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 39/116 2024-09-21T09:21:02.448 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : python3-cryptography-36.0.1-4.el9.x86_64 40/116 2024-09-21T09:21:02.448 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : python3-ply-3.11-14.el9.noarch 41/116 2024-09-21T09:21:02.448 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 42/116 2024-09-21T09:21:02.448 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : python3-requests-2.25.1-8.el9.noarch 43/116 2024-09-21T09:21:02.448 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : python3-urllib3-1.26.5-6.el9.noarch 44/116 2024-09-21T09:21:02.448 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : flexiblas-3.0.4-8.el9.x86_64 45/116 2024-09-21T09:21:02.448 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : flexiblas-netlib-3.0.4-8.el9.x86_64 46/116 2024-09-21T09:21:02.448 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 47/116 2024-09-21T09:21:02.448 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 48/116 2024-09-21T09:21:02.448 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 49/116 2024-09-21T09:21:02.448 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 50/116 2024-09-21T09:21:02.448 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 51/116 2024-09-21T09:21:02.448 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : openblas-0.3.26-2.el9.x86_64 52/116 2024-09-21T09:21:02.449 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : openblas-openmp-0.3.26-2.el9.x86_64 53/116 2024-09-21T09:21:02.449 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : python3-attrs-20.3.0-7.el9.noarch 54/116 2024-09-21T09:21:02.449 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 55/116 2024-09-21T09:21:02.449 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : python3-devel-3.9.19-8.el9.x86_64 56/116 2024-09-21T09:21:02.449 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : python3-iniconfig-1.1.1-7.el9.noarch 57/116 2024-09-21T09:21:02.449 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : python3-jinja2-2.11.3-6.el9.noarch 58/116 2024-09-21T09:21:02.449 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : python3-jsonpatch-1.21-16.el9.noarch 59/116 2024-09-21T09:21:02.449 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : python3-jsonpointer-2.0-4.el9.noarch 60/116 2024-09-21T09:21:02.449 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : python3-mako-1.1.4-6.el9.noarch 61/116 2024-09-21T09:21:02.449 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 62/116 2024-09-21T09:21:02.450 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : python3-numpy-1:1.20.1-5.el9.x86_64 63/116 2024-09-21T09:21:02.450 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 64/116 2024-09-21T09:21:02.450 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : python3-oauthlib-3.1.1-5.el9.noarch 65/116 2024-09-21T09:21:02.450 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : python3-packaging-20.9-5.el9.noarch 66/116 2024-09-21T09:21:02.450 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : python3-pluggy-0.13.1-7.el9.noarch 67/116 2024-09-21T09:21:02.450 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : python3-prettytable-0.7.2-27.el9.noarch 68/116 2024-09-21T09:21:02.450 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : python3-py-1.10.0-6.el9.noarch 69/116 2024-09-21T09:21:02.450 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : python3-pyasn1-0.4.8-6.el9.noarch 70/116 2024-09-21T09:21:02.450 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : python3-pyasn1-modules-0.4.8-6.el9.noarch 71/116 2024-09-21T09:21:02.450 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : python3-pytest-6.2.2-7.el9.noarch 72/116 2024-09-21T09:21:02.450 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : python3-pytz-2021.1-5.el9.noarch 73/116 2024-09-21T09:21:02.450 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 74/116 2024-09-21T09:21:02.451 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : python3-scipy-1.6.2-8.el9.x86_64 75/116 2024-09-21T09:21:02.451 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 76/116 2024-09-21T09:21:02.451 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : socat-1.7.4.1-6.el9.x86_64 77/116 2024-09-21T09:21:02.451 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 78/116 2024-09-21T09:21:02.451 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : gperftools-libs-2.9.1-3.el9.x86_64 79/116 2024-09-21T09:21:02.451 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : liboath-2.6.7-2.el9.x86_64 80/116 2024-09-21T09:21:02.451 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 81/116 2024-09-21T09:21:02.451 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : python3-asyncssh-2.13.2-5.el9.noarch 82/116 2024-09-21T09:21:02.451 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 83/116 2024-09-21T09:21:02.451 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 84/116 2024-09-21T09:21:02.451 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : python3-certifi-2023.05.07-4.el9.noarch 85/116 2024-09-21T09:21:02.451 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : python3-cheroot-8.6.0-4.el9.noarch 86/116 2024-09-21T09:21:02.451 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 87/116 2024-09-21T09:21:02.451 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : python3-google-auth-1:2.34.0-1.el9.noarch 88/116 2024-09-21T09:21:02.452 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 89/116 2024-09-21T09:21:02.452 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 90/116 2024-09-21T09:21:02.452 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 91/116 2024-09-21T09:21:02.452 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 92/116 2024-09-21T09:21:02.452 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : python3-jaraco-text-3.2.0-6.el9.noarch 93/116 2024-09-21T09:21:02.452 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : python3-jwt+crypto-2.4.0-1.el9.noarch 94/116 2024-09-21T09:21:02.452 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : python3-jwt-2.4.0-1.el9.noarch 95/116 2024-09-21T09:21:02.452 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : python3-kubernetes-1:26.1.0-2.el9.noarch 96/116 2024-09-21T09:21:02.452 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : python3-logutils-0.3.5-21.el9.noarch 97/116 2024-09-21T09:21:02.452 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 98/116 2024-09-21T09:21:02.452 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 99/116 2024-09-21T09:21:02.452 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : python3-pecan-1.4.2-3.el9.noarch 100/116 2024-09-21T09:21:02.452 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 101/116 2024-09-21T09:21:02.452 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 102/116 2024-09-21T09:21:02.453 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 103/116 2024-09-21T09:21:02.453 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 104/116 2024-09-21T09:21:02.453 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 105/116 2024-09-21T09:21:02.453 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 106/116 2024-09-21T09:21:02.453 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : python3-typing-extensions-4.12.2-2.el9.noarch 107/116 2024-09-21T09:21:02.453 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : python3-webob-1.8.8-2.el9.noarch 108/116 2024-09-21T09:21:02.453 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 109/116 2024-09-21T09:21:02.453 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : python3-werkzeug-2.0.3-3.el9.1.noarch 110/116 2024-09-21T09:21:02.453 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 111/116 2024-09-21T09:21:02.453 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : thrift-0.15.0-2.el9.x86_64 112/116 2024-09-21T09:21:02.453 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : librados2-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 113/116 2024-09-21T09:21:02.453 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 114/116 2024-09-21T09:21:02.454 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : librbd1-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 115/116 2024-09-21T09:21:04.508 INFO:teuthology.orchestra.run.smithi022.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 116/116 2024-09-21T09:21:04.508 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-09-21T09:21:04.508 INFO:teuthology.orchestra.run.smithi022.stdout:Upgraded: 2024-09-21T09:21:04.508 INFO:teuthology.orchestra.run.smithi022.stdout: librados2-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:21:04.508 INFO:teuthology.orchestra.run.smithi022.stdout: librbd1-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:21:04.508 INFO:teuthology.orchestra.run.smithi022.stdout:Installed: 2024-09-21T09:21:04.509 INFO:teuthology.orchestra.run.smithi022.stdout: ceph-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:21:04.509 INFO:teuthology.orchestra.run.smithi022.stdout: ceph-base-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:21:04.509 INFO:teuthology.orchestra.run.smithi022.stdout: ceph-common-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:21:04.509 INFO:teuthology.orchestra.run.smithi022.stdout: ceph-fuse-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:21:04.509 INFO:teuthology.orchestra.run.smithi022.stdout: ceph-grafana-dashboards-2:17.2.7-1649.gfbd1a6ab.el9.noarch 2024-09-21T09:21:04.509 INFO:teuthology.orchestra.run.smithi022.stdout: ceph-immutable-object-cache-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:21:04.509 INFO:teuthology.orchestra.run.smithi022.stdout: ceph-mds-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:21:04.509 INFO:teuthology.orchestra.run.smithi022.stdout: ceph-mgr-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:21:04.509 INFO:teuthology.orchestra.run.smithi022.stdout: ceph-mgr-cephadm-2:17.2.7-1649.gfbd1a6ab.el9.noarch 2024-09-21T09:21:04.509 INFO:teuthology.orchestra.run.smithi022.stdout: ceph-mgr-dashboard-2:17.2.7-1649.gfbd1a6ab.el9.noarch 2024-09-21T09:21:04.510 INFO:teuthology.orchestra.run.smithi022.stdout: ceph-mgr-diskprediction-local-2:17.2.7-1649.gfbd1a6ab.el9.noarch 2024-09-21T09:21:04.510 INFO:teuthology.orchestra.run.smithi022.stdout: ceph-mgr-modules-core-2:17.2.7-1649.gfbd1a6ab.el9.noarch 2024-09-21T09:21:04.510 INFO:teuthology.orchestra.run.smithi022.stdout: ceph-mgr-rook-2:17.2.7-1649.gfbd1a6ab.el9.noarch 2024-09-21T09:21:04.510 INFO:teuthology.orchestra.run.smithi022.stdout: ceph-mon-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:21:04.510 INFO:teuthology.orchestra.run.smithi022.stdout: ceph-osd-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:21:04.510 INFO:teuthology.orchestra.run.smithi022.stdout: ceph-prometheus-alerts-2:17.2.7-1649.gfbd1a6ab.el9.noarch 2024-09-21T09:21:04.510 INFO:teuthology.orchestra.run.smithi022.stdout: ceph-radosgw-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:21:04.510 INFO:teuthology.orchestra.run.smithi022.stdout: ceph-selinux-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:21:04.510 INFO:teuthology.orchestra.run.smithi022.stdout: ceph-test-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:21:04.510 INFO:teuthology.orchestra.run.smithi022.stdout: cephadm-2:17.2.7-1649.gfbd1a6ab.el9.noarch 2024-09-21T09:21:04.510 INFO:teuthology.orchestra.run.smithi022.stdout: flexiblas-3.0.4-8.el9.x86_64 2024-09-21T09:21:04.510 INFO:teuthology.orchestra.run.smithi022.stdout: flexiblas-netlib-3.0.4-8.el9.x86_64 2024-09-21T09:21:04.511 INFO:teuthology.orchestra.run.smithi022.stdout: flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 2024-09-21T09:21:04.511 INFO:teuthology.orchestra.run.smithi022.stdout: fuse-2.9.9-16.el9.x86_64 2024-09-21T09:21:04.511 INFO:teuthology.orchestra.run.smithi022.stdout: gperftools-libs-2.9.1-3.el9.x86_64 2024-09-21T09:21:04.511 INFO:teuthology.orchestra.run.smithi022.stdout: libcephfs-devel-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:21:04.511 INFO:teuthology.orchestra.run.smithi022.stdout: libcephfs2-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:21:04.511 INFO:teuthology.orchestra.run.smithi022.stdout: libcephsqlite-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:21:04.511 INFO:teuthology.orchestra.run.smithi022.stdout: libgfortran-11.5.0-2.el9.x86_64 2024-09-21T09:21:04.511 INFO:teuthology.orchestra.run.smithi022.stdout: liboath-2.6.7-2.el9.x86_64 2024-09-21T09:21:04.511 INFO:teuthology.orchestra.run.smithi022.stdout: libpmemobj-1.12.1-1.el9.x86_64 2024-09-21T09:21:04.511 INFO:teuthology.orchestra.run.smithi022.stdout: libquadmath-11.5.0-2.el9.x86_64 2024-09-21T09:21:04.511 INFO:teuthology.orchestra.run.smithi022.stdout: librabbitmq-0.11.0-7.el9.x86_64 2024-09-21T09:21:04.511 INFO:teuthology.orchestra.run.smithi022.stdout: librados-devel-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:21:04.511 INFO:teuthology.orchestra.run.smithi022.stdout: libradosstriper1-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:21:04.512 INFO:teuthology.orchestra.run.smithi022.stdout: librdkafka-1.6.1-102.el9.x86_64 2024-09-21T09:21:04.512 INFO:teuthology.orchestra.run.smithi022.stdout: librgw2-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:21:04.512 INFO:teuthology.orchestra.run.smithi022.stdout: libunwind-1.6.2-1.el9.x86_64 2024-09-21T09:21:04.512 INFO:teuthology.orchestra.run.smithi022.stdout: lttng-ust-2.12.0-6.el9.x86_64 2024-09-21T09:21:04.512 INFO:teuthology.orchestra.run.smithi022.stdout: openblas-0.3.26-2.el9.x86_64 2024-09-21T09:21:04.512 INFO:teuthology.orchestra.run.smithi022.stdout: openblas-openmp-0.3.26-2.el9.x86_64 2024-09-21T09:21:04.512 INFO:teuthology.orchestra.run.smithi022.stdout: python3-asyncssh-2.13.2-5.el9.noarch 2024-09-21T09:21:04.512 INFO:teuthology.orchestra.run.smithi022.stdout: python3-attrs-20.3.0-7.el9.noarch 2024-09-21T09:21:04.512 INFO:teuthology.orchestra.run.smithi022.stdout: python3-babel-2.9.1-2.el9.noarch 2024-09-21T09:21:04.512 INFO:teuthology.orchestra.run.smithi022.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2024-09-21T09:21:04.512 INFO:teuthology.orchestra.run.smithi022.stdout: python3-cachetools-4.2.4-1.el9.noarch 2024-09-21T09:21:04.512 INFO:teuthology.orchestra.run.smithi022.stdout: python3-ceph-argparse-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:21:04.512 INFO:teuthology.orchestra.run.smithi022.stdout: python3-ceph-common-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:21:04.513 INFO:teuthology.orchestra.run.smithi022.stdout: python3-cephfs-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:21:04.513 INFO:teuthology.orchestra.run.smithi022.stdout: python3-certifi-2023.05.07-4.el9.noarch 2024-09-21T09:21:04.513 INFO:teuthology.orchestra.run.smithi022.stdout: python3-cffi-1.14.5-5.el9.x86_64 2024-09-21T09:21:04.513 INFO:teuthology.orchestra.run.smithi022.stdout: python3-cheroot-8.6.0-4.el9.noarch 2024-09-21T09:21:04.513 INFO:teuthology.orchestra.run.smithi022.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2024-09-21T09:21:04.513 INFO:teuthology.orchestra.run.smithi022.stdout: python3-cryptography-36.0.1-4.el9.x86_64 2024-09-21T09:21:04.513 INFO:teuthology.orchestra.run.smithi022.stdout: python3-devel-3.9.19-8.el9.x86_64 2024-09-21T09:21:04.513 INFO:teuthology.orchestra.run.smithi022.stdout: python3-google-auth-1:2.34.0-1.el9.noarch 2024-09-21T09:21:04.513 INFO:teuthology.orchestra.run.smithi022.stdout: python3-iniconfig-1.1.1-7.el9.noarch 2024-09-21T09:21:04.513 INFO:teuthology.orchestra.run.smithi022.stdout: python3-jaraco-8.2.1-3.el9.noarch 2024-09-21T09:21:04.513 INFO:teuthology.orchestra.run.smithi022.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2024-09-21T09:21:04.513 INFO:teuthology.orchestra.run.smithi022.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2024-09-21T09:21:04.513 INFO:teuthology.orchestra.run.smithi022.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2024-09-21T09:21:04.513 INFO:teuthology.orchestra.run.smithi022.stdout: python3-jaraco-text-3.2.0-6.el9.noarch 2024-09-21T09:21:04.514 INFO:teuthology.orchestra.run.smithi022.stdout: python3-jinja2-2.11.3-6.el9.noarch 2024-09-21T09:21:04.514 INFO:teuthology.orchestra.run.smithi022.stdout: python3-jsonpatch-1.21-16.el9.noarch 2024-09-21T09:21:04.514 INFO:teuthology.orchestra.run.smithi022.stdout: python3-jsonpointer-2.0-4.el9.noarch 2024-09-21T09:21:04.514 INFO:teuthology.orchestra.run.smithi022.stdout: python3-jwt-2.4.0-1.el9.noarch 2024-09-21T09:21:04.514 INFO:teuthology.orchestra.run.smithi022.stdout: python3-jwt+crypto-2.4.0-1.el9.noarch 2024-09-21T09:21:04.514 INFO:teuthology.orchestra.run.smithi022.stdout: python3-kubernetes-1:26.1.0-2.el9.noarch 2024-09-21T09:21:04.514 INFO:teuthology.orchestra.run.smithi022.stdout: python3-logutils-0.3.5-21.el9.noarch 2024-09-21T09:21:04.514 INFO:teuthology.orchestra.run.smithi022.stdout: python3-mako-1.1.4-6.el9.noarch 2024-09-21T09:21:04.514 INFO:teuthology.orchestra.run.smithi022.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2024-09-21T09:21:04.514 INFO:teuthology.orchestra.run.smithi022.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2024-09-21T09:21:04.514 INFO:teuthology.orchestra.run.smithi022.stdout: python3-natsort-7.1.1-5.el9.noarch 2024-09-21T09:21:04.514 INFO:teuthology.orchestra.run.smithi022.stdout: python3-numpy-1:1.20.1-5.el9.x86_64 2024-09-21T09:21:04.514 INFO:teuthology.orchestra.run.smithi022.stdout: python3-numpy-f2py-1:1.20.1-5.el9.x86_64 2024-09-21T09:21:04.514 INFO:teuthology.orchestra.run.smithi022.stdout: python3-oauthlib-3.1.1-5.el9.noarch 2024-09-21T09:21:04.515 INFO:teuthology.orchestra.run.smithi022.stdout: python3-packaging-20.9-5.el9.noarch 2024-09-21T09:21:04.515 INFO:teuthology.orchestra.run.smithi022.stdout: python3-pecan-1.4.2-3.el9.noarch 2024-09-21T09:21:04.515 INFO:teuthology.orchestra.run.smithi022.stdout: python3-pluggy-0.13.1-7.el9.noarch 2024-09-21T09:21:04.515 INFO:teuthology.orchestra.run.smithi022.stdout: python3-ply-3.11-14.el9.noarch 2024-09-21T09:21:04.515 INFO:teuthology.orchestra.run.smithi022.stdout: python3-portend-3.1.0-2.el9.noarch 2024-09-21T09:21:04.515 INFO:teuthology.orchestra.run.smithi022.stdout: python3-prettytable-0.7.2-27.el9.noarch 2024-09-21T09:21:04.515 INFO:teuthology.orchestra.run.smithi022.stdout: python3-py-1.10.0-6.el9.noarch 2024-09-21T09:21:04.515 INFO:teuthology.orchestra.run.smithi022.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2024-09-21T09:21:04.515 INFO:teuthology.orchestra.run.smithi022.stdout: python3-pyasn1-0.4.8-6.el9.noarch 2024-09-21T09:21:04.515 INFO:teuthology.orchestra.run.smithi022.stdout: python3-pyasn1-modules-0.4.8-6.el9.noarch 2024-09-21T09:21:04.515 INFO:teuthology.orchestra.run.smithi022.stdout: python3-pycparser-2.20-6.el9.noarch 2024-09-21T09:21:04.515 INFO:teuthology.orchestra.run.smithi022.stdout: python3-pytest-6.2.2-7.el9.noarch 2024-09-21T09:21:04.515 INFO:teuthology.orchestra.run.smithi022.stdout: python3-pytz-2021.1-5.el9.noarch 2024-09-21T09:21:04.516 INFO:teuthology.orchestra.run.smithi022.stdout: python3-rados-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:21:04.516 INFO:teuthology.orchestra.run.smithi022.stdout: python3-rbd-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:21:04.516 INFO:teuthology.orchestra.run.smithi022.stdout: python3-repoze-lru-0.7-16.el9.noarch 2024-09-21T09:21:04.516 INFO:teuthology.orchestra.run.smithi022.stdout: python3-requests-2.25.1-8.el9.noarch 2024-09-21T09:21:04.516 INFO:teuthology.orchestra.run.smithi022.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2024-09-21T09:21:04.516 INFO:teuthology.orchestra.run.smithi022.stdout: python3-rgw-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:21:04.516 INFO:teuthology.orchestra.run.smithi022.stdout: python3-routes-2.5.1-5.el9.noarch 2024-09-21T09:21:04.516 INFO:teuthology.orchestra.run.smithi022.stdout: python3-rsa-4.9-2.el9.noarch 2024-09-21T09:21:04.516 INFO:teuthology.orchestra.run.smithi022.stdout: python3-scipy-1.6.2-8.el9.x86_64 2024-09-21T09:21:04.516 INFO:teuthology.orchestra.run.smithi022.stdout: python3-tempora-5.0.0-2.el9.noarch 2024-09-21T09:21:04.516 INFO:teuthology.orchestra.run.smithi022.stdout: python3-toml-0.10.2-6.el9.noarch 2024-09-21T09:21:04.516 INFO:teuthology.orchestra.run.smithi022.stdout: python3-typing-extensions-4.12.2-2.el9.noarch 2024-09-21T09:21:04.516 INFO:teuthology.orchestra.run.smithi022.stdout: python3-urllib3-1.26.5-6.el9.noarch 2024-09-21T09:21:04.517 INFO:teuthology.orchestra.run.smithi022.stdout: python3-webob-1.8.8-2.el9.noarch 2024-09-21T09:21:04.517 INFO:teuthology.orchestra.run.smithi022.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2024-09-21T09:21:04.517 INFO:teuthology.orchestra.run.smithi022.stdout: python3-werkzeug-2.0.3-3.el9.1.noarch 2024-09-21T09:21:04.517 INFO:teuthology.orchestra.run.smithi022.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2024-09-21T09:21:04.517 INFO:teuthology.orchestra.run.smithi022.stdout: rbd-fuse-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:21:04.517 INFO:teuthology.orchestra.run.smithi022.stdout: rbd-mirror-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:21:04.517 INFO:teuthology.orchestra.run.smithi022.stdout: rbd-nbd-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:21:04.517 INFO:teuthology.orchestra.run.smithi022.stdout: socat-1.7.4.1-6.el9.x86_64 2024-09-21T09:21:04.517 INFO:teuthology.orchestra.run.smithi022.stdout: thrift-0.15.0-2.el9.x86_64 2024-09-21T09:21:04.517 INFO:teuthology.orchestra.run.smithi022.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2024-09-21T09:21:04.518 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-09-21T09:21:04.518 INFO:teuthology.orchestra.run.smithi022.stdout:Complete! 2024-09-21T09:21:04.921 DEBUG:teuthology.parallel:result is None 2024-09-21T09:21:06.033 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 116/116 2024-09-21T09:21:06.033 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : ceph-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 1/116 2024-09-21T09:21:06.034 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : ceph-base-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2/116 2024-09-21T09:21:06.034 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : ceph-common-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 3/116 2024-09-21T09:21:06.034 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : ceph-fuse-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 4/116 2024-09-21T09:21:06.034 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : ceph-immutable-object-cache-2:17.2.7-1649.gfbd1a 5/116 2024-09-21T09:21:06.034 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : ceph-mds-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 6/116 2024-09-21T09:21:06.034 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : ceph-mgr-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 7/116 2024-09-21T09:21:06.034 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : ceph-mon-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 8/116 2024-09-21T09:21:06.034 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : ceph-osd-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 9/116 2024-09-21T09:21:06.034 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : ceph-radosgw-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 10/116 2024-09-21T09:21:06.034 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : ceph-selinux-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 11/116 2024-09-21T09:21:06.034 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : ceph-test-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 12/116 2024-09-21T09:21:06.035 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : libcephfs-devel-2:17.2.7-1649.gfbd1a6ab.el9.x86_ 13/116 2024-09-21T09:21:06.035 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : libcephfs2-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 14/116 2024-09-21T09:21:06.035 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : libcephsqlite-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 15/116 2024-09-21T09:21:06.035 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : librados-devel-2:17.2.7-1649.gfbd1a6ab.el9.x86_6 16/116 2024-09-21T09:21:06.035 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : libradosstriper1-2:17.2.7-1649.gfbd1a6ab.el9.x86 17/116 2024-09-21T09:21:06.035 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : librgw2-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 18/116 2024-09-21T09:21:06.035 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-ceph-argparse-2:17.2.7-1649.gfbd1a6ab.el 19/116 2024-09-21T09:21:06.035 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-ceph-common-2:17.2.7-1649.gfbd1a6ab.el9. 20/116 2024-09-21T09:21:06.035 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-cephfs-2:17.2.7-1649.gfbd1a6ab.el9.x86_6 21/116 2024-09-21T09:21:06.035 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-rados-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 22/116 2024-09-21T09:21:06.035 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-rbd-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 23/116 2024-09-21T09:21:06.035 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-rgw-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 24/116 2024-09-21T09:21:06.035 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : rbd-fuse-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 25/116 2024-09-21T09:21:06.036 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : rbd-mirror-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 26/116 2024-09-21T09:21:06.036 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : rbd-nbd-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 27/116 2024-09-21T09:21:06.036 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : ceph-grafana-dashboards-2:17.2.7-1649.gfbd1a6ab. 28/116 2024-09-21T09:21:06.036 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : ceph-mgr-cephadm-2:17.2.7-1649.gfbd1a6ab.el9.noa 29/116 2024-09-21T09:21:06.036 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : ceph-mgr-dashboard-2:17.2.7-1649.gfbd1a6ab.el9.n 30/116 2024-09-21T09:21:06.036 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : ceph-mgr-diskprediction-local-2:17.2.7-1649.gfbd 31/116 2024-09-21T09:21:06.036 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : ceph-mgr-modules-core-2:17.2.7-1649.gfbd1a6ab.el 32/116 2024-09-21T09:21:06.036 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : ceph-mgr-rook-2:17.2.7-1649.gfbd1a6ab.el9.noarch 33/116 2024-09-21T09:21:06.036 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : ceph-prometheus-alerts-2:17.2.7-1649.gfbd1a6ab.e 34/116 2024-09-21T09:21:06.036 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : cephadm-2:17.2.7-1649.gfbd1a6ab.el9.noarch 35/116 2024-09-21T09:21:06.036 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : fuse-2.9.9-16.el9.x86_64 36/116 2024-09-21T09:21:06.036 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : libgfortran-11.5.0-2.el9.x86_64 37/116 2024-09-21T09:21:06.036 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : libquadmath-11.5.0-2.el9.x86_64 38/116 2024-09-21T09:21:06.036 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 39/116 2024-09-21T09:21:06.037 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-cryptography-36.0.1-4.el9.x86_64 40/116 2024-09-21T09:21:06.037 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-ply-3.11-14.el9.noarch 41/116 2024-09-21T09:21:06.037 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 42/116 2024-09-21T09:21:06.037 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-requests-2.25.1-8.el9.noarch 43/116 2024-09-21T09:21:06.037 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-urllib3-1.26.5-6.el9.noarch 44/116 2024-09-21T09:21:06.037 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : flexiblas-3.0.4-8.el9.x86_64 45/116 2024-09-21T09:21:06.037 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : flexiblas-netlib-3.0.4-8.el9.x86_64 46/116 2024-09-21T09:21:06.037 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 47/116 2024-09-21T09:21:06.037 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 48/116 2024-09-21T09:21:06.037 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 49/116 2024-09-21T09:21:06.037 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 50/116 2024-09-21T09:21:06.037 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 51/116 2024-09-21T09:21:06.037 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : openblas-0.3.26-2.el9.x86_64 52/116 2024-09-21T09:21:06.038 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : openblas-openmp-0.3.26-2.el9.x86_64 53/116 2024-09-21T09:21:06.038 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-attrs-20.3.0-7.el9.noarch 54/116 2024-09-21T09:21:06.038 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 55/116 2024-09-21T09:21:06.038 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-devel-3.9.19-8.el9.x86_64 56/116 2024-09-21T09:21:06.038 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-iniconfig-1.1.1-7.el9.noarch 57/116 2024-09-21T09:21:06.038 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-jinja2-2.11.3-6.el9.noarch 58/116 2024-09-21T09:21:06.038 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-jsonpatch-1.21-16.el9.noarch 59/116 2024-09-21T09:21:06.038 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-jsonpointer-2.0-4.el9.noarch 60/116 2024-09-21T09:21:06.038 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-mako-1.1.4-6.el9.noarch 61/116 2024-09-21T09:21:06.038 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 62/116 2024-09-21T09:21:06.038 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-numpy-1:1.20.1-5.el9.x86_64 63/116 2024-09-21T09:21:06.038 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 64/116 2024-09-21T09:21:06.038 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-oauthlib-3.1.1-5.el9.noarch 65/116 2024-09-21T09:21:06.039 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-packaging-20.9-5.el9.noarch 66/116 2024-09-21T09:21:06.039 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-pluggy-0.13.1-7.el9.noarch 67/116 2024-09-21T09:21:06.039 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-prettytable-0.7.2-27.el9.noarch 68/116 2024-09-21T09:21:06.039 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-py-1.10.0-6.el9.noarch 69/116 2024-09-21T09:21:06.039 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-pyasn1-0.4.8-6.el9.noarch 70/116 2024-09-21T09:21:06.039 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-pyasn1-modules-0.4.8-6.el9.noarch 71/116 2024-09-21T09:21:06.039 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-pytest-6.2.2-7.el9.noarch 72/116 2024-09-21T09:21:06.039 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-pytz-2021.1-5.el9.noarch 73/116 2024-09-21T09:21:06.039 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 74/116 2024-09-21T09:21:06.039 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-scipy-1.6.2-8.el9.x86_64 75/116 2024-09-21T09:21:06.039 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 76/116 2024-09-21T09:21:06.039 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : socat-1.7.4.1-6.el9.x86_64 77/116 2024-09-21T09:21:06.039 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 78/116 2024-09-21T09:21:06.039 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : gperftools-libs-2.9.1-3.el9.x86_64 79/116 2024-09-21T09:21:06.040 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : liboath-2.6.7-2.el9.x86_64 80/116 2024-09-21T09:21:06.040 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 81/116 2024-09-21T09:21:06.040 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-asyncssh-2.13.2-5.el9.noarch 82/116 2024-09-21T09:21:06.040 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 83/116 2024-09-21T09:21:06.040 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 84/116 2024-09-21T09:21:06.040 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-certifi-2023.05.07-4.el9.noarch 85/116 2024-09-21T09:21:06.040 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-cheroot-8.6.0-4.el9.noarch 86/116 2024-09-21T09:21:06.040 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 87/116 2024-09-21T09:21:06.040 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-google-auth-1:2.34.0-1.el9.noarch 88/116 2024-09-21T09:21:06.040 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 89/116 2024-09-21T09:21:06.040 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 90/116 2024-09-21T09:21:06.040 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 91/116 2024-09-21T09:21:06.040 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 92/116 2024-09-21T09:21:06.041 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-jaraco-text-3.2.0-6.el9.noarch 93/116 2024-09-21T09:21:06.041 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-jwt+crypto-2.4.0-1.el9.noarch 94/116 2024-09-21T09:21:06.041 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-jwt-2.4.0-1.el9.noarch 95/116 2024-09-21T09:21:06.041 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-kubernetes-1:26.1.0-2.el9.noarch 96/116 2024-09-21T09:21:06.041 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-logutils-0.3.5-21.el9.noarch 97/116 2024-09-21T09:21:06.041 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 98/116 2024-09-21T09:21:06.041 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 99/116 2024-09-21T09:21:06.041 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-pecan-1.4.2-3.el9.noarch 100/116 2024-09-21T09:21:06.041 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 101/116 2024-09-21T09:21:06.041 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 102/116 2024-09-21T09:21:06.041 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 103/116 2024-09-21T09:21:06.041 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 104/116 2024-09-21T09:21:06.041 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 105/116 2024-09-21T09:21:06.041 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 106/116 2024-09-21T09:21:06.041 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-typing-extensions-4.12.2-2.el9.noarch 107/116 2024-09-21T09:21:06.041 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-webob-1.8.8-2.el9.noarch 108/116 2024-09-21T09:21:06.042 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 109/116 2024-09-21T09:21:06.042 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-werkzeug-2.0.3-3.el9.1.noarch 110/116 2024-09-21T09:21:06.042 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 111/116 2024-09-21T09:21:06.042 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : thrift-0.15.0-2.el9.x86_64 112/116 2024-09-21T09:21:06.042 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : librados2-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 113/116 2024-09-21T09:21:06.042 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 114/116 2024-09-21T09:21:06.042 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : librbd1-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 115/116 2024-09-21T09:21:07.781 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 116/116 2024-09-21T09:21:07.781 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-09-21T09:21:07.782 INFO:teuthology.orchestra.run.smithi064.stdout:Upgraded: 2024-09-21T09:21:07.782 INFO:teuthology.orchestra.run.smithi064.stdout: librados2-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:21:07.782 INFO:teuthology.orchestra.run.smithi064.stdout: librbd1-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:21:07.782 INFO:teuthology.orchestra.run.smithi064.stdout:Installed: 2024-09-21T09:21:07.782 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:21:07.782 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-base-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:21:07.782 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-common-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:21:07.782 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-fuse-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:21:07.782 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-grafana-dashboards-2:17.2.7-1649.gfbd1a6ab.el9.noarch 2024-09-21T09:21:07.782 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-immutable-object-cache-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:21:07.782 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-mds-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:21:07.782 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-mgr-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:21:07.782 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-mgr-cephadm-2:17.2.7-1649.gfbd1a6ab.el9.noarch 2024-09-21T09:21:07.783 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-mgr-dashboard-2:17.2.7-1649.gfbd1a6ab.el9.noarch 2024-09-21T09:21:07.783 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-mgr-diskprediction-local-2:17.2.7-1649.gfbd1a6ab.el9.noarch 2024-09-21T09:21:07.783 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-mgr-modules-core-2:17.2.7-1649.gfbd1a6ab.el9.noarch 2024-09-21T09:21:07.783 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-mgr-rook-2:17.2.7-1649.gfbd1a6ab.el9.noarch 2024-09-21T09:21:07.783 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-mon-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:21:07.783 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-osd-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:21:07.783 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-prometheus-alerts-2:17.2.7-1649.gfbd1a6ab.el9.noarch 2024-09-21T09:21:07.783 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-radosgw-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:21:07.783 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-selinux-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:21:07.783 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-test-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:21:07.783 INFO:teuthology.orchestra.run.smithi064.stdout: cephadm-2:17.2.7-1649.gfbd1a6ab.el9.noarch 2024-09-21T09:21:07.783 INFO:teuthology.orchestra.run.smithi064.stdout: flexiblas-3.0.4-8.el9.x86_64 2024-09-21T09:21:07.783 INFO:teuthology.orchestra.run.smithi064.stdout: flexiblas-netlib-3.0.4-8.el9.x86_64 2024-09-21T09:21:07.784 INFO:teuthology.orchestra.run.smithi064.stdout: flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 2024-09-21T09:21:07.784 INFO:teuthology.orchestra.run.smithi064.stdout: fuse-2.9.9-16.el9.x86_64 2024-09-21T09:21:07.784 INFO:teuthology.orchestra.run.smithi064.stdout: gperftools-libs-2.9.1-3.el9.x86_64 2024-09-21T09:21:07.784 INFO:teuthology.orchestra.run.smithi064.stdout: libcephfs-devel-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:21:07.784 INFO:teuthology.orchestra.run.smithi064.stdout: libcephfs2-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:21:07.784 INFO:teuthology.orchestra.run.smithi064.stdout: libcephsqlite-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:21:07.784 INFO:teuthology.orchestra.run.smithi064.stdout: libgfortran-11.5.0-2.el9.x86_64 2024-09-21T09:21:07.784 INFO:teuthology.orchestra.run.smithi064.stdout: liboath-2.6.7-2.el9.x86_64 2024-09-21T09:21:07.784 INFO:teuthology.orchestra.run.smithi064.stdout: libpmemobj-1.12.1-1.el9.x86_64 2024-09-21T09:21:07.784 INFO:teuthology.orchestra.run.smithi064.stdout: libquadmath-11.5.0-2.el9.x86_64 2024-09-21T09:21:07.785 INFO:teuthology.orchestra.run.smithi064.stdout: librabbitmq-0.11.0-7.el9.x86_64 2024-09-21T09:21:07.785 INFO:teuthology.orchestra.run.smithi064.stdout: librados-devel-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:21:07.785 INFO:teuthology.orchestra.run.smithi064.stdout: libradosstriper1-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:21:07.785 INFO:teuthology.orchestra.run.smithi064.stdout: librdkafka-1.6.1-102.el9.x86_64 2024-09-21T09:21:07.785 INFO:teuthology.orchestra.run.smithi064.stdout: librgw2-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:21:07.785 INFO:teuthology.orchestra.run.smithi064.stdout: libunwind-1.6.2-1.el9.x86_64 2024-09-21T09:21:07.786 INFO:teuthology.orchestra.run.smithi064.stdout: lttng-ust-2.12.0-6.el9.x86_64 2024-09-21T09:21:07.786 INFO:teuthology.orchestra.run.smithi064.stdout: openblas-0.3.26-2.el9.x86_64 2024-09-21T09:21:07.786 INFO:teuthology.orchestra.run.smithi064.stdout: openblas-openmp-0.3.26-2.el9.x86_64 2024-09-21T09:21:07.786 INFO:teuthology.orchestra.run.smithi064.stdout: python3-asyncssh-2.13.2-5.el9.noarch 2024-09-21T09:21:07.786 INFO:teuthology.orchestra.run.smithi064.stdout: python3-attrs-20.3.0-7.el9.noarch 2024-09-21T09:21:07.786 INFO:teuthology.orchestra.run.smithi064.stdout: python3-babel-2.9.1-2.el9.noarch 2024-09-21T09:21:07.786 INFO:teuthology.orchestra.run.smithi064.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2024-09-21T09:21:07.787 INFO:teuthology.orchestra.run.smithi064.stdout: python3-cachetools-4.2.4-1.el9.noarch 2024-09-21T09:21:07.787 INFO:teuthology.orchestra.run.smithi064.stdout: python3-ceph-argparse-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:21:07.787 INFO:teuthology.orchestra.run.smithi064.stdout: python3-ceph-common-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:21:07.787 INFO:teuthology.orchestra.run.smithi064.stdout: python3-cephfs-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:21:07.787 INFO:teuthology.orchestra.run.smithi064.stdout: python3-certifi-2023.05.07-4.el9.noarch 2024-09-21T09:21:07.787 INFO:teuthology.orchestra.run.smithi064.stdout: python3-cffi-1.14.5-5.el9.x86_64 2024-09-21T09:21:07.787 INFO:teuthology.orchestra.run.smithi064.stdout: python3-cheroot-8.6.0-4.el9.noarch 2024-09-21T09:21:07.787 INFO:teuthology.orchestra.run.smithi064.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2024-09-21T09:21:07.788 INFO:teuthology.orchestra.run.smithi064.stdout: python3-cryptography-36.0.1-4.el9.x86_64 2024-09-21T09:21:07.788 INFO:teuthology.orchestra.run.smithi064.stdout: python3-devel-3.9.19-8.el9.x86_64 2024-09-21T09:21:07.788 INFO:teuthology.orchestra.run.smithi064.stdout: python3-google-auth-1:2.34.0-1.el9.noarch 2024-09-21T09:21:07.788 INFO:teuthology.orchestra.run.smithi064.stdout: python3-iniconfig-1.1.1-7.el9.noarch 2024-09-21T09:21:07.788 INFO:teuthology.orchestra.run.smithi064.stdout: python3-jaraco-8.2.1-3.el9.noarch 2024-09-21T09:21:07.788 INFO:teuthology.orchestra.run.smithi064.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2024-09-21T09:21:07.788 INFO:teuthology.orchestra.run.smithi064.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2024-09-21T09:21:07.788 INFO:teuthology.orchestra.run.smithi064.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2024-09-21T09:21:07.789 INFO:teuthology.orchestra.run.smithi064.stdout: python3-jaraco-text-3.2.0-6.el9.noarch 2024-09-21T09:21:07.789 INFO:teuthology.orchestra.run.smithi064.stdout: python3-jinja2-2.11.3-6.el9.noarch 2024-09-21T09:21:07.789 INFO:teuthology.orchestra.run.smithi064.stdout: python3-jsonpatch-1.21-16.el9.noarch 2024-09-21T09:21:07.789 INFO:teuthology.orchestra.run.smithi064.stdout: python3-jsonpointer-2.0-4.el9.noarch 2024-09-21T09:21:07.789 INFO:teuthology.orchestra.run.smithi064.stdout: python3-jwt-2.4.0-1.el9.noarch 2024-09-21T09:21:07.789 INFO:teuthology.orchestra.run.smithi064.stdout: python3-jwt+crypto-2.4.0-1.el9.noarch 2024-09-21T09:21:07.790 INFO:teuthology.orchestra.run.smithi064.stdout: python3-kubernetes-1:26.1.0-2.el9.noarch 2024-09-21T09:21:07.790 INFO:teuthology.orchestra.run.smithi064.stdout: python3-logutils-0.3.5-21.el9.noarch 2024-09-21T09:21:07.790 INFO:teuthology.orchestra.run.smithi064.stdout: python3-mako-1.1.4-6.el9.noarch 2024-09-21T09:21:07.790 INFO:teuthology.orchestra.run.smithi064.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2024-09-21T09:21:07.790 INFO:teuthology.orchestra.run.smithi064.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2024-09-21T09:21:07.790 INFO:teuthology.orchestra.run.smithi064.stdout: python3-natsort-7.1.1-5.el9.noarch 2024-09-21T09:21:07.790 INFO:teuthology.orchestra.run.smithi064.stdout: python3-numpy-1:1.20.1-5.el9.x86_64 2024-09-21T09:21:07.791 INFO:teuthology.orchestra.run.smithi064.stdout: python3-numpy-f2py-1:1.20.1-5.el9.x86_64 2024-09-21T09:21:07.791 INFO:teuthology.orchestra.run.smithi064.stdout: python3-oauthlib-3.1.1-5.el9.noarch 2024-09-21T09:21:07.791 INFO:teuthology.orchestra.run.smithi064.stdout: python3-packaging-20.9-5.el9.noarch 2024-09-21T09:21:07.791 INFO:teuthology.orchestra.run.smithi064.stdout: python3-pecan-1.4.2-3.el9.noarch 2024-09-21T09:21:07.791 INFO:teuthology.orchestra.run.smithi064.stdout: python3-pluggy-0.13.1-7.el9.noarch 2024-09-21T09:21:07.792 INFO:teuthology.orchestra.run.smithi064.stdout: python3-ply-3.11-14.el9.noarch 2024-09-21T09:21:07.792 INFO:teuthology.orchestra.run.smithi064.stdout: python3-portend-3.1.0-2.el9.noarch 2024-09-21T09:21:07.792 INFO:teuthology.orchestra.run.smithi064.stdout: python3-prettytable-0.7.2-27.el9.noarch 2024-09-21T09:21:07.792 INFO:teuthology.orchestra.run.smithi064.stdout: python3-py-1.10.0-6.el9.noarch 2024-09-21T09:21:07.792 INFO:teuthology.orchestra.run.smithi064.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2024-09-21T09:21:07.792 INFO:teuthology.orchestra.run.smithi064.stdout: python3-pyasn1-0.4.8-6.el9.noarch 2024-09-21T09:21:07.792 INFO:teuthology.orchestra.run.smithi064.stdout: python3-pyasn1-modules-0.4.8-6.el9.noarch 2024-09-21T09:21:07.792 INFO:teuthology.orchestra.run.smithi064.stdout: python3-pycparser-2.20-6.el9.noarch 2024-09-21T09:21:07.792 INFO:teuthology.orchestra.run.smithi064.stdout: python3-pytest-6.2.2-7.el9.noarch 2024-09-21T09:21:07.792 INFO:teuthology.orchestra.run.smithi064.stdout: python3-pytz-2021.1-5.el9.noarch 2024-09-21T09:21:07.793 INFO:teuthology.orchestra.run.smithi064.stdout: python3-rados-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:21:07.793 INFO:teuthology.orchestra.run.smithi064.stdout: python3-rbd-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:21:07.793 INFO:teuthology.orchestra.run.smithi064.stdout: python3-repoze-lru-0.7-16.el9.noarch 2024-09-21T09:21:07.793 INFO:teuthology.orchestra.run.smithi064.stdout: python3-requests-2.25.1-8.el9.noarch 2024-09-21T09:21:07.793 INFO:teuthology.orchestra.run.smithi064.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2024-09-21T09:21:07.793 INFO:teuthology.orchestra.run.smithi064.stdout: python3-rgw-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:21:07.793 INFO:teuthology.orchestra.run.smithi064.stdout: python3-routes-2.5.1-5.el9.noarch 2024-09-21T09:21:07.793 INFO:teuthology.orchestra.run.smithi064.stdout: python3-rsa-4.9-2.el9.noarch 2024-09-21T09:21:07.793 INFO:teuthology.orchestra.run.smithi064.stdout: python3-scipy-1.6.2-8.el9.x86_64 2024-09-21T09:21:07.793 INFO:teuthology.orchestra.run.smithi064.stdout: python3-tempora-5.0.0-2.el9.noarch 2024-09-21T09:21:07.793 INFO:teuthology.orchestra.run.smithi064.stdout: python3-toml-0.10.2-6.el9.noarch 2024-09-21T09:21:07.793 INFO:teuthology.orchestra.run.smithi064.stdout: python3-typing-extensions-4.12.2-2.el9.noarch 2024-09-21T09:21:07.794 INFO:teuthology.orchestra.run.smithi064.stdout: python3-urllib3-1.26.5-6.el9.noarch 2024-09-21T09:21:07.794 INFO:teuthology.orchestra.run.smithi064.stdout: python3-webob-1.8.8-2.el9.noarch 2024-09-21T09:21:07.794 INFO:teuthology.orchestra.run.smithi064.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2024-09-21T09:21:07.794 INFO:teuthology.orchestra.run.smithi064.stdout: python3-werkzeug-2.0.3-3.el9.1.noarch 2024-09-21T09:21:07.794 INFO:teuthology.orchestra.run.smithi064.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2024-09-21T09:21:07.794 INFO:teuthology.orchestra.run.smithi064.stdout: rbd-fuse-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:21:07.794 INFO:teuthology.orchestra.run.smithi064.stdout: rbd-mirror-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:21:07.794 INFO:teuthology.orchestra.run.smithi064.stdout: rbd-nbd-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:21:07.794 INFO:teuthology.orchestra.run.smithi064.stdout: socat-1.7.4.1-6.el9.x86_64 2024-09-21T09:21:07.794 INFO:teuthology.orchestra.run.smithi064.stdout: thrift-0.15.0-2.el9.x86_64 2024-09-21T09:21:07.795 INFO:teuthology.orchestra.run.smithi064.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2024-09-21T09:21:07.795 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-09-21T09:21:07.795 INFO:teuthology.orchestra.run.smithi064.stdout:Complete! 2024-09-21T09:21:08.105 DEBUG:teuthology.parallel:result is None 2024-09-21T09:21:08.105 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-21T09:21:08.106 INFO:teuthology.packaging:ref: None 2024-09-21T09:21:08.106 INFO:teuthology.packaging:tag: None 2024-09-21T09:21:08.106 INFO:teuthology.packaging:branch: quincy 2024-09-21T09:21:08.106 INFO:teuthology.packaging:sha1: 62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:21:08.106 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=quincy 2024-09-21T09:21:08.344 DEBUG:teuthology.orchestra.run.smithi022:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2024-09-21T09:21:08.371 INFO:teuthology.orchestra.run.smithi022.stdout:17.2.7-1649.gfbd1a6ab.el9 2024-09-21T09:21:08.372 INFO:teuthology.packaging:The installed version of ceph is 17.2.7-1649.gfbd1a6ab.el9 2024-09-21T09:21:08.372 INFO:teuthology.task.install:The correct ceph version 17.2.7-1649.gfbd1a6ab is installed. 2024-09-21T09:21:08.374 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-21T09:21:08.374 INFO:teuthology.packaging:ref: None 2024-09-21T09:21:08.374 INFO:teuthology.packaging:tag: None 2024-09-21T09:21:08.374 INFO:teuthology.packaging:branch: quincy 2024-09-21T09:21:08.374 INFO:teuthology.packaging:sha1: 62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:21:08.375 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=quincy 2024-09-21T09:21:08.570 DEBUG:teuthology.orchestra.run.smithi064:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2024-09-21T09:21:08.595 INFO:teuthology.orchestra.run.smithi064.stdout:17.2.7-1649.gfbd1a6ab.el9 2024-09-21T09:21:08.595 INFO:teuthology.packaging:The installed version of ceph is 17.2.7-1649.gfbd1a6ab.el9 2024-09-21T09:21:08.595 INFO:teuthology.task.install:The correct ceph version 17.2.7-1649.gfbd1a6ab is installed. 2024-09-21T09:21:08.597 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2024-09-21T09:21:08.597 DEBUG:teuthology.orchestra.run.smithi022:> set -ex 2024-09-21T09:21:08.597 DEBUG:teuthology.orchestra.run.smithi022:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-09-21T09:21:08.631 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-09-21T09:21:08.632 DEBUG:teuthology.orchestra.run.smithi064:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-09-21T09:21:08.666 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2024-09-21T09:21:08.666 DEBUG:teuthology.orchestra.run.smithi022:> set -ex 2024-09-21T09:21:08.666 DEBUG:teuthology.orchestra.run.smithi022:> sudo dd of=/usr/bin/daemon-helper 2024-09-21T09:21:08.701 DEBUG:teuthology.orchestra.run.smithi022:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-09-21T09:21:08.770 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-09-21T09:21:08.770 DEBUG:teuthology.orchestra.run.smithi064:> sudo dd of=/usr/bin/daemon-helper 2024-09-21T09:21:08.798 DEBUG:teuthology.orchestra.run.smithi064:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-09-21T09:21:08.865 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2024-09-21T09:21:08.865 DEBUG:teuthology.orchestra.run.smithi022:> set -ex 2024-09-21T09:21:08.866 DEBUG:teuthology.orchestra.run.smithi022:> sudo dd of=/usr/bin/adjust-ulimits 2024-09-21T09:21:08.894 DEBUG:teuthology.orchestra.run.smithi022:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-09-21T09:21:08.964 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-09-21T09:21:08.964 DEBUG:teuthology.orchestra.run.smithi064:> sudo dd of=/usr/bin/adjust-ulimits 2024-09-21T09:21:08.992 DEBUG:teuthology.orchestra.run.smithi064:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-09-21T09:21:09.059 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2024-09-21T09:21:09.059 DEBUG:teuthology.orchestra.run.smithi022:> set -ex 2024-09-21T09:21:09.059 DEBUG:teuthology.orchestra.run.smithi022:> sudo dd of=/usr/bin/stdin-killer 2024-09-21T09:21:09.087 DEBUG:teuthology.orchestra.run.smithi022:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-09-21T09:21:09.155 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-09-21T09:21:09.155 DEBUG:teuthology.orchestra.run.smithi064:> sudo dd of=/usr/bin/stdin-killer 2024-09-21T09:21:09.182 DEBUG:teuthology.orchestra.run.smithi064:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-09-21T09:21:09.249 INFO:teuthology.run_tasks:Running task print... 2024-09-21T09:21:09.258 INFO:teuthology.task.print:**** done install task... 2024-09-21T09:21:09.258 INFO:teuthology.run_tasks:Running task print... 2024-09-21T09:21:09.278 INFO:teuthology.task.print:**** done start installing quincy cephadm ... 2024-09-21T09:21:09.278 INFO:teuthology.run_tasks:Running task cephadm... 2024-09-21T09:21:09.381 INFO:tasks.cephadm:Config: {'cephadm_branch': 'quincy', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'conf': {'osd': {'osd_class_default_list': '*', 'osd_class_load_list': '*', 'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}, 'global': {'mon election default strategy': 1}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}}, 'image': 'quay.ceph.io/ceph-ci/ceph:quincy', 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(POOL_APP_NOT_ENABLED\\)', 'OSD_DOWN', 'mons down', 'mon down', 'MON_DOWN', 'out of quorum', 'PG_DEGRADED', 'Reduced data availability', 'Degraded data redundancy', 'OSDMAP_FLAGS', 'mons down', 'mon down', 'MON_DOWN', 'out of quorum', 'PG_AVAILABILITY'], 'sha1': '62da54b4b83a96ca9976b7e1d0c8272a564a2208'} 2024-09-21T09:21:09.381 INFO:tasks.cephadm:Cluster image is quay.ceph.io/ceph-ci/ceph:quincy 2024-09-21T09:21:09.382 INFO:tasks.cephadm:Cluster fsid is d65e801e-77fa-11ef-baf6-efdc52797490 2024-09-21T09:21:09.382 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-09-21T09:21:09.382 INFO:tasks.cephadm:Monitor IPs: {'mon.a': '172.21.15.22', 'mon.c': '[v2:172.21.15.22:3301,v1:172.21.15.22:6790]', 'mon.b': '172.21.15.64'} 2024-09-21T09:21:09.382 INFO:tasks.cephadm:First mon is mon.a on smithi022 2024-09-21T09:21:09.382 INFO:tasks.cephadm:First mgr is y 2024-09-21T09:21:09.382 INFO:tasks.cephadm:Normalizing hostnames... 2024-09-21T09:21:09.383 DEBUG:teuthology.orchestra.run.smithi022:> sudo hostname $(hostname -s) 2024-09-21T09:21:09.412 DEBUG:teuthology.orchestra.run.smithi064:> sudo hostname $(hostname -s) 2024-09-21T09:21:09.440 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref quincy)... 2024-09-21T09:21:09.440 DEBUG:teuthology.orchestra.run.smithi022:> curl --silent https://raw.githubusercontent.com/ceph/ceph/quincy/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-09-21T09:21:09.585 INFO:teuthology.orchestra.run.smithi022.stdout:-rw-r--r--. 1 ubuntu ubuntu 378585 Sep 21 09:21 /home/ubuntu/cephtest/cephadm 2024-09-21T09:21:09.585 DEBUG:teuthology.orchestra.run.smithi064:> curl --silent https://raw.githubusercontent.com/ceph/ceph/quincy/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-09-21T09:21:09.719 INFO:teuthology.orchestra.run.smithi064.stdout:-rw-r--r--. 1 ubuntu ubuntu 378585 Sep 21 09:21 /home/ubuntu/cephtest/cephadm 2024-09-21T09:21:09.719 DEBUG:teuthology.orchestra.run.smithi022:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-09-21T09:21:09.739 DEBUG:teuthology.orchestra.run.smithi064:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-09-21T09:21:09.767 INFO:tasks.cephadm:Pulling image quay.ceph.io/ceph-ci/ceph:quincy on all hosts... 2024-09-21T09:21:09.767 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy pull 2024-09-21T09:21:09.782 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy pull 2024-09-21T09:21:11.391 INFO:teuthology.orchestra.run.smithi064.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2024-09-21T09:21:11.393 INFO:teuthology.orchestra.run.smithi022.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2024-09-21T09:21:37.507 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-09-21T09:21:37.507 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph_version": "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)", 2024-09-21T09:21:37.508 INFO:teuthology.orchestra.run.smithi022.stdout: "image_id": "fc3ec4a4f6be575045de0fd8a2fe84939e888152c447bdedb90add7841f7bef9", 2024-09-21T09:21:37.508 INFO:teuthology.orchestra.run.smithi022.stdout: "repo_digests": [ 2024-09-21T09:21:37.508 INFO:teuthology.orchestra.run.smithi022.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2" 2024-09-21T09:21:37.508 INFO:teuthology.orchestra.run.smithi022.stdout: ] 2024-09-21T09:21:37.508 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-09-21T09:21:44.298 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-09-21T09:21:44.298 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph_version": "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)", 2024-09-21T09:21:44.298 INFO:teuthology.orchestra.run.smithi064.stdout: "image_id": "fc3ec4a4f6be575045de0fd8a2fe84939e888152c447bdedb90add7841f7bef9", 2024-09-21T09:21:44.298 INFO:teuthology.orchestra.run.smithi064.stdout: "repo_digests": [ 2024-09-21T09:21:44.298 INFO:teuthology.orchestra.run.smithi064.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2" 2024-09-21T09:21:44.298 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-09-21T09:21:44.298 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-09-21T09:21:44.321 DEBUG:teuthology.orchestra.run.smithi022:> sudo mkdir -p /etc/ceph 2024-09-21T09:21:44.356 DEBUG:teuthology.orchestra.run.smithi064:> sudo mkdir -p /etc/ceph 2024-09-21T09:21:44.387 DEBUG:teuthology.orchestra.run.smithi022:> sudo chmod 777 /etc/ceph 2024-09-21T09:21:44.425 DEBUG:teuthology.orchestra.run.smithi064:> sudo chmod 777 /etc/ceph 2024-09-21T09:21:44.455 INFO:tasks.cephadm:Writing seed config... 2024-09-21T09:21:44.456 INFO:tasks.cephadm: override: [osd] osd_class_default_list = * 2024-09-21T09:21:44.456 INFO:tasks.cephadm: override: [osd] osd_class_load_list = * 2024-09-21T09:21:44.456 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-09-21T09:21:44.456 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-09-21T09:21:44.456 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2024-09-21T09:21:44.456 INFO:tasks.cephadm: override: [global] mon election default strategy = 1 2024-09-21T09:21:44.456 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-09-21T09:21:44.456 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-09-21T09:21:44.456 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-09-21T09:21:44.456 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-09-21T09:21:44.457 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-09-21T09:21:44.457 DEBUG:teuthology.orchestra.run.smithi022:> set -ex 2024-09-21T09:21:44.457 DEBUG:teuthology.orchestra.run.smithi022:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-09-21T09:21:44.483 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log to file = true mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = d65e801e-77fa-11ef-baf6-efdc52797490 mon election default strategy = 1 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true osd_class_default_list = * osd_class_load_list = * debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2024-09-21T09:21:44.483 DEBUG:teuthology.orchestra.run.smithi022:mon.a> sudo journalctl -f -n 0 -u ceph-d65e801e-77fa-11ef-baf6-efdc52797490@mon.a.service 2024-09-21T09:21:44.526 DEBUG:teuthology.orchestra.run.smithi022:mgr.y> sudo journalctl -f -n 0 -u ceph-d65e801e-77fa-11ef-baf6-efdc52797490@mgr.y.service 2024-09-21T09:21:44.569 INFO:tasks.cephadm:Bootstrapping... 2024-09-21T09:21:44.569 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy -v bootstrap --fsid d65e801e-77fa-11ef-baf6-efdc52797490 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-id a --mgr-id y --orphan-initial-daemons --skip-monitoring-stack --mon-ip 172.21.15.22 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-09-21T09:21:44.779 INFO:teuthology.orchestra.run.smithi022.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:21:44.779 INFO:teuthology.orchestra.run.smithi022.stdout:cephadm ['--image', 'quay.ceph.io/ceph-ci/ceph:quincy', '-v', 'bootstrap', '--fsid', 'd65e801e-77fa-11ef-baf6-efdc52797490', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-id', 'a', '--mgr-id', 'y', '--orphan-initial-daemons', '--skip-monitoring-stack', '--mon-ip', '172.21.15.22', '--skip-admin-label'] 2024-09-21T09:21:44.810 INFO:teuthology.orchestra.run.smithi022.stdout:/bin/podman: stdout 5.2.2 2024-09-21T09:21:44.811 INFO:teuthology.orchestra.run.smithi022.stderr:Specifying an fsid for your cluster offers no advantages and may increase the likelihood of fsid conflicts. 2024-09-21T09:21:44.811 INFO:teuthology.orchestra.run.smithi022.stdout:Verifying podman|docker is present... 2024-09-21T09:21:44.834 INFO:teuthology.orchestra.run.smithi022.stdout:/bin/podman: stdout 5.2.2 2024-09-21T09:21:44.834 INFO:teuthology.orchestra.run.smithi022.stdout:Verifying lvm2 is present... 2024-09-21T09:21:44.834 INFO:teuthology.orchestra.run.smithi022.stdout:Verifying time synchronization is in place... 2024-09-21T09:21:44.842 INFO:teuthology.orchestra.run.smithi022.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2024-09-21T09:21:44.842 INFO:teuthology.orchestra.run.smithi022.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2024-09-21T09:21:44.849 INFO:teuthology.orchestra.run.smithi022.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2024-09-21T09:21:44.849 INFO:teuthology.orchestra.run.smithi022.stdout:systemctl: stdout inactive 2024-09-21T09:21:44.856 INFO:teuthology.orchestra.run.smithi022.stdout:systemctl: stdout enabled 2024-09-21T09:21:44.863 INFO:teuthology.orchestra.run.smithi022.stdout:systemctl: stdout active 2024-09-21T09:21:44.863 INFO:teuthology.orchestra.run.smithi022.stdout:Unit chronyd.service is enabled and running 2024-09-21T09:21:44.864 INFO:teuthology.orchestra.run.smithi022.stdout:Repeating the final host check... 2024-09-21T09:21:44.887 INFO:teuthology.orchestra.run.smithi022.stdout:/bin/podman: stdout 5.2.2 2024-09-21T09:21:44.887 INFO:teuthology.orchestra.run.smithi022.stdout:podman (/bin/podman) version 5.2.2 is present 2024-09-21T09:21:44.887 INFO:teuthology.orchestra.run.smithi022.stdout:systemctl is present 2024-09-21T09:21:44.887 INFO:teuthology.orchestra.run.smithi022.stdout:lvcreate is present 2024-09-21T09:21:44.894 INFO:teuthology.orchestra.run.smithi022.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2024-09-21T09:21:44.894 INFO:teuthology.orchestra.run.smithi022.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2024-09-21T09:21:44.901 INFO:teuthology.orchestra.run.smithi022.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2024-09-21T09:21:44.901 INFO:teuthology.orchestra.run.smithi022.stdout:systemctl: stdout inactive 2024-09-21T09:21:44.908 INFO:teuthology.orchestra.run.smithi022.stdout:systemctl: stdout enabled 2024-09-21T09:21:44.915 INFO:teuthology.orchestra.run.smithi022.stdout:systemctl: stdout active 2024-09-21T09:21:44.915 INFO:teuthology.orchestra.run.smithi022.stdout:Unit chronyd.service is enabled and running 2024-09-21T09:21:44.915 INFO:teuthology.orchestra.run.smithi022.stdout:Host looks OK 2024-09-21T09:21:44.915 INFO:teuthology.orchestra.run.smithi022.stdout:Cluster fsid: d65e801e-77fa-11ef-baf6-efdc52797490 2024-09-21T09:21:44.915 INFO:teuthology.orchestra.run.smithi022.stdout:Acquiring lock 140124823630752 on /run/cephadm/d65e801e-77fa-11ef-baf6-efdc52797490.lock 2024-09-21T09:21:44.915 INFO:teuthology.orchestra.run.smithi022.stdout:Lock 140124823630752 acquired on /run/cephadm/d65e801e-77fa-11ef-baf6-efdc52797490.lock 2024-09-21T09:21:44.916 INFO:teuthology.orchestra.run.smithi022.stdout:Verifying IP 172.21.15.22 port 3300 ... 2024-09-21T09:21:44.916 INFO:teuthology.orchestra.run.smithi022.stdout:Verifying IP 172.21.15.22 port 6789 ... 2024-09-21T09:21:44.916 INFO:teuthology.orchestra.run.smithi022.stdout:Base mon IP(s) is [172.21.15.22:3300, 172.21.15.22:6789], mon addrv is [v2:172.21.15.22:3300,v1:172.21.15.22:6789] 2024-09-21T09:21:44.920 INFO:teuthology.orchestra.run.smithi022.stdout:/sbin/ip: stdout default via 172.21.15.254 dev ens1f0 proto dhcp src 172.21.15.22 metric 100 2024-09-21T09:21:44.920 INFO:teuthology.orchestra.run.smithi022.stdout:/sbin/ip: stdout 172.21.0.0/20 dev ens1f0 proto kernel scope link src 172.21.15.22 metric 100 2024-09-21T09:21:44.923 INFO:teuthology.orchestra.run.smithi022.stdout:/sbin/ip: stdout ::1 dev lo proto kernel metric 256 pref medium 2024-09-21T09:21:44.923 INFO:teuthology.orchestra.run.smithi022.stdout:/sbin/ip: stdout fe80::/64 dev ens1f0 proto kernel metric 256 pref medium 2024-09-21T09:21:44.923 INFO:teuthology.orchestra.run.smithi022.stdout:/sbin/ip: stdout default via fe80::327c:5e00:6487:71e0 dev ens1f0 proto ra metric 1024 expires 1779sec hoplimit 64 pref medium 2024-09-21T09:21:44.925 INFO:teuthology.orchestra.run.smithi022.stdout:/sbin/ip: stdout 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-09-21T09:21:44.925 INFO:teuthology.orchestra.run.smithi022.stdout:/sbin/ip: stdout inet6 ::1/128 scope host 2024-09-21T09:21:44.925 INFO:teuthology.orchestra.run.smithi022.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2024-09-21T09:21:44.926 INFO:teuthology.orchestra.run.smithi022.stdout:/sbin/ip: stdout 4: ens1f0: mtu 1500 state UP qlen 1000 2024-09-21T09:21:44.926 INFO:teuthology.orchestra.run.smithi022.stdout:/sbin/ip: stdout inet6 fe80::ec4:7aff:fe59:5de4/64 scope link 2024-09-21T09:21:44.926 INFO:teuthology.orchestra.run.smithi022.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2024-09-21T09:21:44.926 INFO:teuthology.orchestra.run.smithi022.stdout:Mon IP `172.21.15.22` is in CIDR network `172.21.0.0/20` 2024-09-21T09:21:44.926 INFO:teuthology.orchestra.run.smithi022.stdout:Mon IP `172.21.15.22` is in CIDR network `172.21.0.0/20` 2024-09-21T09:21:44.926 INFO:teuthology.orchestra.run.smithi022.stdout:Inferred mon public CIDR from local network configuration ['172.21.0.0/20', '172.21.0.0/20'] 2024-09-21T09:21:44.927 INFO:teuthology.orchestra.run.smithi022.stdout:Internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-09-21T09:21:44.927 INFO:teuthology.orchestra.run.smithi022.stdout:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2024-09-21T09:21:45.431 INFO:teuthology.orchestra.run.smithi022.stdout:/bin/podman: stdout fc3ec4a4f6be575045de0fd8a2fe84939e888152c447bdedb90add7841f7bef9 2024-09-21T09:21:45.431 INFO:teuthology.orchestra.run.smithi022.stdout:/bin/podman: stderr Trying to pull quay.ceph.io/ceph-ci/ceph:quincy... 2024-09-21T09:21:45.431 INFO:teuthology.orchestra.run.smithi022.stdout:/bin/podman: stderr Getting image source signatures 2024-09-21T09:21:45.431 INFO:teuthology.orchestra.run.smithi022.stdout:/bin/podman: stderr Copying blob sha256:3ba00a516579693832719224d909a7cd5333511394d4f35f95e899e50f0bfaeb 2024-09-21T09:21:45.431 INFO:teuthology.orchestra.run.smithi022.stdout:/bin/podman: stderr Copying blob sha256:e8b54c863393b7a8216a71737771b73b16a8e43ec7acf927c7faa175c255e551 2024-09-21T09:21:45.431 INFO:teuthology.orchestra.run.smithi022.stdout:/bin/podman: stderr Copying config sha256:fc3ec4a4f6be575045de0fd8a2fe84939e888152c447bdedb90add7841f7bef9 2024-09-21T09:21:45.431 INFO:teuthology.orchestra.run.smithi022.stdout:/bin/podman: stderr Writing manifest to image destination 2024-09-21T09:21:46.344 INFO:teuthology.orchestra.run.smithi022.stdout:ceph: stdout ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable) 2024-09-21T09:21:46.344 INFO:teuthology.orchestra.run.smithi022.stdout:Ceph version: ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable) 2024-09-21T09:21:46.344 INFO:teuthology.orchestra.run.smithi022.stdout:Extracting ceph user uid/gid from container image... 2024-09-21T09:21:47.324 INFO:teuthology.orchestra.run.smithi022.stdout:stat: stdout 167 167 2024-09-21T09:21:47.325 INFO:teuthology.orchestra.run.smithi022.stdout:Creating initial keys... 2024-09-21T09:21:48.371 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph-authtool: stdout AQArkO5mwFcVKBAA7IQ3bD1b5K4tCesHSGzmhw== 2024-09-21T09:21:49.311 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph-authtool: stdout AQAskO5mQXptKxAAruWfQKR5HOXkX79Z/kP/pQ== 2024-09-21T09:21:58.485 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph-authtool: stdout AQAwkO5m0d0tBRAAm2h1AlLA0lShBwZaX+npSw== 2024-09-21T09:21:58.485 INFO:teuthology.orchestra.run.smithi022.stdout:Creating initial monmap... 2024-09-21T09:22:01.778 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: monmap file /tmp/monmap 2024-09-21T09:22:01.778 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/monmaptool: stdout setting min_mon_release = octopus 2024-09-21T09:22:01.779 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: set fsid to d65e801e-77fa-11ef-baf6-efdc52797490 2024-09-21T09:22:01.779 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-09-21T09:22:01.779 INFO:teuthology.orchestra.run.smithi022.stdout:monmaptool for a [v2:172.21.15.22:3300,v1:172.21.15.22:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-09-21T09:22:01.779 INFO:teuthology.orchestra.run.smithi022.stdout:setting min_mon_release = octopus 2024-09-21T09:22:01.779 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/monmaptool: set fsid to d65e801e-77fa-11ef-baf6-efdc52797490 2024-09-21T09:22:01.779 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-09-21T09:22:01.779 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-09-21T09:22:01.779 INFO:teuthology.orchestra.run.smithi022.stdout:Creating mon... 2024-09-21T09:22:02.683 INFO:teuthology.orchestra.run.smithi022.stdout:create mon.a on 2024-09-21T09:22:02.878 INFO:teuthology.orchestra.run.smithi022.stdout:systemctl: stderr Removed "/etc/systemd/system/multi-user.target.wants/ceph.target". 2024-09-21T09:22:03.023 INFO:teuthology.orchestra.run.smithi022.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2024-09-21T09:22:03.179 INFO:teuthology.orchestra.run.smithi022.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph-d65e801e-77fa-11ef-baf6-efdc52797490.target → /etc/systemd/system/ceph-d65e801e-77fa-11ef-baf6-efdc52797490.target. 2024-09-21T09:22:03.179 INFO:teuthology.orchestra.run.smithi022.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph.target.wants/ceph-d65e801e-77fa-11ef-baf6-efdc52797490.target → /etc/systemd/system/ceph-d65e801e-77fa-11ef-baf6-efdc52797490.target. 2024-09-21T09:22:03.358 INFO:teuthology.orchestra.run.smithi022.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-d65e801e-77fa-11ef-baf6-efdc52797490@mon.a 2024-09-21T09:22:03.358 INFO:teuthology.orchestra.run.smithi022.stdout:systemctl: stderr Failed to reset failed state of unit ceph-d65e801e-77fa-11ef-baf6-efdc52797490@mon.a.service: Unit ceph-d65e801e-77fa-11ef-baf6-efdc52797490@mon.a.service not loaded. 2024-09-21T09:22:03.518 INFO:teuthology.orchestra.run.smithi022.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-d65e801e-77fa-11ef-baf6-efdc52797490.target.wants/ceph-d65e801e-77fa-11ef-baf6-efdc52797490@mon.a.service → /etc/systemd/system/ceph-d65e801e-77fa-11ef-baf6-efdc52797490@.service. 2024-09-21T09:22:03.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:03 smithi022 podman[27257]: 2024-09-21 09:22:03.664623775 +0000 UTC m=+0.016643610 image pull fc3ec4a4f6be575045de0fd8a2fe84939e888152c447bdedb90add7841f7bef9 quay.ceph.io/ceph-ci/ceph:quincy 2024-09-21T09:22:04.478 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:04 smithi022 podman[27257]: 2024-09-21 09:22:04.222540581 +0000 UTC m=+0.574560444 container create 5968f1e67cf8d8e029a773dd84fc3f042d7ef9844875dc7e7b43ca68ce3030cc (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mon-a, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=quincy-fbd1a6a, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-09-21T09:22:04.479 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:04 smithi022 podman[27257]: 2024-09-21 09:22:04.432943081 +0000 UTC m=+0.784962910 container init 5968f1e67cf8d8e029a773dd84fc3f042d7ef9844875dc7e7b43ca68ce3030cc (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mon-a, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, ceph=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, RELEASE=quincy-fbd1a6a) 2024-09-21T09:22:04.479 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:04 smithi022 podman[27257]: 2024-09-21 09:22:04.438024658 +0000 UTC m=+0.790044479 container start 5968f1e67cf8d8e029a773dd84fc3f042d7ef9844875dc7e7b43ca68ce3030cc (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mon-a, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=quincy-fbd1a6a) 2024-09-21T09:22:04.558 INFO:teuthology.orchestra.run.smithi022.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-09-21T09:22:04.558 INFO:teuthology.orchestra.run.smithi022.stdout:systemctl: stdout disabled 2024-09-21T09:22:04.565 INFO:teuthology.orchestra.run.smithi022.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-09-21T09:22:04.565 INFO:teuthology.orchestra.run.smithi022.stdout:systemctl: stdout inactive 2024-09-21T09:22:04.566 INFO:teuthology.orchestra.run.smithi022.stdout:firewalld.service is not enabled 2024-09-21T09:22:04.566 INFO:teuthology.orchestra.run.smithi022.stdout:Not possible to enable service . firewalld.service is not available 2024-09-21T09:22:04.566 INFO:teuthology.orchestra.run.smithi022.stdout:Waiting for mon to start... 2024-09-21T09:22:04.566 INFO:teuthology.orchestra.run.smithi022.stdout:Waiting for mon... 2024-09-21T09:22:04.770 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:04 smithi022 ceph-mon[27295]: mkfs d65e801e-77fa-11ef-baf6-efdc52797490 2024-09-21T09:22:04.770 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:04 smithi022 ceph-mon[27295]: mon.a is new leader, mons a in quorum (ranks 0) 2024-09-21T09:22:04.770 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:04 smithi022 bash[27257]: 5968f1e67cf8d8e029a773dd84fc3f042d7ef9844875dc7e7b43ca68ce3030cc 2024-09-21T09:22:04.770 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:04 smithi022 systemd[1]: Started Ceph mon.a for d65e801e-77fa-11ef-baf6-efdc52797490. 2024-09-21T09:22:05.624 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:05 smithi022 ceph-mon[27295]: mon.a is new leader, mons a in quorum (ranks 0) 2024-09-21T09:22:05.624 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:05 smithi022 ceph-mon[27295]: monmap e1: 1 mons at {a=[v2:172.21.15.22:3300/0,v1:172.21.15.22:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-21T09:22:05.624 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:05 smithi022 ceph-mon[27295]: fsmap 2024-09-21T09:22:05.624 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:05 smithi022 ceph-mon[27295]: osdmap e1: 0 total, 0 up, 0 in 2024-09-21T09:22:05.624 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:05 smithi022 ceph-mon[27295]: mgrmap e1: no daemons active 2024-09-21T09:22:05.624 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:05 smithi022 ceph-mon[27295]: from='client.? 172.21.15.22:0/3840679249' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-09-21T09:22:05.627 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout cluster: 2024-09-21T09:22:05.628 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout id: d65e801e-77fa-11ef-baf6-efdc52797490 2024-09-21T09:22:05.628 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout health: HEALTH_OK 2024-09-21T09:22:05.628 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout 2024-09-21T09:22:05.628 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout services: 2024-09-21T09:22:05.628 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout mon: 1 daemons, quorum a (age 0.58355s) 2024-09-21T09:22:05.628 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout mgr: no daemons active 2024-09-21T09:22:05.628 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout osd: 0 osds: 0 up, 0 in 2024-09-21T09:22:05.628 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout 2024-09-21T09:22:05.628 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout data: 2024-09-21T09:22:05.628 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout pools: 0 pools, 0 pgs 2024-09-21T09:22:05.628 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout objects: 0 objects, 0 B 2024-09-21T09:22:05.628 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout usage: 0 B used, 0 B / 0 B avail 2024-09-21T09:22:05.628 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout pgs: 2024-09-21T09:22:05.629 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout 2024-09-21T09:22:05.629 INFO:teuthology.orchestra.run.smithi022.stdout:mon is available 2024-09-21T09:22:05.629 INFO:teuthology.orchestra.run.smithi022.stdout:Assimilating anything we can from ceph.conf... 2024-09-21T09:22:06.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:06 smithi022 ceph-mon[27295]: from='client.? 172.21.15.22:0/603037409' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-09-21T09:22:06.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:06 smithi022 ceph-mon[27295]: from='client.? 172.21.15.22:0/603037409' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-09-21T09:22:07.614 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout 2024-09-21T09:22:07.615 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout [global] 2024-09-21T09:22:07.615 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout fsid = d65e801e-77fa-11ef-baf6-efdc52797490 2024-09-21T09:22:07.615 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout mon_cluster_log_file_level = debug 2024-09-21T09:22:07.615 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout mon_host = [v2:172.21.15.22:3300,v1:172.21.15.22:6789] 2024-09-21T09:22:07.615 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2024-09-21T09:22:07.615 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2024-09-21T09:22:07.615 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2024-09-21T09:22:07.615 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2024-09-21T09:22:07.615 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout 2024-09-21T09:22:07.615 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout [mgr] 2024-09-21T09:22:07.615 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2024-09-21T09:22:07.616 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout 2024-09-21T09:22:07.616 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout [osd] 2024-09-21T09:22:07.616 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2024-09-21T09:22:07.616 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2024-09-21T09:22:07.616 INFO:teuthology.orchestra.run.smithi022.stdout:Generating new minimal ceph.conf... 2024-09-21T09:22:08.307 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:08 smithi022 ceph-mon[27295]: from='client.? 172.21.15.22:0/2782661620' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:22:08.727 INFO:teuthology.orchestra.run.smithi022.stdout:Restarting the monitor... 2024-09-21T09:22:09.022 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:08 smithi022 systemd[1]: Stopping Ceph mon.a for d65e801e-77fa-11ef-baf6-efdc52797490... 2024-09-21T09:22:09.307 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:09 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mon-a[27272]: 2024-09-21T09:22:09.019+0000 7f9903fd3640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T09:22:09.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:09 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mon-a[27272]: 2024-09-21T09:22:09.019+0000 7f9903fd3640 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2024-09-21T09:22:09.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:09 smithi022 podman[27613]: 2024-09-21 09:22:09.132168207 +0000 UTC m=+0.236197541 container died 5968f1e67cf8d8e029a773dd84fc3f042d7ef9844875dc7e7b43ca68ce3030cc (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mon-a, org.label-schema.license=GPLv2, ceph=True, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=quincy-fbd1a6a) 2024-09-21T09:22:09.637 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:09 smithi022 podman[27613]: 2024-09-21 09:22:09.458432215 +0000 UTC m=+0.562461548 container cleanup 5968f1e67cf8d8e029a773dd84fc3f042d7ef9844875dc7e7b43ca68ce3030cc (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mon-a, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=quincy-fbd1a6a, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=) 2024-09-21T09:22:09.637 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:09 smithi022 bash[27613]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mon-a 2024-09-21T09:22:09.637 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:09 smithi022 podman[27630]: 2024-09-21 09:22:09.634522436 +0000 UTC m=+0.505283447 container remove 5968f1e67cf8d8e029a773dd84fc3f042d7ef9844875dc7e7b43ca68ce3030cc (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mon-a, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, ceph=True, GIT_CLEAN=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=quincy-fbd1a6a, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T09:22:10.221 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:09 smithi022 systemd[1]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490@mon.a.service: Deactivated successfully. 2024-09-21T09:22:10.222 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:09 smithi022 systemd[1]: Stopped Ceph mon.a for d65e801e-77fa-11ef-baf6-efdc52797490. 2024-09-21T09:22:10.222 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:09 smithi022 systemd[1]: Starting Ceph mon.a for d65e801e-77fa-11ef-baf6-efdc52797490... 2024-09-21T09:22:10.222 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 podman[27726]: 2024-09-21 09:22:10.006354503 +0000 UTC m=+0.016596142 image pull fc3ec4a4f6be575045de0fd8a2fe84939e888152c447bdedb90add7841f7bef9 quay.ceph.io/ceph-ci/ceph:quincy 2024-09-21T09:22:10.222 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 podman[27726]: 2024-09-21 09:22:10.110229769 +0000 UTC m=+0.120471439 container create 1e0a2d74e66ed0875bbbe448f9186ef3a086f3147ceee15bc593a101f91895e0 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mon-a, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.schema-version=1.0, RELEASE=quincy-fbd1a6a, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T09:22:10.359 INFO:teuthology.orchestra.run.smithi022.stdout:Setting public_network to 172.21.0.0/20 in mon config section 2024-09-21T09:22:10.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 podman[27726]: 2024-09-21 09:22:10.279165634 +0000 UTC m=+0.289407266 container init 1e0a2d74e66ed0875bbbe448f9186ef3a086f3147ceee15bc593a101f91895e0 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mon-a, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , RELEASE=quincy-fbd1a6a) 2024-09-21T09:22:10.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 podman[27726]: 2024-09-21 09:22:10.283907598 +0000 UTC m=+0.294149223 container start 1e0a2d74e66ed0875bbbe448f9186ef3a086f3147ceee15bc593a101f91895e0 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mon-a, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=quincy-fbd1a6a, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T09:22:10.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: set uid:gid to 167:167 (ceph:ceph) 2024-09-21T09:22:10.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable), process ceph-mon, pid 7 2024-09-21T09:22:10.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: pidfile_write: ignore empty --pid-file 2024-09-21T09:22:10.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: load: jerasure load: lrc 2024-09-21T09:22:10.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: RocksDB version: 6.15.5 2024-09-21T09:22:10.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-09-21T09:22:10.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Compile date Sep 20 2024 2024-09-21T09:22:10.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: DB SUMMARY 2024-09-21T09:22:10.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: DB Session ID: D5S5VXU896KYWAPI6FTP 2024-09-21T09:22:10.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: CURRENT file: CURRENT 2024-09-21T09:22:10.560 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: IDENTITY file: IDENTITY 2024-09-21T09:22:10.560 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: MANIFEST file: MANIFEST-000009 size: 131 Bytes 2024-09-21T09:22:10.560 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000008.sst 2024-09-21T09:22:10.560 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000010.log size: 83682 ; 2024-09-21T09:22:10.560 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.error_if_exists: 0 2024-09-21T09:22:10.560 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.create_if_missing: 0 2024-09-21T09:22:10.560 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.paranoid_checks: 1 2024-09-21T09:22:10.560 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-21T09:22:10.560 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.env: 0x5595e128d440 2024-09-21T09:22:10.560 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.fs: Posix File System 2024-09-21T09:22:10.560 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.info_log: 0x5595e3388ec0 2024-09-21T09:22:10.560 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.max_file_opening_threads: 16 2024-09-21T09:22:10.560 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.statistics: (nil) 2024-09-21T09:22:10.561 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.use_fsync: 0 2024-09-21T09:22:10.561 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.max_log_file_size: 0 2024-09-21T09:22:10.561 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-21T09:22:10.561 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-21T09:22:10.561 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.keep_log_file_num: 1000 2024-09-21T09:22:10.561 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.recycle_log_file_num: 0 2024-09-21T09:22:10.561 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.allow_fallocate: 1 2024-09-21T09:22:10.561 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.allow_mmap_reads: 0 2024-09-21T09:22:10.561 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.allow_mmap_writes: 0 2024-09-21T09:22:10.561 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.use_direct_reads: 0 2024-09-21T09:22:10.561 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-21T09:22:10.561 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.create_missing_column_families: 0 2024-09-21T09:22:10.562 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.db_log_dir: 2024-09-21T09:22:10.562 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2024-09-21T09:22:10.562 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-21T09:22:10.562 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-21T09:22:10.562 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-21T09:22:10.562 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-21T09:22:10.562 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-21T09:22:10.562 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-21T09:22:10.562 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.advise_random_on_open: 1 2024-09-21T09:22:10.562 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.db_write_buffer_size: 0 2024-09-21T09:22:10.562 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.write_buffer_manager: 0x5595e34467e0 2024-09-21T09:22:10.562 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-21T09:22:10.562 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-09-21T09:22:10.563 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-21T09:22:10.563 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-21T09:22:10.563 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.rate_limiter: (nil) 2024-09-21T09:22:10.563 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-21T09:22:10.563 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.wal_recovery_mode: 2 2024-09-21T09:22:10.563 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.enable_thread_tracking: 0 2024-09-21T09:22:10.563 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.enable_pipelined_write: 0 2024-09-21T09:22:10.563 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.unordered_write: 0 2024-09-21T09:22:10.563 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-21T09:22:10.563 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-21T09:22:10.563 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-21T09:22:10.563 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-21T09:22:10.563 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.row_cache: None 2024-09-21T09:22:10.563 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.wal_filter: None 2024-09-21T09:22:10.563 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-21T09:22:10.564 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.allow_ingest_behind: 0 2024-09-21T09:22:10.564 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.preserve_deletes: 0 2024-09-21T09:22:10.565 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.two_write_queues: 0 2024-09-21T09:22:10.565 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.manual_wal_flush: 0 2024-09-21T09:22:10.565 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.atomic_flush: 0 2024-09-21T09:22:10.565 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-21T09:22:10.565 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-21T09:22:10.565 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-21T09:22:10.565 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.log_readahead_size: 0 2024-09-21T09:22:10.565 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-21T09:22:10.565 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.best_efforts_recovery: 0 2024-09-21T09:22:10.565 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-21T09:22:10.565 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-21T09:22:10.565 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.allow_data_in_errors: 0 2024-09-21T09:22:10.566 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.db_host_id: __hostname__ 2024-09-21T09:22:10.566 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.max_background_jobs: 2 2024-09-21T09:22:10.566 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.max_background_compactions: -1 2024-09-21T09:22:10.566 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.max_subcompactions: 1 2024-09-21T09:22:10.566 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-21T09:22:10.566 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-21T09:22:10.566 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-21T09:22:10.566 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.max_total_wal_size: 0 2024-09-21T09:22:10.566 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-21T09:22:10.566 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-21T09:22:10.566 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-21T09:22:10.566 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-21T09:22:10.567 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.max_open_files: -1 2024-09-21T09:22:10.567 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.bytes_per_sync: 0 2024-09-21T09:22:10.567 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-21T09:22:10.567 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-21T09:22:10.567 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.compaction_readahead_size: 0 2024-09-21T09:22:10.567 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.max_background_flushes: -1 2024-09-21T09:22:10.567 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Compression algorithms supported: 2024-09-21T09:22:10.567 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: kZSTD supported: 0 2024-09-21T09:22:10.567 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: kXpressCompression supported: 0 2024-09-21T09:22:10.567 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: kBZip2Compression supported: 0 2024-09-21T09:22:10.567 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-21T09:22:10.568 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: kLZ4Compression supported: 1 2024-09-21T09:22:10.568 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: kZlibCompression supported: 1 2024-09-21T09:22:10.568 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: kLZ4HCCompression supported: 1 2024-09-21T09:22:10.568 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: kSnappyCompression supported: 1 2024-09-21T09:22:10.568 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-21T09:22:10.568 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 2024-09-21T09:22:10.568 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2024-09-21T09:22:10.568 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-21T09:22:10.568 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.merge_operator: 2024-09-21T09:22:10.568 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.compaction_filter: None 2024-09-21T09:22:10.568 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.compaction_filter_factory: None 2024-09-21T09:22:10.568 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.sst_partitioner_factory: None 2024-09-21T09:22:10.568 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-21T09:22:10.568 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-21T09:22:10.569 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5595e332fa38) 2024-09-21T09:22:10.569 INFO:journalctl@ceph.mon.a.smithi022.stdout: cache_index_and_filter_blocks: 1 2024-09-21T09:22:10.569 INFO:journalctl@ceph.mon.a.smithi022.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-21T09:22:10.569 INFO:journalctl@ceph.mon.a.smithi022.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-21T09:22:10.569 INFO:journalctl@ceph.mon.a.smithi022.stdout: pin_top_level_index_and_filter: 1 2024-09-21T09:22:10.569 INFO:journalctl@ceph.mon.a.smithi022.stdout: index_type: 0 2024-09-21T09:22:10.569 INFO:journalctl@ceph.mon.a.smithi022.stdout: data_block_index_type: 0 2024-09-21T09:22:10.569 INFO:journalctl@ceph.mon.a.smithi022.stdout: index_shortening: 1 2024-09-21T09:22:10.569 INFO:journalctl@ceph.mon.a.smithi022.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-21T09:22:10.569 INFO:journalctl@ceph.mon.a.smithi022.stdout: hash_index_allow_collision: 1 2024-09-21T09:22:10.569 INFO:journalctl@ceph.mon.a.smithi022.stdout: checksum: 1 2024-09-21T09:22:10.570 INFO:journalctl@ceph.mon.a.smithi022.stdout: no_block_cache: 0 2024-09-21T09:22:10.570 INFO:journalctl@ceph.mon.a.smithi022.stdout: block_cache: 0x5595e3365350 2024-09-21T09:22:10.570 INFO:journalctl@ceph.mon.a.smithi022.stdout: block_cache_name: BinnedLRUCache 2024-09-21T09:22:10.570 INFO:journalctl@ceph.mon.a.smithi022.stdout: block_cache_options: 2024-09-21T09:22:10.570 INFO:journalctl@ceph.mon.a.smithi022.stdout: capacity : 536870912 2024-09-21T09:22:10.570 INFO:journalctl@ceph.mon.a.smithi022.stdout: num_shard_bits : 4 2024-09-21T09:22:10.570 INFO:journalctl@ceph.mon.a.smithi022.stdout: strict_capacity_limit : 0 2024-09-21T09:22:10.570 INFO:journalctl@ceph.mon.a.smithi022.stdout: high_pri_pool_ratio: 0.000 2024-09-21T09:22:10.570 INFO:journalctl@ceph.mon.a.smithi022.stdout: block_cache_compressed: (nil) 2024-09-21T09:22:10.570 INFO:journalctl@ceph.mon.a.smithi022.stdout: persistent_cache: (nil) 2024-09-21T09:22:10.570 INFO:journalctl@ceph.mon.a.smithi022.stdout: block_size: 4096 2024-09-21T09:22:10.570 INFO:journalctl@ceph.mon.a.smithi022.stdout: block_size_deviation: 10 2024-09-21T09:22:10.570 INFO:journalctl@ceph.mon.a.smithi022.stdout: block_restart_interval: 16 2024-09-21T09:22:10.571 INFO:journalctl@ceph.mon.a.smithi022.stdout: index_block_restart_interval: 1 2024-09-21T09:22:10.571 INFO:journalctl@ceph.mon.a.smithi022.stdout: metadata_block_size: 4096 2024-09-21T09:22:10.571 INFO:journalctl@ceph.mon.a.smithi022.stdout: partition_filters: 0 2024-09-21T09:22:10.571 INFO:journalctl@ceph.mon.a.smithi022.stdout: use_delta_encoding: 1 2024-09-21T09:22:10.571 INFO:journalctl@ceph.mon.a.smithi022.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-09-21T09:22:10.571 INFO:journalctl@ceph.mon.a.smithi022.stdout: whole_key_filtering: 1 2024-09-21T09:22:10.571 INFO:journalctl@ceph.mon.a.smithi022.stdout: verify_compression: 0 2024-09-21T09:22:10.571 INFO:journalctl@ceph.mon.a.smithi022.stdout: read_amp_bytes_per_bit: 0 2024-09-21T09:22:10.571 INFO:journalctl@ceph.mon.a.smithi022.stdout: format_version: 4 2024-09-21T09:22:10.571 INFO:journalctl@ceph.mon.a.smithi022.stdout: enable_index_compression: 1 2024-09-21T09:22:10.571 INFO:journalctl@ceph.mon.a.smithi022.stdout: block_align: 0 2024-09-21T09:22:10.571 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.write_buffer_size: 33554432 2024-09-21T09:22:10.571 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.max_write_buffer_number: 2 2024-09-21T09:22:10.572 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.compression: NoCompression 2024-09-21T09:22:10.572 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.bottommost_compression: Disabled 2024-09-21T09:22:10.572 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.prefix_extractor: nullptr 2024-09-21T09:22:10.572 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-21T09:22:10.572 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.num_levels: 7 2024-09-21T09:22:10.572 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-21T09:22:10.572 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-21T09:22:10.572 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-21T09:22:10.572 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-21T09:22:10.572 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-21T09:22:10.572 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-21T09:22:10.572 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-21T09:22:10.572 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-21T09:22:10.572 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-21T09:22:10.572 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-21T09:22:10.573 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-21T09:22:10.573 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.compression_opts.level: 32767 2024-09-21T09:22:10.573 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.compression_opts.strategy: 0 2024-09-21T09:22:10.573 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-21T09:22:10.573 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-21T09:22:10.573 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-21T09:22:10.573 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.compression_opts.enabled: false 2024-09-21T09:22:10.573 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-21T09:22:10.573 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-21T09:22:10.573 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-21T09:22:10.573 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.target_file_size_base: 67108864 2024-09-21T09:22:10.573 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-21T09:22:10.573 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-21T09:22:10.573 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-21T09:22:10.574 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-21T09:22:10.574 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-21T09:22:10.574 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-21T09:22:10.574 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-21T09:22:10.574 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-21T09:22:10.574 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-21T09:22:10.574 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-21T09:22:10.574 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-21T09:22:10.574 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-21T09:22:10.574 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-21T09:22:10.574 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.arena_block_size: 4194304 2024-09-21T09:22:10.574 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-21T09:22:10.574 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-21T09:22:10.574 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-09-21T09:22:10.575 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.disable_auto_compactions: 0 2024-09-21T09:22:10.575 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-21T09:22:10.575 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-21T09:22:10.575 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-21T09:22:10.575 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-21T09:22:10.575 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-21T09:22:10.575 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-21T09:22:10.575 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-21T09:22:10.575 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-21T09:22:10.575 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-21T09:22:10.575 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-21T09:22:10.575 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-21T09:22:10.575 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.inplace_update_support: 0 2024-09-21T09:22:10.575 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-21T09:22:10.575 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-21T09:22:10.576 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-21T09:22:10.576 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-21T09:22:10.576 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.bloom_locality: 0 2024-09-21T09:22:10.576 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.max_successive_merges: 0 2024-09-21T09:22:10.576 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-21T09:22:10.576 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.paranoid_file_checks: 0 2024-09-21T09:22:10.576 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.force_consistency_checks: 1 2024-09-21T09:22:10.576 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.report_bg_io_stats: 0 2024-09-21T09:22:10.576 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.ttl: 2592000 2024-09-21T09:22:10.576 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-21T09:22:10.576 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.enable_blob_files: false 2024-09-21T09:22:10.576 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.min_blob_size: 0 2024-09-21T09:22:10.576 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.blob_file_size: 268435456 2024-09-21T09:22:10.576 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-21T09:22:10.576 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-21T09:22:10.577 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-21T09:22:10.577 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 11, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-21T09:22:10.577 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 5 2024-09-21T09:22:10.577 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: [db/version_set.cc:4082] Creating manifest 13 2024-09-21T09:22:10.577 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726910530307295, "job": 1, "event": "recovery_started", "wal_files": [10]} 2024-09-21T09:22:10.577 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #10 mode 2 2024-09-21T09:22:10.577 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-09-21T09:22:10.577 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726910530308077, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 14, "file_size": 80255, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 78497, "index_size": 220, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 10130, "raw_average_key_size": 47, "raw_value_size": 72710, "raw_average_value_size": 341, "num_data_blocks": 10, "num_entries": 213, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1726910530, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "b9b914da-1973-42af-b0a1-d06ce239017b", "db_session_id": "D5S5VXU896KYWAPI6FTP"}} 2024-09-21T09:22:10.577 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: [db/version_set.cc:4082] Creating manifest 15 2024-09-21T09:22:10.577 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726910530308439, "job": 1, "event": "recovery_finished"} 2024-09-21T09:22:10.577 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000010.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-21T09:22:10.578 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x5595e3398700 2024-09-21T09:22:10.578 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: DB pointer 0x5595e3368000 2024-09-21T09:22:10.578 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2024-09-21T09:22:10.578 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-09-21T09:22:10.578 INFO:journalctl@ceph.mon.a.smithi022.stdout: ** DB Stats ** 2024-09-21T09:22:10.578 INFO:journalctl@ceph.mon.a.smithi022.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-21T09:22:10.578 INFO:journalctl@ceph.mon.a.smithi022.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-21T09:22:10.578 INFO:journalctl@ceph.mon.a.smithi022.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-21T09:22:10.578 INFO:journalctl@ceph.mon.a.smithi022.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-21T09:22:10.578 INFO:journalctl@ceph.mon.a.smithi022.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-21T09:22:10.578 INFO:journalctl@ceph.mon.a.smithi022.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-09-21T09:22:10.578 INFO:journalctl@ceph.mon.a.smithi022.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-21T09:22:10.578 INFO:journalctl@ceph.mon.a.smithi022.stdout: 2024-09-21T09:22:10.579 INFO:journalctl@ceph.mon.a.smithi022.stdout: ** Compaction Stats [default] ** 2024-09-21T09:22:10.579 INFO:journalctl@ceph.mon.a.smithi022.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-21T09:22:10.579 INFO:journalctl@ceph.mon.a.smithi022.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-21T09:22:10.579 INFO:journalctl@ceph.mon.a.smithi022.stdout: L0 2/0 80.16 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 148.6 0.00 0.00 1 0.001 0 0 2024-09-21T09:22:10.579 INFO:journalctl@ceph.mon.a.smithi022.stdout: Sum 2/0 80.16 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 148.6 0.00 0.00 1 0.001 0 0 2024-09-21T09:22:10.579 INFO:journalctl@ceph.mon.a.smithi022.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 148.6 0.00 0.00 1 0.001 0 0 2024-09-21T09:22:10.579 INFO:journalctl@ceph.mon.a.smithi022.stdout: 2024-09-21T09:22:10.579 INFO:journalctl@ceph.mon.a.smithi022.stdout: ** Compaction Stats [default] ** 2024-09-21T09:22:10.579 INFO:journalctl@ceph.mon.a.smithi022.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-21T09:22:10.579 INFO:journalctl@ceph.mon.a.smithi022.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-21T09:22:10.579 INFO:journalctl@ceph.mon.a.smithi022.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 148.6 0.00 0.00 1 0.001 0 0 2024-09-21T09:22:10.579 INFO:journalctl@ceph.mon.a.smithi022.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-21T09:22:10.579 INFO:journalctl@ceph.mon.a.smithi022.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-21T09:22:10.579 INFO:journalctl@ceph.mon.a.smithi022.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-21T09:22:10.580 INFO:journalctl@ceph.mon.a.smithi022.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-21T09:22:10.580 INFO:journalctl@ceph.mon.a.smithi022.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-21T09:22:10.580 INFO:journalctl@ceph.mon.a.smithi022.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-21T09:22:10.580 INFO:journalctl@ceph.mon.a.smithi022.stdout: Cumulative compaction: 0.00 GB write, 24.26 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T09:22:10.580 INFO:journalctl@ceph.mon.a.smithi022.stdout: Interval compaction: 0.00 GB write, 24.26 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T09:22:10.580 INFO:journalctl@ceph.mon.a.smithi022.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-21T09:22:10.580 INFO:journalctl@ceph.mon.a.smithi022.stdout: 2024-09-21T09:22:10.580 INFO:journalctl@ceph.mon.a.smithi022.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-21T09:22:10.580 INFO:journalctl@ceph.mon.a.smithi022.stdout: 2024-09-21T09:22:10.580 INFO:journalctl@ceph.mon.a.smithi022.stdout: ** Compaction Stats [default] ** 2024-09-21T09:22:10.580 INFO:journalctl@ceph.mon.a.smithi022.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-21T09:22:10.580 INFO:journalctl@ceph.mon.a.smithi022.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-21T09:22:10.580 INFO:journalctl@ceph.mon.a.smithi022.stdout: L0 2/0 80.16 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 148.6 0.00 0.00 1 0.001 0 0 2024-09-21T09:22:10.580 INFO:journalctl@ceph.mon.a.smithi022.stdout: Sum 2/0 80.16 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 148.6 0.00 0.00 1 0.001 0 0 2024-09-21T09:22:10.580 INFO:journalctl@ceph.mon.a.smithi022.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-09-21T09:22:10.581 INFO:journalctl@ceph.mon.a.smithi022.stdout: 2024-09-21T09:22:10.581 INFO:journalctl@ceph.mon.a.smithi022.stdout: ** Compaction Stats [default] ** 2024-09-21T09:22:10.581 INFO:journalctl@ceph.mon.a.smithi022.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-21T09:22:10.581 INFO:journalctl@ceph.mon.a.smithi022.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-21T09:22:10.581 INFO:journalctl@ceph.mon.a.smithi022.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 148.6 0.00 0.00 1 0.001 0 0 2024-09-21T09:22:10.581 INFO:journalctl@ceph.mon.a.smithi022.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-21T09:22:10.581 INFO:journalctl@ceph.mon.a.smithi022.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-21T09:22:10.581 INFO:journalctl@ceph.mon.a.smithi022.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-21T09:22:10.581 INFO:journalctl@ceph.mon.a.smithi022.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-21T09:22:10.581 INFO:journalctl@ceph.mon.a.smithi022.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-21T09:22:10.581 INFO:journalctl@ceph.mon.a.smithi022.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-21T09:22:10.581 INFO:journalctl@ceph.mon.a.smithi022.stdout: Cumulative compaction: 0.00 GB write, 24.03 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T09:22:10.581 INFO:journalctl@ceph.mon.a.smithi022.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T09:22:10.581 INFO:journalctl@ceph.mon.a.smithi022.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-21T09:22:10.581 INFO:journalctl@ceph.mon.a.smithi022.stdout: 2024-09-21T09:22:10.582 INFO:journalctl@ceph.mon.a.smithi022.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-21T09:22:10.582 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: starting mon.a rank 0 at public addrs [v2:172.21.15.22:3300/0,v1:172.21.15.22:6789/0] at bind addrs [v2:172.21.15.22:3300/0,v1:172.21.15.22:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid d65e801e-77fa-11ef-baf6-efdc52797490 2024-09-21T09:22:10.582 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: mon.a@-1(???) e1 preinit fsid d65e801e-77fa-11ef-baf6-efdc52797490 2024-09-21T09:22:10.582 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: mon.a@-1(???).mds e1 new map 2024-09-21T09:22:10.582 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: mon.a@-1(???).mds e1 print_map 2024-09-21T09:22:10.582 INFO:journalctl@ceph.mon.a.smithi022.stdout: e1 2024-09-21T09:22:10.582 INFO:journalctl@ceph.mon.a.smithi022.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-21T09:22:10.582 INFO:journalctl@ceph.mon.a.smithi022.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-21T09:22:10.582 INFO:journalctl@ceph.mon.a.smithi022.stdout: legacy client fscid: -1 2024-09-21T09:22:10.582 INFO:journalctl@ceph.mon.a.smithi022.stdout: 2024-09-21T09:22:10.582 INFO:journalctl@ceph.mon.a.smithi022.stdout: No filesystems configured 2024-09-21T09:22:10.582 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: mon.a@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-09-21T09:22:10.582 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-21T09:22:10.582 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-21T09:22:10.583 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-21T09:22:10.583 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: mon.a@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-09-21T09:22:10.583 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 bash[27726]: 1e0a2d74e66ed0875bbbe448f9186ef3a086f3147ceee15bc593a101f91895e0 2024-09-21T09:22:10.583 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 systemd[1]: Started Ceph mon.a for d65e801e-77fa-11ef-baf6-efdc52797490. 2024-09-21T09:22:10.583 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: mon.a is new leader, mons a in quorum (ranks 0) 2024-09-21T09:22:10.583 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: monmap e1: 1 mons at {a=[v2:172.21.15.22:3300/0,v1:172.21.15.22:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-21T09:22:10.583 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: fsmap 2024-09-21T09:22:10.583 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: osdmap e1: 0 total, 0 up, 0 in 2024-09-21T09:22:10.583 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:10 smithi022 ceph-mon[27763]: mgrmap e1: no daemons active 2024-09-21T09:22:11.356 INFO:teuthology.orchestra.run.smithi022.stdout:Wrote config to /etc/ceph/ceph.conf 2024-09-21T09:22:11.356 INFO:teuthology.orchestra.run.smithi022.stdout:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-09-21T09:22:11.356 INFO:teuthology.orchestra.run.smithi022.stdout:Creating mgr... 2024-09-21T09:22:11.357 INFO:teuthology.orchestra.run.smithi022.stdout:Verifying port 0.0.0.0:9283 ... 2024-09-21T09:22:11.357 INFO:teuthology.orchestra.run.smithi022.stdout:Verifying port 0.0.0.0:8765 ... 2024-09-21T09:22:11.536 INFO:teuthology.orchestra.run.smithi022.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-d65e801e-77fa-11ef-baf6-efdc52797490@mgr.y 2024-09-21T09:22:11.536 INFO:teuthology.orchestra.run.smithi022.stdout:systemctl: stderr Failed to reset failed state of unit ceph-d65e801e-77fa-11ef-baf6-efdc52797490@mgr.y.service: Unit ceph-d65e801e-77fa-11ef-baf6-efdc52797490@mgr.y.service not loaded. 2024-09-21T09:22:11.687 INFO:teuthology.orchestra.run.smithi022.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-d65e801e-77fa-11ef-baf6-efdc52797490.target.wants/ceph-d65e801e-77fa-11ef-baf6-efdc52797490@mgr.y.service → /etc/systemd/system/ceph-d65e801e-77fa-11ef-baf6-efdc52797490@.service. 2024-09-21T09:22:11.807 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:11 smithi022 systemd[1]: Starting Ceph mgr.y for d65e801e-77fa-11ef-baf6-efdc52797490... 2024-09-21T09:22:12.190 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:11 smithi022 ceph-mon[27763]: from='client.? 172.21.15.22:0/1991193948' entity='client.admin' 2024-09-21T09:22:12.190 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:11 smithi022 podman[27997]: 2024-09-21 09:22:11.82288445 +0000 UTC m=+0.015890151 image pull fc3ec4a4f6be575045de0fd8a2fe84939e888152c447bdedb90add7841f7bef9 quay.ceph.io/ceph-ci/ceph:quincy 2024-09-21T09:22:12.190 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:11 smithi022 podman[27997]: 2024-09-21 09:22:11.96974591 +0000 UTC m=+0.162751625 container create 5bc449c0676ffbba20383552b4d91185ae197d22b0feaaa35cf8ba1ea83482b8 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=quincy-fbd1a6a, io.buildah.version=1.37.2, ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731) 2024-09-21T09:22:12.190 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:12 smithi022 podman[27997]: 2024-09-21 09:22:12.188591943 +0000 UTC m=+0.381597632 container init 5bc449c0676ffbba20383552b4d91185ae197d22b0feaaa35cf8ba1ea83482b8 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=quincy-fbd1a6a, ceph=True, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-09-21T09:22:12.303 INFO:teuthology.orchestra.run.smithi022.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-09-21T09:22:12.303 INFO:teuthology.orchestra.run.smithi022.stdout:systemctl: stdout disabled 2024-09-21T09:22:12.311 INFO:teuthology.orchestra.run.smithi022.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-09-21T09:22:12.311 INFO:teuthology.orchestra.run.smithi022.stdout:systemctl: stdout inactive 2024-09-21T09:22:12.311 INFO:teuthology.orchestra.run.smithi022.stdout:firewalld.service is not enabled 2024-09-21T09:22:12.311 INFO:teuthology.orchestra.run.smithi022.stdout:Not possible to enable service . firewalld.service is not available 2024-09-21T09:22:12.321 INFO:teuthology.orchestra.run.smithi022.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-09-21T09:22:12.321 INFO:teuthology.orchestra.run.smithi022.stdout:systemctl: stdout disabled 2024-09-21T09:22:12.329 INFO:teuthology.orchestra.run.smithi022.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-09-21T09:22:12.329 INFO:teuthology.orchestra.run.smithi022.stdout:systemctl: stdout inactive 2024-09-21T09:22:12.329 INFO:teuthology.orchestra.run.smithi022.stdout:firewalld.service is not enabled 2024-09-21T09:22:12.329 INFO:teuthology.orchestra.run.smithi022.stdout:Not possible to open ports <[9283, 8765]>. firewalld.service is not available 2024-09-21T09:22:12.329 INFO:teuthology.orchestra.run.smithi022.stdout:Waiting for mgr to start... 2024-09-21T09:22:12.329 INFO:teuthology.orchestra.run.smithi022.stdout:Waiting for mgr... 2024-09-21T09:22:12.443 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:12 smithi022 podman[27997]: 2024-09-21 09:22:12.193597969 +0000 UTC m=+0.386603639 container start 5bc449c0676ffbba20383552b4d91185ae197d22b0feaaa35cf8ba1ea83482b8 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y, GIT_BRANCH=HEAD, RELEASE=quincy-fbd1a6a, org.label-schema.license=GPLv2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, ceph=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T09:22:12.443 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:12 smithi022 bash[27997]: 5bc449c0676ffbba20383552b4d91185ae197d22b0feaaa35cf8ba1ea83482b8 2024-09-21T09:22:12.443 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:12 smithi022 systemd[1]: Started Ceph mgr.y for d65e801e-77fa-11ef-baf6-efdc52797490. 2024-09-21T09:22:12.444 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:12 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:22:12.312+0000 7fdd4b9dcec0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-21T09:22:12.444 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:12 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:22:12.373+0000 7fdd4b9dcec0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-21T09:22:12.444 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:12 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:22:12.441+0000 7fdd4b9dcec0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-21T09:22:12.701 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:12 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:22:12.576+0000 7fdd4b9dcec0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-21T09:22:13.005 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:12 smithi022 ceph-mon[27763]: from='client.? 172.21.15.22:0/2339425661' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-21T09:22:13.006 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:12 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:22:12.725+0000 7fdd4b9dcec0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-21T09:22:13.006 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:13 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:22:13.003+0000 7fdd4b9dcec0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-21T09:22:13.308 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:13 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:22:13.067+0000 7fdd4b9dcec0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-21T09:22:13.308 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:13 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:22:13.238+0000 7fdd4b9dcec0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-21T09:22:13.431 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout 2024-09-21T09:22:13.434 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout { 2024-09-21T09:22:13.434 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "fsid": "d65e801e-77fa-11ef-baf6-efdc52797490", 2024-09-21T09:22:13.434 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "health": { 2024-09-21T09:22:13.434 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-09-21T09:22:13.434 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-09-21T09:22:13.434 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-09-21T09:22:13.434 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout }, 2024-09-21T09:22:13.435 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-09-21T09:22:13.435 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-09-21T09:22:13.435 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout 0 2024-09-21T09:22:13.435 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout ], 2024-09-21T09:22:13.435 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-09-21T09:22:13.435 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "a" 2024-09-21T09:22:13.435 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout ], 2024-09-21T09:22:13.435 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "quorum_age": 2, 2024-09-21T09:22:13.435 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "monmap": { 2024-09-21T09:22:13.435 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-21T09:22:13.435 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2024-09-21T09:22:13.435 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-09-21T09:22:13.435 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout }, 2024-09-21T09:22:13.435 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-09-21T09:22:13.435 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-21T09:22:13.436 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-09-21T09:22:13.436 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-09-21T09:22:13.436 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-09-21T09:22:13.436 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-09-21T09:22:13.436 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-09-21T09:22:13.436 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-09-21T09:22:13.439 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout }, 2024-09-21T09:22:13.439 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-09-21T09:22:13.439 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-09-21T09:22:13.439 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-09-21T09:22:13.439 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-09-21T09:22:13.439 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-09-21T09:22:13.439 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-09-21T09:22:13.440 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-09-21T09:22:13.440 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-09-21T09:22:13.440 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-09-21T09:22:13.440 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout }, 2024-09-21T09:22:13.440 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-09-21T09:22:13.440 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-21T09:22:13.440 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-09-21T09:22:13.440 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-09-21T09:22:13.440 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout }, 2024-09-21T09:22:13.440 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-09-21T09:22:13.440 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "available": false, 2024-09-21T09:22:13.440 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-09-21T09:22:13.440 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "modules": [ 2024-09-21T09:22:13.440 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "iostat", 2024-09-21T09:22:13.440 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "nfs", 2024-09-21T09:22:13.441 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "restful" 2024-09-21T09:22:13.441 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout ], 2024-09-21T09:22:13.441 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "services": {} 2024-09-21T09:22:13.441 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout }, 2024-09-21T09:22:13.441 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-09-21T09:22:13.441 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-21T09:22:13.441 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "modified": "2024-09-21T09:22:04.475392+0000", 2024-09-21T09:22:13.441 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "services": {} 2024-09-21T09:22:13.441 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout }, 2024-09-21T09:22:13.441 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-09-21T09:22:13.441 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout } 2024-09-21T09:22:13.441 INFO:teuthology.orchestra.run.smithi022.stdout:mgr not available, waiting (1/15)... 2024-09-21T09:22:14.557 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:14 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:22:14.131+0000 7fdd4b9dcec0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-21T09:22:14.557 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:14 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:22:14.198+0000 7fdd4b9dcec0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-21T09:22:15.261 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:14 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:22:14.954+0000 7fdd4b9dcec0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-21T09:22:15.261 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:15 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:22:15.071+0000 7fdd4b9dcec0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-21T09:22:15.261 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:15 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:22:15.197+0000 7fdd4b9dcec0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-21T09:22:15.261 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:15 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:22:15.259+0000 7fdd4b9dcec0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-21T09:22:15.526 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:15 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:22:15.390+0000 7fdd4b9dcec0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-21T09:22:15.526 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:15 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:22:15.462+0000 7fdd4b9dcec0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-21T09:22:15.526 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:15 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:22:15.524+0000 7fdd4b9dcec0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-21T09:22:15.808 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:15 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:22:15.667+0000 7fdd4b9dcec0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-21T09:22:15.808 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:15 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:22:15.739+0000 7fdd4b9dcec0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-21T09:22:16.062 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:15 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:22:15.808+0000 7fdd4b9dcec0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-21T09:22:16.062 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:15 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:22:15.894+0000 7fdd4b9dcec0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-21T09:22:16.062 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:15 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:22:15.957+0000 7fdd4b9dcec0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-21T09:22:16.062 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:16 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:22:16.018+0000 7fdd4b9dcec0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-21T09:22:16.370 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:16 smithi022 ceph-mon[27763]: from='client.? 172.21.15.22:0/2260417771' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-21T09:22:16.625 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout 2024-09-21T09:22:16.625 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout { 2024-09-21T09:22:16.625 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "fsid": "d65e801e-77fa-11ef-baf6-efdc52797490", 2024-09-21T09:22:16.625 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "health": { 2024-09-21T09:22:16.625 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-09-21T09:22:16.625 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-09-21T09:22:16.625 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-09-21T09:22:16.625 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout }, 2024-09-21T09:22:16.625 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-09-21T09:22:16.625 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-09-21T09:22:16.626 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout 0 2024-09-21T09:22:16.626 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout ], 2024-09-21T09:22:16.626 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-09-21T09:22:16.626 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "a" 2024-09-21T09:22:16.626 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout ], 2024-09-21T09:22:16.626 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "quorum_age": 5, 2024-09-21T09:22:16.626 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "monmap": { 2024-09-21T09:22:16.626 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-21T09:22:16.626 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2024-09-21T09:22:16.626 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-09-21T09:22:16.626 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout }, 2024-09-21T09:22:16.627 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-09-21T09:22:16.627 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-21T09:22:16.627 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-09-21T09:22:16.627 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-09-21T09:22:16.627 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-09-21T09:22:16.627 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-09-21T09:22:16.627 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-09-21T09:22:16.627 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-09-21T09:22:16.630 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout }, 2024-09-21T09:22:16.630 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-09-21T09:22:16.630 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-09-21T09:22:16.630 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-09-21T09:22:16.631 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-09-21T09:22:16.631 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-09-21T09:22:16.631 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-09-21T09:22:16.631 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-09-21T09:22:16.631 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-09-21T09:22:16.631 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-09-21T09:22:16.631 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout }, 2024-09-21T09:22:16.631 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-09-21T09:22:16.631 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-21T09:22:16.631 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-09-21T09:22:16.631 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-09-21T09:22:16.631 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout }, 2024-09-21T09:22:16.632 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-09-21T09:22:16.632 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "available": false, 2024-09-21T09:22:16.632 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-09-21T09:22:16.632 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "modules": [ 2024-09-21T09:22:16.632 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "iostat", 2024-09-21T09:22:16.632 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "nfs", 2024-09-21T09:22:16.632 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "restful" 2024-09-21T09:22:16.632 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout ], 2024-09-21T09:22:16.632 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "services": {} 2024-09-21T09:22:16.632 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout }, 2024-09-21T09:22:16.632 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-09-21T09:22:16.632 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-21T09:22:16.632 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "modified": "2024-09-21T09:22:04.475392+0000", 2024-09-21T09:22:16.632 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "services": {} 2024-09-21T09:22:16.632 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout }, 2024-09-21T09:22:16.632 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-09-21T09:22:16.633 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout } 2024-09-21T09:22:16.633 INFO:teuthology.orchestra.run.smithi022.stdout:mgr not available, waiting (2/15)... 2024-09-21T09:22:17.058 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:16 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:22:16.818+0000 7fdd4b9dcec0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-21T09:22:17.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:17 smithi022 ceph-mon[27763]: Activating manager daemon y 2024-09-21T09:22:17.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:17 smithi022 ceph-mon[27763]: mgrmap e2: y(active, starting, since 0.00284198s) 2024-09-21T09:22:17.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:17 smithi022 ceph-mon[27763]: from='mgr.14100 172.21.15.22:0/93945452' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T09:22:17.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:17 smithi022 ceph-mon[27763]: from='mgr.14100 172.21.15.22:0/93945452' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T09:22:17.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:17 smithi022 ceph-mon[27763]: from='mgr.14100 172.21.15.22:0/93945452' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T09:22:17.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:17 smithi022 ceph-mon[27763]: from='mgr.14100 172.21.15.22:0/93945452' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:22:17.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:17 smithi022 ceph-mon[27763]: from='mgr.14100 172.21.15.22:0/93945452' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-21T09:22:17.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:17 smithi022 ceph-mon[27763]: Manager daemon y is now available 2024-09-21T09:22:17.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:17 smithi022 ceph-mon[27763]: from='mgr.14100 172.21.15.22:0/93945452' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-21T09:22:17.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:17 smithi022 ceph-mon[27763]: from='mgr.14100 172.21.15.22:0/93945452' entity='mgr.y' 2024-09-21T09:22:17.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:17 smithi022 ceph-mon[27763]: from='mgr.14100 172.21.15.22:0/93945452' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-21T09:22:17.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:17 smithi022 ceph-mon[27763]: from='mgr.14100 172.21.15.22:0/93945452' entity='mgr.y' 2024-09-21T09:22:17.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:17 smithi022 ceph-mon[27763]: from='mgr.14100 172.21.15.22:0/93945452' entity='mgr.y' 2024-09-21T09:22:19.006 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:18 smithi022 ceph-mon[27763]: mgrmap e3: y(active, since 1.00671s) 2024-09-21T09:22:19.926 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:19 smithi022 ceph-mon[27763]: mgrmap e4: y(active, since 2s) 2024-09-21T09:22:19.926 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:19 smithi022 ceph-mon[27763]: from='client.? 172.21.15.22:0/29955038' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-21T09:22:19.937 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout 2024-09-21T09:22:19.938 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout { 2024-09-21T09:22:19.938 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "fsid": "d65e801e-77fa-11ef-baf6-efdc52797490", 2024-09-21T09:22:19.938 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "health": { 2024-09-21T09:22:19.938 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-09-21T09:22:19.938 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-09-21T09:22:19.939 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-09-21T09:22:19.939 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout }, 2024-09-21T09:22:19.939 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-09-21T09:22:19.939 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-09-21T09:22:19.939 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout 0 2024-09-21T09:22:19.939 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout ], 2024-09-21T09:22:19.940 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-09-21T09:22:19.940 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "a" 2024-09-21T09:22:19.940 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout ], 2024-09-21T09:22:19.940 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "quorum_age": 9, 2024-09-21T09:22:19.940 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "monmap": { 2024-09-21T09:22:19.940 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-21T09:22:19.941 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2024-09-21T09:22:19.941 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-09-21T09:22:19.941 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout }, 2024-09-21T09:22:19.941 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-09-21T09:22:19.941 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-21T09:22:19.941 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-09-21T09:22:19.941 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-09-21T09:22:19.941 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-09-21T09:22:19.942 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-09-21T09:22:19.942 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-09-21T09:22:19.942 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-09-21T09:22:19.942 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout }, 2024-09-21T09:22:19.942 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-09-21T09:22:19.942 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-09-21T09:22:19.942 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-09-21T09:22:19.942 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-09-21T09:22:19.942 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-09-21T09:22:19.943 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-09-21T09:22:19.943 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-09-21T09:22:19.943 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-09-21T09:22:19.943 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-09-21T09:22:19.943 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout }, 2024-09-21T09:22:19.943 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-09-21T09:22:19.943 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-21T09:22:19.943 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-09-21T09:22:19.943 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-09-21T09:22:19.944 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout }, 2024-09-21T09:22:19.944 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-09-21T09:22:19.944 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "available": true, 2024-09-21T09:22:19.944 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-09-21T09:22:19.944 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "modules": [ 2024-09-21T09:22:19.944 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "iostat", 2024-09-21T09:22:19.944 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "nfs", 2024-09-21T09:22:19.944 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "restful" 2024-09-21T09:22:19.945 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout ], 2024-09-21T09:22:19.945 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "services": {} 2024-09-21T09:22:19.945 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout }, 2024-09-21T09:22:19.945 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-09-21T09:22:19.945 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-21T09:22:19.945 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "modified": "2024-09-21T09:22:04.475392+0000", 2024-09-21T09:22:19.945 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "services": {} 2024-09-21T09:22:19.946 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout }, 2024-09-21T09:22:19.946 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-09-21T09:22:19.946 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout } 2024-09-21T09:22:19.946 INFO:teuthology.orchestra.run.smithi022.stdout:mgr is available 2024-09-21T09:22:21.165 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:20 smithi022 ceph-mon[27763]: from='client.? 172.21.15.22:0/181927087' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-09-21T09:22:21.169 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout 2024-09-21T09:22:21.169 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout [global] 2024-09-21T09:22:21.169 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout fsid = d65e801e-77fa-11ef-baf6-efdc52797490 2024-09-21T09:22:21.169 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout mon_cluster_log_file_level = debug 2024-09-21T09:22:21.169 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2024-09-21T09:22:21.170 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2024-09-21T09:22:21.170 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2024-09-21T09:22:21.170 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2024-09-21T09:22:21.170 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout 2024-09-21T09:22:21.170 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout [mgr] 2024-09-21T09:22:21.170 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2024-09-21T09:22:21.170 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout 2024-09-21T09:22:21.170 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout [osd] 2024-09-21T09:22:21.170 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2024-09-21T09:22:21.170 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2024-09-21T09:22:21.170 INFO:teuthology.orchestra.run.smithi022.stdout:Enabling cephadm module... 2024-09-21T09:22:22.156 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:21 smithi022 ceph-mon[27763]: from='client.? 172.21.15.22:0/1452316437' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-09-21T09:22:22.156 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:21 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: ignoring --setuser ceph since I am not root 2024-09-21T09:22:22.156 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:21 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: ignoring --setgroup ceph since I am not root 2024-09-21T09:22:22.157 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:22 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:22:22.028+0000 7f4f25535ec0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-21T09:22:22.157 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:22 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:22:22.085+0000 7f4f25535ec0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-21T09:22:22.157 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:22 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:22:22.154+0000 7f4f25535ec0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-21T09:22:22.429 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:22 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:22:22.284+0000 7f4f25535ec0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-21T09:22:22.429 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:22 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:22:22.427+0000 7f4f25535ec0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-21T09:22:23.057 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:22 smithi022 ceph-mon[27763]: from='client.? 172.21.15.22:0/1452316437' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-09-21T09:22:23.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:22 smithi022 ceph-mon[27763]: mgrmap e5: y(active, since 5s) 2024-09-21T09:22:23.058 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:22 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:22:22.700+0000 7f4f25535ec0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-21T09:22:23.058 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:22 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:22:22.766+0000 7f4f25535ec0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-21T09:22:23.058 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:22 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:22:22.938+0000 7f4f25535ec0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-21T09:22:23.797 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout { 2024-09-21T09:22:23.797 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "epoch": 5, 2024-09-21T09:22:23.797 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "available": true, 2024-09-21T09:22:23.797 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "active_name": "y", 2024-09-21T09:22:23.797 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "num_standby": 0 2024-09-21T09:22:23.797 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout } 2024-09-21T09:22:23.797 INFO:teuthology.orchestra.run.smithi022.stdout:Waiting for the mgr to restart... 2024-09-21T09:22:23.798 INFO:teuthology.orchestra.run.smithi022.stdout:Waiting for mgr epoch 5... 2024-09-21T09:22:24.057 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:23 smithi022 ceph-mon[27763]: from='client.? 172.21.15.22:0/4005917663' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-09-21T09:22:24.057 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:23 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:22:23.831+0000 7f4f25535ec0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-21T09:22:24.057 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:23 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:22:23.899+0000 7f4f25535ec0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-21T09:22:24.954 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:24 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:22:24.648+0000 7f4f25535ec0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-21T09:22:24.955 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:24 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:22:24.764+0000 7f4f25535ec0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-21T09:22:24.955 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:24 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:22:24.890+0000 7f4f25535ec0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-21T09:22:24.955 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:24 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:22:24.952+0000 7f4f25535ec0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-21T09:22:25.216 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:25 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:22:25.084+0000 7f4f25535ec0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-21T09:22:25.216 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:25 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:22:25.153+0000 7f4f25535ec0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-21T09:22:25.216 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:25 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:22:25.214+0000 7f4f25535ec0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-21T09:22:25.489 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:25 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:22:25.354+0000 7f4f25535ec0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-21T09:22:25.490 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:25 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:22:25.420+0000 7f4f25535ec0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-21T09:22:25.490 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:25 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:22:25.487+0000 7f4f25535ec0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-21T09:22:25.807 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:25 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:22:25.572+0000 7f4f25535ec0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-21T09:22:25.808 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:25 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:22:25.632+0000 7f4f25535ec0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-21T09:22:25.808 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:25 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:22:25.691+0000 7f4f25535ec0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-21T09:22:26.808 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:26 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:22:26.470+0000 7f4f25535ec0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-21T09:22:26.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:26 smithi022 ceph-mon[27763]: Active manager daemon y restarted 2024-09-21T09:22:26.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:26 smithi022 ceph-mon[27763]: Activating manager daemon y 2024-09-21T09:22:26.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:26 smithi022 ceph-mon[27763]: osdmap e2: 0 total, 0 up, 0 in 2024-09-21T09:22:26.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:26 smithi022 ceph-mon[27763]: mgrmap e6: y(active, starting, since 0.0032865s) 2024-09-21T09:22:26.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:26 smithi022 ceph-mon[27763]: from='mgr.14118 172.21.15.22:0/4059432701' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:22:26.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:26 smithi022 ceph-mon[27763]: from='mgr.14118 172.21.15.22:0/4059432701' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-21T09:22:26.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:26 smithi022 ceph-mon[27763]: from='mgr.14118 172.21.15.22:0/4059432701' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T09:22:26.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:26 smithi022 ceph-mon[27763]: from='mgr.14118 172.21.15.22:0/4059432701' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T09:22:26.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:26 smithi022 ceph-mon[27763]: from='mgr.14118 172.21.15.22:0/4059432701' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T09:22:26.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:26 smithi022 ceph-mon[27763]: Manager daemon y is now available 2024-09-21T09:22:26.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:26 smithi022 ceph-mon[27763]: from='mgr.14118 172.21.15.22:0/4059432701' entity='mgr.y' 2024-09-21T09:22:26.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:26 smithi022 ceph-mon[27763]: from='mgr.14118 172.21.15.22:0/4059432701' entity='mgr.y' 2024-09-21T09:22:26.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:26 smithi022 ceph-mon[27763]: from='mgr.14118 172.21.15.22:0/4059432701' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:22:26.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:26 smithi022 ceph-mon[27763]: from='mgr.14118 172.21.15.22:0/4059432701' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:22:26.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:26 smithi022 ceph-mon[27763]: from='mgr.14118 172.21.15.22:0/4059432701' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-21T09:22:26.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:26 smithi022 ceph-mon[27763]: from='mgr.14118 172.21.15.22:0/4059432701' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-21T09:22:27.799 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:27 smithi022 ceph-mon[27763]: Found migration_current of "None". Setting to last migration. 2024-09-21T09:22:27.799 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:27 smithi022 ceph-mon[27763]: mgrmap e7: y(active, since 1.00532s) 2024-09-21T09:22:28.044 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout { 2024-09-21T09:22:28.044 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 7, 2024-09-21T09:22:28.044 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "initialized": true 2024-09-21T09:22:28.044 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout } 2024-09-21T09:22:28.044 INFO:teuthology.orchestra.run.smithi022.stdout:mgr epoch 5 is available 2024-09-21T09:22:28.044 INFO:teuthology.orchestra.run.smithi022.stdout:Setting orchestrator backend to cephadm... 2024-09-21T09:22:28.704 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:28 smithi022 ceph-mon[27763]: from='client.14122 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-09-21T09:22:28.704 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:28 smithi022 ceph-mon[27763]: from='client.14122 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-09-21T09:22:29.039 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:28 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:22:28.806+0000 7f4ed78a4640 -1 log_channel(cephadm) log [ERR] : [21/Sep/2024:09:22:28] ENGINE Error in HTTPServer.serve 2024-09-21T09:22:29.039 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:28 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: Traceback (most recent call last): 2024-09-21T09:22:29.039 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:28 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T09:22:29.039 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:28 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: self._connections.run(self.expiration_interval) 2024-09-21T09:22:29.039 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:28 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T09:22:29.039 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:28 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: self._run(expiration_interval) 2024-09-21T09:22:29.040 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:28 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T09:22:29.040 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:28 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T09:22:29.040 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:28 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T09:22:29.040 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:28 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T09:22:29.040 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:28 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T09:22:29.040 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:28 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: s = self.context.wrap_socket( 2024-09-21T09:22:29.040 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:28 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T09:22:29.040 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:28 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: return self.sslsocket_class._create( 2024-09-21T09:22:29.040 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:28 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T09:22:29.040 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:28 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: self.do_handshake() 2024-09-21T09:22:29.040 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:28 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T09:22:29.040 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:28 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: self._sslobj.do_handshake() 2024-09-21T09:22:29.040 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:28 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T09:22:29.040 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:28 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:22:29.556 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:29 smithi022 ceph-mon[27763]: mgrmap e8: y(active, since 2s) 2024-09-21T09:22:29.556 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:29 smithi022 ceph-mon[27763]: [21/Sep/2024:09:22:28] ENGINE Bus STARTING 2024-09-21T09:22:29.556 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:29 smithi022 ceph-mon[27763]: from='client.14130 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:22:29.556 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:29 smithi022 ceph-mon[27763]: from='mgr.14118 172.21.15.22:0/4059432701' entity='mgr.y' 2024-09-21T09:22:29.557 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:29 smithi022 ceph-mon[27763]: from='mgr.14118 172.21.15.22:0/4059432701' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:22:29.557 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:29 smithi022 ceph-mon[27763]: [21/Sep/2024:09:22:28] ENGINE Serving on https://172.21.15.22:7150 2024-09-21T09:22:29.557 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:29 smithi022 ceph-mon[27763]: [21/Sep/2024:09:22:28] ENGINE Bus STARTED 2024-09-21T09:22:29.557 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:29 smithi022 ceph-mon[27763]: [21/Sep/2024:09:22:28] ENGINE Error in HTTPServer.serve 2024-09-21T09:22:29.557 INFO:journalctl@ceph.mon.a.smithi022.stdout: Traceback (most recent call last): 2024-09-21T09:22:29.557 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T09:22:29.557 INFO:journalctl@ceph.mon.a.smithi022.stdout: self._connections.run(self.expiration_interval) 2024-09-21T09:22:29.557 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T09:22:29.557 INFO:journalctl@ceph.mon.a.smithi022.stdout: self._run(expiration_interval) 2024-09-21T09:22:29.557 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T09:22:29.558 INFO:journalctl@ceph.mon.a.smithi022.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T09:22:29.558 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T09:22:29.558 INFO:journalctl@ceph.mon.a.smithi022.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T09:22:29.558 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T09:22:29.558 INFO:journalctl@ceph.mon.a.smithi022.stdout: s = self.context.wrap_socket( 2024-09-21T09:22:29.558 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T09:22:29.558 INFO:journalctl@ceph.mon.a.smithi022.stdout: return self.sslsocket_class._create( 2024-09-21T09:22:29.558 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T09:22:29.558 INFO:journalctl@ceph.mon.a.smithi022.stdout: self.do_handshake() 2024-09-21T09:22:29.558 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T09:22:29.558 INFO:journalctl@ceph.mon.a.smithi022.stdout: self._sslobj.do_handshake() 2024-09-21T09:22:29.558 INFO:journalctl@ceph.mon.a.smithi022.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T09:22:29.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:29 smithi022 ceph-mon[27763]: from='mgr.14118 172.21.15.22:0/4059432701' entity='mgr.y' 2024-09-21T09:22:29.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:29 smithi022 ceph-mon[27763]: from='mgr.14118 172.21.15.22:0/4059432701' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:22:30.506 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout value unchanged 2024-09-21T09:22:30.506 INFO:teuthology.orchestra.run.smithi022.stdout:Generating ssh key... 2024-09-21T09:22:30.762 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:30 smithi022 ceph-mon[27763]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:22:31.668 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:31 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: Generating public/private rsa key pair. 2024-09-21T09:22:31.668 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:31 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: Your identification has been saved in /tmp/tmpjivoug5h/key 2024-09-21T09:22:31.668 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:31 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: Your public key has been saved in /tmp/tmpjivoug5h/key.pub 2024-09-21T09:22:31.668 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:31 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: The key fingerprint is: 2024-09-21T09:22:31.668 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:31 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: SHA256:blVHLFy04qeCMSUCJsKJ8LetwMoxQ9mRNvnKzeNMcSk ceph-d65e801e-77fa-11ef-baf6-efdc52797490 2024-09-21T09:22:31.668 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:31 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: The key's randomart image is: 2024-09-21T09:22:31.668 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:31 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: +---[RSA 3072]----+ 2024-09-21T09:22:31.668 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:31 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: |* o.= . ++ | 2024-09-21T09:22:31.668 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:31 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: |.=oB.. o... | 2024-09-21T09:22:31.668 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:31 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: | oo.+ . o . o.o | 2024-09-21T09:22:31.668 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:31 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: |.. . E + o o o | 2024-09-21T09:22:31.668 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:31 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: | ++ = = S . . . | 2024-09-21T09:22:31.668 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:31 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: |..++ * . = o | 2024-09-21T09:22:31.669 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:31 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: |.. = . + . . | 2024-09-21T09:22:31.669 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:31 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: | o . . | 2024-09-21T09:22:31.669 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:31 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: | | 2024-09-21T09:22:31.669 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:31 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: +----[SHA256]-----+ 2024-09-21T09:22:32.540 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:32 smithi022 ceph-mon[27763]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:22:32.540 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:32 smithi022 ceph-mon[27763]: Generating ssh key... 2024-09-21T09:22:32.540 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:32 smithi022 ceph-mon[27763]: from='mgr.14118 172.21.15.22:0/4059432701' entity='mgr.y' 2024-09-21T09:22:32.540 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:32 smithi022 ceph-mon[27763]: from='mgr.14118 172.21.15.22:0/4059432701' entity='mgr.y' 2024-09-21T09:22:33.092 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCgtj26TUUPKZfHHYW2674nRj6bLjTmZXKFQDPqTDJcuWAx/brs2+q6sRqH+0T8ZOflWLc3iKe82vSid3oGp2O7xGj/XuXwWAR5bvNTx7zHAJDB3+oxR/PCs4kkmBlIc2rsLYWyk0iY4IgiOw99DTE0BaZ5Ino089oWjDqU3S3fv0hlONoWClicHLHEnW3emQh9kY7C40xNSoSD8jECioSYorATTDqMEYR5G+1zLn3Jnlg0E6YyETLFXolyDcKs2zZIe1z9/hSNq6wm0Wfg3cgAKHb0/Wmv4wtOL7dv4iseK7V1fz+LuB/txX6OdXElDSA2zwM/LjIPhtKSE2uJ4uJ1X1VafwuEhvlNaIpTASiLWuOVR4c01/CDWCSdCfB/ruGsRSq2pVPluHKcKIISPiqdv/Grj1GJfGYpCn9WGS0CeLN5v+n02ZZG8b7vIUW8IFBwqI77ta6CjLeCTm13Lab6GvrZ5zhQ95ZYPy/T0fKtSWB3OETqe1qmx177jPwamKM= ceph-d65e801e-77fa-11ef-baf6-efdc52797490 2024-09-21T09:22:33.093 INFO:teuthology.orchestra.run.smithi022.stdout:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-09-21T09:22:33.093 INFO:teuthology.orchestra.run.smithi022.stdout:Adding key to root@localhost authorized_keys... 2024-09-21T09:22:33.100 INFO:teuthology.orchestra.run.smithi022.stdout:Adding host smithi022... 2024-09-21T09:22:33.807 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:33 smithi022 ceph-mon[27763]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:22:34.433 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:34 smithi022 ceph-mon[27763]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi022", "addr": "172.21.15.22", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:22:35.670 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:35 smithi022 ceph-mon[27763]: Deploying cephadm binary to smithi022 2024-09-21T09:22:37.726 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout Added host 'smithi022' with addr '172.21.15.22' 2024-09-21T09:22:37.726 INFO:teuthology.orchestra.run.smithi022.stdout:Deploying unmanaged mon service... 2024-09-21T09:22:38.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:38 smithi022 ceph-mon[27763]: from='mgr.14118 172.21.15.22:0/4059432701' entity='mgr.y' 2024-09-21T09:22:38.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:38 smithi022 ceph-mon[27763]: Added host smithi022 2024-09-21T09:22:38.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:38 smithi022 ceph-mon[27763]: from='mgr.14118 172.21.15.22:0/4059432701' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:22:39.725 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:39 smithi022 ceph-mon[27763]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:22:39.725 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:39 smithi022 ceph-mon[27763]: Saving service mon spec with placement count:5 2024-09-21T09:22:39.725 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:39 smithi022 ceph-mon[27763]: from='mgr.14118 172.21.15.22:0/4059432701' entity='mgr.y' 2024-09-21T09:22:40.181 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout Scheduled mon update... 2024-09-21T09:22:40.181 INFO:teuthology.orchestra.run.smithi022.stdout:Deploying unmanaged mgr service... 2024-09-21T09:22:41.057 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:40 smithi022 ceph-mon[27763]: from='mgr.14118 172.21.15.22:0/4059432701' entity='mgr.y' 2024-09-21T09:22:41.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:40 smithi022 ceph-mon[27763]: from='mgr.14118 172.21.15.22:0/4059432701' entity='mgr.y' 2024-09-21T09:22:42.154 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:41 smithi022 ceph-mon[27763]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:22:42.154 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:41 smithi022 ceph-mon[27763]: Saving service mgr spec with placement count:2 2024-09-21T09:22:42.154 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:41 smithi022 ceph-mon[27763]: from='mgr.14118 172.21.15.22:0/4059432701' entity='mgr.y' 2024-09-21T09:22:42.154 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:41 smithi022 ceph-mon[27763]: from='mgr.14118 172.21.15.22:0/4059432701' entity='mgr.y' 2024-09-21T09:22:42.156 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout Scheduled mgr update... 2024-09-21T09:22:43.700 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:43 smithi022 ceph-mon[27763]: from='mgr.14118 172.21.15.22:0/4059432701' entity='mgr.y' 2024-09-21T09:22:43.700 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:43 smithi022 ceph-mon[27763]: from='client.? 172.21.15.22:0/3353182403' entity='client.admin' 2024-09-21T09:22:44.720 INFO:teuthology.orchestra.run.smithi022.stdout:Enabling the dashboard module... 2024-09-21T09:22:45.557 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:45 smithi022 ceph-mon[27763]: from='client.? 172.21.15.22:0/2856295511' entity='client.admin' 2024-09-21T09:22:46.585 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:46 smithi022 ceph-mon[27763]: from='client.? 172.21.15.22:0/1516635890' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-09-21T09:22:46.585 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:46 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: ignoring --setuser ceph since I am not root 2024-09-21T09:22:46.586 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:46 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: ignoring --setgroup ceph since I am not root 2024-09-21T09:22:46.586 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:46 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:22:46.453+0000 7fc298da2ec0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-21T09:22:46.586 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:46 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:22:46.510+0000 7fc298da2ec0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-21T09:22:46.586 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:46 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:22:46.583+0000 7fc298da2ec0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-21T09:22:46.856 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:46 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:22:46.712+0000 7fc298da2ec0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-21T09:22:46.856 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:46 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:22:46.854+0000 7fc298da2ec0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-21T09:22:47.369 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:47 smithi022 ceph-mon[27763]: from='client.? 172.21.15.22:0/1516635890' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-09-21T09:22:47.369 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:47 smithi022 ceph-mon[27763]: mgrmap e9: y(active, since 19s) 2024-09-21T09:22:47.369 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:47 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:22:47.131+0000 7fc298da2ec0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-21T09:22:47.369 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:47 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:22:47.193+0000 7fc298da2ec0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-21T09:22:47.667 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:47 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:22:47.367+0000 7fc298da2ec0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-21T09:22:48.245 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout { 2024-09-21T09:22:48.303 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "epoch": 9, 2024-09-21T09:22:48.303 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "available": true, 2024-09-21T09:22:48.303 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "active_name": "y", 2024-09-21T09:22:48.303 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "num_standby": 0 2024-09-21T09:22:48.303 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout } 2024-09-21T09:22:48.303 INFO:teuthology.orchestra.run.smithi022.stdout:Waiting for the mgr to restart... 2024-09-21T09:22:48.303 INFO:teuthology.orchestra.run.smithi022.stdout:Waiting for mgr epoch 9... 2024-09-21T09:22:48.492 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:48 smithi022 ceph-mon[27763]: from='client.? 172.21.15.22:0/1424980127' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-09-21T09:22:48.493 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:48 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:22:48.260+0000 7fc298da2ec0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-21T09:22:48.493 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:48 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:22:48.329+0000 7fc298da2ec0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-21T09:22:49.382 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:49 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:22:49.077+0000 7fc298da2ec0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-21T09:22:49.382 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:49 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:22:49.193+0000 7fc298da2ec0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-21T09:22:49.382 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:49 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:22:49.318+0000 7fc298da2ec0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-21T09:22:49.382 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:49 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:22:49.380+0000 7fc298da2ec0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-21T09:22:49.642 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:49 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:22:49.511+0000 7fc298da2ec0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-21T09:22:49.643 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:49 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:22:49.580+0000 7fc298da2ec0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-21T09:22:49.643 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:49 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:22:49.640+0000 7fc298da2ec0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-21T09:22:49.916 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:49 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:22:49.780+0000 7fc298da2ec0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-21T09:22:49.916 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:49 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:22:49.846+0000 7fc298da2ec0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-21T09:22:49.916 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:49 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:22:49.914+0000 7fc298da2ec0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-21T09:22:50.307 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:49 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:22:49.998+0000 7fc298da2ec0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-21T09:22:50.308 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:50 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:22:50.057+0000 7fc298da2ec0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-21T09:22:50.308 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:50 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:22:50.116+0000 7fc298da2ec0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-21T09:22:50.910 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:50 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:22:50.908+0000 7fc298da2ec0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-21T09:22:51.163 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:50 smithi022 ceph-mon[27763]: Active manager daemon y restarted 2024-09-21T09:22:51.164 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:50 smithi022 ceph-mon[27763]: Activating manager daemon y 2024-09-21T09:22:51.164 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:50 smithi022 ceph-mon[27763]: osdmap e3: 0 total, 0 up, 0 in 2024-09-21T09:22:51.164 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:50 smithi022 ceph-mon[27763]: mgrmap e10: y(active, starting, since 0.00348079s) 2024-09-21T09:22:51.164 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:50 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:22:51.164 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:50 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-21T09:22:51.164 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:50 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T09:22:51.164 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:50 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T09:22:51.164 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:50 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T09:22:51.164 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:50 smithi022 ceph-mon[27763]: Manager daemon y is now available 2024-09-21T09:22:51.164 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:50 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:22:51.164 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:50 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-21T09:22:51.164 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:50 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-21T09:22:52.912 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout { 2024-09-21T09:22:52.913 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 11, 2024-09-21T09:22:52.913 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout "initialized": true 2024-09-21T09:22:52.913 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout } 2024-09-21T09:22:52.913 INFO:teuthology.orchestra.run.smithi022.stdout:mgr epoch 9 is available 2024-09-21T09:22:52.913 INFO:teuthology.orchestra.run.smithi022.stdout:Generating a dashboard self-signed certificate... 2024-09-21T09:22:53.165 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:52 smithi022 ceph-mon[27763]: from='client.14154 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-09-21T09:22:53.165 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:52 smithi022 ceph-mon[27763]: mgrmap e11: y(active, since 1.00558s) 2024-09-21T09:22:53.166 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:52 smithi022 ceph-mon[27763]: from='client.14154 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-09-21T09:22:53.166 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:52 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:22:53.558 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:53 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:22:53.300+0000 7fc249900640 -1 log_channel(cephadm) log [ERR] : [21/Sep/2024:09:22:53] ENGINE Error in HTTPServer.serve 2024-09-21T09:22:53.558 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:53 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: Traceback (most recent call last): 2024-09-21T09:22:53.558 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:53 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T09:22:53.559 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:53 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: self._connections.run(self.expiration_interval) 2024-09-21T09:22:53.559 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:53 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T09:22:53.559 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:53 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: self._run(expiration_interval) 2024-09-21T09:22:53.559 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:53 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T09:22:53.559 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:53 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T09:22:53.559 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:53 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T09:22:53.559 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:53 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T09:22:53.559 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:53 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T09:22:53.559 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:53 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: s = self.context.wrap_socket( 2024-09-21T09:22:53.559 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:53 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T09:22:53.559 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:53 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: return self.sslsocket_class._create( 2024-09-21T09:22:53.559 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:53 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T09:22:53.559 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:53 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: self.do_handshake() 2024-09-21T09:22:53.559 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:53 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T09:22:53.560 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:53 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: self._sslobj.do_handshake() 2024-09-21T09:22:53.560 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:53 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T09:22:53.560 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:22:53 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:22:54.260 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout Self-signed certificate created 2024-09-21T09:22:54.260 INFO:teuthology.orchestra.run.smithi022.stdout:Creating initial admin user... 2024-09-21T09:22:54.557 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:54 smithi022 ceph-mon[27763]: [21/Sep/2024:09:22:53] ENGINE Bus STARTING 2024-09-21T09:22:54.557 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:54 smithi022 ceph-mon[27763]: [21/Sep/2024:09:22:53] ENGINE Serving on https://172.21.15.22:7150 2024-09-21T09:22:54.557 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:54 smithi022 ceph-mon[27763]: [21/Sep/2024:09:22:53] ENGINE Bus STARTED 2024-09-21T09:22:54.557 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:54 smithi022 ceph-mon[27763]: [21/Sep/2024:09:22:53] ENGINE Error in HTTPServer.serve 2024-09-21T09:22:54.557 INFO:journalctl@ceph.mon.a.smithi022.stdout: Traceback (most recent call last): 2024-09-21T09:22:54.557 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T09:22:54.557 INFO:journalctl@ceph.mon.a.smithi022.stdout: self._connections.run(self.expiration_interval) 2024-09-21T09:22:54.557 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T09:22:54.557 INFO:journalctl@ceph.mon.a.smithi022.stdout: self._run(expiration_interval) 2024-09-21T09:22:54.558 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T09:22:54.558 INFO:journalctl@ceph.mon.a.smithi022.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T09:22:54.558 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T09:22:54.558 INFO:journalctl@ceph.mon.a.smithi022.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T09:22:54.558 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T09:22:54.558 INFO:journalctl@ceph.mon.a.smithi022.stdout: s = self.context.wrap_socket( 2024-09-21T09:22:54.558 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T09:22:54.558 INFO:journalctl@ceph.mon.a.smithi022.stdout: return self.sslsocket_class._create( 2024-09-21T09:22:54.558 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T09:22:54.558 INFO:journalctl@ceph.mon.a.smithi022.stdout: self.do_handshake() 2024-09-21T09:22:54.558 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T09:22:54.558 INFO:journalctl@ceph.mon.a.smithi022.stdout: self._sslobj.do_handshake() 2024-09-21T09:22:54.558 INFO:journalctl@ceph.mon.a.smithi022.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T09:22:54.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:54 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:22:54.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:54 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:22:54.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:54 smithi022 ceph-mon[27763]: mgrmap e12: y(active, since 2s) 2024-09-21T09:22:54.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:54 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:22:54.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:54 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:22:55.557 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:55 smithi022 ceph-mon[27763]: from='client.14162 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:22:55.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:55 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:22:56.139 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout {"username": "admin", "password": "$2b$12$bd906w9Wy5CjU0uOHpHeAeKl7mFFiOTU6u4NkERJKS.eWFyTwglTi", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1726910575, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-09-21T09:22:56.139 INFO:teuthology.orchestra.run.smithi022.stdout:Fetching dashboard port number... 2024-09-21T09:22:56.388 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:56 smithi022 ceph-mon[27763]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:22:57.557 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:57 smithi022 ceph-mon[27763]: from='client.? 172.21.15.22:0/324845974' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-09-21T09:22:57.857 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stdout 8443 2024-09-21T09:22:57.867 INFO:teuthology.orchestra.run.smithi022.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-09-21T09:22:57.867 INFO:teuthology.orchestra.run.smithi022.stdout:systemctl: stdout disabled 2024-09-21T09:22:57.876 INFO:teuthology.orchestra.run.smithi022.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-09-21T09:22:57.876 INFO:teuthology.orchestra.run.smithi022.stdout:systemctl: stdout inactive 2024-09-21T09:22:57.876 INFO:teuthology.orchestra.run.smithi022.stdout:firewalld.service is not enabled 2024-09-21T09:22:57.876 INFO:teuthology.orchestra.run.smithi022.stdout:Not possible to open ports <[8443]>. firewalld.service is not available 2024-09-21T09:22:57.878 INFO:teuthology.orchestra.run.smithi022.stdout:Ceph Dashboard is now available at: 2024-09-21T09:22:57.878 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-09-21T09:22:57.878 INFO:teuthology.orchestra.run.smithi022.stdout: URL: https://smithi022.front.sepia.ceph.com:8443/ 2024-09-21T09:22:57.878 INFO:teuthology.orchestra.run.smithi022.stdout: User: admin 2024-09-21T09:22:57.878 INFO:teuthology.orchestra.run.smithi022.stdout: Password: gh7nno7zpb 2024-09-21T09:22:57.878 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-09-21T09:22:57.879 INFO:teuthology.orchestra.run.smithi022.stdout:Saving cluster configuration to /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/config directory 2024-09-21T09:22:57.879 INFO:teuthology.orchestra.run.smithi022.stdout:Enabling autotune for osd_memory_target 2024-09-21T09:22:58.530 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:22:58 smithi022 ceph-mon[27763]: mgrmap e13: y(active, since 6s) 2024-09-21T09:23:00.444 INFO:teuthology.orchestra.run.smithi022.stdout:/usr/bin/ceph: stderr set mgr/dashboard/cluster/status 2024-09-21T09:23:00.445 INFO:teuthology.orchestra.run.smithi022.stdout:You can access the Ceph CLI as following in case of multi-cluster or non-default config: 2024-09-21T09:23:00.445 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-09-21T09:23:00.445 INFO:teuthology.orchestra.run.smithi022.stdout: sudo /home/ubuntu/cephtest/cephadm shell --fsid d65e801e-77fa-11ef-baf6-efdc52797490 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-09-21T09:23:00.445 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-09-21T09:23:00.445 INFO:teuthology.orchestra.run.smithi022.stdout:Or, if you are only running a single cluster on this host: 2024-09-21T09:23:00.445 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-09-21T09:23:00.445 INFO:teuthology.orchestra.run.smithi022.stdout: sudo /home/ubuntu/cephtest/cephadm shell 2024-09-21T09:23:00.445 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-09-21T09:23:00.445 INFO:teuthology.orchestra.run.smithi022.stdout:Please consider enabling telemetry to help improve Ceph: 2024-09-21T09:23:00.445 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-09-21T09:23:00.445 INFO:teuthology.orchestra.run.smithi022.stdout: ceph telemetry on 2024-09-21T09:23:00.445 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-09-21T09:23:00.445 INFO:teuthology.orchestra.run.smithi022.stdout:For more information see: 2024-09-21T09:23:00.446 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-09-21T09:23:00.446 INFO:teuthology.orchestra.run.smithi022.stdout: https://docs.ceph.com/docs/master/mgr/telemetry/ 2024-09-21T09:23:00.446 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-09-21T09:23:00.446 INFO:teuthology.orchestra.run.smithi022.stdout:Bootstrap complete. 2024-09-21T09:23:00.489 INFO:tasks.cephadm:Fetching config... 2024-09-21T09:23:00.490 DEBUG:teuthology.orchestra.run.smithi022:> set -ex 2024-09-21T09:23:00.490 DEBUG:teuthology.orchestra.run.smithi022:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-09-21T09:23:00.508 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-09-21T09:23:00.508 DEBUG:teuthology.orchestra.run.smithi022:> set -ex 2024-09-21T09:23:00.508 DEBUG:teuthology.orchestra.run.smithi022:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-09-21T09:23:00.564 INFO:tasks.cephadm:Fetching mon keyring... 2024-09-21T09:23:00.565 DEBUG:teuthology.orchestra.run.smithi022:> set -ex 2024-09-21T09:23:00.565 DEBUG:teuthology.orchestra.run.smithi022:> sudo dd if=/var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/mon.a/keyring of=/dev/stdout 2024-09-21T09:23:00.631 INFO:tasks.cephadm:Fetching pub ssh key... 2024-09-21T09:23:00.631 DEBUG:teuthology.orchestra.run.smithi022:> set -ex 2024-09-21T09:23:00.631 DEBUG:teuthology.orchestra.run.smithi022:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-09-21T09:23:00.688 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-09-21T09:23:00.688 DEBUG:teuthology.orchestra.run.smithi022:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCgtj26TUUPKZfHHYW2674nRj6bLjTmZXKFQDPqTDJcuWAx/brs2+q6sRqH+0T8ZOflWLc3iKe82vSid3oGp2O7xGj/XuXwWAR5bvNTx7zHAJDB3+oxR/PCs4kkmBlIc2rsLYWyk0iY4IgiOw99DTE0BaZ5Ino089oWjDqU3S3fv0hlONoWClicHLHEnW3emQh9kY7C40xNSoSD8jECioSYorATTDqMEYR5G+1zLn3Jnlg0E6YyETLFXolyDcKs2zZIe1z9/hSNq6wm0Wfg3cgAKHb0/Wmv4wtOL7dv4iseK7V1fz+LuB/txX6OdXElDSA2zwM/LjIPhtKSE2uJ4uJ1X1VafwuEhvlNaIpTASiLWuOVR4c01/CDWCSdCfB/ruGsRSq2pVPluHKcKIISPiqdv/Grj1GJfGYpCn9WGS0CeLN5v+n02ZZG8b7vIUW8IFBwqI77ta6CjLeCTm13Lab6GvrZ5zhQ95ZYPy/T0fKtSWB3OETqe1qmx177jPwamKM= ceph-d65e801e-77fa-11ef-baf6-efdc52797490' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-09-21T09:23:00.776 INFO:teuthology.orchestra.run.smithi022.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCgtj26TUUPKZfHHYW2674nRj6bLjTmZXKFQDPqTDJcuWAx/brs2+q6sRqH+0T8ZOflWLc3iKe82vSid3oGp2O7xGj/XuXwWAR5bvNTx7zHAJDB3+oxR/PCs4kkmBlIc2rsLYWyk0iY4IgiOw99DTE0BaZ5Ino089oWjDqU3S3fv0hlONoWClicHLHEnW3emQh9kY7C40xNSoSD8jECioSYorATTDqMEYR5G+1zLn3Jnlg0E6YyETLFXolyDcKs2zZIe1z9/hSNq6wm0Wfg3cgAKHb0/Wmv4wtOL7dv4iseK7V1fz+LuB/txX6OdXElDSA2zwM/LjIPhtKSE2uJ4uJ1X1VafwuEhvlNaIpTASiLWuOVR4c01/CDWCSdCfB/ruGsRSq2pVPluHKcKIISPiqdv/Grj1GJfGYpCn9WGS0CeLN5v+n02ZZG8b7vIUW8IFBwqI77ta6CjLeCTm13Lab6GvrZ5zhQ95ZYPy/T0fKtSWB3OETqe1qmx177jPwamKM= ceph-d65e801e-77fa-11ef-baf6-efdc52797490 2024-09-21T09:23:00.806 DEBUG:teuthology.orchestra.run.smithi064:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCgtj26TUUPKZfHHYW2674nRj6bLjTmZXKFQDPqTDJcuWAx/brs2+q6sRqH+0T8ZOflWLc3iKe82vSid3oGp2O7xGj/XuXwWAR5bvNTx7zHAJDB3+oxR/PCs4kkmBlIc2rsLYWyk0iY4IgiOw99DTE0BaZ5Ino089oWjDqU3S3fv0hlONoWClicHLHEnW3emQh9kY7C40xNSoSD8jECioSYorATTDqMEYR5G+1zLn3Jnlg0E6YyETLFXolyDcKs2zZIe1z9/hSNq6wm0Wfg3cgAKHb0/Wmv4wtOL7dv4iseK7V1fz+LuB/txX6OdXElDSA2zwM/LjIPhtKSE2uJ4uJ1X1VafwuEhvlNaIpTASiLWuOVR4c01/CDWCSdCfB/ruGsRSq2pVPluHKcKIISPiqdv/Grj1GJfGYpCn9WGS0CeLN5v+n02ZZG8b7vIUW8IFBwqI77ta6CjLeCTm13Lab6GvrZ5zhQ95ZYPy/T0fKtSWB3OETqe1qmx177jPwamKM= ceph-d65e801e-77fa-11ef-baf6-efdc52797490' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-09-21T09:23:00.860 INFO:teuthology.orchestra.run.smithi064.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCgtj26TUUPKZfHHYW2674nRj6bLjTmZXKFQDPqTDJcuWAx/brs2+q6sRqH+0T8ZOflWLc3iKe82vSid3oGp2O7xGj/XuXwWAR5bvNTx7zHAJDB3+oxR/PCs4kkmBlIc2rsLYWyk0iY4IgiOw99DTE0BaZ5Ino089oWjDqU3S3fv0hlONoWClicHLHEnW3emQh9kY7C40xNSoSD8jECioSYorATTDqMEYR5G+1zLn3Jnlg0E6YyETLFXolyDcKs2zZIe1z9/hSNq6wm0Wfg3cgAKHb0/Wmv4wtOL7dv4iseK7V1fz+LuB/txX6OdXElDSA2zwM/LjIPhtKSE2uJ4uJ1X1VafwuEhvlNaIpTASiLWuOVR4c01/CDWCSdCfB/ruGsRSq2pVPluHKcKIISPiqdv/Grj1GJfGYpCn9WGS0CeLN5v+n02ZZG8b7vIUW8IFBwqI77ta6CjLeCTm13Lab6GvrZ5zhQ95ZYPy/T0fKtSWB3OETqe1qmx177jPwamKM= ceph-d65e801e-77fa-11ef-baf6-efdc52797490 2024-09-21T09:23:00.873 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d65e801e-77fa-11ef-baf6-efdc52797490 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-09-21T09:23:01.044 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:00 smithi022 ceph-mon[27763]: from='client.? 172.21.15.22:0/180381826' entity='client.admin' 2024-09-21T09:23:01.125 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:23:03.900 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2024-09-21T09:23:03.900 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d65e801e-77fa-11ef-baf6-efdc52797490 -- ceph orch client-keyring set client.admin '*' --mode 0755 2024-09-21T09:23:04.139 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:04 smithi022 ceph-mon[27763]: from='client.? 172.21.15.22:0/830494254' entity='client.admin' 2024-09-21T09:23:04.376 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:23:07.057 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:06 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:07.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:06 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:07.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:06 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi022", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:23:07.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:06 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:07.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:06 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:23:07.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:06 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:07.578 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi064 2024-09-21T09:23:07.578 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-09-21T09:23:07.578 DEBUG:teuthology.orchestra.run.smithi064:> dd of=/etc/ceph/ceph.conf 2024-09-21T09:23:07.598 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-09-21T09:23:07.598 DEBUG:teuthology.orchestra.run.smithi064:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:23:07.657 INFO:tasks.cephadm:Adding host smithi064 to orchestrator... 2024-09-21T09:23:07.657 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d65e801e-77fa-11ef-baf6-efdc52797490 -- ceph orch host add smithi064 2024-09-21T09:23:07.907 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:23:08.129 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:07 smithi022 ceph-mon[27763]: from='client.14174 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:23:08.129 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:07 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:08.129 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:07 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:23:08.129 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:07 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:23:08.129 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:07 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:23:08.129 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:07 smithi022 ceph-mon[27763]: Updating smithi022:/etc/ceph/ceph.conf 2024-09-21T09:23:08.129 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:07 smithi022 ceph-mon[27763]: Updating smithi022:/var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:23:08.130 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:07 smithi022 ceph-mon[27763]: Updating smithi022:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:23:08.130 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:07 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:08.130 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:07 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:08.130 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:07 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:08.957 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:08 smithi022 ceph-mon[27763]: Updating smithi022:/var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:23:10.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:10 smithi022 ceph-mon[27763]: from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi064", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:23:12.307 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:11 smithi022 ceph-mon[27763]: Deploying cephadm binary to smithi064 2024-09-21T09:23:12.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:11 smithi022 ceph-mon[27763]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:23:12.954 INFO:teuthology.orchestra.run.smithi022.stdout:Added host 'smithi064' with addr '172.21.15.64' 2024-09-21T09:23:13.583 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d65e801e-77fa-11ef-baf6-efdc52797490 -- ceph orch host ls --format=json 2024-09-21T09:23:13.833 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:23:14.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:13 smithi022 ceph-mon[27763]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:23:14.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:13 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:14.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:13 smithi022 ceph-mon[27763]: Added host smithi064 2024-09-21T09:23:14.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:13 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:23:14.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:13 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:14.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:13 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:15.523 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-09-21T09:23:15.523 INFO:teuthology.orchestra.run.smithi022.stdout:[{"addr": "172.21.15.22", "hostname": "smithi022", "labels": [], "status": ""}, {"addr": "172.21.15.64", "hostname": "smithi064", "labels": [], "status": ""}] 2024-09-21T09:23:15.523 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:15 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:16.134 INFO:tasks.cephadm:Setting crush tunables to default 2024-09-21T09:23:16.134 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d65e801e-77fa-11ef-baf6-efdc52797490 -- ceph osd crush tunables default 2024-09-21T09:23:16.381 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:23:16.501 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:16 smithi022 ceph-mon[27763]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:23:17.557 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:17 smithi022 ceph-mon[27763]: from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:23:17.557 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:17 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:17.557 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:17 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:17.557 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:17 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:17.557 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:17 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:17.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:17 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi064", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:23:17.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:17 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:23:17.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:17 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:23:18.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:18 smithi022 ceph-mon[27763]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:23:18.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:18 smithi022 ceph-mon[27763]: Updating smithi064:/etc/ceph/ceph.conf 2024-09-21T09:23:18.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:18 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:18.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:18 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:18.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:18 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:18.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:18 smithi022 ceph-mon[27763]: from='client.? 172.21.15.22:0/1710393031' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-09-21T09:23:19.071 INFO:teuthology.orchestra.run.smithi022.stderr:adjusted tunables profile to default 2024-09-21T09:23:19.640 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:19 smithi022 ceph-mon[27763]: Updating smithi064:/var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:23:19.640 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:19 smithi022 ceph-mon[27763]: Updating smithi064:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:23:19.640 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:19 smithi022 ceph-mon[27763]: Updating smithi064:/var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:23:19.640 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:19 smithi022 ceph-mon[27763]: from='client.? 172.21.15.22:0/1710393031' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-09-21T09:23:19.640 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:19 smithi022 ceph-mon[27763]: osdmap e4: 0 total, 0 up, 0 in 2024-09-21T09:23:19.671 INFO:tasks.cephadm:Adding mon.a on smithi022 2024-09-21T09:23:19.671 INFO:tasks.cephadm:Adding mon.c on smithi022 2024-09-21T09:23:19.671 INFO:tasks.cephadm:Adding mon.b on smithi064 2024-09-21T09:23:19.671 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d65e801e-77fa-11ef-baf6-efdc52797490 -- ceph orch apply mon '3;smithi022:172.21.15.22=a;smithi022:[v2:172.21.15.22:3301,v1:172.21.15.22:6790]=c;smithi064:172.21.15.64=b' 2024-09-21T09:23:19.927 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:23:20.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:20 smithi022 ceph-mon[27763]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:23:21.684 INFO:teuthology.orchestra.run.smithi064.stdout:Scheduled mon update... 2024-09-21T09:23:22.247 DEBUG:teuthology.orchestra.run.smithi022:mon.c> sudo journalctl -f -n 0 -u ceph-d65e801e-77fa-11ef-baf6-efdc52797490@mon.c.service 2024-09-21T09:23:22.250 DEBUG:teuthology.orchestra.run.smithi064:mon.b> sudo journalctl -f -n 0 -u ceph-d65e801e-77fa-11ef-baf6-efdc52797490@mon.b.service 2024-09-21T09:23:22.252 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-09-21T09:23:22.252 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d65e801e-77fa-11ef-baf6-efdc52797490 -- ceph mon dump -f json 2024-09-21T09:23:22.557 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:22 smithi022 ceph-mon[27763]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:23:22.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:22 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:22.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:22 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:23:22.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:22 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:23:22.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:22 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:23:22.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:22 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:22.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:22 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T09:23:22.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:22 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:23:22.994 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:23:23.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:23 smithi022 ceph-mon[27763]: from='client.14182 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi022:172.21.15.22=a;smithi022:[v2:172.21.15.22:3301,v1:172.21.15.22:6790]=c;smithi064:172.21.15.64=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:23:23.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:23 smithi022 ceph-mon[27763]: Saving service mon spec with placement smithi022:172.21.15.22=a;smithi022:[v2:172.21.15.22:3301,v1:172.21.15.22:6790]=c;smithi064:172.21.15.64=b;count:3 2024-09-21T09:23:23.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:23 smithi022 ceph-mon[27763]: Deploying daemon mon.b on smithi064 2024-09-21T09:23:24.807 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:24 smithi022 ceph-mon[27763]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:23:25.211 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-09-21T09:23:25.211 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":1,"fsid":"d65e801e-77fa-11ef-baf6-efdc52797490","modified":"2024-09-21T09:22:01.045656Z","created":"2024-09-21T09:22:01.045656Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:3300","nonce":0},{"type":"v1","addr":"172.21.15.22:6789","nonce":0}]},"addr":"172.21.15.22:6789/0","public_addr":"172.21.15.22:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-21T09:23:25.212 INFO:teuthology.orchestra.run.smithi064.stderr:dumped monmap epoch 1 2024-09-21T09:23:25.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:25 smithi022 ceph-mon[27763]: from='client.? 172.21.15.64:0/3562629943' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-21T09:23:26.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:26 smithi022 ceph-mon[27763]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:23:27.309 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 podman[31086]: 2024-09-21 09:23:27.10599297 +0000 UTC m=+0.016206725 image pull fc3ec4a4f6be575045de0fd8a2fe84939e888152c447bdedb90add7841f7bef9 quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2 2024-09-21T09:23:27.647 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-09-21T09:23:27.647 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d65e801e-77fa-11ef-baf6-efdc52797490 -- ceph mon dump -f json 2024-09-21T09:23:27.672 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 podman[31086]: 2024-09-21 09:23:27.413290553 +0000 UTC m=+0.323504297 container init ed126533cd64ff6a282f3cb73136801422203655f4f16fbb5683e4986c899996 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mon-b, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, RELEASE=quincy-fbd1a6a, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-09-21T09:23:27.672 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 podman[31086]: 2024-09-21 09:23:27.417873736 +0000 UTC m=+0.328087480 container start ed126533cd64ff6a282f3cb73136801422203655f4f16fbb5683e4986c899996 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mon-b, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=quincy-fbd1a6a, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_CLEAN=True, ceph=True, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731) 2024-09-21T09:23:27.672 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: set uid:gid to 167:167 (ceph:ceph) 2024-09-21T09:23:27.672 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable), process ceph-mon, pid 7 2024-09-21T09:23:27.672 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: pidfile_write: ignore empty --pid-file 2024-09-21T09:23:27.672 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: load: jerasure load: lrc 2024-09-21T09:23:27.672 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: RocksDB version: 6.15.5 2024-09-21T09:23:27.672 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-09-21T09:23:27.672 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Compile date Sep 20 2024 2024-09-21T09:23:27.672 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: DB SUMMARY 2024-09-21T09:23:27.673 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: DB Session ID: ODUAUJ3WNKP803OB4HBH 2024-09-21T09:23:27.673 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: CURRENT file: CURRENT 2024-09-21T09:23:27.673 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: IDENTITY file: IDENTITY 2024-09-21T09:23:27.673 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2024-09-21T09:23:27.673 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 0, files: 2024-09-21T09:23:27.673 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000004.log size: 511 ; 2024-09-21T09:23:27.673 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.error_if_exists: 0 2024-09-21T09:23:27.673 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.create_if_missing: 0 2024-09-21T09:23:27.673 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.paranoid_checks: 1 2024-09-21T09:23:27.673 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-21T09:23:27.673 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.env: 0x5643016e7440 2024-09-21T09:23:27.673 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.fs: Posix File System 2024-09-21T09:23:27.673 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.info_log: 0x564303da6ec0 2024-09-21T09:23:27.673 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.max_file_opening_threads: 16 2024-09-21T09:23:27.674 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.statistics: (nil) 2024-09-21T09:23:27.674 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.use_fsync: 0 2024-09-21T09:23:27.674 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.max_log_file_size: 0 2024-09-21T09:23:27.674 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-21T09:23:27.674 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-21T09:23:27.674 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.keep_log_file_num: 1000 2024-09-21T09:23:27.675 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.recycle_log_file_num: 0 2024-09-21T09:23:27.675 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.allow_fallocate: 1 2024-09-21T09:23:27.675 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.allow_mmap_reads: 0 2024-09-21T09:23:27.675 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.allow_mmap_writes: 0 2024-09-21T09:23:27.675 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.use_direct_reads: 0 2024-09-21T09:23:27.675 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-21T09:23:27.675 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.create_missing_column_families: 0 2024-09-21T09:23:27.675 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.db_log_dir: 2024-09-21T09:23:27.675 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-b/store.db 2024-09-21T09:23:27.675 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-21T09:23:27.675 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-21T09:23:27.675 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-21T09:23:27.675 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-21T09:23:27.676 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-21T09:23:27.676 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-21T09:23:27.676 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.advise_random_on_open: 1 2024-09-21T09:23:27.676 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.db_write_buffer_size: 0 2024-09-21T09:23:27.676 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.write_buffer_manager: 0x564303e647e0 2024-09-21T09:23:27.676 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-21T09:23:27.676 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-09-21T09:23:27.676 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-21T09:23:27.676 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-21T09:23:27.676 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.rate_limiter: (nil) 2024-09-21T09:23:27.676 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-21T09:23:27.676 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.wal_recovery_mode: 2 2024-09-21T09:23:27.676 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.enable_thread_tracking: 0 2024-09-21T09:23:27.676 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.enable_pipelined_write: 0 2024-09-21T09:23:27.677 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.unordered_write: 0 2024-09-21T09:23:27.677 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-21T09:23:27.677 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-21T09:23:27.677 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-21T09:23:27.677 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-21T09:23:27.677 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.row_cache: None 2024-09-21T09:23:27.677 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.wal_filter: None 2024-09-21T09:23:27.677 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-21T09:23:27.677 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.allow_ingest_behind: 0 2024-09-21T09:23:27.677 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.preserve_deletes: 0 2024-09-21T09:23:27.677 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.two_write_queues: 0 2024-09-21T09:23:27.678 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.manual_wal_flush: 0 2024-09-21T09:23:27.678 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.atomic_flush: 0 2024-09-21T09:23:27.678 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-21T09:23:27.678 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-21T09:23:27.678 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-21T09:23:27.678 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.log_readahead_size: 0 2024-09-21T09:23:27.678 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-21T09:23:27.678 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.best_efforts_recovery: 0 2024-09-21T09:23:27.678 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-21T09:23:27.678 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-21T09:23:27.678 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.allow_data_in_errors: 0 2024-09-21T09:23:27.678 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.db_host_id: __hostname__ 2024-09-21T09:23:27.678 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.max_background_jobs: 2 2024-09-21T09:23:27.678 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.max_background_compactions: -1 2024-09-21T09:23:27.679 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.max_subcompactions: 1 2024-09-21T09:23:27.679 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-21T09:23:27.679 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-21T09:23:27.679 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-21T09:23:27.679 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.max_total_wal_size: 0 2024-09-21T09:23:27.679 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-21T09:23:27.679 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-21T09:23:27.679 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-21T09:23:27.679 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-21T09:23:27.679 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.max_open_files: -1 2024-09-21T09:23:27.679 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.bytes_per_sync: 0 2024-09-21T09:23:27.679 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-21T09:23:27.679 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-21T09:23:27.680 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.compaction_readahead_size: 0 2024-09-21T09:23:27.680 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.max_background_flushes: -1 2024-09-21T09:23:27.680 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Compression algorithms supported: 2024-09-21T09:23:27.680 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: kZSTD supported: 0 2024-09-21T09:23:27.680 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: kXpressCompression supported: 0 2024-09-21T09:23:27.680 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: kBZip2Compression supported: 0 2024-09-21T09:23:27.680 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-21T09:23:27.680 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: kLZ4Compression supported: 1 2024-09-21T09:23:27.680 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: kZlibCompression supported: 1 2024-09-21T09:23:27.680 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: kLZ4HCCompression supported: 1 2024-09-21T09:23:27.680 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: kSnappyCompression supported: 1 2024-09-21T09:23:27.680 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-21T09:23:27.680 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000003 2024-09-21T09:23:27.680 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2024-09-21T09:23:27.680 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-21T09:23:27.681 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.merge_operator: 2024-09-21T09:23:27.681 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.compaction_filter: None 2024-09-21T09:23:27.681 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.compaction_filter_factory: None 2024-09-21T09:23:27.681 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.sst_partitioner_factory: None 2024-09-21T09:23:27.681 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-21T09:23:27.681 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-21T09:23:27.681 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x564303d4da38) 2024-09-21T09:23:27.681 INFO:journalctl@ceph.mon.b.smithi064.stdout: cache_index_and_filter_blocks: 1 2024-09-21T09:23:27.681 INFO:journalctl@ceph.mon.b.smithi064.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-21T09:23:27.681 INFO:journalctl@ceph.mon.b.smithi064.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-21T09:23:27.681 INFO:journalctl@ceph.mon.b.smithi064.stdout: pin_top_level_index_and_filter: 1 2024-09-21T09:23:27.681 INFO:journalctl@ceph.mon.b.smithi064.stdout: index_type: 0 2024-09-21T09:23:27.681 INFO:journalctl@ceph.mon.b.smithi064.stdout: data_block_index_type: 0 2024-09-21T09:23:27.681 INFO:journalctl@ceph.mon.b.smithi064.stdout: index_shortening: 1 2024-09-21T09:23:27.681 INFO:journalctl@ceph.mon.b.smithi064.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-21T09:23:27.681 INFO:journalctl@ceph.mon.b.smithi064.stdout: hash_index_allow_collision: 1 2024-09-21T09:23:27.682 INFO:journalctl@ceph.mon.b.smithi064.stdout: checksum: 1 2024-09-21T09:23:27.682 INFO:journalctl@ceph.mon.b.smithi064.stdout: no_block_cache: 0 2024-09-21T09:23:27.682 INFO:journalctl@ceph.mon.b.smithi064.stdout: block_cache: 0x564303d831f0 2024-09-21T09:23:27.682 INFO:journalctl@ceph.mon.b.smithi064.stdout: block_cache_name: BinnedLRUCache 2024-09-21T09:23:27.682 INFO:journalctl@ceph.mon.b.smithi064.stdout: block_cache_options: 2024-09-21T09:23:27.682 INFO:journalctl@ceph.mon.b.smithi064.stdout: capacity : 536870912 2024-09-21T09:23:27.682 INFO:journalctl@ceph.mon.b.smithi064.stdout: num_shard_bits : 4 2024-09-21T09:23:27.682 INFO:journalctl@ceph.mon.b.smithi064.stdout: strict_capacity_limit : 0 2024-09-21T09:23:27.682 INFO:journalctl@ceph.mon.b.smithi064.stdout: high_pri_pool_ratio: 0.000 2024-09-21T09:23:27.682 INFO:journalctl@ceph.mon.b.smithi064.stdout: block_cache_compressed: (nil) 2024-09-21T09:23:27.682 INFO:journalctl@ceph.mon.b.smithi064.stdout: persistent_cache: (nil) 2024-09-21T09:23:27.683 INFO:journalctl@ceph.mon.b.smithi064.stdout: block_size: 4096 2024-09-21T09:23:27.683 INFO:journalctl@ceph.mon.b.smithi064.stdout: block_size_deviation: 10 2024-09-21T09:23:27.683 INFO:journalctl@ceph.mon.b.smithi064.stdout: block_restart_interval: 16 2024-09-21T09:23:27.683 INFO:journalctl@ceph.mon.b.smithi064.stdout: index_block_restart_interval: 1 2024-09-21T09:23:27.683 INFO:journalctl@ceph.mon.b.smithi064.stdout: metadata_block_size: 4096 2024-09-21T09:23:27.683 INFO:journalctl@ceph.mon.b.smithi064.stdout: partition_filters: 0 2024-09-21T09:23:27.683 INFO:journalctl@ceph.mon.b.smithi064.stdout: use_delta_encoding: 1 2024-09-21T09:23:27.683 INFO:journalctl@ceph.mon.b.smithi064.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-09-21T09:23:27.683 INFO:journalctl@ceph.mon.b.smithi064.stdout: whole_key_filtering: 1 2024-09-21T09:23:27.683 INFO:journalctl@ceph.mon.b.smithi064.stdout: verify_compression: 0 2024-09-21T09:23:27.683 INFO:journalctl@ceph.mon.b.smithi064.stdout: read_amp_bytes_per_bit: 0 2024-09-21T09:23:27.683 INFO:journalctl@ceph.mon.b.smithi064.stdout: format_version: 4 2024-09-21T09:23:27.684 INFO:journalctl@ceph.mon.b.smithi064.stdout: enable_index_compression: 1 2024-09-21T09:23:27.684 INFO:journalctl@ceph.mon.b.smithi064.stdout: block_align: 0 2024-09-21T09:23:27.684 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.write_buffer_size: 33554432 2024-09-21T09:23:27.684 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.max_write_buffer_number: 2 2024-09-21T09:23:27.684 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.compression: NoCompression 2024-09-21T09:23:27.684 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.bottommost_compression: Disabled 2024-09-21T09:23:27.684 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.prefix_extractor: nullptr 2024-09-21T09:23:27.685 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-21T09:23:27.685 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.num_levels: 7 2024-09-21T09:23:27.685 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-21T09:23:27.685 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-21T09:23:27.685 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-21T09:23:27.685 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-21T09:23:27.685 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-21T09:23:27.685 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-21T09:23:27.685 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-21T09:23:27.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-21T09:23:27.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-21T09:23:27.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-21T09:23:27.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-21T09:23:27.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.compression_opts.level: 32767 2024-09-21T09:23:27.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.compression_opts.strategy: 0 2024-09-21T09:23:27.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-21T09:23:27.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-21T09:23:27.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-21T09:23:27.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.compression_opts.enabled: false 2024-09-21T09:23:27.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-21T09:23:27.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-21T09:23:27.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-21T09:23:27.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.target_file_size_base: 67108864 2024-09-21T09:23:27.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-21T09:23:27.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-21T09:23:27.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-21T09:23:27.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-21T09:23:27.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-21T09:23:27.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-21T09:23:27.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-21T09:23:27.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-21T09:23:27.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-21T09:23:27.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-21T09:23:27.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-21T09:23:27.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-21T09:23:27.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-21T09:23:27.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.arena_block_size: 4194304 2024-09-21T09:23:27.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-21T09:23:27.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-21T09:23:27.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-09-21T09:23:27.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.disable_auto_compactions: 0 2024-09-21T09:23:27.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-21T09:23:27.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-21T09:23:27.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-21T09:23:27.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-21T09:23:27.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-21T09:23:27.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-21T09:23:27.689 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-21T09:23:27.689 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-21T09:23:27.689 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-21T09:23:27.689 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-21T09:23:27.689 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-21T09:23:27.689 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.inplace_update_support: 0 2024-09-21T09:23:27.689 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-21T09:23:27.689 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-21T09:23:27.689 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-21T09:23:27.689 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-21T09:23:27.689 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.bloom_locality: 0 2024-09-21T09:23:27.689 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.max_successive_merges: 0 2024-09-21T09:23:27.689 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-21T09:23:27.689 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.paranoid_file_checks: 0 2024-09-21T09:23:27.689 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.force_consistency_checks: 1 2024-09-21T09:23:27.689 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.report_bg_io_stats: 0 2024-09-21T09:23:27.690 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.ttl: 2592000 2024-09-21T09:23:27.690 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-21T09:23:27.690 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.enable_blob_files: false 2024-09-21T09:23:27.690 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.min_blob_size: 0 2024-09-21T09:23:27.690 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.blob_file_size: 268435456 2024-09-21T09:23:27.690 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-21T09:23:27.690 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-21T09:23:27.690 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-21T09:23:27.690 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-21T09:23:27.690 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 0 2024-09-21T09:23:27.690 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: [db/version_set.cc:4082] Creating manifest 7 2024-09-21T09:23:27.690 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726910607442004, "job": 1, "event": "recovery_started", "wal_files": [4]} 2024-09-21T09:23:27.690 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #4 mode 2 2024-09-21T09:23:27.690 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-09-21T09:23:27.690 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726910607442309, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1571, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 523, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 401, "raw_average_value_size": 80, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1726910607, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "5b01eb0b-2464-4205-99f7-9da5ccd663ef", "db_session_id": "ODUAUJ3WNKP803OB4HBH"}} 2024-09-21T09:23:27.690 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: [db/version_set.cc:4082] Creating manifest 9 2024-09-21T09:23:27.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726910607442633, "job": 1, "event": "recovery_finished"} 2024-09-21T09:23:27.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-21T09:23:27.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x564303db6700 2024-09-21T09:23:27.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: DB pointer 0x564303d86000 2024-09-21T09:23:27.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2024-09-21T09:23:27.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-09-21T09:23:27.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: ** DB Stats ** 2024-09-21T09:23:27.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-21T09:23:27.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-21T09:23:27.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-21T09:23:27.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-21T09:23:27.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-21T09:23:27.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-09-21T09:23:27.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-21T09:23:27.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: 2024-09-21T09:23:27.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: ** Compaction Stats [default] ** 2024-09-21T09:23:27.692 INFO:journalctl@ceph.mon.b.smithi064.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-21T09:23:27.692 INFO:journalctl@ceph.mon.b.smithi064.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-21T09:23:27.692 INFO:journalctl@ceph.mon.b.smithi064.stdout: L0 1/0 1.53 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.2 0.00 0.00 1 0.000 0 0 2024-09-21T09:23:27.692 INFO:journalctl@ceph.mon.b.smithi064.stdout: Sum 1/0 1.53 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.2 0.00 0.00 1 0.000 0 0 2024-09-21T09:23:27.692 INFO:journalctl@ceph.mon.b.smithi064.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.2 0.00 0.00 1 0.000 0 0 2024-09-21T09:23:27.692 INFO:journalctl@ceph.mon.b.smithi064.stdout: 2024-09-21T09:23:27.692 INFO:journalctl@ceph.mon.b.smithi064.stdout: ** Compaction Stats [default] ** 2024-09-21T09:23:27.692 INFO:journalctl@ceph.mon.b.smithi064.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-21T09:23:27.692 INFO:journalctl@ceph.mon.b.smithi064.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-21T09:23:27.692 INFO:journalctl@ceph.mon.b.smithi064.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.2 0.00 0.00 1 0.000 0 0 2024-09-21T09:23:27.692 INFO:journalctl@ceph.mon.b.smithi064.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-21T09:23:27.692 INFO:journalctl@ceph.mon.b.smithi064.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-21T09:23:27.692 INFO:journalctl@ceph.mon.b.smithi064.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-21T09:23:27.692 INFO:journalctl@ceph.mon.b.smithi064.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-21T09:23:27.692 INFO:journalctl@ceph.mon.b.smithi064.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-21T09:23:27.692 INFO:journalctl@ceph.mon.b.smithi064.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-21T09:23:27.693 INFO:journalctl@ceph.mon.b.smithi064.stdout: Cumulative compaction: 0.00 GB write, 0.55 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T09:23:27.693 INFO:journalctl@ceph.mon.b.smithi064.stdout: Interval compaction: 0.00 GB write, 0.55 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T09:23:27.693 INFO:journalctl@ceph.mon.b.smithi064.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-21T09:23:27.693 INFO:journalctl@ceph.mon.b.smithi064.stdout: 2024-09-21T09:23:27.693 INFO:journalctl@ceph.mon.b.smithi064.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-21T09:23:27.693 INFO:journalctl@ceph.mon.b.smithi064.stdout: 2024-09-21T09:23:27.693 INFO:journalctl@ceph.mon.b.smithi064.stdout: ** Compaction Stats [default] ** 2024-09-21T09:23:27.693 INFO:journalctl@ceph.mon.b.smithi064.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-21T09:23:27.693 INFO:journalctl@ceph.mon.b.smithi064.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-21T09:23:27.693 INFO:journalctl@ceph.mon.b.smithi064.stdout: L0 1/0 1.53 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.2 0.00 0.00 1 0.000 0 0 2024-09-21T09:23:27.693 INFO:journalctl@ceph.mon.b.smithi064.stdout: Sum 1/0 1.53 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.2 0.00 0.00 1 0.000 0 0 2024-09-21T09:23:27.693 INFO:journalctl@ceph.mon.b.smithi064.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-09-21T09:23:27.693 INFO:journalctl@ceph.mon.b.smithi064.stdout: 2024-09-21T09:23:27.693 INFO:journalctl@ceph.mon.b.smithi064.stdout: ** Compaction Stats [default] ** 2024-09-21T09:23:27.693 INFO:journalctl@ceph.mon.b.smithi064.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-21T09:23:27.694 INFO:journalctl@ceph.mon.b.smithi064.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-21T09:23:27.694 INFO:journalctl@ceph.mon.b.smithi064.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.2 0.00 0.00 1 0.000 0 0 2024-09-21T09:23:27.694 INFO:journalctl@ceph.mon.b.smithi064.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-21T09:23:27.694 INFO:journalctl@ceph.mon.b.smithi064.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-21T09:23:27.694 INFO:journalctl@ceph.mon.b.smithi064.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-21T09:23:27.694 INFO:journalctl@ceph.mon.b.smithi064.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-21T09:23:27.694 INFO:journalctl@ceph.mon.b.smithi064.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-21T09:23:27.694 INFO:journalctl@ceph.mon.b.smithi064.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-21T09:23:27.694 INFO:journalctl@ceph.mon.b.smithi064.stdout: Cumulative compaction: 0.00 GB write, 0.55 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T09:23:27.694 INFO:journalctl@ceph.mon.b.smithi064.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T09:23:27.694 INFO:journalctl@ceph.mon.b.smithi064.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-21T09:23:27.695 INFO:journalctl@ceph.mon.b.smithi064.stdout: 2024-09-21T09:23:27.695 INFO:journalctl@ceph.mon.b.smithi064.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-21T09:23:27.695 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: mon.b does not exist in monmap, will attempt to join an existing cluster 2024-09-21T09:23:27.695 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: using public_addr v2:172.21.15.64:0/0 -> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] 2024-09-21T09:23:27.695 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: starting mon.b rank -1 at public addrs [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] at bind addrs [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid d65e801e-77fa-11ef-baf6-efdc52797490 2024-09-21T09:23:27.695 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: mon.b@-1(???) e0 preinit fsid d65e801e-77fa-11ef-baf6-efdc52797490 2024-09-21T09:23:27.695 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: mon.b@-1(synchronizing).mds e1 new map 2024-09-21T09:23:27.695 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: mon.b@-1(synchronizing).mds e1 print_map 2024-09-21T09:23:27.695 INFO:journalctl@ceph.mon.b.smithi064.stdout: e1 2024-09-21T09:23:27.695 INFO:journalctl@ceph.mon.b.smithi064.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-21T09:23:27.695 INFO:journalctl@ceph.mon.b.smithi064.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-21T09:23:27.695 INFO:journalctl@ceph.mon.b.smithi064.stdout: legacy client fscid: -1 2024-09-21T09:23:27.695 INFO:journalctl@ceph.mon.b.smithi064.stdout: 2024-09-21T09:23:27.695 INFO:journalctl@ceph.mon.b.smithi064.stdout: No filesystems configured 2024-09-21T09:23:27.695 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: mon.b@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-09-21T09:23:27.696 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: mon.b@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-09-21T09:23:27.696 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: mon.b@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-09-21T09:23:27.696 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: mon.b@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-09-21T09:23:27.696 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: mon.b@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-09-21T09:23:27.696 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: mon.b@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-09-21T09:23:27.696 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: mon.b@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2024-09-21T09:23:27.696 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-09-21T09:23:27.696 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-09-21T09:23:27.696 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-09-21T09:23:27.696 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:23:27.696 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:27.696 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:27.696 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:27.696 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:27.696 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi064", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:23:27.696 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:23:27.697 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:23:27.697 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:23:27.697 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: Updating smithi064:/etc/ceph/ceph.conf 2024-09-21T09:23:27.697 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:27.697 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:27.697 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:27.697 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: from='client.? 172.21.15.22:0/1710393031' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-09-21T09:23:27.697 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: Updating smithi064:/var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:23:27.697 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: Updating smithi064:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:23:27.697 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: Updating smithi064:/var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:23:27.697 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: from='client.? 172.21.15.22:0/1710393031' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-09-21T09:23:27.697 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: osdmap e4: 0 total, 0 up, 0 in 2024-09-21T09:23:27.697 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:23:27.697 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:23:27.697 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:27.697 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:23:27.698 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:23:27.698 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:23:27.698 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:27.698 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T09:23:27.698 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:23:27.698 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: from='client.14182 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi022:172.21.15.22=a;smithi022:[v2:172.21.15.22:3301,v1:172.21.15.22:6790]=c;smithi064:172.21.15.64=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:23:27.698 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: Saving service mon spec with placement smithi022:172.21.15.22=a;smithi022:[v2:172.21.15.22:3301,v1:172.21.15.22:6790]=c;smithi064:172.21.15.64=b;count:3 2024-09-21T09:23:27.698 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: Deploying daemon mon.b on smithi064 2024-09-21T09:23:27.698 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:23:27.698 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: from='client.? 172.21.15.64:0/3562629943' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-21T09:23:27.698 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:23:27.698 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 ceph-mon[31124]: mon.b@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2024-09-21T09:23:27.698 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 bash[31086]: ed126533cd64ff6a282f3cb73136801422203655f4f16fbb5683e4986c899996 2024-09-21T09:23:27.698 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:27 smithi064 systemd[1]: Started Ceph mon.b for d65e801e-77fa-11ef-baf6-efdc52797490. 2024-09-21T09:23:27.903 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:23:32.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:32 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:23:32.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:32 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:23:32.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:32 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:23:32.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:32 smithi022 ceph-mon[27763]: mon.a calling monitor election 2024-09-21T09:23:32.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:32 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:23:32.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:32 smithi022 ceph-mon[27763]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:23:32.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:32 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:23:32.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:32 smithi022 ceph-mon[27763]: mon.b calling monitor election 2024-09-21T09:23:32.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:32 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:23:32.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:32 smithi022 ceph-mon[27763]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:23:32.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:32 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:23:32.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:32 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:23:32.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:32 smithi022 ceph-mon[27763]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-09-21T09:23:32.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:32 smithi022 ceph-mon[27763]: monmap e2: 2 mons at {a=[v2:172.21.15.22:3300/0,v1:172.21.15.22:6789/0],b=[v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-21T09:23:32.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:32 smithi022 ceph-mon[27763]: fsmap 2024-09-21T09:23:32.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:32 smithi022 ceph-mon[27763]: osdmap e4: 0 total, 0 up, 0 in 2024-09-21T09:23:32.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:32 smithi022 ceph-mon[27763]: mgrmap e13: y(active, since 41s) 2024-09-21T09:23:32.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:32 smithi022 ceph-mon[27763]: overall HEALTH_OK 2024-09-21T09:23:32.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:32 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:32.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:32 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:32.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:32 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:32.810 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:32 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T09:23:32.810 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:32 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:23:32.872 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-09-21T09:23:32.872 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":2,"fsid":"d65e801e-77fa-11ef-baf6-efdc52797490","modified":"2024-09-21T09:23:27.468350Z","created":"2024-09-21T09:22:01.045656Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:3300","nonce":0},{"type":"v1","addr":"172.21.15.22:6789","nonce":0}]},"addr":"172.21.15.22:6789/0","public_addr":"172.21.15.22:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:3300","nonce":0},{"type":"v1","addr":"172.21.15.64:6789","nonce":0}]},"addr":"172.21.15.64:6789/0","public_addr":"172.21.15.64:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-09-21T09:23:32.873 INFO:teuthology.orchestra.run.smithi064.stderr:dumped monmap epoch 2 2024-09-21T09:23:32.873 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:32 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:23:32.873 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:32 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:23:32.873 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:32 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:23:32.873 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:32 smithi064 ceph-mon[31124]: mon.a calling monitor election 2024-09-21T09:23:32.873 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:32 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:23:32.873 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:32 smithi064 ceph-mon[31124]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:23:32.874 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:32 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:23:32.874 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:32 smithi064 ceph-mon[31124]: mon.b calling monitor election 2024-09-21T09:23:32.874 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:32 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:23:32.874 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:32 smithi064 ceph-mon[31124]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:23:32.874 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:32 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:23:32.874 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:32 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:23:32.874 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:32 smithi064 ceph-mon[31124]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-09-21T09:23:32.874 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:32 smithi064 ceph-mon[31124]: monmap e2: 2 mons at {a=[v2:172.21.15.22:3300/0,v1:172.21.15.22:6789/0],b=[v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-21T09:23:32.874 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:32 smithi064 ceph-mon[31124]: fsmap 2024-09-21T09:23:32.874 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:32 smithi064 ceph-mon[31124]: osdmap e4: 0 total, 0 up, 0 in 2024-09-21T09:23:32.874 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:32 smithi064 ceph-mon[31124]: mgrmap e13: y(active, since 41s) 2024-09-21T09:23:32.874 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:32 smithi064 ceph-mon[31124]: overall HEALTH_OK 2024-09-21T09:23:32.875 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:32 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:32.875 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:32 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:32.875 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:32 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:32.875 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:32 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T09:23:32.875 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:32 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:23:33.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:33 smithi022 ceph-mon[27763]: Deploying daemon mon.c on smithi022 2024-09-21T09:23:33.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:33 smithi022 ceph-mon[27763]: from='client.? 172.21.15.64:0/2835446397' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-21T09:23:33.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:33 smithi022 ceph-mon[27763]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:23:33.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:33 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:23:33.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:33 smithi064 ceph-mon[31124]: Deploying daemon mon.c on smithi022 2024-09-21T09:23:33.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:33 smithi064 ceph-mon[31124]: from='client.? 172.21.15.64:0/2835446397' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-21T09:23:33.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:33 smithi064 ceph-mon[31124]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:23:33.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:33 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:23:34.602 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-09-21T09:23:34.602 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d65e801e-77fa-11ef-baf6-efdc52797490 -- ceph mon dump -f json 2024-09-21T09:23:34.854 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:23:35.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 podman[39487]: 2024-09-21 09:23:35.701926943 +0000 UTC m=+0.117259539 container create 79f307e3b64ae5b1c912931bbf795d4a334d924c2b0dc757e944822a4c6e984f (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mon-c, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=quincy-fbd1a6a, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731) 2024-09-21T09:23:36.008 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:35 smithi064 ceph-mon[31124]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:23:36.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[27763]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:23:36.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 podman[39487]: 2024-09-21 09:23:35.929632623 +0000 UTC m=+0.344965224 container init 79f307e3b64ae5b1c912931bbf795d4a334d924c2b0dc757e944822a4c6e984f (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mon-c, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, ceph=True, RELEASE=quincy-fbd1a6a, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731) 2024-09-21T09:23:36.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 podman[39487]: 2024-09-21 09:23:35.934650809 +0000 UTC m=+0.349983408 container start 79f307e3b64ae5b1c912931bbf795d4a334d924c2b0dc757e944822a4c6e984f (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mon-c, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, ceph=True, RELEASE=quincy-fbd1a6a, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T09:23:36.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: set uid:gid to 167:167 (ceph:ceph) 2024-09-21T09:23:36.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable), process ceph-mon, pid 7 2024-09-21T09:23:36.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: pidfile_write: ignore empty --pid-file 2024-09-21T09:23:36.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: load: jerasure load: lrc 2024-09-21T09:23:36.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: RocksDB version: 6.15.5 2024-09-21T09:23:36.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-09-21T09:23:36.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Compile date Sep 20 2024 2024-09-21T09:23:36.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: DB SUMMARY 2024-09-21T09:23:36.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: DB Session ID: YNIB9ZWE27EOHDY2691O 2024-09-21T09:23:36.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: CURRENT file: CURRENT 2024-09-21T09:23:36.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: IDENTITY file: IDENTITY 2024-09-21T09:23:36.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2024-09-21T09:23:36.310 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 0, files: 2024-09-21T09:23:36.310 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000004.log size: 636 ; 2024-09-21T09:23:36.310 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.error_if_exists: 0 2024-09-21T09:23:36.310 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.create_if_missing: 0 2024-09-21T09:23:36.310 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.paranoid_checks: 1 2024-09-21T09:23:36.310 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-21T09:23:36.310 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.env: 0x562994eb5440 2024-09-21T09:23:36.310 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.fs: Posix File System 2024-09-21T09:23:36.310 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.info_log: 0x5629965b6ec0 2024-09-21T09:23:36.310 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.max_file_opening_threads: 16 2024-09-21T09:23:36.310 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.statistics: (nil) 2024-09-21T09:23:36.311 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.use_fsync: 0 2024-09-21T09:23:36.311 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.max_log_file_size: 0 2024-09-21T09:23:36.311 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-21T09:23:36.311 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-21T09:23:36.311 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.keep_log_file_num: 1000 2024-09-21T09:23:36.311 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.recycle_log_file_num: 0 2024-09-21T09:23:36.311 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.allow_fallocate: 1 2024-09-21T09:23:36.311 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.allow_mmap_reads: 0 2024-09-21T09:23:36.311 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.allow_mmap_writes: 0 2024-09-21T09:23:36.311 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.use_direct_reads: 0 2024-09-21T09:23:36.311 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-21T09:23:36.311 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.create_missing_column_families: 0 2024-09-21T09:23:36.311 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.db_log_dir: 2024-09-21T09:23:36.312 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-c/store.db 2024-09-21T09:23:36.312 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-21T09:23:36.312 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-21T09:23:36.312 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-21T09:23:36.312 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-21T09:23:36.312 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-21T09:23:36.312 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-21T09:23:36.312 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.advise_random_on_open: 1 2024-09-21T09:23:36.312 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.db_write_buffer_size: 0 2024-09-21T09:23:36.312 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.write_buffer_manager: 0x5629966747b0 2024-09-21T09:23:36.312 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-21T09:23:36.312 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-09-21T09:23:36.313 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-21T09:23:36.313 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-21T09:23:36.313 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.rate_limiter: (nil) 2024-09-21T09:23:36.313 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-21T09:23:36.313 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.wal_recovery_mode: 2 2024-09-21T09:23:36.313 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.enable_thread_tracking: 0 2024-09-21T09:23:36.313 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.enable_pipelined_write: 0 2024-09-21T09:23:36.313 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.unordered_write: 0 2024-09-21T09:23:36.313 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-21T09:23:36.313 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-21T09:23:36.313 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-21T09:23:36.313 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-21T09:23:36.313 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.row_cache: None 2024-09-21T09:23:36.313 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.wal_filter: None 2024-09-21T09:23:36.313 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-21T09:23:36.315 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.allow_ingest_behind: 0 2024-09-21T09:23:36.315 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.preserve_deletes: 0 2024-09-21T09:23:36.315 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.two_write_queues: 0 2024-09-21T09:23:36.315 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.manual_wal_flush: 0 2024-09-21T09:23:36.315 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.atomic_flush: 0 2024-09-21T09:23:36.316 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-21T09:23:36.316 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-21T09:23:36.316 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-21T09:23:36.316 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.log_readahead_size: 0 2024-09-21T09:23:36.316 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-21T09:23:36.316 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.best_efforts_recovery: 0 2024-09-21T09:23:36.316 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-21T09:23:36.316 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-21T09:23:36.316 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.allow_data_in_errors: 0 2024-09-21T09:23:36.316 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.db_host_id: __hostname__ 2024-09-21T09:23:36.316 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.max_background_jobs: 2 2024-09-21T09:23:36.316 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.max_background_compactions: -1 2024-09-21T09:23:36.316 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.max_subcompactions: 1 2024-09-21T09:23:36.316 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-21T09:23:36.316 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-21T09:23:36.317 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-21T09:23:36.317 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.max_total_wal_size: 0 2024-09-21T09:23:36.317 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-21T09:23:36.317 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-21T09:23:36.317 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-21T09:23:36.317 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-21T09:23:36.317 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.max_open_files: -1 2024-09-21T09:23:36.317 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.bytes_per_sync: 0 2024-09-21T09:23:36.317 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-21T09:23:36.317 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-21T09:23:36.317 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.compaction_readahead_size: 0 2024-09-21T09:23:36.317 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.max_background_flushes: -1 2024-09-21T09:23:36.317 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Compression algorithms supported: 2024-09-21T09:23:36.317 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: kZSTD supported: 0 2024-09-21T09:23:36.318 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: kXpressCompression supported: 0 2024-09-21T09:23:36.318 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: kBZip2Compression supported: 0 2024-09-21T09:23:36.318 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-21T09:23:36.318 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: kLZ4Compression supported: 1 2024-09-21T09:23:36.318 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: kZlibCompression supported: 1 2024-09-21T09:23:36.318 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: kLZ4HCCompression supported: 1 2024-09-21T09:23:36.318 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: kSnappyCompression supported: 1 2024-09-21T09:23:36.318 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-21T09:23:36.318 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000003 2024-09-21T09:23:36.318 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2024-09-21T09:23:36.318 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-21T09:23:36.318 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.merge_operator: 2024-09-21T09:23:36.318 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.compaction_filter: None 2024-09-21T09:23:36.318 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.compaction_filter_factory: None 2024-09-21T09:23:36.318 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.sst_partitioner_factory: None 2024-09-21T09:23:36.319 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-21T09:23:36.319 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-21T09:23:36.319 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x56299655da38) 2024-09-21T09:23:36.319 INFO:journalctl@ceph.mon.c.smithi022.stdout: cache_index_and_filter_blocks: 1 2024-09-21T09:23:36.319 INFO:journalctl@ceph.mon.c.smithi022.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-21T09:23:36.319 INFO:journalctl@ceph.mon.c.smithi022.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-21T09:23:36.319 INFO:journalctl@ceph.mon.c.smithi022.stdout: pin_top_level_index_and_filter: 1 2024-09-21T09:23:36.319 INFO:journalctl@ceph.mon.c.smithi022.stdout: index_type: 0 2024-09-21T09:23:36.319 INFO:journalctl@ceph.mon.c.smithi022.stdout: data_block_index_type: 0 2024-09-21T09:23:36.319 INFO:journalctl@ceph.mon.c.smithi022.stdout: index_shortening: 1 2024-09-21T09:23:36.319 INFO:journalctl@ceph.mon.c.smithi022.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-21T09:23:36.319 INFO:journalctl@ceph.mon.c.smithi022.stdout: hash_index_allow_collision: 1 2024-09-21T09:23:36.320 INFO:journalctl@ceph.mon.c.smithi022.stdout: checksum: 1 2024-09-21T09:23:36.320 INFO:journalctl@ceph.mon.c.smithi022.stdout: no_block_cache: 0 2024-09-21T09:23:36.320 INFO:journalctl@ceph.mon.c.smithi022.stdout: block_cache: 0x562996593350 2024-09-21T09:23:36.320 INFO:journalctl@ceph.mon.c.smithi022.stdout: block_cache_name: BinnedLRUCache 2024-09-21T09:23:36.320 INFO:journalctl@ceph.mon.c.smithi022.stdout: block_cache_options: 2024-09-21T09:23:36.320 INFO:journalctl@ceph.mon.c.smithi022.stdout: capacity : 536870912 2024-09-21T09:23:36.320 INFO:journalctl@ceph.mon.c.smithi022.stdout: num_shard_bits : 4 2024-09-21T09:23:36.320 INFO:journalctl@ceph.mon.c.smithi022.stdout: strict_capacity_limit : 0 2024-09-21T09:23:36.320 INFO:journalctl@ceph.mon.c.smithi022.stdout: high_pri_pool_ratio: 0.000 2024-09-21T09:23:36.320 INFO:journalctl@ceph.mon.c.smithi022.stdout: block_cache_compressed: (nil) 2024-09-21T09:23:36.320 INFO:journalctl@ceph.mon.c.smithi022.stdout: persistent_cache: (nil) 2024-09-21T09:23:36.320 INFO:journalctl@ceph.mon.c.smithi022.stdout: block_size: 4096 2024-09-21T09:23:36.320 INFO:journalctl@ceph.mon.c.smithi022.stdout: block_size_deviation: 10 2024-09-21T09:23:36.320 INFO:journalctl@ceph.mon.c.smithi022.stdout: block_restart_interval: 16 2024-09-21T09:23:36.320 INFO:journalctl@ceph.mon.c.smithi022.stdout: index_block_restart_interval: 1 2024-09-21T09:23:36.320 INFO:journalctl@ceph.mon.c.smithi022.stdout: metadata_block_size: 4096 2024-09-21T09:23:36.321 INFO:journalctl@ceph.mon.c.smithi022.stdout: partition_filters: 0 2024-09-21T09:23:36.321 INFO:journalctl@ceph.mon.c.smithi022.stdout: use_delta_encoding: 1 2024-09-21T09:23:36.321 INFO:journalctl@ceph.mon.c.smithi022.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-09-21T09:23:36.321 INFO:journalctl@ceph.mon.c.smithi022.stdout: whole_key_filtering: 1 2024-09-21T09:23:36.321 INFO:journalctl@ceph.mon.c.smithi022.stdout: verify_compression: 0 2024-09-21T09:23:36.321 INFO:journalctl@ceph.mon.c.smithi022.stdout: read_amp_bytes_per_bit: 0 2024-09-21T09:23:36.321 INFO:journalctl@ceph.mon.c.smithi022.stdout: format_version: 4 2024-09-21T09:23:36.321 INFO:journalctl@ceph.mon.c.smithi022.stdout: enable_index_compression: 1 2024-09-21T09:23:36.321 INFO:journalctl@ceph.mon.c.smithi022.stdout: block_align: 0 2024-09-21T09:23:36.321 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.write_buffer_size: 33554432 2024-09-21T09:23:36.321 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.max_write_buffer_number: 2 2024-09-21T09:23:36.321 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.compression: NoCompression 2024-09-21T09:23:36.321 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.bottommost_compression: Disabled 2024-09-21T09:23:36.321 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.prefix_extractor: nullptr 2024-09-21T09:23:36.321 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-21T09:23:36.321 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.num_levels: 7 2024-09-21T09:23:36.322 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-21T09:23:36.322 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-21T09:23:36.322 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-21T09:23:36.322 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-21T09:23:36.322 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-21T09:23:36.322 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-21T09:23:36.322 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-21T09:23:36.322 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-21T09:23:36.322 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-21T09:23:36.322 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-21T09:23:36.322 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-21T09:23:36.322 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.compression_opts.level: 32767 2024-09-21T09:23:36.322 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.compression_opts.strategy: 0 2024-09-21T09:23:36.323 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-21T09:23:36.323 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-21T09:23:36.323 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-21T09:23:36.323 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.compression_opts.enabled: false 2024-09-21T09:23:36.323 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-21T09:23:36.323 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-21T09:23:36.323 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-21T09:23:36.323 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.target_file_size_base: 67108864 2024-09-21T09:23:36.323 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-21T09:23:36.323 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-21T09:23:36.323 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-21T09:23:36.323 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-21T09:23:36.323 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-21T09:23:36.323 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-21T09:23:36.323 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-21T09:23:36.324 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-21T09:23:36.324 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-21T09:23:36.324 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-21T09:23:36.324 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-21T09:23:36.324 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-21T09:23:36.324 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-21T09:23:36.324 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.arena_block_size: 4194304 2024-09-21T09:23:36.324 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-21T09:23:36.324 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-21T09:23:36.324 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-09-21T09:23:36.324 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.disable_auto_compactions: 0 2024-09-21T09:23:36.324 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-21T09:23:36.324 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-21T09:23:36.325 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-21T09:23:36.325 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-21T09:23:36.325 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-21T09:23:36.325 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-21T09:23:36.325 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-21T09:23:36.325 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-21T09:23:36.325 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-21T09:23:36.325 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-21T09:23:36.325 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-21T09:23:36.325 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.inplace_update_support: 0 2024-09-21T09:23:36.325 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-21T09:23:36.325 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-21T09:23:36.325 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-21T09:23:36.325 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-21T09:23:36.326 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.bloom_locality: 0 2024-09-21T09:23:36.326 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.max_successive_merges: 0 2024-09-21T09:23:36.326 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-21T09:23:36.326 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.paranoid_file_checks: 0 2024-09-21T09:23:36.326 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.force_consistency_checks: 1 2024-09-21T09:23:36.326 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.report_bg_io_stats: 0 2024-09-21T09:23:36.326 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.ttl: 2592000 2024-09-21T09:23:36.326 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-21T09:23:36.326 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.enable_blob_files: false 2024-09-21T09:23:36.326 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.min_blob_size: 0 2024-09-21T09:23:36.326 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.blob_file_size: 268435456 2024-09-21T09:23:36.327 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-21T09:23:36.327 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-21T09:23:36.327 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-21T09:23:36.327 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-21T09:23:36.327 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 0 2024-09-21T09:23:36.327 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: [db/version_set.cc:4082] Creating manifest 7 2024-09-21T09:23:36.327 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726910615959371, "job": 1, "event": "recovery_started", "wal_files": [4]} 2024-09-21T09:23:36.327 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #4 mode 2 2024-09-21T09:23:36.327 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-09-21T09:23:36.327 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726910615959688, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1696, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 648, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 526, "raw_average_value_size": 105, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1726910615, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "5ff9eb41-e42e-4fb5-87fa-84f29ffa0ff0", "db_session_id": "YNIB9ZWE27EOHDY2691O"}} 2024-09-21T09:23:36.327 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: [db/version_set.cc:4082] Creating manifest 9 2024-09-21T09:23:36.327 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726910615960048, "job": 1, "event": "recovery_finished"} 2024-09-21T09:23:36.327 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-21T09:23:36.327 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x5629965c6700 2024-09-21T09:23:36.328 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: DB pointer 0x562996596000 2024-09-21T09:23:36.328 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2024-09-21T09:23:36.328 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-09-21T09:23:36.328 INFO:journalctl@ceph.mon.c.smithi022.stdout: ** DB Stats ** 2024-09-21T09:23:36.328 INFO:journalctl@ceph.mon.c.smithi022.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-21T09:23:36.328 INFO:journalctl@ceph.mon.c.smithi022.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-21T09:23:36.328 INFO:journalctl@ceph.mon.c.smithi022.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-21T09:23:36.328 INFO:journalctl@ceph.mon.c.smithi022.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-21T09:23:36.328 INFO:journalctl@ceph.mon.c.smithi022.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-21T09:23:36.328 INFO:journalctl@ceph.mon.c.smithi022.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-09-21T09:23:36.328 INFO:journalctl@ceph.mon.c.smithi022.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-21T09:23:36.328 INFO:journalctl@ceph.mon.c.smithi022.stdout: 2024-09-21T09:23:36.328 INFO:journalctl@ceph.mon.c.smithi022.stdout: ** Compaction Stats [default] ** 2024-09-21T09:23:36.329 INFO:journalctl@ceph.mon.c.smithi022.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-21T09:23:36.329 INFO:journalctl@ceph.mon.c.smithi022.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-21T09:23:36.329 INFO:journalctl@ceph.mon.c.smithi022.stdout: L0 1/0 1.66 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.4 0.00 0.00 1 0.000 0 0 2024-09-21T09:23:36.329 INFO:journalctl@ceph.mon.c.smithi022.stdout: Sum 1/0 1.66 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.4 0.00 0.00 1 0.000 0 0 2024-09-21T09:23:36.329 INFO:journalctl@ceph.mon.c.smithi022.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.4 0.00 0.00 1 0.000 0 0 2024-09-21T09:23:36.329 INFO:journalctl@ceph.mon.c.smithi022.stdout: 2024-09-21T09:23:36.329 INFO:journalctl@ceph.mon.c.smithi022.stdout: ** Compaction Stats [default] ** 2024-09-21T09:23:36.329 INFO:journalctl@ceph.mon.c.smithi022.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-21T09:23:36.329 INFO:journalctl@ceph.mon.c.smithi022.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-21T09:23:36.329 INFO:journalctl@ceph.mon.c.smithi022.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.4 0.00 0.00 1 0.000 0 0 2024-09-21T09:23:36.329 INFO:journalctl@ceph.mon.c.smithi022.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-21T09:23:36.329 INFO:journalctl@ceph.mon.c.smithi022.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-21T09:23:36.329 INFO:journalctl@ceph.mon.c.smithi022.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-21T09:23:36.329 INFO:journalctl@ceph.mon.c.smithi022.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-21T09:23:36.330 INFO:journalctl@ceph.mon.c.smithi022.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-21T09:23:36.330 INFO:journalctl@ceph.mon.c.smithi022.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-21T09:23:36.330 INFO:journalctl@ceph.mon.c.smithi022.stdout: Cumulative compaction: 0.00 GB write, 0.58 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T09:23:36.330 INFO:journalctl@ceph.mon.c.smithi022.stdout: Interval compaction: 0.00 GB write, 0.58 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T09:23:36.330 INFO:journalctl@ceph.mon.c.smithi022.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-21T09:23:36.330 INFO:journalctl@ceph.mon.c.smithi022.stdout: 2024-09-21T09:23:36.330 INFO:journalctl@ceph.mon.c.smithi022.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-21T09:23:36.330 INFO:journalctl@ceph.mon.c.smithi022.stdout: 2024-09-21T09:23:36.330 INFO:journalctl@ceph.mon.c.smithi022.stdout: ** Compaction Stats [default] ** 2024-09-21T09:23:36.330 INFO:journalctl@ceph.mon.c.smithi022.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-21T09:23:36.330 INFO:journalctl@ceph.mon.c.smithi022.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-21T09:23:36.330 INFO:journalctl@ceph.mon.c.smithi022.stdout: L0 1/0 1.66 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.4 0.00 0.00 1 0.000 0 0 2024-09-21T09:23:36.330 INFO:journalctl@ceph.mon.c.smithi022.stdout: Sum 1/0 1.66 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.4 0.00 0.00 1 0.000 0 0 2024-09-21T09:23:36.330 INFO:journalctl@ceph.mon.c.smithi022.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-09-21T09:23:36.331 INFO:journalctl@ceph.mon.c.smithi022.stdout: 2024-09-21T09:23:36.331 INFO:journalctl@ceph.mon.c.smithi022.stdout: ** Compaction Stats [default] ** 2024-09-21T09:23:36.331 INFO:journalctl@ceph.mon.c.smithi022.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-21T09:23:36.331 INFO:journalctl@ceph.mon.c.smithi022.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-21T09:23:36.331 INFO:journalctl@ceph.mon.c.smithi022.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.4 0.00 0.00 1 0.000 0 0 2024-09-21T09:23:36.331 INFO:journalctl@ceph.mon.c.smithi022.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-21T09:23:36.331 INFO:journalctl@ceph.mon.c.smithi022.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-21T09:23:36.331 INFO:journalctl@ceph.mon.c.smithi022.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-21T09:23:36.331 INFO:journalctl@ceph.mon.c.smithi022.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-21T09:23:36.331 INFO:journalctl@ceph.mon.c.smithi022.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-21T09:23:36.331 INFO:journalctl@ceph.mon.c.smithi022.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-21T09:23:36.331 INFO:journalctl@ceph.mon.c.smithi022.stdout: Cumulative compaction: 0.00 GB write, 0.57 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T09:23:36.331 INFO:journalctl@ceph.mon.c.smithi022.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T09:23:36.331 INFO:journalctl@ceph.mon.c.smithi022.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-21T09:23:36.332 INFO:journalctl@ceph.mon.c.smithi022.stdout: 2024-09-21T09:23:36.332 INFO:journalctl@ceph.mon.c.smithi022.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-21T09:23:36.332 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: mon.c does not exist in monmap, will attempt to join an existing cluster 2024-09-21T09:23:36.332 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: using public_addrv [v2:172.21.15.22:3301/0,v1:172.21.15.22:6790/0] 2024-09-21T09:23:36.332 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: starting mon.c rank -1 at public addrs [v2:172.21.15.22:3301/0,v1:172.21.15.22:6790/0] at bind addrs [v2:172.21.15.22:3301/0,v1:172.21.15.22:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid d65e801e-77fa-11ef-baf6-efdc52797490 2024-09-21T09:23:36.332 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: mon.c@-1(???) e0 preinit fsid d65e801e-77fa-11ef-baf6-efdc52797490 2024-09-21T09:23:36.332 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: mon.c@-1(synchronizing).mds e1 new map 2024-09-21T09:23:36.332 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: mon.c@-1(synchronizing).mds e1 print_map 2024-09-21T09:23:36.332 INFO:journalctl@ceph.mon.c.smithi022.stdout: e1 2024-09-21T09:23:36.332 INFO:journalctl@ceph.mon.c.smithi022.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-21T09:23:36.332 INFO:journalctl@ceph.mon.c.smithi022.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-21T09:23:36.332 INFO:journalctl@ceph.mon.c.smithi022.stdout: legacy client fscid: -1 2024-09-21T09:23:36.332 INFO:journalctl@ceph.mon.c.smithi022.stdout: 2024-09-21T09:23:36.332 INFO:journalctl@ceph.mon.c.smithi022.stdout: No filesystems configured 2024-09-21T09:23:36.332 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: mon.c@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-09-21T09:23:36.333 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: mon.c@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-09-21T09:23:36.333 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: mon.c@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-09-21T09:23:36.333 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: mon.c@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-09-21T09:23:36.333 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: mon.c@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-09-21T09:23:36.333 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: mon.c@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-09-21T09:23:36.333 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: mon.c@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2024-09-21T09:23:36.333 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-09-21T09:23:36.333 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-09-21T09:23:36.333 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-09-21T09:23:36.333 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:23:36.333 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:36.333 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:36.333 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:36.333 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:36.334 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi064", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:23:36.334 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:23:36.334 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:23:36.334 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:23:36.334 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: Updating smithi064:/etc/ceph/ceph.conf 2024-09-21T09:23:36.334 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:36.334 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:36.334 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:36.334 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: from='client.? 172.21.15.22:0/1710393031' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-09-21T09:23:36.334 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: Updating smithi064:/var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:23:36.334 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: Updating smithi064:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:23:36.334 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: Updating smithi064:/var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:23:36.334 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: from='client.? 172.21.15.22:0/1710393031' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-09-21T09:23:36.335 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: osdmap e4: 0 total, 0 up, 0 in 2024-09-21T09:23:36.335 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:23:36.335 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:23:36.335 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:36.335 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:23:36.335 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:23:36.335 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:23:36.335 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:36.335 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T09:23:36.335 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:23:36.335 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: from='client.14182 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi022:172.21.15.22=a;smithi022:[v2:172.21.15.22:3301,v1:172.21.15.22:6790]=c;smithi064:172.21.15.64=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:23:36.335 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: Saving service mon spec with placement smithi022:172.21.15.22=a;smithi022:[v2:172.21.15.22:3301,v1:172.21.15.22:6790]=c;smithi064:172.21.15.64=b;count:3 2024-09-21T09:23:36.335 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: Deploying daemon mon.b on smithi064 2024-09-21T09:23:36.335 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:23:36.336 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: from='client.? 172.21.15.64:0/3562629943' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-21T09:23:36.336 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:23:36.336 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:23:36.336 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:23:36.336 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:23:36.336 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: mon.a calling monitor election 2024-09-21T09:23:36.336 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:23:36.336 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:23:36.336 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:23:36.336 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: mon.b calling monitor election 2024-09-21T09:23:36.336 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:23:36.336 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:23:36.336 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:23:36.337 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:23:36.337 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-09-21T09:23:36.337 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: monmap e2: 2 mons at {a=[v2:172.21.15.22:3300/0,v1:172.21.15.22:6789/0],b=[v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-21T09:23:36.337 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: fsmap 2024-09-21T09:23:36.337 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: osdmap e4: 0 total, 0 up, 0 in 2024-09-21T09:23:36.337 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: mgrmap e13: y(active, since 41s) 2024-09-21T09:23:36.337 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: overall HEALTH_OK 2024-09-21T09:23:36.337 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:36.338 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:36.338 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:36.338 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T09:23:36.338 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:23:36.338 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: Deploying daemon mon.c on smithi022 2024-09-21T09:23:36.338 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: from='client.? 172.21.15.64:0/2835446397' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-21T09:23:36.338 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:23:36.338 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:23:36.338 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:23:36.338 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:35 smithi022 ceph-mon[39525]: mon.c@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2024-09-21T09:23:36.338 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:36 smithi022 bash[39487]: 79f307e3b64ae5b1c912931bbf795d4a334d924c2b0dc757e944822a4c6e984f 2024-09-21T09:23:36.338 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:36 smithi022 systemd[1]: Started Ceph mon.c for d65e801e-77fa-11ef-baf6-efdc52797490. 2024-09-21T09:23:41.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:41 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:23:41.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:41 smithi022 ceph-mon[27763]: mon.a calling monitor election 2024-09-21T09:23:41.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:41 smithi022 ceph-mon[27763]: mon.b calling monitor election 2024-09-21T09:23:41.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:41 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:23:41.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:41 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:23:41.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:41 smithi022 ceph-mon[27763]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:23:41.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:41 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:23:41.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:41 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:23:41.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:41 smithi022 ceph-mon[27763]: mon.c calling monitor election 2024-09-21T09:23:41.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:41 smithi022 ceph-mon[27763]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:23:41.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:41 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:23:41.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:41 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:23:41.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:41 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:23:41.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:41 smithi022 ceph-mon[27763]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-21T09:23:41.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:41 smithi022 ceph-mon[27763]: monmap e3: 3 mons at {a=[v2:172.21.15.22:3300/0,v1:172.21.15.22:6789/0],b=[v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0],c=[v2:172.21.15.22:3301/0,v1:172.21.15.22:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-21T09:23:41.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:41 smithi022 ceph-mon[27763]: fsmap 2024-09-21T09:23:41.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:41 smithi022 ceph-mon[27763]: osdmap e4: 0 total, 0 up, 0 in 2024-09-21T09:23:41.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:41 smithi022 ceph-mon[27763]: mgrmap e13: y(active, since 50s) 2024-09-21T09:23:41.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:41 smithi022 ceph-mon[27763]: overall HEALTH_OK 2024-09-21T09:23:41.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:41 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:41.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:41 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:41.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:41 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:41.310 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:41 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:41.310 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:41 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:23:41.310 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:41 smithi022 ceph-mon[39525]: mon.a calling monitor election 2024-09-21T09:23:41.310 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:41 smithi022 ceph-mon[39525]: mon.b calling monitor election 2024-09-21T09:23:41.310 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:41 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:23:41.310 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:41 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:23:41.310 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:41 smithi022 ceph-mon[39525]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:23:41.310 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:41 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:23:41.310 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:41 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:23:41.310 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:41 smithi022 ceph-mon[39525]: mon.c calling monitor election 2024-09-21T09:23:41.310 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:41 smithi022 ceph-mon[39525]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:23:41.311 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:41 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:23:41.311 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:41 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:23:41.311 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:41 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:23:41.311 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:41 smithi022 ceph-mon[39525]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-21T09:23:41.311 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:41 smithi022 ceph-mon[39525]: monmap e3: 3 mons at {a=[v2:172.21.15.22:3300/0,v1:172.21.15.22:6789/0],b=[v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0],c=[v2:172.21.15.22:3301/0,v1:172.21.15.22:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-21T09:23:41.311 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:41 smithi022 ceph-mon[39525]: fsmap 2024-09-21T09:23:41.311 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:41 smithi022 ceph-mon[39525]: osdmap e4: 0 total, 0 up, 0 in 2024-09-21T09:23:41.311 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:41 smithi022 ceph-mon[39525]: mgrmap e13: y(active, since 50s) 2024-09-21T09:23:41.311 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:41 smithi022 ceph-mon[39525]: overall HEALTH_OK 2024-09-21T09:23:41.311 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:41 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:41.311 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:41 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:41.311 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:41 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:41.311 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:41 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:41.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:41 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:23:41.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:41 smithi064 ceph-mon[31124]: mon.a calling monitor election 2024-09-21T09:23:41.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:41 smithi064 ceph-mon[31124]: mon.b calling monitor election 2024-09-21T09:23:41.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:41 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:23:41.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:41 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:23:41.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:41 smithi064 ceph-mon[31124]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:23:41.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:41 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:23:41.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:41 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:23:41.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:41 smithi064 ceph-mon[31124]: mon.c calling monitor election 2024-09-21T09:23:41.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:41 smithi064 ceph-mon[31124]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:23:41.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:41 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:23:41.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:41 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:23:41.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:41 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:23:41.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:41 smithi064 ceph-mon[31124]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-21T09:23:41.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:41 smithi064 ceph-mon[31124]: monmap e3: 3 mons at {a=[v2:172.21.15.22:3300/0,v1:172.21.15.22:6789/0],b=[v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0],c=[v2:172.21.15.22:3301/0,v1:172.21.15.22:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-21T09:23:41.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:41 smithi064 ceph-mon[31124]: fsmap 2024-09-21T09:23:41.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:41 smithi064 ceph-mon[31124]: osdmap e4: 0 total, 0 up, 0 in 2024-09-21T09:23:41.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:41 smithi064 ceph-mon[31124]: mgrmap e13: y(active, since 50s) 2024-09-21T09:23:41.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:41 smithi064 ceph-mon[31124]: overall HEALTH_OK 2024-09-21T09:23:41.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:41 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:41.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:41 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:41.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:41 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:41.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:41 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:42.307 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:42 smithi022 ceph-mon[27763]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:23:42.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:42 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:23:42.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:42 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:23:42.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:42 smithi022 ceph-mon[39525]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:23:42.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:42 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:23:42.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:42 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:23:42.436 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-09-21T09:23:42.436 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":3,"fsid":"d65e801e-77fa-11ef-baf6-efdc52797490","modified":"2024-09-21T09:23:35.984741Z","created":"2024-09-21T09:22:01.045656Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:3300","nonce":0},{"type":"v1","addr":"172.21.15.22:6789","nonce":0}]},"addr":"172.21.15.22:6789/0","public_addr":"172.21.15.22:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:3300","nonce":0},{"type":"v1","addr":"172.21.15.64:6789","nonce":0}]},"addr":"172.21.15.64:6789/0","public_addr":"172.21.15.64:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":2,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:3301","nonce":0},{"type":"v1","addr":"172.21.15.22:6790","nonce":0}]},"addr":"172.21.15.22:6790/0","public_addr":"172.21.15.22:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1,2]} 2024-09-21T09:23:42.437 INFO:teuthology.orchestra.run.smithi064.stderr:dumped monmap epoch 3 2024-09-21T09:23:42.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:42 smithi064 ceph-mon[31124]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:23:42.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:42 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:23:42.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:42 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:23:43.082 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-09-21T09:23:43.082 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d65e801e-77fa-11ef-baf6-efdc52797490 -- ceph config generate-minimal-conf 2024-09-21T09:23:43.430 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:43 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:43.430 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:43 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:43.430 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:43 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:43.430 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:43 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:43.431 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:43 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:23:43.431 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:43 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:23:43.431 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:43 smithi022 ceph-mon[39525]: Updating smithi022:/etc/ceph/ceph.conf 2024-09-21T09:23:43.431 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:43 smithi022 ceph-mon[39525]: Updating smithi064:/etc/ceph/ceph.conf 2024-09-21T09:23:43.431 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:43 smithi022 ceph-mon[39525]: from='client.? 172.21.15.64:0/4089299117' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-21T09:23:43.431 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:43 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:43.431 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:43 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:43.431 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:43 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:43.431 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:43 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:43.431 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:43 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:43.431 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:43 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T09:23:43.431 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:43 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T09:23:43.431 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:43 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:23:43.432 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:43 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:43.432 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:43 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:43.432 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:43 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:43.432 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:43 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:43.432 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:43 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:23:43.432 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:43 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:23:43.432 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:43 smithi022 ceph-mon[27763]: Updating smithi022:/etc/ceph/ceph.conf 2024-09-21T09:23:43.432 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:43 smithi022 ceph-mon[27763]: Updating smithi064:/etc/ceph/ceph.conf 2024-09-21T09:23:43.432 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:43 smithi022 ceph-mon[27763]: from='client.? 172.21.15.64:0/4089299117' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-21T09:23:43.432 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:43 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:43.432 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:43 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:43.432 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:43 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:43.433 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:43 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:43.433 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:43 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:43.433 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:43 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T09:23:43.433 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:43 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T09:23:43.433 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:43 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:23:43.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:43 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:43.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:43 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:43.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:43 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:43.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:43 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:43.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:43 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:23:43.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:43 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:23:43.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:43 smithi064 ceph-mon[31124]: Updating smithi022:/etc/ceph/ceph.conf 2024-09-21T09:23:43.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:43 smithi064 ceph-mon[31124]: Updating smithi064:/etc/ceph/ceph.conf 2024-09-21T09:23:43.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:43 smithi064 ceph-mon[31124]: from='client.? 172.21.15.64:0/4089299117' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-21T09:23:43.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:43 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:43.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:43 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:43.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:43 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:43.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:43 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:43.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:43 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:43.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:43 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T09:23:43.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:43 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T09:23:43.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:43 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:23:43.763 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:23:44.135 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:44 smithi022 ceph-mon[27763]: Updating smithi022:/var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:23:44.135 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:44 smithi022 ceph-mon[27763]: Updating smithi064:/var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:23:44.135 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:44 smithi022 ceph-mon[27763]: Reconfiguring mon.a (unknown last config time)... 2024-09-21T09:23:44.135 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:44 smithi022 ceph-mon[27763]: Reconfiguring daemon mon.a on smithi022 2024-09-21T09:23:44.136 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:44 smithi022 ceph-mon[27763]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:23:44.136 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:44 smithi022 ceph-mon[39525]: Updating smithi022:/var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:23:44.136 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:44 smithi022 ceph-mon[39525]: Updating smithi064:/var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:23:44.136 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:44 smithi022 ceph-mon[39525]: Reconfiguring mon.a (unknown last config time)... 2024-09-21T09:23:44.136 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:44 smithi022 ceph-mon[39525]: Reconfiguring daemon mon.a on smithi022 2024-09-21T09:23:44.136 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:44 smithi022 ceph-mon[39525]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:23:44.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:44 smithi064 ceph-mon[31124]: Updating smithi022:/var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:23:44.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:44 smithi064 ceph-mon[31124]: Updating smithi064:/var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:23:44.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:44 smithi064 ceph-mon[31124]: Reconfiguring mon.a (unknown last config time)... 2024-09-21T09:23:44.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:44 smithi064 ceph-mon[31124]: Reconfiguring daemon mon.a on smithi022 2024-09-21T09:23:44.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:44 smithi064 ceph-mon[31124]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:23:45.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:45 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:45.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:45 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:45.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:45 smithi022 ceph-mon[27763]: Reconfiguring mon.c (monmap changed)... 2024-09-21T09:23:45.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:45 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T09:23:45.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:45 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T09:23:45.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:45 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:23:45.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:45 smithi022 ceph-mon[27763]: Reconfiguring daemon mon.c on smithi022 2024-09-21T09:23:45.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:45 smithi022 ceph-mon[27763]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:23:45.560 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:45 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:45.560 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:45 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:45.560 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:45 smithi022 ceph-mon[39525]: Reconfiguring mon.c (monmap changed)... 2024-09-21T09:23:45.560 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:45 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T09:23:45.560 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:45 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T09:23:45.560 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:45 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:23:45.560 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:45 smithi022 ceph-mon[39525]: Reconfiguring daemon mon.c on smithi022 2024-09-21T09:23:45.560 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:45 smithi022 ceph-mon[39525]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:23:45.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:45 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:45.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:45 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:45.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:45 smithi064 ceph-mon[31124]: Reconfiguring mon.c (monmap changed)... 2024-09-21T09:23:45.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:45 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T09:23:45.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:45 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T09:23:45.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:45 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:23:45.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:45 smithi064 ceph-mon[31124]: Reconfiguring daemon mon.c on smithi022 2024-09-21T09:23:45.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:45 smithi064 ceph-mon[31124]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:23:46.941 INFO:teuthology.orchestra.run.smithi022.stdout:# minimal ceph.conf for d65e801e-77fa-11ef-baf6-efdc52797490 2024-09-21T09:23:46.942 INFO:teuthology.orchestra.run.smithi022.stdout:[global] 2024-09-21T09:23:46.942 INFO:teuthology.orchestra.run.smithi022.stdout: fsid = d65e801e-77fa-11ef-baf6-efdc52797490 2024-09-21T09:23:46.942 INFO:teuthology.orchestra.run.smithi022.stdout: mon_host = [v2:172.21.15.22:3300/0,v1:172.21.15.22:6789/0] [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] [v2:172.21.15.22:3301/0,v1:172.21.15.22:6790/0] 2024-09-21T09:23:47.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:46 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:47.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:46 smithi022 ceph-mon[27763]: from='client.? 172.21.15.22:0/832004841' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:23:47.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:46 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:47.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:46 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:47.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:46 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T09:23:47.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:46 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T09:23:47.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:46 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:23:47.059 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:46 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:47.059 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:46 smithi022 ceph-mon[39525]: from='client.? 172.21.15.22:0/832004841' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:23:47.059 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:46 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:47.059 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:46 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:47.059 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:46 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T09:23:47.059 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:46 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T09:23:47.059 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:46 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:23:47.246 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:46 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:47.246 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:46 smithi064 ceph-mon[31124]: from='client.? 172.21.15.22:0/832004841' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:23:47.246 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:46 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:47.246 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:46 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:47.246 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:46 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T09:23:47.246 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:46 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T09:23:47.247 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:46 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:23:47.634 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-09-21T09:23:47.635 DEBUG:teuthology.orchestra.run.smithi022:> set -ex 2024-09-21T09:23:47.635 DEBUG:teuthology.orchestra.run.smithi022:> sudo dd of=/etc/ceph/ceph.conf 2024-09-21T09:23:47.666 DEBUG:teuthology.orchestra.run.smithi022:> set -ex 2024-09-21T09:23:47.666 DEBUG:teuthology.orchestra.run.smithi022:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:23:47.735 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-09-21T09:23:47.735 DEBUG:teuthology.orchestra.run.smithi064:> sudo dd of=/etc/ceph/ceph.conf 2024-09-21T09:23:47.764 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-09-21T09:23:47.764 DEBUG:teuthology.orchestra.run.smithi064:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:23:47.830 INFO:tasks.cephadm:Adding mgr.y on smithi022 2024-09-21T09:23:47.830 INFO:tasks.cephadm:Adding mgr.x on smithi064 2024-09-21T09:23:47.831 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d65e801e-77fa-11ef-baf6-efdc52797490 -- ceph orch apply mgr '2;smithi022=y;smithi064=x' 2024-09-21T09:23:48.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:47 smithi064 ceph-mon[31124]: Reconfiguring mon.b (monmap changed)... 2024-09-21T09:23:48.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:47 smithi064 ceph-mon[31124]: Reconfiguring daemon mon.b on smithi064 2024-09-21T09:23:48.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:47 smithi064 ceph-mon[31124]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:23:48.299 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:23:48.307 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:47 smithi022 ceph-mon[27763]: Reconfiguring mon.b (monmap changed)... 2024-09-21T09:23:48.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:47 smithi022 ceph-mon[27763]: Reconfiguring daemon mon.b on smithi064 2024-09-21T09:23:48.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:47 smithi022 ceph-mon[27763]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:23:48.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:47 smithi022 ceph-mon[39525]: Reconfiguring mon.b (monmap changed)... 2024-09-21T09:23:48.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:47 smithi022 ceph-mon[39525]: Reconfiguring daemon mon.b on smithi064 2024-09-21T09:23:48.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:47 smithi022 ceph-mon[39525]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:23:49.386 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:49 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:49.386 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:49 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:49.386 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:49 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:23:49.386 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:49 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:23:49.386 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:49 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:23:49.386 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:49 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:49.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:49 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:49.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:49 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:49.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:49 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:23:49.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:49 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:23:49.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:49 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:23:49.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:49 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:49.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:49 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:49.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:49 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:49.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:49 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:23:49.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:49 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:23:49.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:49 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:23:49.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:49 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:50.051 INFO:teuthology.orchestra.run.smithi064.stdout:Scheduled mgr update... 2024-09-21T09:23:50.359 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:50 smithi064 ceph-mon[31124]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:23:50.359 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:50 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:50.359 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:50 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:23:50.359 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:50 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:23:50.360 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:50 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:23:50.360 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:50 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:50.360 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:50 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T09:23:50.360 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:50 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-09-21T09:23:50.360 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:50 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T09:23:50.360 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:50 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:23:50.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:50 smithi022 ceph-mon[39525]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:23:50.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:50 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:50.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:50 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:23:50.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:50 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:23:50.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:50 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:23:50.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:50 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:50.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:50 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T09:23:50.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:50 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-09-21T09:23:50.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:50 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T09:23:50.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:50 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:23:50.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:50 smithi022 ceph-mon[27763]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:23:50.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:50 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:50.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:50 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:23:50.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:50 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:23:50.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:50 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:23:50.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:50 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:50.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:50 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T09:23:50.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:50 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-09-21T09:23:50.560 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:50 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T09:23:50.560 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:50 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:23:50.684 DEBUG:teuthology.orchestra.run.smithi064:mgr.x> sudo journalctl -f -n 0 -u ceph-d65e801e-77fa-11ef-baf6-efdc52797490@mgr.x.service 2024-09-21T09:23:50.687 DEBUG:tasks.cephadm:set 0 configs 2024-09-21T09:23:50.687 INFO:tasks.cephadm:Deploying OSDs... 2024-09-21T09:23:50.688 DEBUG:teuthology.orchestra.run.smithi022:> set -ex 2024-09-21T09:23:50.688 DEBUG:teuthology.orchestra.run.smithi022:> dd if=/scratch_devs of=/dev/stdout 2024-09-21T09:23:50.704 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-09-21T09:23:50.704 DEBUG:teuthology.orchestra.run.smithi022:> stat /dev/vg_nvme/lv_1 2024-09-21T09:23:50.760 INFO:teuthology.orchestra.run.smithi022.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-09-21T09:23:50.760 INFO:teuthology.orchestra.run.smithi022.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-21T09:23:50.760 INFO:teuthology.orchestra.run.smithi022.stdout:Device: 5h/5d Inode: 973 Links: 1 2024-09-21T09:23:50.760 INFO:teuthology.orchestra.run.smithi022.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-21T09:23:50.760 INFO:teuthology.orchestra.run.smithi022.stdout:Context: system_u:object_r:device_t:s0 2024-09-21T09:23:50.760 INFO:teuthology.orchestra.run.smithi022.stdout:Access: 2024-09-21 09:23:35.426141197 +0000 2024-09-21T09:23:50.760 INFO:teuthology.orchestra.run.smithi022.stdout:Modify: 2024-09-21 09:19:41.220018669 +0000 2024-09-21T09:23:50.760 INFO:teuthology.orchestra.run.smithi022.stdout:Change: 2024-09-21 09:19:41.220018669 +0000 2024-09-21T09:23:50.760 INFO:teuthology.orchestra.run.smithi022.stdout: Birth: 2024-09-21 09:19:41.220018669 +0000 2024-09-21T09:23:50.761 DEBUG:teuthology.orchestra.run.smithi022:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-09-21T09:23:50.825 INFO:teuthology.orchestra.run.smithi022.stderr:1+0 records in 2024-09-21T09:23:50.825 INFO:teuthology.orchestra.run.smithi022.stderr:1+0 records out 2024-09-21T09:23:50.826 INFO:teuthology.orchestra.run.smithi022.stderr:512 bytes copied, 0.000108527 s, 4.7 MB/s 2024-09-21T09:23:50.827 DEBUG:teuthology.orchestra.run.smithi022:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-09-21T09:23:50.884 DEBUG:teuthology.orchestra.run.smithi022:> stat /dev/vg_nvme/lv_2 2024-09-21T09:23:50.940 INFO:teuthology.orchestra.run.smithi022.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-09-21T09:23:50.940 INFO:teuthology.orchestra.run.smithi022.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-21T09:23:50.940 INFO:teuthology.orchestra.run.smithi022.stdout:Device: 5h/5d Inode: 976 Links: 1 2024-09-21T09:23:50.940 INFO:teuthology.orchestra.run.smithi022.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-21T09:23:50.940 INFO:teuthology.orchestra.run.smithi022.stdout:Context: system_u:object_r:device_t:s0 2024-09-21T09:23:50.940 INFO:teuthology.orchestra.run.smithi022.stdout:Access: 2024-09-21 09:23:35.426141197 +0000 2024-09-21T09:23:50.941 INFO:teuthology.orchestra.run.smithi022.stdout:Modify: 2024-09-21 09:19:41.220018669 +0000 2024-09-21T09:23:50.941 INFO:teuthology.orchestra.run.smithi022.stdout:Change: 2024-09-21 09:19:41.220018669 +0000 2024-09-21T09:23:50.941 INFO:teuthology.orchestra.run.smithi022.stdout: Birth: 2024-09-21 09:19:41.220018669 +0000 2024-09-21T09:23:50.941 DEBUG:teuthology.orchestra.run.smithi022:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-09-21T09:23:51.004 INFO:teuthology.orchestra.run.smithi022.stderr:1+0 records in 2024-09-21T09:23:51.004 INFO:teuthology.orchestra.run.smithi022.stderr:1+0 records out 2024-09-21T09:23:51.004 INFO:teuthology.orchestra.run.smithi022.stderr:512 bytes copied, 8.7405e-05 s, 5.9 MB/s 2024-09-21T09:23:51.006 DEBUG:teuthology.orchestra.run.smithi022:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-09-21T09:23:51.062 DEBUG:teuthology.orchestra.run.smithi022:> stat /dev/vg_nvme/lv_3 2024-09-21T09:23:51.118 INFO:teuthology.orchestra.run.smithi022.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-09-21T09:23:51.118 INFO:teuthology.orchestra.run.smithi022.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-21T09:23:51.118 INFO:teuthology.orchestra.run.smithi022.stdout:Device: 5h/5d Inode: 961 Links: 1 2024-09-21T09:23:51.119 INFO:teuthology.orchestra.run.smithi022.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-21T09:23:51.119 INFO:teuthology.orchestra.run.smithi022.stdout:Context: system_u:object_r:device_t:s0 2024-09-21T09:23:51.119 INFO:teuthology.orchestra.run.smithi022.stdout:Access: 2024-09-21 09:23:35.427141184 +0000 2024-09-21T09:23:51.119 INFO:teuthology.orchestra.run.smithi022.stdout:Modify: 2024-09-21 09:19:41.218018608 +0000 2024-09-21T09:23:51.119 INFO:teuthology.orchestra.run.smithi022.stdout:Change: 2024-09-21 09:19:41.218018608 +0000 2024-09-21T09:23:51.119 INFO:teuthology.orchestra.run.smithi022.stdout: Birth: 2024-09-21 09:19:41.218018608 +0000 2024-09-21T09:23:51.119 DEBUG:teuthology.orchestra.run.smithi022:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-09-21T09:23:51.182 INFO:teuthology.orchestra.run.smithi022.stderr:1+0 records in 2024-09-21T09:23:51.182 INFO:teuthology.orchestra.run.smithi022.stderr:1+0 records out 2024-09-21T09:23:51.182 INFO:teuthology.orchestra.run.smithi022.stderr:512 bytes copied, 0.000104278 s, 4.9 MB/s 2024-09-21T09:23:51.183 DEBUG:teuthology.orchestra.run.smithi022:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-09-21T09:23:51.240 DEBUG:teuthology.orchestra.run.smithi022:> stat /dev/vg_nvme/lv_4 2024-09-21T09:23:51.295 INFO:teuthology.orchestra.run.smithi022.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-09-21T09:23:51.295 INFO:teuthology.orchestra.run.smithi022.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-21T09:23:51.295 INFO:teuthology.orchestra.run.smithi022.stdout:Device: 5h/5d Inode: 964 Links: 1 2024-09-21T09:23:51.295 INFO:teuthology.orchestra.run.smithi022.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-21T09:23:51.295 INFO:teuthology.orchestra.run.smithi022.stdout:Context: system_u:object_r:device_t:s0 2024-09-21T09:23:51.296 INFO:teuthology.orchestra.run.smithi022.stdout:Access: 2024-09-21 09:23:35.427141184 +0000 2024-09-21T09:23:51.296 INFO:teuthology.orchestra.run.smithi022.stdout:Modify: 2024-09-21 09:19:41.219018639 +0000 2024-09-21T09:23:51.296 INFO:teuthology.orchestra.run.smithi022.stdout:Change: 2024-09-21 09:19:41.219018639 +0000 2024-09-21T09:23:51.296 INFO:teuthology.orchestra.run.smithi022.stdout: Birth: 2024-09-21 09:19:41.219018639 +0000 2024-09-21T09:23:51.296 DEBUG:teuthology.orchestra.run.smithi022:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-09-21T09:23:51.359 INFO:teuthology.orchestra.run.smithi022.stderr:1+0 records in 2024-09-21T09:23:51.359 INFO:teuthology.orchestra.run.smithi022.stderr:1+0 records out 2024-09-21T09:23:51.359 INFO:teuthology.orchestra.run.smithi022.stderr:512 bytes copied, 9.9932e-05 s, 5.1 MB/s 2024-09-21T09:23:51.361 DEBUG:teuthology.orchestra.run.smithi022:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-09-21T09:23:51.417 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-09-21T09:23:51.417 DEBUG:teuthology.orchestra.run.smithi064:> dd if=/scratch_devs of=/dev/stdout 2024-09-21T09:23:51.435 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-09-21T09:23:51.435 DEBUG:teuthology.orchestra.run.smithi064:> stat /dev/vg_nvme/lv_1 2024-09-21T09:23:51.492 INFO:teuthology.orchestra.run.smithi064.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-09-21T09:23:51.492 INFO:teuthology.orchestra.run.smithi064.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-21T09:23:51.492 INFO:teuthology.orchestra.run.smithi064.stdout:Device: 5h/5d Inode: 971 Links: 1 2024-09-21T09:23:51.492 INFO:teuthology.orchestra.run.smithi064.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-21T09:23:51.492 INFO:teuthology.orchestra.run.smithi064.stdout:Context: system_u:object_r:device_t:s0 2024-09-21T09:23:51.492 INFO:teuthology.orchestra.run.smithi064.stdout:Access: 2024-09-21 09:23:26.905673244 +0000 2024-09-21T09:23:51.492 INFO:teuthology.orchestra.run.smithi064.stdout:Modify: 2024-09-21 09:19:42.091098901 +0000 2024-09-21T09:23:51.492 INFO:teuthology.orchestra.run.smithi064.stdout:Change: 2024-09-21 09:19:42.091098901 +0000 2024-09-21T09:23:51.492 INFO:teuthology.orchestra.run.smithi064.stdout: Birth: 2024-09-21 09:19:42.091098901 +0000 2024-09-21T09:23:51.493 DEBUG:teuthology.orchestra.run.smithi064:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-09-21T09:23:51.555 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:51 smithi064 ceph-mon[31124]: from='client.24106 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi022=y;smithi064=x", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:23:51.555 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:51 smithi064 ceph-mon[31124]: Saving service mgr spec with placement smithi022=y;smithi064=x;count:2 2024-09-21T09:23:51.555 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:51 smithi064 ceph-mon[31124]: Deploying daemon mgr.x on smithi064 2024-09-21T09:23:51.557 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:51 smithi022 ceph-mon[27763]: from='client.24106 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi022=y;smithi064=x", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:23:51.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:51 smithi022 ceph-mon[27763]: Saving service mgr spec with placement smithi022=y;smithi064=x;count:2 2024-09-21T09:23:51.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:51 smithi022 ceph-mon[27763]: Deploying daemon mgr.x on smithi064 2024-09-21T09:23:51.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:51 smithi022 ceph-mon[39525]: from='client.24106 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi022=y;smithi064=x", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:23:51.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:51 smithi022 ceph-mon[39525]: Saving service mgr spec with placement smithi022=y;smithi064=x;count:2 2024-09-21T09:23:51.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:51 smithi022 ceph-mon[39525]: Deploying daemon mgr.x on smithi064 2024-09-21T09:23:51.558 INFO:teuthology.orchestra.run.smithi064.stderr:1+0 records in 2024-09-21T09:23:51.559 INFO:teuthology.orchestra.run.smithi064.stderr:1+0 records out 2024-09-21T09:23:51.559 INFO:teuthology.orchestra.run.smithi064.stderr:512 bytes copied, 0.000193549 s, 2.6 MB/s 2024-09-21T09:23:51.559 DEBUG:teuthology.orchestra.run.smithi064:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-09-21T09:23:51.616 DEBUG:teuthology.orchestra.run.smithi064:> stat /dev/vg_nvme/lv_2 2024-09-21T09:23:51.672 INFO:teuthology.orchestra.run.smithi064.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-09-21T09:23:51.672 INFO:teuthology.orchestra.run.smithi064.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-21T09:23:51.672 INFO:teuthology.orchestra.run.smithi064.stdout:Device: 5h/5d Inode: 967 Links: 1 2024-09-21T09:23:51.672 INFO:teuthology.orchestra.run.smithi064.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-21T09:23:51.672 INFO:teuthology.orchestra.run.smithi064.stdout:Context: system_u:object_r:device_t:s0 2024-09-21T09:23:51.672 INFO:teuthology.orchestra.run.smithi064.stdout:Access: 2024-09-21 09:23:26.905673244 +0000 2024-09-21T09:23:51.672 INFO:teuthology.orchestra.run.smithi064.stdout:Modify: 2024-09-21 09:19:42.091098901 +0000 2024-09-21T09:23:51.672 INFO:teuthology.orchestra.run.smithi064.stdout:Change: 2024-09-21 09:19:42.091098901 +0000 2024-09-21T09:23:51.672 INFO:teuthology.orchestra.run.smithi064.stdout: Birth: 2024-09-21 09:19:42.091098901 +0000 2024-09-21T09:23:51.673 DEBUG:teuthology.orchestra.run.smithi064:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-09-21T09:23:51.736 INFO:teuthology.orchestra.run.smithi064.stderr:1+0 records in 2024-09-21T09:23:51.736 INFO:teuthology.orchestra.run.smithi064.stderr:1+0 records out 2024-09-21T09:23:51.736 INFO:teuthology.orchestra.run.smithi064.stderr:512 bytes copied, 0.000218434 s, 2.3 MB/s 2024-09-21T09:23:51.738 DEBUG:teuthology.orchestra.run.smithi064:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-09-21T09:23:51.795 DEBUG:teuthology.orchestra.run.smithi064:> stat /dev/vg_nvme/lv_3 2024-09-21T09:23:51.851 INFO:teuthology.orchestra.run.smithi064.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-09-21T09:23:51.852 INFO:teuthology.orchestra.run.smithi064.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-21T09:23:51.852 INFO:teuthology.orchestra.run.smithi064.stdout:Device: 5h/5d Inode: 975 Links: 1 2024-09-21T09:23:51.852 INFO:teuthology.orchestra.run.smithi064.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-21T09:23:51.852 INFO:teuthology.orchestra.run.smithi064.stdout:Context: system_u:object_r:device_t:s0 2024-09-21T09:23:51.852 INFO:teuthology.orchestra.run.smithi064.stdout:Access: 2024-09-21 09:23:26.906673224 +0000 2024-09-21T09:23:51.852 INFO:teuthology.orchestra.run.smithi064.stdout:Modify: 2024-09-21 09:19:42.091098901 +0000 2024-09-21T09:23:51.852 INFO:teuthology.orchestra.run.smithi064.stdout:Change: 2024-09-21 09:19:42.091098901 +0000 2024-09-21T09:23:51.852 INFO:teuthology.orchestra.run.smithi064.stdout: Birth: 2024-09-21 09:19:42.091098901 +0000 2024-09-21T09:23:51.852 DEBUG:teuthology.orchestra.run.smithi064:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-09-21T09:23:51.916 INFO:teuthology.orchestra.run.smithi064.stderr:1+0 records in 2024-09-21T09:23:51.917 INFO:teuthology.orchestra.run.smithi064.stderr:1+0 records out 2024-09-21T09:23:51.917 INFO:teuthology.orchestra.run.smithi064.stderr:512 bytes copied, 0.000199246 s, 2.6 MB/s 2024-09-21T09:23:51.918 DEBUG:teuthology.orchestra.run.smithi064:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-09-21T09:23:51.981 DEBUG:teuthology.orchestra.run.smithi064:> stat /dev/vg_nvme/lv_4 2024-09-21T09:23:52.036 INFO:teuthology.orchestra.run.smithi064.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-09-21T09:23:52.037 INFO:teuthology.orchestra.run.smithi064.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-21T09:23:52.037 INFO:teuthology.orchestra.run.smithi064.stdout:Device: 5h/5d Inode: 980 Links: 1 2024-09-21T09:23:52.037 INFO:teuthology.orchestra.run.smithi064.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-21T09:23:52.037 INFO:teuthology.orchestra.run.smithi064.stdout:Context: system_u:object_r:device_t:s0 2024-09-21T09:23:52.037 INFO:teuthology.orchestra.run.smithi064.stdout:Access: 2024-09-21 09:23:26.906673224 +0000 2024-09-21T09:23:52.037 INFO:teuthology.orchestra.run.smithi064.stdout:Modify: 2024-09-21 09:19:42.092098881 +0000 2024-09-21T09:23:52.037 INFO:teuthology.orchestra.run.smithi064.stdout:Change: 2024-09-21 09:19:42.092098881 +0000 2024-09-21T09:23:52.037 INFO:teuthology.orchestra.run.smithi064.stdout: Birth: 2024-09-21 09:19:42.092098881 +0000 2024-09-21T09:23:52.037 DEBUG:teuthology.orchestra.run.smithi064:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-09-21T09:23:52.103 INFO:teuthology.orchestra.run.smithi064.stderr:1+0 records in 2024-09-21T09:23:52.103 INFO:teuthology.orchestra.run.smithi064.stderr:1+0 records out 2024-09-21T09:23:52.103 INFO:teuthology.orchestra.run.smithi064.stderr:512 bytes copied, 0.000178213 s, 2.9 MB/s 2024-09-21T09:23:52.104 DEBUG:teuthology.orchestra.run.smithi064:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-09-21T09:23:52.168 INFO:tasks.cephadm:Deploying osd.0 on smithi022 with /dev/vg_nvme/lv_4... 2024-09-21T09:23:52.168 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d65e801e-77fa-11ef-baf6-efdc52797490 -- lvm zap /dev/vg_nvme/lv_4 2024-09-21T09:23:52.430 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:23:52.534 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:52 smithi064 ceph-mon[31124]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:23:52.534 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:23:52 smithi064 podman[33198]: 2024-09-21 09:23:52.532767901 +0000 UTC m=+0.143765516 container create c7e735bed227093eb6967a1f874f3ca6f0ce3147269d5c2af46a72c02ac3dcf0 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=quincy-fbd1a6a, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, ceph=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T09:23:52.551 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:52 smithi022 ceph-mon[27763]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:23:52.551 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:52 smithi022 ceph-mon[39525]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:23:52.796 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:23:52 smithi064 podman[33198]: 2024-09-21 09:23:52.768290796 +0000 UTC m=+0.379288388 container init c7e735bed227093eb6967a1f874f3ca6f0ce3147269d5c2af46a72c02ac3dcf0 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=quincy-fbd1a6a, org.label-schema.license=GPLv2, ceph=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS) 2024-09-21T09:23:52.796 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:23:52 smithi064 podman[33198]: 2024-09-21 09:23:52.773288093 +0000 UTC m=+0.384285666 container start c7e735bed227093eb6967a1f874f3ca6f0ce3147269d5c2af46a72c02ac3dcf0 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x, GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, RELEASE=quincy-fbd1a6a, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_CLEAN=True) 2024-09-21T09:23:53.154 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:23:52 smithi064 ceph-mgr[33236]: -- 172.21.15.64:0/644452886 <== mon.1 v2:172.21.15.64:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55959bb72340 con 0x55959bb7e800 2024-09-21T09:23:53.154 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:23:52 smithi064 bash[33198]: c7e735bed227093eb6967a1f874f3ca6f0ce3147269d5c2af46a72c02ac3dcf0 2024-09-21T09:23:53.154 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:23:52 smithi064 systemd[1]: Started Ceph mgr.x for d65e801e-77fa-11ef-baf6-efdc52797490. 2024-09-21T09:23:53.155 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:23:52 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[33213]: 2024-09-21T09:23:52.891+0000 7f85ea59dec0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-21T09:23:53.155 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:23:52 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[33213]: 2024-09-21T09:23:52.951+0000 7f85ea59dec0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-21T09:23:53.155 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:23:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[33213]: 2024-09-21T09:23:53.020+0000 7f85ea59dec0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-21T09:23:53.155 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:23:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[33213]: 2024-09-21T09:23:53.152+0000 7f85ea59dec0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-21T09:23:53.441 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:23:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[33213]: 2024-09-21T09:23:53.303+0000 7f85ea59dec0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-21T09:23:53.881 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:23:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[33213]: 2024-09-21T09:23:53.591+0000 7f85ea59dec0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-21T09:23:53.881 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:23:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[33213]: 2024-09-21T09:23:53.661+0000 7f85ea59dec0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-21T09:23:53.881 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:23:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[33213]: 2024-09-21T09:23:53.834+0000 7f85ea59dec0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-21T09:23:54.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:53 smithi064 ceph-mon[31124]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:23:54.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:53 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:54.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:53 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:54.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:53 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:54.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:53 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:54.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:53 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:23:54.196 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:53 smithi022 ceph-mon[39525]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:23:54.196 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:53 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:54.197 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:53 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:54.197 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:53 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:54.197 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:53 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:54.197 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:53 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:23:54.197 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:53 smithi022 ceph-mon[27763]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:23:54.197 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:53 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:54.197 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:53 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:54.197 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:53 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:54.197 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:53 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:54.197 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:53 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:23:55.074 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-09-21T09:23:55.096 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d65e801e-77fa-11ef-baf6-efdc52797490 -- ceph orch daemon add osd smithi022:vg_nvme/lv_4 2024-09-21T09:23:55.159 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:23:54 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[33213]: 2024-09-21T09:23:54.735+0000 7f85ea59dec0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-21T09:23:55.159 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:23:54 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[33213]: 2024-09-21T09:23:54.802+0000 7f85ea59dec0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-21T09:23:55.398 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:55 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:55.398 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:55 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:55.398 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:55 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:23:55.399 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:55 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:23:55.399 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:55 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:55.399 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:55 smithi022 ceph-mon[39525]: Reconfiguring mgr.y (unknown last config time)... 2024-09-21T09:23:55.399 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:55 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T09:23:55.399 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:55 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T09:23:55.399 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:55 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:23:55.399 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:55 smithi022 ceph-mon[39525]: Reconfiguring daemon mgr.y on smithi022 2024-09-21T09:23:55.400 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:55 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:55.400 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:55 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:55.400 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:55 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:23:55.400 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:55 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:23:55.400 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:55 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:55.400 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:55 smithi022 ceph-mon[27763]: Reconfiguring mgr.y (unknown last config time)... 2024-09-21T09:23:55.401 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:55 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T09:23:55.401 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:55 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T09:23:55.401 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:55 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:23:55.401 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:55 smithi022 ceph-mon[27763]: Reconfiguring daemon mgr.y on smithi022 2024-09-21T09:23:55.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:55 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:55.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:55 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:55.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:55 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:23:55.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:55 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:23:55.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:55 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:55.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:55 smithi064 ceph-mon[31124]: Reconfiguring mgr.y (unknown last config time)... 2024-09-21T09:23:55.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:55 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T09:23:55.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:55 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T09:23:55.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:55 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:23:55.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:55 smithi064 ceph-mon[31124]: Reconfiguring daemon mgr.y on smithi022 2024-09-21T09:23:55.846 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:23:55.862 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:23:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[33213]: 2024-09-21T09:23:55.556+0000 7f85ea59dec0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-21T09:23:55.862 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:23:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[33213]: 2024-09-21T09:23:55.673+0000 7f85ea59dec0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-21T09:23:55.862 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:23:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[33213]: 2024-09-21T09:23:55.798+0000 7f85ea59dec0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-21T09:23:55.862 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:23:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[33213]: 2024-09-21T09:23:55.860+0000 7f85ea59dec0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-21T09:23:56.124 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:23:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[33213]: 2024-09-21T09:23:55.992+0000 7f85ea59dec0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-21T09:23:56.124 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:23:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[33213]: 2024-09-21T09:23:56.061+0000 7f85ea59dec0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-21T09:23:56.124 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:23:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[33213]: 2024-09-21T09:23:56.122+0000 7f85ea59dec0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-21T09:23:56.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:56 smithi022 ceph-mon[27763]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:23:56.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:56 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:56.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:56 smithi022 ceph-mon[39525]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:23:56.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:56 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:56.398 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:56 smithi064 ceph-mon[31124]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:23:56.399 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:56 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:56.399 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:23:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[33213]: 2024-09-21T09:23:56.263+0000 7f85ea59dec0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-21T09:23:56.399 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:23:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[33213]: 2024-09-21T09:23:56.329+0000 7f85ea59dec0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-21T09:23:56.399 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:23:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[33213]: 2024-09-21T09:23:56.397+0000 7f85ea59dec0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-21T09:23:56.691 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:23:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[33213]: 2024-09-21T09:23:56.481+0000 7f85ea59dec0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-21T09:23:56.691 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:23:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[33213]: 2024-09-21T09:23:56.541+0000 7f85ea59dec0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-21T09:23:56.691 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:23:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[33213]: 2024-09-21T09:23:56.601+0000 7f85ea59dec0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-21T09:23:57.691 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:23:57 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[33213]: 2024-09-21T09:23:57.386+0000 7f85ea59dec0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-21T09:23:57.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:57 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:57.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:57 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:57.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:57 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:23:57.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:57 smithi064 ceph-mon[31124]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:23:57.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:57 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:23:57.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:57 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:23:57.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:57 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:57.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:57 smithi064 ceph-mon[31124]: Standby manager daemon x started 2024-09-21T09:23:57.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:57 smithi064 ceph-mon[31124]: from='mgr.? 172.21.15.64:0/20392080' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-21T09:23:57.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:57 smithi064 ceph-mon[31124]: from='mgr.? 172.21.15.64:0/20392080' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:23:57.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:57 smithi064 ceph-mon[31124]: from='mgr.? 172.21.15.64:0/20392080' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-21T09:23:57.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:57 smithi064 ceph-mon[31124]: from='mgr.? 172.21.15.64:0/20392080' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:23:57.783 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:57 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:57.783 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:57 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:57.783 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:57 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:23:57.783 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:57 smithi022 ceph-mon[27763]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:23:57.783 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:57 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:23:57.783 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:57 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:23:57.783 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:57 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:57.783 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:57 smithi022 ceph-mon[27763]: Standby manager daemon x started 2024-09-21T09:23:57.783 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:57 smithi022 ceph-mon[27763]: from='mgr.? 172.21.15.64:0/20392080' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-21T09:23:57.783 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:57 smithi022 ceph-mon[27763]: from='mgr.? 172.21.15.64:0/20392080' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:23:57.783 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:57 smithi022 ceph-mon[27763]: from='mgr.? 172.21.15.64:0/20392080' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-21T09:23:57.783 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:57 smithi022 ceph-mon[27763]: from='mgr.? 172.21.15.64:0/20392080' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:23:57.784 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:57 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:57.784 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:57 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:57.784 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:57 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:23:57.784 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:57 smithi022 ceph-mon[39525]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:23:57.784 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:57 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:23:57.784 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:57 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:23:57.784 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:57 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:23:57.784 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:57 smithi022 ceph-mon[39525]: Standby manager daemon x started 2024-09-21T09:23:57.784 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:57 smithi022 ceph-mon[39525]: from='mgr.? 172.21.15.64:0/20392080' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-21T09:23:57.784 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:57 smithi022 ceph-mon[39525]: from='mgr.? 172.21.15.64:0/20392080' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:23:57.785 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:57 smithi022 ceph-mon[39525]: from='mgr.? 172.21.15.64:0/20392080' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-21T09:23:57.785 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:57 smithi022 ceph-mon[39525]: from='mgr.? 172.21.15.64:0/20392080' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:23:59.257 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:59 smithi022 ceph-mon[39525]: mgrmap e14: y(active, since 67s), standbys: x 2024-09-21T09:23:59.257 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:59 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-21T09:23:59.257 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:59 smithi022 ceph-mon[39525]: from='client.24116 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi022:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:23:59.258 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:59 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:23:59.258 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:59 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:23:59.258 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:23:59 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:23:59.258 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:59 smithi022 ceph-mon[27763]: mgrmap e14: y(active, since 67s), standbys: x 2024-09-21T09:23:59.258 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:59 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-21T09:23:59.258 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:59 smithi022 ceph-mon[27763]: from='client.24116 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi022:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:23:59.259 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:59 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:23:59.259 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:59 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:23:59.259 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:23:59 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:23:59.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:59 smithi064 ceph-mon[31124]: mgrmap e14: y(active, since 67s), standbys: x 2024-09-21T09:23:59.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:59 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-21T09:23:59.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:59 smithi064 ceph-mon[31124]: from='client.24116 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi022:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:23:59.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:59 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:23:59.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:59 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:23:59.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:23:59 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:24:00.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:00 smithi064 ceph-mon[31124]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:24:00.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:00 smithi022 ceph-mon[27763]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:24:00.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:00 smithi022 ceph-mon[39525]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:24:01.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:01 smithi064 ceph-mon[31124]: from='client.? 172.21.15.22:0/3077293398' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a1fbb74d-ffae-4d5f-81ab-9677d258a95c"}]: dispatch 2024-09-21T09:24:01.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:01 smithi064 ceph-mon[31124]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a1fbb74d-ffae-4d5f-81ab-9677d258a95c"}]: dispatch 2024-09-21T09:24:01.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:01 smithi064 ceph-mon[31124]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a1fbb74d-ffae-4d5f-81ab-9677d258a95c"}]': finished 2024-09-21T09:24:01.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:01 smithi064 ceph-mon[31124]: osdmap e5: 1 total, 0 up, 1 in 2024-09-21T09:24:01.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:01 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:24:01.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:01 smithi064 ceph-mon[31124]: from='client.? 172.21.15.22:0/1262044397' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:24:01.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:01 smithi022 ceph-mon[27763]: from='client.? 172.21.15.22:0/3077293398' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a1fbb74d-ffae-4d5f-81ab-9677d258a95c"}]: dispatch 2024-09-21T09:24:01.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:01 smithi022 ceph-mon[27763]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a1fbb74d-ffae-4d5f-81ab-9677d258a95c"}]: dispatch 2024-09-21T09:24:01.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:01 smithi022 ceph-mon[27763]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a1fbb74d-ffae-4d5f-81ab-9677d258a95c"}]': finished 2024-09-21T09:24:01.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:01 smithi022 ceph-mon[27763]: osdmap e5: 1 total, 0 up, 1 in 2024-09-21T09:24:01.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:01 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:24:01.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:01 smithi022 ceph-mon[27763]: from='client.? 172.21.15.22:0/1262044397' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:24:01.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:01 smithi022 ceph-mon[39525]: from='client.? 172.21.15.22:0/3077293398' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a1fbb74d-ffae-4d5f-81ab-9677d258a95c"}]: dispatch 2024-09-21T09:24:01.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:01 smithi022 ceph-mon[39525]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a1fbb74d-ffae-4d5f-81ab-9677d258a95c"}]: dispatch 2024-09-21T09:24:01.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:01 smithi022 ceph-mon[39525]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a1fbb74d-ffae-4d5f-81ab-9677d258a95c"}]': finished 2024-09-21T09:24:01.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:01 smithi022 ceph-mon[39525]: osdmap e5: 1 total, 0 up, 1 in 2024-09-21T09:24:01.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:01 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:24:01.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:01 smithi022 ceph-mon[39525]: from='client.? 172.21.15.22:0/1262044397' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:24:02.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:02 smithi064 ceph-mon[31124]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:24:02.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:02 smithi022 ceph-mon[39525]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:24:02.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:02 smithi022 ceph-mon[27763]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:24:04.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:04 smithi022 ceph-mon[27763]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:24:04.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:04 smithi022 ceph-mon[39525]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:24:04.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:04 smithi064 ceph-mon[31124]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:24:06.388 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:06 smithi022 ceph-mon[27763]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:24:06.388 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:06 smithi022 ceph-mon[39525]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:24:06.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:06 smithi064 ceph-mon[31124]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:24:07.339 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:07 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-21T09:24:07.339 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:07 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:24:07.340 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:07 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-21T09:24:07.340 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:07 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:24:07.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:07 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-21T09:24:07.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:07 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:24:08.417 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:08 smithi022 ceph-mon[39525]: Deploying daemon osd.0 on smithi022 2024-09-21T09:24:08.417 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:08 smithi022 ceph-mon[39525]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:24:08.417 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:08 smithi022 ceph-mon[27763]: Deploying daemon osd.0 on smithi022 2024-09-21T09:24:08.417 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:08 smithi022 ceph-mon[27763]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:24:08.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:08 smithi064 ceph-mon[31124]: Deploying daemon osd.0 on smithi022 2024-09-21T09:24:08.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:08 smithi064 ceph-mon[31124]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:24:10.238 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:10 smithi022 ceph-mon[27763]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:24:10.239 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:10 smithi022 ceph-mon[39525]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:24:10.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:10 smithi064 ceph-mon[31124]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:24:12.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:12 smithi064 ceph-mon[31124]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:24:12.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:12 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:24:12.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:12 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:24:12.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:12 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:24:12.448 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:12 smithi022 ceph-mon[27763]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:24:12.448 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:12 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:24:12.448 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:12 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:24:12.448 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:12 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:24:12.449 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:12 smithi022 ceph-mon[39525]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:24:12.449 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:12 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:24:12.449 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:12 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:24:12.449 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:12 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:24:14.437 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:14 smithi022 ceph-mon[27763]: pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:24:14.438 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:14 smithi022 ceph-mon[39525]: pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:24:14.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:14 smithi064 ceph-mon[31124]: pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:24:15.055 INFO:teuthology.orchestra.run.smithi022.stdout:Created osd(s) 0 on host 'smithi022' 2024-09-21T09:24:15.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:15 smithi022 ceph-mon[27763]: from='osd.0 [v2:172.21.15.22:6802/3576301437,v1:172.21.15.22:6803/3576301437]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-21T09:24:15.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:15 smithi022 ceph-mon[27763]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-21T09:24:15.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:15 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:24:15.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:15 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:24:15.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:15 smithi022 ceph-mon[39525]: from='osd.0 [v2:172.21.15.22:6802/3576301437,v1:172.21.15.22:6803/3576301437]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-21T09:24:15.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:15 smithi022 ceph-mon[39525]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-21T09:24:15.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:15 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:24:15.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:15 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:24:15.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:15 smithi064 ceph-mon[31124]: from='osd.0 [v2:172.21.15.22:6802/3576301437,v1:172.21.15.22:6803/3576301437]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-21T09:24:15.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:15 smithi064 ceph-mon[31124]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-21T09:24:15.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:15 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:24:15.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:15 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:24:15.703 DEBUG:teuthology.orchestra.run.smithi022:osd.0> sudo journalctl -f -n 0 -u ceph-d65e801e-77fa-11ef-baf6-efdc52797490@osd.0.service 2024-09-21T09:24:15.705 INFO:tasks.cephadm:Deploying osd.1 on smithi022 with /dev/vg_nvme/lv_3... 2024-09-21T09:24:15.705 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d65e801e-77fa-11ef-baf6-efdc52797490 -- lvm zap /dev/vg_nvme/lv_3 2024-09-21T09:24:16.017 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:24:16.414 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 21 09:24:16 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-0[43934]: 2024-09-21T09:24:16.158+0000 7ff75c0d0640 -1 osd.0 0 waiting for initial osdmap 2024-09-21T09:24:16.414 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 21 09:24:16 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-0[43934]: 2024-09-21T09:24:16.160+0000 7ff7536de640 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T09:24:16.414 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:16 smithi022 ceph-mon[39525]: pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:24:16.414 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:16 smithi022 ceph-mon[39525]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-21T09:24:16.414 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:16 smithi022 ceph-mon[39525]: osdmap e6: 1 total, 0 up, 1 in 2024-09-21T09:24:16.414 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:16 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:24:16.414 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:16 smithi022 ceph-mon[39525]: from='osd.0 [v2:172.21.15.22:6802/3576301437,v1:172.21.15.22:6803/3576301437]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]: dispatch 2024-09-21T09:24:16.414 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:16 smithi022 ceph-mon[39525]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]: dispatch 2024-09-21T09:24:16.415 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:16 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:24:16.415 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:16 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:24:16.415 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:16 smithi022 ceph-mon[27763]: pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:24:16.415 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:16 smithi022 ceph-mon[27763]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-21T09:24:16.415 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:16 smithi022 ceph-mon[27763]: osdmap e6: 1 total, 0 up, 1 in 2024-09-21T09:24:16.415 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:16 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:24:16.415 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:16 smithi022 ceph-mon[27763]: from='osd.0 [v2:172.21.15.22:6802/3576301437,v1:172.21.15.22:6803/3576301437]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]: dispatch 2024-09-21T09:24:16.415 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:16 smithi022 ceph-mon[27763]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]: dispatch 2024-09-21T09:24:16.415 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:16 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:24:16.415 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:16 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:24:16.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:16 smithi064 ceph-mon[31124]: pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:24:16.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:16 smithi064 ceph-mon[31124]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-21T09:24:16.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:16 smithi064 ceph-mon[31124]: osdmap e6: 1 total, 0 up, 1 in 2024-09-21T09:24:16.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:16 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:24:16.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:16 smithi064 ceph-mon[31124]: from='osd.0 [v2:172.21.15.22:6802/3576301437,v1:172.21.15.22:6803/3576301437]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]: dispatch 2024-09-21T09:24:16.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:16 smithi064 ceph-mon[31124]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]: dispatch 2024-09-21T09:24:16.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:16 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:24:16.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:16 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:24:17.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:17 smithi064 ceph-mon[31124]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]': finished 2024-09-21T09:24:17.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:17 smithi064 ceph-mon[31124]: osdmap e7: 1 total, 0 up, 1 in 2024-09-21T09:24:17.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:17 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:24:17.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:17 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:24:17.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:17 smithi022 ceph-mon[27763]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]': finished 2024-09-21T09:24:17.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:17 smithi022 ceph-mon[27763]: osdmap e7: 1 total, 0 up, 1 in 2024-09-21T09:24:17.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:17 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:24:17.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:17 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:24:17.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:17 smithi022 ceph-mon[39525]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]': finished 2024-09-21T09:24:17.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:17 smithi022 ceph-mon[39525]: osdmap e7: 1 total, 0 up, 1 in 2024-09-21T09:24:17.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:17 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:24:17.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:17 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:24:18.441 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:18 smithi022 ceph-mon[27763]: purged_snaps scrub starts 2024-09-21T09:24:18.441 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:18 smithi022 ceph-mon[27763]: purged_snaps scrub ok 2024-09-21T09:24:18.441 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:18 smithi022 ceph-mon[27763]: pgmap v40: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:24:18.441 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:18 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:24:18.442 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:18 smithi022 ceph-mon[27763]: osd.0 [v2:172.21.15.22:6802/3576301437,v1:172.21.15.22:6803/3576301437] boot 2024-09-21T09:24:18.442 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:18 smithi022 ceph-mon[27763]: osdmap e8: 1 total, 1 up, 1 in 2024-09-21T09:24:18.442 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:18 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:24:18.442 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:18 smithi022 ceph-mon[39525]: purged_snaps scrub starts 2024-09-21T09:24:18.442 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:18 smithi022 ceph-mon[39525]: purged_snaps scrub ok 2024-09-21T09:24:18.442 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:18 smithi022 ceph-mon[39525]: pgmap v40: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:24:18.442 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:18 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:24:18.442 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:18 smithi022 ceph-mon[39525]: osd.0 [v2:172.21.15.22:6802/3576301437,v1:172.21.15.22:6803/3576301437] boot 2024-09-21T09:24:18.442 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:18 smithi022 ceph-mon[39525]: osdmap e8: 1 total, 1 up, 1 in 2024-09-21T09:24:18.442 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:18 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:24:18.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:18 smithi064 ceph-mon[31124]: purged_snaps scrub starts 2024-09-21T09:24:18.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:18 smithi064 ceph-mon[31124]: purged_snaps scrub ok 2024-09-21T09:24:18.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:18 smithi064 ceph-mon[31124]: pgmap v40: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:24:18.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:18 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:24:18.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:18 smithi064 ceph-mon[31124]: osd.0 [v2:172.21.15.22:6802/3576301437,v1:172.21.15.22:6803/3576301437] boot 2024-09-21T09:24:18.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:18 smithi064 ceph-mon[31124]: osdmap e8: 1 total, 1 up, 1 in 2024-09-21T09:24:18.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:18 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:24:19.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:19 smithi022 ceph-mon[27763]: osdmap e9: 1 total, 1 up, 1 in 2024-09-21T09:24:19.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:19 smithi022 ceph-mon[39525]: osdmap e9: 1 total, 1 up, 1 in 2024-09-21T09:24:19.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:19 smithi064 ceph-mon[31124]: osdmap e9: 1 total, 1 up, 1 in 2024-09-21T09:24:19.670 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-09-21T09:24:19.693 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d65e801e-77fa-11ef-baf6-efdc52797490 -- ceph orch daemon add osd smithi022:vg_nvme/lv_3 2024-09-21T09:24:20.259 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:24:20.307 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:20 smithi022 ceph-mon[27763]: pgmap v43: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:24:20.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:20 smithi022 ceph-mon[39525]: pgmap v43: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:24:20.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:20 smithi064 ceph-mon[31124]: pgmap v43: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:24:22.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:22 smithi022 ceph-mon[39525]: pgmap v44: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:24:22.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:22 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:24:22.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:22 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:24:22.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:22 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:24:22.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:22 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:24:22.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:22 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:24:22.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:22 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:24:22.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:22 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:24:22.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:22 smithi022 ceph-mon[27763]: pgmap v44: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:24:22.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:22 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:24:22.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:22 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:24:22.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:22 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:24:22.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:22 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:24:22.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:22 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:24:22.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:22 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:24:22.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:22 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:24:22.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:22 smithi064 ceph-mon[31124]: pgmap v44: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:24:22.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:22 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:24:22.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:22 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:24:22.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:22 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:24:22.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:22 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:24:22.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:22 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:24:22.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:22 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:24:22.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:22 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:24:23.434 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:23 smithi022 ceph-mon[39525]: Detected new or changed devices on smithi022 2024-09-21T09:24:23.434 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:23 smithi022 ceph-mon[39525]: Adjusting osd_memory_target on smithi022 to 16068M 2024-09-21T09:24:23.434 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:23 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:24:23.434 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:23 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:24:23.434 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:23 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:24:23.434 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:23 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:24:23.434 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:23 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:24:23.434 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:23 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:24:23.434 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:23 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:24:23.435 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:23 smithi022 ceph-mon[27763]: Detected new or changed devices on smithi022 2024-09-21T09:24:23.435 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:23 smithi022 ceph-mon[27763]: Adjusting osd_memory_target on smithi022 to 16068M 2024-09-21T09:24:23.435 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:23 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:24:23.435 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:23 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:24:23.435 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:23 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:24:23.435 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:23 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:24:23.435 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:23 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:24:23.435 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:23 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:24:23.435 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:23 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:24:23.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:23 smithi064 ceph-mon[31124]: Detected new or changed devices on smithi022 2024-09-21T09:24:23.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:23 smithi064 ceph-mon[31124]: Adjusting osd_memory_target on smithi022 to 16068M 2024-09-21T09:24:23.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:23 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:24:23.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:23 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:24:23.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:23 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:24:23.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:23 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:24:23.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:23 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:24:23.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:23 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:24:23.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:23 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:24:24.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:24 smithi064 ceph-mon[31124]: from='client.14238 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi022:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:24:24.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:24 smithi064 ceph-mon[31124]: pgmap v45: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:24:24.446 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:24 smithi022 ceph-mon[39525]: from='client.14238 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi022:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:24:24.446 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:24 smithi022 ceph-mon[39525]: pgmap v45: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:24:24.446 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:24 smithi022 ceph-mon[27763]: from='client.14238 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi022:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:24:24.446 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:24 smithi022 ceph-mon[27763]: pgmap v45: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:24:25.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:25 smithi064 ceph-mon[31124]: from='client.? 172.21.15.22:0/898007389' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fa4c9385-ac93-4983-92b4-f23f0a3731db"}]: dispatch 2024-09-21T09:24:25.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:25 smithi064 ceph-mon[31124]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fa4c9385-ac93-4983-92b4-f23f0a3731db"}]: dispatch 2024-09-21T09:24:25.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:25 smithi064 ceph-mon[31124]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "fa4c9385-ac93-4983-92b4-f23f0a3731db"}]': finished 2024-09-21T09:24:25.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:25 smithi064 ceph-mon[31124]: osdmap e10: 2 total, 1 up, 2 in 2024-09-21T09:24:25.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:25 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:24:25.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:25 smithi022 ceph-mon[27763]: from='client.? 172.21.15.22:0/898007389' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fa4c9385-ac93-4983-92b4-f23f0a3731db"}]: dispatch 2024-09-21T09:24:25.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:25 smithi022 ceph-mon[27763]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fa4c9385-ac93-4983-92b4-f23f0a3731db"}]: dispatch 2024-09-21T09:24:25.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:25 smithi022 ceph-mon[27763]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "fa4c9385-ac93-4983-92b4-f23f0a3731db"}]': finished 2024-09-21T09:24:25.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:25 smithi022 ceph-mon[27763]: osdmap e10: 2 total, 1 up, 2 in 2024-09-21T09:24:25.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:25 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:24:25.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:25 smithi022 ceph-mon[39525]: from='client.? 172.21.15.22:0/898007389' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fa4c9385-ac93-4983-92b4-f23f0a3731db"}]: dispatch 2024-09-21T09:24:25.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:25 smithi022 ceph-mon[39525]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fa4c9385-ac93-4983-92b4-f23f0a3731db"}]: dispatch 2024-09-21T09:24:25.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:25 smithi022 ceph-mon[39525]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "fa4c9385-ac93-4983-92b4-f23f0a3731db"}]': finished 2024-09-21T09:24:25.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:25 smithi022 ceph-mon[39525]: osdmap e10: 2 total, 1 up, 2 in 2024-09-21T09:24:25.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:25 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:24:26.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:26 smithi064 ceph-mon[31124]: pgmap v46: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:24:26.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:26 smithi064 ceph-mon[31124]: from='client.? 172.21.15.22:0/3880824955' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:24:26.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:26 smithi022 ceph-mon[27763]: pgmap v46: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:24:26.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:26 smithi022 ceph-mon[27763]: from='client.? 172.21.15.22:0/3880824955' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:24:26.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:26 smithi022 ceph-mon[39525]: pgmap v46: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:24:26.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:26 smithi022 ceph-mon[39525]: from='client.? 172.21.15.22:0/3880824955' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:24:28.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:28 smithi064 ceph-mon[31124]: pgmap v48: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:24:28.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:28 smithi022 ceph-mon[27763]: pgmap v48: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:24:28.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:28 smithi022 ceph-mon[39525]: pgmap v48: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:24:30.298 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:30 smithi022 ceph-mon[27763]: pgmap v49: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:24:30.298 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:30 smithi022 ceph-mon[39525]: pgmap v49: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:24:30.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:30 smithi064 ceph-mon[31124]: pgmap v49: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:24:32.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:32 smithi022 ceph-mon[39525]: pgmap v50: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:24:32.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:32 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-21T09:24:32.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:32 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:24:32.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:32 smithi022 ceph-mon[39525]: Deploying daemon osd.1 on smithi022 2024-09-21T09:24:32.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:32 smithi022 ceph-mon[27763]: pgmap v50: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:24:32.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:32 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-21T09:24:32.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:32 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:24:32.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:32 smithi022 ceph-mon[27763]: Deploying daemon osd.1 on smithi022 2024-09-21T09:24:32.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:32 smithi064 ceph-mon[31124]: pgmap v50: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:24:32.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:32 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-21T09:24:32.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:32 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:24:32.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:32 smithi064 ceph-mon[31124]: Deploying daemon osd.1 on smithi022 2024-09-21T09:24:34.285 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:34 smithi022 ceph-mon[27763]: pgmap v51: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:24:34.285 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:34 smithi022 ceph-mon[39525]: pgmap v51: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:24:34.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:34 smithi064 ceph-mon[31124]: pgmap v51: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:24:36.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:36 smithi022 ceph-mon[27763]: pgmap v52: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:24:36.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:36 smithi022 ceph-mon[39525]: pgmap v52: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:24:36.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:36 smithi064 ceph-mon[31124]: pgmap v52: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:24:37.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:37 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:24:37.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:37 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:24:37.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:37 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:24:37.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:37 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:24:37.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:37 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:24:37.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:37 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:24:37.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:37 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:24:37.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:37 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:24:37.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:37 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:24:38.486 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:38 smithi022 ceph-mon[27763]: pgmap v53: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:24:38.486 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:38 smithi022 ceph-mon[39525]: pgmap v53: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:24:38.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:38 smithi064 ceph-mon[31124]: pgmap v53: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:24:39.812 INFO:teuthology.orchestra.run.smithi022.stdout:Created osd(s) 1 on host 'smithi022' 2024-09-21T09:24:40.422 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:40 smithi022 ceph-mon[27763]: pgmap v54: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:24:40.422 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:40 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:24:40.422 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:40 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:24:40.422 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:40 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:24:40.422 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:40 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:24:40.422 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:40 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:24:40.422 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:40 smithi022 ceph-mon[27763]: from='osd.1 [v2:172.21.15.22:6810/3562679765,v1:172.21.15.22:6811/3562679765]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-21T09:24:40.422 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:40 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:24:40.422 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:40 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:24:40.423 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:40 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:24:40.423 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:40 smithi022 ceph-mon[39525]: pgmap v54: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:24:40.423 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:40 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:24:40.423 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:40 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:24:40.423 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:40 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:24:40.423 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:40 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:24:40.423 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:40 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:24:40.423 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:40 smithi022 ceph-mon[39525]: from='osd.1 [v2:172.21.15.22:6810/3562679765,v1:172.21.15.22:6811/3562679765]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-21T09:24:40.423 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:40 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:24:40.423 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:40 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:24:40.423 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:40 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:24:40.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:40 smithi064 ceph-mon[31124]: pgmap v54: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:24:40.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:40 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:24:40.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:40 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:24:40.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:40 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:24:40.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:40 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:24:40.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:40 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:24:40.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:40 smithi064 ceph-mon[31124]: from='osd.1 [v2:172.21.15.22:6810/3562679765,v1:172.21.15.22:6811/3562679765]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-21T09:24:40.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:40 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:24:40.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:40 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:24:40.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:40 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:24:40.447 DEBUG:teuthology.orchestra.run.smithi022:osd.1> sudo journalctl -f -n 0 -u ceph-d65e801e-77fa-11ef-baf6-efdc52797490@osd.1.service 2024-09-21T09:24:40.449 INFO:tasks.cephadm:Deploying osd.2 on smithi022 with /dev/vg_nvme/lv_2... 2024-09-21T09:24:40.449 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d65e801e-77fa-11ef-baf6-efdc52797490 -- lvm zap /dev/vg_nvme/lv_2 2024-09-21T09:24:41.259 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:24:41.499 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:41 smithi022 ceph-mon[39525]: from='osd.1 [v2:172.21.15.22:6810/3562679765,v1:172.21.15.22:6811/3562679765]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-21T09:24:41.499 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:41 smithi022 ceph-mon[39525]: osdmap e11: 2 total, 1 up, 2 in 2024-09-21T09:24:41.499 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:41 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:24:41.499 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:41 smithi022 ceph-mon[39525]: from='osd.1 [v2:172.21.15.22:6810/3562679765,v1:172.21.15.22:6811/3562679765]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]: dispatch 2024-09-21T09:24:41.499 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:41 smithi022 ceph-mon[27763]: from='osd.1 [v2:172.21.15.22:6810/3562679765,v1:172.21.15.22:6811/3562679765]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-21T09:24:41.499 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:41 smithi022 ceph-mon[27763]: osdmap e11: 2 total, 1 up, 2 in 2024-09-21T09:24:41.499 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:41 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:24:41.499 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:41 smithi022 ceph-mon[27763]: from='osd.1 [v2:172.21.15.22:6810/3562679765,v1:172.21.15.22:6811/3562679765]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]: dispatch 2024-09-21T09:24:41.500 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 21 09:24:41 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-1[50027]: 2024-09-21T09:24:41.202+0000 7f3a7ce62640 -1 osd.1 0 waiting for initial osdmap 2024-09-21T09:24:41.500 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 21 09:24:41 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-1[50027]: 2024-09-21T09:24:41.206+0000 7f3a77498640 -1 osd.1 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T09:24:41.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:41 smithi064 ceph-mon[31124]: from='osd.1 [v2:172.21.15.22:6810/3562679765,v1:172.21.15.22:6811/3562679765]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-21T09:24:41.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:41 smithi064 ceph-mon[31124]: osdmap e11: 2 total, 1 up, 2 in 2024-09-21T09:24:41.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:41 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:24:41.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:41 smithi064 ceph-mon[31124]: from='osd.1 [v2:172.21.15.22:6810/3562679765,v1:172.21.15.22:6811/3562679765]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]: dispatch 2024-09-21T09:24:42.431 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:42 smithi022 ceph-mon[39525]: pgmap v56: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:24:42.431 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:42 smithi022 ceph-mon[39525]: from='osd.1 [v2:172.21.15.22:6810/3562679765,v1:172.21.15.22:6811/3562679765]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]': finished 2024-09-21T09:24:42.431 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:42 smithi022 ceph-mon[39525]: osdmap e12: 2 total, 1 up, 2 in 2024-09-21T09:24:42.431 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:42 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:24:42.432 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:42 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:24:42.432 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:42 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:24:42.432 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:42 smithi022 ceph-mon[27763]: pgmap v56: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:24:42.432 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:42 smithi022 ceph-mon[27763]: from='osd.1 [v2:172.21.15.22:6810/3562679765,v1:172.21.15.22:6811/3562679765]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]': finished 2024-09-21T09:24:42.432 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:42 smithi022 ceph-mon[27763]: osdmap e12: 2 total, 1 up, 2 in 2024-09-21T09:24:42.432 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:42 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:24:42.432 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:42 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:24:42.432 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:42 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:24:42.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:42 smithi064 ceph-mon[31124]: pgmap v56: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:24:42.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:42 smithi064 ceph-mon[31124]: from='osd.1 [v2:172.21.15.22:6810/3562679765,v1:172.21.15.22:6811/3562679765]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]': finished 2024-09-21T09:24:42.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:42 smithi064 ceph-mon[31124]: osdmap e12: 2 total, 1 up, 2 in 2024-09-21T09:24:42.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:42 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:24:42.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:42 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:24:42.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:42 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:24:43.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:43 smithi022 ceph-mon[39525]: purged_snaps scrub starts 2024-09-21T09:24:43.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:43 smithi022 ceph-mon[39525]: purged_snaps scrub ok 2024-09-21T09:24:43.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:43 smithi022 ceph-mon[39525]: osd.1 [v2:172.21.15.22:6810/3562679765,v1:172.21.15.22:6811/3562679765] boot 2024-09-21T09:24:43.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:43 smithi022 ceph-mon[39525]: osdmap e13: 2 total, 2 up, 2 in 2024-09-21T09:24:43.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:43 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:24:43.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:43 smithi022 ceph-mon[27763]: purged_snaps scrub starts 2024-09-21T09:24:43.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:43 smithi022 ceph-mon[27763]: purged_snaps scrub ok 2024-09-21T09:24:43.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:43 smithi022 ceph-mon[27763]: osd.1 [v2:172.21.15.22:6810/3562679765,v1:172.21.15.22:6811/3562679765] boot 2024-09-21T09:24:43.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:43 smithi022 ceph-mon[27763]: osdmap e13: 2 total, 2 up, 2 in 2024-09-21T09:24:43.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:43 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:24:43.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:43 smithi064 ceph-mon[31124]: purged_snaps scrub starts 2024-09-21T09:24:43.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:43 smithi064 ceph-mon[31124]: purged_snaps scrub ok 2024-09-21T09:24:43.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:43 smithi064 ceph-mon[31124]: osd.1 [v2:172.21.15.22:6810/3562679765,v1:172.21.15.22:6811/3562679765] boot 2024-09-21T09:24:43.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:43 smithi064 ceph-mon[31124]: osdmap e13: 2 total, 2 up, 2 in 2024-09-21T09:24:43.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:43 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:24:44.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:44 smithi022 ceph-mon[39525]: pgmap v59: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:24:44.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:44 smithi022 ceph-mon[39525]: osdmap e14: 2 total, 2 up, 2 in 2024-09-21T09:24:44.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:44 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:24:44.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:44 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:24:44.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:44 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:24:44.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:44 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:24:44.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:44 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:24:44.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:44 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:24:44.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:44 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:24:44.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:44 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:24:44.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:44 smithi022 ceph-mon[27763]: pgmap v59: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:24:44.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:44 smithi022 ceph-mon[27763]: osdmap e14: 2 total, 2 up, 2 in 2024-09-21T09:24:44.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:44 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:24:44.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:44 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:24:44.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:44 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:24:44.560 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:44 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:24:44.560 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:44 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:24:44.560 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:44 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:24:44.560 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:44 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:24:44.560 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:44 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:24:44.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:44 smithi064 ceph-mon[31124]: pgmap v59: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:24:44.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:44 smithi064 ceph-mon[31124]: osdmap e14: 2 total, 2 up, 2 in 2024-09-21T09:24:44.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:44 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:24:44.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:44 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:24:44.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:44 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:24:44.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:44 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:24:44.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:44 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:24:44.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:44 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:24:44.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:44 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:24:44.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:44 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:24:45.275 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:45 smithi022 ceph-mon[39525]: Detected new or changed devices on smithi022 2024-09-21T09:24:45.275 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:45 smithi022 ceph-mon[39525]: Adjusting osd_memory_target on smithi022 to 8034M 2024-09-21T09:24:45.275 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:45 smithi022 ceph-mon[27763]: Detected new or changed devices on smithi022 2024-09-21T09:24:45.275 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:45 smithi022 ceph-mon[27763]: Adjusting osd_memory_target on smithi022 to 8034M 2024-09-21T09:24:45.281 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-09-21T09:24:45.304 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d65e801e-77fa-11ef-baf6-efdc52797490 -- ceph orch daemon add osd smithi022:vg_nvme/lv_2 2024-09-21T09:24:45.562 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:24:45.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:45 smithi064 ceph-mon[31124]: Detected new or changed devices on smithi022 2024-09-21T09:24:45.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:45 smithi064 ceph-mon[31124]: Adjusting osd_memory_target on smithi022 to 8034M 2024-09-21T09:24:46.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:46 smithi022 ceph-mon[27763]: pgmap v61: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-21T09:24:46.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:46 smithi022 ceph-mon[39525]: pgmap v61: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-21T09:24:46.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:46 smithi064 ceph-mon[31124]: pgmap v61: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-21T09:24:48.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:48 smithi022 ceph-mon[39525]: pgmap v62: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-21T09:24:48.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:48 smithi022 ceph-mon[39525]: from='client.24170 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi022:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:24:48.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:48 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:24:48.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:48 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:24:48.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:48 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:24:48.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:48 smithi022 ceph-mon[27763]: pgmap v62: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-21T09:24:48.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:48 smithi022 ceph-mon[27763]: from='client.24170 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi022:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:24:48.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:48 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:24:48.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:48 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:24:48.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:48 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:24:48.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:48 smithi064 ceph-mon[31124]: pgmap v62: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-21T09:24:48.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:48 smithi064 ceph-mon[31124]: from='client.24170 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi022:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:24:48.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:48 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:24:48.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:48 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:24:48.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:48 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:24:50.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:50 smithi022 ceph-mon[39525]: pgmap v63: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-21T09:24:50.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:50 smithi022 ceph-mon[39525]: from='client.? 172.21.15.22:0/249114971' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1c27d056-2218-47dd-8f13-bc1c13888483"}]: dispatch 2024-09-21T09:24:50.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:50 smithi022 ceph-mon[39525]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1c27d056-2218-47dd-8f13-bc1c13888483"}]: dispatch 2024-09-21T09:24:50.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:50 smithi022 ceph-mon[39525]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "1c27d056-2218-47dd-8f13-bc1c13888483"}]': finished 2024-09-21T09:24:50.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:50 smithi022 ceph-mon[39525]: osdmap e15: 3 total, 2 up, 3 in 2024-09-21T09:24:50.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:50 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:24:50.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:50 smithi022 ceph-mon[27763]: pgmap v63: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-21T09:24:50.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:50 smithi022 ceph-mon[27763]: from='client.? 172.21.15.22:0/249114971' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1c27d056-2218-47dd-8f13-bc1c13888483"}]: dispatch 2024-09-21T09:24:50.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:50 smithi022 ceph-mon[27763]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1c27d056-2218-47dd-8f13-bc1c13888483"}]: dispatch 2024-09-21T09:24:50.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:50 smithi022 ceph-mon[27763]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "1c27d056-2218-47dd-8f13-bc1c13888483"}]': finished 2024-09-21T09:24:50.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:50 smithi022 ceph-mon[27763]: osdmap e15: 3 total, 2 up, 3 in 2024-09-21T09:24:50.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:50 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:24:50.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:50 smithi064 ceph-mon[31124]: pgmap v63: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-21T09:24:50.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:50 smithi064 ceph-mon[31124]: from='client.? 172.21.15.22:0/249114971' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1c27d056-2218-47dd-8f13-bc1c13888483"}]: dispatch 2024-09-21T09:24:50.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:50 smithi064 ceph-mon[31124]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1c27d056-2218-47dd-8f13-bc1c13888483"}]: dispatch 2024-09-21T09:24:50.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:50 smithi064 ceph-mon[31124]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "1c27d056-2218-47dd-8f13-bc1c13888483"}]': finished 2024-09-21T09:24:50.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:50 smithi064 ceph-mon[31124]: osdmap e15: 3 total, 2 up, 3 in 2024-09-21T09:24:50.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:50 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:24:51.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:51 smithi022 ceph-mon[39525]: from='client.? 172.21.15.22:0/1837067455' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:24:51.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:51 smithi022 ceph-mon[27763]: from='client.? 172.21.15.22:0/1837067455' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:24:51.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:51 smithi064 ceph-mon[31124]: from='client.? 172.21.15.22:0/1837067455' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:24:52.486 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:52 smithi022 ceph-mon[39525]: pgmap v65: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-21T09:24:52.486 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:52 smithi022 ceph-mon[27763]: pgmap v65: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-21T09:24:52.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:52 smithi064 ceph-mon[31124]: pgmap v65: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-21T09:24:54.546 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:54 smithi022 ceph-mon[27763]: pgmap v66: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-21T09:24:54.546 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:54 smithi022 ceph-mon[39525]: pgmap v66: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-21T09:24:54.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:54 smithi064 ceph-mon[31124]: pgmap v66: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-21T09:24:56.483 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:56 smithi022 ceph-mon[39525]: pgmap v67: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-21T09:24:56.483 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:56 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-21T09:24:56.483 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:56 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:24:56.483 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:56 smithi022 ceph-mon[27763]: pgmap v67: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-21T09:24:56.483 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:56 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-21T09:24:56.483 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:56 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:24:56.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:56 smithi064 ceph-mon[31124]: pgmap v67: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-21T09:24:56.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:56 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-21T09:24:56.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:56 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:24:57.354 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:57 smithi022 ceph-mon[39525]: Deploying daemon osd.2 on smithi022 2024-09-21T09:24:57.354 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:57 smithi022 ceph-mon[27763]: Deploying daemon osd.2 on smithi022 2024-09-21T09:24:57.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:57 smithi064 ceph-mon[31124]: Deploying daemon osd.2 on smithi022 2024-09-21T09:24:58.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:24:58 smithi022 ceph-mon[27763]: pgmap v68: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-21T09:24:58.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:24:58 smithi022 ceph-mon[39525]: pgmap v68: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-21T09:24:58.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:24:58 smithi064 ceph-mon[31124]: pgmap v68: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-21T09:25:00.507 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:00 smithi022 ceph-mon[27763]: pgmap v69: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-21T09:25:00.507 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:00 smithi022 ceph-mon[39525]: pgmap v69: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-21T09:25:00.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:00 smithi064 ceph-mon[31124]: pgmap v69: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-21T09:25:01.471 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:01 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:25:01.471 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:01 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:25:01.471 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:01 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:25:01.471 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:01 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:25:01.471 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:01 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:25:01.471 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:01 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:25:01.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:01 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:25:01.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:01 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:25:01.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:01 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:25:02.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:02 smithi022 ceph-mon[27763]: pgmap v70: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-21T09:25:02.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:02 smithi022 ceph-mon[39525]: pgmap v70: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-21T09:25:02.592 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:02 smithi064 ceph-mon[31124]: pgmap v70: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-21T09:25:04.450 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:04 smithi022 ceph-mon[27763]: pgmap v71: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-21T09:25:04.450 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:04 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:25:04.450 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:04 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:25:04.450 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:04 smithi022 ceph-mon[27763]: from='osd.2 [v2:172.21.15.22:6818/695486895,v1:172.21.15.22:6819/695486895]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-21T09:25:04.450 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:04 smithi022 ceph-mon[27763]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-21T09:25:04.450 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:04 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:25:04.450 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:04 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:25:04.450 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:04 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:25:04.450 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:04 smithi022 ceph-mon[39525]: pgmap v71: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-21T09:25:04.450 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:04 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:25:04.451 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:04 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:25:04.451 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:04 smithi022 ceph-mon[39525]: from='osd.2 [v2:172.21.15.22:6818/695486895,v1:172.21.15.22:6819/695486895]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-21T09:25:04.451 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:04 smithi022 ceph-mon[39525]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-21T09:25:04.451 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:04 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:25:04.451 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:04 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:25:04.451 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:04 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:25:04.485 INFO:teuthology.orchestra.run.smithi022.stdout:Created osd(s) 2 on host 'smithi022' 2024-09-21T09:25:04.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:04 smithi064 ceph-mon[31124]: pgmap v71: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-21T09:25:04.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:04 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:25:04.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:04 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:25:04.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:04 smithi064 ceph-mon[31124]: from='osd.2 [v2:172.21.15.22:6818/695486895,v1:172.21.15.22:6819/695486895]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-21T09:25:04.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:04 smithi064 ceph-mon[31124]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-21T09:25:04.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:04 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:25:04.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:04 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:25:04.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:04 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:25:05.145 DEBUG:teuthology.orchestra.run.smithi022:osd.2> sudo journalctl -f -n 0 -u ceph-d65e801e-77fa-11ef-baf6-efdc52797490@osd.2.service 2024-09-21T09:25:05.147 INFO:tasks.cephadm:Deploying osd.3 on smithi022 with /dev/vg_nvme/lv_1... 2024-09-21T09:25:05.147 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d65e801e-77fa-11ef-baf6-efdc52797490 -- lvm zap /dev/vg_nvme/lv_1 2024-09-21T09:25:05.444 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:05 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:25:05.444 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:05 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:25:05.444 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:05 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:25:05.444 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:05 smithi022 ceph-mon[39525]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-21T09:25:05.444 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:05 smithi022 ceph-mon[39525]: osdmap e16: 3 total, 2 up, 3 in 2024-09-21T09:25:05.444 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:05 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:25:05.444 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:05 smithi022 ceph-mon[39525]: from='osd.2 [v2:172.21.15.22:6818/695486895,v1:172.21.15.22:6819/695486895]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]: dispatch 2024-09-21T09:25:05.444 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:05 smithi022 ceph-mon[39525]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]: dispatch 2024-09-21T09:25:05.445 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:05 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:25:05.445 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:05 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:25:05.445 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:05 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:25:05.445 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:05 smithi022 ceph-mon[27763]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-21T09:25:05.445 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:05 smithi022 ceph-mon[27763]: osdmap e16: 3 total, 2 up, 3 in 2024-09-21T09:25:05.445 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:05 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:25:05.445 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:05 smithi022 ceph-mon[27763]: from='osd.2 [v2:172.21.15.22:6818/695486895,v1:172.21.15.22:6819/695486895]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]: dispatch 2024-09-21T09:25:05.445 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:05 smithi022 ceph-mon[27763]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]: dispatch 2024-09-21T09:25:05.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:05 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:25:05.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:05 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:25:05.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:05 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:25:05.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:05 smithi064 ceph-mon[31124]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-21T09:25:05.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:05 smithi064 ceph-mon[31124]: osdmap e16: 3 total, 2 up, 3 in 2024-09-21T09:25:05.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:05 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:25:05.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:05 smithi064 ceph-mon[31124]: from='osd.2 [v2:172.21.15.22:6818/695486895,v1:172.21.15.22:6819/695486895]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]: dispatch 2024-09-21T09:25:05.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:05 smithi064 ceph-mon[31124]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]: dispatch 2024-09-21T09:25:05.808 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 21 09:25:05 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-2[56454]: 2024-09-21T09:25:05.646+0000 7f0831a59640 -1 osd.2 0 waiting for initial osdmap 2024-09-21T09:25:05.808 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 21 09:25:05 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-2[56454]: 2024-09-21T09:25:05.650+0000 7f082c86e640 -1 osd.2 17 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T09:25:05.906 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:25:06.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:06 smithi022 ceph-mon[39525]: pgmap v73: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-21T09:25:06.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:06 smithi022 ceph-mon[39525]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]': finished 2024-09-21T09:25:06.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:06 smithi022 ceph-mon[39525]: osdmap e17: 3 total, 2 up, 3 in 2024-09-21T09:25:06.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:06 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:25:06.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:06 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:25:06.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:06 smithi022 ceph-mon[27763]: pgmap v73: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-21T09:25:06.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:06 smithi022 ceph-mon[27763]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]': finished 2024-09-21T09:25:06.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:06 smithi022 ceph-mon[27763]: osdmap e17: 3 total, 2 up, 3 in 2024-09-21T09:25:06.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:06 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:25:06.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:06 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:25:06.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:06 smithi064 ceph-mon[31124]: pgmap v73: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-21T09:25:06.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:06 smithi064 ceph-mon[31124]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]': finished 2024-09-21T09:25:06.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:06 smithi064 ceph-mon[31124]: osdmap e17: 3 total, 2 up, 3 in 2024-09-21T09:25:06.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:06 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:25:06.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:06 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:25:07.938 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:07 smithi022 ceph-mon[39525]: purged_snaps scrub starts 2024-09-21T09:25:07.938 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:07 smithi022 ceph-mon[39525]: purged_snaps scrub ok 2024-09-21T09:25:07.939 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:07 smithi022 ceph-mon[39525]: osd.2 [v2:172.21.15.22:6818/695486895,v1:172.21.15.22:6819/695486895] boot 2024-09-21T09:25:07.939 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:07 smithi022 ceph-mon[39525]: osdmap e18: 3 total, 3 up, 3 in 2024-09-21T09:25:07.939 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:07 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:25:07.939 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:07 smithi022 ceph-mon[39525]: pgmap v76: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-21T09:25:07.939 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:07 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-09-21T09:25:07.939 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:07 smithi022 ceph-mon[27763]: purged_snaps scrub starts 2024-09-21T09:25:07.939 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:07 smithi022 ceph-mon[27763]: purged_snaps scrub ok 2024-09-21T09:25:07.939 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:07 smithi022 ceph-mon[27763]: osd.2 [v2:172.21.15.22:6818/695486895,v1:172.21.15.22:6819/695486895] boot 2024-09-21T09:25:07.939 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:07 smithi022 ceph-mon[27763]: osdmap e18: 3 total, 3 up, 3 in 2024-09-21T09:25:07.939 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:07 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:25:07.939 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:07 smithi022 ceph-mon[27763]: pgmap v76: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-21T09:25:07.940 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:07 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-09-21T09:25:07.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:07 smithi064 ceph-mon[31124]: purged_snaps scrub starts 2024-09-21T09:25:07.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:07 smithi064 ceph-mon[31124]: purged_snaps scrub ok 2024-09-21T09:25:07.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:07 smithi064 ceph-mon[31124]: osd.2 [v2:172.21.15.22:6818/695486895,v1:172.21.15.22:6819/695486895] boot 2024-09-21T09:25:07.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:07 smithi064 ceph-mon[31124]: osdmap e18: 3 total, 3 up, 3 in 2024-09-21T09:25:07.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:07 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:25:07.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:07 smithi064 ceph-mon[31124]: pgmap v76: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-21T09:25:07.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:07 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-09-21T09:25:08.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:08 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-09-21T09:25:08.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:08 smithi064 ceph-mon[31124]: osdmap e19: 3 total, 3 up, 3 in 2024-09-21T09:25:08.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:08 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-21T09:25:09.002 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:08 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-09-21T09:25:09.003 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:08 smithi022 ceph-mon[39525]: osdmap e19: 3 total, 3 up, 3 in 2024-09-21T09:25:09.003 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:08 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-21T09:25:09.003 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:08 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-09-21T09:25:09.003 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:08 smithi022 ceph-mon[27763]: osdmap e19: 3 total, 3 up, 3 in 2024-09-21T09:25:09.003 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:08 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-21T09:25:09.558 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 21 09:25:09 smithi022 sudo[60360]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-21T09:25:09.558 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 21 09:25:09 smithi022 sudo[60360]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-21T09:25:09.558 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 21 09:25:09 smithi022 sudo[60360]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-21T09:25:09.558 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 21 09:25:09 smithi022 sudo[60360]: pam_unix(sudo:session): session closed for user root 2024-09-21T09:25:09.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:09 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-09-21T09:25:09.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:09 smithi064 ceph-mon[31124]: Detected new or changed devices on smithi022 2024-09-21T09:25:09.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:09 smithi064 ceph-mon[31124]: osdmap e20: 3 total, 3 up, 3 in 2024-09-21T09:25:09.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:09 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:25:09.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:09 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:25:09.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:09 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:25:09.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:09 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:25:09.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:09 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:25:09.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:09 smithi064 ceph-mon[31124]: Adjusting osd_memory_target on smithi022 to 5356M 2024-09-21T09:25:09.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:09 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:25:09.943 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:09 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:25:09.943 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:09 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:25:09.943 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:09 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:25:09.943 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:09 smithi064 ceph-mon[31124]: pgmap v79: 1 pgs: 1 creating+peering; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-09-21T09:25:10.057 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 21 09:25:09 smithi022 sudo[60476]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-21T09:25:10.057 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 21 09:25:09 smithi022 sudo[60476]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-21T09:25:10.057 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 21 09:25:09 smithi022 sudo[60476]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-21T09:25:10.057 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 21 09:25:09 smithi022 sudo[60476]: pam_unix(sudo:session): session closed for user root 2024-09-21T09:25:10.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:09 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-09-21T09:25:10.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:09 smithi022 ceph-mon[39525]: Detected new or changed devices on smithi022 2024-09-21T09:25:10.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:09 smithi022 ceph-mon[39525]: osdmap e20: 3 total, 3 up, 3 in 2024-09-21T09:25:10.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:09 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:25:10.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:09 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:25:10.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:09 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:25:10.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:09 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:25:10.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:09 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:25:10.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:09 smithi022 ceph-mon[39525]: Adjusting osd_memory_target on smithi022 to 5356M 2024-09-21T09:25:10.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:09 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:25:10.059 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:09 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:25:10.059 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:09 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:25:10.059 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:09 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:25:10.059 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:09 smithi022 ceph-mon[39525]: pgmap v79: 1 pgs: 1 creating+peering; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-09-21T09:25:10.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:09 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-09-21T09:25:10.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:09 smithi022 ceph-mon[27763]: Detected new or changed devices on smithi022 2024-09-21T09:25:10.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:09 smithi022 ceph-mon[27763]: osdmap e20: 3 total, 3 up, 3 in 2024-09-21T09:25:10.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:09 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:25:10.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:09 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:25:10.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:09 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:25:10.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:09 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:25:10.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:09 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:25:10.060 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:09 smithi022 ceph-mon[27763]: Adjusting osd_memory_target on smithi022 to 5356M 2024-09-21T09:25:10.060 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:09 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:25:10.060 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:09 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:25:10.060 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:09 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:25:10.060 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:09 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:25:10.060 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:09 smithi022 ceph-mon[27763]: pgmap v79: 1 pgs: 1 creating+peering; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-09-21T09:25:10.064 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-09-21T09:25:10.083 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d65e801e-77fa-11ef-baf6-efdc52797490 -- ceph orch daemon add osd smithi022:vg_nvme/lv_1 2024-09-21T09:25:10.308 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 21 09:25:10 smithi022 sudo[60500]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-21T09:25:10.308 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 21 09:25:10 smithi022 sudo[60500]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-21T09:25:10.308 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 21 09:25:10 smithi022 sudo[60500]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-21T09:25:10.308 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 21 09:25:10 smithi022 sudo[60500]: pam_unix(sudo:session): session closed for user root 2024-09-21T09:25:10.342 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:25:10.777 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:10 smithi022 ceph-mon[27763]: mgrmap e15: y(active, since 2m), standbys: x 2024-09-21T09:25:10.777 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:10 smithi022 ceph-mon[27763]: osdmap e21: 3 total, 3 up, 3 in 2024-09-21T09:25:10.777 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:10 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:25:10.777 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:10 smithi022 ceph-mon[39525]: mgrmap e15: y(active, since 2m), standbys: x 2024-09-21T09:25:10.778 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:10 smithi022 ceph-mon[39525]: osdmap e21: 3 total, 3 up, 3 in 2024-09-21T09:25:10.778 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:10 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:25:10.778 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 21 09:25:10 smithi022 sudo[60575]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-21T09:25:10.778 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 21 09:25:10 smithi022 sudo[60575]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-21T09:25:10.778 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 21 09:25:10 smithi022 sudo[60575]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-21T09:25:10.778 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 21 09:25:10 smithi022 sudo[60575]: pam_unix(sudo:session): session closed for user root 2024-09-21T09:25:10.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:10 smithi064 ceph-mon[31124]: mgrmap e15: y(active, since 2m), standbys: x 2024-09-21T09:25:10.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:10 smithi064 ceph-mon[31124]: osdmap e21: 3 total, 3 up, 3 in 2024-09-21T09:25:10.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:10 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:25:11.539 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 21 09:25:11 smithi022 sudo[60595]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-21T09:25:11.539 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 21 09:25:11 smithi022 sudo[60595]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-21T09:25:11.539 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 21 09:25:11 smithi022 sudo[60595]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-21T09:25:11.539 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 21 09:25:11 smithi022 sudo[60595]: pam_unix(sudo:session): session closed for user root 2024-09-21T09:25:11.808 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 21 09:25:11 smithi022 sudo[60610]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-21T09:25:11.808 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 21 09:25:11 smithi022 sudo[60610]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-21T09:25:11.808 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 21 09:25:11 smithi022 sudo[60610]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-21T09:25:11.808 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 21 09:25:11 smithi022 sudo[60610]: pam_unix(sudo:session): session closed for user root 2024-09-21T09:25:12.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:12 smithi022 sudo[60659]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-21T09:25:12.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:12 smithi022 sudo[60659]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-21T09:25:12.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:12 smithi022 sudo[60659]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-21T09:25:12.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:12 smithi022 sudo[60659]: pam_unix(sudo:session): session closed for user root 2024-09-21T09:25:12.807 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:12 smithi022 sudo[60682]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-21T09:25:12.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:12 smithi022 sudo[60682]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-21T09:25:12.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:12 smithi022 sudo[60682]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-21T09:25:12.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:12 smithi022 sudo[60682]: pam_unix(sudo:session): session closed for user root 2024-09-21T09:25:13.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:12 smithi064 sudo[33886]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-21T09:25:13.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:12 smithi064 sudo[33886]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-21T09:25:13.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:12 smithi064 sudo[33886]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-21T09:25:13.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:12 smithi064 sudo[33886]: pam_unix(sudo:session): session closed for user root 2024-09-21T09:25:13.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:13 smithi022 ceph-mon[27763]: pgmap v81: 1 pgs: 1 creating+peering; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-09-21T09:25:13.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:13 smithi022 ceph-mon[27763]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-21T09:25:13.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:13 smithi022 ceph-mon[39525]: pgmap v81: 1 pgs: 1 creating+peering; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-09-21T09:25:13.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:13 smithi022 ceph-mon[39525]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-21T09:25:13.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:13 smithi064 sudo[33889]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-21T09:25:13.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:13 smithi064 sudo[33889]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-21T09:25:13.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:13 smithi064 sudo[33889]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-21T09:25:13.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:13 smithi064 sudo[33889]: pam_unix(sudo:session): session closed for user root 2024-09-21T09:25:13.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:13 smithi064 ceph-mon[31124]: pgmap v81: 1 pgs: 1 creating+peering; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-09-21T09:25:13.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:13 smithi064 ceph-mon[31124]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-21T09:25:13.957 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:13 smithi022 sudo[60852]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-21T09:25:13.957 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:13 smithi022 sudo[60852]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-21T09:25:13.957 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:13 smithi022 sudo[60852]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-21T09:25:13.957 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:13 smithi022 sudo[60852]: pam_unix(sudo:session): session closed for user root 2024-09-21T09:25:14.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:14 smithi022 ceph-mon[27763]: from='client.24203 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi022:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:25:14.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:14 smithi022 ceph-mon[27763]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-21T09:25:14.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:14 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:25:14.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:14 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:25:14.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:14 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:25:14.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:14 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:25:14.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:14 smithi022 ceph-mon[27763]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-21T09:25:14.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:14 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:25:14.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:14 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:25:14.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:14 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:25:14.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:14 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:25:14.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:14 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:25:14.560 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:14 smithi022 ceph-mon[27763]: pgmap v82: 1 pgs: 1 creating+peering; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-09-21T09:25:14.560 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:14 smithi022 ceph-mon[27763]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-21T09:25:14.560 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:14 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:25:14.560 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:14 smithi022 ceph-mon[27763]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-21T09:25:14.560 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:14 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:25:14.560 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:14 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:25:14.560 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:14 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:25:14.560 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:14 smithi022 ceph-mon[27763]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-21T09:25:14.560 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:14 smithi022 sudo[60895]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-21T09:25:14.560 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:14 smithi022 sudo[60895]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-21T09:25:14.561 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:14 smithi022 sudo[60895]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-21T09:25:14.561 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:14 smithi022 sudo[60895]: pam_unix(sudo:session): session closed for user root 2024-09-21T09:25:14.561 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:14 smithi022 ceph-mon[39525]: from='client.24203 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi022:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:25:14.561 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:14 smithi022 ceph-mon[39525]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-21T09:25:14.561 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:14 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:25:14.561 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:14 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:25:14.561 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:14 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:25:14.561 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:14 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:25:14.561 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:14 smithi022 ceph-mon[39525]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-21T09:25:14.561 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:14 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:25:14.561 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:14 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:25:14.561 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:14 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:25:14.561 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:14 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:25:14.561 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:14 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:25:14.562 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:14 smithi022 ceph-mon[39525]: pgmap v82: 1 pgs: 1 creating+peering; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-09-21T09:25:14.562 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:14 smithi022 ceph-mon[39525]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-21T09:25:14.562 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:14 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:25:14.562 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:14 smithi022 ceph-mon[39525]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-21T09:25:14.562 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:14 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:25:14.562 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:14 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:25:14.562 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:14 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:25:14.562 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:14 smithi022 ceph-mon[39525]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-21T09:25:14.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:14 smithi064 ceph-mon[31124]: from='client.24203 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi022:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:25:14.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:14 smithi064 ceph-mon[31124]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-21T09:25:14.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:14 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:25:14.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:14 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:25:14.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:14 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:25:14.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:14 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:25:14.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:14 smithi064 ceph-mon[31124]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-21T09:25:14.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:14 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:25:14.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:14 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:25:14.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:14 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:25:14.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:14 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:25:14.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:14 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:25:14.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:14 smithi064 ceph-mon[31124]: pgmap v82: 1 pgs: 1 creating+peering; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-09-21T09:25:14.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:14 smithi064 ceph-mon[31124]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-21T09:25:14.693 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:14 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:25:14.693 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:14 smithi064 ceph-mon[31124]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-21T09:25:14.693 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:14 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:25:14.693 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:14 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:25:14.693 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:14 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:25:14.693 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:14 smithi064 ceph-mon[31124]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-21T09:25:15.495 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:15 smithi022 ceph-mon[39525]: from='client.? 172.21.15.22:0/264939242' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8668a232-df4a-46ba-b97d-3ed2f6ddeab2"}]: dispatch 2024-09-21T09:25:15.495 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:15 smithi022 ceph-mon[39525]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8668a232-df4a-46ba-b97d-3ed2f6ddeab2"}]: dispatch 2024-09-21T09:25:15.495 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:15 smithi022 ceph-mon[39525]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "8668a232-df4a-46ba-b97d-3ed2f6ddeab2"}]': finished 2024-09-21T09:25:15.495 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:15 smithi022 ceph-mon[39525]: osdmap e22: 4 total, 3 up, 4 in 2024-09-21T09:25:15.496 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:15 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:25:15.496 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:15 smithi022 ceph-mon[27763]: from='client.? 172.21.15.22:0/264939242' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8668a232-df4a-46ba-b97d-3ed2f6ddeab2"}]: dispatch 2024-09-21T09:25:15.496 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:15 smithi022 ceph-mon[27763]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8668a232-df4a-46ba-b97d-3ed2f6ddeab2"}]: dispatch 2024-09-21T09:25:15.496 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:15 smithi022 ceph-mon[27763]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "8668a232-df4a-46ba-b97d-3ed2f6ddeab2"}]': finished 2024-09-21T09:25:15.496 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:15 smithi022 ceph-mon[27763]: osdmap e22: 4 total, 3 up, 4 in 2024-09-21T09:25:15.496 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:15 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:25:15.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:15 smithi064 ceph-mon[31124]: from='client.? 172.21.15.22:0/264939242' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8668a232-df4a-46ba-b97d-3ed2f6ddeab2"}]: dispatch 2024-09-21T09:25:15.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:15 smithi064 ceph-mon[31124]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8668a232-df4a-46ba-b97d-3ed2f6ddeab2"}]: dispatch 2024-09-21T09:25:15.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:15 smithi064 ceph-mon[31124]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "8668a232-df4a-46ba-b97d-3ed2f6ddeab2"}]': finished 2024-09-21T09:25:15.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:15 smithi064 ceph-mon[31124]: osdmap e22: 4 total, 3 up, 4 in 2024-09-21T09:25:15.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:15 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:25:16.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:16 smithi064 ceph-mon[31124]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-21T09:25:16.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:16 smithi064 ceph-mon[31124]: from='client.? 172.21.15.22:0/2393231845' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:25:16.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:16 smithi022 ceph-mon[39525]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-21T09:25:16.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:16 smithi022 ceph-mon[39525]: from='client.? 172.21.15.22:0/2393231845' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:25:16.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:16 smithi022 ceph-mon[27763]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-21T09:25:16.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:16 smithi022 ceph-mon[27763]: from='client.? 172.21.15.22:0/2393231845' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:25:17.685 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:17 smithi022 ceph-mon[39525]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-21T09:25:17.685 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:17 smithi022 ceph-mon[27763]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-21T09:25:17.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:17 smithi064 ceph-mon[31124]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-21T09:25:20.274 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:19 smithi022 ceph-mon[27763]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 4.5 KiB/s rd, 195 KiB/s wr, 13 op/s 2024-09-21T09:25:20.274 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:19 smithi022 ceph-mon[39525]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 4.5 KiB/s rd, 195 KiB/s wr, 13 op/s 2024-09-21T09:25:20.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:19 smithi064 ceph-mon[31124]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 4.5 KiB/s rd, 195 KiB/s wr, 13 op/s 2024-09-21T09:25:22.049 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:21 smithi022 ceph-mon[39525]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 3.0 KiB/s rd, 130 KiB/s wr, 9 op/s 2024-09-21T09:25:22.049 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:21 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-21T09:25:22.049 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:21 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:25:22.049 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:21 smithi022 ceph-mon[39525]: Deploying daemon osd.3 on smithi022 2024-09-21T09:25:22.050 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:21 smithi022 ceph-mon[27763]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 3.0 KiB/s rd, 130 KiB/s wr, 9 op/s 2024-09-21T09:25:22.050 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:21 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-21T09:25:22.050 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:21 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:25:22.050 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:21 smithi022 ceph-mon[27763]: Deploying daemon osd.3 on smithi022 2024-09-21T09:25:22.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:21 smithi064 ceph-mon[31124]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 3.0 KiB/s rd, 130 KiB/s wr, 9 op/s 2024-09-21T09:25:22.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:21 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-21T09:25:22.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:21 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:25:22.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:21 smithi064 ceph-mon[31124]: Deploying daemon osd.3 on smithi022 2024-09-21T09:25:23.997 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:23 smithi022 ceph-mon[27763]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-09-21T09:25:24.293 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:23 smithi022 ceph-mon[39525]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-09-21T09:25:24.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:23 smithi064 ceph-mon[31124]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-09-21T09:25:26.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:25 smithi022 ceph-mon[27763]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-09-21T09:25:26.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:25 smithi022 ceph-mon[39525]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-09-21T09:25:26.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:25 smithi064 ceph-mon[31124]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-09-21T09:25:27.204 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:26 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:25:27.205 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:26 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:25:27.205 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:26 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:25:27.205 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:26 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:25:27.205 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:26 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:25:27.205 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:26 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:25:27.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:26 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:25:27.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:26 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:25:27.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:26 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:25:28.307 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:28 smithi022 ceph-mon[27763]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-09-21T09:25:28.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:28 smithi022 ceph-mon[39525]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-09-21T09:25:28.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:28 smithi064 ceph-mon[31124]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-09-21T09:25:30.096 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:30 smithi022 ceph-mon[27763]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-09-21T09:25:30.096 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:30 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:25:30.096 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:30 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:25:30.096 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:30 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:25:30.096 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:30 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:25:30.096 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:30 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:25:30.096 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:30 smithi022 ceph-mon[27763]: from='osd.3 [v2:172.21.15.22:6826/1394296125,v1:172.21.15.22:6827/1394296125]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-21T09:25:30.096 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:30 smithi022 ceph-mon[27763]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-21T09:25:30.096 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:30 smithi022 ceph-mon[39525]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-09-21T09:25:30.097 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:30 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:25:30.097 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:30 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:25:30.097 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:30 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:25:30.097 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:30 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:25:30.097 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:30 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:25:30.097 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:30 smithi022 ceph-mon[39525]: from='osd.3 [v2:172.21.15.22:6826/1394296125,v1:172.21.15.22:6827/1394296125]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-21T09:25:30.097 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:30 smithi022 ceph-mon[39525]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-21T09:25:30.134 INFO:teuthology.orchestra.run.smithi022.stdout:Created osd(s) 3 on host 'smithi022' 2024-09-21T09:25:30.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:30 smithi064 ceph-mon[31124]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-09-21T09:25:30.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:30 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:25:30.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:30 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:25:30.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:30 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:25:30.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:30 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:25:30.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:30 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:25:30.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:30 smithi064 ceph-mon[31124]: from='osd.3 [v2:172.21.15.22:6826/1394296125,v1:172.21.15.22:6827/1394296125]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-21T09:25:30.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:30 smithi064 ceph-mon[31124]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-21T09:25:30.911 DEBUG:teuthology.orchestra.run.smithi022:osd.3> sudo journalctl -f -n 0 -u ceph-d65e801e-77fa-11ef-baf6-efdc52797490@osd.3.service 2024-09-21T09:25:30.914 INFO:tasks.cephadm:Deploying osd.4 on smithi064 with /dev/vg_nvme/lv_4... 2024-09-21T09:25:30.915 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d65e801e-77fa-11ef-baf6-efdc52797490 -- lvm zap /dev/vg_nvme/lv_4 2024-09-21T09:25:31.189 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:25:31.225 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:31 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:25:31.225 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:31 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:25:31.225 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:31 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:25:31.225 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:31 smithi022 ceph-mon[39525]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-21T09:25:31.225 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:31 smithi022 ceph-mon[39525]: osdmap e23: 4 total, 3 up, 4 in 2024-09-21T09:25:31.225 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:31 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:25:31.225 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:31 smithi022 ceph-mon[39525]: from='osd.3 [v2:172.21.15.22:6826/1394296125,v1:172.21.15.22:6827/1394296125]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]: dispatch 2024-09-21T09:25:31.225 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:31 smithi022 ceph-mon[39525]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]: dispatch 2024-09-21T09:25:31.225 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:31 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:25:31.226 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:31 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:25:31.226 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:31 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:25:31.226 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:31 smithi022 ceph-mon[27763]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-21T09:25:31.226 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:31 smithi022 ceph-mon[27763]: osdmap e23: 4 total, 3 up, 4 in 2024-09-21T09:25:31.226 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:31 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:25:31.226 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:31 smithi022 ceph-mon[27763]: from='osd.3 [v2:172.21.15.22:6826/1394296125,v1:172.21.15.22:6827/1394296125]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]: dispatch 2024-09-21T09:25:31.226 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:31 smithi022 ceph-mon[27763]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]: dispatch 2024-09-21T09:25:31.309 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:31 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:25:31.310 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:31 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:25:31.310 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:31 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:25:31.310 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:31 smithi064 ceph-mon[31124]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-21T09:25:31.310 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:31 smithi064 ceph-mon[31124]: osdmap e23: 4 total, 3 up, 4 in 2024-09-21T09:25:31.310 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:31 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:25:31.310 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:31 smithi064 ceph-mon[31124]: from='osd.3 [v2:172.21.15.22:6826/1394296125,v1:172.21.15.22:6827/1394296125]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]: dispatch 2024-09-21T09:25:31.310 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:31 smithi064 ceph-mon[31124]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]: dispatch 2024-09-21T09:25:31.808 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 21 09:25:31 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-3[62753]: 2024-09-21T09:25:31.507+0000 7fda09a6c640 -1 osd.3 0 waiting for initial osdmap 2024-09-21T09:25:31.808 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 21 09:25:31 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-3[62753]: 2024-09-21T09:25:31.511+0000 7fda058a5640 -1 osd.3 24 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T09:25:32.123 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:32 smithi022 ceph-mon[39525]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-21T09:25:32.123 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:32 smithi022 ceph-mon[39525]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]': finished 2024-09-21T09:25:32.123 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:32 smithi022 ceph-mon[39525]: osdmap e24: 4 total, 3 up, 4 in 2024-09-21T09:25:32.123 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:32 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:25:32.123 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:32 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:25:32.123 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:32 smithi022 ceph-mon[27763]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-21T09:25:32.123 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:32 smithi022 ceph-mon[27763]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]': finished 2024-09-21T09:25:32.123 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:32 smithi022 ceph-mon[27763]: osdmap e24: 4 total, 3 up, 4 in 2024-09-21T09:25:32.123 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:32 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:25:32.123 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:32 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:25:32.406 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:32 smithi064 ceph-mon[31124]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-21T09:25:32.406 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:32 smithi064 ceph-mon[31124]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi022", "root=default"]}]': finished 2024-09-21T09:25:32.406 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:32 smithi064 ceph-mon[31124]: osdmap e24: 4 total, 3 up, 4 in 2024-09-21T09:25:32.406 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:32 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:25:32.407 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:32 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:25:33.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:33 smithi022 ceph-mon[27763]: purged_snaps scrub starts 2024-09-21T09:25:33.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:33 smithi022 ceph-mon[27763]: purged_snaps scrub ok 2024-09-21T09:25:33.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:33 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:25:33.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:33 smithi022 ceph-mon[27763]: osd.3 [v2:172.21.15.22:6826/1394296125,v1:172.21.15.22:6827/1394296125] boot 2024-09-21T09:25:33.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:33 smithi022 ceph-mon[27763]: osdmap e25: 4 total, 4 up, 4 in 2024-09-21T09:25:33.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:33 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:25:33.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:33 smithi022 ceph-mon[39525]: purged_snaps scrub starts 2024-09-21T09:25:33.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:33 smithi022 ceph-mon[39525]: purged_snaps scrub ok 2024-09-21T09:25:33.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:33 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:25:33.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:33 smithi022 ceph-mon[39525]: osd.3 [v2:172.21.15.22:6826/1394296125,v1:172.21.15.22:6827/1394296125] boot 2024-09-21T09:25:33.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:33 smithi022 ceph-mon[39525]: osdmap e25: 4 total, 4 up, 4 in 2024-09-21T09:25:33.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:33 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:25:33.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:33 smithi064 ceph-mon[31124]: purged_snaps scrub starts 2024-09-21T09:25:33.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:33 smithi064 ceph-mon[31124]: purged_snaps scrub ok 2024-09-21T09:25:33.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:33 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:25:33.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:33 smithi064 ceph-mon[31124]: osd.3 [v2:172.21.15.22:6826/1394296125,v1:172.21.15.22:6827/1394296125] boot 2024-09-21T09:25:33.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:33 smithi064 ceph-mon[31124]: osdmap e25: 4 total, 4 up, 4 in 2024-09-21T09:25:33.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:33 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:25:33.869 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-09-21T09:25:33.888 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d65e801e-77fa-11ef-baf6-efdc52797490 -- ceph orch daemon add osd smithi064:vg_nvme/lv_4 2024-09-21T09:25:34.148 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:25:34.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:34 smithi064 ceph-mon[31124]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-21T09:25:34.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:34 smithi064 ceph-mon[31124]: osdmap e26: 4 total, 4 up, 4 in 2024-09-21T09:25:34.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:34 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:25:34.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:34 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:25:34.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:34 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:25:34.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:34 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:25:34.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:34 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:25:34.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:34 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:25:34.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:34 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:25:34.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:34 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:25:34.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:34 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:25:34.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:34 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:25:34.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:34 smithi022 ceph-mon[39525]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-21T09:25:34.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:34 smithi022 ceph-mon[39525]: osdmap e26: 4 total, 4 up, 4 in 2024-09-21T09:25:34.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:34 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:25:34.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:34 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:25:34.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:34 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:25:34.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:34 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:25:34.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:34 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:25:34.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:34 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:25:34.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:34 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:25:34.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:34 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:25:34.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:34 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:25:34.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:34 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:25:34.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:34 smithi022 ceph-mon[27763]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-21T09:25:34.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:34 smithi022 ceph-mon[27763]: osdmap e26: 4 total, 4 up, 4 in 2024-09-21T09:25:34.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:34 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:25:34.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:34 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:25:34.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:34 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:25:34.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:34 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:25:34.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:34 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:25:34.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:34 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:25:34.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:34 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:25:34.310 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:34 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:25:34.310 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:34 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:25:34.310 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:34 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:25:35.277 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:35 smithi064 ceph-mon[31124]: Detected new or changed devices on smithi022 2024-09-21T09:25:35.277 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:35 smithi064 ceph-mon[31124]: Adjusting osd_memory_target on smithi022 to 4017M 2024-09-21T09:25:35.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:35 smithi022 ceph-mon[39525]: Detected new or changed devices on smithi022 2024-09-21T09:25:35.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:35 smithi022 ceph-mon[39525]: Adjusting osd_memory_target on smithi022 to 4017M 2024-09-21T09:25:35.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:35 smithi022 ceph-mon[27763]: Detected new or changed devices on smithi022 2024-09-21T09:25:35.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:35 smithi022 ceph-mon[27763]: Adjusting osd_memory_target on smithi022 to 4017M 2024-09-21T09:25:36.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:36 smithi064 ceph-mon[31124]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-21T09:25:36.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:36 smithi022 ceph-mon[39525]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-21T09:25:36.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:36 smithi022 ceph-mon[27763]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-21T09:25:37.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:37 smithi064 ceph-mon[31124]: from='client.24187 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi064:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:25:37.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:37 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:25:37.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:37 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:25:37.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:37 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:25:37.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:37 smithi022 ceph-mon[39525]: from='client.24187 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi064:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:25:37.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:37 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:25:37.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:37 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:25:37.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:37 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:25:37.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:37 smithi022 ceph-mon[27763]: from='client.24187 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi064:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:25:37.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:37 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:25:37.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:37 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:25:37.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:37 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:25:38.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:38 smithi064 ceph-mon[31124]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-21T09:25:38.307 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:38 smithi022 ceph-mon[27763]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-21T09:25:38.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:38 smithi022 ceph-mon[39525]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-21T09:25:39.106 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:39 smithi064 ceph-mon[31124]: from='client.? 172.21.15.64:0/4061202341' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "44dd8dcc-dbf7-41f7-9ec2-e66922e6c521"}]: dispatch 2024-09-21T09:25:39.107 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:39 smithi064 ceph-mon[31124]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "44dd8dcc-dbf7-41f7-9ec2-e66922e6c521"}]: dispatch 2024-09-21T09:25:39.107 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:39 smithi064 ceph-mon[31124]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "44dd8dcc-dbf7-41f7-9ec2-e66922e6c521"}]': finished 2024-09-21T09:25:39.107 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:39 smithi064 ceph-mon[31124]: osdmap e27: 5 total, 4 up, 5 in 2024-09-21T09:25:39.107 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:39 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:25:39.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:39 smithi022 ceph-mon[39525]: from='client.? 172.21.15.64:0/4061202341' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "44dd8dcc-dbf7-41f7-9ec2-e66922e6c521"}]: dispatch 2024-09-21T09:25:39.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:39 smithi022 ceph-mon[39525]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "44dd8dcc-dbf7-41f7-9ec2-e66922e6c521"}]: dispatch 2024-09-21T09:25:39.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:39 smithi022 ceph-mon[39525]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "44dd8dcc-dbf7-41f7-9ec2-e66922e6c521"}]': finished 2024-09-21T09:25:39.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:39 smithi022 ceph-mon[39525]: osdmap e27: 5 total, 4 up, 5 in 2024-09-21T09:25:39.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:39 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:25:39.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:39 smithi022 ceph-mon[27763]: from='client.? 172.21.15.64:0/4061202341' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "44dd8dcc-dbf7-41f7-9ec2-e66922e6c521"}]: dispatch 2024-09-21T09:25:39.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:39 smithi022 ceph-mon[27763]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "44dd8dcc-dbf7-41f7-9ec2-e66922e6c521"}]: dispatch 2024-09-21T09:25:39.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:39 smithi022 ceph-mon[27763]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "44dd8dcc-dbf7-41f7-9ec2-e66922e6c521"}]': finished 2024-09-21T09:25:39.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:39 smithi022 ceph-mon[27763]: osdmap e27: 5 total, 4 up, 5 in 2024-09-21T09:25:39.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:39 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:25:40.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:40 smithi022 ceph-mon[27763]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-21T09:25:40.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:40 smithi022 ceph-mon[27763]: from='client.? 172.21.15.64:0/209771283' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:25:40.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:40 smithi022 ceph-mon[39525]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-21T09:25:40.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:40 smithi022 ceph-mon[39525]: from='client.? 172.21.15.64:0/209771283' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:25:40.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:40 smithi064 ceph-mon[31124]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-21T09:25:40.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:40 smithi064 ceph-mon[31124]: from='client.? 172.21.15.64:0/209771283' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:25:42.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:42 smithi064 ceph-mon[31124]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-21T09:25:42.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:42 smithi022 ceph-mon[27763]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-21T09:25:42.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:42 smithi022 ceph-mon[39525]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-21T09:25:44.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:44 smithi064 ceph-mon[31124]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-21T09:25:44.307 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:44 smithi022 ceph-mon[39525]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-21T09:25:44.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:44 smithi022 ceph-mon[27763]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-21T09:25:45.291 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:45 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-21T09:25:45.291 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:45 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:25:45.307 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:45 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-21T09:25:45.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:45 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:25:45.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:45 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-21T09:25:45.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:45 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:25:46.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:46 smithi064 ceph-mon[31124]: Deploying daemon osd.4 on smithi064 2024-09-21T09:25:46.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:46 smithi064 ceph-mon[31124]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-21T09:25:46.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:46 smithi022 ceph-mon[27763]: Deploying daemon osd.4 on smithi064 2024-09-21T09:25:46.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:46 smithi022 ceph-mon[27763]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-21T09:25:46.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:46 smithi022 ceph-mon[39525]: Deploying daemon osd.4 on smithi064 2024-09-21T09:25:46.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:46 smithi022 ceph-mon[39525]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-21T09:25:48.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:48 smithi022 ceph-mon[27763]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-21T09:25:48.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:48 smithi022 ceph-mon[39525]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-21T09:25:48.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:48 smithi064 ceph-mon[31124]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-21T09:25:50.310 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:50 smithi022 ceph-mon[27763]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-21T09:25:50.310 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:50 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:25:50.310 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:50 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:25:50.311 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:50 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:25:50.311 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:50 smithi022 ceph-mon[39525]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-21T09:25:50.311 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:50 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:25:50.311 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:50 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:25:50.311 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:50 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:25:50.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:50 smithi064 ceph-mon[31124]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-21T09:25:50.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:50 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:25:50.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:50 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:25:50.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:50 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:25:52.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:52 smithi064 ceph-mon[31124]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-21T09:25:52.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:52 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:25:52.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:52 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:25:52.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:52 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:25:52.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:52 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:25:52.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:52 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:25:52.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:52 smithi022 ceph-mon[39525]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-21T09:25:52.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:52 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:25:52.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:52 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:25:52.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:52 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:25:52.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:52 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:25:52.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:52 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:25:52.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:52 smithi022 ceph-mon[27763]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-21T09:25:52.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:52 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:25:52.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:52 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:25:52.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:52 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:25:52.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:52 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:25:52.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:52 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:25:52.753 INFO:teuthology.orchestra.run.smithi064.stdout:Created osd(s) 4 on host 'smithi064' 2024-09-21T09:25:53.336 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:53 smithi064 ceph-mon[31124]: from='osd.4 [v2:172.21.15.64:6800/2736095169,v1:172.21.15.64:6801/2736095169]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-21T09:25:53.337 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:53 smithi064 ceph-mon[31124]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-21T09:25:53.337 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:53 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:25:53.337 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:53 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:25:53.337 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:53 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:25:53.362 DEBUG:teuthology.orchestra.run.smithi064:osd.4> sudo journalctl -f -n 0 -u ceph-d65e801e-77fa-11ef-baf6-efdc52797490@osd.4.service 2024-09-21T09:25:53.364 INFO:tasks.cephadm:Deploying osd.5 on smithi064 with /dev/vg_nvme/lv_3... 2024-09-21T09:25:53.364 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d65e801e-77fa-11ef-baf6-efdc52797490 -- lvm zap /dev/vg_nvme/lv_3 2024-09-21T09:25:53.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:53 smithi022 ceph-mon[27763]: from='osd.4 [v2:172.21.15.64:6800/2736095169,v1:172.21.15.64:6801/2736095169]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-21T09:25:53.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:53 smithi022 ceph-mon[27763]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-21T09:25:53.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:53 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:25:53.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:53 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:25:53.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:53 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:25:53.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:53 smithi022 ceph-mon[39525]: from='osd.4 [v2:172.21.15.64:6800/2736095169,v1:172.21.15.64:6801/2736095169]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-21T09:25:53.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:53 smithi022 ceph-mon[39525]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-21T09:25:53.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:53 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:25:53.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:53 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:25:53.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:53 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:25:54.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:54 smithi064 ceph-mon[31124]: pgmap v108: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-21T09:25:54.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:54 smithi064 ceph-mon[31124]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-21T09:25:54.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:54 smithi064 ceph-mon[31124]: osdmap e28: 5 total, 4 up, 5 in 2024-09-21T09:25:54.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:54 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:25:54.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:54 smithi064 ceph-mon[31124]: from='osd.4 [v2:172.21.15.64:6800/2736095169,v1:172.21.15.64:6801/2736095169]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2024-09-21T09:25:54.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:54 smithi064 ceph-mon[31124]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2024-09-21T09:25:54.442 INFO:journalctl@ceph.osd.4.smithi064.stdout:Sep 21 09:25:54 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-4[36572]: 2024-09-21T09:25:54.159+0000 7fb73eacc640 -1 osd.4 0 waiting for initial osdmap 2024-09-21T09:25:54.442 INFO:journalctl@ceph.osd.4.smithi064.stdout:Sep 21 09:25:54 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-4[36572]: 2024-09-21T09:25:54.164+0000 7fb739903640 -1 osd.4 29 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T09:25:54.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:54 smithi022 ceph-mon[27763]: pgmap v108: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-21T09:25:54.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:54 smithi022 ceph-mon[27763]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-21T09:25:54.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:54 smithi022 ceph-mon[27763]: osdmap e28: 5 total, 4 up, 5 in 2024-09-21T09:25:54.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:54 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:25:54.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:54 smithi022 ceph-mon[27763]: from='osd.4 [v2:172.21.15.64:6800/2736095169,v1:172.21.15.64:6801/2736095169]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2024-09-21T09:25:54.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:54 smithi022 ceph-mon[27763]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2024-09-21T09:25:54.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:54 smithi022 ceph-mon[39525]: pgmap v108: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-21T09:25:54.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:54 smithi022 ceph-mon[39525]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-21T09:25:54.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:54 smithi022 ceph-mon[39525]: osdmap e28: 5 total, 4 up, 5 in 2024-09-21T09:25:54.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:54 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:25:54.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:54 smithi022 ceph-mon[39525]: from='osd.4 [v2:172.21.15.64:6800/2736095169,v1:172.21.15.64:6801/2736095169]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2024-09-21T09:25:54.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:54 smithi022 ceph-mon[39525]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2024-09-21T09:25:55.005 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:25:55.398 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:55 smithi064 ceph-mon[31124]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]': finished 2024-09-21T09:25:55.398 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:55 smithi064 ceph-mon[31124]: osdmap e29: 5 total, 4 up, 5 in 2024-09-21T09:25:55.398 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:55 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:25:55.398 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:55 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:25:55.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:55 smithi022 ceph-mon[27763]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]': finished 2024-09-21T09:25:55.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:55 smithi022 ceph-mon[27763]: osdmap e29: 5 total, 4 up, 5 in 2024-09-21T09:25:55.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:55 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:25:55.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:55 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:25:55.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:55 smithi022 ceph-mon[39525]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]': finished 2024-09-21T09:25:55.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:55 smithi022 ceph-mon[39525]: osdmap e29: 5 total, 4 up, 5 in 2024-09-21T09:25:55.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:55 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:25:55.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:55 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:25:56.354 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:56 smithi064 ceph-mon[31124]: purged_snaps scrub starts 2024-09-21T09:25:56.355 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:56 smithi064 ceph-mon[31124]: purged_snaps scrub ok 2024-09-21T09:25:56.355 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:56 smithi064 ceph-mon[31124]: pgmap v111: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-21T09:25:56.355 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:56 smithi064 ceph-mon[31124]: osd.4 [v2:172.21.15.64:6800/2736095169,v1:172.21.15.64:6801/2736095169] boot 2024-09-21T09:25:56.355 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:56 smithi064 ceph-mon[31124]: osdmap e30: 5 total, 5 up, 5 in 2024-09-21T09:25:56.355 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:56 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:25:56.355 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:56 smithi064 ceph-mon[31124]: osdmap e31: 5 total, 5 up, 5 in 2024-09-21T09:25:56.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:56 smithi022 ceph-mon[27763]: purged_snaps scrub starts 2024-09-21T09:25:56.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:56 smithi022 ceph-mon[27763]: purged_snaps scrub ok 2024-09-21T09:25:56.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:56 smithi022 ceph-mon[27763]: pgmap v111: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-21T09:25:56.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:56 smithi022 ceph-mon[27763]: osd.4 [v2:172.21.15.64:6800/2736095169,v1:172.21.15.64:6801/2736095169] boot 2024-09-21T09:25:56.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:56 smithi022 ceph-mon[27763]: osdmap e30: 5 total, 5 up, 5 in 2024-09-21T09:25:56.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:56 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:25:56.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:56 smithi022 ceph-mon[27763]: osdmap e31: 5 total, 5 up, 5 in 2024-09-21T09:25:56.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:56 smithi022 ceph-mon[39525]: purged_snaps scrub starts 2024-09-21T09:25:56.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:56 smithi022 ceph-mon[39525]: purged_snaps scrub ok 2024-09-21T09:25:56.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:56 smithi022 ceph-mon[39525]: pgmap v111: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-21T09:25:56.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:56 smithi022 ceph-mon[39525]: osd.4 [v2:172.21.15.64:6800/2736095169,v1:172.21.15.64:6801/2736095169] boot 2024-09-21T09:25:56.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:56 smithi022 ceph-mon[39525]: osdmap e30: 5 total, 5 up, 5 in 2024-09-21T09:25:56.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:56 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:25:56.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:56 smithi022 ceph-mon[39525]: osdmap e31: 5 total, 5 up, 5 in 2024-09-21T09:25:57.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:57 smithi064 ceph-mon[31124]: osdmap e32: 5 total, 5 up, 5 in 2024-09-21T09:25:57.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:57 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:25:57.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:57 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:25:57.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:57 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:25:57.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:57 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:25:57.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:57 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:25:57.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:57 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:25:57.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:57 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:25:57.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:57 smithi022 ceph-mon[27763]: osdmap e32: 5 total, 5 up, 5 in 2024-09-21T09:25:57.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:57 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:25:57.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:57 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:25:57.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:57 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:25:57.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:57 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:25:57.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:57 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:25:57.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:57 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:25:57.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:57 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:25:57.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:57 smithi022 ceph-mon[39525]: osdmap e32: 5 total, 5 up, 5 in 2024-09-21T09:25:57.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:57 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:25:57.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:57 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:25:57.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:57 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:25:57.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:57 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:25:57.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:57 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:25:57.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:57 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:25:57.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:57 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:25:58.067 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-09-21T09:25:58.087 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d65e801e-77fa-11ef-baf6-efdc52797490 -- ceph orch daemon add osd smithi064:vg_nvme/lv_3 2024-09-21T09:25:58.340 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:25:58.614 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:58 smithi064 ceph-mon[31124]: Detected new or changed devices on smithi064 2024-09-21T09:25:58.614 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:58 smithi064 ceph-mon[31124]: Adjusting osd_memory_target on smithi064 to 17088M 2024-09-21T09:25:58.614 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:25:58 smithi064 ceph-mon[31124]: pgmap v115: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-21T09:25:58.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:58 smithi022 ceph-mon[27763]: Detected new or changed devices on smithi064 2024-09-21T09:25:58.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:58 smithi022 ceph-mon[27763]: Adjusting osd_memory_target on smithi064 to 17088M 2024-09-21T09:25:58.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:25:58 smithi022 ceph-mon[27763]: pgmap v115: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-21T09:25:58.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:58 smithi022 ceph-mon[39525]: Detected new or changed devices on smithi064 2024-09-21T09:25:58.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:58 smithi022 ceph-mon[39525]: Adjusting osd_memory_target on smithi064 to 17088M 2024-09-21T09:25:58.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:25:58 smithi022 ceph-mon[39525]: pgmap v115: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-21T09:26:00.659 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:00 smithi064 ceph-mon[31124]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 120 KiB/s, 0 objects/s recovering 2024-09-21T09:26:00.660 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:00 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:26:00.660 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:00 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:26:00.660 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:00 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:26:00.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:00 smithi022 ceph-mon[27763]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 120 KiB/s, 0 objects/s recovering 2024-09-21T09:26:00.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:00 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:26:00.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:00 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:26:00.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:00 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:26:00.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:00 smithi022 ceph-mon[39525]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 120 KiB/s, 0 objects/s recovering 2024-09-21T09:26:00.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:00 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:26:00.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:00 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:26:00.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:00 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:26:01.500 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:01 smithi064 ceph-mon[31124]: from='client.24214 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi064:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:26:01.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:01 smithi022 ceph-mon[27763]: from='client.24214 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi064:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:26:01.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:01 smithi022 ceph-mon[39525]: from='client.24214 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi064:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:26:02.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:02 smithi064 ceph-mon[31124]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-09-21T09:26:02.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:02 smithi022 ceph-mon[27763]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-09-21T09:26:02.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:02 smithi022 ceph-mon[39525]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-09-21T09:26:03.682 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:03 smithi064 ceph-mon[31124]: from='client.? 172.21.15.64:0/4084850536' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a920292e-6301-4b67-b2a1-30f4788f8fd3"}]: dispatch 2024-09-21T09:26:03.682 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:03 smithi064 ceph-mon[31124]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a920292e-6301-4b67-b2a1-30f4788f8fd3"}]: dispatch 2024-09-21T09:26:03.682 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:03 smithi064 ceph-mon[31124]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a920292e-6301-4b67-b2a1-30f4788f8fd3"}]': finished 2024-09-21T09:26:03.682 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:03 smithi064 ceph-mon[31124]: osdmap e33: 6 total, 5 up, 6 in 2024-09-21T09:26:03.682 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:03 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:26:03.682 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:03 smithi064 ceph-mon[31124]: from='client.? 172.21.15.64:0/724970902' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:26:03.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:03 smithi022 ceph-mon[27763]: from='client.? 172.21.15.64:0/4084850536' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a920292e-6301-4b67-b2a1-30f4788f8fd3"}]: dispatch 2024-09-21T09:26:03.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:03 smithi022 ceph-mon[27763]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a920292e-6301-4b67-b2a1-30f4788f8fd3"}]: dispatch 2024-09-21T09:26:03.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:03 smithi022 ceph-mon[27763]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a920292e-6301-4b67-b2a1-30f4788f8fd3"}]': finished 2024-09-21T09:26:03.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:03 smithi022 ceph-mon[27763]: osdmap e33: 6 total, 5 up, 6 in 2024-09-21T09:26:03.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:03 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:26:03.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:03 smithi022 ceph-mon[27763]: from='client.? 172.21.15.64:0/724970902' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:26:03.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:03 smithi022 ceph-mon[39525]: from='client.? 172.21.15.64:0/4084850536' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a920292e-6301-4b67-b2a1-30f4788f8fd3"}]: dispatch 2024-09-21T09:26:03.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:03 smithi022 ceph-mon[39525]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a920292e-6301-4b67-b2a1-30f4788f8fd3"}]: dispatch 2024-09-21T09:26:03.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:03 smithi022 ceph-mon[39525]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a920292e-6301-4b67-b2a1-30f4788f8fd3"}]': finished 2024-09-21T09:26:03.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:03 smithi022 ceph-mon[39525]: osdmap e33: 6 total, 5 up, 6 in 2024-09-21T09:26:03.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:03 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:26:03.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:03 smithi022 ceph-mon[39525]: from='client.? 172.21.15.64:0/724970902' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:26:04.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:04 smithi064 ceph-mon[31124]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 76 KiB/s, 0 objects/s recovering 2024-09-21T09:26:04.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:04 smithi022 ceph-mon[39525]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 76 KiB/s, 0 objects/s recovering 2024-09-21T09:26:04.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:04 smithi022 ceph-mon[27763]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 76 KiB/s, 0 objects/s recovering 2024-09-21T09:26:06.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:06 smithi064 ceph-mon[31124]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 67 KiB/s, 0 objects/s recovering 2024-09-21T09:26:06.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:06 smithi022 ceph-mon[39525]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 67 KiB/s, 0 objects/s recovering 2024-09-21T09:26:06.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:06 smithi022 ceph-mon[27763]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 67 KiB/s, 0 objects/s recovering 2024-09-21T09:26:08.504 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:08 smithi064 ceph-mon[31124]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-21T09:26:08.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:08 smithi022 ceph-mon[39525]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-21T09:26:08.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:08 smithi022 ceph-mon[27763]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-21T09:26:09.635 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:09 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-21T09:26:09.636 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:09 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:26:09.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:09 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-21T09:26:09.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:09 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:26:09.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:09 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-21T09:26:09.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:09 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:26:10.506 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:10 smithi064 ceph-mon[31124]: Deploying daemon osd.5 on smithi064 2024-09-21T09:26:10.506 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:10 smithi064 ceph-mon[31124]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-21T09:26:10.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:10 smithi022 ceph-mon[27763]: Deploying daemon osd.5 on smithi064 2024-09-21T09:26:10.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:10 smithi022 ceph-mon[27763]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-21T09:26:10.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:10 smithi022 ceph-mon[39525]: Deploying daemon osd.5 on smithi064 2024-09-21T09:26:10.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:10 smithi022 ceph-mon[39525]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-21T09:26:12.615 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:12 smithi064 ceph-mon[31124]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-21T09:26:12.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:12 smithi022 ceph-mon[27763]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-21T09:26:12.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:12 smithi022 ceph-mon[39525]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-21T09:26:14.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:14 smithi064 ceph-mon[31124]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-21T09:26:14.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:14 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:26:14.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:14 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:26:14.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:14 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:26:14.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:14 smithi022 ceph-mon[27763]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-21T09:26:14.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:14 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:26:14.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:14 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:26:14.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:14 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:26:14.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:14 smithi022 ceph-mon[39525]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-21T09:26:14.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:14 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:26:14.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:14 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:26:14.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:14 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:26:16.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:16 smithi064 ceph-mon[31124]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-21T09:26:16.807 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:16 smithi022 ceph-mon[27763]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-21T09:26:16.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:16 smithi022 ceph-mon[39525]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-21T09:26:16.818 INFO:teuthology.orchestra.run.smithi064.stdout:Created osd(s) 5 on host 'smithi064' 2024-09-21T09:26:17.414 DEBUG:teuthology.orchestra.run.smithi064:osd.5> sudo journalctl -f -n 0 -u ceph-d65e801e-77fa-11ef-baf6-efdc52797490@osd.5.service 2024-09-21T09:26:17.418 INFO:tasks.cephadm:Deploying osd.6 on smithi064 with /dev/vg_nvme/lv_2... 2024-09-21T09:26:17.418 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d65e801e-77fa-11ef-baf6-efdc52797490 -- lvm zap /dev/vg_nvme/lv_2 2024-09-21T09:26:17.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:17 smithi064 ceph-mon[31124]: from='osd.5 [v2:172.21.15.64:6808/2673197529,v1:172.21.15.64:6809/2673197529]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-21T09:26:17.693 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:17 smithi064 ceph-mon[31124]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-21T09:26:17.693 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:17 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:26:17.693 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:17 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:26:17.726 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:26:17.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:17 smithi022 ceph-mon[39525]: from='osd.5 [v2:172.21.15.64:6808/2673197529,v1:172.21.15.64:6809/2673197529]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-21T09:26:17.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:17 smithi022 ceph-mon[39525]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-21T09:26:17.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:17 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:26:17.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:17 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:26:17.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:17 smithi022 ceph-mon[27763]: from='osd.5 [v2:172.21.15.64:6808/2673197529,v1:172.21.15.64:6809/2673197529]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-21T09:26:17.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:17 smithi022 ceph-mon[27763]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-21T09:26:17.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:17 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:26:17.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:17 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:26:18.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:18 smithi064 ceph-mon[31124]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-21T09:26:18.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:18 smithi064 ceph-mon[31124]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-21T09:26:18.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:18 smithi064 ceph-mon[31124]: osdmap e34: 6 total, 5 up, 6 in 2024-09-21T09:26:18.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:18 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:26:18.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:18 smithi064 ceph-mon[31124]: from='osd.5 [v2:172.21.15.64:6808/2673197529,v1:172.21.15.64:6809/2673197529]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2024-09-21T09:26:18.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:18 smithi064 ceph-mon[31124]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2024-09-21T09:26:18.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:18 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:26:18.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:18 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:26:18.692 INFO:journalctl@ceph.osd.5.smithi064.stdout:Sep 21 09:26:18 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-5[42862]: 2024-09-21T09:26:18.397+0000 7f1b5fd7b640 -1 osd.5 0 waiting for initial osdmap 2024-09-21T09:26:18.692 INFO:journalctl@ceph.osd.5.smithi064.stdout:Sep 21 09:26:18 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-5[42862]: 2024-09-21T09:26:18.403+0000 7f1b5a3b1640 -1 osd.5 35 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T09:26:18.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:18 smithi022 ceph-mon[27763]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-21T09:26:18.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:18 smithi022 ceph-mon[27763]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-21T09:26:18.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:18 smithi022 ceph-mon[27763]: osdmap e34: 6 total, 5 up, 6 in 2024-09-21T09:26:18.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:18 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:26:18.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:18 smithi022 ceph-mon[27763]: from='osd.5 [v2:172.21.15.64:6808/2673197529,v1:172.21.15.64:6809/2673197529]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2024-09-21T09:26:18.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:18 smithi022 ceph-mon[27763]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2024-09-21T09:26:18.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:18 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:26:18.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:18 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:26:18.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:18 smithi022 ceph-mon[39525]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-21T09:26:18.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:18 smithi022 ceph-mon[39525]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-21T09:26:18.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:18 smithi022 ceph-mon[39525]: osdmap e34: 6 total, 5 up, 6 in 2024-09-21T09:26:18.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:18 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:26:18.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:18 smithi022 ceph-mon[39525]: from='osd.5 [v2:172.21.15.64:6808/2673197529,v1:172.21.15.64:6809/2673197529]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2024-09-21T09:26:18.810 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:18 smithi022 ceph-mon[39525]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2024-09-21T09:26:18.810 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:18 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:26:18.810 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:18 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:26:19.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:19 smithi064 ceph-mon[31124]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]': finished 2024-09-21T09:26:19.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:19 smithi064 ceph-mon[31124]: osdmap e35: 6 total, 5 up, 6 in 2024-09-21T09:26:19.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:19 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:26:19.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:19 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:26:19.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:19 smithi022 ceph-mon[27763]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]': finished 2024-09-21T09:26:19.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:19 smithi022 ceph-mon[27763]: osdmap e35: 6 total, 5 up, 6 in 2024-09-21T09:26:19.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:19 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:26:19.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:19 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:26:19.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:19 smithi022 ceph-mon[39525]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]': finished 2024-09-21T09:26:19.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:19 smithi022 ceph-mon[39525]: osdmap e35: 6 total, 5 up, 6 in 2024-09-21T09:26:19.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:19 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:26:19.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:19 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:26:20.358 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-09-21T09:26:20.383 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d65e801e-77fa-11ef-baf6-efdc52797490 -- ceph orch daemon add osd smithi064:vg_nvme/lv_2 2024-09-21T09:26:20.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:20 smithi064 ceph-mon[31124]: purged_snaps scrub starts 2024-09-21T09:26:20.836 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:20 smithi064 ceph-mon[31124]: purged_snaps scrub ok 2024-09-21T09:26:20.836 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:20 smithi064 ceph-mon[31124]: pgmap v129: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-21T09:26:20.836 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:20 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:26:20.836 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:20 smithi064 ceph-mon[31124]: osd.5 [v2:172.21.15.64:6808/2673197529,v1:172.21.15.64:6809/2673197529] boot 2024-09-21T09:26:20.836 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:20 smithi064 ceph-mon[31124]: osdmap e36: 6 total, 6 up, 6 in 2024-09-21T09:26:20.836 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:20 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:26:20.836 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:20 smithi064 ceph-mon[31124]: osdmap e37: 6 total, 6 up, 6 in 2024-09-21T09:26:20.837 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:20 smithi022 ceph-mon[39525]: purged_snaps scrub starts 2024-09-21T09:26:20.837 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:20 smithi022 ceph-mon[39525]: purged_snaps scrub ok 2024-09-21T09:26:20.837 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:20 smithi022 ceph-mon[39525]: pgmap v129: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-21T09:26:20.837 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:20 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:26:20.837 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:20 smithi022 ceph-mon[39525]: osd.5 [v2:172.21.15.64:6808/2673197529,v1:172.21.15.64:6809/2673197529] boot 2024-09-21T09:26:20.837 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:20 smithi022 ceph-mon[39525]: osdmap e36: 6 total, 6 up, 6 in 2024-09-21T09:26:20.837 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:20 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:26:20.838 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:20 smithi022 ceph-mon[39525]: osdmap e37: 6 total, 6 up, 6 in 2024-09-21T09:26:20.838 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:20 smithi022 ceph-mon[27763]: purged_snaps scrub starts 2024-09-21T09:26:20.838 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:20 smithi022 ceph-mon[27763]: purged_snaps scrub ok 2024-09-21T09:26:20.838 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:20 smithi022 ceph-mon[27763]: pgmap v129: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-21T09:26:20.838 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:20 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:26:20.838 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:20 smithi022 ceph-mon[27763]: osd.5 [v2:172.21.15.64:6808/2673197529,v1:172.21.15.64:6809/2673197529] boot 2024-09-21T09:26:20.838 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:20 smithi022 ceph-mon[27763]: osdmap e36: 6 total, 6 up, 6 in 2024-09-21T09:26:20.838 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:20 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:26:20.838 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:20 smithi022 ceph-mon[27763]: osdmap e37: 6 total, 6 up, 6 in 2024-09-21T09:26:20.882 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:26:22.690 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:22 smithi064 ceph-mon[31124]: pgmap v132: 1 pgs: 1 peering; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:26:22.690 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:22 smithi064 ceph-mon[31124]: osdmap e38: 6 total, 6 up, 6 in 2024-09-21T09:26:22.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:22 smithi022 ceph-mon[39525]: pgmap v132: 1 pgs: 1 peering; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:26:22.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:22 smithi022 ceph-mon[39525]: osdmap e38: 6 total, 6 up, 6 in 2024-09-21T09:26:22.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:22 smithi022 ceph-mon[27763]: pgmap v132: 1 pgs: 1 peering; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:26:22.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:22 smithi022 ceph-mon[27763]: osdmap e38: 6 total, 6 up, 6 in 2024-09-21T09:26:24.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:23 smithi022 ceph-mon[39525]: Detected new or changed devices on smithi064 2024-09-21T09:26:24.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:23 smithi022 ceph-mon[39525]: pgmap v134: 1 pgs: 1 peering; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:26:24.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:23 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:26:24.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:23 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:26:24.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:23 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:26:24.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:23 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:26:24.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:23 smithi022 ceph-mon[39525]: Adjusting osd_memory_target on smithi064 to 8544M 2024-09-21T09:26:24.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:23 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:26:24.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:23 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:26:24.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:23 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:26:24.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:23 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:26:24.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:23 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:26:24.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:23 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:26:24.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:23 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:26:24.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:23 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:26:24.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:23 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:26:24.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:23 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:26:24.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:23 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:26:24.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:23 smithi022 ceph-mon[27763]: Detected new or changed devices on smithi064 2024-09-21T09:26:24.310 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:23 smithi022 ceph-mon[27763]: pgmap v134: 1 pgs: 1 peering; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:26:24.310 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:23 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:26:24.310 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:23 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:26:24.310 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:23 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:26:24.310 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:23 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:26:24.310 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:23 smithi022 ceph-mon[27763]: Adjusting osd_memory_target on smithi064 to 8544M 2024-09-21T09:26:24.310 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:23 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:26:24.310 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:23 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:26:24.310 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:23 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:26:24.310 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:23 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:26:24.310 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:23 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:26:24.310 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:23 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:26:24.311 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:23 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:26:24.311 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:23 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:26:24.311 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:23 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:26:24.311 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:23 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:26:24.311 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:23 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:26:24.312 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:23 smithi064 ceph-mon[31124]: Detected new or changed devices on smithi064 2024-09-21T09:26:24.313 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:23 smithi064 ceph-mon[31124]: pgmap v134: 1 pgs: 1 peering; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:26:24.313 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:23 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:26:24.313 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:23 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:26:24.313 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:23 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:26:24.313 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:23 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:26:24.313 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:23 smithi064 ceph-mon[31124]: Adjusting osd_memory_target on smithi064 to 8544M 2024-09-21T09:26:24.313 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:23 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:26:24.313 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:23 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:26:24.313 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:23 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:26:24.313 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:23 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:26:24.313 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:23 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:26:24.313 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:23 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:26:24.313 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:23 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:26:24.313 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:23 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:26:24.314 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:23 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:26:24.314 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:23 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:26:24.314 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:23 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:26:25.075 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:24 smithi064 ceph-mon[31124]: from='client.24241 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi064:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:26:25.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:24 smithi022 ceph-mon[39525]: from='client.24241 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi064:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:26:25.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:24 smithi022 ceph-mon[27763]: from='client.24241 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi064:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:26:26.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:25 smithi064 ceph-mon[31124]: pgmap v135: 1 pgs: 1 peering; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:26:26.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:25 smithi022 ceph-mon[39525]: pgmap v135: 1 pgs: 1 peering; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:26:26.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:25 smithi022 ceph-mon[27763]: pgmap v135: 1 pgs: 1 peering; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:26:27.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:26 smithi022 ceph-mon[39525]: from='client.? 172.21.15.64:0/3667020571' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ed01fe25-9b7d-48b9-b054-e054280e5826"}]: dispatch 2024-09-21T09:26:27.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:26 smithi022 ceph-mon[39525]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ed01fe25-9b7d-48b9-b054-e054280e5826"}]: dispatch 2024-09-21T09:26:27.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:26 smithi022 ceph-mon[39525]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ed01fe25-9b7d-48b9-b054-e054280e5826"}]': finished 2024-09-21T09:26:27.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:26 smithi022 ceph-mon[39525]: osdmap e39: 7 total, 6 up, 7 in 2024-09-21T09:26:27.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:26 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:26:27.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:26 smithi022 ceph-mon[39525]: from='client.? 172.21.15.64:0/1371755306' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:26:27.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:26 smithi022 ceph-mon[27763]: from='client.? 172.21.15.64:0/3667020571' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ed01fe25-9b7d-48b9-b054-e054280e5826"}]: dispatch 2024-09-21T09:26:27.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:26 smithi022 ceph-mon[27763]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ed01fe25-9b7d-48b9-b054-e054280e5826"}]: dispatch 2024-09-21T09:26:27.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:26 smithi022 ceph-mon[27763]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ed01fe25-9b7d-48b9-b054-e054280e5826"}]': finished 2024-09-21T09:26:27.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:26 smithi022 ceph-mon[27763]: osdmap e39: 7 total, 6 up, 7 in 2024-09-21T09:26:27.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:26 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:26:27.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:26 smithi022 ceph-mon[27763]: from='client.? 172.21.15.64:0/1371755306' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:26:27.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:26 smithi064 ceph-mon[31124]: from='client.? 172.21.15.64:0/3667020571' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ed01fe25-9b7d-48b9-b054-e054280e5826"}]: dispatch 2024-09-21T09:26:27.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:26 smithi064 ceph-mon[31124]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ed01fe25-9b7d-48b9-b054-e054280e5826"}]: dispatch 2024-09-21T09:26:27.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:26 smithi064 ceph-mon[31124]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ed01fe25-9b7d-48b9-b054-e054280e5826"}]': finished 2024-09-21T09:26:27.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:26 smithi064 ceph-mon[31124]: osdmap e39: 7 total, 6 up, 7 in 2024-09-21T09:26:27.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:26 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:26:27.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:26 smithi064 ceph-mon[31124]: from='client.? 172.21.15.64:0/1371755306' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:26:28.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:28 smithi022 ceph-mon[39525]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:26:28.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:28 smithi022 ceph-mon[27763]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:26:28.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:28 smithi064 ceph-mon[31124]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:26:30.151 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:30 smithi064 ceph-mon[31124]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:26:30.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:30 smithi022 ceph-mon[27763]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:26:30.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:30 smithi022 ceph-mon[39525]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:26:32.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:32 smithi022 ceph-mon[27763]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:26:32.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:32 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-21T09:26:32.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:32 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:26:32.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:32 smithi022 ceph-mon[39525]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:26:32.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:32 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-21T09:26:32.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:32 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:26:32.401 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:32 smithi064 ceph-mon[31124]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:26:32.402 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:32 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-21T09:26:32.402 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:32 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:26:33.340 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:33 smithi064 ceph-mon[31124]: Deploying daemon osd.6 on smithi064 2024-09-21T09:26:33.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:33 smithi022 ceph-mon[39525]: Deploying daemon osd.6 on smithi064 2024-09-21T09:26:33.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:33 smithi022 ceph-mon[27763]: Deploying daemon osd.6 on smithi064 2024-09-21T09:26:34.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:34 smithi064 ceph-mon[31124]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:26:34.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:34 smithi022 ceph-mon[27763]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:26:34.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:34 smithi022 ceph-mon[39525]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:26:36.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:36 smithi064 ceph-mon[31124]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:26:36.557 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:36 smithi022 ceph-mon[27763]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:26:36.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:36 smithi022 ceph-mon[39525]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:26:37.331 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:37 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:26:37.332 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:37 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:26:37.332 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:37 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:26:37.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:37 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:26:37.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:37 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:26:37.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:37 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:26:37.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:37 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:26:37.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:37 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:26:37.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:37 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:26:38.387 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:38 smithi064 ceph-mon[31124]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:26:38.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:38 smithi022 ceph-mon[27763]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:26:38.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:38 smithi022 ceph-mon[39525]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:26:39.939 INFO:teuthology.orchestra.run.smithi064.stdout:Created osd(s) 6 on host 'smithi064' 2024-09-21T09:26:40.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:40 smithi022 ceph-mon[39525]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:26:40.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:40 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:26:40.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:40 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:26:40.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:40 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:26:40.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:40 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:26:40.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:40 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:26:40.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:40 smithi022 ceph-mon[39525]: from='osd.6 [v2:172.21.15.64:6816/1977044182,v1:172.21.15.64:6817/1977044182]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-21T09:26:40.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:40 smithi022 ceph-mon[39525]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-21T09:26:40.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:40 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:26:40.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:40 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:26:40.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:40 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:26:40.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:40 smithi022 ceph-mon[27763]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:26:40.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:40 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:26:40.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:40 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:26:40.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:40 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:26:40.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:40 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:26:40.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:40 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:26:40.560 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:40 smithi022 ceph-mon[27763]: from='osd.6 [v2:172.21.15.64:6816/1977044182,v1:172.21.15.64:6817/1977044182]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-21T09:26:40.560 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:40 smithi022 ceph-mon[27763]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-21T09:26:40.560 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:40 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:26:40.560 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:40 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:26:40.560 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:40 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:26:40.599 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:40 smithi064 ceph-mon[31124]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:26:40.599 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:40 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:26:40.599 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:40 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:26:40.599 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:40 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:26:40.599 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:40 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:26:40.599 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:40 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:26:40.600 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:40 smithi064 ceph-mon[31124]: from='osd.6 [v2:172.21.15.64:6816/1977044182,v1:172.21.15.64:6817/1977044182]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-21T09:26:40.600 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:40 smithi064 ceph-mon[31124]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-21T09:26:40.600 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:40 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:26:40.600 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:40 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:26:40.600 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:40 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:26:40.629 DEBUG:teuthology.orchestra.run.smithi064:osd.6> sudo journalctl -f -n 0 -u ceph-d65e801e-77fa-11ef-baf6-efdc52797490@osd.6.service 2024-09-21T09:26:40.632 INFO:tasks.cephadm:Deploying osd.7 on smithi064 with /dev/vg_nvme/lv_1... 2024-09-21T09:26:40.632 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d65e801e-77fa-11ef-baf6-efdc52797490 -- lvm zap /dev/vg_nvme/lv_1 2024-09-21T09:26:41.343 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:26:41.780 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:41 smithi064 ceph-mon[31124]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-21T09:26:41.780 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:41 smithi064 ceph-mon[31124]: osdmap e40: 7 total, 6 up, 7 in 2024-09-21T09:26:41.780 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:41 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:26:41.780 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:41 smithi064 ceph-mon[31124]: from='osd.6 [v2:172.21.15.64:6816/1977044182,v1:172.21.15.64:6817/1977044182]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2024-09-21T09:26:41.780 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:41 smithi064 ceph-mon[31124]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2024-09-21T09:26:41.780 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:41 smithi064 ceph-mon[31124]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:26:41.781 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:26:41 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-6[49241]: 2024-09-21T09:26:41.687+0000 7fdaff6b1640 -1 osd.6 0 waiting for initial osdmap 2024-09-21T09:26:41.781 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:26:41 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-6[49241]: 2024-09-21T09:26:41.695+0000 7fdaf8ce5640 -1 osd.6 41 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T09:26:42.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:41 smithi022 ceph-mon[39525]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-21T09:26:42.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:41 smithi022 ceph-mon[39525]: osdmap e40: 7 total, 6 up, 7 in 2024-09-21T09:26:42.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:41 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:26:42.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:41 smithi022 ceph-mon[39525]: from='osd.6 [v2:172.21.15.64:6816/1977044182,v1:172.21.15.64:6817/1977044182]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2024-09-21T09:26:42.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:41 smithi022 ceph-mon[39525]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2024-09-21T09:26:42.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:41 smithi022 ceph-mon[39525]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:26:42.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:41 smithi022 ceph-mon[27763]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-21T09:26:42.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:41 smithi022 ceph-mon[27763]: osdmap e40: 7 total, 6 up, 7 in 2024-09-21T09:26:42.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:41 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:26:42.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:41 smithi022 ceph-mon[27763]: from='osd.6 [v2:172.21.15.64:6816/1977044182,v1:172.21.15.64:6817/1977044182]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2024-09-21T09:26:42.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:41 smithi022 ceph-mon[27763]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2024-09-21T09:26:42.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:41 smithi022 ceph-mon[27763]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:26:42.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:42 smithi064 ceph-mon[31124]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]': finished 2024-09-21T09:26:42.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:42 smithi064 ceph-mon[31124]: osdmap e41: 7 total, 6 up, 7 in 2024-09-21T09:26:42.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:42 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:26:42.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:42 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:26:43.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:42 smithi022 ceph-mon[39525]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]': finished 2024-09-21T09:26:43.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:42 smithi022 ceph-mon[39525]: osdmap e41: 7 total, 6 up, 7 in 2024-09-21T09:26:43.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:42 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:26:43.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:42 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:26:43.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:42 smithi022 ceph-mon[27763]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]': finished 2024-09-21T09:26:43.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:42 smithi022 ceph-mon[27763]: osdmap e41: 7 total, 6 up, 7 in 2024-09-21T09:26:43.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:42 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:26:43.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:42 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:26:43.924 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:43 smithi064 ceph-mon[31124]: purged_snaps scrub starts 2024-09-21T09:26:43.924 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:43 smithi064 ceph-mon[31124]: purged_snaps scrub ok 2024-09-21T09:26:43.924 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:43 smithi064 ceph-mon[31124]: osd.6 [v2:172.21.15.64:6816/1977044182,v1:172.21.15.64:6817/1977044182] boot 2024-09-21T09:26:43.924 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:43 smithi064 ceph-mon[31124]: osdmap e42: 7 total, 7 up, 7 in 2024-09-21T09:26:43.925 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:43 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:26:43.925 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:43 smithi064 ceph-mon[31124]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-21T09:26:43.925 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:43 smithi064 ceph-mon[31124]: osdmap e43: 7 total, 7 up, 7 in 2024-09-21T09:26:44.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:43 smithi022 ceph-mon[27763]: purged_snaps scrub starts 2024-09-21T09:26:44.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:43 smithi022 ceph-mon[27763]: purged_snaps scrub ok 2024-09-21T09:26:44.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:43 smithi022 ceph-mon[27763]: osd.6 [v2:172.21.15.64:6816/1977044182,v1:172.21.15.64:6817/1977044182] boot 2024-09-21T09:26:44.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:43 smithi022 ceph-mon[27763]: osdmap e42: 7 total, 7 up, 7 in 2024-09-21T09:26:44.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:43 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:26:44.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:43 smithi022 ceph-mon[27763]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-21T09:26:44.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:43 smithi022 ceph-mon[27763]: osdmap e43: 7 total, 7 up, 7 in 2024-09-21T09:26:44.059 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:43 smithi022 ceph-mon[39525]: purged_snaps scrub starts 2024-09-21T09:26:44.059 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:43 smithi022 ceph-mon[39525]: purged_snaps scrub ok 2024-09-21T09:26:44.059 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:43 smithi022 ceph-mon[39525]: osd.6 [v2:172.21.15.64:6816/1977044182,v1:172.21.15.64:6817/1977044182] boot 2024-09-21T09:26:44.059 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:43 smithi022 ceph-mon[39525]: osdmap e42: 7 total, 7 up, 7 in 2024-09-21T09:26:44.059 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:43 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:26:44.059 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:43 smithi022 ceph-mon[39525]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-21T09:26:44.059 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:43 smithi022 ceph-mon[39525]: osdmap e43: 7 total, 7 up, 7 in 2024-09-21T09:26:44.214 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-09-21T09:26:44.236 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d65e801e-77fa-11ef-baf6-efdc52797490 -- ceph orch daemon add osd smithi064:vg_nvme/lv_1 2024-09-21T09:26:44.796 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:26:45.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:45 smithi064 ceph-mon[31124]: osdmap e44: 7 total, 7 up, 7 in 2024-09-21T09:26:45.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:45 smithi064 ceph-mon[31124]: pgmap v151: 1 pgs: 1 remapped+peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-21T09:26:46.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:45 smithi022 ceph-mon[27763]: osdmap e44: 7 total, 7 up, 7 in 2024-09-21T09:26:46.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:45 smithi022 ceph-mon[27763]: pgmap v151: 1 pgs: 1 remapped+peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-21T09:26:46.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:45 smithi022 ceph-mon[39525]: osdmap e44: 7 total, 7 up, 7 in 2024-09-21T09:26:46.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:45 smithi022 ceph-mon[39525]: pgmap v151: 1 pgs: 1 remapped+peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-21T09:26:46.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:46 smithi064 ceph-mon[31124]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-21T09:26:46.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:46 smithi064 ceph-mon[31124]: Detected new or changed devices on smithi064 2024-09-21T09:26:46.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:46 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:26:46.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:46 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:26:46.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:46 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:26:46.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:46 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:26:46.943 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:46 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:26:46.943 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:46 smithi064 ceph-mon[31124]: Adjusting osd_memory_target on smithi064 to 5696M 2024-09-21T09:26:46.943 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:46 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:26:46.943 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:46 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:26:46.943 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:46 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:26:46.943 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:46 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:26:47.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:46 smithi022 ceph-mon[27763]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-21T09:26:47.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:46 smithi022 ceph-mon[27763]: Detected new or changed devices on smithi064 2024-09-21T09:26:47.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:46 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:26:47.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:46 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:26:47.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:46 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:26:47.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:46 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:26:47.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:46 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:26:47.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:46 smithi022 ceph-mon[27763]: Adjusting osd_memory_target on smithi064 to 5696M 2024-09-21T09:26:47.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:46 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:26:47.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:46 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:26:47.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:46 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:26:47.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:46 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:26:47.059 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:46 smithi022 ceph-mon[39525]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-21T09:26:47.059 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:46 smithi022 ceph-mon[39525]: Detected new or changed devices on smithi064 2024-09-21T09:26:47.059 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:46 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:26:47.059 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:46 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:26:47.059 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:46 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:26:47.060 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:46 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:26:47.060 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:46 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:26:47.060 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:46 smithi022 ceph-mon[39525]: Adjusting osd_memory_target on smithi064 to 5696M 2024-09-21T09:26:47.060 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:46 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:26:47.060 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:46 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:26:47.060 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:46 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:26:47.060 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:46 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:26:47.818 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:47 smithi064 ceph-mon[31124]: pgmap v152: 1 pgs: 1 remapped+peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-21T09:26:47.818 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:47 smithi064 ceph-mon[31124]: from='client.24268 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi064:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:26:47.818 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:47 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:26:47.818 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:47 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:26:47.818 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:47 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:26:48.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:47 smithi022 ceph-mon[27763]: pgmap v152: 1 pgs: 1 remapped+peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-21T09:26:48.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:47 smithi022 ceph-mon[27763]: from='client.24268 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi064:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:26:48.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:47 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:26:48.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:47 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:26:48.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:47 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:26:48.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:47 smithi022 ceph-mon[39525]: pgmap v152: 1 pgs: 1 remapped+peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-21T09:26:48.059 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:47 smithi022 ceph-mon[39525]: from='client.24268 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi064:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:26:48.059 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:47 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:26:48.059 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:47 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:26:48.059 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:47 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:26:49.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:49 smithi064 ceph-mon[31124]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-21T09:26:49.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:49 smithi064 ceph-mon[31124]: Cluster is now healthy 2024-09-21T09:26:49.557 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:49 smithi022 ceph-mon[39525]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-21T09:26:49.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:49 smithi022 ceph-mon[39525]: Cluster is now healthy 2024-09-21T09:26:49.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:49 smithi022 ceph-mon[27763]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-21T09:26:49.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:49 smithi022 ceph-mon[27763]: Cluster is now healthy 2024-09-21T09:26:50.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:50 smithi064 ceph-mon[31124]: pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-21T09:26:50.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:50 smithi064 ceph-mon[31124]: from='client.? 172.21.15.64:0/3419284402' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a9433be2-fcc7-46f5-91ee-13fbe71bf2f7"}]: dispatch 2024-09-21T09:26:50.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:50 smithi064 ceph-mon[31124]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a9433be2-fcc7-46f5-91ee-13fbe71bf2f7"}]: dispatch 2024-09-21T09:26:50.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:50 smithi064 ceph-mon[31124]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a9433be2-fcc7-46f5-91ee-13fbe71bf2f7"}]': finished 2024-09-21T09:26:50.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:50 smithi064 ceph-mon[31124]: osdmap e45: 8 total, 7 up, 8 in 2024-09-21T09:26:50.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:50 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:26:50.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:50 smithi064 ceph-mon[31124]: from='client.? 172.21.15.64:0/1698794083' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:26:50.459 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:50 smithi022 ceph-mon[39525]: pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-21T09:26:50.459 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:50 smithi022 ceph-mon[39525]: from='client.? 172.21.15.64:0/3419284402' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a9433be2-fcc7-46f5-91ee-13fbe71bf2f7"}]: dispatch 2024-09-21T09:26:50.459 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:50 smithi022 ceph-mon[39525]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a9433be2-fcc7-46f5-91ee-13fbe71bf2f7"}]: dispatch 2024-09-21T09:26:50.459 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:50 smithi022 ceph-mon[39525]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a9433be2-fcc7-46f5-91ee-13fbe71bf2f7"}]': finished 2024-09-21T09:26:50.459 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:50 smithi022 ceph-mon[39525]: osdmap e45: 8 total, 7 up, 8 in 2024-09-21T09:26:50.459 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:50 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:26:50.459 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:50 smithi022 ceph-mon[39525]: from='client.? 172.21.15.64:0/1698794083' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:26:50.460 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:50 smithi022 ceph-mon[27763]: pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-21T09:26:50.460 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:50 smithi022 ceph-mon[27763]: from='client.? 172.21.15.64:0/3419284402' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a9433be2-fcc7-46f5-91ee-13fbe71bf2f7"}]: dispatch 2024-09-21T09:26:50.460 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:50 smithi022 ceph-mon[27763]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a9433be2-fcc7-46f5-91ee-13fbe71bf2f7"}]: dispatch 2024-09-21T09:26:50.460 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:50 smithi022 ceph-mon[27763]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a9433be2-fcc7-46f5-91ee-13fbe71bf2f7"}]': finished 2024-09-21T09:26:50.460 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:50 smithi022 ceph-mon[27763]: osdmap e45: 8 total, 7 up, 8 in 2024-09-21T09:26:50.460 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:50 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:26:50.460 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:50 smithi022 ceph-mon[27763]: from='client.? 172.21.15.64:0/1698794083' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:26:52.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:52 smithi064 ceph-mon[31124]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 79 KiB/s, 0 objects/s recovering 2024-09-21T09:26:52.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:52 smithi022 ceph-mon[39525]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 79 KiB/s, 0 objects/s recovering 2024-09-21T09:26:52.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:52 smithi022 ceph-mon[27763]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 79 KiB/s, 0 objects/s recovering 2024-09-21T09:26:54.195 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:54 smithi064 ceph-mon[31124]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 70 KiB/s, 0 objects/s recovering 2024-09-21T09:26:54.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:54 smithi022 ceph-mon[27763]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 70 KiB/s, 0 objects/s recovering 2024-09-21T09:26:54.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:54 smithi022 ceph-mon[39525]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 70 KiB/s, 0 objects/s recovering 2024-09-21T09:26:56.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:56 smithi064 ceph-mon[31124]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-21T09:26:56.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:56 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-21T09:26:56.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:56 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:26:56.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:56 smithi022 ceph-mon[27763]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-21T09:26:56.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:56 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-21T09:26:56.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:56 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:26:56.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:56 smithi022 ceph-mon[39525]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-21T09:26:56.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:56 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-21T09:26:56.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:56 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:26:57.298 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:57 smithi064 ceph-mon[31124]: Deploying daemon osd.7 on smithi064 2024-09-21T09:26:57.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:57 smithi022 ceph-mon[27763]: Deploying daemon osd.7 on smithi064 2024-09-21T09:26:57.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:57 smithi022 ceph-mon[39525]: Deploying daemon osd.7 on smithi064 2024-09-21T09:26:58.253 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:26:58 smithi064 ceph-mon[31124]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-21T09:26:58.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:26:58 smithi022 ceph-mon[27763]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-21T09:26:58.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:26:58 smithi022 ceph-mon[39525]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-21T09:27:00.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:00 smithi064 ceph-mon[31124]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-21T09:27:00.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:00 smithi022 ceph-mon[27763]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-21T09:27:00.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:00 smithi022 ceph-mon[39525]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-21T09:27:01.417 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:01 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:27:01.417 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:01 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:27:01.417 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:01 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:27:01.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:01 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:27:01.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:01 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:27:01.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:01 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:27:01.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:01 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:27:01.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:01 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:27:01.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:01 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:27:02.633 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:02 smithi064 ceph-mon[31124]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-21T09:27:02.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:02 smithi022 ceph-mon[27763]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-21T09:27:02.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:02 smithi022 ceph-mon[39525]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-21T09:27:03.445 INFO:teuthology.orchestra.run.smithi064.stdout:Created osd(s) 7 on host 'smithi064' 2024-09-21T09:27:04.139 DEBUG:teuthology.orchestra.run.smithi064:osd.7> sudo journalctl -f -n 0 -u ceph-d65e801e-77fa-11ef-baf6-efdc52797490@osd.7.service 2024-09-21T09:27:04.141 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2024-09-21T09:27:04.142 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d65e801e-77fa-11ef-baf6-efdc52797490 -- ceph osd stat -f json 2024-09-21T09:27:04.405 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:04 smithi064 ceph-mon[31124]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-21T09:27:04.405 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:04 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:27:04.405 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:04 smithi064 ceph-mon[31124]: from='osd.7 [v2:172.21.15.64:6824/4274465032,v1:172.21.15.64:6825/4274465032]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-21T09:27:04.405 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:04 smithi064 ceph-mon[31124]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-21T09:27:04.405 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:04 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:27:04.405 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:04 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:27:04.405 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:04 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:27:04.408 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:27:04.529 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:04 smithi022 ceph-mon[27763]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-21T09:27:04.530 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:04 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:27:04.530 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:04 smithi022 ceph-mon[27763]: from='osd.7 [v2:172.21.15.64:6824/4274465032,v1:172.21.15.64:6825/4274465032]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-21T09:27:04.530 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:04 smithi022 ceph-mon[27763]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-21T09:27:04.530 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:04 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:27:04.530 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:04 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:27:04.530 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:04 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:27:04.530 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:04 smithi022 ceph-mon[39525]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-21T09:27:04.530 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:04 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:27:04.530 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:04 smithi022 ceph-mon[39525]: from='osd.7 [v2:172.21.15.64:6824/4274465032,v1:172.21.15.64:6825/4274465032]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-21T09:27:04.530 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:04 smithi022 ceph-mon[39525]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-21T09:27:04.531 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:04 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:27:04.531 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:04 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:27:04.531 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:04 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:27:05.543 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:05 smithi064 ceph-mon[31124]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-21T09:27:05.543 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:05 smithi064 ceph-mon[31124]: osdmap e46: 8 total, 7 up, 8 in 2024-09-21T09:27:05.543 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:05 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:27:05.543 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:05 smithi064 ceph-mon[31124]: from='osd.7 [v2:172.21.15.64:6824/4274465032,v1:172.21.15.64:6825/4274465032]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2024-09-21T09:27:05.543 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:05 smithi064 ceph-mon[31124]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2024-09-21T09:27:05.543 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:05 smithi064 ceph-mon[31124]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-21T09:27:05.543 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:27:05 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7[55468]: 2024-09-21T09:27:05.447+0000 7f2d9192c640 -1 osd.7 0 waiting for initial osdmap 2024-09-21T09:27:05.543 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:27:05 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7[55468]: 2024-09-21T09:27:05.454+0000 7f2d8bf62640 -1 osd.7 47 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T09:27:05.716 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:05 smithi022 ceph-mon[39525]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-21T09:27:05.716 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:05 smithi022 ceph-mon[39525]: osdmap e46: 8 total, 7 up, 8 in 2024-09-21T09:27:05.716 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:05 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:27:05.716 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:05 smithi022 ceph-mon[39525]: from='osd.7 [v2:172.21.15.64:6824/4274465032,v1:172.21.15.64:6825/4274465032]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2024-09-21T09:27:05.716 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:05 smithi022 ceph-mon[39525]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2024-09-21T09:27:05.716 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:05 smithi022 ceph-mon[39525]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-21T09:27:05.717 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:05 smithi022 ceph-mon[27763]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-21T09:27:05.717 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:05 smithi022 ceph-mon[27763]: osdmap e46: 8 total, 7 up, 8 in 2024-09-21T09:27:05.717 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:05 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:27:05.717 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:05 smithi022 ceph-mon[27763]: from='osd.7 [v2:172.21.15.64:6824/4274465032,v1:172.21.15.64:6825/4274465032]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2024-09-21T09:27:05.717 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:05 smithi022 ceph-mon[27763]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2024-09-21T09:27:05.717 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:05 smithi022 ceph-mon[27763]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-21T09:27:06.520 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-09-21T09:27:06.562 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:06 smithi064 ceph-mon[31124]: purged_snaps scrub starts 2024-09-21T09:27:06.562 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:06 smithi064 ceph-mon[31124]: purged_snaps scrub ok 2024-09-21T09:27:06.562 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:06 smithi064 ceph-mon[31124]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]': finished 2024-09-21T09:27:06.562 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:06 smithi064 ceph-mon[31124]: osdmap e47: 8 total, 7 up, 8 in 2024-09-21T09:27:06.562 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:06 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:27:06.562 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:06 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:27:06.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:06 smithi022 ceph-mon[27763]: purged_snaps scrub starts 2024-09-21T09:27:06.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:06 smithi022 ceph-mon[27763]: purged_snaps scrub ok 2024-09-21T09:27:06.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:06 smithi022 ceph-mon[27763]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]': finished 2024-09-21T09:27:06.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:06 smithi022 ceph-mon[27763]: osdmap e47: 8 total, 7 up, 8 in 2024-09-21T09:27:06.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:06 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:27:06.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:06 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:27:06.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:06 smithi022 ceph-mon[39525]: purged_snaps scrub starts 2024-09-21T09:27:06.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:06 smithi022 ceph-mon[39525]: purged_snaps scrub ok 2024-09-21T09:27:06.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:06 smithi022 ceph-mon[39525]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]': finished 2024-09-21T09:27:06.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:06 smithi022 ceph-mon[39525]: osdmap e47: 8 total, 7 up, 8 in 2024-09-21T09:27:06.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:06 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:27:06.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:06 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:27:07.264 INFO:teuthology.orchestra.run.smithi022.stdout:{"epoch":48,"num_osds":8,"num_up_osds":8,"osd_up_since":1726910826,"num_in_osds":8,"osd_in_since":1726910809,"num_remapped_pgs":1} 2024-09-21T09:27:07.264 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid d65e801e-77fa-11ef-baf6-efdc52797490 -- ceph osd dump --format=json 2024-09-21T09:27:07.520 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:27:07.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:07 smithi022 ceph-mon[39525]: osd.7 [v2:172.21.15.64:6824/4274465032,v1:172.21.15.64:6825/4274465032] boot 2024-09-21T09:27:07.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:07 smithi022 ceph-mon[39525]: osdmap e48: 8 total, 8 up, 8 in 2024-09-21T09:27:07.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:07 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:27:07.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:07 smithi022 ceph-mon[39525]: from='client.? 172.21.15.22:0/2978664786' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-21T09:27:07.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:07 smithi022 ceph-mon[39525]: Detected new or changed devices on smithi064 2024-09-21T09:27:07.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:07 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:27:07.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:07 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:27:07.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:07 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:27:07.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:07 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:27:07.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:07 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:27:07.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:07 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:27:07.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:07 smithi022 ceph-mon[39525]: Adjusting osd_memory_target on smithi064 to 4272M 2024-09-21T09:27:07.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:07 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:27:07.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:07 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:27:07.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:07 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:27:07.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:07 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:27:07.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:07 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:27:07.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:07 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:27:07.560 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:07 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:27:07.560 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:07 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:27:07.560 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:07 smithi022 ceph-mon[39525]: pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:27:07.560 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:07 smithi022 ceph-mon[27763]: osd.7 [v2:172.21.15.64:6824/4274465032,v1:172.21.15.64:6825/4274465032] boot 2024-09-21T09:27:07.560 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:07 smithi022 ceph-mon[27763]: osdmap e48: 8 total, 8 up, 8 in 2024-09-21T09:27:07.560 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:07 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:27:07.560 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:07 smithi022 ceph-mon[27763]: from='client.? 172.21.15.22:0/2978664786' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-21T09:27:07.560 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:07 smithi022 ceph-mon[27763]: Detected new or changed devices on smithi064 2024-09-21T09:27:07.560 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:07 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:27:07.560 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:07 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:27:07.561 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:07 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:27:07.561 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:07 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:27:07.561 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:07 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:27:07.561 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:07 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:27:07.561 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:07 smithi022 ceph-mon[27763]: Adjusting osd_memory_target on smithi064 to 4272M 2024-09-21T09:27:07.561 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:07 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:27:07.561 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:07 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:27:07.561 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:07 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:27:07.561 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:07 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:27:07.561 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:07 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:27:07.561 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:07 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:27:07.561 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:07 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:27:07.561 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:07 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:27:07.561 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:07 smithi022 ceph-mon[27763]: pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:27:07.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:07 smithi064 ceph-mon[31124]: osd.7 [v2:172.21.15.64:6824/4274465032,v1:172.21.15.64:6825/4274465032] boot 2024-09-21T09:27:07.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:07 smithi064 ceph-mon[31124]: osdmap e48: 8 total, 8 up, 8 in 2024-09-21T09:27:07.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:07 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:27:07.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:07 smithi064 ceph-mon[31124]: from='client.? 172.21.15.22:0/2978664786' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-21T09:27:07.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:07 smithi064 ceph-mon[31124]: Detected new or changed devices on smithi064 2024-09-21T09:27:07.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:07 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:27:07.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:07 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:27:07.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:07 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:27:07.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:07 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:27:07.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:07 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:27:07.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:07 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:27:07.943 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:07 smithi064 ceph-mon[31124]: Adjusting osd_memory_target on smithi064 to 4272M 2024-09-21T09:27:07.943 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:07 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:27:07.943 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:07 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:27:07.943 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:07 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:27:07.943 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:07 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:27:07.943 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:07 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:27:07.943 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:07 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:27:07.943 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:07 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:27:07.943 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:07 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:27:07.943 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:07 smithi064 ceph-mon[31124]: pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:27:08.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:08 smithi022 ceph-mon[27763]: osdmap e49: 8 total, 8 up, 8 in 2024-09-21T09:27:08.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:08 smithi022 ceph-mon[27763]: osdmap e50: 8 total, 8 up, 8 in 2024-09-21T09:27:08.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:08 smithi022 ceph-mon[39525]: osdmap e49: 8 total, 8 up, 8 in 2024-09-21T09:27:08.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:08 smithi022 ceph-mon[39525]: osdmap e50: 8 total, 8 up, 8 in 2024-09-21T09:27:08.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:08 smithi064 ceph-mon[31124]: osdmap e49: 8 total, 8 up, 8 in 2024-09-21T09:27:08.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:08 smithi064 ceph-mon[31124]: osdmap e50: 8 total, 8 up, 8 in 2024-09-21T09:27:09.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:09 smithi022 ceph-mon[27763]: pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:27:09.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:09 smithi022 ceph-mon[39525]: pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:27:09.626 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-09-21T09:27:09.626 INFO:teuthology.orchestra.run.smithi022.stdout:{"epoch":50,"fsid":"d65e801e-77fa-11ef-baf6-efdc52797490","created":"2024-09-21T09:22:04.474917+0000","modified":"2024-09-21T09:27:08.447043+0000","last_up_change":"2024-09-21T09:27:06.441515+0000","last_in_change":"2024-09-21T09:26:49.376072+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-21T09:25:07.405775+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"a1fbb74d-ffae-4d5f-81ab-9677d258a95c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6802","nonce":3576301437},{"type":"v1","addr":"172.21.15.22:6803","nonce":3576301437}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6804","nonce":3576301437},{"type":"v1","addr":"172.21.15.22:6805","nonce":3576301437}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6808","nonce":3576301437},{"type":"v1","addr":"172.21.15.22:6809","nonce":3576301437}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6806","nonce":3576301437},{"type":"v1","addr":"172.21.15.22:6807","nonce":3576301437}]},"public_addr":"172.21.15.22:6803/3576301437","cluster_addr":"172.21.15.22:6805/3576301437","heartbeat_back_addr":"172.21.15.22:6809/3576301437","heartbeat_front_addr":"172.21.15.22:6807/3576301437","state":["exists","up"]},{"osd":1,"uuid":"fa4c9385-ac93-4983-92b4-f23f0a3731db","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6810","nonce":3562679765},{"type":"v1","addr":"172.21.15.22:6811","nonce":3562679765}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6812","nonce":3562679765},{"type":"v1","addr":"172.21.15.22:6813","nonce":3562679765}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6816","nonce":3562679765},{"type":"v1","addr":"172.21.15.22:6817","nonce":3562679765}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6814","nonce":3562679765},{"type":"v1","addr":"172.21.15.22:6815","nonce":3562679765}]},"public_addr":"172.21.15.22:6811/3562679765","cluster_addr":"172.21.15.22:6813/3562679765","heartbeat_back_addr":"172.21.15.22:6817/3562679765","heartbeat_front_addr":"172.21.15.22:6815/3562679765","state":["exists","up"]},{"osd":2,"uuid":"1c27d056-2218-47dd-8f13-bc1c13888483","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6818","nonce":695486895},{"type":"v1","addr":"172.21.15.22:6819","nonce":695486895}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6820","nonce":695486895},{"type":"v1","addr":"172.21.15.22:6821","nonce":695486895}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6824","nonce":695486895},{"type":"v1","addr":"172.21.15.22:6825","nonce":695486895}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6822","nonce":695486895},{"type":"v1","addr":"172.21.15.22:6823","nonce":695486895}]},"public_addr":"172.21.15.22:6819/695486895","cluster_addr":"172.21.15.22:6821/695486895","heartbeat_back_addr":"172.21.15.22:6825/695486895","heartbeat_front_addr":"172.21.15.22:6823/695486895","state":["exists","up"]},{"osd":3,"uuid":"8668a232-df4a-46ba-b97d-3ed2f6ddeab2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6826","nonce":1394296125},{"type":"v1","addr":"172.21.15.22:6827","nonce":1394296125}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6828","nonce":1394296125},{"type":"v1","addr":"172.21.15.22:6829","nonce":1394296125}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6832","nonce":1394296125},{"type":"v1","addr":"172.21.15.22:6833","nonce":1394296125}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6830","nonce":1394296125},{"type":"v1","addr":"172.21.15.22:6831","nonce":1394296125}]},"public_addr":"172.21.15.22:6827/1394296125","cluster_addr":"172.21.15.22:6829/1394296125","heartbeat_back_addr":"172.21.15.22:6833/1394296125","heartbeat_front_addr":"172.21.15.22:6831/1394296125","state":["exists","up"]},{"osd":4,"uuid":"44dd8dcc-dbf7-41f7-9ec2-e66922e6c521","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2736095169},{"type":"v1","addr":"172.21.15.64:6801","nonce":2736095169}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6802","nonce":2736095169},{"type":"v1","addr":"172.21.15.64:6803","nonce":2736095169}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6806","nonce":2736095169},{"type":"v1","addr":"172.21.15.64:6807","nonce":2736095169}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":2736095169},{"type":"v1","addr":"172.21.15.64:6805","nonce":2736095169}]},"public_addr":"172.21.15.64:6801/2736095169","cluster_addr":"172.21.15.64:6803/2736095169","heartbeat_back_addr":"172.21.15.64:6807/2736095169","heartbeat_front_addr":"172.21.15.64:6805/2736095169","state":["exists","up"]},{"osd":5,"uuid":"a920292e-6301-4b67-b2a1-30f4788f8fd3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":37,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":2673197529},{"type":"v1","addr":"172.21.15.64:6809","nonce":2673197529}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2673197529},{"type":"v1","addr":"172.21.15.64:6811","nonce":2673197529}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2673197529},{"type":"v1","addr":"172.21.15.64:6815","nonce":2673197529}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6812","nonce":2673197529},{"type":"v1","addr":"172.21.15.64:6813","nonce":2673197529}]},"public_addr":"172.21.15.64:6809/2673197529","cluster_addr":"172.21.15.64:6811/2673197529","heartbeat_back_addr":"172.21.15.64:6815/2673197529","heartbeat_front_addr":"172.21.15.64:6813/2673197529","state":["exists","up"]},{"osd":6,"uuid":"ed01fe25-9b7d-48b9-b054-e054280e5826","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":43,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1977044182},{"type":"v1","addr":"172.21.15.64:6817","nonce":1977044182}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1977044182},{"type":"v1","addr":"172.21.15.64:6819","nonce":1977044182}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1977044182},{"type":"v1","addr":"172.21.15.64:6823","nonce":1977044182}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1977044182},{"type":"v1","addr":"172.21.15.64:6821","nonce":1977044182}]},"public_addr":"172.21.15.64:6817/1977044182","cluster_addr":"172.21.15.64:6819/1977044182","heartbeat_back_addr":"172.21.15.64:6823/1977044182","heartbeat_front_addr":"172.21.15.64:6821/1977044182","state":["exists","up"]},{"osd":7,"uuid":"a9433be2-fcc7-46f5-91ee-13fbe71bf2f7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":4274465032},{"type":"v1","addr":"172.21.15.64:6825","nonce":4274465032}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":4274465032},{"type":"v1","addr":"172.21.15.64:6827","nonce":4274465032}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":4274465032},{"type":"v1","addr":"172.21.15.64:6831","nonce":4274465032}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":4274465032},{"type":"v1","addr":"172.21.15.64:6829","nonce":4274465032}]},"public_addr":"172.21.15.64:6825/4274465032","cluster_addr":"172.21.15.64:6827/4274465032","heartbeat_back_addr":"172.21.15.64:6831/4274465032","heartbeat_front_addr":"172.21.15.64:6829/4274465032","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:24:15.825542+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:24:40.497779+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:25:04.918091+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:25:30.663725+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:25:53.521949+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:26:17.565302+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:26:40.876016+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:27:04.436407+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.22:6801/2821000087":"2024-09-22T09:22:50.910682+0000","172.21.15.22:6800/2821000087":"2024-09-22T09:22:50.910682+0000","172.21.15.22:0/4200462415":"2024-09-22T09:22:50.910682+0000","172.21.15.22:0/2669480478":"2024-09-22T09:22:50.910682+0000","172.21.15.22:0/1728251114":"2024-09-22T09:22:26.473640+0000","172.21.15.22:0/3766882712":"2024-09-22T09:22:50.910682+0000","172.21.15.22:0/1467628521":"2024-09-22T09:22:26.473640+0000","172.21.15.22:0/1022854075":"2024-09-22T09:22:26.473640+0000","172.21.15.22:6801/2983422368":"2024-09-22T09:22:26.473640+0000","172.21.15.22:6800/2983422368":"2024-09-22T09:22:26.473640+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-21T09:27:09.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:09 smithi064 ceph-mon[31124]: pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:27:10.341 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-09-21T09:25:07.405775+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '21', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2024-09-21T09:27:10.342 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid d65e801e-77fa-11ef-baf6-efdc52797490 -- ceph osd pool get .mgr pg_num 2024-09-21T09:27:10.599 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:27:10.720 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:10 smithi022 ceph-mon[39525]: from='client.? 172.21.15.22:0/3740362764' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T09:27:10.720 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:10 smithi022 ceph-mon[27763]: from='client.? 172.21.15.22:0/3740362764' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T09:27:10.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:10 smithi064 ceph-mon[31124]: from='client.? 172.21.15.22:0/3740362764' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T09:27:11.744 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:11 smithi022 ceph-mon[39525]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:27:11.744 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:11 smithi022 ceph-mon[27763]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:27:11.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:11 smithi064 ceph-mon[31124]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:27:12.419 INFO:teuthology.orchestra.run.smithi022.stdout:pg_num: 1 2024-09-21T09:27:12.800 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:12 smithi022 ceph-mon[39525]: from='client.? 172.21.15.22:0/1267961398' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-21T09:27:12.800 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:12 smithi022 ceph-mon[27763]: from='client.? 172.21.15.22:0/1267961398' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-21T09:27:12.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:12 smithi064 ceph-mon[31124]: from='client.? 172.21.15.22:0/1267961398' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-21T09:27:13.195 INFO:tasks.cephadm:Adding mds.a on smithi022 2024-09-21T09:27:13.196 INFO:tasks.cephadm:Adding mds.b on smithi064 2024-09-21T09:27:13.196 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d65e801e-77fa-11ef-baf6-efdc52797490 -- ceph orch apply mds all '2;smithi022=a;smithi064=b' 2024-09-21T09:27:13.462 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:27:13.584 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:13 smithi064 ceph-mon[31124]: pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:27:13.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:13 smithi022 ceph-mon[39525]: pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:27:13.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:13 smithi022 ceph-mon[27763]: pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:27:15.168 INFO:teuthology.orchestra.run.smithi064.stdout:Scheduled mds.all update... 2024-09-21T09:27:15.795 DEBUG:teuthology.orchestra.run.smithi022:mds.a> sudo journalctl -f -n 0 -u ceph-d65e801e-77fa-11ef-baf6-efdc52797490@mds.a.service 2024-09-21T09:27:15.798 DEBUG:teuthology.orchestra.run.smithi064:mds.b> sudo journalctl -f -n 0 -u ceph-d65e801e-77fa-11ef-baf6-efdc52797490@mds.b.service 2024-09-21T09:27:15.800 INFO:tasks.cephadm:Setting up CephFS filesystem(s)... 2024-09-21T09:27:15.800 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid d65e801e-77fa-11ef-baf6-efdc52797490 -- ceph fs dump --format=json 2024-09-21T09:27:16.413 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:16 smithi022 ceph-mon[39525]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:27:16.413 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:16 smithi022 ceph-mon[39525]: from='client.24310 -' entity='client.admin' cmd=[{"prefix": "orch apply mds", "fs_name": "all", "placement": "2;smithi022=a;smithi064=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:27:16.413 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:16 smithi022 ceph-mon[39525]: Saving service mds.all spec with placement smithi022=a;smithi064=b;count:2 2024-09-21T09:27:16.413 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:16 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:27:16.413 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:16 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:27:16.413 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:16 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:27:16.413 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:16 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:27:16.413 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:16 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:27:16.413 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:16 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-21T09:27:16.414 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:16 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-21T09:27:16.414 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:16 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:27:16.414 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:16 smithi022 ceph-mon[27763]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:27:16.414 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:16 smithi022 ceph-mon[27763]: from='client.24310 -' entity='client.admin' cmd=[{"prefix": "orch apply mds", "fs_name": "all", "placement": "2;smithi022=a;smithi064=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:27:16.414 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:16 smithi022 ceph-mon[27763]: Saving service mds.all spec with placement smithi022=a;smithi064=b;count:2 2024-09-21T09:27:16.414 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:16 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:27:16.414 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:16 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:27:16.414 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:16 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:27:16.414 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:16 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:27:16.414 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:16 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:27:16.415 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:16 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-21T09:27:16.415 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:16 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-21T09:27:16.415 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:16 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:27:16.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:16 smithi064 ceph-mon[31124]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:27:16.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:16 smithi064 ceph-mon[31124]: from='client.24310 -' entity='client.admin' cmd=[{"prefix": "orch apply mds", "fs_name": "all", "placement": "2;smithi022=a;smithi064=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:27:16.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:16 smithi064 ceph-mon[31124]: Saving service mds.all spec with placement smithi022=a;smithi064=b;count:2 2024-09-21T09:27:16.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:16 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:27:16.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:16 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:27:16.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:16 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:27:16.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:16 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:27:16.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:16 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:27:16.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:16 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-21T09:27:16.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:16 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-21T09:27:16.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:16 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:27:17.096 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:27:17.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:17 smithi064 ceph-mon[31124]: Deploying daemon mds.a on smithi022 2024-09-21T09:27:17.479 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:17 smithi022 ceph-mon[39525]: Deploying daemon mds.a on smithi022 2024-09-21T09:27:17.479 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:17 smithi022 ceph-mon[27763]: Deploying daemon mds.a on smithi022 2024-09-21T09:27:18.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:18 smithi064 ceph-mon[31124]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:27:18.508 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:18 smithi022 ceph-mon[39525]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:27:18.508 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:18 smithi022 ceph-mon[27763]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:27:18.777 INFO:journalctl@ceph.mds.a.smithi022.stdout:Sep 21 09:27:18 smithi022 podman[68112]: 2024-09-21 09:27:18.658805367 +0000 UTC m=+0.019929688 image pull fc3ec4a4f6be575045de0fd8a2fe84939e888152c447bdedb90add7841f7bef9 quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2 2024-09-21T09:27:18.777 INFO:journalctl@ceph.mds.a.smithi022.stdout:Sep 21 09:27:18 smithi022 podman[68112]: 2024-09-21 09:27:18.775304826 +0000 UTC m=+0.136429144 container create 34e4221f3a2777f03221291ba32091db4dc7dbfda30062481512245696e66950 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mds-a, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, org.label-schema.schema-version=1.0, RELEASE=quincy-fbd1a6a, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux ) 2024-09-21T09:27:19.310 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-09-21T09:27:19.310 INFO:teuthology.orchestra.run.smithi022.stdout:{"epoch":2,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14424,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.22:6835/3558312740","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6834","nonce":3558312740},{"type":"v1","addr":"172.21.15.22:6835","nonce":3558312740}]},"join_fscid":-1,"export_targets":[],"features":4540138320759226367,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":2}],"filesystems":[]} 2024-09-21T09:27:19.310 INFO:teuthology.orchestra.run.smithi022.stderr:dumped fsmap epoch 2 2024-09-21T09:27:19.558 INFO:journalctl@ceph.mds.a.smithi022.stdout:Sep 21 09:27:19 smithi022 podman[68112]: 2024-09-21 09:27:19.095365836 +0000 UTC m=+0.456490155 container init 34e4221f3a2777f03221291ba32091db4dc7dbfda30062481512245696e66950 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mds-a, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, RELEASE=quincy-fbd1a6a, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, maintainer=Guillaume Abrioux ) 2024-09-21T09:27:19.558 INFO:journalctl@ceph.mds.a.smithi022.stdout:Sep 21 09:27:19 smithi022 podman[68112]: 2024-09-21 09:27:19.101579785 +0000 UTC m=+0.462704103 container start 34e4221f3a2777f03221291ba32091db4dc7dbfda30062481512245696e66950 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mds-a, RELEASE=quincy-fbd1a6a, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, ceph=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD) 2024-09-21T09:27:19.558 INFO:journalctl@ceph.mds.a.smithi022.stdout:Sep 21 09:27:19 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mds-a[68145]: starting mds.a at 2024-09-21T09:27:19.558 INFO:journalctl@ceph.mds.a.smithi022.stdout:Sep 21 09:27:19 smithi022 bash[68112]: 34e4221f3a2777f03221291ba32091db4dc7dbfda30062481512245696e66950 2024-09-21T09:27:19.558 INFO:journalctl@ceph.mds.a.smithi022.stdout:Sep 21 09:27:19 smithi022 systemd[1]: Started Ceph mds.a for d65e801e-77fa-11ef-baf6-efdc52797490. 2024-09-21T09:27:20.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:20 smithi022 ceph-mon[27763]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:27:20.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:20 smithi022 ceph-mon[27763]: mds.? [v2:172.21.15.22:6834/3558312740,v1:172.21.15.22:6835/3558312740] up:boot 2024-09-21T09:27:20.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:20 smithi022 ceph-mon[27763]: fsmap 1 up:standby 2024-09-21T09:27:20.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:20 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-09-21T09:27:20.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:20 smithi022 ceph-mon[27763]: from='client.? 172.21.15.22:0/3179499838' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-21T09:27:20.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:20 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:27:20.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:20 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:27:20.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:20 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:27:20.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:20 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-21T09:27:20.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:20 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-21T09:27:20.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:20 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:27:20.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:20 smithi022 ceph-mon[27763]: Deploying daemon mds.b on smithi064 2024-09-21T09:27:20.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:20 smithi022 ceph-mon[39525]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:27:20.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:20 smithi022 ceph-mon[39525]: mds.? [v2:172.21.15.22:6834/3558312740,v1:172.21.15.22:6835/3558312740] up:boot 2024-09-21T09:27:20.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:20 smithi022 ceph-mon[39525]: fsmap 1 up:standby 2024-09-21T09:27:20.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:20 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-09-21T09:27:20.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:20 smithi022 ceph-mon[39525]: from='client.? 172.21.15.22:0/3179499838' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-21T09:27:20.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:20 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:27:20.310 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:20 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:27:20.310 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:20 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:27:20.310 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:20 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-21T09:27:20.310 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:20 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-21T09:27:20.310 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:20 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:27:20.310 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:20 smithi022 ceph-mon[39525]: Deploying daemon mds.b on smithi064 2024-09-21T09:27:20.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:20 smithi064 ceph-mon[31124]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:27:20.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:20 smithi064 ceph-mon[31124]: mds.? [v2:172.21.15.22:6834/3558312740,v1:172.21.15.22:6835/3558312740] up:boot 2024-09-21T09:27:20.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:20 smithi064 ceph-mon[31124]: fsmap 1 up:standby 2024-09-21T09:27:20.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:20 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-09-21T09:27:20.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:20 smithi064 ceph-mon[31124]: from='client.? 172.21.15.22:0/3179499838' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-21T09:27:20.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:20 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:27:20.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:20 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:27:20.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:20 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:27:20.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:20 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-21T09:27:20.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:20 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-21T09:27:20.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:20 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:27:20.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:20 smithi064 ceph-mon[31124]: Deploying daemon mds.b on smithi064 2024-09-21T09:27:21.580 INFO:journalctl@ceph.mds.b.smithi064.stdout:Sep 21 09:27:21 smithi064 podman[59613]: 2024-09-21 09:27:21.463610928 +0000 UTC m=+0.119921188 container create 0f846b3ec6947704b96b602066283327e11f659fdf60a4ce5c5e0f40a4c41781 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mds-b, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, RELEASE=quincy-fbd1a6a, org.label-schema.vendor=CentOS, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2) 2024-09-21T09:27:22.038 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid d65e801e-77fa-11ef-baf6-efdc52797490 -- ceph fs dump --format=json 2024-09-21T09:27:22.134 INFO:journalctl@ceph.mds.b.smithi064.stdout:Sep 21 09:27:21 smithi064 podman[59613]: 2024-09-21 09:27:21.716267866 +0000 UTC m=+0.372578138 container init 0f846b3ec6947704b96b602066283327e11f659fdf60a4ce5c5e0f40a4c41781 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mds-b, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240731, RELEASE=quincy-fbd1a6a, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.license=GPLv2) 2024-09-21T09:27:22.135 INFO:journalctl@ceph.mds.b.smithi064.stdout:Sep 21 09:27:21 smithi064 podman[59613]: 2024-09-21 09:27:21.721149187 +0000 UTC m=+0.377459452 container start 0f846b3ec6947704b96b602066283327e11f659fdf60a4ce5c5e0f40a4c41781 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mds-b, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-fbd1a6a, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2) 2024-09-21T09:27:22.135 INFO:journalctl@ceph.mds.b.smithi064.stdout:Sep 21 09:27:21 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mds-b[59631]: starting mds.b at 2024-09-21T09:27:22.135 INFO:journalctl@ceph.mds.b.smithi064.stdout:Sep 21 09:27:21 smithi064 bash[59613]: 0f846b3ec6947704b96b602066283327e11f659fdf60a4ce5c5e0f40a4c41781 2024-09-21T09:27:22.135 INFO:journalctl@ceph.mds.b.smithi064.stdout:Sep 21 09:27:21 smithi064 systemd[1]: Started Ceph mds.b for d65e801e-77fa-11ef-baf6-efdc52797490. 2024-09-21T09:27:22.305 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:27:22.367 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:22 smithi022 ceph-mon[39525]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:27:22.367 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:22 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:27:22.367 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:22 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:27:22.367 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:22 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:27:22.367 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:22 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:27:22.367 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:22 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:27:22.367 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:22 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:27:22.367 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:22 smithi022 ceph-mon[27763]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:27:22.367 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:22 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:27:22.368 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:22 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:27:22.368 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:22 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:27:22.368 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:22 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:27:22.368 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:22 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:27:22.368 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:22 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:27:22.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:22 smithi064 ceph-mon[31124]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:27:22.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:22 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:27:22.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:22 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:27:22.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:22 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:27:22.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:22 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:27:22.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:22 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:27:22.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:22 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:27:23.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:23 smithi064 ceph-mon[31124]: mds.? [v2:172.21.15.64:6832/3589447551,v1:172.21.15.64:6833/3589447551] up:boot 2024-09-21T09:27:23.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:23 smithi064 ceph-mon[31124]: fsmap 2 up:standby 2024-09-21T09:27:23.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:23 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-09-21T09:27:23.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:23 smithi022 ceph-mon[39525]: mds.? [v2:172.21.15.64:6832/3589447551,v1:172.21.15.64:6833/3589447551] up:boot 2024-09-21T09:27:23.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:23 smithi022 ceph-mon[39525]: fsmap 2 up:standby 2024-09-21T09:27:23.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:23 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-09-21T09:27:23.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:23 smithi022 ceph-mon[27763]: mds.? [v2:172.21.15.64:6832/3589447551,v1:172.21.15.64:6833/3589447551] up:boot 2024-09-21T09:27:23.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:23 smithi022 ceph-mon[27763]: fsmap 2 up:standby 2024-09-21T09:27:23.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:23 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-09-21T09:27:24.471 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:24 smithi022 ceph-mon[39525]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:27:24.471 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:24 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:27:24.471 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:24 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:27:24.471 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:24 smithi022 ceph-mon[27763]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:27:24.472 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:24 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:27:24.472 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:24 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:27:24.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:24 smithi064 ceph-mon[31124]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:27:24.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:24 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:27:24.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:24 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:27:24.913 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-09-21T09:27:24.913 INFO:teuthology.orchestra.run.smithi022.stdout:{"epoch":3,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14424,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.22:6835/3558312740","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6834","nonce":3558312740},{"type":"v1","addr":"172.21.15.22:6835","nonce":3558312740}]},"join_fscid":-1,"export_targets":[],"features":4540138320759226367,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":2},{"gid":24325,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6833/3589447551","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6832","nonce":3589447551},{"type":"v1","addr":"172.21.15.64:6833","nonce":3589447551}]},"join_fscid":-1,"export_targets":[],"features":4540138320759226367,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":3}],"filesystems":[]} 2024-09-21T09:27:24.913 INFO:teuthology.orchestra.run.smithi022.stderr:dumped fsmap epoch 3 2024-09-21T09:27:25.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:25 smithi022 ceph-mon[39525]: from='client.? 172.21.15.22:0/1456574738' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-21T09:27:25.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:25 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:27:25.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:25 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:27:25.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:25 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:27:25.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:25 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:27:25.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:25 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:27:25.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:25 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:27:25.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:25 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:27:25.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:25 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:27:25.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:25 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:27:25.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:25 smithi022 ceph-mon[27763]: from='client.? 172.21.15.22:0/1456574738' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-21T09:27:25.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:25 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:27:25.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:25 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:27:25.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:25 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:27:25.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:25 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:27:25.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:25 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:27:25.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:25 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:27:25.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:25 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:27:25.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:25 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:27:25.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:25 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:27:25.637 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid d65e801e-77fa-11ef-baf6-efdc52797490 -- ceph --format=json-pretty osd lspools 2024-09-21T09:27:25.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:25 smithi064 ceph-mon[31124]: from='client.? 172.21.15.22:0/1456574738' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-21T09:27:25.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:25 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:27:25.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:25 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:27:25.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:25 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:27:25.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:25 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:27:25.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:25 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:27:25.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:25 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:27:25.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:25 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:27:25.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:25 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:27:25.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:25 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:27:25.890 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:27:26.264 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:26 smithi022 ceph-mon[27763]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:27:26.264 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:26 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:27:26.265 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:26 smithi022 ceph-mon[39525]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:27:26.265 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:26 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:27:26.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:26 smithi064 ceph-mon[31124]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:27:26.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:26 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:27:27.652 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-09-21T09:27:27.652 INFO:teuthology.orchestra.run.smithi022.stdout:[ 2024-09-21T09:27:27.652 INFO:teuthology.orchestra.run.smithi022.stdout: { 2024-09-21T09:27:27.652 INFO:teuthology.orchestra.run.smithi022.stdout: "poolnum": 1, 2024-09-21T09:27:27.652 INFO:teuthology.orchestra.run.smithi022.stdout: "poolname": ".mgr" 2024-09-21T09:27:27.652 INFO:teuthology.orchestra.run.smithi022.stdout: } 2024-09-21T09:27:27.652 INFO:teuthology.orchestra.run.smithi022.stdout:] 2024-09-21T09:27:28.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:28 smithi022 ceph-mon[27763]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:27:28.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:28 smithi022 ceph-mon[27763]: from='client.? 172.21.15.22:0/3823363349' entity='client.admin' cmd=[{"prefix": "osd lspools", "format": "json-pretty"}]: dispatch 2024-09-21T09:27:28.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:28 smithi022 ceph-mon[39525]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:27:28.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:28 smithi022 ceph-mon[39525]: from='client.? 172.21.15.22:0/3823363349' entity='client.admin' cmd=[{"prefix": "osd lspools", "format": "json-pretty"}]: dispatch 2024-09-21T09:27:28.456 DEBUG:tasks.cephfs.filesystem:Creating filesystem 'cephfs' 2024-09-21T09:27:28.457 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid d65e801e-77fa-11ef-baf6-efdc52797490 -- ceph osd pool create cephfs_metadata --pg_num_min 64 2024-09-21T09:27:28.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:28 smithi064 ceph-mon[31124]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:27:28.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:28 smithi064 ceph-mon[31124]: from='client.? 172.21.15.22:0/3823363349' entity='client.admin' cmd=[{"prefix": "osd lspools", "format": "json-pretty"}]: dispatch 2024-09-21T09:27:28.716 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:27:30.544 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:30 smithi022 ceph-mon[39525]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:27:30.545 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:30 smithi022 ceph-mon[27763]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:27:30.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:30 smithi064 ceph-mon[31124]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:27:31.257 INFO:teuthology.orchestra.run.smithi022.stderr:pool 'cephfs_metadata' created 2024-09-21T09:27:31.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:31 smithi022 ceph-mon[39525]: from='client.? 172.21.15.22:0/964771310' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]: dispatch 2024-09-21T09:27:31.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:31 smithi022 ceph-mon[39525]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]: dispatch 2024-09-21T09:27:31.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:31 smithi022 ceph-mon[27763]: from='client.? 172.21.15.22:0/964771310' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]: dispatch 2024-09-21T09:27:31.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:31 smithi022 ceph-mon[27763]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]: dispatch 2024-09-21T09:27:31.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:31 smithi064 ceph-mon[31124]: from='client.? 172.21.15.22:0/964771310' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]: dispatch 2024-09-21T09:27:31.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:31 smithi064 ceph-mon[31124]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]: dispatch 2024-09-21T09:27:31.965 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid d65e801e-77fa-11ef-baf6-efdc52797490 -- ceph osd pool create cephfs_data 64 --pg_num_min 64 --target_size_ratio 0.9 2024-09-21T09:27:32.218 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:27:32.262 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:32 smithi022 ceph-mon[39525]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:27:32.262 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:32 smithi022 ceph-mon[39525]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]': finished 2024-09-21T09:27:32.262 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:32 smithi022 ceph-mon[39525]: osdmap e51: 8 total, 8 up, 8 in 2024-09-21T09:27:32.517 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:32 smithi022 ceph-mon[27763]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:27:32.517 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:32 smithi022 ceph-mon[27763]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]': finished 2024-09-21T09:27:32.517 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:32 smithi022 ceph-mon[27763]: osdmap e51: 8 total, 8 up, 8 in 2024-09-21T09:27:32.593 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:32 smithi064 ceph-mon[31124]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:27:32.593 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:32 smithi064 ceph-mon[31124]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]': finished 2024-09-21T09:27:32.593 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:32 smithi064 ceph-mon[31124]: osdmap e51: 8 total, 8 up, 8 in 2024-09-21T09:27:33.438 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:33 smithi022 ceph-mon[39525]: osdmap e52: 8 total, 8 up, 8 in 2024-09-21T09:27:33.438 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:33 smithi022 ceph-mon[27763]: osdmap e52: 8 total, 8 up, 8 in 2024-09-21T09:27:33.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:33 smithi064 ceph-mon[31124]: osdmap e52: 8 total, 8 up, 8 in 2024-09-21T09:27:34.617 INFO:teuthology.orchestra.run.smithi022.stderr:pool 'cephfs_data' created 2024-09-21T09:27:34.919 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:34 smithi022 ceph-mon[39525]: pgmap v183: 33 pgs: 6 creating+peering, 10 active+clean, 17 unknown; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:27:34.920 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:34 smithi022 ceph-mon[39525]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:27:34.920 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:34 smithi022 ceph-mon[39525]: osdmap e53: 8 total, 8 up, 8 in 2024-09-21T09:27:34.920 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:34 smithi022 ceph-mon[39525]: from='client.? 172.21.15.22:0/843160615' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9}]: dispatch 2024-09-21T09:27:34.920 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:34 smithi022 ceph-mon[27763]: pgmap v183: 33 pgs: 6 creating+peering, 10 active+clean, 17 unknown; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:27:34.920 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:34 smithi022 ceph-mon[27763]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:27:34.920 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:34 smithi022 ceph-mon[27763]: osdmap e53: 8 total, 8 up, 8 in 2024-09-21T09:27:34.920 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:34 smithi022 ceph-mon[27763]: from='client.? 172.21.15.22:0/843160615' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9}]: dispatch 2024-09-21T09:27:34.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:34 smithi064 ceph-mon[31124]: pgmap v183: 33 pgs: 6 creating+peering, 10 active+clean, 17 unknown; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:27:34.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:34 smithi064 ceph-mon[31124]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:27:34.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:34 smithi064 ceph-mon[31124]: osdmap e53: 8 total, 8 up, 8 in 2024-09-21T09:27:34.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:34 smithi064 ceph-mon[31124]: from='client.? 172.21.15.22:0/843160615' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9}]: dispatch 2024-09-21T09:27:35.392 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid d65e801e-77fa-11ef-baf6-efdc52797490 -- ceph fs new cephfs cephfs_metadata cephfs_data 2024-09-21T09:27:35.618 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:35 smithi022 ceph-mon[39525]: from='client.? 172.21.15.22:0/843160615' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9}]': finished 2024-09-21T09:27:35.618 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:35 smithi022 ceph-mon[39525]: osdmap e54: 8 total, 8 up, 8 in 2024-09-21T09:27:35.618 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:35 smithi022 ceph-mon[39525]: pgmap v186: 97 pgs: 6 creating+peering, 21 active+clean, 70 unknown; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:27:35.618 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:35 smithi022 ceph-mon[27763]: from='client.? 172.21.15.22:0/843160615' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9}]': finished 2024-09-21T09:27:35.619 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:35 smithi022 ceph-mon[27763]: osdmap e54: 8 total, 8 up, 8 in 2024-09-21T09:27:35.619 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:35 smithi022 ceph-mon[27763]: pgmap v186: 97 pgs: 6 creating+peering, 21 active+clean, 70 unknown; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:27:35.665 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:27:35.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:35 smithi064 ceph-mon[31124]: from='client.? 172.21.15.22:0/843160615' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9}]': finished 2024-09-21T09:27:35.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:35 smithi064 ceph-mon[31124]: osdmap e54: 8 total, 8 up, 8 in 2024-09-21T09:27:35.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:35 smithi064 ceph-mon[31124]: pgmap v186: 97 pgs: 6 creating+peering, 21 active+clean, 70 unknown; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:27:36.871 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:36 smithi022 ceph-mon[39525]: osdmap e55: 8 total, 8 up, 8 in 2024-09-21T09:27:36.872 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:36 smithi022 ceph-mon[27763]: osdmap e55: 8 total, 8 up, 8 in 2024-09-21T09:27:36.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:36 smithi064 ceph-mon[31124]: osdmap e55: 8 total, 8 up, 8 in 2024-09-21T09:27:37.630 INFO:teuthology.orchestra.run.smithi022.stderr:new fs with metadata pool 2 and data pool 3 2024-09-21T09:27:37.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:37 smithi064 ceph-mon[31124]: osdmap e56: 8 total, 8 up, 8 in 2024-09-21T09:27:37.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:37 smithi064 ceph-mon[31124]: pgmap v189: 97 pgs: 17 creating+peering, 44 active+clean, 36 unknown; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:27:37.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:37 smithi064 ceph-mon[31124]: from='client.? 172.21.15.22:0/2839636282' entity='client.admin' cmd=[{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]: dispatch 2024-09-21T09:27:37.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:37 smithi064 ceph-mon[31124]: from='client.? ' entity='client.admin' cmd=[{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]: dispatch 2024-09-21T09:27:37.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:37 smithi064 ceph-mon[31124]: daemon mds.b assigned to filesystem cephfs as rank 0 2024-09-21T09:27:37.952 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:37 smithi022 ceph-mon[39525]: osdmap e56: 8 total, 8 up, 8 in 2024-09-21T09:27:37.952 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:37 smithi022 ceph-mon[39525]: pgmap v189: 97 pgs: 17 creating+peering, 44 active+clean, 36 unknown; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:27:37.953 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:37 smithi022 ceph-mon[39525]: from='client.? 172.21.15.22:0/2839636282' entity='client.admin' cmd=[{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]: dispatch 2024-09-21T09:27:37.953 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:37 smithi022 ceph-mon[39525]: from='client.? ' entity='client.admin' cmd=[{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]: dispatch 2024-09-21T09:27:37.953 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:37 smithi022 ceph-mon[39525]: daemon mds.b assigned to filesystem cephfs as rank 0 2024-09-21T09:27:37.953 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:37 smithi022 ceph-mon[27763]: osdmap e56: 8 total, 8 up, 8 in 2024-09-21T09:27:37.953 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:37 smithi022 ceph-mon[27763]: pgmap v189: 97 pgs: 17 creating+peering, 44 active+clean, 36 unknown; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:27:37.953 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:37 smithi022 ceph-mon[27763]: from='client.? 172.21.15.22:0/2839636282' entity='client.admin' cmd=[{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]: dispatch 2024-09-21T09:27:37.953 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:37 smithi022 ceph-mon[27763]: from='client.? ' entity='client.admin' cmd=[{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]: dispatch 2024-09-21T09:27:37.953 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:37 smithi022 ceph-mon[27763]: daemon mds.b assigned to filesystem cephfs as rank 0 2024-09-21T09:27:38.349 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid d65e801e-77fa-11ef-baf6-efdc52797490 -- ceph osd dump --format=json 2024-09-21T09:27:38.607 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:27:38.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:38 smithi064 ceph-mon[31124]: osdmap e57: 8 total, 8 up, 8 in 2024-09-21T09:27:38.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:38 smithi064 ceph-mon[31124]: from='client.? ' entity='client.admin' cmd='[{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]': finished 2024-09-21T09:27:38.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:38 smithi064 ceph-mon[31124]: fsmap cephfs:1 {0=b=up:creating} 1 up:standby 2024-09-21T09:27:38.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:38 smithi064 ceph-mon[31124]: daemon mds.b is now active in filesystem cephfs as rank 0 2024-09-21T09:27:38.990 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:38 smithi022 ceph-mon[39525]: osdmap e57: 8 total, 8 up, 8 in 2024-09-21T09:27:38.990 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:38 smithi022 ceph-mon[39525]: from='client.? ' entity='client.admin' cmd='[{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]': finished 2024-09-21T09:27:38.990 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:38 smithi022 ceph-mon[39525]: fsmap cephfs:1 {0=b=up:creating} 1 up:standby 2024-09-21T09:27:38.990 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:38 smithi022 ceph-mon[39525]: daemon mds.b is now active in filesystem cephfs as rank 0 2024-09-21T09:27:38.990 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:38 smithi022 ceph-mon[27763]: osdmap e57: 8 total, 8 up, 8 in 2024-09-21T09:27:38.991 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:38 smithi022 ceph-mon[27763]: from='client.? ' entity='client.admin' cmd='[{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]': finished 2024-09-21T09:27:38.991 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:38 smithi022 ceph-mon[27763]: fsmap cephfs:1 {0=b=up:creating} 1 up:standby 2024-09-21T09:27:38.991 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:38 smithi022 ceph-mon[27763]: daemon mds.b is now active in filesystem cephfs as rank 0 2024-09-21T09:27:39.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:39 smithi022 ceph-mon[39525]: mds.? [v2:172.21.15.64:6832/3589447551,v1:172.21.15.64:6833/3589447551] up:active 2024-09-21T09:27:39.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:39 smithi022 ceph-mon[39525]: fsmap cephfs:1 {0=b=up:active} 1 up:standby 2024-09-21T09:27:39.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:39 smithi022 ceph-mon[39525]: fsmap cephfs:1 {0=b=up:active} 1 up:standby 2024-09-21T09:27:39.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:39 smithi022 ceph-mon[39525]: pgmap v191: 97 pgs: 17 creating+peering, 80 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.8 KiB/s wr, 8 op/s 2024-09-21T09:27:39.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:39 smithi022 ceph-mon[27763]: mds.? [v2:172.21.15.64:6832/3589447551,v1:172.21.15.64:6833/3589447551] up:active 2024-09-21T09:27:39.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:39 smithi022 ceph-mon[27763]: fsmap cephfs:1 {0=b=up:active} 1 up:standby 2024-09-21T09:27:39.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:39 smithi022 ceph-mon[27763]: fsmap cephfs:1 {0=b=up:active} 1 up:standby 2024-09-21T09:27:39.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:39 smithi022 ceph-mon[27763]: pgmap v191: 97 pgs: 17 creating+peering, 80 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.8 KiB/s wr, 8 op/s 2024-09-21T09:27:39.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:39 smithi064 ceph-mon[31124]: mds.? [v2:172.21.15.64:6832/3589447551,v1:172.21.15.64:6833/3589447551] up:active 2024-09-21T09:27:39.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:39 smithi064 ceph-mon[31124]: fsmap cephfs:1 {0=b=up:active} 1 up:standby 2024-09-21T09:27:39.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:39 smithi064 ceph-mon[31124]: fsmap cephfs:1 {0=b=up:active} 1 up:standby 2024-09-21T09:27:39.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:39 smithi064 ceph-mon[31124]: pgmap v191: 97 pgs: 17 creating+peering, 80 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.8 KiB/s wr, 8 op/s 2024-09-21T09:27:40.531 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-09-21T09:27:40.532 INFO:teuthology.orchestra.run.smithi022.stdout:{"epoch":57,"fsid":"d65e801e-77fa-11ef-baf6-efdc52797490","created":"2024-09-21T09:22:04.474917+0000","modified":"2024-09-21T09:27:37.584277+0000","last_up_change":"2024-09-21T09:27:06.441515+0000","last_in_change":"2024-09-21T09:26:49.376072+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-21T09:25:07.405775+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-09-21T09:27:31.151960+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"57","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-09-21T09:27:34.558875+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"57","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}}}],"osds":[{"osd":0,"uuid":"a1fbb74d-ffae-4d5f-81ab-9677d258a95c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":54,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6802","nonce":3576301437},{"type":"v1","addr":"172.21.15.22:6803","nonce":3576301437}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6804","nonce":3576301437},{"type":"v1","addr":"172.21.15.22:6805","nonce":3576301437}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6808","nonce":3576301437},{"type":"v1","addr":"172.21.15.22:6809","nonce":3576301437}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6806","nonce":3576301437},{"type":"v1","addr":"172.21.15.22:6807","nonce":3576301437}]},"public_addr":"172.21.15.22:6803/3576301437","cluster_addr":"172.21.15.22:6805/3576301437","heartbeat_back_addr":"172.21.15.22:6809/3576301437","heartbeat_front_addr":"172.21.15.22:6807/3576301437","state":["exists","up"]},{"osd":1,"uuid":"fa4c9385-ac93-4983-92b4-f23f0a3731db","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":54,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6810","nonce":3562679765},{"type":"v1","addr":"172.21.15.22:6811","nonce":3562679765}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6812","nonce":3562679765},{"type":"v1","addr":"172.21.15.22:6813","nonce":3562679765}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6816","nonce":3562679765},{"type":"v1","addr":"172.21.15.22:6817","nonce":3562679765}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6814","nonce":3562679765},{"type":"v1","addr":"172.21.15.22:6815","nonce":3562679765}]},"public_addr":"172.21.15.22:6811/3562679765","cluster_addr":"172.21.15.22:6813/3562679765","heartbeat_back_addr":"172.21.15.22:6817/3562679765","heartbeat_front_addr":"172.21.15.22:6815/3562679765","state":["exists","up"]},{"osd":2,"uuid":"1c27d056-2218-47dd-8f13-bc1c13888483","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":54,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6818","nonce":695486895},{"type":"v1","addr":"172.21.15.22:6819","nonce":695486895}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6820","nonce":695486895},{"type":"v1","addr":"172.21.15.22:6821","nonce":695486895}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6824","nonce":695486895},{"type":"v1","addr":"172.21.15.22:6825","nonce":695486895}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6822","nonce":695486895},{"type":"v1","addr":"172.21.15.22:6823","nonce":695486895}]},"public_addr":"172.21.15.22:6819/695486895","cluster_addr":"172.21.15.22:6821/695486895","heartbeat_back_addr":"172.21.15.22:6825/695486895","heartbeat_front_addr":"172.21.15.22:6823/695486895","state":["exists","up"]},{"osd":3,"uuid":"8668a232-df4a-46ba-b97d-3ed2f6ddeab2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":54,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6826","nonce":1394296125},{"type":"v1","addr":"172.21.15.22:6827","nonce":1394296125}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6828","nonce":1394296125},{"type":"v1","addr":"172.21.15.22:6829","nonce":1394296125}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6832","nonce":1394296125},{"type":"v1","addr":"172.21.15.22:6833","nonce":1394296125}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6830","nonce":1394296125},{"type":"v1","addr":"172.21.15.22:6831","nonce":1394296125}]},"public_addr":"172.21.15.22:6827/1394296125","cluster_addr":"172.21.15.22:6829/1394296125","heartbeat_back_addr":"172.21.15.22:6833/1394296125","heartbeat_front_addr":"172.21.15.22:6831/1394296125","state":["exists","up"]},{"osd":4,"uuid":"44dd8dcc-dbf7-41f7-9ec2-e66922e6c521","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":54,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2736095169},{"type":"v1","addr":"172.21.15.64:6801","nonce":2736095169}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6802","nonce":2736095169},{"type":"v1","addr":"172.21.15.64:6803","nonce":2736095169}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6806","nonce":2736095169},{"type":"v1","addr":"172.21.15.64:6807","nonce":2736095169}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":2736095169},{"type":"v1","addr":"172.21.15.64:6805","nonce":2736095169}]},"public_addr":"172.21.15.64:6801/2736095169","cluster_addr":"172.21.15.64:6803/2736095169","heartbeat_back_addr":"172.21.15.64:6807/2736095169","heartbeat_front_addr":"172.21.15.64:6805/2736095169","state":["exists","up"]},{"osd":5,"uuid":"a920292e-6301-4b67-b2a1-30f4788f8fd3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":54,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":2673197529},{"type":"v1","addr":"172.21.15.64:6809","nonce":2673197529}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2673197529},{"type":"v1","addr":"172.21.15.64:6811","nonce":2673197529}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2673197529},{"type":"v1","addr":"172.21.15.64:6815","nonce":2673197529}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6812","nonce":2673197529},{"type":"v1","addr":"172.21.15.64:6813","nonce":2673197529}]},"public_addr":"172.21.15.64:6809/2673197529","cluster_addr":"172.21.15.64:6811/2673197529","heartbeat_back_addr":"172.21.15.64:6815/2673197529","heartbeat_front_addr":"172.21.15.64:6813/2673197529","state":["exists","up"]},{"osd":6,"uuid":"ed01fe25-9b7d-48b9-b054-e054280e5826","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":54,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1977044182},{"type":"v1","addr":"172.21.15.64:6817","nonce":1977044182}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1977044182},{"type":"v1","addr":"172.21.15.64:6819","nonce":1977044182}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1977044182},{"type":"v1","addr":"172.21.15.64:6823","nonce":1977044182}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1977044182},{"type":"v1","addr":"172.21.15.64:6821","nonce":1977044182}]},"public_addr":"172.21.15.64:6817/1977044182","cluster_addr":"172.21.15.64:6819/1977044182","heartbeat_back_addr":"172.21.15.64:6823/1977044182","heartbeat_front_addr":"172.21.15.64:6821/1977044182","state":["exists","up"]},{"osd":7,"uuid":"a9433be2-fcc7-46f5-91ee-13fbe71bf2f7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":54,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":4274465032},{"type":"v1","addr":"172.21.15.64:6825","nonce":4274465032}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":4274465032},{"type":"v1","addr":"172.21.15.64:6827","nonce":4274465032}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":4274465032},{"type":"v1","addr":"172.21.15.64:6831","nonce":4274465032}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":4274465032},{"type":"v1","addr":"172.21.15.64:6829","nonce":4274465032}]},"public_addr":"172.21.15.64:6825/4274465032","cluster_addr":"172.21.15.64:6827/4274465032","heartbeat_back_addr":"172.21.15.64:6831/4274465032","heartbeat_front_addr":"172.21.15.64:6829/4274465032","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:24:15.825542+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:24:40.497779+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:25:04.918091+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:25:30.663725+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:25:53.521949+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:26:17.565302+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:26:40.876016+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:27:04.436407+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.22:6801/2821000087":"2024-09-22T09:22:50.910682+0000","172.21.15.22:6800/2821000087":"2024-09-22T09:22:50.910682+0000","172.21.15.22:0/4200462415":"2024-09-22T09:22:50.910682+0000","172.21.15.22:0/2669480478":"2024-09-22T09:22:50.910682+0000","172.21.15.22:0/1728251114":"2024-09-22T09:22:26.473640+0000","172.21.15.22:0/3766882712":"2024-09-22T09:22:50.910682+0000","172.21.15.22:0/1467628521":"2024-09-22T09:22:26.473640+0000","172.21.15.22:0/1022854075":"2024-09-22T09:22:26.473640+0000","172.21.15.22:6801/2983422368":"2024-09-22T09:22:26.473640+0000","172.21.15.22:6800/2983422368":"2024-09-22T09:22:26.473640+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-21T09:27:40.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:40 smithi022 ceph-mon[27763]: Health check cleared: POOL_APP_NOT_ENABLED (was: 2 pool(s) do not have an application enabled) 2024-09-21T09:27:40.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:40 smithi022 ceph-mon[27763]: Cluster is now healthy 2024-09-21T09:27:40.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:40 smithi022 ceph-mon[27763]: from='client.? 172.21.15.22:0/549538834' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T09:27:40.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:40 smithi022 ceph-mon[39525]: Health check cleared: POOL_APP_NOT_ENABLED (was: 2 pool(s) do not have an application enabled) 2024-09-21T09:27:40.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:40 smithi022 ceph-mon[39525]: Cluster is now healthy 2024-09-21T09:27:40.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:40 smithi022 ceph-mon[39525]: from='client.? 172.21.15.22:0/549538834' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T09:27:40.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:40 smithi064 ceph-mon[31124]: Health check cleared: POOL_APP_NOT_ENABLED (was: 2 pool(s) do not have an application enabled) 2024-09-21T09:27:40.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:40 smithi064 ceph-mon[31124]: Cluster is now healthy 2024-09-21T09:27:40.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:40 smithi064 ceph-mon[31124]: from='client.? 172.21.15.22:0/549538834' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T09:27:41.182 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid d65e801e-77fa-11ef-baf6-efdc52797490 -- ceph osd dump --format=json 2024-09-21T09:27:41.438 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:27:41.830 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:41 smithi022 ceph-mon[39525]: pgmap v192: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.2 KiB/s wr, 6 op/s 2024-09-21T09:27:41.831 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:41 smithi022 ceph-mon[27763]: pgmap v192: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.2 KiB/s wr, 6 op/s 2024-09-21T09:27:41.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:41 smithi064 ceph-mon[31124]: pgmap v192: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.2 KiB/s wr, 6 op/s 2024-09-21T09:27:42.441 INFO:journalctl@ceph.mds.b.smithi064.stdout:Sep 21 09:27:42 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mds-b[59631]: 2024-09-21T09:27:42.141+0000 7f76e4f31640 -1 mds.pinger is_rank_lagging: rank=0 was never sent ping request. 2024-09-21T09:27:43.375 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-09-21T09:27:43.375 INFO:teuthology.orchestra.run.smithi022.stdout:{"epoch":57,"fsid":"d65e801e-77fa-11ef-baf6-efdc52797490","created":"2024-09-21T09:22:04.474917+0000","modified":"2024-09-21T09:27:37.584277+0000","last_up_change":"2024-09-21T09:27:06.441515+0000","last_in_change":"2024-09-21T09:26:49.376072+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-21T09:25:07.405775+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-09-21T09:27:31.151960+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"57","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-09-21T09:27:34.558875+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"57","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}}}],"osds":[{"osd":0,"uuid":"a1fbb74d-ffae-4d5f-81ab-9677d258a95c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":54,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6802","nonce":3576301437},{"type":"v1","addr":"172.21.15.22:6803","nonce":3576301437}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6804","nonce":3576301437},{"type":"v1","addr":"172.21.15.22:6805","nonce":3576301437}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6808","nonce":3576301437},{"type":"v1","addr":"172.21.15.22:6809","nonce":3576301437}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6806","nonce":3576301437},{"type":"v1","addr":"172.21.15.22:6807","nonce":3576301437}]},"public_addr":"172.21.15.22:6803/3576301437","cluster_addr":"172.21.15.22:6805/3576301437","heartbeat_back_addr":"172.21.15.22:6809/3576301437","heartbeat_front_addr":"172.21.15.22:6807/3576301437","state":["exists","up"]},{"osd":1,"uuid":"fa4c9385-ac93-4983-92b4-f23f0a3731db","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":54,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6810","nonce":3562679765},{"type":"v1","addr":"172.21.15.22:6811","nonce":3562679765}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6812","nonce":3562679765},{"type":"v1","addr":"172.21.15.22:6813","nonce":3562679765}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6816","nonce":3562679765},{"type":"v1","addr":"172.21.15.22:6817","nonce":3562679765}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6814","nonce":3562679765},{"type":"v1","addr":"172.21.15.22:6815","nonce":3562679765}]},"public_addr":"172.21.15.22:6811/3562679765","cluster_addr":"172.21.15.22:6813/3562679765","heartbeat_back_addr":"172.21.15.22:6817/3562679765","heartbeat_front_addr":"172.21.15.22:6815/3562679765","state":["exists","up"]},{"osd":2,"uuid":"1c27d056-2218-47dd-8f13-bc1c13888483","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":54,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6818","nonce":695486895},{"type":"v1","addr":"172.21.15.22:6819","nonce":695486895}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6820","nonce":695486895},{"type":"v1","addr":"172.21.15.22:6821","nonce":695486895}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6824","nonce":695486895},{"type":"v1","addr":"172.21.15.22:6825","nonce":695486895}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6822","nonce":695486895},{"type":"v1","addr":"172.21.15.22:6823","nonce":695486895}]},"public_addr":"172.21.15.22:6819/695486895","cluster_addr":"172.21.15.22:6821/695486895","heartbeat_back_addr":"172.21.15.22:6825/695486895","heartbeat_front_addr":"172.21.15.22:6823/695486895","state":["exists","up"]},{"osd":3,"uuid":"8668a232-df4a-46ba-b97d-3ed2f6ddeab2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":54,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6826","nonce":1394296125},{"type":"v1","addr":"172.21.15.22:6827","nonce":1394296125}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6828","nonce":1394296125},{"type":"v1","addr":"172.21.15.22:6829","nonce":1394296125}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6832","nonce":1394296125},{"type":"v1","addr":"172.21.15.22:6833","nonce":1394296125}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6830","nonce":1394296125},{"type":"v1","addr":"172.21.15.22:6831","nonce":1394296125}]},"public_addr":"172.21.15.22:6827/1394296125","cluster_addr":"172.21.15.22:6829/1394296125","heartbeat_back_addr":"172.21.15.22:6833/1394296125","heartbeat_front_addr":"172.21.15.22:6831/1394296125","state":["exists","up"]},{"osd":4,"uuid":"44dd8dcc-dbf7-41f7-9ec2-e66922e6c521","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":54,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2736095169},{"type":"v1","addr":"172.21.15.64:6801","nonce":2736095169}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6802","nonce":2736095169},{"type":"v1","addr":"172.21.15.64:6803","nonce":2736095169}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6806","nonce":2736095169},{"type":"v1","addr":"172.21.15.64:6807","nonce":2736095169}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":2736095169},{"type":"v1","addr":"172.21.15.64:6805","nonce":2736095169}]},"public_addr":"172.21.15.64:6801/2736095169","cluster_addr":"172.21.15.64:6803/2736095169","heartbeat_back_addr":"172.21.15.64:6807/2736095169","heartbeat_front_addr":"172.21.15.64:6805/2736095169","state":["exists","up"]},{"osd":5,"uuid":"a920292e-6301-4b67-b2a1-30f4788f8fd3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":54,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":2673197529},{"type":"v1","addr":"172.21.15.64:6809","nonce":2673197529}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2673197529},{"type":"v1","addr":"172.21.15.64:6811","nonce":2673197529}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2673197529},{"type":"v1","addr":"172.21.15.64:6815","nonce":2673197529}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6812","nonce":2673197529},{"type":"v1","addr":"172.21.15.64:6813","nonce":2673197529}]},"public_addr":"172.21.15.64:6809/2673197529","cluster_addr":"172.21.15.64:6811/2673197529","heartbeat_back_addr":"172.21.15.64:6815/2673197529","heartbeat_front_addr":"172.21.15.64:6813/2673197529","state":["exists","up"]},{"osd":6,"uuid":"ed01fe25-9b7d-48b9-b054-e054280e5826","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":54,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1977044182},{"type":"v1","addr":"172.21.15.64:6817","nonce":1977044182}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1977044182},{"type":"v1","addr":"172.21.15.64:6819","nonce":1977044182}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1977044182},{"type":"v1","addr":"172.21.15.64:6823","nonce":1977044182}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1977044182},{"type":"v1","addr":"172.21.15.64:6821","nonce":1977044182}]},"public_addr":"172.21.15.64:6817/1977044182","cluster_addr":"172.21.15.64:6819/1977044182","heartbeat_back_addr":"172.21.15.64:6823/1977044182","heartbeat_front_addr":"172.21.15.64:6821/1977044182","state":["exists","up"]},{"osd":7,"uuid":"a9433be2-fcc7-46f5-91ee-13fbe71bf2f7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":54,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":4274465032},{"type":"v1","addr":"172.21.15.64:6825","nonce":4274465032}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":4274465032},{"type":"v1","addr":"172.21.15.64:6827","nonce":4274465032}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":4274465032},{"type":"v1","addr":"172.21.15.64:6831","nonce":4274465032}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":4274465032},{"type":"v1","addr":"172.21.15.64:6829","nonce":4274465032}]},"public_addr":"172.21.15.64:6825/4274465032","cluster_addr":"172.21.15.64:6827/4274465032","heartbeat_back_addr":"172.21.15.64:6831/4274465032","heartbeat_front_addr":"172.21.15.64:6829/4274465032","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:24:15.825542+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:24:40.497779+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:25:04.918091+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:25:30.663725+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:25:53.521949+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:26:17.565302+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:26:40.876016+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:27:04.436407+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.22:6801/2821000087":"2024-09-22T09:22:50.910682+0000","172.21.15.22:6800/2821000087":"2024-09-22T09:22:50.910682+0000","172.21.15.22:0/4200462415":"2024-09-22T09:22:50.910682+0000","172.21.15.22:0/2669480478":"2024-09-22T09:22:50.910682+0000","172.21.15.22:0/1728251114":"2024-09-22T09:22:26.473640+0000","172.21.15.22:0/3766882712":"2024-09-22T09:22:50.910682+0000","172.21.15.22:0/1467628521":"2024-09-22T09:22:26.473640+0000","172.21.15.22:0/1022854075":"2024-09-22T09:22:26.473640+0000","172.21.15.22:6801/2983422368":"2024-09-22T09:22:26.473640+0000","172.21.15.22:6800/2983422368":"2024-09-22T09:22:26.473640+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-21T09:27:43.987 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid d65e801e-77fa-11ef-baf6-efdc52797490 -- ceph fs set cephfs standby_count_wanted 0 2024-09-21T09:27:44.244 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:27:44.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:44 smithi022 ceph-mon[27763]: pgmap v193: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.8 KiB/s wr, 5 op/s 2024-09-21T09:27:44.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:44 smithi022 ceph-mon[27763]: from='client.? 172.21.15.22:0/1703933742' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T09:27:44.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:44 smithi022 ceph-mon[39525]: pgmap v193: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.8 KiB/s wr, 5 op/s 2024-09-21T09:27:44.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:44 smithi022 ceph-mon[39525]: from='client.? 172.21.15.22:0/1703933742' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T09:27:44.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:44 smithi064 ceph-mon[31124]: pgmap v193: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.8 KiB/s wr, 5 op/s 2024-09-21T09:27:44.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:44 smithi064 ceph-mon[31124]: from='client.? 172.21.15.22:0/1703933742' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T09:27:46.023 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:46 smithi022 ceph-mon[27763]: pgmap v194: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.6 KiB/s wr, 5 op/s 2024-09-21T09:27:46.307 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:46 smithi022 ceph-mon[39525]: pgmap v194: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.6 KiB/s wr, 5 op/s 2024-09-21T09:27:46.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:46 smithi064 ceph-mon[31124]: pgmap v194: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.6 KiB/s wr, 5 op/s 2024-09-21T09:27:47.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:47 smithi022 ceph-mon[39525]: from='client.? 172.21.15.22:0/3626642597' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"}]: dispatch 2024-09-21T09:27:47.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:47 smithi022 ceph-mon[27763]: from='client.? 172.21.15.22:0/3626642597' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"}]: dispatch 2024-09-21T09:27:47.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:47 smithi064 ceph-mon[31124]: from='client.? 172.21.15.22:0/3626642597' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"}]: dispatch 2024-09-21T09:27:47.727 DEBUG:tasks.cephfs.filesystem:fs_config: {} 2024-09-21T09:27:47.728 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid d65e801e-77fa-11ef-baf6-efdc52797490 -- ceph fs set cephfs allow_standby_replay false 2024-09-21T09:27:47.985 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:27:48.307 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:48 smithi022 ceph-mon[27763]: pgmap v195: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-09-21T09:27:48.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:48 smithi022 ceph-mon[27763]: from='client.? 172.21.15.22:0/3626642597' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"}]': finished 2024-09-21T09:27:48.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:48 smithi022 ceph-mon[27763]: fsmap cephfs:1 {0=b=up:active} 1 up:standby 2024-09-21T09:27:48.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:48 smithi022 ceph-mon[39525]: pgmap v195: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-09-21T09:27:48.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:48 smithi022 ceph-mon[39525]: from='client.? 172.21.15.22:0/3626642597' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"}]': finished 2024-09-21T09:27:48.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:48 smithi022 ceph-mon[39525]: fsmap cephfs:1 {0=b=up:active} 1 up:standby 2024-09-21T09:27:48.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:48 smithi064 ceph-mon[31124]: pgmap v195: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-09-21T09:27:48.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:48 smithi064 ceph-mon[31124]: from='client.? 172.21.15.22:0/3626642597' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"}]': finished 2024-09-21T09:27:48.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:48 smithi064 ceph-mon[31124]: fsmap cephfs:1 {0=b=up:active} 1 up:standby 2024-09-21T09:27:50.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:50 smithi022 ceph-mon[27763]: pgmap v196: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-21T09:27:50.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:50 smithi022 ceph-mon[27763]: from='client.? 172.21.15.22:0/1552313510' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "false"}]: dispatch 2024-09-21T09:27:50.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:50 smithi022 ceph-mon[39525]: pgmap v196: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-21T09:27:50.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:50 smithi022 ceph-mon[39525]: from='client.? 172.21.15.22:0/1552313510' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "false"}]: dispatch 2024-09-21T09:27:50.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:50 smithi064 ceph-mon[31124]: pgmap v196: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-21T09:27:50.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:50 smithi064 ceph-mon[31124]: from='client.? 172.21.15.22:0/1552313510' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "false"}]: dispatch 2024-09-21T09:27:50.665 DEBUG:tasks.cephfs.filesystem:Not Creating any subvols for filesystem 'cephfs' 2024-09-21T09:27:50.665 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid d65e801e-77fa-11ef-baf6-efdc52797490 -- ceph fs dump --format=json 2024-09-21T09:27:50.920 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:27:51.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:51 smithi022 ceph-mon[27763]: from='client.? 172.21.15.22:0/1552313510' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "false"}]': finished 2024-09-21T09:27:51.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:51 smithi022 ceph-mon[27763]: fsmap cephfs:1 {0=b=up:active} 1 up:standby 2024-09-21T09:27:51.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:51 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.3", "id": [7, 4]}]: dispatch 2024-09-21T09:27:51.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:51 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.a", "id": [4, 3]}]: dispatch 2024-09-21T09:27:51.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:51 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.b", "id": [0, 2]}]: dispatch 2024-09-21T09:27:51.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:51 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.17", "id": [0, 2]}]: dispatch 2024-09-21T09:27:51.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:51 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.22", "id": [0, 1]}]: dispatch 2024-09-21T09:27:51.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:51 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.24", "id": [4, 2]}]: dispatch 2024-09-21T09:27:51.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:51 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.37", "id": [5, 1]}]: dispatch 2024-09-21T09:27:51.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:51 smithi022 ceph-mon[39525]: from='client.? 172.21.15.22:0/1552313510' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "false"}]': finished 2024-09-21T09:27:51.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:51 smithi022 ceph-mon[39525]: fsmap cephfs:1 {0=b=up:active} 1 up:standby 2024-09-21T09:27:51.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:51 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.3", "id": [7, 4]}]: dispatch 2024-09-21T09:27:51.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:51 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.a", "id": [4, 3]}]: dispatch 2024-09-21T09:27:51.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:51 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.b", "id": [0, 2]}]: dispatch 2024-09-21T09:27:51.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:51 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.17", "id": [0, 2]}]: dispatch 2024-09-21T09:27:51.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:51 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.22", "id": [0, 1]}]: dispatch 2024-09-21T09:27:51.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:51 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.24", "id": [4, 2]}]: dispatch 2024-09-21T09:27:51.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:51 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.37", "id": [5, 1]}]: dispatch 2024-09-21T09:27:51.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:51 smithi064 ceph-mon[31124]: from='client.? 172.21.15.22:0/1552313510' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "false"}]': finished 2024-09-21T09:27:51.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:51 smithi064 ceph-mon[31124]: fsmap cephfs:1 {0=b=up:active} 1 up:standby 2024-09-21T09:27:51.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:51 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.3", "id": [7, 4]}]: dispatch 2024-09-21T09:27:51.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:51 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.a", "id": [4, 3]}]: dispatch 2024-09-21T09:27:51.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:51 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.b", "id": [0, 2]}]: dispatch 2024-09-21T09:27:51.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:51 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.17", "id": [0, 2]}]: dispatch 2024-09-21T09:27:51.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:51 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.22", "id": [0, 1]}]: dispatch 2024-09-21T09:27:51.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:51 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.24", "id": [4, 2]}]: dispatch 2024-09-21T09:27:51.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:51 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.37", "id": [5, 1]}]: dispatch 2024-09-21T09:27:52.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:52 smithi022 ceph-mon[27763]: pgmap v197: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 85 B/s wr, 0 op/s 2024-09-21T09:27:52.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:52 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.3", "id": [7, 4]}]': finished 2024-09-21T09:27:52.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:52 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.a", "id": [4, 3]}]': finished 2024-09-21T09:27:52.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:52 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.b", "id": [0, 2]}]': finished 2024-09-21T09:27:52.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:52 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.17", "id": [0, 2]}]': finished 2024-09-21T09:27:52.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:52 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.22", "id": [0, 1]}]': finished 2024-09-21T09:27:52.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:52 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.24", "id": [4, 2]}]': finished 2024-09-21T09:27:52.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:52 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.37", "id": [5, 1]}]': finished 2024-09-21T09:27:52.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:52 smithi022 ceph-mon[27763]: osdmap e58: 8 total, 8 up, 8 in 2024-09-21T09:27:52.059 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:52 smithi022 ceph-mon[39525]: pgmap v197: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 85 B/s wr, 0 op/s 2024-09-21T09:27:52.059 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:52 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.3", "id": [7, 4]}]': finished 2024-09-21T09:27:52.059 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:52 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.a", "id": [4, 3]}]': finished 2024-09-21T09:27:52.059 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:52 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.b", "id": [0, 2]}]': finished 2024-09-21T09:27:52.059 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:52 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.17", "id": [0, 2]}]': finished 2024-09-21T09:27:52.059 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:52 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.22", "id": [0, 1]}]': finished 2024-09-21T09:27:52.059 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:52 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.24", "id": [4, 2]}]': finished 2024-09-21T09:27:52.059 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:52 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.37", "id": [5, 1]}]': finished 2024-09-21T09:27:52.059 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:52 smithi022 ceph-mon[39525]: osdmap e58: 8 total, 8 up, 8 in 2024-09-21T09:27:52.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:52 smithi064 ceph-mon[31124]: pgmap v197: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 85 B/s wr, 0 op/s 2024-09-21T09:27:52.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:52 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.3", "id": [7, 4]}]': finished 2024-09-21T09:27:52.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:52 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.a", "id": [4, 3]}]': finished 2024-09-21T09:27:52.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:52 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.b", "id": [0, 2]}]': finished 2024-09-21T09:27:52.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:52 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.17", "id": [0, 2]}]': finished 2024-09-21T09:27:52.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:52 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.22", "id": [0, 1]}]': finished 2024-09-21T09:27:52.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:52 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.24", "id": [4, 2]}]': finished 2024-09-21T09:27:52.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:52 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.37", "id": [5, 1]}]': finished 2024-09-21T09:27:52.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:52 smithi064 ceph-mon[31124]: osdmap e58: 8 total, 8 up, 8 in 2024-09-21T09:27:52.949 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-09-21T09:27:52.950 INFO:teuthology.orchestra.run.smithi022.stdout:{"epoch":8,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14424,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.22:6835/3558312740","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6834","nonce":3558312740},{"type":"v1","addr":"172.21.15.22:6835","nonce":3558312740}]},"join_fscid":-1,"export_targets":[],"features":4540138320759226367,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":2}],"filesystems":[{"mdsmap":{"epoch":8,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-09-21T09:27:37.584397+0000","modified":"2024-09-21T09:27:50.029809+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"max_mds":1,"in":[0],"up":{"mds_0":24325},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24325":{"gid":24325,"name":"b","rank":0,"incarnation":4,"state":"up:active","state_seq":5,"addr":"172.21.15.64:6833/3589447551","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6832","nonce":3589447551},{"type":"v1","addr":"172.21.15.64:6833","nonce":3589447551}]},"join_fscid":-1,"export_targets":[],"features":4540138320759226367,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","standby_count_wanted":0},"id":1}]} 2024-09-21T09:27:52.950 INFO:teuthology.orchestra.run.smithi022.stderr:dumped fsmap epoch 8 2024-09-21T09:27:53.253 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:53 smithi022 ceph-mon[39525]: osdmap e59: 8 total, 8 up, 8 in 2024-09-21T09:27:53.253 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:53 smithi022 ceph-mon[39525]: from='client.? 172.21.15.22:0/2190817576' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-21T09:27:53.253 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:53 smithi022 ceph-mon[27763]: osdmap e59: 8 total, 8 up, 8 in 2024-09-21T09:27:53.253 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:53 smithi022 ceph-mon[27763]: from='client.? 172.21.15.22:0/2190817576' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-21T09:27:53.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:53 smithi064 ceph-mon[31124]: osdmap e59: 8 total, 8 up, 8 in 2024-09-21T09:27:53.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:53 smithi064 ceph-mon[31124]: from='client.? 172.21.15.22:0/2190817576' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-21T09:27:53.703 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid d65e801e-77fa-11ef-baf6-efdc52797490 -- ceph osd dump --format=json 2024-09-21T09:27:53.957 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:27:54.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:54 smithi022 ceph-mon[39525]: pgmap v200: 97 pgs: 1 peering, 96 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:27:54.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:54 smithi022 ceph-mon[27763]: pgmap v200: 97 pgs: 1 peering, 96 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:27:54.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:54 smithi064 ceph-mon[31124]: pgmap v200: 97 pgs: 1 peering, 96 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:27:55.767 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-09-21T09:27:55.767 INFO:teuthology.orchestra.run.smithi022.stdout:{"epoch":59,"fsid":"d65e801e-77fa-11ef-baf6-efdc52797490","created":"2024-09-21T09:22:04.474917+0000","modified":"2024-09-21T09:27:52.040938+0000","last_up_change":"2024-09-21T09:27:06.441515+0000","last_in_change":"2024-09-21T09:26:49.376072+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-21T09:25:07.405775+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-09-21T09:27:31.151960+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"57","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-09-21T09:27:34.558875+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"57","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}}}],"osds":[{"osd":0,"uuid":"a1fbb74d-ffae-4d5f-81ab-9677d258a95c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":54,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6802","nonce":3576301437},{"type":"v1","addr":"172.21.15.22:6803","nonce":3576301437}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6804","nonce":3576301437},{"type":"v1","addr":"172.21.15.22:6805","nonce":3576301437}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6808","nonce":3576301437},{"type":"v1","addr":"172.21.15.22:6809","nonce":3576301437}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6806","nonce":3576301437},{"type":"v1","addr":"172.21.15.22:6807","nonce":3576301437}]},"public_addr":"172.21.15.22:6803/3576301437","cluster_addr":"172.21.15.22:6805/3576301437","heartbeat_back_addr":"172.21.15.22:6809/3576301437","heartbeat_front_addr":"172.21.15.22:6807/3576301437","state":["exists","up"]},{"osd":1,"uuid":"fa4c9385-ac93-4983-92b4-f23f0a3731db","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":54,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6810","nonce":3562679765},{"type":"v1","addr":"172.21.15.22:6811","nonce":3562679765}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6812","nonce":3562679765},{"type":"v1","addr":"172.21.15.22:6813","nonce":3562679765}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6816","nonce":3562679765},{"type":"v1","addr":"172.21.15.22:6817","nonce":3562679765}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6814","nonce":3562679765},{"type":"v1","addr":"172.21.15.22:6815","nonce":3562679765}]},"public_addr":"172.21.15.22:6811/3562679765","cluster_addr":"172.21.15.22:6813/3562679765","heartbeat_back_addr":"172.21.15.22:6817/3562679765","heartbeat_front_addr":"172.21.15.22:6815/3562679765","state":["exists","up"]},{"osd":2,"uuid":"1c27d056-2218-47dd-8f13-bc1c13888483","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6818","nonce":695486895},{"type":"v1","addr":"172.21.15.22:6819","nonce":695486895}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6820","nonce":695486895},{"type":"v1","addr":"172.21.15.22:6821","nonce":695486895}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6824","nonce":695486895},{"type":"v1","addr":"172.21.15.22:6825","nonce":695486895}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6822","nonce":695486895},{"type":"v1","addr":"172.21.15.22:6823","nonce":695486895}]},"public_addr":"172.21.15.22:6819/695486895","cluster_addr":"172.21.15.22:6821/695486895","heartbeat_back_addr":"172.21.15.22:6825/695486895","heartbeat_front_addr":"172.21.15.22:6823/695486895","state":["exists","up"]},{"osd":3,"uuid":"8668a232-df4a-46ba-b97d-3ed2f6ddeab2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6826","nonce":1394296125},{"type":"v1","addr":"172.21.15.22:6827","nonce":1394296125}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6828","nonce":1394296125},{"type":"v1","addr":"172.21.15.22:6829","nonce":1394296125}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6832","nonce":1394296125},{"type":"v1","addr":"172.21.15.22:6833","nonce":1394296125}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6830","nonce":1394296125},{"type":"v1","addr":"172.21.15.22:6831","nonce":1394296125}]},"public_addr":"172.21.15.22:6827/1394296125","cluster_addr":"172.21.15.22:6829/1394296125","heartbeat_back_addr":"172.21.15.22:6833/1394296125","heartbeat_front_addr":"172.21.15.22:6831/1394296125","state":["exists","up"]},{"osd":4,"uuid":"44dd8dcc-dbf7-41f7-9ec2-e66922e6c521","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":54,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2736095169},{"type":"v1","addr":"172.21.15.64:6801","nonce":2736095169}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6802","nonce":2736095169},{"type":"v1","addr":"172.21.15.64:6803","nonce":2736095169}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6806","nonce":2736095169},{"type":"v1","addr":"172.21.15.64:6807","nonce":2736095169}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":2736095169},{"type":"v1","addr":"172.21.15.64:6805","nonce":2736095169}]},"public_addr":"172.21.15.64:6801/2736095169","cluster_addr":"172.21.15.64:6803/2736095169","heartbeat_back_addr":"172.21.15.64:6807/2736095169","heartbeat_front_addr":"172.21.15.64:6805/2736095169","state":["exists","up"]},{"osd":5,"uuid":"a920292e-6301-4b67-b2a1-30f4788f8fd3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":2673197529},{"type":"v1","addr":"172.21.15.64:6809","nonce":2673197529}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2673197529},{"type":"v1","addr":"172.21.15.64:6811","nonce":2673197529}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2673197529},{"type":"v1","addr":"172.21.15.64:6815","nonce":2673197529}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6812","nonce":2673197529},{"type":"v1","addr":"172.21.15.64:6813","nonce":2673197529}]},"public_addr":"172.21.15.64:6809/2673197529","cluster_addr":"172.21.15.64:6811/2673197529","heartbeat_back_addr":"172.21.15.64:6815/2673197529","heartbeat_front_addr":"172.21.15.64:6813/2673197529","state":["exists","up"]},{"osd":6,"uuid":"ed01fe25-9b7d-48b9-b054-e054280e5826","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1977044182},{"type":"v1","addr":"172.21.15.64:6817","nonce":1977044182}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1977044182},{"type":"v1","addr":"172.21.15.64:6819","nonce":1977044182}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1977044182},{"type":"v1","addr":"172.21.15.64:6823","nonce":1977044182}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1977044182},{"type":"v1","addr":"172.21.15.64:6821","nonce":1977044182}]},"public_addr":"172.21.15.64:6817/1977044182","cluster_addr":"172.21.15.64:6819/1977044182","heartbeat_back_addr":"172.21.15.64:6823/1977044182","heartbeat_front_addr":"172.21.15.64:6821/1977044182","state":["exists","up"]},{"osd":7,"uuid":"a9433be2-fcc7-46f5-91ee-13fbe71bf2f7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":4274465032},{"type":"v1","addr":"172.21.15.64:6825","nonce":4274465032}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":4274465032},{"type":"v1","addr":"172.21.15.64:6827","nonce":4274465032}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":4274465032},{"type":"v1","addr":"172.21.15.64:6831","nonce":4274465032}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":4274465032},{"type":"v1","addr":"172.21.15.64:6829","nonce":4274465032}]},"public_addr":"172.21.15.64:6825/4274465032","cluster_addr":"172.21.15.64:6827/4274465032","heartbeat_back_addr":"172.21.15.64:6831/4274465032","heartbeat_front_addr":"172.21.15.64:6829/4274465032","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:24:15.825542+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:24:40.497779+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:25:04.918091+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:25:30.663725+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:25:53.521949+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:26:17.565302+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:26:40.876016+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:27:04.436407+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.3","mappings":[{"from":7,"to":4}]},{"pgid":"3.a","mappings":[{"from":4,"to":3}]},{"pgid":"3.b","mappings":[{"from":0,"to":2}]},{"pgid":"3.17","mappings":[{"from":0,"to":2}]},{"pgid":"3.22","mappings":[{"from":0,"to":1}]},{"pgid":"3.24","mappings":[{"from":4,"to":2}]},{"pgid":"3.37","mappings":[{"from":5,"to":1}]}],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.22:6801/2821000087":"2024-09-22T09:22:50.910682+0000","172.21.15.22:6800/2821000087":"2024-09-22T09:22:50.910682+0000","172.21.15.22:0/4200462415":"2024-09-22T09:22:50.910682+0000","172.21.15.22:0/2669480478":"2024-09-22T09:22:50.910682+0000","172.21.15.22:0/1728251114":"2024-09-22T09:22:26.473640+0000","172.21.15.22:0/3766882712":"2024-09-22T09:22:50.910682+0000","172.21.15.22:0/1467628521":"2024-09-22T09:22:26.473640+0000","172.21.15.22:0/1022854075":"2024-09-22T09:22:26.473640+0000","172.21.15.22:6801/2983422368":"2024-09-22T09:22:26.473640+0000","172.21.15.22:6800/2983422368":"2024-09-22T09:22:26.473640+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-21T09:27:56.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:56 smithi064 ceph-mon[31124]: pgmap v201: 97 pgs: 1 peering, 96 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:27:56.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:56 smithi064 ceph-mon[31124]: from='client.? 172.21.15.22:0/308218315' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T09:27:56.442 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:56 smithi022 ceph-mon[39525]: pgmap v201: 97 pgs: 1 peering, 96 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:27:56.443 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:56 smithi022 ceph-mon[39525]: from='client.? 172.21.15.22:0/308218315' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T09:27:56.443 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:56 smithi022 ceph-mon[27763]: pgmap v201: 97 pgs: 1 peering, 96 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:27:56.443 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:56 smithi022 ceph-mon[27763]: from='client.? 172.21.15.22:0/308218315' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T09:27:56.473 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-09-21T09:27:56.473 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid d65e801e-77fa-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T09:27:56.731 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:27:58.219 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:27:58 smithi022 ceph-mon[39525]: pgmap v202: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:27:58.219 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:27:58 smithi022 ceph-mon[27763]: pgmap v202: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:27:58.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:27:58 smithi064 ceph-mon[31124]: pgmap v202: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:27:58.621 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-09-21T09:27:58.622 INFO:teuthology.orchestra.run.smithi022.stderr:dumped all 2024-09-21T09:27:59.405 INFO:teuthology.orchestra.run.smithi022.stdout:{"pg_ready":true,"pg_map":{"version":202,"stamp":"2024-09-21T09:27:56.970110+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592706,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":146,"num_write_kb":1641,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":111,"ondisk_log_size":111,"up":291,"acting":291,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":287,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2385296,"kb_used_data":6428,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747575920,"statfs":{"total":767960285184,"available":765517742080,"internally_reserved":0,"allocated":6582272,"data_stored":3785626,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.001934"},"pg_stats":[{"pgid":"3.3f","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.051004+0000","last_change":"2024-09-21T09:27:35.635695+0000","last_active":"2024-09-21T09:27:51.051004+0000","last_peered":"2024-09-21T09:27:51.051004+0000","last_clean":"2024-09-21T09:27:51.051004+0000","last_became_active":"2024-09-21T09:27:35.635255+0000","last_became_peered":"2024-09-21T09:27:35.635255+0000","last_unstale":"2024-09-21T09:27:51.051004+0000","last_undegraded":"2024-09-21T09:27:51.051004+0000","last_fullsized":"2024-09-21T09:27:51.051004+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:15:50.156743+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3e","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.043647+0000","last_change":"2024-09-21T09:27:35.636115+0000","last_active":"2024-09-21T09:27:51.043647+0000","last_peered":"2024-09-21T09:27:51.043647+0000","last_clean":"2024-09-21T09:27:51.043647+0000","last_became_active":"2024-09-21T09:27:35.635778+0000","last_became_peered":"2024-09-21T09:27:35.635778+0000","last_unstale":"2024-09-21T09:27:51.043647+0000","last_undegraded":"2024-09-21T09:27:51.043647+0000","last_fullsized":"2024-09-21T09:27:51.043647+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:36:35.607057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3d","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.050003+0000","last_change":"2024-09-21T09:27:35.640689+0000","last_active":"2024-09-21T09:27:51.050003+0000","last_peered":"2024-09-21T09:27:51.050003+0000","last_clean":"2024-09-21T09:27:51.050003+0000","last_became_active":"2024-09-21T09:27:35.640415+0000","last_became_peered":"2024-09-21T09:27:35.640415+0000","last_unstale":"2024-09-21T09:27:51.050003+0000","last_undegraded":"2024-09-21T09:27:51.050003+0000","last_fullsized":"2024-09-21T09:27:51.050003+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:12:22.314199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3c","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.045431+0000","last_change":"2024-09-21T09:27:35.644536+0000","last_active":"2024-09-21T09:27:51.045431+0000","last_peered":"2024-09-21T09:27:51.045431+0000","last_clean":"2024-09-21T09:27:51.045431+0000","last_became_active":"2024-09-21T09:27:35.644246+0000","last_became_peered":"2024-09-21T09:27:35.644246+0000","last_unstale":"2024-09-21T09:27:51.045431+0000","last_undegraded":"2024-09-21T09:27:51.045431+0000","last_fullsized":"2024-09-21T09:27:51.045431+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:31:37.886172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3b","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.050078+0000","last_change":"2024-09-21T09:27:35.642779+0000","last_active":"2024-09-21T09:27:51.050078+0000","last_peered":"2024-09-21T09:27:51.050078+0000","last_clean":"2024-09-21T09:27:51.050078+0000","last_became_active":"2024-09-21T09:27:35.642487+0000","last_became_peered":"2024-09-21T09:27:35.642487+0000","last_unstale":"2024-09-21T09:27:51.050078+0000","last_undegraded":"2024-09-21T09:27:51.050078+0000","last_fullsized":"2024-09-21T09:27:51.050078+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:17:29.631740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3a","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.046996+0000","last_change":"2024-09-21T09:27:35.651016+0000","last_active":"2024-09-21T09:27:51.046996+0000","last_peered":"2024-09-21T09:27:51.046996+0000","last_clean":"2024-09-21T09:27:51.046996+0000","last_became_active":"2024-09-21T09:27:35.650742+0000","last_became_peered":"2024-09-21T09:27:35.650742+0000","last_unstale":"2024-09-21T09:27:51.046996+0000","last_undegraded":"2024-09-21T09:27:51.046996+0000","last_fullsized":"2024-09-21T09:27:51.046996+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:11:11.493990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.39","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.050829+0000","last_change":"2024-09-21T09:27:35.647235+0000","last_active":"2024-09-21T09:27:51.050829+0000","last_peered":"2024-09-21T09:27:51.050829+0000","last_clean":"2024-09-21T09:27:51.050829+0000","last_became_active":"2024-09-21T09:27:35.646890+0000","last_became_peered":"2024-09-21T09:27:35.646890+0000","last_unstale":"2024-09-21T09:27:51.050829+0000","last_undegraded":"2024-09-21T09:27:51.050829+0000","last_fullsized":"2024-09-21T09:27:51.050829+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:21:45.995387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.38","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.047996+0000","last_change":"2024-09-21T09:27:35.638161+0000","last_active":"2024-09-21T09:27:51.047996+0000","last_peered":"2024-09-21T09:27:51.047996+0000","last_clean":"2024-09-21T09:27:51.047996+0000","last_became_active":"2024-09-21T09:27:35.637751+0000","last_became_peered":"2024-09-21T09:27:35.637751+0000","last_unstale":"2024-09-21T09:27:51.047996+0000","last_undegraded":"2024-09-21T09:27:51.047996+0000","last_fullsized":"2024-09-21T09:27:51.047996+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:07:40.751242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.37","version":"0'0","reported_seq":23,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-21T09:27:52.079760+0000","last_change":"2024-09-21T09:27:52.079760+0000","last_active":"2024-09-21T09:27:52.079760+0000","last_peered":"2024-09-21T09:27:52.079760+0000","last_clean":"2024-09-21T09:27:52.079760+0000","last_became_active":"2024-09-21T09:27:52.079352+0000","last_became_peered":"2024-09-21T09:27:52.079352+0000","last_unstale":"2024-09-21T09:27:52.079760+0000","last_undegraded":"2024-09-21T09:27:52.079760+0000","last_fullsized":"2024-09-21T09:27:52.079760+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:57:57.194876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.36","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.045365+0000","last_change":"2024-09-21T09:27:35.636094+0000","last_active":"2024-09-21T09:27:51.045365+0000","last_peered":"2024-09-21T09:27:51.045365+0000","last_clean":"2024-09-21T09:27:51.045365+0000","last_became_active":"2024-09-21T09:27:35.635770+0000","last_became_peered":"2024-09-21T09:27:35.635770+0000","last_unstale":"2024-09-21T09:27:51.045365+0000","last_undegraded":"2024-09-21T09:27:51.045365+0000","last_fullsized":"2024-09-21T09:27:51.045365+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:41:52.909549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.35","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.045488+0000","last_change":"2024-09-21T09:27:35.640571+0000","last_active":"2024-09-21T09:27:51.045488+0000","last_peered":"2024-09-21T09:27:51.045488+0000","last_clean":"2024-09-21T09:27:51.045488+0000","last_became_active":"2024-09-21T09:27:35.640253+0000","last_became_peered":"2024-09-21T09:27:35.640253+0000","last_unstale":"2024-09-21T09:27:51.045488+0000","last_undegraded":"2024-09-21T09:27:51.045488+0000","last_fullsized":"2024-09-21T09:27:51.045488+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:40:44.721446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.34","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.048082+0000","last_change":"2024-09-21T09:27:35.639471+0000","last_active":"2024-09-21T09:27:51.048082+0000","last_peered":"2024-09-21T09:27:51.048082+0000","last_clean":"2024-09-21T09:27:51.048082+0000","last_became_active":"2024-09-21T09:27:35.639056+0000","last_became_peered":"2024-09-21T09:27:35.639056+0000","last_unstale":"2024-09-21T09:27:51.048082+0000","last_undegraded":"2024-09-21T09:27:51.048082+0000","last_fullsized":"2024-09-21T09:27:51.048082+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:16:15.724781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.33","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.048493+0000","last_change":"2024-09-21T09:27:35.637331+0000","last_active":"2024-09-21T09:27:51.048493+0000","last_peered":"2024-09-21T09:27:51.048493+0000","last_clean":"2024-09-21T09:27:51.048493+0000","last_became_active":"2024-09-21T09:27:35.637068+0000","last_became_peered":"2024-09-21T09:27:35.637068+0000","last_unstale":"2024-09-21T09:27:51.048493+0000","last_undegraded":"2024-09-21T09:27:51.048493+0000","last_fullsized":"2024-09-21T09:27:51.048493+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:24:52.176104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.32","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.049091+0000","last_change":"2024-09-21T09:27:35.645565+0000","last_active":"2024-09-21T09:27:51.049091+0000","last_peered":"2024-09-21T09:27:51.049091+0000","last_clean":"2024-09-21T09:27:51.049091+0000","last_became_active":"2024-09-21T09:27:35.644114+0000","last_became_peered":"2024-09-21T09:27:35.644114+0000","last_unstale":"2024-09-21T09:27:51.049091+0000","last_undegraded":"2024-09-21T09:27:51.049091+0000","last_fullsized":"2024-09-21T09:27:51.049091+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:27:18.008605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.31","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.042816+0000","last_change":"2024-09-21T09:27:35.634448+0000","last_active":"2024-09-21T09:27:51.042816+0000","last_peered":"2024-09-21T09:27:51.042816+0000","last_clean":"2024-09-21T09:27:51.042816+0000","last_became_active":"2024-09-21T09:27:35.634089+0000","last_became_peered":"2024-09-21T09:27:35.634089+0000","last_unstale":"2024-09-21T09:27:51.042816+0000","last_undegraded":"2024-09-21T09:27:51.042816+0000","last_fullsized":"2024-09-21T09:27:51.042816+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:14:14.726492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.30","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.046109+0000","last_change":"2024-09-21T09:27:35.651997+0000","last_active":"2024-09-21T09:27:51.046109+0000","last_peered":"2024-09-21T09:27:51.046109+0000","last_clean":"2024-09-21T09:27:51.046109+0000","last_became_active":"2024-09-21T09:27:35.651698+0000","last_became_peered":"2024-09-21T09:27:35.651698+0000","last_unstale":"2024-09-21T09:27:51.046109+0000","last_undegraded":"2024-09-21T09:27:51.046109+0000","last_fullsized":"2024-09-21T09:27:51.046109+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:43:39.574788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2f","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.046369+0000","last_change":"2024-09-21T09:27:35.646363+0000","last_active":"2024-09-21T09:27:51.046369+0000","last_peered":"2024-09-21T09:27:51.046369+0000","last_clean":"2024-09-21T09:27:51.046369+0000","last_became_active":"2024-09-21T09:27:35.646084+0000","last_became_peered":"2024-09-21T09:27:35.646084+0000","last_unstale":"2024-09-21T09:27:51.046369+0000","last_undegraded":"2024-09-21T09:27:51.046369+0000","last_fullsized":"2024-09-21T09:27:51.046369+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:35:38.711926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2e","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.042536+0000","last_change":"2024-09-21T09:27:35.636697+0000","last_active":"2024-09-21T09:27:51.042536+0000","last_peered":"2024-09-21T09:27:51.042536+0000","last_clean":"2024-09-21T09:27:51.042536+0000","last_became_active":"2024-09-21T09:27:35.636438+0000","last_became_peered":"2024-09-21T09:27:35.636438+0000","last_unstale":"2024-09-21T09:27:51.042536+0000","last_undegraded":"2024-09-21T09:27:51.042536+0000","last_fullsized":"2024-09-21T09:27:51.042536+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:55:44.287544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,7],"acting":[2,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2d","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.050587+0000","last_change":"2024-09-21T09:27:35.636110+0000","last_active":"2024-09-21T09:27:51.050587+0000","last_peered":"2024-09-21T09:27:51.050587+0000","last_clean":"2024-09-21T09:27:51.050587+0000","last_became_active":"2024-09-21T09:27:35.635807+0000","last_became_peered":"2024-09-21T09:27:35.635807+0000","last_unstale":"2024-09-21T09:27:51.050587+0000","last_undegraded":"2024-09-21T09:27:51.050587+0000","last_fullsized":"2024-09-21T09:27:51.050587+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:22:43.942811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2c","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.045967+0000","last_change":"2024-09-21T09:27:35.650351+0000","last_active":"2024-09-21T09:27:51.045967+0000","last_peered":"2024-09-21T09:27:51.045967+0000","last_clean":"2024-09-21T09:27:51.045967+0000","last_became_active":"2024-09-21T09:27:35.649905+0000","last_became_peered":"2024-09-21T09:27:35.649905+0000","last_unstale":"2024-09-21T09:27:51.045967+0000","last_undegraded":"2024-09-21T09:27:51.045967+0000","last_fullsized":"2024-09-21T09:27:51.045967+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:12:31.390666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2b","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.049389+0000","last_change":"2024-09-21T09:27:35.643345+0000","last_active":"2024-09-21T09:27:51.049389+0000","last_peered":"2024-09-21T09:27:51.049389+0000","last_clean":"2024-09-21T09:27:51.049389+0000","last_became_active":"2024-09-21T09:27:35.642408+0000","last_became_peered":"2024-09-21T09:27:35.642408+0000","last_unstale":"2024-09-21T09:27:51.049389+0000","last_undegraded":"2024-09-21T09:27:51.049389+0000","last_fullsized":"2024-09-21T09:27:51.049389+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:34:59.698334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2a","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.047079+0000","last_change":"2024-09-21T09:27:35.645032+0000","last_active":"2024-09-21T09:27:51.047079+0000","last_peered":"2024-09-21T09:27:51.047079+0000","last_clean":"2024-09-21T09:27:51.047079+0000","last_became_active":"2024-09-21T09:27:35.644322+0000","last_became_peered":"2024-09-21T09:27:35.644322+0000","last_unstale":"2024-09-21T09:27:51.047079+0000","last_undegraded":"2024-09-21T09:27:51.047079+0000","last_fullsized":"2024-09-21T09:27:51.047079+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:10:14.511772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.29","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.048171+0000","last_change":"2024-09-21T09:27:35.638305+0000","last_active":"2024-09-21T09:27:51.048171+0000","last_peered":"2024-09-21T09:27:51.048171+0000","last_clean":"2024-09-21T09:27:51.048171+0000","last_became_active":"2024-09-21T09:27:35.638036+0000","last_became_peered":"2024-09-21T09:27:35.638036+0000","last_unstale":"2024-09-21T09:27:51.048171+0000","last_undegraded":"2024-09-21T09:27:51.048171+0000","last_fullsized":"2024-09-21T09:27:51.048171+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:38:54.627840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.28","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.046739+0000","last_change":"2024-09-21T09:27:35.637277+0000","last_active":"2024-09-21T09:27:51.046739+0000","last_peered":"2024-09-21T09:27:51.046739+0000","last_clean":"2024-09-21T09:27:51.046739+0000","last_became_active":"2024-09-21T09:27:35.636541+0000","last_became_peered":"2024-09-21T09:27:35.636541+0000","last_unstale":"2024-09-21T09:27:51.046739+0000","last_undegraded":"2024-09-21T09:27:51.046739+0000","last_fullsized":"2024-09-21T09:27:51.046739+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:45:47.090531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.27","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.049341+0000","last_change":"2024-09-21T09:27:35.638946+0000","last_active":"2024-09-21T09:27:51.049341+0000","last_peered":"2024-09-21T09:27:51.049341+0000","last_clean":"2024-09-21T09:27:51.049341+0000","last_became_active":"2024-09-21T09:27:35.638663+0000","last_became_peered":"2024-09-21T09:27:35.638663+0000","last_unstale":"2024-09-21T09:27:51.049341+0000","last_undegraded":"2024-09-21T09:27:51.049341+0000","last_fullsized":"2024-09-21T09:27:51.049341+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:49:57.591531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.26","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.045448+0000","last_change":"2024-09-21T09:27:35.638679+0000","last_active":"2024-09-21T09:27:51.045448+0000","last_peered":"2024-09-21T09:27:51.045448+0000","last_clean":"2024-09-21T09:27:51.045448+0000","last_became_active":"2024-09-21T09:27:35.638383+0000","last_became_peered":"2024-09-21T09:27:35.638383+0000","last_unstale":"2024-09-21T09:27:51.045448+0000","last_undegraded":"2024-09-21T09:27:51.045448+0000","last_fullsized":"2024-09-21T09:27:51.045448+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:16:11.663503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,4],"acting":[6,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.25","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.043664+0000","last_change":"2024-09-21T09:27:35.632894+0000","last_active":"2024-09-21T09:27:51.043664+0000","last_peered":"2024-09-21T09:27:51.043664+0000","last_clean":"2024-09-21T09:27:51.043664+0000","last_became_active":"2024-09-21T09:27:35.632569+0000","last_became_peered":"2024-09-21T09:27:35.632569+0000","last_unstale":"2024-09-21T09:27:51.043664+0000","last_undegraded":"2024-09-21T09:27:51.043664+0000","last_fullsized":"2024-09-21T09:27:51.043664+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:25:22.833535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.24","version":"0'0","reported_seq":23,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-21T09:27:52.079786+0000","last_change":"2024-09-21T09:27:52.079786+0000","last_active":"2024-09-21T09:27:52.079786+0000","last_peered":"2024-09-21T09:27:52.079786+0000","last_clean":"2024-09-21T09:27:52.079786+0000","last_became_active":"2024-09-21T09:27:52.079395+0000","last_became_peered":"2024-09-21T09:27:52.079395+0000","last_unstale":"2024-09-21T09:27:52.079786+0000","last_undegraded":"2024-09-21T09:27:52.079786+0000","last_fullsized":"2024-09-21T09:27:52.079786+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:17:07.388900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.23","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.049581+0000","last_change":"2024-09-21T09:27:35.634785+0000","last_active":"2024-09-21T09:27:51.049581+0000","last_peered":"2024-09-21T09:27:51.049581+0000","last_clean":"2024-09-21T09:27:51.049581+0000","last_became_active":"2024-09-21T09:27:35.634065+0000","last_became_peered":"2024-09-21T09:27:35.634065+0000","last_unstale":"2024-09-21T09:27:51.049581+0000","last_undegraded":"2024-09-21T09:27:51.049581+0000","last_fullsized":"2024-09-21T09:27:51.049581+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:54:21.482132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.22","version":"0'0","reported_seq":23,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-21T09:27:52.049243+0000","last_change":"2024-09-21T09:27:52.049243+0000","last_active":"2024-09-21T09:27:52.049243+0000","last_peered":"2024-09-21T09:27:52.049243+0000","last_clean":"2024-09-21T09:27:52.049243+0000","last_became_active":"2024-09-21T09:27:52.048885+0000","last_became_peered":"2024-09-21T09:27:52.048885+0000","last_unstale":"2024-09-21T09:27:52.049243+0000","last_undegraded":"2024-09-21T09:27:52.049243+0000","last_fullsized":"2024-09-21T09:27:52.049243+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:44:22.262414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.21","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.042354+0000","last_change":"2024-09-21T09:27:35.636222+0000","last_active":"2024-09-21T09:27:51.042354+0000","last_peered":"2024-09-21T09:27:51.042354+0000","last_clean":"2024-09-21T09:27:51.042354+0000","last_became_active":"2024-09-21T09:27:35.635935+0000","last_became_peered":"2024-09-21T09:27:35.635935+0000","last_unstale":"2024-09-21T09:27:51.042354+0000","last_undegraded":"2024-09-21T09:27:51.042354+0000","last_fullsized":"2024-09-21T09:27:51.042354+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:05:50.402224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.20","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.044885+0000","last_change":"2024-09-21T09:27:35.641740+0000","last_active":"2024-09-21T09:27:51.044885+0000","last_peered":"2024-09-21T09:27:51.044885+0000","last_clean":"2024-09-21T09:27:51.044885+0000","last_became_active":"2024-09-21T09:27:35.641409+0000","last_became_peered":"2024-09-21T09:27:35.641409+0000","last_unstale":"2024-09-21T09:27:51.044885+0000","last_undegraded":"2024-09-21T09:27:51.044885+0000","last_fullsized":"2024-09-21T09:27:51.044885+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:59:30.487730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1a","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.047761+0000","last_change":"2024-09-21T09:27:35.629525+0000","last_active":"2024-09-21T09:27:51.047761+0000","last_peered":"2024-09-21T09:27:51.047761+0000","last_clean":"2024-09-21T09:27:51.047761+0000","last_became_active":"2024-09-21T09:27:35.629167+0000","last_became_peered":"2024-09-21T09:27:35.629167+0000","last_unstale":"2024-09-21T09:27:51.047761+0000","last_undegraded":"2024-09-21T09:27:51.047761+0000","last_fullsized":"2024-09-21T09:27:51.047761+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:06:24.323825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1b","version":"57'1","reported_seq":17,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.048995+0000","last_change":"2024-09-21T09:27:32.265235+0000","last_active":"2024-09-21T09:27:51.048995+0000","last_peered":"2024-09-21T09:27:51.048995+0000","last_clean":"2024-09-21T09:27:51.048995+0000","last_became_active":"2024-09-21T09:27:32.264963+0000","last_became_peered":"2024-09-21T09:27:32.264963+0000","last_unstale":"2024-09-21T09:27:51.048995+0000","last_undegraded":"2024-09-21T09:27:51.048995+0000","last_fullsized":"2024-09-21T09:27:51.048995+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:09:03.927341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.049658+0000","last_change":"2024-09-21T09:27:35.642154+0000","last_active":"2024-09-21T09:27:51.049658+0000","last_peered":"2024-09-21T09:27:51.049658+0000","last_clean":"2024-09-21T09:27:51.049658+0000","last_became_active":"2024-09-21T09:27:35.641801+0000","last_became_peered":"2024-09-21T09:27:35.641801+0000","last_unstale":"2024-09-21T09:27:51.049658+0000","last_undegraded":"2024-09-21T09:27:51.049658+0000","last_fullsized":"2024-09-21T09:27:51.049658+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:50:45.075506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1a","version":"57'1","reported_seq":17,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.047707+0000","last_change":"2024-09-21T09:27:32.269358+0000","last_active":"2024-09-21T09:27:51.047707+0000","last_peered":"2024-09-21T09:27:51.047707+0000","last_clean":"2024-09-21T09:27:51.047707+0000","last_became_active":"2024-09-21T09:27:32.269026+0000","last_became_peered":"2024-09-21T09:27:32.269026+0000","last_unstale":"2024-09-21T09:27:51.047707+0000","last_undegraded":"2024-09-21T09:27:51.047707+0000","last_fullsized":"2024-09-21T09:27:51.047707+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:59:17.916370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.19","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.048323+0000","last_change":"2024-09-21T09:27:32.264601+0000","last_active":"2024-09-21T09:27:51.048323+0000","last_peered":"2024-09-21T09:27:51.048323+0000","last_clean":"2024-09-21T09:27:51.048323+0000","last_became_active":"2024-09-21T09:27:32.264381+0000","last_became_peered":"2024-09-21T09:27:32.264381+0000","last_unstale":"2024-09-21T09:27:51.048323+0000","last_undegraded":"2024-09-21T09:27:51.048323+0000","last_fullsized":"2024-09-21T09:27:51.048323+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:35:57.463156+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.048044+0000","last_change":"2024-09-21T09:27:35.634475+0000","last_active":"2024-09-21T09:27:51.048044+0000","last_peered":"2024-09-21T09:27:51.048044+0000","last_clean":"2024-09-21T09:27:51.048044+0000","last_became_active":"2024-09-21T09:27:35.634224+0000","last_became_peered":"2024-09-21T09:27:35.634224+0000","last_unstale":"2024-09-21T09:27:51.048044+0000","last_undegraded":"2024-09-21T09:27:51.048044+0000","last_fullsized":"2024-09-21T09:27:51.048044+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:38:21.241590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.18","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.046834+0000","last_change":"2024-09-21T09:27:32.267805+0000","last_active":"2024-09-21T09:27:51.046834+0000","last_peered":"2024-09-21T09:27:51.046834+0000","last_clean":"2024-09-21T09:27:51.046834+0000","last_became_active":"2024-09-21T09:27:32.267484+0000","last_became_peered":"2024-09-21T09:27:32.267484+0000","last_unstale":"2024-09-21T09:27:51.046834+0000","last_undegraded":"2024-09-21T09:27:51.046834+0000","last_fullsized":"2024-09-21T09:27:51.046834+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:00:43.105362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,7],"acting":[5,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.042506+0000","last_change":"2024-09-21T09:27:35.631350+0000","last_active":"2024-09-21T09:27:51.042506+0000","last_peered":"2024-09-21T09:27:51.042506+0000","last_clean":"2024-09-21T09:27:51.042506+0000","last_became_active":"2024-09-21T09:27:35.630723+0000","last_became_peered":"2024-09-21T09:27:35.630723+0000","last_unstale":"2024-09-21T09:27:51.042506+0000","last_undegraded":"2024-09-21T09:27:51.042506+0000","last_fullsized":"2024-09-21T09:27:51.042506+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:46:32.135037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.17","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.045755+0000","last_change":"2024-09-21T09:27:32.266307+0000","last_active":"2024-09-21T09:27:51.045755+0000","last_peered":"2024-09-21T09:27:51.045755+0000","last_clean":"2024-09-21T09:27:51.045755+0000","last_became_active":"2024-09-21T09:27:32.265995+0000","last_became_peered":"2024-09-21T09:27:32.265995+0000","last_unstale":"2024-09-21T09:27:51.045755+0000","last_undegraded":"2024-09-21T09:27:51.045755+0000","last_fullsized":"2024-09-21T09:27:51.045755+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:24:28.036534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.045841+0000","last_change":"2024-09-21T09:27:35.647952+0000","last_active":"2024-09-21T09:27:51.045841+0000","last_peered":"2024-09-21T09:27:51.045841+0000","last_clean":"2024-09-21T09:27:51.045841+0000","last_became_active":"2024-09-21T09:27:35.647649+0000","last_became_peered":"2024-09-21T09:27:35.647649+0000","last_unstale":"2024-09-21T09:27:51.045841+0000","last_undegraded":"2024-09-21T09:27:51.045841+0000","last_fullsized":"2024-09-21T09:27:51.045841+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:12:29.850835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.16","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.046149+0000","last_change":"2024-09-21T09:27:32.269137+0000","last_active":"2024-09-21T09:27:51.046149+0000","last_peered":"2024-09-21T09:27:51.046149+0000","last_clean":"2024-09-21T09:27:51.046149+0000","last_became_active":"2024-09-21T09:27:32.268832+0000","last_became_peered":"2024-09-21T09:27:32.268832+0000","last_unstale":"2024-09-21T09:27:51.046149+0000","last_undegraded":"2024-09-21T09:27:51.046149+0000","last_fullsized":"2024-09-21T09:27:51.046149+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:51:58.668390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":10,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-21T09:27:52.050121+0000","last_change":"2024-09-21T09:27:52.050121+0000","last_active":"2024-09-21T09:27:52.050121+0000","last_peered":"2024-09-21T09:27:52.050121+0000","last_clean":"2024-09-21T09:27:52.050121+0000","last_became_active":"2024-09-21T09:27:52.049549+0000","last_became_peered":"2024-09-21T09:27:52.049549+0000","last_unstale":"2024-09-21T09:27:52.050121+0000","last_undegraded":"2024-09-21T09:27:52.050121+0000","last_fullsized":"2024-09-21T09:27:52.050121+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:25:27.303500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.15","version":"57'3","reported_seq":19,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.043540+0000","last_change":"2024-09-21T09:27:32.270145+0000","last_active":"2024-09-21T09:27:51.043540+0000","last_peered":"2024-09-21T09:27:51.043540+0000","last_clean":"2024-09-21T09:27:51.043540+0000","last_became_active":"2024-09-21T09:27:32.269808+0000","last_became_peered":"2024-09-21T09:27:32.269808+0000","last_unstale":"2024-09-21T09:27:51.043540+0000","last_undegraded":"2024-09-21T09:27:51.043540+0000","last_fullsized":"2024-09-21T09:27:51.043540+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:35:53.397343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.048177+0000","last_change":"2024-09-21T09:27:35.640314+0000","last_active":"2024-09-21T09:27:51.048177+0000","last_peered":"2024-09-21T09:27:51.048177+0000","last_clean":"2024-09-21T09:27:51.048177+0000","last_became_active":"2024-09-21T09:27:35.639976+0000","last_became_peered":"2024-09-21T09:27:35.639976+0000","last_unstale":"2024-09-21T09:27:51.048177+0000","last_undegraded":"2024-09-21T09:27:51.048177+0000","last_fullsized":"2024-09-21T09:27:51.048177+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:55:21.229874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.14","version":"57'6","reported_seq":22,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.046070+0000","last_change":"2024-09-21T09:27:32.266499+0000","last_active":"2024-09-21T09:27:51.046070+0000","last_peered":"2024-09-21T09:27:51.046070+0000","last_clean":"2024-09-21T09:27:51.046070+0000","last_became_active":"2024-09-21T09:27:32.265712+0000","last_became_peered":"2024-09-21T09:27:32.265712+0000","last_unstale":"2024-09-21T09:27:51.046070+0000","last_undegraded":"2024-09-21T09:27:51.046070+0000","last_fullsized":"2024-09-21T09:27:51.046070+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:19:17.014990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1552,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.050296+0000","last_change":"2024-09-21T09:27:35.641626+0000","last_active":"2024-09-21T09:27:51.050296+0000","last_peered":"2024-09-21T09:27:51.050296+0000","last_clean":"2024-09-21T09:27:51.050296+0000","last_became_active":"2024-09-21T09:27:35.641272+0000","last_became_peered":"2024-09-21T09:27:35.641272+0000","last_unstale":"2024-09-21T09:27:51.050296+0000","last_undegraded":"2024-09-21T09:27:51.050296+0000","last_fullsized":"2024-09-21T09:27:51.050296+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:15:05.108072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.13","version":"57'2","reported_seq":18,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.049744+0000","last_change":"2024-09-21T09:27:32.267412+0000","last_active":"2024-09-21T09:27:51.049744+0000","last_peered":"2024-09-21T09:27:51.049744+0000","last_clean":"2024-09-21T09:27:51.049744+0000","last_became_active":"2024-09-21T09:27:32.267108+0000","last_became_peered":"2024-09-21T09:27:32.267108+0000","last_unstale":"2024-09-21T09:27:51.049744+0000","last_undegraded":"2024-09-21T09:27:51.049744+0000","last_fullsized":"2024-09-21T09:27:51.049744+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:54:11.228702+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.049165+0000","last_change":"2024-09-21T09:27:35.641572+0000","last_active":"2024-09-21T09:27:51.049165+0000","last_peered":"2024-09-21T09:27:51.049165+0000","last_clean":"2024-09-21T09:27:51.049165+0000","last_became_active":"2024-09-21T09:27:35.641185+0000","last_became_peered":"2024-09-21T09:27:35.641185+0000","last_unstale":"2024-09-21T09:27:51.049165+0000","last_undegraded":"2024-09-21T09:27:51.049165+0000","last_fullsized":"2024-09-21T09:27:51.049165+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:52:30.993491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.12","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.048014+0000","last_change":"2024-09-21T09:27:32.267644+0000","last_active":"2024-09-21T09:27:51.048014+0000","last_peered":"2024-09-21T09:27:51.048014+0000","last_clean":"2024-09-21T09:27:51.048014+0000","last_became_active":"2024-09-21T09:27:32.267232+0000","last_became_peered":"2024-09-21T09:27:32.267232+0000","last_unstale":"2024-09-21T09:27:51.048014+0000","last_undegraded":"2024-09-21T09:27:51.048014+0000","last_fullsized":"2024-09-21T09:27:51.048014+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:14:19.578564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.050123+0000","last_change":"2024-09-21T09:27:35.644217+0000","last_active":"2024-09-21T09:27:51.050123+0000","last_peered":"2024-09-21T09:27:51.050123+0000","last_clean":"2024-09-21T09:27:51.050123+0000","last_became_active":"2024-09-21T09:27:35.643681+0000","last_became_peered":"2024-09-21T09:27:35.643681+0000","last_unstale":"2024-09-21T09:27:51.050123+0000","last_undegraded":"2024-09-21T09:27:51.050123+0000","last_fullsized":"2024-09-21T09:27:51.050123+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:07:58.418951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.11","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.045294+0000","last_change":"2024-09-21T09:27:32.270501+0000","last_active":"2024-09-21T09:27:51.045294+0000","last_peered":"2024-09-21T09:27:51.045294+0000","last_clean":"2024-09-21T09:27:51.045294+0000","last_became_active":"2024-09-21T09:27:32.269808+0000","last_became_peered":"2024-09-21T09:27:32.269808+0000","last_unstale":"2024-09-21T09:27:51.045294+0000","last_undegraded":"2024-09-21T09:27:51.045294+0000","last_fullsized":"2024-09-21T09:27:51.045294+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:28:03.705631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.10","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.044945+0000","last_change":"2024-09-21T09:27:35.641508+0000","last_active":"2024-09-21T09:27:51.044945+0000","last_peered":"2024-09-21T09:27:51.044945+0000","last_clean":"2024-09-21T09:27:51.044945+0000","last_became_active":"2024-09-21T09:27:35.640996+0000","last_became_peered":"2024-09-21T09:27:35.640996+0000","last_unstale":"2024-09-21T09:27:51.044945+0000","last_undegraded":"2024-09-21T09:27:51.044945+0000","last_fullsized":"2024-09-21T09:27:51.044945+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:02:24.674295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.10","version":"57'8","reported_seq":24,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.042476+0000","last_change":"2024-09-21T09:27:32.264620+0000","last_active":"2024-09-21T09:27:51.042476+0000","last_peered":"2024-09-21T09:27:51.042476+0000","last_clean":"2024-09-21T09:27:51.042476+0000","last_became_active":"2024-09-21T09:27:32.261324+0000","last_became_peered":"2024-09-21T09:27:32.261324+0000","last_unstale":"2024-09-21T09:27:51.042476+0000","last_undegraded":"2024-09-21T09:27:51.042476+0000","last_fullsized":"2024-09-21T09:27:51.042476+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:11:04.933873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.050795+0000","last_change":"2024-09-21T09:27:35.641576+0000","last_active":"2024-09-21T09:27:51.050795+0000","last_peered":"2024-09-21T09:27:51.050795+0000","last_clean":"2024-09-21T09:27:51.050795+0000","last_became_active":"2024-09-21T09:27:35.641261+0000","last_became_peered":"2024-09-21T09:27:35.641261+0000","last_unstale":"2024-09-21T09:27:51.050795+0000","last_undegraded":"2024-09-21T09:27:51.050795+0000","last_fullsized":"2024-09-21T09:27:51.050795+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:59:48.868788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.f","version":"57'1","reported_seq":17,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.049183+0000","last_change":"2024-09-21T09:27:32.270055+0000","last_active":"2024-09-21T09:27:51.049183+0000","last_peered":"2024-09-21T09:27:51.049183+0000","last_clean":"2024-09-21T09:27:51.049183+0000","last_became_active":"2024-09-21T09:27:32.269748+0000","last_became_peered":"2024-09-21T09:27:32.269748+0000","last_unstale":"2024-09-21T09:27:51.049183+0000","last_undegraded":"2024-09-21T09:27:51.049183+0000","last_fullsized":"2024-09-21T09:27:51.049183+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:50:54.853319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.050943+0000","last_change":"2024-09-21T09:27:35.641790+0000","last_active":"2024-09-21T09:27:51.050943+0000","last_peered":"2024-09-21T09:27:51.050943+0000","last_clean":"2024-09-21T09:27:51.050943+0000","last_became_active":"2024-09-21T09:27:35.641348+0000","last_became_peered":"2024-09-21T09:27:35.641348+0000","last_unstale":"2024-09-21T09:27:51.050943+0000","last_undegraded":"2024-09-21T09:27:51.050943+0000","last_fullsized":"2024-09-21T09:27:51.050943+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:25:10.635353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.e","version":"57'2","reported_seq":18,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.042989+0000","last_change":"2024-09-21T09:27:32.264276+0000","last_active":"2024-09-21T09:27:51.042989+0000","last_peered":"2024-09-21T09:27:51.042989+0000","last_clean":"2024-09-21T09:27:51.042989+0000","last_became_active":"2024-09-21T09:27:32.263984+0000","last_became_peered":"2024-09-21T09:27:32.263984+0000","last_unstale":"2024-09-21T09:27:51.042989+0000","last_undegraded":"2024-09-21T09:27:51.042989+0000","last_fullsized":"2024-09-21T09:27:51.042989+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:18:50.045457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.051063+0000","last_change":"2024-09-21T09:27:35.649353+0000","last_active":"2024-09-21T09:27:51.051063+0000","last_peered":"2024-09-21T09:27:51.051063+0000","last_clean":"2024-09-21T09:27:51.051063+0000","last_became_active":"2024-09-21T09:27:35.649033+0000","last_became_peered":"2024-09-21T09:27:35.649033+0000","last_unstale":"2024-09-21T09:27:51.051063+0000","last_undegraded":"2024-09-21T09:27:51.051063+0000","last_fullsized":"2024-09-21T09:27:51.051063+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:13:12.611019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.d","version":"57'1","reported_seq":17,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.043640+0000","last_change":"2024-09-21T09:27:32.260033+0000","last_active":"2024-09-21T09:27:51.043640+0000","last_peered":"2024-09-21T09:27:51.043640+0000","last_clean":"2024-09-21T09:27:51.043640+0000","last_became_active":"2024-09-21T09:27:32.259209+0000","last_became_peered":"2024-09-21T09:27:32.259209+0000","last_unstale":"2024-09-21T09:27:51.043640+0000","last_undegraded":"2024-09-21T09:27:51.043640+0000","last_fullsized":"2024-09-21T09:27:51.043640+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:16:09.835879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.045659+0000","last_change":"2024-09-21T09:27:35.639076+0000","last_active":"2024-09-21T09:27:51.045659+0000","last_peered":"2024-09-21T09:27:51.045659+0000","last_clean":"2024-09-21T09:27:51.045659+0000","last_became_active":"2024-09-21T09:27:35.638802+0000","last_became_peered":"2024-09-21T09:27:35.638802+0000","last_unstale":"2024-09-21T09:27:51.045659+0000","last_undegraded":"2024-09-21T09:27:51.045659+0000","last_fullsized":"2024-09-21T09:27:51.045659+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:36:18.524320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.c","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.042975+0000","last_change":"2024-09-21T09:27:32.270060+0000","last_active":"2024-09-21T09:27:51.042975+0000","last_peered":"2024-09-21T09:27:51.042975+0000","last_clean":"2024-09-21T09:27:51.042975+0000","last_became_active":"2024-09-21T09:27:32.269717+0000","last_became_peered":"2024-09-21T09:27:32.269717+0000","last_unstale":"2024-09-21T09:27:51.042975+0000","last_undegraded":"2024-09-21T09:27:51.042975+0000","last_fullsized":"2024-09-21T09:27:51.042975+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:59:27.725031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.050384+0000","last_change":"2024-09-21T09:27:35.640059+0000","last_active":"2024-09-21T09:27:51.050384+0000","last_peered":"2024-09-21T09:27:51.050384+0000","last_clean":"2024-09-21T09:27:51.050384+0000","last_became_active":"2024-09-21T09:27:35.639746+0000","last_became_peered":"2024-09-21T09:27:35.639746+0000","last_unstale":"2024-09-21T09:27:51.050384+0000","last_undegraded":"2024-09-21T09:27:51.050384+0000","last_fullsized":"2024-09-21T09:27:51.050384+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:33:15.402410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"21'72","reported_seq":25,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.050815+0000","last_change":"2024-09-21T09:27:08.460023+0000","last_active":"2024-09-21T09:27:51.050815+0000","last_peered":"2024-09-21T09:27:51.050815+0000","last_clean":"2024-09-21T09:27:51.050815+0000","last_became_active":"2024-09-21T09:27:08.454770+0000","last_became_peered":"2024-09-21T09:27:08.454770+0000","last_unstale":"2024-09-21T09:27:51.050815+0000","last_undegraded":"2024-09-21T09:27:51.050815+0000","last_fullsized":"2024-09-21T09:27:51.050815+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:25:07.646659+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:25:07.646659+0000","last_clean_scrub_stamp":"2024-09-21T09:25:07.646659+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:08:45.658466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":26,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-21T09:27:52.081088+0000","last_change":"2024-09-21T09:27:52.081088+0000","last_active":"2024-09-21T09:27:52.081088+0000","last_peered":"2024-09-21T09:27:52.081088+0000","last_clean":"2024-09-21T09:27:52.081088+0000","last_became_active":"2024-09-21T09:27:52.080680+0000","last_became_peered":"2024-09-21T09:27:52.080680+0000","last_unstale":"2024-09-21T09:27:52.081088+0000","last_undegraded":"2024-09-21T09:27:52.081088+0000","last_fullsized":"2024-09-21T09:27:52.081088+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:01:04.317396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.048243+0000","last_change":"2024-09-21T09:27:35.636200+0000","last_active":"2024-09-21T09:27:51.048243+0000","last_peered":"2024-09-21T09:27:51.048243+0000","last_clean":"2024-09-21T09:27:51.048243+0000","last_became_active":"2024-09-21T09:27:35.635939+0000","last_became_peered":"2024-09-21T09:27:35.635939+0000","last_unstale":"2024-09-21T09:27:51.048243+0000","last_undegraded":"2024-09-21T09:27:51.048243+0000","last_fullsized":"2024-09-21T09:27:51.048243+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:02:06.551561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.050696+0000","last_change":"2024-09-21T09:27:32.268500+0000","last_active":"2024-09-21T09:27:51.050696+0000","last_peered":"2024-09-21T09:27:51.050696+0000","last_clean":"2024-09-21T09:27:51.050696+0000","last_became_active":"2024-09-21T09:27:32.268026+0000","last_became_peered":"2024-09-21T09:27:32.268026+0000","last_unstale":"2024-09-21T09:27:51.050696+0000","last_undegraded":"2024-09-21T09:27:51.050696+0000","last_fullsized":"2024-09-21T09:27:51.050696+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:44:43.463552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.048336+0000","last_change":"2024-09-21T09:27:35.634025+0000","last_active":"2024-09-21T09:27:51.048336+0000","last_peered":"2024-09-21T09:27:51.048336+0000","last_clean":"2024-09-21T09:27:51.048336+0000","last_became_active":"2024-09-21T09:27:35.633746+0000","last_became_peered":"2024-09-21T09:27:35.633746+0000","last_unstale":"2024-09-21T09:27:51.048336+0000","last_undegraded":"2024-09-21T09:27:51.048336+0000","last_fullsized":"2024-09-21T09:27:51.048336+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:18:40.511292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.042366+0000","last_change":"2024-09-21T09:27:32.262645+0000","last_active":"2024-09-21T09:27:51.042366+0000","last_peered":"2024-09-21T09:27:51.042366+0000","last_clean":"2024-09-21T09:27:51.042366+0000","last_became_active":"2024-09-21T09:27:32.262373+0000","last_became_peered":"2024-09-21T09:27:32.262373+0000","last_unstale":"2024-09-21T09:27:51.042366+0000","last_undegraded":"2024-09-21T09:27:51.042366+0000","last_fullsized":"2024-09-21T09:27:51.042366+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:37:23.684561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.042540+0000","last_change":"2024-09-21T09:27:35.636105+0000","last_active":"2024-09-21T09:27:51.042540+0000","last_peered":"2024-09-21T09:27:51.042540+0000","last_clean":"2024-09-21T09:27:51.042540+0000","last_became_active":"2024-09-21T09:27:35.635805+0000","last_became_peered":"2024-09-21T09:27:35.635805+0000","last_unstale":"2024-09-21T09:27:51.042540+0000","last_undegraded":"2024-09-21T09:27:51.042540+0000","last_fullsized":"2024-09-21T09:27:51.042540+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:22:53.711130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.048097+0000","last_change":"2024-09-21T09:27:32.265210+0000","last_active":"2024-09-21T09:27:51.048097+0000","last_peered":"2024-09-21T09:27:51.048097+0000","last_clean":"2024-09-21T09:27:51.048097+0000","last_became_active":"2024-09-21T09:27:32.264740+0000","last_became_peered":"2024-09-21T09:27:32.264740+0000","last_unstale":"2024-09-21T09:27:51.048097+0000","last_undegraded":"2024-09-21T09:27:51.048097+0000","last_fullsized":"2024-09-21T09:27:51.048097+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:14:01.491316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.048823+0000","last_change":"2024-09-21T09:27:35.638582+0000","last_active":"2024-09-21T09:27:51.048823+0000","last_peered":"2024-09-21T09:27:51.048823+0000","last_clean":"2024-09-21T09:27:51.048823+0000","last_became_active":"2024-09-21T09:27:35.638269+0000","last_became_peered":"2024-09-21T09:27:35.638269+0000","last_unstale":"2024-09-21T09:27:51.048823+0000","last_undegraded":"2024-09-21T09:27:51.048823+0000","last_fullsized":"2024-09-21T09:27:51.048823+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:52:20.849743+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.043473+0000","last_change":"2024-09-21T09:27:32.264482+0000","last_active":"2024-09-21T09:27:51.043473+0000","last_peered":"2024-09-21T09:27:51.043473+0000","last_clean":"2024-09-21T09:27:51.043473+0000","last_became_active":"2024-09-21T09:27:32.264201+0000","last_became_peered":"2024-09-21T09:27:32.264201+0000","last_unstale":"2024-09-21T09:27:51.043473+0000","last_undegraded":"2024-09-21T09:27:51.043473+0000","last_fullsized":"2024-09-21T09:27:51.043473+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:37:15.565839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.045599+0000","last_change":"2024-09-21T09:27:35.634935+0000","last_active":"2024-09-21T09:27:51.045599+0000","last_peered":"2024-09-21T09:27:51.045599+0000","last_clean":"2024-09-21T09:27:51.045599+0000","last_became_active":"2024-09-21T09:27:35.634594+0000","last_became_peered":"2024-09-21T09:27:35.634594+0000","last_unstale":"2024-09-21T09:27:51.045599+0000","last_undegraded":"2024-09-21T09:27:51.045599+0000","last_fullsized":"2024-09-21T09:27:51.045599+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:53:03.606929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.050709+0000","last_change":"2024-09-21T09:27:32.274427+0000","last_active":"2024-09-21T09:27:51.050709+0000","last_peered":"2024-09-21T09:27:51.050709+0000","last_clean":"2024-09-21T09:27:51.050709+0000","last_became_active":"2024-09-21T09:27:32.274058+0000","last_became_peered":"2024-09-21T09:27:32.274058+0000","last_unstale":"2024-09-21T09:27:51.050709+0000","last_undegraded":"2024-09-21T09:27:51.050709+0000","last_fullsized":"2024-09-21T09:27:51.050709+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:09:38.714482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.043384+0000","last_change":"2024-09-21T09:27:35.643081+0000","last_active":"2024-09-21T09:27:51.043384+0000","last_peered":"2024-09-21T09:27:51.043384+0000","last_clean":"2024-09-21T09:27:51.043384+0000","last_became_active":"2024-09-21T09:27:35.642792+0000","last_became_peered":"2024-09-21T09:27:35.642792+0000","last_unstale":"2024-09-21T09:27:51.043384+0000","last_undegraded":"2024-09-21T09:27:51.043384+0000","last_fullsized":"2024-09-21T09:27:51.043384+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:05:03.157102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"57'1","reported_seq":17,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.043735+0000","last_change":"2024-09-21T09:27:32.265476+0000","last_active":"2024-09-21T09:27:51.043735+0000","last_peered":"2024-09-21T09:27:51.043735+0000","last_clean":"2024-09-21T09:27:51.043735+0000","last_became_active":"2024-09-21T09:27:32.265171+0000","last_became_peered":"2024-09-21T09:27:32.265171+0000","last_unstale":"2024-09-21T09:27:51.043735+0000","last_undegraded":"2024-09-21T09:27:51.043735+0000","last_fullsized":"2024-09-21T09:27:51.043735+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:42:48.279357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.049424+0000","last_change":"2024-09-21T09:27:35.629856+0000","last_active":"2024-09-21T09:27:51.049424+0000","last_peered":"2024-09-21T09:27:51.049424+0000","last_clean":"2024-09-21T09:27:51.049424+0000","last_became_active":"2024-09-21T09:27:35.629345+0000","last_became_peered":"2024-09-21T09:27:35.629345+0000","last_unstale":"2024-09-21T09:27:51.049424+0000","last_undegraded":"2024-09-21T09:27:51.049424+0000","last_fullsized":"2024-09-21T09:27:51.049424+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:30:14.361278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"57'2","reported_seq":18,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.045847+0000","last_change":"2024-09-21T09:27:32.264763+0000","last_active":"2024-09-21T09:27:51.045847+0000","last_peered":"2024-09-21T09:27:51.045847+0000","last_clean":"2024-09-21T09:27:51.045847+0000","last_became_active":"2024-09-21T09:27:32.264444+0000","last_became_peered":"2024-09-21T09:27:32.264444+0000","last_unstale":"2024-09-21T09:27:51.045847+0000","last_undegraded":"2024-09-21T09:27:51.045847+0000","last_fullsized":"2024-09-21T09:27:51.045847+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:01:52.871773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.050094+0000","last_change":"2024-09-21T09:27:35.637964+0000","last_active":"2024-09-21T09:27:51.050094+0000","last_peered":"2024-09-21T09:27:51.050094+0000","last_clean":"2024-09-21T09:27:51.050094+0000","last_became_active":"2024-09-21T09:27:35.634680+0000","last_became_peered":"2024-09-21T09:27:35.634680+0000","last_unstale":"2024-09-21T09:27:51.050094+0000","last_undegraded":"2024-09-21T09:27:51.050094+0000","last_fullsized":"2024-09-21T09:27:51.050094+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:50:36.642394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.8","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.049792+0000","last_change":"2024-09-21T09:27:32.270810+0000","last_active":"2024-09-21T09:27:51.049792+0000","last_peered":"2024-09-21T09:27:51.049792+0000","last_clean":"2024-09-21T09:27:51.049792+0000","last_became_active":"2024-09-21T09:27:32.268720+0000","last_became_peered":"2024-09-21T09:27:32.268720+0000","last_unstale":"2024-09-21T09:27:51.049792+0000","last_undegraded":"2024-09-21T09:27:51.049792+0000","last_fullsized":"2024-09-21T09:27:51.049792+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:58:54.820070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.047482+0000","last_change":"2024-09-21T09:27:35.637942+0000","last_active":"2024-09-21T09:27:51.047482+0000","last_peered":"2024-09-21T09:27:51.047482+0000","last_clean":"2024-09-21T09:27:51.047482+0000","last_became_active":"2024-09-21T09:27:35.637064+0000","last_became_peered":"2024-09-21T09:27:35.637064+0000","last_unstale":"2024-09-21T09:27:51.047482+0000","last_undegraded":"2024-09-21T09:27:51.047482+0000","last_fullsized":"2024-09-21T09:27:51.047482+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:16:53.911245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.9","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.042877+0000","last_change":"2024-09-21T09:27:32.262848+0000","last_active":"2024-09-21T09:27:51.042877+0000","last_peered":"2024-09-21T09:27:51.042877+0000","last_clean":"2024-09-21T09:27:51.042877+0000","last_became_active":"2024-09-21T09:27:32.262555+0000","last_became_peered":"2024-09-21T09:27:32.262555+0000","last_unstale":"2024-09-21T09:27:51.042877+0000","last_undegraded":"2024-09-21T09:27:51.042877+0000","last_fullsized":"2024-09-21T09:27:51.042877+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:27:02.670310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.048071+0000","last_change":"2024-09-21T09:27:35.625899+0000","last_active":"2024-09-21T09:27:51.048071+0000","last_peered":"2024-09-21T09:27:51.048071+0000","last_clean":"2024-09-21T09:27:51.048071+0000","last_became_active":"2024-09-21T09:27:35.625619+0000","last_became_peered":"2024-09-21T09:27:35.625619+0000","last_unstale":"2024-09-21T09:27:51.048071+0000","last_undegraded":"2024-09-21T09:27:51.048071+0000","last_fullsized":"2024-09-21T09:27:51.048071+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:38:48.577776+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.a","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.043557+0000","last_change":"2024-09-21T09:27:32.264667+0000","last_active":"2024-09-21T09:27:51.043557+0000","last_peered":"2024-09-21T09:27:51.043557+0000","last_clean":"2024-09-21T09:27:51.043557+0000","last_became_active":"2024-09-21T09:27:32.264353+0000","last_became_peered":"2024-09-21T09:27:32.264353+0000","last_unstale":"2024-09-21T09:27:51.043557+0000","last_undegraded":"2024-09-21T09:27:51.043557+0000","last_fullsized":"2024-09-21T09:27:51.043557+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:21:53.938525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"0'0","reported_seq":23,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-21T09:27:52.081164+0000","last_change":"2024-09-21T09:27:52.081164+0000","last_active":"2024-09-21T09:27:52.081164+0000","last_peered":"2024-09-21T09:27:52.081164+0000","last_clean":"2024-09-21T09:27:52.081164+0000","last_became_active":"2024-09-21T09:27:52.080726+0000","last_became_peered":"2024-09-21T09:27:52.080726+0000","last_unstale":"2024-09-21T09:27:52.081164+0000","last_undegraded":"2024-09-21T09:27:52.081164+0000","last_fullsized":"2024-09-21T09:27:52.081164+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:37:01.620726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.b","version":"57'2","reported_seq":18,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.050288+0000","last_change":"2024-09-21T09:27:32.274404+0000","last_active":"2024-09-21T09:27:51.050288+0000","last_peered":"2024-09-21T09:27:51.050288+0000","last_clean":"2024-09-21T09:27:51.050288+0000","last_became_active":"2024-09-21T09:27:32.274109+0000","last_became_peered":"2024-09-21T09:27:32.274109+0000","last_unstale":"2024-09-21T09:27:51.050288+0000","last_undegraded":"2024-09-21T09:27:51.050288+0000","last_fullsized":"2024-09-21T09:27:51.050288+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:24:06.607203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":23,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-21T09:27:52.050465+0000","last_change":"2024-09-21T09:27:52.050465+0000","last_active":"2024-09-21T09:27:52.050465+0000","last_peered":"2024-09-21T09:27:52.050465+0000","last_clean":"2024-09-21T09:27:52.050465+0000","last_became_active":"2024-09-21T09:27:52.050052+0000","last_became_peered":"2024-09-21T09:27:52.050052+0000","last_unstale":"2024-09-21T09:27:52.050465+0000","last_undegraded":"2024-09-21T09:27:52.050465+0000","last_fullsized":"2024-09-21T09:27:52.050465+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:58:35.370217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1d","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.045494+0000","last_change":"2024-09-21T09:27:35.637946+0000","last_active":"2024-09-21T09:27:51.045494+0000","last_peered":"2024-09-21T09:27:51.045494+0000","last_clean":"2024-09-21T09:27:51.045494+0000","last_became_active":"2024-09-21T09:27:35.637664+0000","last_became_peered":"2024-09-21T09:27:35.637664+0000","last_unstale":"2024-09-21T09:27:51.045494+0000","last_undegraded":"2024-09-21T09:27:51.045494+0000","last_fullsized":"2024-09-21T09:27:51.045494+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:22:17.260192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1c","version":"57'2","reported_seq":18,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.050172+0000","last_change":"2024-09-21T09:27:32.268336+0000","last_active":"2024-09-21T09:27:51.050172+0000","last_peered":"2024-09-21T09:27:51.050172+0000","last_clean":"2024-09-21T09:27:51.050172+0000","last_became_active":"2024-09-21T09:27:32.268038+0000","last_became_peered":"2024-09-21T09:27:32.268038+0000","last_unstale":"2024-09-21T09:27:51.050172+0000","last_undegraded":"2024-09-21T09:27:51.050172+0000","last_fullsized":"2024-09-21T09:27:51.050172+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:57:13.991593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1c","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.045543+0000","last_change":"2024-09-21T09:27:35.642780+0000","last_active":"2024-09-21T09:27:51.045543+0000","last_peered":"2024-09-21T09:27:51.045543+0000","last_clean":"2024-09-21T09:27:51.045543+0000","last_became_active":"2024-09-21T09:27:35.642487+0000","last_became_peered":"2024-09-21T09:27:35.642487+0000","last_unstale":"2024-09-21T09:27:51.045543+0000","last_undegraded":"2024-09-21T09:27:51.045543+0000","last_fullsized":"2024-09-21T09:27:51.045543+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:28:54.864609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1d","version":"57'2","reported_seq":18,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.050487+0000","last_change":"2024-09-21T09:27:32.270398+0000","last_active":"2024-09-21T09:27:51.050487+0000","last_peered":"2024-09-21T09:27:51.050487+0000","last_clean":"2024-09-21T09:27:51.050487+0000","last_became_active":"2024-09-21T09:27:32.269469+0000","last_became_peered":"2024-09-21T09:27:32.269469+0000","last_unstale":"2024-09-21T09:27:51.050487+0000","last_undegraded":"2024-09-21T09:27:51.050487+0000","last_fullsized":"2024-09-21T09:27:51.050487+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:11:17.381607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1f","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.050305+0000","last_change":"2024-09-21T09:27:35.646662+0000","last_active":"2024-09-21T09:27:51.050305+0000","last_peered":"2024-09-21T09:27:51.050305+0000","last_clean":"2024-09-21T09:27:51.050305+0000","last_became_active":"2024-09-21T09:27:35.646141+0000","last_became_peered":"2024-09-21T09:27:35.646141+0000","last_unstale":"2024-09-21T09:27:51.050305+0000","last_undegraded":"2024-09-21T09:27:51.050305+0000","last_fullsized":"2024-09-21T09:27:51.050305+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:22:15.834922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1e","version":"57'1","reported_seq":17,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.049450+0000","last_change":"2024-09-21T09:27:32.270481+0000","last_active":"2024-09-21T09:27:51.049450+0000","last_peered":"2024-09-21T09:27:51.049450+0000","last_clean":"2024-09-21T09:27:51.049450+0000","last_became_active":"2024-09-21T09:27:32.269813+0000","last_became_peered":"2024-09-21T09:27:32.269813+0000","last_unstale":"2024-09-21T09:27:51.049450+0000","last_undegraded":"2024-09-21T09:27:51.049450+0000","last_fullsized":"2024-09-21T09:27:51.049450+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:28:41.266917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.048769+0000","last_change":"2024-09-21T09:27:35.636905+0000","last_active":"2024-09-21T09:27:51.048769+0000","last_peered":"2024-09-21T09:27:51.048769+0000","last_clean":"2024-09-21T09:27:51.048769+0000","last_became_active":"2024-09-21T09:27:35.636625+0000","last_became_peered":"2024-09-21T09:27:35.636625+0000","last_unstale":"2024-09-21T09:27:51.048769+0000","last_undegraded":"2024-09-21T09:27:51.048769+0000","last_fullsized":"2024-09-21T09:27:51.048769+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:19:39.022799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1f","version":"57'4","reported_seq":20,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.050451+0000","last_change":"2024-09-21T09:27:32.267938+0000","last_active":"2024-09-21T09:27:51.050451+0000","last_peered":"2024-09-21T09:27:51.050451+0000","last_clean":"2024-09-21T09:27:51.050451+0000","last_became_active":"2024-09-21T09:27:32.267583+0000","last_became_peered":"2024-09-21T09:27:32.267583+0000","last_unstale":"2024-09-21T09:27:51.050451+0000","last_undegraded":"2024-09-21T09:27:51.050451+0000","last_fullsized":"2024-09-21T09:27:51.050451+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:35:00.591469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":3,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":192,"acting":192,"num_store_stats":8},{"poolid":2,"num_pg":32,"stat_sum":{"num_bytes":2338,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":7014,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3563520,"data_stored":3542208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430220,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298520,"kb_used_data":1164,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446632,"statfs":{"total":95995035648,"available":95689351168,"internally_reserved":0,"allocated":1191936,"data_stored":842056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.57999999999999996}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.46000000000000002}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.51100000000000001}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.47799999999999998}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.48899999999999999}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.38800000000000001}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.56499999999999995}]}]},{"osd":6,"up_from":42,"seq":180388626448,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298524,"kb_used_data":1164,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446628,"statfs":{"total":95995035648,"available":95689347072,"internally_reserved":0,"allocated":1191936,"data_stored":842701,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:27:45 2024","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.70799999999999996,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.78000000000000003},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.83699999999999997,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.63200000000000001}]},{"osd":1,"last update":"Sat Sep 21 09:27:45 2024","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.70199999999999996},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.76500000000000001,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.71599999999999997}]},{"osd":2,"last update":"Sat Sep 21 09:27:45 2024","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.66600000000000004},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"min":{"1min":0.441,"5min":0.441,"15min":0.441},"max":{"1min":0.74099999999999999,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.68300000000000005}]},{"osd":3,"last update":"Sat Sep 21 09:27:45 2024","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.65300000000000002},{"interface":"front","average":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.83199999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.754}]},{"osd":4,"last update":"Sat Sep 21 09:27:45 2024","interfaces":[{"interface":"back","average":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.69899999999999995,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":0.45800000000000002},{"interface":"front","average":{"1min":0.438,"5min":0.438,"15min":0.438},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.53900000000000003}]},{"osd":5,"last update":"Sat Sep 21 09:27:45 2024","interfaces":[{"interface":"back","average":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.51200000000000001},{"interface":"front","average":{"1min":0.48699999999999999,"5min":0.48699999999999999,"15min":0.48699999999999999},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.85499999999999998,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.48099999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.61099999999999999}]}]},{"osd":5,"up_from":36,"seq":154618822678,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297944,"kb_used_data":584,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447208,"statfs":{"total":95995035648,"available":95689940992,"internally_reserved":0,"allocated":598016,"data_stored":252108,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:27:21 2024","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.54200000000000004},{"interface":"front","average":{"1min":0.52000000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.69099999999999995,"5min":0.69099999999999995,"15min":0.69099999999999995},"last":0.42399999999999999}]},{"osd":1,"last update":"Sat Sep 21 09:27:21 2024","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.39700000000000002},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.69899999999999995,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":0.52500000000000002}]},{"osd":2,"last update":"Sat Sep 21 09:27:21 2024","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.75,"5min":0.75,"15min":0.75},"last":0.35099999999999998},{"interface":"front","average":{"1min":0.505,"5min":0.505,"15min":0.505},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.72999999999999998,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.505}]},{"osd":3,"last update":"Sat Sep 21 09:27:21 2024","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.79000000000000004,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.46300000000000002},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.442}]},{"osd":4,"last update":"Sat Sep 21 09:27:21 2024","interfaces":[{"interface":"back","average":{"1min":0.377,"5min":0.377,"15min":0.377},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"last":0.58199999999999996},{"interface":"front","average":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":0.65500000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"last":0.60199999999999998}]},{"osd":6,"last update":"Sat Sep 21 09:27:45 2024","interfaces":[{"interface":"back","average":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.72099999999999997,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.65800000000000003},{"interface":"front","average":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.68100000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"last":0.56200000000000006}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.63}]}]},{"osd":4,"up_from":30,"seq":128849018906,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297952,"kb_used_data":592,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447200,"statfs":{"total":95995035648,"available":95689932800,"internally_reserved":0,"allocated":606208,"data_stored":251792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:26:55 2024","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.85499999999999998,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.63700000000000001},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.62}]},{"osd":1,"last update":"Sat Sep 21 09:26:55 2024","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.78700000000000003,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.67500000000000004},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.79700000000000004,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.66700000000000004}]},{"osd":2,"last update":"Sat Sep 21 09:26:55 2024","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.91200000000000003,"5min":0.91200000000000003,"15min":0.91200000000000003},"last":0.51500000000000001},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.77100000000000002,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.72899999999999998}]},{"osd":3,"last update":"Sat Sep 21 09:26:55 2024","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.82399999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.65600000000000003},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.81399999999999995,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":0.71199999999999997}]},{"osd":5,"last update":"Sat Sep 21 09:27:24 2024","interfaces":[{"interface":"back","average":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.66300000000000003,"5min":0.66300000000000003,"15min":0.66300000000000003},"last":0.69099999999999995},{"interface":"front","average":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"last":0.60799999999999998}]},{"osd":6,"last update":"Sat Sep 21 09:27:44 2024","interfaces":[{"interface":"back","average":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.747,"5min":0.747,"15min":0.747},"last":0.57999999999999996},{"interface":"front","average":{"1min":0.46700000000000003,"5min":0.46700000000000003,"15min":0.46700000000000003},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.81399999999999995,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":0.56699999999999995}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.54300000000000004}]}]},{"osd":3,"up_from":25,"seq":107374182431,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297956,"kb_used_data":596,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447196,"statfs":{"total":95995035648,"available":95689928704,"internally_reserved":0,"allocated":610304,"data_stored":252752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:27:39 2024","interfaces":[{"interface":"back","average":{"1min":0.38,"5min":0.36599999999999999,"15min":0.36399999999999999},"min":{"1min":0.253,"5min":0.25,"15min":0.25},"max":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"last":0.315},{"interface":"front","average":{"1min":0.41399999999999998,"5min":0.39400000000000002,"15min":0.39100000000000001},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"last":0.40000000000000002}]},{"osd":1,"last update":"Sat Sep 21 09:27:39 2024","interfaces":[{"interface":"back","average":{"1min":0.44600000000000001,"5min":0.434,"15min":0.432},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.63900000000000001,"5min":0.63900000000000001,"15min":0.63900000000000001},"last":0.36199999999999999},{"interface":"front","average":{"1min":0.40300000000000002,"5min":0.41299999999999998,"15min":0.41499999999999998},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":0.66700000000000004,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.28899999999999998}]},{"osd":2,"last update":"Sat Sep 21 09:27:39 2024","interfaces":[{"interface":"back","average":{"1min":0.439,"5min":0.44400000000000001,"15min":0.44500000000000001},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.625,"5min":0.625,"15min":0.625},"last":0.42799999999999999},{"interface":"front","average":{"1min":0.41899999999999998,"5min":0.42099999999999999,"15min":0.42099999999999999},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"last":0.38100000000000001}]},{"osd":4,"last update":"Sat Sep 21 09:26:57 2024","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.68999999999999995,"5min":0.68999999999999995,"15min":0.68999999999999995},"last":0.54000000000000004},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.71599999999999997,"5min":0.71599999999999997,"15min":0.71599999999999997},"last":0.60499999999999998}]},{"osd":5,"last update":"Sat Sep 21 09:27:19 2024","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.83299999999999996,"5min":0.83299999999999996,"15min":0.83299999999999996},"last":0.65400000000000003},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.60599999999999998,"15min":0.60599999999999998},"min":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":0.77300000000000002,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.624}]},{"osd":6,"last update":"Sat Sep 21 09:27:46 2024","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.85999999999999999,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.58899999999999997},{"interface":"front","average":{"1min":0.62,"5min":0.62,"15min":0.62},"min":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.68300000000000005}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.72199999999999998}]}]},{"osd":2,"up_from":18,"seq":77309411364,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297932,"kb_used_data":576,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447220,"statfs":{"total":95995035648,"available":95689953280,"internally_reserved":0,"allocated":589824,"data_stored":251096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:27:12 2024","interfaces":[{"interface":"back","average":{"1min":0.36599999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.47499999999999998,"5min":0.54400000000000004,"15min":0.54400000000000004},"last":0.35199999999999998},{"interface":"front","average":{"1min":0.40899999999999997,"5min":0.39700000000000002,"15min":0.39500000000000002},"min":{"1min":0.33000000000000002,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.50700000000000001,"5min":0.53800000000000003,"15min":0.53800000000000003},"last":0.33300000000000002}]},{"osd":1,"last update":"Sat Sep 21 09:27:12 2024","interfaces":[{"interface":"back","average":{"1min":0.40400000000000003,"5min":0.374,"15min":0.36899999999999999},"min":{"1min":0.26800000000000002,"5min":0.219,"15min":0.219},"max":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"last":0.376},{"interface":"front","average":{"1min":0.46700000000000003,"5min":0.42199999999999999,"15min":0.41399999999999998},"min":{"1min":0.376,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.625,"5min":0.625,"15min":0.625},"last":0.39300000000000002}]},{"osd":3,"last update":"Sat Sep 21 09:27:35 2024","interfaces":[{"interface":"back","average":{"1min":0.376,"5min":0.437,"15min":0.44700000000000001},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.54200000000000004,"5min":0.61599999999999999,"15min":0.61599999999999999},"last":0.41199999999999998},{"interface":"front","average":{"1min":0.44400000000000001,"5min":0.46800000000000003,"15min":0.47199999999999998},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.64000000000000001,"5min":0.65200000000000002,"15min":0.65200000000000002},"last":0.42599999999999999}]},{"osd":4,"last update":"Sat Sep 21 09:26:57 2024","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"min":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"last":0.76200000000000001},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"min":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":0.85099999999999998,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.68500000000000005}]},{"osd":5,"last update":"Sat Sep 21 09:27:22 2024","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"min":{"1min":0.48399999999999999,"5min":0.48399999999999999,"15min":0.48399999999999999},"max":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.745},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":0.75700000000000001,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.61399999999999999}]},{"osd":6,"last update":"Sat Sep 21 09:27:43 2024","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.72199999999999998},{"interface":"front","average":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"min":{"1min":0.47399999999999998,"5min":0.47399999999999998,"15min":0.47399999999999998},"max":{"1min":0.79900000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.58499999999999996}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.78200000000000003}]}]},{"osd":1,"up_from":13,"seq":55834574889,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297944,"kb_used_data":584,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447208,"statfs":{"total":95995035648,"available":95689940992,"internally_reserved":0,"allocated":598016,"data_stored":251256,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:27:46 2024","interfaces":[{"interface":"back","average":{"1min":0.40500000000000003,"5min":0.35899999999999999,"15min":0.33600000000000002},"min":{"1min":0.246,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":0.60599999999999998,"5min":0.61399999999999999,"15min":0.61399999999999999},"last":0.33500000000000002},{"interface":"front","average":{"1min":0.41099999999999998,"5min":0.39100000000000001,"15min":0.36899999999999999},"min":{"1min":0.28299999999999997,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.58099999999999996,"5min":0.63,"15min":0.63},"last":0.40200000000000002}]},{"osd":2,"last update":"Sat Sep 21 09:27:12 2024","interfaces":[{"interface":"back","average":{"1min":0.41799999999999998,"5min":0.37,"15min":0.36199999999999999},"min":{"1min":0.28699999999999998,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"last":0.45500000000000002},{"interface":"front","average":{"1min":0.41499999999999998,"5min":0.379,"15min":0.373},"min":{"1min":0.27400000000000002,"5min":0.245,"15min":0.245},"max":{"1min":0.67100000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.35299999999999998}]},{"osd":3,"last update":"Sat Sep 21 09:27:35 2024","interfaces":[{"interface":"back","average":{"1min":0.40200000000000002,"5min":0.41999999999999998,"15min":0.42299999999999999},"min":{"1min":0.309,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"last":0.29699999999999999},{"interface":"front","average":{"1min":0.46500000000000002,"5min":0.46999999999999997,"15min":0.47099999999999997},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.76800000000000002,"5min":0.76800000000000002,"15min":0.76800000000000002},"last":0.38100000000000001}]},{"osd":4,"last update":"Sat Sep 21 09:26:55 2024","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.97599999999999998,"5min":0.97599999999999998,"15min":0.97599999999999998},"last":0.56999999999999995},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.95699999999999996,"5min":0.95699999999999996,"15min":0.95699999999999996},"last":0.47299999999999998}]},{"osd":5,"last update":"Sat Sep 21 09:27:20 2024","interfaces":[{"interface":"back","average":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"min":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":1.014,"5min":1.014,"15min":1.014},"last":0.51800000000000002},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.93500000000000005,"5min":0.93500000000000005,"15min":0.93500000000000005},"last":0.44}]},{"osd":6,"last update":"Sat Sep 21 09:27:46 2024","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.74099999999999999,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.58599999999999997},{"interface":"front","average":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"min":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.49199999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.53500000000000003}]}]},{"osd":0,"up_from":8,"seq":34359738413,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298524,"kb_used_data":1168,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446628,"statfs":{"total":95995035648,"available":95689347072,"internally_reserved":0,"allocated":1196032,"data_stored":841865,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 21 09:27:47 2024","interfaces":[{"interface":"back","average":{"1min":0.33200000000000002,"5min":0.32000000000000001,"15min":0.31},"min":{"1min":0.20200000000000001,"5min":0.185,"15min":0.185},"max":{"1min":0.51300000000000001,"5min":0.54300000000000004,"15min":0.54300000000000004},"last":0.46999999999999997},{"interface":"front","average":{"1min":0.34499999999999997,"5min":0.34200000000000003,"15min":0.33300000000000002},"min":{"1min":0.24299999999999999,"5min":0.185,"15min":0.185},"max":{"1min":0.51600000000000001,"5min":0.625,"15min":0.625},"last":0.35599999999999998}]},{"osd":2,"last update":"Sat Sep 21 09:27:12 2024","interfaces":[{"interface":"back","average":{"1min":0.39300000000000002,"5min":0.373,"15min":0.36899999999999999},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"last":0.58599999999999997},{"interface":"front","average":{"1min":0.44,"5min":0.42299999999999999,"15min":0.41999999999999998},"min":{"1min":0.30499999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.56200000000000006,"5min":0.69399999999999995,"15min":0.69399999999999995},"last":0.42399999999999999}]},{"osd":3,"last update":"Sat Sep 21 09:27:36 2024","interfaces":[{"interface":"back","average":{"1min":0.39900000000000002,"5min":0.42399999999999999,"15min":0.42799999999999999},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"last":0.38500000000000001},{"interface":"front","average":{"1min":0.42099999999999999,"5min":0.42899999999999999,"15min":0.42999999999999999},"min":{"1min":0.32400000000000001,"5min":0.31,"15min":0.31},"max":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"last":0.52600000000000002}]},{"osd":4,"last update":"Sat Sep 21 09:26:55 2024","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"last":0.67400000000000004},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"last":0.751}]},{"osd":5,"last update":"Sat Sep 21 09:27:23 2024","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.76600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.66100000000000003},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.84199999999999997,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":0.73699999999999999}]},{"osd":6,"last update":"Sat Sep 21 09:27:44 2024","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.876,"5min":0.876,"15min":0.876},"last":0.56699999999999995},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"min":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":0.753,"5min":0.753,"15min":0.753},"last":0.71999999999999997}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.70499999999999996}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":716,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":1656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1642,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":1552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T09:27:59.409 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid d65e801e-77fa-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T09:27:59.674 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:28:00.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:00 smithi022 ceph-mon[39525]: from='client.24410 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:28:00.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:00 smithi022 ceph-mon[39525]: pgmap v203: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:28:00.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:00 smithi022 ceph-mon[27763]: from='client.24410 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:28:00.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:00 smithi022 ceph-mon[27763]: pgmap v203: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:28:00.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:00 smithi064 ceph-mon[31124]: from='client.24410 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:28:00.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:00 smithi064 ceph-mon[31124]: pgmap v203: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:28:01.600 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-09-21T09:28:01.601 INFO:teuthology.orchestra.run.smithi022.stderr:dumped all 2024-09-21T09:28:02.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:02 smithi022 ceph-mon[39525]: pgmap v204: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:28:02.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:02 smithi022 ceph-mon[27763]: pgmap v204: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:28:02.345 INFO:teuthology.orchestra.run.smithi022.stdout:{"pg_ready":true,"pg_map":{"version":204,"stamp":"2024-09-21T09:28:00.971368+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592706,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":146,"num_write_kb":1641,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":111,"ondisk_log_size":111,"up":291,"acting":291,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":291,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2385296,"kb_used_data":6428,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747575920,"statfs":{"total":767960285184,"available":765517742080,"internally_reserved":0,"allocated":6582272,"data_stored":3785626,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.930165"},"pg_stats":[{"pgid":"3.3f","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.051004+0000","last_change":"2024-09-21T09:27:35.635695+0000","last_active":"2024-09-21T09:27:51.051004+0000","last_peered":"2024-09-21T09:27:51.051004+0000","last_clean":"2024-09-21T09:27:51.051004+0000","last_became_active":"2024-09-21T09:27:35.635255+0000","last_became_peered":"2024-09-21T09:27:35.635255+0000","last_unstale":"2024-09-21T09:27:51.051004+0000","last_undegraded":"2024-09-21T09:27:51.051004+0000","last_fullsized":"2024-09-21T09:27:51.051004+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:15:50.156743+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3e","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.043647+0000","last_change":"2024-09-21T09:27:35.636115+0000","last_active":"2024-09-21T09:27:51.043647+0000","last_peered":"2024-09-21T09:27:51.043647+0000","last_clean":"2024-09-21T09:27:51.043647+0000","last_became_active":"2024-09-21T09:27:35.635778+0000","last_became_peered":"2024-09-21T09:27:35.635778+0000","last_unstale":"2024-09-21T09:27:51.043647+0000","last_undegraded":"2024-09-21T09:27:51.043647+0000","last_fullsized":"2024-09-21T09:27:51.043647+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:36:35.607057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3d","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.050003+0000","last_change":"2024-09-21T09:27:35.640689+0000","last_active":"2024-09-21T09:27:51.050003+0000","last_peered":"2024-09-21T09:27:51.050003+0000","last_clean":"2024-09-21T09:27:51.050003+0000","last_became_active":"2024-09-21T09:27:35.640415+0000","last_became_peered":"2024-09-21T09:27:35.640415+0000","last_unstale":"2024-09-21T09:27:51.050003+0000","last_undegraded":"2024-09-21T09:27:51.050003+0000","last_fullsized":"2024-09-21T09:27:51.050003+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:12:22.314199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3c","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.045431+0000","last_change":"2024-09-21T09:27:35.644536+0000","last_active":"2024-09-21T09:27:51.045431+0000","last_peered":"2024-09-21T09:27:51.045431+0000","last_clean":"2024-09-21T09:27:51.045431+0000","last_became_active":"2024-09-21T09:27:35.644246+0000","last_became_peered":"2024-09-21T09:27:35.644246+0000","last_unstale":"2024-09-21T09:27:51.045431+0000","last_undegraded":"2024-09-21T09:27:51.045431+0000","last_fullsized":"2024-09-21T09:27:51.045431+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:31:37.886172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3b","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.050078+0000","last_change":"2024-09-21T09:27:35.642779+0000","last_active":"2024-09-21T09:27:51.050078+0000","last_peered":"2024-09-21T09:27:51.050078+0000","last_clean":"2024-09-21T09:27:51.050078+0000","last_became_active":"2024-09-21T09:27:35.642487+0000","last_became_peered":"2024-09-21T09:27:35.642487+0000","last_unstale":"2024-09-21T09:27:51.050078+0000","last_undegraded":"2024-09-21T09:27:51.050078+0000","last_fullsized":"2024-09-21T09:27:51.050078+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:17:29.631740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3a","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.046996+0000","last_change":"2024-09-21T09:27:35.651016+0000","last_active":"2024-09-21T09:27:51.046996+0000","last_peered":"2024-09-21T09:27:51.046996+0000","last_clean":"2024-09-21T09:27:51.046996+0000","last_became_active":"2024-09-21T09:27:35.650742+0000","last_became_peered":"2024-09-21T09:27:35.650742+0000","last_unstale":"2024-09-21T09:27:51.046996+0000","last_undegraded":"2024-09-21T09:27:51.046996+0000","last_fullsized":"2024-09-21T09:27:51.046996+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:11:11.493990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.39","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.050829+0000","last_change":"2024-09-21T09:27:35.647235+0000","last_active":"2024-09-21T09:27:51.050829+0000","last_peered":"2024-09-21T09:27:51.050829+0000","last_clean":"2024-09-21T09:27:51.050829+0000","last_became_active":"2024-09-21T09:27:35.646890+0000","last_became_peered":"2024-09-21T09:27:35.646890+0000","last_unstale":"2024-09-21T09:27:51.050829+0000","last_undegraded":"2024-09-21T09:27:51.050829+0000","last_fullsized":"2024-09-21T09:27:51.050829+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:21:45.995387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.38","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.047996+0000","last_change":"2024-09-21T09:27:35.638161+0000","last_active":"2024-09-21T09:27:51.047996+0000","last_peered":"2024-09-21T09:27:51.047996+0000","last_clean":"2024-09-21T09:27:51.047996+0000","last_became_active":"2024-09-21T09:27:35.637751+0000","last_became_peered":"2024-09-21T09:27:35.637751+0000","last_unstale":"2024-09-21T09:27:51.047996+0000","last_undegraded":"2024-09-21T09:27:51.047996+0000","last_fullsized":"2024-09-21T09:27:51.047996+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:07:40.751242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.37","version":"0'0","reported_seq":23,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-21T09:27:52.079760+0000","last_change":"2024-09-21T09:27:52.079760+0000","last_active":"2024-09-21T09:27:52.079760+0000","last_peered":"2024-09-21T09:27:52.079760+0000","last_clean":"2024-09-21T09:27:52.079760+0000","last_became_active":"2024-09-21T09:27:52.079352+0000","last_became_peered":"2024-09-21T09:27:52.079352+0000","last_unstale":"2024-09-21T09:27:52.079760+0000","last_undegraded":"2024-09-21T09:27:52.079760+0000","last_fullsized":"2024-09-21T09:27:52.079760+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:57:57.194876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.36","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.045365+0000","last_change":"2024-09-21T09:27:35.636094+0000","last_active":"2024-09-21T09:27:51.045365+0000","last_peered":"2024-09-21T09:27:51.045365+0000","last_clean":"2024-09-21T09:27:51.045365+0000","last_became_active":"2024-09-21T09:27:35.635770+0000","last_became_peered":"2024-09-21T09:27:35.635770+0000","last_unstale":"2024-09-21T09:27:51.045365+0000","last_undegraded":"2024-09-21T09:27:51.045365+0000","last_fullsized":"2024-09-21T09:27:51.045365+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:41:52.909549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.35","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.045488+0000","last_change":"2024-09-21T09:27:35.640571+0000","last_active":"2024-09-21T09:27:51.045488+0000","last_peered":"2024-09-21T09:27:51.045488+0000","last_clean":"2024-09-21T09:27:51.045488+0000","last_became_active":"2024-09-21T09:27:35.640253+0000","last_became_peered":"2024-09-21T09:27:35.640253+0000","last_unstale":"2024-09-21T09:27:51.045488+0000","last_undegraded":"2024-09-21T09:27:51.045488+0000","last_fullsized":"2024-09-21T09:27:51.045488+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:40:44.721446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.34","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.048082+0000","last_change":"2024-09-21T09:27:35.639471+0000","last_active":"2024-09-21T09:27:51.048082+0000","last_peered":"2024-09-21T09:27:51.048082+0000","last_clean":"2024-09-21T09:27:51.048082+0000","last_became_active":"2024-09-21T09:27:35.639056+0000","last_became_peered":"2024-09-21T09:27:35.639056+0000","last_unstale":"2024-09-21T09:27:51.048082+0000","last_undegraded":"2024-09-21T09:27:51.048082+0000","last_fullsized":"2024-09-21T09:27:51.048082+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:16:15.724781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.33","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.048493+0000","last_change":"2024-09-21T09:27:35.637331+0000","last_active":"2024-09-21T09:27:51.048493+0000","last_peered":"2024-09-21T09:27:51.048493+0000","last_clean":"2024-09-21T09:27:51.048493+0000","last_became_active":"2024-09-21T09:27:35.637068+0000","last_became_peered":"2024-09-21T09:27:35.637068+0000","last_unstale":"2024-09-21T09:27:51.048493+0000","last_undegraded":"2024-09-21T09:27:51.048493+0000","last_fullsized":"2024-09-21T09:27:51.048493+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:24:52.176104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.32","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.049091+0000","last_change":"2024-09-21T09:27:35.645565+0000","last_active":"2024-09-21T09:27:51.049091+0000","last_peered":"2024-09-21T09:27:51.049091+0000","last_clean":"2024-09-21T09:27:51.049091+0000","last_became_active":"2024-09-21T09:27:35.644114+0000","last_became_peered":"2024-09-21T09:27:35.644114+0000","last_unstale":"2024-09-21T09:27:51.049091+0000","last_undegraded":"2024-09-21T09:27:51.049091+0000","last_fullsized":"2024-09-21T09:27:51.049091+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:27:18.008605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.31","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.042816+0000","last_change":"2024-09-21T09:27:35.634448+0000","last_active":"2024-09-21T09:27:51.042816+0000","last_peered":"2024-09-21T09:27:51.042816+0000","last_clean":"2024-09-21T09:27:51.042816+0000","last_became_active":"2024-09-21T09:27:35.634089+0000","last_became_peered":"2024-09-21T09:27:35.634089+0000","last_unstale":"2024-09-21T09:27:51.042816+0000","last_undegraded":"2024-09-21T09:27:51.042816+0000","last_fullsized":"2024-09-21T09:27:51.042816+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:14:14.726492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.30","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.046109+0000","last_change":"2024-09-21T09:27:35.651997+0000","last_active":"2024-09-21T09:27:51.046109+0000","last_peered":"2024-09-21T09:27:51.046109+0000","last_clean":"2024-09-21T09:27:51.046109+0000","last_became_active":"2024-09-21T09:27:35.651698+0000","last_became_peered":"2024-09-21T09:27:35.651698+0000","last_unstale":"2024-09-21T09:27:51.046109+0000","last_undegraded":"2024-09-21T09:27:51.046109+0000","last_fullsized":"2024-09-21T09:27:51.046109+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:43:39.574788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2f","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.046369+0000","last_change":"2024-09-21T09:27:35.646363+0000","last_active":"2024-09-21T09:27:51.046369+0000","last_peered":"2024-09-21T09:27:51.046369+0000","last_clean":"2024-09-21T09:27:51.046369+0000","last_became_active":"2024-09-21T09:27:35.646084+0000","last_became_peered":"2024-09-21T09:27:35.646084+0000","last_unstale":"2024-09-21T09:27:51.046369+0000","last_undegraded":"2024-09-21T09:27:51.046369+0000","last_fullsized":"2024-09-21T09:27:51.046369+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:35:38.711926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2e","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.042536+0000","last_change":"2024-09-21T09:27:35.636697+0000","last_active":"2024-09-21T09:27:51.042536+0000","last_peered":"2024-09-21T09:27:51.042536+0000","last_clean":"2024-09-21T09:27:51.042536+0000","last_became_active":"2024-09-21T09:27:35.636438+0000","last_became_peered":"2024-09-21T09:27:35.636438+0000","last_unstale":"2024-09-21T09:27:51.042536+0000","last_undegraded":"2024-09-21T09:27:51.042536+0000","last_fullsized":"2024-09-21T09:27:51.042536+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:55:44.287544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,7],"acting":[2,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2d","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.050587+0000","last_change":"2024-09-21T09:27:35.636110+0000","last_active":"2024-09-21T09:27:51.050587+0000","last_peered":"2024-09-21T09:27:51.050587+0000","last_clean":"2024-09-21T09:27:51.050587+0000","last_became_active":"2024-09-21T09:27:35.635807+0000","last_became_peered":"2024-09-21T09:27:35.635807+0000","last_unstale":"2024-09-21T09:27:51.050587+0000","last_undegraded":"2024-09-21T09:27:51.050587+0000","last_fullsized":"2024-09-21T09:27:51.050587+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:22:43.942811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2c","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.045967+0000","last_change":"2024-09-21T09:27:35.650351+0000","last_active":"2024-09-21T09:27:51.045967+0000","last_peered":"2024-09-21T09:27:51.045967+0000","last_clean":"2024-09-21T09:27:51.045967+0000","last_became_active":"2024-09-21T09:27:35.649905+0000","last_became_peered":"2024-09-21T09:27:35.649905+0000","last_unstale":"2024-09-21T09:27:51.045967+0000","last_undegraded":"2024-09-21T09:27:51.045967+0000","last_fullsized":"2024-09-21T09:27:51.045967+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:12:31.390666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2b","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.049389+0000","last_change":"2024-09-21T09:27:35.643345+0000","last_active":"2024-09-21T09:27:51.049389+0000","last_peered":"2024-09-21T09:27:51.049389+0000","last_clean":"2024-09-21T09:27:51.049389+0000","last_became_active":"2024-09-21T09:27:35.642408+0000","last_became_peered":"2024-09-21T09:27:35.642408+0000","last_unstale":"2024-09-21T09:27:51.049389+0000","last_undegraded":"2024-09-21T09:27:51.049389+0000","last_fullsized":"2024-09-21T09:27:51.049389+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:34:59.698334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2a","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.047079+0000","last_change":"2024-09-21T09:27:35.645032+0000","last_active":"2024-09-21T09:27:51.047079+0000","last_peered":"2024-09-21T09:27:51.047079+0000","last_clean":"2024-09-21T09:27:51.047079+0000","last_became_active":"2024-09-21T09:27:35.644322+0000","last_became_peered":"2024-09-21T09:27:35.644322+0000","last_unstale":"2024-09-21T09:27:51.047079+0000","last_undegraded":"2024-09-21T09:27:51.047079+0000","last_fullsized":"2024-09-21T09:27:51.047079+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:10:14.511772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.29","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.048171+0000","last_change":"2024-09-21T09:27:35.638305+0000","last_active":"2024-09-21T09:27:51.048171+0000","last_peered":"2024-09-21T09:27:51.048171+0000","last_clean":"2024-09-21T09:27:51.048171+0000","last_became_active":"2024-09-21T09:27:35.638036+0000","last_became_peered":"2024-09-21T09:27:35.638036+0000","last_unstale":"2024-09-21T09:27:51.048171+0000","last_undegraded":"2024-09-21T09:27:51.048171+0000","last_fullsized":"2024-09-21T09:27:51.048171+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:38:54.627840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.28","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.046739+0000","last_change":"2024-09-21T09:27:35.637277+0000","last_active":"2024-09-21T09:27:51.046739+0000","last_peered":"2024-09-21T09:27:51.046739+0000","last_clean":"2024-09-21T09:27:51.046739+0000","last_became_active":"2024-09-21T09:27:35.636541+0000","last_became_peered":"2024-09-21T09:27:35.636541+0000","last_unstale":"2024-09-21T09:27:51.046739+0000","last_undegraded":"2024-09-21T09:27:51.046739+0000","last_fullsized":"2024-09-21T09:27:51.046739+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:45:47.090531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.27","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.049341+0000","last_change":"2024-09-21T09:27:35.638946+0000","last_active":"2024-09-21T09:27:51.049341+0000","last_peered":"2024-09-21T09:27:51.049341+0000","last_clean":"2024-09-21T09:27:51.049341+0000","last_became_active":"2024-09-21T09:27:35.638663+0000","last_became_peered":"2024-09-21T09:27:35.638663+0000","last_unstale":"2024-09-21T09:27:51.049341+0000","last_undegraded":"2024-09-21T09:27:51.049341+0000","last_fullsized":"2024-09-21T09:27:51.049341+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:49:57.591531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.26","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.045448+0000","last_change":"2024-09-21T09:27:35.638679+0000","last_active":"2024-09-21T09:27:51.045448+0000","last_peered":"2024-09-21T09:27:51.045448+0000","last_clean":"2024-09-21T09:27:51.045448+0000","last_became_active":"2024-09-21T09:27:35.638383+0000","last_became_peered":"2024-09-21T09:27:35.638383+0000","last_unstale":"2024-09-21T09:27:51.045448+0000","last_undegraded":"2024-09-21T09:27:51.045448+0000","last_fullsized":"2024-09-21T09:27:51.045448+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:16:11.663503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,4],"acting":[6,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.25","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.043664+0000","last_change":"2024-09-21T09:27:35.632894+0000","last_active":"2024-09-21T09:27:51.043664+0000","last_peered":"2024-09-21T09:27:51.043664+0000","last_clean":"2024-09-21T09:27:51.043664+0000","last_became_active":"2024-09-21T09:27:35.632569+0000","last_became_peered":"2024-09-21T09:27:35.632569+0000","last_unstale":"2024-09-21T09:27:51.043664+0000","last_undegraded":"2024-09-21T09:27:51.043664+0000","last_fullsized":"2024-09-21T09:27:51.043664+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:25:22.833535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.24","version":"0'0","reported_seq":23,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-21T09:27:52.079786+0000","last_change":"2024-09-21T09:27:52.079786+0000","last_active":"2024-09-21T09:27:52.079786+0000","last_peered":"2024-09-21T09:27:52.079786+0000","last_clean":"2024-09-21T09:27:52.079786+0000","last_became_active":"2024-09-21T09:27:52.079395+0000","last_became_peered":"2024-09-21T09:27:52.079395+0000","last_unstale":"2024-09-21T09:27:52.079786+0000","last_undegraded":"2024-09-21T09:27:52.079786+0000","last_fullsized":"2024-09-21T09:27:52.079786+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:17:07.388900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.23","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.049581+0000","last_change":"2024-09-21T09:27:35.634785+0000","last_active":"2024-09-21T09:27:51.049581+0000","last_peered":"2024-09-21T09:27:51.049581+0000","last_clean":"2024-09-21T09:27:51.049581+0000","last_became_active":"2024-09-21T09:27:35.634065+0000","last_became_peered":"2024-09-21T09:27:35.634065+0000","last_unstale":"2024-09-21T09:27:51.049581+0000","last_undegraded":"2024-09-21T09:27:51.049581+0000","last_fullsized":"2024-09-21T09:27:51.049581+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:54:21.482132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.22","version":"0'0","reported_seq":23,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-21T09:27:52.049243+0000","last_change":"2024-09-21T09:27:52.049243+0000","last_active":"2024-09-21T09:27:52.049243+0000","last_peered":"2024-09-21T09:27:52.049243+0000","last_clean":"2024-09-21T09:27:52.049243+0000","last_became_active":"2024-09-21T09:27:52.048885+0000","last_became_peered":"2024-09-21T09:27:52.048885+0000","last_unstale":"2024-09-21T09:27:52.049243+0000","last_undegraded":"2024-09-21T09:27:52.049243+0000","last_fullsized":"2024-09-21T09:27:52.049243+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:44:22.262414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.21","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.042354+0000","last_change":"2024-09-21T09:27:35.636222+0000","last_active":"2024-09-21T09:27:51.042354+0000","last_peered":"2024-09-21T09:27:51.042354+0000","last_clean":"2024-09-21T09:27:51.042354+0000","last_became_active":"2024-09-21T09:27:35.635935+0000","last_became_peered":"2024-09-21T09:27:35.635935+0000","last_unstale":"2024-09-21T09:27:51.042354+0000","last_undegraded":"2024-09-21T09:27:51.042354+0000","last_fullsized":"2024-09-21T09:27:51.042354+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:05:50.402224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.20","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.044885+0000","last_change":"2024-09-21T09:27:35.641740+0000","last_active":"2024-09-21T09:27:51.044885+0000","last_peered":"2024-09-21T09:27:51.044885+0000","last_clean":"2024-09-21T09:27:51.044885+0000","last_became_active":"2024-09-21T09:27:35.641409+0000","last_became_peered":"2024-09-21T09:27:35.641409+0000","last_unstale":"2024-09-21T09:27:51.044885+0000","last_undegraded":"2024-09-21T09:27:51.044885+0000","last_fullsized":"2024-09-21T09:27:51.044885+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:59:30.487730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1a","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.047761+0000","last_change":"2024-09-21T09:27:35.629525+0000","last_active":"2024-09-21T09:27:51.047761+0000","last_peered":"2024-09-21T09:27:51.047761+0000","last_clean":"2024-09-21T09:27:51.047761+0000","last_became_active":"2024-09-21T09:27:35.629167+0000","last_became_peered":"2024-09-21T09:27:35.629167+0000","last_unstale":"2024-09-21T09:27:51.047761+0000","last_undegraded":"2024-09-21T09:27:51.047761+0000","last_fullsized":"2024-09-21T09:27:51.047761+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:06:24.323825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1b","version":"57'1","reported_seq":17,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.048995+0000","last_change":"2024-09-21T09:27:32.265235+0000","last_active":"2024-09-21T09:27:51.048995+0000","last_peered":"2024-09-21T09:27:51.048995+0000","last_clean":"2024-09-21T09:27:51.048995+0000","last_became_active":"2024-09-21T09:27:32.264963+0000","last_became_peered":"2024-09-21T09:27:32.264963+0000","last_unstale":"2024-09-21T09:27:51.048995+0000","last_undegraded":"2024-09-21T09:27:51.048995+0000","last_fullsized":"2024-09-21T09:27:51.048995+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:09:03.927341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.049658+0000","last_change":"2024-09-21T09:27:35.642154+0000","last_active":"2024-09-21T09:27:51.049658+0000","last_peered":"2024-09-21T09:27:51.049658+0000","last_clean":"2024-09-21T09:27:51.049658+0000","last_became_active":"2024-09-21T09:27:35.641801+0000","last_became_peered":"2024-09-21T09:27:35.641801+0000","last_unstale":"2024-09-21T09:27:51.049658+0000","last_undegraded":"2024-09-21T09:27:51.049658+0000","last_fullsized":"2024-09-21T09:27:51.049658+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:50:45.075506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1a","version":"57'1","reported_seq":17,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.047707+0000","last_change":"2024-09-21T09:27:32.269358+0000","last_active":"2024-09-21T09:27:51.047707+0000","last_peered":"2024-09-21T09:27:51.047707+0000","last_clean":"2024-09-21T09:27:51.047707+0000","last_became_active":"2024-09-21T09:27:32.269026+0000","last_became_peered":"2024-09-21T09:27:32.269026+0000","last_unstale":"2024-09-21T09:27:51.047707+0000","last_undegraded":"2024-09-21T09:27:51.047707+0000","last_fullsized":"2024-09-21T09:27:51.047707+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:59:17.916370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.19","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.048323+0000","last_change":"2024-09-21T09:27:32.264601+0000","last_active":"2024-09-21T09:27:51.048323+0000","last_peered":"2024-09-21T09:27:51.048323+0000","last_clean":"2024-09-21T09:27:51.048323+0000","last_became_active":"2024-09-21T09:27:32.264381+0000","last_became_peered":"2024-09-21T09:27:32.264381+0000","last_unstale":"2024-09-21T09:27:51.048323+0000","last_undegraded":"2024-09-21T09:27:51.048323+0000","last_fullsized":"2024-09-21T09:27:51.048323+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:35:57.463156+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.048044+0000","last_change":"2024-09-21T09:27:35.634475+0000","last_active":"2024-09-21T09:27:51.048044+0000","last_peered":"2024-09-21T09:27:51.048044+0000","last_clean":"2024-09-21T09:27:51.048044+0000","last_became_active":"2024-09-21T09:27:35.634224+0000","last_became_peered":"2024-09-21T09:27:35.634224+0000","last_unstale":"2024-09-21T09:27:51.048044+0000","last_undegraded":"2024-09-21T09:27:51.048044+0000","last_fullsized":"2024-09-21T09:27:51.048044+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:38:21.241590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.18","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.046834+0000","last_change":"2024-09-21T09:27:32.267805+0000","last_active":"2024-09-21T09:27:51.046834+0000","last_peered":"2024-09-21T09:27:51.046834+0000","last_clean":"2024-09-21T09:27:51.046834+0000","last_became_active":"2024-09-21T09:27:32.267484+0000","last_became_peered":"2024-09-21T09:27:32.267484+0000","last_unstale":"2024-09-21T09:27:51.046834+0000","last_undegraded":"2024-09-21T09:27:51.046834+0000","last_fullsized":"2024-09-21T09:27:51.046834+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:00:43.105362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,7],"acting":[5,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.042506+0000","last_change":"2024-09-21T09:27:35.631350+0000","last_active":"2024-09-21T09:27:51.042506+0000","last_peered":"2024-09-21T09:27:51.042506+0000","last_clean":"2024-09-21T09:27:51.042506+0000","last_became_active":"2024-09-21T09:27:35.630723+0000","last_became_peered":"2024-09-21T09:27:35.630723+0000","last_unstale":"2024-09-21T09:27:51.042506+0000","last_undegraded":"2024-09-21T09:27:51.042506+0000","last_fullsized":"2024-09-21T09:27:51.042506+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:46:32.135037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.17","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.045755+0000","last_change":"2024-09-21T09:27:32.266307+0000","last_active":"2024-09-21T09:27:51.045755+0000","last_peered":"2024-09-21T09:27:51.045755+0000","last_clean":"2024-09-21T09:27:51.045755+0000","last_became_active":"2024-09-21T09:27:32.265995+0000","last_became_peered":"2024-09-21T09:27:32.265995+0000","last_unstale":"2024-09-21T09:27:51.045755+0000","last_undegraded":"2024-09-21T09:27:51.045755+0000","last_fullsized":"2024-09-21T09:27:51.045755+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:24:28.036534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.045841+0000","last_change":"2024-09-21T09:27:35.647952+0000","last_active":"2024-09-21T09:27:51.045841+0000","last_peered":"2024-09-21T09:27:51.045841+0000","last_clean":"2024-09-21T09:27:51.045841+0000","last_became_active":"2024-09-21T09:27:35.647649+0000","last_became_peered":"2024-09-21T09:27:35.647649+0000","last_unstale":"2024-09-21T09:27:51.045841+0000","last_undegraded":"2024-09-21T09:27:51.045841+0000","last_fullsized":"2024-09-21T09:27:51.045841+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:12:29.850835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.16","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.046149+0000","last_change":"2024-09-21T09:27:32.269137+0000","last_active":"2024-09-21T09:27:51.046149+0000","last_peered":"2024-09-21T09:27:51.046149+0000","last_clean":"2024-09-21T09:27:51.046149+0000","last_became_active":"2024-09-21T09:27:32.268832+0000","last_became_peered":"2024-09-21T09:27:32.268832+0000","last_unstale":"2024-09-21T09:27:51.046149+0000","last_undegraded":"2024-09-21T09:27:51.046149+0000","last_fullsized":"2024-09-21T09:27:51.046149+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:51:58.668390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":10,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-21T09:27:52.050121+0000","last_change":"2024-09-21T09:27:52.050121+0000","last_active":"2024-09-21T09:27:52.050121+0000","last_peered":"2024-09-21T09:27:52.050121+0000","last_clean":"2024-09-21T09:27:52.050121+0000","last_became_active":"2024-09-21T09:27:52.049549+0000","last_became_peered":"2024-09-21T09:27:52.049549+0000","last_unstale":"2024-09-21T09:27:52.050121+0000","last_undegraded":"2024-09-21T09:27:52.050121+0000","last_fullsized":"2024-09-21T09:27:52.050121+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:25:27.303500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.15","version":"57'3","reported_seq":19,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.043540+0000","last_change":"2024-09-21T09:27:32.270145+0000","last_active":"2024-09-21T09:27:51.043540+0000","last_peered":"2024-09-21T09:27:51.043540+0000","last_clean":"2024-09-21T09:27:51.043540+0000","last_became_active":"2024-09-21T09:27:32.269808+0000","last_became_peered":"2024-09-21T09:27:32.269808+0000","last_unstale":"2024-09-21T09:27:51.043540+0000","last_undegraded":"2024-09-21T09:27:51.043540+0000","last_fullsized":"2024-09-21T09:27:51.043540+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:35:53.397343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.048177+0000","last_change":"2024-09-21T09:27:35.640314+0000","last_active":"2024-09-21T09:27:51.048177+0000","last_peered":"2024-09-21T09:27:51.048177+0000","last_clean":"2024-09-21T09:27:51.048177+0000","last_became_active":"2024-09-21T09:27:35.639976+0000","last_became_peered":"2024-09-21T09:27:35.639976+0000","last_unstale":"2024-09-21T09:27:51.048177+0000","last_undegraded":"2024-09-21T09:27:51.048177+0000","last_fullsized":"2024-09-21T09:27:51.048177+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:55:21.229874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.14","version":"57'6","reported_seq":22,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.046070+0000","last_change":"2024-09-21T09:27:32.266499+0000","last_active":"2024-09-21T09:27:51.046070+0000","last_peered":"2024-09-21T09:27:51.046070+0000","last_clean":"2024-09-21T09:27:51.046070+0000","last_became_active":"2024-09-21T09:27:32.265712+0000","last_became_peered":"2024-09-21T09:27:32.265712+0000","last_unstale":"2024-09-21T09:27:51.046070+0000","last_undegraded":"2024-09-21T09:27:51.046070+0000","last_fullsized":"2024-09-21T09:27:51.046070+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:19:17.014990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1552,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.050296+0000","last_change":"2024-09-21T09:27:35.641626+0000","last_active":"2024-09-21T09:27:51.050296+0000","last_peered":"2024-09-21T09:27:51.050296+0000","last_clean":"2024-09-21T09:27:51.050296+0000","last_became_active":"2024-09-21T09:27:35.641272+0000","last_became_peered":"2024-09-21T09:27:35.641272+0000","last_unstale":"2024-09-21T09:27:51.050296+0000","last_undegraded":"2024-09-21T09:27:51.050296+0000","last_fullsized":"2024-09-21T09:27:51.050296+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:15:05.108072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.13","version":"57'2","reported_seq":18,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.049744+0000","last_change":"2024-09-21T09:27:32.267412+0000","last_active":"2024-09-21T09:27:51.049744+0000","last_peered":"2024-09-21T09:27:51.049744+0000","last_clean":"2024-09-21T09:27:51.049744+0000","last_became_active":"2024-09-21T09:27:32.267108+0000","last_became_peered":"2024-09-21T09:27:32.267108+0000","last_unstale":"2024-09-21T09:27:51.049744+0000","last_undegraded":"2024-09-21T09:27:51.049744+0000","last_fullsized":"2024-09-21T09:27:51.049744+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:54:11.228702+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.049165+0000","last_change":"2024-09-21T09:27:35.641572+0000","last_active":"2024-09-21T09:27:51.049165+0000","last_peered":"2024-09-21T09:27:51.049165+0000","last_clean":"2024-09-21T09:27:51.049165+0000","last_became_active":"2024-09-21T09:27:35.641185+0000","last_became_peered":"2024-09-21T09:27:35.641185+0000","last_unstale":"2024-09-21T09:27:51.049165+0000","last_undegraded":"2024-09-21T09:27:51.049165+0000","last_fullsized":"2024-09-21T09:27:51.049165+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:52:30.993491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.12","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.048014+0000","last_change":"2024-09-21T09:27:32.267644+0000","last_active":"2024-09-21T09:27:51.048014+0000","last_peered":"2024-09-21T09:27:51.048014+0000","last_clean":"2024-09-21T09:27:51.048014+0000","last_became_active":"2024-09-21T09:27:32.267232+0000","last_became_peered":"2024-09-21T09:27:32.267232+0000","last_unstale":"2024-09-21T09:27:51.048014+0000","last_undegraded":"2024-09-21T09:27:51.048014+0000","last_fullsized":"2024-09-21T09:27:51.048014+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:14:19.578564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.050123+0000","last_change":"2024-09-21T09:27:35.644217+0000","last_active":"2024-09-21T09:27:51.050123+0000","last_peered":"2024-09-21T09:27:51.050123+0000","last_clean":"2024-09-21T09:27:51.050123+0000","last_became_active":"2024-09-21T09:27:35.643681+0000","last_became_peered":"2024-09-21T09:27:35.643681+0000","last_unstale":"2024-09-21T09:27:51.050123+0000","last_undegraded":"2024-09-21T09:27:51.050123+0000","last_fullsized":"2024-09-21T09:27:51.050123+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:07:58.418951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.11","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.045294+0000","last_change":"2024-09-21T09:27:32.270501+0000","last_active":"2024-09-21T09:27:51.045294+0000","last_peered":"2024-09-21T09:27:51.045294+0000","last_clean":"2024-09-21T09:27:51.045294+0000","last_became_active":"2024-09-21T09:27:32.269808+0000","last_became_peered":"2024-09-21T09:27:32.269808+0000","last_unstale":"2024-09-21T09:27:51.045294+0000","last_undegraded":"2024-09-21T09:27:51.045294+0000","last_fullsized":"2024-09-21T09:27:51.045294+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:28:03.705631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.10","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.044945+0000","last_change":"2024-09-21T09:27:35.641508+0000","last_active":"2024-09-21T09:27:51.044945+0000","last_peered":"2024-09-21T09:27:51.044945+0000","last_clean":"2024-09-21T09:27:51.044945+0000","last_became_active":"2024-09-21T09:27:35.640996+0000","last_became_peered":"2024-09-21T09:27:35.640996+0000","last_unstale":"2024-09-21T09:27:51.044945+0000","last_undegraded":"2024-09-21T09:27:51.044945+0000","last_fullsized":"2024-09-21T09:27:51.044945+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:02:24.674295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.10","version":"57'8","reported_seq":24,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.042476+0000","last_change":"2024-09-21T09:27:32.264620+0000","last_active":"2024-09-21T09:27:51.042476+0000","last_peered":"2024-09-21T09:27:51.042476+0000","last_clean":"2024-09-21T09:27:51.042476+0000","last_became_active":"2024-09-21T09:27:32.261324+0000","last_became_peered":"2024-09-21T09:27:32.261324+0000","last_unstale":"2024-09-21T09:27:51.042476+0000","last_undegraded":"2024-09-21T09:27:51.042476+0000","last_fullsized":"2024-09-21T09:27:51.042476+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:11:04.933873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.050795+0000","last_change":"2024-09-21T09:27:35.641576+0000","last_active":"2024-09-21T09:27:51.050795+0000","last_peered":"2024-09-21T09:27:51.050795+0000","last_clean":"2024-09-21T09:27:51.050795+0000","last_became_active":"2024-09-21T09:27:35.641261+0000","last_became_peered":"2024-09-21T09:27:35.641261+0000","last_unstale":"2024-09-21T09:27:51.050795+0000","last_undegraded":"2024-09-21T09:27:51.050795+0000","last_fullsized":"2024-09-21T09:27:51.050795+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:59:48.868788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.f","version":"57'1","reported_seq":17,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.049183+0000","last_change":"2024-09-21T09:27:32.270055+0000","last_active":"2024-09-21T09:27:51.049183+0000","last_peered":"2024-09-21T09:27:51.049183+0000","last_clean":"2024-09-21T09:27:51.049183+0000","last_became_active":"2024-09-21T09:27:32.269748+0000","last_became_peered":"2024-09-21T09:27:32.269748+0000","last_unstale":"2024-09-21T09:27:51.049183+0000","last_undegraded":"2024-09-21T09:27:51.049183+0000","last_fullsized":"2024-09-21T09:27:51.049183+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:50:54.853319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.050943+0000","last_change":"2024-09-21T09:27:35.641790+0000","last_active":"2024-09-21T09:27:51.050943+0000","last_peered":"2024-09-21T09:27:51.050943+0000","last_clean":"2024-09-21T09:27:51.050943+0000","last_became_active":"2024-09-21T09:27:35.641348+0000","last_became_peered":"2024-09-21T09:27:35.641348+0000","last_unstale":"2024-09-21T09:27:51.050943+0000","last_undegraded":"2024-09-21T09:27:51.050943+0000","last_fullsized":"2024-09-21T09:27:51.050943+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:25:10.635353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.e","version":"57'2","reported_seq":18,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.042989+0000","last_change":"2024-09-21T09:27:32.264276+0000","last_active":"2024-09-21T09:27:51.042989+0000","last_peered":"2024-09-21T09:27:51.042989+0000","last_clean":"2024-09-21T09:27:51.042989+0000","last_became_active":"2024-09-21T09:27:32.263984+0000","last_became_peered":"2024-09-21T09:27:32.263984+0000","last_unstale":"2024-09-21T09:27:51.042989+0000","last_undegraded":"2024-09-21T09:27:51.042989+0000","last_fullsized":"2024-09-21T09:27:51.042989+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:18:50.045457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.051063+0000","last_change":"2024-09-21T09:27:35.649353+0000","last_active":"2024-09-21T09:27:51.051063+0000","last_peered":"2024-09-21T09:27:51.051063+0000","last_clean":"2024-09-21T09:27:51.051063+0000","last_became_active":"2024-09-21T09:27:35.649033+0000","last_became_peered":"2024-09-21T09:27:35.649033+0000","last_unstale":"2024-09-21T09:27:51.051063+0000","last_undegraded":"2024-09-21T09:27:51.051063+0000","last_fullsized":"2024-09-21T09:27:51.051063+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:13:12.611019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.d","version":"57'1","reported_seq":17,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.043640+0000","last_change":"2024-09-21T09:27:32.260033+0000","last_active":"2024-09-21T09:27:51.043640+0000","last_peered":"2024-09-21T09:27:51.043640+0000","last_clean":"2024-09-21T09:27:51.043640+0000","last_became_active":"2024-09-21T09:27:32.259209+0000","last_became_peered":"2024-09-21T09:27:32.259209+0000","last_unstale":"2024-09-21T09:27:51.043640+0000","last_undegraded":"2024-09-21T09:27:51.043640+0000","last_fullsized":"2024-09-21T09:27:51.043640+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:16:09.835879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.045659+0000","last_change":"2024-09-21T09:27:35.639076+0000","last_active":"2024-09-21T09:27:51.045659+0000","last_peered":"2024-09-21T09:27:51.045659+0000","last_clean":"2024-09-21T09:27:51.045659+0000","last_became_active":"2024-09-21T09:27:35.638802+0000","last_became_peered":"2024-09-21T09:27:35.638802+0000","last_unstale":"2024-09-21T09:27:51.045659+0000","last_undegraded":"2024-09-21T09:27:51.045659+0000","last_fullsized":"2024-09-21T09:27:51.045659+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:36:18.524320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.c","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.042975+0000","last_change":"2024-09-21T09:27:32.270060+0000","last_active":"2024-09-21T09:27:51.042975+0000","last_peered":"2024-09-21T09:27:51.042975+0000","last_clean":"2024-09-21T09:27:51.042975+0000","last_became_active":"2024-09-21T09:27:32.269717+0000","last_became_peered":"2024-09-21T09:27:32.269717+0000","last_unstale":"2024-09-21T09:27:51.042975+0000","last_undegraded":"2024-09-21T09:27:51.042975+0000","last_fullsized":"2024-09-21T09:27:51.042975+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:59:27.725031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.050384+0000","last_change":"2024-09-21T09:27:35.640059+0000","last_active":"2024-09-21T09:27:51.050384+0000","last_peered":"2024-09-21T09:27:51.050384+0000","last_clean":"2024-09-21T09:27:51.050384+0000","last_became_active":"2024-09-21T09:27:35.639746+0000","last_became_peered":"2024-09-21T09:27:35.639746+0000","last_unstale":"2024-09-21T09:27:51.050384+0000","last_undegraded":"2024-09-21T09:27:51.050384+0000","last_fullsized":"2024-09-21T09:27:51.050384+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:33:15.402410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"21'72","reported_seq":25,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.050815+0000","last_change":"2024-09-21T09:27:08.460023+0000","last_active":"2024-09-21T09:27:51.050815+0000","last_peered":"2024-09-21T09:27:51.050815+0000","last_clean":"2024-09-21T09:27:51.050815+0000","last_became_active":"2024-09-21T09:27:08.454770+0000","last_became_peered":"2024-09-21T09:27:08.454770+0000","last_unstale":"2024-09-21T09:27:51.050815+0000","last_undegraded":"2024-09-21T09:27:51.050815+0000","last_fullsized":"2024-09-21T09:27:51.050815+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:25:07.646659+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:25:07.646659+0000","last_clean_scrub_stamp":"2024-09-21T09:25:07.646659+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:08:45.658466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":26,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-21T09:27:52.081088+0000","last_change":"2024-09-21T09:27:52.081088+0000","last_active":"2024-09-21T09:27:52.081088+0000","last_peered":"2024-09-21T09:27:52.081088+0000","last_clean":"2024-09-21T09:27:52.081088+0000","last_became_active":"2024-09-21T09:27:52.080680+0000","last_became_peered":"2024-09-21T09:27:52.080680+0000","last_unstale":"2024-09-21T09:27:52.081088+0000","last_undegraded":"2024-09-21T09:27:52.081088+0000","last_fullsized":"2024-09-21T09:27:52.081088+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:01:04.317396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.048243+0000","last_change":"2024-09-21T09:27:35.636200+0000","last_active":"2024-09-21T09:27:51.048243+0000","last_peered":"2024-09-21T09:27:51.048243+0000","last_clean":"2024-09-21T09:27:51.048243+0000","last_became_active":"2024-09-21T09:27:35.635939+0000","last_became_peered":"2024-09-21T09:27:35.635939+0000","last_unstale":"2024-09-21T09:27:51.048243+0000","last_undegraded":"2024-09-21T09:27:51.048243+0000","last_fullsized":"2024-09-21T09:27:51.048243+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:02:06.551561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.050696+0000","last_change":"2024-09-21T09:27:32.268500+0000","last_active":"2024-09-21T09:27:51.050696+0000","last_peered":"2024-09-21T09:27:51.050696+0000","last_clean":"2024-09-21T09:27:51.050696+0000","last_became_active":"2024-09-21T09:27:32.268026+0000","last_became_peered":"2024-09-21T09:27:32.268026+0000","last_unstale":"2024-09-21T09:27:51.050696+0000","last_undegraded":"2024-09-21T09:27:51.050696+0000","last_fullsized":"2024-09-21T09:27:51.050696+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:44:43.463552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.048336+0000","last_change":"2024-09-21T09:27:35.634025+0000","last_active":"2024-09-21T09:27:51.048336+0000","last_peered":"2024-09-21T09:27:51.048336+0000","last_clean":"2024-09-21T09:27:51.048336+0000","last_became_active":"2024-09-21T09:27:35.633746+0000","last_became_peered":"2024-09-21T09:27:35.633746+0000","last_unstale":"2024-09-21T09:27:51.048336+0000","last_undegraded":"2024-09-21T09:27:51.048336+0000","last_fullsized":"2024-09-21T09:27:51.048336+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:18:40.511292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.042366+0000","last_change":"2024-09-21T09:27:32.262645+0000","last_active":"2024-09-21T09:27:51.042366+0000","last_peered":"2024-09-21T09:27:51.042366+0000","last_clean":"2024-09-21T09:27:51.042366+0000","last_became_active":"2024-09-21T09:27:32.262373+0000","last_became_peered":"2024-09-21T09:27:32.262373+0000","last_unstale":"2024-09-21T09:27:51.042366+0000","last_undegraded":"2024-09-21T09:27:51.042366+0000","last_fullsized":"2024-09-21T09:27:51.042366+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:37:23.684561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.042540+0000","last_change":"2024-09-21T09:27:35.636105+0000","last_active":"2024-09-21T09:27:51.042540+0000","last_peered":"2024-09-21T09:27:51.042540+0000","last_clean":"2024-09-21T09:27:51.042540+0000","last_became_active":"2024-09-21T09:27:35.635805+0000","last_became_peered":"2024-09-21T09:27:35.635805+0000","last_unstale":"2024-09-21T09:27:51.042540+0000","last_undegraded":"2024-09-21T09:27:51.042540+0000","last_fullsized":"2024-09-21T09:27:51.042540+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:22:53.711130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.048097+0000","last_change":"2024-09-21T09:27:32.265210+0000","last_active":"2024-09-21T09:27:51.048097+0000","last_peered":"2024-09-21T09:27:51.048097+0000","last_clean":"2024-09-21T09:27:51.048097+0000","last_became_active":"2024-09-21T09:27:32.264740+0000","last_became_peered":"2024-09-21T09:27:32.264740+0000","last_unstale":"2024-09-21T09:27:51.048097+0000","last_undegraded":"2024-09-21T09:27:51.048097+0000","last_fullsized":"2024-09-21T09:27:51.048097+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:14:01.491316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.048823+0000","last_change":"2024-09-21T09:27:35.638582+0000","last_active":"2024-09-21T09:27:51.048823+0000","last_peered":"2024-09-21T09:27:51.048823+0000","last_clean":"2024-09-21T09:27:51.048823+0000","last_became_active":"2024-09-21T09:27:35.638269+0000","last_became_peered":"2024-09-21T09:27:35.638269+0000","last_unstale":"2024-09-21T09:27:51.048823+0000","last_undegraded":"2024-09-21T09:27:51.048823+0000","last_fullsized":"2024-09-21T09:27:51.048823+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:52:20.849743+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.043473+0000","last_change":"2024-09-21T09:27:32.264482+0000","last_active":"2024-09-21T09:27:51.043473+0000","last_peered":"2024-09-21T09:27:51.043473+0000","last_clean":"2024-09-21T09:27:51.043473+0000","last_became_active":"2024-09-21T09:27:32.264201+0000","last_became_peered":"2024-09-21T09:27:32.264201+0000","last_unstale":"2024-09-21T09:27:51.043473+0000","last_undegraded":"2024-09-21T09:27:51.043473+0000","last_fullsized":"2024-09-21T09:27:51.043473+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:37:15.565839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.045599+0000","last_change":"2024-09-21T09:27:35.634935+0000","last_active":"2024-09-21T09:27:51.045599+0000","last_peered":"2024-09-21T09:27:51.045599+0000","last_clean":"2024-09-21T09:27:51.045599+0000","last_became_active":"2024-09-21T09:27:35.634594+0000","last_became_peered":"2024-09-21T09:27:35.634594+0000","last_unstale":"2024-09-21T09:27:51.045599+0000","last_undegraded":"2024-09-21T09:27:51.045599+0000","last_fullsized":"2024-09-21T09:27:51.045599+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:53:03.606929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.050709+0000","last_change":"2024-09-21T09:27:32.274427+0000","last_active":"2024-09-21T09:27:51.050709+0000","last_peered":"2024-09-21T09:27:51.050709+0000","last_clean":"2024-09-21T09:27:51.050709+0000","last_became_active":"2024-09-21T09:27:32.274058+0000","last_became_peered":"2024-09-21T09:27:32.274058+0000","last_unstale":"2024-09-21T09:27:51.050709+0000","last_undegraded":"2024-09-21T09:27:51.050709+0000","last_fullsized":"2024-09-21T09:27:51.050709+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:09:38.714482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.043384+0000","last_change":"2024-09-21T09:27:35.643081+0000","last_active":"2024-09-21T09:27:51.043384+0000","last_peered":"2024-09-21T09:27:51.043384+0000","last_clean":"2024-09-21T09:27:51.043384+0000","last_became_active":"2024-09-21T09:27:35.642792+0000","last_became_peered":"2024-09-21T09:27:35.642792+0000","last_unstale":"2024-09-21T09:27:51.043384+0000","last_undegraded":"2024-09-21T09:27:51.043384+0000","last_fullsized":"2024-09-21T09:27:51.043384+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:05:03.157102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"57'1","reported_seq":17,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.043735+0000","last_change":"2024-09-21T09:27:32.265476+0000","last_active":"2024-09-21T09:27:51.043735+0000","last_peered":"2024-09-21T09:27:51.043735+0000","last_clean":"2024-09-21T09:27:51.043735+0000","last_became_active":"2024-09-21T09:27:32.265171+0000","last_became_peered":"2024-09-21T09:27:32.265171+0000","last_unstale":"2024-09-21T09:27:51.043735+0000","last_undegraded":"2024-09-21T09:27:51.043735+0000","last_fullsized":"2024-09-21T09:27:51.043735+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:42:48.279357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.049424+0000","last_change":"2024-09-21T09:27:35.629856+0000","last_active":"2024-09-21T09:27:51.049424+0000","last_peered":"2024-09-21T09:27:51.049424+0000","last_clean":"2024-09-21T09:27:51.049424+0000","last_became_active":"2024-09-21T09:27:35.629345+0000","last_became_peered":"2024-09-21T09:27:35.629345+0000","last_unstale":"2024-09-21T09:27:51.049424+0000","last_undegraded":"2024-09-21T09:27:51.049424+0000","last_fullsized":"2024-09-21T09:27:51.049424+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:30:14.361278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"57'2","reported_seq":18,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.045847+0000","last_change":"2024-09-21T09:27:32.264763+0000","last_active":"2024-09-21T09:27:51.045847+0000","last_peered":"2024-09-21T09:27:51.045847+0000","last_clean":"2024-09-21T09:27:51.045847+0000","last_became_active":"2024-09-21T09:27:32.264444+0000","last_became_peered":"2024-09-21T09:27:32.264444+0000","last_unstale":"2024-09-21T09:27:51.045847+0000","last_undegraded":"2024-09-21T09:27:51.045847+0000","last_fullsized":"2024-09-21T09:27:51.045847+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:01:52.871773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.050094+0000","last_change":"2024-09-21T09:27:35.637964+0000","last_active":"2024-09-21T09:27:51.050094+0000","last_peered":"2024-09-21T09:27:51.050094+0000","last_clean":"2024-09-21T09:27:51.050094+0000","last_became_active":"2024-09-21T09:27:35.634680+0000","last_became_peered":"2024-09-21T09:27:35.634680+0000","last_unstale":"2024-09-21T09:27:51.050094+0000","last_undegraded":"2024-09-21T09:27:51.050094+0000","last_fullsized":"2024-09-21T09:27:51.050094+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:50:36.642394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.8","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.049792+0000","last_change":"2024-09-21T09:27:32.270810+0000","last_active":"2024-09-21T09:27:51.049792+0000","last_peered":"2024-09-21T09:27:51.049792+0000","last_clean":"2024-09-21T09:27:51.049792+0000","last_became_active":"2024-09-21T09:27:32.268720+0000","last_became_peered":"2024-09-21T09:27:32.268720+0000","last_unstale":"2024-09-21T09:27:51.049792+0000","last_undegraded":"2024-09-21T09:27:51.049792+0000","last_fullsized":"2024-09-21T09:27:51.049792+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:58:54.820070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.047482+0000","last_change":"2024-09-21T09:27:35.637942+0000","last_active":"2024-09-21T09:27:51.047482+0000","last_peered":"2024-09-21T09:27:51.047482+0000","last_clean":"2024-09-21T09:27:51.047482+0000","last_became_active":"2024-09-21T09:27:35.637064+0000","last_became_peered":"2024-09-21T09:27:35.637064+0000","last_unstale":"2024-09-21T09:27:51.047482+0000","last_undegraded":"2024-09-21T09:27:51.047482+0000","last_fullsized":"2024-09-21T09:27:51.047482+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:16:53.911245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.9","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.042877+0000","last_change":"2024-09-21T09:27:32.262848+0000","last_active":"2024-09-21T09:27:51.042877+0000","last_peered":"2024-09-21T09:27:51.042877+0000","last_clean":"2024-09-21T09:27:51.042877+0000","last_became_active":"2024-09-21T09:27:32.262555+0000","last_became_peered":"2024-09-21T09:27:32.262555+0000","last_unstale":"2024-09-21T09:27:51.042877+0000","last_undegraded":"2024-09-21T09:27:51.042877+0000","last_fullsized":"2024-09-21T09:27:51.042877+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:27:02.670310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.048071+0000","last_change":"2024-09-21T09:27:35.625899+0000","last_active":"2024-09-21T09:27:51.048071+0000","last_peered":"2024-09-21T09:27:51.048071+0000","last_clean":"2024-09-21T09:27:51.048071+0000","last_became_active":"2024-09-21T09:27:35.625619+0000","last_became_peered":"2024-09-21T09:27:35.625619+0000","last_unstale":"2024-09-21T09:27:51.048071+0000","last_undegraded":"2024-09-21T09:27:51.048071+0000","last_fullsized":"2024-09-21T09:27:51.048071+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:38:48.577776+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.a","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.043557+0000","last_change":"2024-09-21T09:27:32.264667+0000","last_active":"2024-09-21T09:27:51.043557+0000","last_peered":"2024-09-21T09:27:51.043557+0000","last_clean":"2024-09-21T09:27:51.043557+0000","last_became_active":"2024-09-21T09:27:32.264353+0000","last_became_peered":"2024-09-21T09:27:32.264353+0000","last_unstale":"2024-09-21T09:27:51.043557+0000","last_undegraded":"2024-09-21T09:27:51.043557+0000","last_fullsized":"2024-09-21T09:27:51.043557+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:21:53.938525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"0'0","reported_seq":23,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-21T09:27:52.081164+0000","last_change":"2024-09-21T09:27:52.081164+0000","last_active":"2024-09-21T09:27:52.081164+0000","last_peered":"2024-09-21T09:27:52.081164+0000","last_clean":"2024-09-21T09:27:52.081164+0000","last_became_active":"2024-09-21T09:27:52.080726+0000","last_became_peered":"2024-09-21T09:27:52.080726+0000","last_unstale":"2024-09-21T09:27:52.081164+0000","last_undegraded":"2024-09-21T09:27:52.081164+0000","last_fullsized":"2024-09-21T09:27:52.081164+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:37:01.620726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.b","version":"57'2","reported_seq":18,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.050288+0000","last_change":"2024-09-21T09:27:32.274404+0000","last_active":"2024-09-21T09:27:51.050288+0000","last_peered":"2024-09-21T09:27:51.050288+0000","last_clean":"2024-09-21T09:27:51.050288+0000","last_became_active":"2024-09-21T09:27:32.274109+0000","last_became_peered":"2024-09-21T09:27:32.274109+0000","last_unstale":"2024-09-21T09:27:51.050288+0000","last_undegraded":"2024-09-21T09:27:51.050288+0000","last_fullsized":"2024-09-21T09:27:51.050288+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:24:06.607203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":23,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-21T09:27:52.050465+0000","last_change":"2024-09-21T09:27:52.050465+0000","last_active":"2024-09-21T09:27:52.050465+0000","last_peered":"2024-09-21T09:27:52.050465+0000","last_clean":"2024-09-21T09:27:52.050465+0000","last_became_active":"2024-09-21T09:27:52.050052+0000","last_became_peered":"2024-09-21T09:27:52.050052+0000","last_unstale":"2024-09-21T09:27:52.050465+0000","last_undegraded":"2024-09-21T09:27:52.050465+0000","last_fullsized":"2024-09-21T09:27:52.050465+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:58:35.370217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1d","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.045494+0000","last_change":"2024-09-21T09:27:35.637946+0000","last_active":"2024-09-21T09:27:51.045494+0000","last_peered":"2024-09-21T09:27:51.045494+0000","last_clean":"2024-09-21T09:27:51.045494+0000","last_became_active":"2024-09-21T09:27:35.637664+0000","last_became_peered":"2024-09-21T09:27:35.637664+0000","last_unstale":"2024-09-21T09:27:51.045494+0000","last_undegraded":"2024-09-21T09:27:51.045494+0000","last_fullsized":"2024-09-21T09:27:51.045494+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:22:17.260192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1c","version":"57'2","reported_seq":18,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.050172+0000","last_change":"2024-09-21T09:27:32.268336+0000","last_active":"2024-09-21T09:27:51.050172+0000","last_peered":"2024-09-21T09:27:51.050172+0000","last_clean":"2024-09-21T09:27:51.050172+0000","last_became_active":"2024-09-21T09:27:32.268038+0000","last_became_peered":"2024-09-21T09:27:32.268038+0000","last_unstale":"2024-09-21T09:27:51.050172+0000","last_undegraded":"2024-09-21T09:27:51.050172+0000","last_fullsized":"2024-09-21T09:27:51.050172+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:57:13.991593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1c","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.045543+0000","last_change":"2024-09-21T09:27:35.642780+0000","last_active":"2024-09-21T09:27:51.045543+0000","last_peered":"2024-09-21T09:27:51.045543+0000","last_clean":"2024-09-21T09:27:51.045543+0000","last_became_active":"2024-09-21T09:27:35.642487+0000","last_became_peered":"2024-09-21T09:27:35.642487+0000","last_unstale":"2024-09-21T09:27:51.045543+0000","last_undegraded":"2024-09-21T09:27:51.045543+0000","last_fullsized":"2024-09-21T09:27:51.045543+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:28:54.864609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1d","version":"57'2","reported_seq":18,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.050487+0000","last_change":"2024-09-21T09:27:32.270398+0000","last_active":"2024-09-21T09:27:51.050487+0000","last_peered":"2024-09-21T09:27:51.050487+0000","last_clean":"2024-09-21T09:27:51.050487+0000","last_became_active":"2024-09-21T09:27:32.269469+0000","last_became_peered":"2024-09-21T09:27:32.269469+0000","last_unstale":"2024-09-21T09:27:51.050487+0000","last_undegraded":"2024-09-21T09:27:51.050487+0000","last_fullsized":"2024-09-21T09:27:51.050487+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:11:17.381607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1f","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.050305+0000","last_change":"2024-09-21T09:27:35.646662+0000","last_active":"2024-09-21T09:27:51.050305+0000","last_peered":"2024-09-21T09:27:51.050305+0000","last_clean":"2024-09-21T09:27:51.050305+0000","last_became_active":"2024-09-21T09:27:35.646141+0000","last_became_peered":"2024-09-21T09:27:35.646141+0000","last_unstale":"2024-09-21T09:27:51.050305+0000","last_undegraded":"2024-09-21T09:27:51.050305+0000","last_fullsized":"2024-09-21T09:27:51.050305+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:22:15.834922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1e","version":"57'1","reported_seq":17,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.049450+0000","last_change":"2024-09-21T09:27:32.270481+0000","last_active":"2024-09-21T09:27:51.049450+0000","last_peered":"2024-09-21T09:27:51.049450+0000","last_clean":"2024-09-21T09:27:51.049450+0000","last_became_active":"2024-09-21T09:27:32.269813+0000","last_became_peered":"2024-09-21T09:27:32.269813+0000","last_unstale":"2024-09-21T09:27:51.049450+0000","last_undegraded":"2024-09-21T09:27:51.049450+0000","last_fullsized":"2024-09-21T09:27:51.049450+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:28:41.266917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.048769+0000","last_change":"2024-09-21T09:27:35.636905+0000","last_active":"2024-09-21T09:27:51.048769+0000","last_peered":"2024-09-21T09:27:51.048769+0000","last_clean":"2024-09-21T09:27:51.048769+0000","last_became_active":"2024-09-21T09:27:35.636625+0000","last_became_peered":"2024-09-21T09:27:35.636625+0000","last_unstale":"2024-09-21T09:27:51.048769+0000","last_undegraded":"2024-09-21T09:27:51.048769+0000","last_fullsized":"2024-09-21T09:27:51.048769+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:19:39.022799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1f","version":"57'4","reported_seq":20,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:27:51.050451+0000","last_change":"2024-09-21T09:27:32.267938+0000","last_active":"2024-09-21T09:27:51.050451+0000","last_peered":"2024-09-21T09:27:51.050451+0000","last_clean":"2024-09-21T09:27:51.050451+0000","last_became_active":"2024-09-21T09:27:32.267583+0000","last_became_peered":"2024-09-21T09:27:32.267583+0000","last_unstale":"2024-09-21T09:27:51.050451+0000","last_undegraded":"2024-09-21T09:27:51.050451+0000","last_fullsized":"2024-09-21T09:27:51.050451+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:35:00.591469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":3,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":192,"acting":192,"num_store_stats":8},{"poolid":2,"num_pg":32,"stat_sum":{"num_bytes":2338,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":7014,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3563520,"data_stored":3542208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430220,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298520,"kb_used_data":1164,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446632,"statfs":{"total":95995035648,"available":95689351168,"internally_reserved":0,"allocated":1191936,"data_stored":842056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.57999999999999996}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.46000000000000002}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.51100000000000001}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.47799999999999998}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.48899999999999999}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.38800000000000001}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.56499999999999995}]}]},{"osd":6,"up_from":42,"seq":180388626449,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298524,"kb_used_data":1164,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446628,"statfs":{"total":95995035648,"available":95689347072,"internally_reserved":0,"allocated":1191936,"data_stored":842701,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:27:45 2024","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.70799999999999996,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.83699999999999997,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.53500000000000003}]},{"osd":1,"last update":"Sat Sep 21 09:27:45 2024","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.56100000000000005},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.76500000000000001,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.63700000000000001}]},{"osd":2,"last update":"Sat Sep 21 09:27:45 2024","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.52400000000000002},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"min":{"1min":0.441,"5min":0.441,"15min":0.441},"max":{"1min":0.74099999999999999,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.60699999999999998}]},{"osd":3,"last update":"Sat Sep 21 09:27:45 2024","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.626},{"interface":"front","average":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.83199999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.67000000000000004}]},{"osd":4,"last update":"Sat Sep 21 09:27:45 2024","interfaces":[{"interface":"back","average":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.69899999999999995,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":0.46600000000000003},{"interface":"front","average":{"1min":0.438,"5min":0.438,"15min":0.438},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.504}]},{"osd":5,"last update":"Sat Sep 21 09:27:45 2024","interfaces":[{"interface":"back","average":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.40799999999999997},{"interface":"front","average":{"1min":0.48699999999999999,"5min":0.48699999999999999,"15min":0.48699999999999999},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.85499999999999998,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.38500000000000001}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.437}]}]},{"osd":5,"up_from":36,"seq":154618822679,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297944,"kb_used_data":584,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447208,"statfs":{"total":95995035648,"available":95689940992,"internally_reserved":0,"allocated":598016,"data_stored":252108,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:27:21 2024","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.54000000000000004},{"interface":"front","average":{"1min":0.52000000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.69099999999999995,"5min":0.69099999999999995,"15min":0.69099999999999995},"last":0.57499999999999996}]},{"osd":1,"last update":"Sat Sep 21 09:27:21 2024","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.68400000000000005},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.69899999999999995,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":0.52600000000000002}]},{"osd":2,"last update":"Sat Sep 21 09:27:21 2024","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.75,"5min":0.75,"15min":0.75},"last":0.61299999999999999},{"interface":"front","average":{"1min":0.505,"5min":0.505,"15min":0.505},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.72999999999999998,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.63400000000000001}]},{"osd":3,"last update":"Sat Sep 21 09:27:21 2024","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.79000000000000004,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.66400000000000003},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.59799999999999998}]},{"osd":4,"last update":"Sat Sep 21 09:27:21 2024","interfaces":[{"interface":"back","average":{"1min":0.377,"5min":0.377,"15min":0.377},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"last":0.371},{"interface":"front","average":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":0.65500000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"last":0.47799999999999998}]},{"osd":6,"last update":"Sat Sep 21 09:27:45 2024","interfaces":[{"interface":"back","average":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.72099999999999997,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.42599999999999999},{"interface":"front","average":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.68100000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"last":0.44}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.55700000000000005}]}]},{"osd":4,"up_from":30,"seq":128849018907,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297952,"kb_used_data":592,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447200,"statfs":{"total":95995035648,"available":95689932800,"internally_reserved":0,"allocated":606208,"data_stored":251792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:27:56 2024","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.55000000000000004,"15min":0.55200000000000005},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.70999999999999996,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.33400000000000002},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.56499999999999995,"15min":0.56299999999999994},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.75700000000000001,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.47099999999999997}]},{"osd":1,"last update":"Sat Sep 21 09:27:56 2024","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":0.58299999999999996,"15min":0.58199999999999996},"min":{"1min":0.44400000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.70399999999999996,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.41599999999999998},{"interface":"front","average":{"1min":0.57599999999999996,"5min":0.54500000000000004,"15min":0.54000000000000004},"min":{"1min":0.42399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.75700000000000001,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.53200000000000003}]},{"osd":2,"last update":"Sat Sep 21 09:27:56 2024","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.56100000000000005,"15min":0.56399999999999995},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.69299999999999995,"5min":0.91200000000000003,"15min":0.91200000000000003},"last":0.39000000000000001},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.55100000000000005,"15min":0.54100000000000004},"min":{"1min":0.44900000000000001,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.437}]},{"osd":3,"last update":"Sat Sep 21 09:27:56 2024","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.56899999999999995,"15min":0.56499999999999995},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.78800000000000003,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.55400000000000005},{"interface":"front","average":{"1min":0.57599999999999996,"5min":0.56499999999999995,"15min":0.56299999999999994},"min":{"1min":0.41399999999999998,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.76400000000000001,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":0.46000000000000002}]},{"osd":5,"last update":"Sat Sep 21 09:27:24 2024","interfaces":[{"interface":"back","average":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.66300000000000003,"5min":0.66300000000000003,"15min":0.66300000000000003},"last":0.51800000000000002},{"interface":"front","average":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"last":0.57699999999999996}]},{"osd":6,"last update":"Sat Sep 21 09:27:44 2024","interfaces":[{"interface":"back","average":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.747,"5min":0.747,"15min":0.747},"last":0.48299999999999998},{"interface":"front","average":{"1min":0.46700000000000003,"5min":0.46700000000000003,"15min":0.46700000000000003},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.81399999999999995,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":0.56799999999999995}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.48899999999999999}]}]},{"osd":3,"up_from":25,"seq":107374182432,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297956,"kb_used_data":596,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447196,"statfs":{"total":95995035648,"available":95689928704,"internally_reserved":0,"allocated":610304,"data_stored":252752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:27:39 2024","interfaces":[{"interface":"back","average":{"1min":0.38,"5min":0.36599999999999999,"15min":0.36399999999999999},"min":{"1min":0.253,"5min":0.25,"15min":0.25},"max":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"last":0.59699999999999998},{"interface":"front","average":{"1min":0.41399999999999998,"5min":0.39400000000000002,"15min":0.39100000000000001},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"last":0.40400000000000003}]},{"osd":1,"last update":"Sat Sep 21 09:27:39 2024","interfaces":[{"interface":"back","average":{"1min":0.44600000000000001,"5min":0.434,"15min":0.432},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.63900000000000001,"5min":0.63900000000000001,"15min":0.63900000000000001},"last":0.35099999999999998},{"interface":"front","average":{"1min":0.40300000000000002,"5min":0.41299999999999998,"15min":0.41499999999999998},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":0.66700000000000004,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.36799999999999999}]},{"osd":2,"last update":"Sat Sep 21 09:27:39 2024","interfaces":[{"interface":"back","average":{"1min":0.439,"5min":0.44400000000000001,"15min":0.44500000000000001},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.625,"5min":0.625,"15min":0.625},"last":0.39600000000000002},{"interface":"front","average":{"1min":0.41899999999999998,"5min":0.42099999999999999,"15min":0.42099999999999999},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"last":0.45600000000000002}]},{"osd":4,"last update":"Sat Sep 21 09:27:57 2024","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":0.55600000000000005,"15min":0.54500000000000004},"min":{"1min":0.42899999999999999,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.80300000000000005,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.63700000000000001},{"interface":"front","average":{"1min":0.61299999999999999,"5min":0.55300000000000005,"15min":0.54300000000000004},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.80200000000000005,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.496}]},{"osd":5,"last update":"Sat Sep 21 09:27:19 2024","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.83299999999999996,"5min":0.83299999999999996,"15min":0.83299999999999996},"last":0.56100000000000005},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.60599999999999998,"15min":0.60599999999999998},"min":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":0.77300000000000002,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.57499999999999996}]},{"osd":6,"last update":"Sat Sep 21 09:27:46 2024","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.85999999999999999,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.51500000000000001},{"interface":"front","average":{"1min":0.62,"5min":0.62,"15min":0.62},"min":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.60799999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.52800000000000002}]}]},{"osd":2,"up_from":18,"seq":77309411365,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297932,"kb_used_data":576,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447220,"statfs":{"total":95995035648,"available":95689953280,"internally_reserved":0,"allocated":589824,"data_stored":251096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:27:12 2024","interfaces":[{"interface":"back","average":{"1min":0.36599999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.47499999999999998,"5min":0.54400000000000004,"15min":0.54400000000000004},"last":0.28699999999999998},{"interface":"front","average":{"1min":0.40899999999999997,"5min":0.39700000000000002,"15min":0.39500000000000002},"min":{"1min":0.33000000000000002,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.50700000000000001,"5min":0.53800000000000003,"15min":0.53800000000000003},"last":0.26800000000000002}]},{"osd":1,"last update":"Sat Sep 21 09:27:12 2024","interfaces":[{"interface":"back","average":{"1min":0.40400000000000003,"5min":0.374,"15min":0.36899999999999999},"min":{"1min":0.26800000000000002,"5min":0.219,"15min":0.219},"max":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"last":0.19600000000000001},{"interface":"front","average":{"1min":0.46700000000000003,"5min":0.42199999999999999,"15min":0.41399999999999998},"min":{"1min":0.376,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.625,"5min":0.625,"15min":0.625},"last":0.30199999999999999}]},{"osd":3,"last update":"Sat Sep 21 09:27:35 2024","interfaces":[{"interface":"back","average":{"1min":0.376,"5min":0.437,"15min":0.44700000000000001},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.54200000000000004,"5min":0.61599999999999999,"15min":0.61599999999999999},"last":0.32100000000000001},{"interface":"front","average":{"1min":0.44400000000000001,"5min":0.46800000000000003,"15min":0.47199999999999998},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.64000000000000001,"5min":0.65200000000000002,"15min":0.65200000000000002},"last":0.253}]},{"osd":4,"last update":"Sat Sep 21 09:26:57 2024","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"min":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"last":0.499},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"min":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":0.85099999999999998,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.57799999999999996}]},{"osd":5,"last update":"Sat Sep 21 09:27:22 2024","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"min":{"1min":0.48399999999999999,"5min":0.48399999999999999,"15min":0.48399999999999999},"max":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.54900000000000004},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":0.75700000000000001,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.46700000000000003}]},{"osd":6,"last update":"Sat Sep 21 09:27:43 2024","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.51200000000000001},{"interface":"front","average":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"min":{"1min":0.47399999999999998,"5min":0.47399999999999998,"15min":0.47399999999999998},"max":{"1min":0.79900000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.56000000000000005}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.58699999999999997}]}]},{"osd":1,"up_from":13,"seq":55834574890,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297944,"kb_used_data":584,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447208,"statfs":{"total":95995035648,"available":95689940992,"internally_reserved":0,"allocated":598016,"data_stored":251256,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:27:46 2024","interfaces":[{"interface":"back","average":{"1min":0.40500000000000003,"5min":0.35899999999999999,"15min":0.33600000000000002},"min":{"1min":0.246,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":0.60599999999999998,"5min":0.61399999999999999,"15min":0.61399999999999999},"last":0.219},{"interface":"front","average":{"1min":0.41099999999999998,"5min":0.39100000000000001,"15min":0.36899999999999999},"min":{"1min":0.28299999999999997,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.58099999999999996,"5min":0.63,"15min":0.63},"last":0.313}]},{"osd":2,"last update":"Sat Sep 21 09:27:12 2024","interfaces":[{"interface":"back","average":{"1min":0.41799999999999998,"5min":0.37,"15min":0.36199999999999999},"min":{"1min":0.28699999999999998,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"last":0.29499999999999998},{"interface":"front","average":{"1min":0.41499999999999998,"5min":0.379,"15min":0.373},"min":{"1min":0.27400000000000002,"5min":0.245,"15min":0.245},"max":{"1min":0.67100000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.35499999999999998}]},{"osd":3,"last update":"Sat Sep 21 09:27:35 2024","interfaces":[{"interface":"back","average":{"1min":0.40200000000000002,"5min":0.41999999999999998,"15min":0.42299999999999999},"min":{"1min":0.309,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"last":0.33800000000000002},{"interface":"front","average":{"1min":0.46500000000000002,"5min":0.46999999999999997,"15min":0.47099999999999997},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.76800000000000002,"5min":0.76800000000000002,"15min":0.76800000000000002},"last":0.32800000000000001}]},{"osd":4,"last update":"Sat Sep 21 09:26:55 2024","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.97599999999999998,"5min":0.97599999999999998,"15min":0.97599999999999998},"last":0.495},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.95699999999999996,"5min":0.95699999999999996,"15min":0.95699999999999996},"last":0.41299999999999998}]},{"osd":5,"last update":"Sat Sep 21 09:27:20 2024","interfaces":[{"interface":"back","average":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"min":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":1.014,"5min":1.014,"15min":1.014},"last":0.46000000000000002},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.93500000000000005,"5min":0.93500000000000005,"15min":0.93500000000000005},"last":0.54600000000000004}]},{"osd":6,"last update":"Sat Sep 21 09:27:46 2024","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.74099999999999999,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.52200000000000002},{"interface":"front","average":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"min":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.42299999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.50700000000000001}]}]},{"osd":0,"up_from":8,"seq":34359738414,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298524,"kb_used_data":1168,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446628,"statfs":{"total":95995035648,"available":95689347072,"internally_reserved":0,"allocated":1196032,"data_stored":841865,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 21 09:27:47 2024","interfaces":[{"interface":"back","average":{"1min":0.33200000000000002,"5min":0.32000000000000001,"15min":0.31},"min":{"1min":0.20200000000000001,"5min":0.185,"15min":0.185},"max":{"1min":0.51300000000000001,"5min":0.54300000000000004,"15min":0.54300000000000004},"last":0.24399999999999999},{"interface":"front","average":{"1min":0.34499999999999997,"5min":0.34200000000000003,"15min":0.33300000000000002},"min":{"1min":0.24299999999999999,"5min":0.185,"15min":0.185},"max":{"1min":0.51600000000000001,"5min":0.625,"15min":0.625},"last":0.39700000000000002}]},{"osd":2,"last update":"Sat Sep 21 09:27:12 2024","interfaces":[{"interface":"back","average":{"1min":0.39300000000000002,"5min":0.373,"15min":0.36899999999999999},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"last":0.33200000000000002},{"interface":"front","average":{"1min":0.44,"5min":0.42299999999999999,"15min":0.41999999999999998},"min":{"1min":0.30499999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.56200000000000006,"5min":0.69399999999999995,"15min":0.69399999999999995},"last":0.35599999999999998}]},{"osd":3,"last update":"Sat Sep 21 09:27:36 2024","interfaces":[{"interface":"back","average":{"1min":0.39900000000000002,"5min":0.42399999999999999,"15min":0.42799999999999999},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"last":0.28100000000000003},{"interface":"front","average":{"1min":0.42099999999999999,"5min":0.42899999999999999,"15min":0.42999999999999999},"min":{"1min":0.32400000000000001,"5min":0.31,"15min":0.31},"max":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"last":0.29599999999999999}]},{"osd":4,"last update":"Sat Sep 21 09:27:56 2024","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.52500000000000002,"15min":0.51600000000000001},"min":{"1min":0.42799999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.80900000000000005,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.54700000000000004},{"interface":"front","average":{"1min":0.628,"5min":0.57599999999999996,"15min":0.56799999999999995},"min":{"1min":0.443,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.83299999999999996,"5min":0.83299999999999996,"15min":0.83299999999999996},"last":0.53700000000000003}]},{"osd":5,"last update":"Sat Sep 21 09:27:23 2024","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.76600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.44900000000000001},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.84199999999999997,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":0.51900000000000002}]},{"osd":6,"last update":"Sat Sep 21 09:27:44 2024","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.876,"5min":0.876,"15min":0.876},"last":0.63500000000000001},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"min":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":0.753,"5min":0.753,"15min":0.753},"last":0.47199999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.49299999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":716,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":1656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1642,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":1552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T09:28:02.349 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-09-21T09:28:02.349 INFO:tasks.cephadm:Adding prometheus.a on smithi064 2024-09-21T09:28:02.350 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d65e801e-77fa-11ef-baf6-efdc52797490 -- ceph orch apply prometheus '1;smithi064=a' 2024-09-21T09:28:02.376 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:02 smithi064 ceph-mon[31124]: pgmap v204: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:28:02.610 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:28:03.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:03 smithi064 ceph-mon[31124]: from='client.14505 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:28:03.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:03 smithi022 ceph-mon[39525]: from='client.14505 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:28:03.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:03 smithi022 ceph-mon[27763]: from='client.14505 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:28:04.419 INFO:teuthology.orchestra.run.smithi064.stdout:Scheduled prometheus update... 2024-09-21T09:28:04.420 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:04 smithi064 ceph-mon[31124]: pgmap v205: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:28:04.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:04 smithi022 ceph-mon[27763]: pgmap v205: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:28:04.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:04 smithi022 ceph-mon[39525]: pgmap v205: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:28:04.988 DEBUG:teuthology.orchestra.run.smithi064:prometheus.a> sudo journalctl -f -n 0 -u ceph-d65e801e-77fa-11ef-baf6-efdc52797490@prometheus.a.service 2024-09-21T09:28:05.031 INFO:tasks.cephadm:Adding node-exporter.a on smithi022 2024-09-21T09:28:05.031 INFO:tasks.cephadm:Adding node-exporter.b on smithi064 2024-09-21T09:28:05.031 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d65e801e-77fa-11ef-baf6-efdc52797490 -- ceph orch apply node-exporter '2;smithi022=a;smithi064=b' 2024-09-21T09:28:05.329 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:28:05.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:05 smithi064 ceph-mon[31124]: from='client.24385 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi064=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:28:05.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:05 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:28:05.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:05 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:28:05.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:05 smithi022 ceph-mon[27763]: from='client.24385 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi064=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:28:05.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:05 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:28:05.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:05 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:28:05.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:05 smithi022 ceph-mon[39525]: from='client.24385 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi064=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:28:05.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:05 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:28:05.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:05 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:28:06.395 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:06 smithi064 ceph-mon[31124]: Saving service prometheus spec with placement smithi064=a;count:1 2024-09-21T09:28:06.395 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:06 smithi064 ceph-mon[31124]: pgmap v206: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:28:06.395 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:06 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:28:06.395 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:06 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:28:06.395 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:06 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:28:06.395 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:06 smithi064 ceph-mon[31124]: Deploying daemon prometheus.a on smithi064 2024-09-21T09:28:06.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:06 smithi022 ceph-mon[39525]: Saving service prometheus spec with placement smithi064=a;count:1 2024-09-21T09:28:06.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:06 smithi022 ceph-mon[39525]: pgmap v206: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:28:06.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:06 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:28:06.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:06 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:28:06.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:06 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:28:06.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:06 smithi022 ceph-mon[39525]: Deploying daemon prometheus.a on smithi064 2024-09-21T09:28:06.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:06 smithi022 ceph-mon[27763]: Saving service prometheus spec with placement smithi064=a;count:1 2024-09-21T09:28:06.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:06 smithi022 ceph-mon[27763]: pgmap v206: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:28:06.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:06 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:28:06.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:06 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:28:06.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:06 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:28:06.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:06 smithi022 ceph-mon[27763]: Deploying daemon prometheus.a on smithi064 2024-09-21T09:28:07.114 INFO:teuthology.orchestra.run.smithi064.stdout:Scheduled node-exporter update... 2024-09-21T09:28:08.267 DEBUG:teuthology.orchestra.run.smithi022:node-exporter.a> sudo journalctl -f -n 0 -u ceph-d65e801e-77fa-11ef-baf6-efdc52797490@node-exporter.a.service 2024-09-21T09:28:08.270 DEBUG:teuthology.orchestra.run.smithi064:node-exporter.b> sudo journalctl -f -n 0 -u ceph-d65e801e-77fa-11ef-baf6-efdc52797490@node-exporter.b.service 2024-09-21T09:28:08.272 INFO:tasks.cephadm:Adding alertmanager.a on smithi022 2024-09-21T09:28:08.273 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d65e801e-77fa-11ef-baf6-efdc52797490 -- ceph orch apply alertmanager '1;smithi022=a' 2024-09-21T09:28:08.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:08 smithi064 ceph-mon[31124]: pgmap v207: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:28:08.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:08 smithi064 ceph-mon[31124]: from='client.24391 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi022=a;smithi064=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:28:08.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:08 smithi064 ceph-mon[31124]: Saving service node-exporter spec with placement smithi022=a;smithi064=b;count:2 2024-09-21T09:28:08.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:08 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:28:08.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:08 smithi022 ceph-mon[39525]: pgmap v207: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:28:08.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:08 smithi022 ceph-mon[39525]: from='client.24391 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi022=a;smithi064=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:28:08.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:08 smithi022 ceph-mon[39525]: Saving service node-exporter spec with placement smithi022=a;smithi064=b;count:2 2024-09-21T09:28:08.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:08 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:28:08.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:08 smithi022 ceph-mon[27763]: pgmap v207: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:28:08.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:08 smithi022 ceph-mon[27763]: from='client.24391 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi022=a;smithi064=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:28:08.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:08 smithi022 ceph-mon[27763]: Saving service node-exporter spec with placement smithi022=a;smithi064=b;count:2 2024-09-21T09:28:08.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:08 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:28:08.636 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:28:10.368 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:10 smithi064 ceph-mon[31124]: pgmap v208: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:28:10.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:10 smithi022 ceph-mon[39525]: pgmap v208: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:28:10.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:10 smithi022 ceph-mon[27763]: pgmap v208: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:28:11.028 INFO:teuthology.orchestra.run.smithi064.stdout:Scheduled alertmanager update... 2024-09-21T09:28:12.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:12 smithi022 ceph-mon[39525]: pgmap v209: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:28:12.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:12 smithi022 ceph-mon[39525]: from='client.24397 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi022=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:28:12.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:12 smithi022 ceph-mon[39525]: Saving service alertmanager spec with placement smithi022=a;count:1 2024-09-21T09:28:12.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:12 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:28:12.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:12 smithi022 ceph-mon[27763]: pgmap v209: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:28:12.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:12 smithi022 ceph-mon[27763]: from='client.24397 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi022=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:28:12.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:12 smithi022 ceph-mon[27763]: Saving service alertmanager spec with placement smithi022=a;count:1 2024-09-21T09:28:12.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:12 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:28:12.377 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:12 smithi064 ceph-mon[31124]: pgmap v209: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:28:12.377 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:12 smithi064 ceph-mon[31124]: from='client.24397 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi022=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:28:12.377 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:12 smithi064 ceph-mon[31124]: Saving service alertmanager spec with placement smithi022=a;count:1 2024-09-21T09:28:12.377 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:12 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:28:12.930 DEBUG:teuthology.orchestra.run.smithi022:alertmanager.a> sudo journalctl -f -n 0 -u ceph-d65e801e-77fa-11ef-baf6-efdc52797490@alertmanager.a.service 2024-09-21T09:28:12.933 INFO:tasks.cephadm:Adding grafana.a on smithi064 2024-09-21T09:28:12.933 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d65e801e-77fa-11ef-baf6-efdc52797490 -- ceph orch apply grafana '1;smithi064=a' 2024-09-21T09:28:13.416 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:28:14.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:14 smithi064 ceph-mon[31124]: pgmap v210: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:28:14.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:14 smithi022 ceph-mon[39525]: pgmap v210: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:28:14.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:14 smithi022 ceph-mon[27763]: pgmap v210: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:28:16.131 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:16 smithi064 ceph-mon[31124]: pgmap v211: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:28:16.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:16 smithi022 ceph-mon[39525]: pgmap v211: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:28:16.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:16 smithi022 ceph-mon[27763]: pgmap v211: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:28:17.096 INFO:teuthology.orchestra.run.smithi064.stdout:Scheduled grafana update... 2024-09-21T09:28:18.419 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:18 smithi064 ceph-mon[31124]: pgmap v212: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:28:18.513 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:18 smithi064 ceph-mon[31124]: from='client.24403 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi064=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:28:18.513 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:18 smithi064 ceph-mon[31124]: Saving service grafana spec with placement smithi064=a;count:1 2024-09-21T09:28:18.514 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:18 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:28:18.515 DEBUG:teuthology.orchestra.run.smithi064:grafana.a> sudo journalctl -f -n 0 -u ceph-d65e801e-77fa-11ef-baf6-efdc52797490@grafana.a.service 2024-09-21T09:28:18.517 INFO:tasks.cephadm:Setting up client nodes... 2024-09-21T09:28:18.518 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d65e801e-77fa-11ef-baf6-efdc52797490 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-09-21T09:28:18.545 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:18 smithi022 ceph-mon[39525]: pgmap v212: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:28:18.545 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:18 smithi022 ceph-mon[39525]: from='client.24403 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi064=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:28:18.545 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:18 smithi022 ceph-mon[39525]: Saving service grafana spec with placement smithi064=a;count:1 2024-09-21T09:28:18.545 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:18 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:28:18.546 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:18 smithi022 ceph-mon[27763]: pgmap v212: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:28:18.546 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:18 smithi022 ceph-mon[27763]: from='client.24403 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi064=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:28:18.546 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:18 smithi022 ceph-mon[27763]: Saving service grafana spec with placement smithi064=a;count:1 2024-09-21T09:28:18.546 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:18 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:28:18.782 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:28:19.614 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:28:19 smithi064 systemd[1]: Starting Ceph prometheus.a for d65e801e-77fa-11ef-baf6-efdc52797490... 2024-09-21T09:28:19.941 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:28:19 smithi064 podman[61661]: 2024-09-21 09:28:19.51650908 +0000 UTC m=+0.017257323 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-09-21T09:28:19.941 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:28:19 smithi064 podman[61661]: 2024-09-21 09:28:19.613968418 +0000 UTC m=+0.114716646 container create 5f305effcaf8002b2da448af66a8a7486ec77a2ddcb5ce39551ad51709f6a622 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:28:20.290 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:20 smithi022 ceph-mon[39525]: pgmap v213: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:28:20.290 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:20 smithi022 ceph-mon[27763]: pgmap v213: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:28:20.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:20 smithi064 ceph-mon[31124]: pgmap v213: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:28:20.442 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:28:20 smithi064 podman[61661]: 2024-09-21 09:28:20.16669814 +0000 UTC m=+0.667446372 container init 5f305effcaf8002b2da448af66a8a7486ec77a2ddcb5ce39551ad51709f6a622 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:28:20.442 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:28:20 smithi064 podman[61661]: 2024-09-21 09:28:20.173323026 +0000 UTC m=+0.674071257 container start 5f305effcaf8002b2da448af66a8a7486ec77a2ddcb5ce39551ad51709f6a622 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:28:20.442 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:28:20 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[61677]: ts=2024-09-21T09:28:20.200Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-09-21T09:28:20.442 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:28:20 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[61677]: ts=2024-09-21T09:28:20.200Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-09-21T09:28:20.442 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:28:20 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[61677]: ts=2024-09-21T09:28:20.200Z caller=main.go:570 level=info host_details="(Linux 5.14.0-508.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024 x86_64 smithi064 (none))" 2024-09-21T09:28:20.442 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:28:20 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[61677]: ts=2024-09-21T09:28:20.200Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-09-21T09:28:20.443 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:28:20 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[61677]: ts=2024-09-21T09:28:20.200Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-21T09:28:20.443 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:28:20 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[61677]: ts=2024-09-21T09:28:20.202Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-09-21T09:28:20.443 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:28:20 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[61677]: ts=2024-09-21T09:28:20.202Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-09-21T09:28:20.443 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:28:20 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[61677]: ts=2024-09-21T09:28:20.203Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-09-21T09:28:20.443 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:28:20 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[61677]: ts=2024-09-21T09:28:20.203Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-09-21T09:28:20.443 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:28:20 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[61677]: ts=2024-09-21T09:28:20.206Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-09-21T09:28:20.443 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:28:20 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[61677]: ts=2024-09-21T09:28:20.206Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.994µs 2024-09-21T09:28:20.443 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:28:20 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[61677]: ts=2024-09-21T09:28:20.206Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-09-21T09:28:20.443 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:28:20 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[61677]: ts=2024-09-21T09:28:20.206Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=0 2024-09-21T09:28:20.443 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:28:20 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[61677]: ts=2024-09-21T09:28:20.206Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=22.476µs wal_replay_duration=288.804µs wbl_replay_duration=97ns total_replay_duration=331.169µs 2024-09-21T09:28:20.443 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:28:20 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[61677]: ts=2024-09-21T09:28:20.207Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-09-21T09:28:20.443 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:28:20 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[61677]: ts=2024-09-21T09:28:20.207Z caller=main.go:1029 level=info msg="TSDB started" 2024-09-21T09:28:20.443 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:28:20 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[61677]: ts=2024-09-21T09:28:20.207Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-21T09:28:20.443 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:28:20 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[61677]: ts=2024-09-21T09:28:20.223Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=15.717965ms db_storage=890ns remote_storage=1.293µs web_handler=369ns query_engine=640ns scrape=439.954µs scrape_sd=20.789µs notify=695ns notify_sd=1.881µs rules=14.947677ms tracing=24.7µs 2024-09-21T09:28:20.444 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:28:20 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[61677]: ts=2024-09-21T09:28:20.223Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-09-21T09:28:20.444 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:28:20 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[61677]: ts=2024-09-21T09:28:20.223Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-09-21T09:28:20.781 INFO:teuthology.orchestra.run.smithi022.stdout:[client.0] 2024-09-21T09:28:20.781 INFO:teuthology.orchestra.run.smithi022.stdout: key = AQC0ke5mFEVDLhAAif7BRRX38mHRMtcfj1ICiw== 2024-09-21T09:28:21.370 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:21 smithi022 ceph-mon[27763]: from='client.? 172.21.15.22:0/1657156194' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-21T09:28:21.371 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:21 smithi022 ceph-mon[27763]: from='client.? 172.21.15.22:0/1657156194' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-21T09:28:21.371 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:21 smithi022 ceph-mon[39525]: from='client.? 172.21.15.22:0/1657156194' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-21T09:28:21.371 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:21 smithi022 ceph-mon[39525]: from='client.? 172.21.15.22:0/1657156194' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-21T09:28:21.398 DEBUG:teuthology.orchestra.run.smithi022:> set -ex 2024-09-21T09:28:21.398 DEBUG:teuthology.orchestra.run.smithi022:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-09-21T09:28:21.399 DEBUG:teuthology.orchestra.run.smithi022:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-09-21T09:28:21.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:21 smithi064 ceph-mon[31124]: from='client.? 172.21.15.22:0/1657156194' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-21T09:28:21.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:21 smithi064 ceph-mon[31124]: from='client.? 172.21.15.22:0/1657156194' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-21T09:28:21.442 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:28:21 smithi064 bash[61661]: 5f305effcaf8002b2da448af66a8a7486ec77a2ddcb5ce39551ad51709f6a622 2024-09-21T09:28:21.442 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:28:21 smithi064 systemd[1]: Started Ceph prometheus.a for d65e801e-77fa-11ef-baf6-efdc52797490. 2024-09-21T09:28:21.481 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d65e801e-77fa-11ef-baf6-efdc52797490 -- ceph auth get-or-create client.1 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-09-21T09:28:21.751 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:28:22.392 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:22 smithi064 ceph-mon[31124]: pgmap v214: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:28:22.392 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:22 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:28:22.392 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:22 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:28:22.392 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:22 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:28:22.392 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:22 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-09-21T09:28:22.392 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:28:22 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[33213]: ignoring --setuser ceph since I am not root 2024-09-21T09:28:22.393 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:28:22 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[33213]: ignoring --setgroup ceph since I am not root 2024-09-21T09:28:22.393 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:28:22 smithi064 ceph-mgr[33236]: -- 172.21.15.64:0/160409493 <== mon.1 v2:172.21.15.64:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5633ab5d6340 con 0x5633ab5e2c00 2024-09-21T09:28:22.393 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:28:22 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[33213]: 2024-09-21T09:28:22.332+0000 7f527507aec0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-21T09:28:22.393 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:28:22 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[33213]: 2024-09-21T09:28:22.389+0000 7f527507aec0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-21T09:28:22.460 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:22 smithi022 ceph-mon[39525]: pgmap v214: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:28:22.460 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:22 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:28:22.460 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:22 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:28:22.460 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:22 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:28:22.460 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:22 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-09-21T09:28:22.461 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:22 smithi022 ceph-mon[27763]: pgmap v214: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:28:22.461 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:22 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:28:22.461 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:22 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:28:22.461 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:22 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' 2024-09-21T09:28:22.461 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:22 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-09-21T09:28:22.461 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:28:22 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: ignoring --setuser ceph since I am not root 2024-09-21T09:28:22.461 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:28:22 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: ignoring --setgroup ceph since I am not root 2024-09-21T09:28:22.461 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:28:22 smithi022 ceph-mgr[28035]: -- 172.21.15.22:0/3499251730 <== mon.0 v2:172.21.15.22:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55cee9cbe340 con 0x55cee9ccb000 2024-09-21T09:28:22.462 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:28:22 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:28:22.333+0000 7fe55f424ec0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-21T09:28:22.462 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:28:22 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:28:22.390+0000 7fe55f424ec0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-21T09:28:22.462 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:28:22 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:28:22.458+0000 7fe55f424ec0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-21T09:28:22.691 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:28:22 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[33213]: 2024-09-21T09:28:22.457+0000 7f527507aec0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-21T09:28:22.691 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:28:22 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[33213]: 2024-09-21T09:28:22.584+0000 7f527507aec0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-21T09:28:22.731 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:28:22 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:28:22.586+0000 7fe55f424ec0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-21T09:28:22.996 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:28:22 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:28:22.729+0000 7fe55f424ec0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-21T09:28:22.996 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:28:22 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:28:22.994+0000 7fe55f424ec0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-21T09:28:22.998 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:28:22 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[33213]: 2024-09-21T09:28:22.730+0000 7f527507aec0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-21T09:28:23.270 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:23 smithi064 ceph-mon[31124]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-09-21T09:28:23.270 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:23 smithi064 ceph-mon[31124]: mgrmap e16: y(active, since 5m), standbys: x 2024-09-21T09:28:23.270 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:28:22 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[33213]: 2024-09-21T09:28:22.996+0000 7f527507aec0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-21T09:28:23.270 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:28:23 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[33213]: 2024-09-21T09:28:23.058+0000 7f527507aec0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-21T09:28:23.271 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:28:23 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[33213]: 2024-09-21T09:28:23.237+0000 7f527507aec0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-21T09:28:23.307 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:23 smithi022 ceph-mon[39525]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-09-21T09:28:23.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:23 smithi022 ceph-mon[39525]: mgrmap e16: y(active, since 5m), standbys: x 2024-09-21T09:28:23.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:23 smithi022 ceph-mon[27763]: from='mgr.14150 172.21.15.22:0/1301077178' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-09-21T09:28:23.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:23 smithi022 ceph-mon[27763]: mgrmap e16: y(active, since 5m), standbys: x 2024-09-21T09:28:23.308 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:28:23 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:28:23.056+0000 7fe55f424ec0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-21T09:28:23.308 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:28:23 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:28:23.224+0000 7fe55f424ec0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-21T09:28:23.872 INFO:teuthology.orchestra.run.smithi064.stdout:[client.1] 2024-09-21T09:28:23.872 INFO:teuthology.orchestra.run.smithi064.stdout: key = AQC3ke5m/uu4MxAAszwqxo40U30Xuj2UQhpp3g== 2024-09-21T09:28:24.142 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:28:24 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[33213]: 2024-09-21T09:28:24.139+0000 7f527507aec0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-21T09:28:24.441 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:28:24 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[33213]: 2024-09-21T09:28:24.207+0000 7f527507aec0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-21T09:28:24.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:24 smithi064 ceph-mon[31124]: from='client.? 172.21.15.64:0/1201351914' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-21T09:28:24.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:24 smithi064 ceph-mon[31124]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-21T09:28:24.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:24 smithi064 ceph-mon[31124]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-21T09:28:24.557 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:24 smithi022 ceph-mon[39525]: from='client.? 172.21.15.64:0/1201351914' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-21T09:28:24.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:24 smithi022 ceph-mon[39525]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-21T09:28:24.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:24 smithi022 ceph-mon[39525]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-21T09:28:24.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:24 smithi022 ceph-mon[27763]: from='client.? 172.21.15.64:0/1201351914' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-21T09:28:24.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:24 smithi022 ceph-mon[27763]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-21T09:28:24.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:24 smithi022 ceph-mon[27763]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-21T09:28:24.558 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:28:24 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:28:24.105+0000 7fe55f424ec0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-21T09:28:24.559 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:28:24 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:28:24.172+0000 7fe55f424ec0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-21T09:28:24.632 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-09-21T09:28:24.632 DEBUG:teuthology.orchestra.run.smithi064:> sudo dd of=/etc/ceph/ceph.client.1.keyring 2024-09-21T09:28:24.632 DEBUG:teuthology.orchestra.run.smithi064:> sudo chmod 0644 /etc/ceph/ceph.client.1.keyring 2024-09-21T09:28:24.671 INFO:tasks.cephadm:Creating RBD pool 2024-09-21T09:28:24.671 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d65e801e-77fa-11ef-baf6-efdc52797490 -- ceph osd pool create rbd 8 2024-09-21T09:28:24.933 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:28:25.043 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:28:24 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:28:24.920+0000 7fe55f424ec0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-21T09:28:25.043 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:28:25 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:28:25.040+0000 7fe55f424ec0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-21T09:28:25.256 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:28:24 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[33213]: 2024-09-21T09:28:24.950+0000 7f527507aec0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-21T09:28:25.256 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:28:25 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[33213]: 2024-09-21T09:28:25.065+0000 7f527507aec0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-21T09:28:25.256 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:28:25 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[33213]: 2024-09-21T09:28:25.191+0000 7f527507aec0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-21T09:28:25.256 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:28:25 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[33213]: 2024-09-21T09:28:25.253+0000 7f527507aec0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-21T09:28:25.308 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:28:25 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:28:25.167+0000 7fe55f424ec0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-21T09:28:25.308 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:28:25 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:28:25.232+0000 7fe55f424ec0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-21T09:28:25.516 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:28:25 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[33213]: 2024-09-21T09:28:25.383+0000 7f527507aec0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-21T09:28:25.517 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:28:25 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[33213]: 2024-09-21T09:28:25.453+0000 7f527507aec0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-21T09:28:25.517 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:28:25 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[33213]: 2024-09-21T09:28:25.514+0000 7f527507aec0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-21T09:28:25.647 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:28:25 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:28:25.364+0000 7fe55f424ec0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-21T09:28:25.647 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:28:25 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:28:25.435+0000 7fe55f424ec0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-21T09:28:25.647 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:28:25 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:28:25.502+0000 7fe55f424ec0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-21T09:28:25.647 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:28:25 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:28:25.645+0000 7fe55f424ec0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-21T09:28:25.790 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:28:25 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[33213]: 2024-09-21T09:28:25.654+0000 7f527507aec0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-21T09:28:25.790 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:28:25 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[33213]: 2024-09-21T09:28:25.720+0000 7f527507aec0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-21T09:28:25.790 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:28:25 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[33213]: 2024-09-21T09:28:25.787+0000 7f527507aec0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-21T09:28:25.925 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:28:25 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:28:25.711+0000 7fe55f424ec0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-21T09:28:25.925 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:28:25 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:28:25.778+0000 7fe55f424ec0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-21T09:28:25.925 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:28:25 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:28:25.863+0000 7fe55f424ec0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-21T09:28:25.925 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:28:25 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:28:25.923+0000 7fe55f424ec0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-21T09:28:26.191 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:28:25 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[33213]: 2024-09-21T09:28:25.871+0000 7f527507aec0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-21T09:28:26.191 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:28:25 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[33213]: 2024-09-21T09:28:25.931+0000 7f527507aec0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-21T09:28:26.191 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:28:25 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[33213]: 2024-09-21T09:28:25.990+0000 7f527507aec0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-21T09:28:26.236 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:28:25 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:28:25.983+0000 7fe55f424ec0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-21T09:28:26.808 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:28:26 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:28:26.775+0000 7fe55f424ec0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-21T09:28:27.098 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:28:26 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[33213]: 2024-09-21T09:28:26.767+0000 7f527507aec0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-21T09:28:27.098 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:28:26 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[33213]: [21/Sep/2024:09:28:26] ENGINE Bus STARTING 2024-09-21T09:28:27.098 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:28:26 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[33213]: CherryPy Checker: 2024-09-21T09:28:27.098 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:28:26 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[33213]: The Application mounted at '' has an empty config. 2024-09-21T09:28:27.098 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:28:26 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[33213]: 2024-09-21T09:28:27.098 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:28:26 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[33213]: [21/Sep/2024:09:28:26] ENGINE Serving on http://:::9283 2024-09-21T09:28:27.098 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:28:26 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[33213]: [21/Sep/2024:09:28:26] ENGINE Bus STARTED 2024-09-21T09:28:27.098 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:26 smithi064 ceph-mon[31124]: Standby manager daemon x restarted 2024-09-21T09:28:27.099 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:26 smithi064 ceph-mon[31124]: Standby manager daemon x started 2024-09-21T09:28:27.099 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:26 smithi064 ceph-mon[31124]: from='mgr.? 172.21.15.64:0/135146349' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-21T09:28:27.099 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:26 smithi064 ceph-mon[31124]: from='mgr.? 172.21.15.64:0/135146349' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:28:27.099 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:26 smithi064 ceph-mon[31124]: from='mgr.? 172.21.15.64:0/135146349' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-21T09:28:27.099 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:26 smithi064 ceph-mon[31124]: from='mgr.? 172.21.15.64:0/135146349' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:28:27.099 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:26 smithi064 ceph-mon[31124]: Active manager daemon y restarted 2024-09-21T09:28:27.099 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:26 smithi064 ceph-mon[31124]: Activating manager daemon y 2024-09-21T09:28:27.099 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:26 smithi064 ceph-mon[31124]: osdmap e60: 8 total, 8 up, 8 in 2024-09-21T09:28:27.099 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:26 smithi064 ceph-mon[31124]: mgrmap e17: y(active, starting, since 0.00876528s), standbys: x 2024-09-21T09:28:27.099 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:26 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:28:27.099 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:26 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:28:27.099 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:26 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:28:27.100 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:26 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-09-21T09:28:27.100 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:26 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-09-21T09:28:27.100 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:26 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-21T09:28:27.100 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:26 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-21T09:28:27.100 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:26 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:28:27.100 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:26 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:28:27.100 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:26 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:28:27.100 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:26 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:28:27.100 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:26 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:28:27.100 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:26 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:28:27.100 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:26 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:28:27.100 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:26 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:28:27.100 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:26 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T09:28:27.100 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:26 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T09:28:27.100 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:26 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T09:28:27.101 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:26 smithi064 ceph-mon[31124]: Manager daemon y is now available 2024-09-21T09:28:27.101 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:26 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:28:27.167 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:28:26 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: [21/Sep/2024:09:28:26] ENGINE Bus STARTING 2024-09-21T09:28:27.167 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:28:26 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: CherryPy Checker: 2024-09-21T09:28:27.167 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:28:26 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: The Application mounted at '' has an empty config. 2024-09-21T09:28:27.167 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:28:26 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:28:27.167 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:28:26 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:28:26.878+0000 7fe510683640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:28:27.167 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:28:26 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:28:26.878+0000 7fe510683640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:28:27.167 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:28:26 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:28:26.878+0000 7fe510683640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:28:27.167 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:28:26 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:28:26.878+0000 7fe510683640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:28:27.167 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:28:26 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:28:26.878+0000 7fe510683640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:28:27.167 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:28:26 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:28:26.878+0000 7fe513689640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:28:27.168 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:28:26 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:28:26.878+0000 7fe513689640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:28:27.168 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:28:26 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:28:26.878+0000 7fe513689640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:28:27.168 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:28:26 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:28:26.878+0000 7fe513689640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:28:27.168 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:28:26 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:28:26.878+0000 7fe513689640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:28:27.168 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:28:26 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: [21/Sep/2024:09:28:26] ENGINE Serving on http://:::9283 2024-09-21T09:28:27.168 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:28:26 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: [21/Sep/2024:09:28:26] ENGINE Bus STARTED 2024-09-21T09:28:27.168 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:26 smithi022 ceph-mon[27763]: Standby manager daemon x restarted 2024-09-21T09:28:27.168 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:26 smithi022 ceph-mon[27763]: Standby manager daemon x started 2024-09-21T09:28:27.168 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:26 smithi022 ceph-mon[27763]: from='mgr.? 172.21.15.64:0/135146349' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-21T09:28:27.168 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:26 smithi022 ceph-mon[27763]: from='mgr.? 172.21.15.64:0/135146349' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:28:27.169 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:26 smithi022 ceph-mon[27763]: from='mgr.? 172.21.15.64:0/135146349' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-21T09:28:27.169 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:26 smithi022 ceph-mon[27763]: from='mgr.? 172.21.15.64:0/135146349' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:28:27.169 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:26 smithi022 ceph-mon[27763]: Active manager daemon y restarted 2024-09-21T09:28:27.169 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:26 smithi022 ceph-mon[27763]: Activating manager daemon y 2024-09-21T09:28:27.169 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:26 smithi022 ceph-mon[27763]: osdmap e60: 8 total, 8 up, 8 in 2024-09-21T09:28:27.169 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:26 smithi022 ceph-mon[27763]: mgrmap e17: y(active, starting, since 0.00876528s), standbys: x 2024-09-21T09:28:27.169 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:26 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:28:27.169 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:26 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:28:27.169 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:26 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:28:27.169 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:26 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-09-21T09:28:27.169 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:26 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-09-21T09:28:27.169 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:26 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-21T09:28:27.169 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:26 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-21T09:28:27.169 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:26 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:28:27.170 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:26 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:28:27.170 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:26 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:28:27.170 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:26 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:28:27.170 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:26 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:28:27.170 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:26 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:28:27.170 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:26 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:28:27.170 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:26 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:28:27.170 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:26 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T09:28:27.170 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:26 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T09:28:27.170 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:26 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T09:28:27.170 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:26 smithi022 ceph-mon[27763]: Manager daemon y is now available 2024-09-21T09:28:27.170 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:26 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:28:27.170 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:26 smithi022 ceph-mon[39525]: Standby manager daemon x restarted 2024-09-21T09:28:27.171 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:26 smithi022 ceph-mon[39525]: Standby manager daemon x started 2024-09-21T09:28:27.171 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:26 smithi022 ceph-mon[39525]: from='mgr.? 172.21.15.64:0/135146349' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-21T09:28:27.171 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:26 smithi022 ceph-mon[39525]: from='mgr.? 172.21.15.64:0/135146349' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:28:27.171 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:26 smithi022 ceph-mon[39525]: from='mgr.? 172.21.15.64:0/135146349' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-21T09:28:27.171 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:26 smithi022 ceph-mon[39525]: from='mgr.? 172.21.15.64:0/135146349' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:28:27.171 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:26 smithi022 ceph-mon[39525]: Active manager daemon y restarted 2024-09-21T09:28:27.171 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:26 smithi022 ceph-mon[39525]: Activating manager daemon y 2024-09-21T09:28:27.171 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:26 smithi022 ceph-mon[39525]: osdmap e60: 8 total, 8 up, 8 in 2024-09-21T09:28:27.171 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:26 smithi022 ceph-mon[39525]: mgrmap e17: y(active, starting, since 0.00876528s), standbys: x 2024-09-21T09:28:27.171 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:26 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:28:27.171 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:26 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:28:27.171 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:26 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:28:27.171 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:26 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-09-21T09:28:27.171 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:26 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-09-21T09:28:27.171 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:26 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-21T09:28:27.171 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:26 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-21T09:28:27.172 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:26 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:28:27.172 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:26 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:28:27.172 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:26 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:28:27.172 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:26 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:28:27.172 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:26 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:28:27.172 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:26 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:28:27.172 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:26 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:28:27.172 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:26 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:28:27.172 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:26 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T09:28:27.172 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:26 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T09:28:27.172 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:26 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T09:28:27.172 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:26 smithi022 ceph-mon[39525]: Manager daemon y is now available 2024-09-21T09:28:27.172 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:26 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:28:27.840 INFO:teuthology.orchestra.run.smithi022.stderr:pool 'rbd' created 2024-09-21T09:28:28.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:27 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:28:28.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:27 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:28:28.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:27 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-21T09:28:28.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:27 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-21T09:28:28.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:27 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-21T09:28:28.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:27 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-21T09:28:28.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:27 smithi064 ceph-mon[31124]: from='client.? 172.21.15.22:0/2621826761' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-09-21T09:28:28.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:27 smithi064 ceph-mon[31124]: mgrmap e18: y(active, since 1.01204s), standbys: x 2024-09-21T09:28:28.234 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:27 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:28:28.234 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:27 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:28:28.234 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:27 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-21T09:28:28.234 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:27 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-21T09:28:28.234 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:27 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-21T09:28:28.234 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:27 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-21T09:28:28.234 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:27 smithi022 ceph-mon[39525]: from='client.? 172.21.15.22:0/2621826761' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-09-21T09:28:28.234 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:27 smithi022 ceph-mon[39525]: mgrmap e18: y(active, since 1.01204s), standbys: x 2024-09-21T09:28:28.235 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:27 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:28:28.235 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:27 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:28:28.235 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:27 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-21T09:28:28.235 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:27 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-21T09:28:28.235 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:27 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-21T09:28:28.235 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:27 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-21T09:28:28.235 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:27 smithi022 ceph-mon[27763]: from='client.? 172.21.15.22:0/2621826761' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-09-21T09:28:28.235 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:27 smithi022 ceph-mon[27763]: mgrmap e18: y(active, since 1.01204s), standbys: x 2024-09-21T09:28:28.644 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d65e801e-77fa-11ef-baf6-efdc52797490 -- rbd pool init rbd 2024-09-21T09:28:29.042 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:28:29.049 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:28 smithi022 ceph-mon[39525]: pgmap v3: 97 pgs: 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:28:29.049 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:28 smithi022 ceph-mon[39525]: from='client.? 172.21.15.22:0/2621826761' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-09-21T09:28:29.049 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:28 smithi022 ceph-mon[39525]: osdmap e61: 8 total, 8 up, 8 in 2024-09-21T09:28:29.049 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:28 smithi022 ceph-mon[27763]: pgmap v3: 97 pgs: 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:28:29.050 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:28 smithi022 ceph-mon[27763]: from='client.? 172.21.15.22:0/2621826761' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-09-21T09:28:29.050 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:28 smithi022 ceph-mon[27763]: osdmap e61: 8 total, 8 up, 8 in 2024-09-21T09:28:29.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:28 smithi064 ceph-mon[31124]: pgmap v3: 97 pgs: 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:28:29.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:28 smithi064 ceph-mon[31124]: from='client.? 172.21.15.22:0/2621826761' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-09-21T09:28:29.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:28 smithi064 ceph-mon[31124]: osdmap e61: 8 total, 8 up, 8 in 2024-09-21T09:28:29.306 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:28:29 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:28:29.047+0000 7fe4f418b640 -1 log_channel(cephadm) log [ERR] : [21/Sep/2024:09:28:29] ENGINE Error in HTTPServer.serve 2024-09-21T09:28:29.307 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:28:29 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: Traceback (most recent call last): 2024-09-21T09:28:29.307 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:28:29 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T09:28:29.307 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:28:29 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: self._connections.run(self.expiration_interval) 2024-09-21T09:28:29.307 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:28:29 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T09:28:29.307 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:28:29 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: self._run(expiration_interval) 2024-09-21T09:28:29.307 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:28:29 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T09:28:29.307 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:28:29 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T09:28:29.307 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:28:29 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T09:28:29.307 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:28:29 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T09:28:29.307 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:28:29 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T09:28:29.307 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:28:29 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: s = self.context.wrap_socket( 2024-09-21T09:28:29.308 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:28:29 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T09:28:29.308 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:28:29 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: return self.sslsocket_class._create( 2024-09-21T09:28:29.308 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:28:29 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T09:28:29.308 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:28:29 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: self.do_handshake() 2024-09-21T09:28:29.308 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:28:29 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T09:28:29.308 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:28:29 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: self._sslobj.do_handshake() 2024-09-21T09:28:29.308 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:28:29 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T09:28:29.308 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:28:29 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:28:30.147 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:29 smithi022 ceph-mon[39525]: osdmap e62: 8 total, 8 up, 8 in 2024-09-21T09:28:30.147 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:29 smithi022 ceph-mon[39525]: mgrmap e19: y(active, since 2s), standbys: x 2024-09-21T09:28:30.148 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:29 smithi022 ceph-mon[39525]: [21/Sep/2024:09:28:28] ENGINE Bus STARTING 2024-09-21T09:28:30.148 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:29 smithi022 ceph-mon[39525]: [21/Sep/2024:09:28:29] ENGINE Serving on https://172.21.15.22:7150 2024-09-21T09:28:30.148 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:29 smithi022 ceph-mon[39525]: [21/Sep/2024:09:28:29] ENGINE Bus STARTED 2024-09-21T09:28:30.148 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:29 smithi022 ceph-mon[39525]: [21/Sep/2024:09:28:29] ENGINE Error in HTTPServer.serve 2024-09-21T09:28:30.148 INFO:journalctl@ceph.mon.c.smithi022.stdout: Traceback (most recent call last): 2024-09-21T09:28:30.148 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T09:28:30.149 INFO:journalctl@ceph.mon.c.smithi022.stdout: self._connections.run(self.expiration_interval) 2024-09-21T09:28:30.149 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T09:28:30.149 INFO:journalctl@ceph.mon.c.smithi022.stdout: self._run(expiration_interval) 2024-09-21T09:28:30.149 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T09:28:30.149 INFO:journalctl@ceph.mon.c.smithi022.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T09:28:30.149 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T09:28:30.150 INFO:journalctl@ceph.mon.c.smithi022.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T09:28:30.150 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T09:28:30.150 INFO:journalctl@ceph.mon.c.smithi022.stdout: s = self.context.wrap_socket( 2024-09-21T09:28:30.150 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T09:28:30.150 INFO:journalctl@ceph.mon.c.smithi022.stdout: return self.sslsocket_class._create( 2024-09-21T09:28:30.150 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T09:28:30.151 INFO:journalctl@ceph.mon.c.smithi022.stdout: self.do_handshake() 2024-09-21T09:28:30.151 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T09:28:30.151 INFO:journalctl@ceph.mon.c.smithi022.stdout: self._sslobj.do_handshake() 2024-09-21T09:28:30.151 INFO:journalctl@ceph.mon.c.smithi022.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T09:28:30.151 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:29 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:28:30.151 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:29 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:28:30.152 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:29 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:28:30.152 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:29 smithi022 ceph-mon[39525]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:28:30.152 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:29 smithi022 ceph-mon[39525]: osdmap e63: 8 total, 8 up, 8 in 2024-09-21T09:28:30.152 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:29 smithi022 ceph-mon[27763]: osdmap e62: 8 total, 8 up, 8 in 2024-09-21T09:28:30.152 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:29 smithi022 ceph-mon[27763]: mgrmap e19: y(active, since 2s), standbys: x 2024-09-21T09:28:30.153 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:29 smithi022 ceph-mon[27763]: [21/Sep/2024:09:28:28] ENGINE Bus STARTING 2024-09-21T09:28:30.153 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:29 smithi022 ceph-mon[27763]: [21/Sep/2024:09:28:29] ENGINE Serving on https://172.21.15.22:7150 2024-09-21T09:28:30.153 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:29 smithi022 ceph-mon[27763]: [21/Sep/2024:09:28:29] ENGINE Bus STARTED 2024-09-21T09:28:30.153 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:29 smithi022 ceph-mon[27763]: [21/Sep/2024:09:28:29] ENGINE Error in HTTPServer.serve 2024-09-21T09:28:30.153 INFO:journalctl@ceph.mon.a.smithi022.stdout: Traceback (most recent call last): 2024-09-21T09:28:30.153 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T09:28:30.153 INFO:journalctl@ceph.mon.a.smithi022.stdout: self._connections.run(self.expiration_interval) 2024-09-21T09:28:30.154 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T09:28:30.154 INFO:journalctl@ceph.mon.a.smithi022.stdout: self._run(expiration_interval) 2024-09-21T09:28:30.154 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T09:28:30.154 INFO:journalctl@ceph.mon.a.smithi022.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T09:28:30.154 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T09:28:30.154 INFO:journalctl@ceph.mon.a.smithi022.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T09:28:30.155 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T09:28:30.155 INFO:journalctl@ceph.mon.a.smithi022.stdout: s = self.context.wrap_socket( 2024-09-21T09:28:30.155 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T09:28:30.155 INFO:journalctl@ceph.mon.a.smithi022.stdout: return self.sslsocket_class._create( 2024-09-21T09:28:30.155 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T09:28:30.155 INFO:journalctl@ceph.mon.a.smithi022.stdout: self.do_handshake() 2024-09-21T09:28:30.156 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T09:28:30.156 INFO:journalctl@ceph.mon.a.smithi022.stdout: self._sslobj.do_handshake() 2024-09-21T09:28:30.156 INFO:journalctl@ceph.mon.a.smithi022.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T09:28:30.156 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:29 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:28:30.156 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:29 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:28:30.156 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:29 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:28:30.156 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:29 smithi022 ceph-mon[27763]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:28:30.157 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:29 smithi022 ceph-mon[27763]: osdmap e63: 8 total, 8 up, 8 in 2024-09-21T09:28:30.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:29 smithi064 ceph-mon[31124]: osdmap e62: 8 total, 8 up, 8 in 2024-09-21T09:28:30.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:29 smithi064 ceph-mon[31124]: mgrmap e19: y(active, since 2s), standbys: x 2024-09-21T09:28:30.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:29 smithi064 ceph-mon[31124]: [21/Sep/2024:09:28:28] ENGINE Bus STARTING 2024-09-21T09:28:30.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:29 smithi064 ceph-mon[31124]: [21/Sep/2024:09:28:29] ENGINE Serving on https://172.21.15.22:7150 2024-09-21T09:28:30.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:29 smithi064 ceph-mon[31124]: [21/Sep/2024:09:28:29] ENGINE Bus STARTED 2024-09-21T09:28:30.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:29 smithi064 ceph-mon[31124]: [21/Sep/2024:09:28:29] ENGINE Error in HTTPServer.serve 2024-09-21T09:28:30.192 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-09-21T09:28:30.192 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T09:28:30.192 INFO:journalctl@ceph.mon.b.smithi064.stdout: self._connections.run(self.expiration_interval) 2024-09-21T09:28:30.193 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T09:28:30.193 INFO:journalctl@ceph.mon.b.smithi064.stdout: self._run(expiration_interval) 2024-09-21T09:28:30.193 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T09:28:30.193 INFO:journalctl@ceph.mon.b.smithi064.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T09:28:30.193 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T09:28:30.193 INFO:journalctl@ceph.mon.b.smithi064.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T09:28:30.193 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T09:28:30.193 INFO:journalctl@ceph.mon.b.smithi064.stdout: s = self.context.wrap_socket( 2024-09-21T09:28:30.193 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T09:28:30.193 INFO:journalctl@ceph.mon.b.smithi064.stdout: return self.sslsocket_class._create( 2024-09-21T09:28:30.193 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T09:28:30.193 INFO:journalctl@ceph.mon.b.smithi064.stdout: self.do_handshake() 2024-09-21T09:28:30.193 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T09:28:30.193 INFO:journalctl@ceph.mon.b.smithi064.stdout: self._sslobj.do_handshake() 2024-09-21T09:28:30.193 INFO:journalctl@ceph.mon.b.smithi064.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T09:28:30.194 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:29 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:28:30.194 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:29 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:28:30.194 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:29 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:28:30.194 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:29 smithi064 ceph-mon[31124]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:28:30.194 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:29 smithi064 ceph-mon[31124]: osdmap e63: 8 total, 8 up, 8 in 2024-09-21T09:28:30.941 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:28:30 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[33213]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:28:30] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-21T09:28:31.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:31 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:28:31.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:31 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:28:31.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:31 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:28:31.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:31 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:28:31.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:31 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:28:31.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:31 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:28:31.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:31 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:28:31.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:31 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:28:31.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:31 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:28:31.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:31 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:28:31.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:31 smithi022 ceph-mon[39525]: Adjusting osd_memory_target on smithi064 to 2992M 2024-09-21T09:28:31.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:31 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:28:31.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:31 smithi022 ceph-mon[39525]: from='client.? 172.21.15.22:0/2709109029' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]: dispatch 2024-09-21T09:28:31.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:31 smithi022 ceph-mon[39525]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]: dispatch 2024-09-21T09:28:31.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:31 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:28:31.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:31 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:28:31.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:31 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:28:31.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:31 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:28:31.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:31 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:28:31.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:31 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:28:31.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:31 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:28:31.310 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:31 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:28:31.310 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:31 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:28:31.310 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:31 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:28:31.310 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:31 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:28:31.310 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:31 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:28:31.310 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:31 smithi022 ceph-mon[27763]: Adjusting osd_memory_target on smithi064 to 2992M 2024-09-21T09:28:31.310 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:31 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:28:31.310 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:31 smithi022 ceph-mon[27763]: from='client.? 172.21.15.22:0/2709109029' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]: dispatch 2024-09-21T09:28:31.310 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:31 smithi022 ceph-mon[27763]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]: dispatch 2024-09-21T09:28:31.310 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:31 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:28:31.310 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:31 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:28:31.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:31 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:28:31.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:31 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:28:31.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:31 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:28:31.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:31 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:28:31.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:31 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:28:31.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:31 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:28:31.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:31 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:28:31.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:31 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:28:31.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:31 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:28:31.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:31 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:28:31.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:31 smithi064 ceph-mon[31124]: Adjusting osd_memory_target on smithi064 to 2992M 2024-09-21T09:28:31.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:31 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:28:31.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:31 smithi064 ceph-mon[31124]: from='client.? 172.21.15.22:0/2709109029' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]: dispatch 2024-09-21T09:28:31.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:31 smithi064 ceph-mon[31124]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]: dispatch 2024-09-21T09:28:31.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:31 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:28:31.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:31 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:28:32.343 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:32 smithi064 ceph-mon[31124]: pgmap v8: 105 pgs: 8 unknown, 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:28:32.344 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:32 smithi064 ceph-mon[31124]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]': finished 2024-09-21T09:28:32.344 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:32 smithi064 ceph-mon[31124]: osdmap e64: 8 total, 8 up, 8 in 2024-09-21T09:28:32.344 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:32 smithi064 ceph-mon[31124]: mgrmap e20: y(active, since 4s), standbys: x 2024-09-21T09:28:32.344 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:32 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:28:32.344 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:32 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:28:32.344 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:32 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:28:32.344 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:32 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:28:32.344 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:32 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:28:32.344 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:32 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:28:32.344 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:32 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:28:32.344 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:32 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:28:32.344 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:32 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:28:32.345 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:32 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:28:32.345 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:32 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:28:32.345 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:32 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:28:32.345 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:32 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:28:32.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:32 smithi022 ceph-mon[39525]: pgmap v8: 105 pgs: 8 unknown, 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:28:32.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:32 smithi022 ceph-mon[39525]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]': finished 2024-09-21T09:28:32.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:32 smithi022 ceph-mon[39525]: osdmap e64: 8 total, 8 up, 8 in 2024-09-21T09:28:32.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:32 smithi022 ceph-mon[39525]: mgrmap e20: y(active, since 4s), standbys: x 2024-09-21T09:28:32.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:32 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:28:32.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:32 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:28:32.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:32 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:28:32.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:32 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:28:32.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:32 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:28:32.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:32 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:28:32.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:32 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:28:32.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:32 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:28:32.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:32 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:28:32.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:32 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:28:32.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:32 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:28:32.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:32 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:28:32.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:32 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:28:32.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:32 smithi022 ceph-mon[27763]: pgmap v8: 105 pgs: 8 unknown, 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:28:32.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:32 smithi022 ceph-mon[27763]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]': finished 2024-09-21T09:28:32.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:32 smithi022 ceph-mon[27763]: osdmap e64: 8 total, 8 up, 8 in 2024-09-21T09:28:32.560 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:32 smithi022 ceph-mon[27763]: mgrmap e20: y(active, since 4s), standbys: x 2024-09-21T09:28:32.560 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:32 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:28:32.560 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:32 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:28:32.560 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:32 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:28:32.560 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:32 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:28:32.560 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:32 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:28:32.560 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:32 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:28:32.560 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:32 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:28:32.560 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:32 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:28:32.560 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:32 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:28:32.560 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:32 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:28:32.560 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:32 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:28:32.560 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:32 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:28:32.560 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:32 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:28:33.417 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:33 smithi022 ceph-mon[39525]: Adjusting osd_memory_target on smithi022 to 2993M 2024-09-21T09:28:33.417 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:33 smithi022 ceph-mon[39525]: Updating smithi022:/etc/ceph/ceph.conf 2024-09-21T09:28:33.417 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:33 smithi022 ceph-mon[39525]: Updating smithi064:/etc/ceph/ceph.conf 2024-09-21T09:28:33.417 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:33 smithi022 ceph-mon[39525]: Updating smithi022:/var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:28:33.417 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:33 smithi022 ceph-mon[39525]: Updating smithi064:/var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:28:33.418 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:33 smithi022 ceph-mon[39525]: Updating smithi022:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:28:33.418 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:33 smithi022 ceph-mon[39525]: Updating smithi064:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:28:33.418 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:33 smithi022 ceph-mon[39525]: osdmap e65: 8 total, 8 up, 8 in 2024-09-21T09:28:33.418 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:33 smithi022 ceph-mon[39525]: Updating smithi022:/var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:28:33.418 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:33 smithi022 ceph-mon[39525]: Updating smithi064:/var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:28:33.418 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:33 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:28:33.418 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:33 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:28:33.418 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:33 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:28:33.418 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:33 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:28:33.418 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:33 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:28:33.418 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:33 smithi022 ceph-mon[27763]: Adjusting osd_memory_target on smithi022 to 2993M 2024-09-21T09:28:33.419 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:33 smithi022 ceph-mon[27763]: Updating smithi022:/etc/ceph/ceph.conf 2024-09-21T09:28:33.419 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:33 smithi022 ceph-mon[27763]: Updating smithi064:/etc/ceph/ceph.conf 2024-09-21T09:28:33.419 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:33 smithi022 ceph-mon[27763]: Updating smithi022:/var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:28:33.419 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:33 smithi022 ceph-mon[27763]: Updating smithi064:/var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:28:33.419 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:33 smithi022 ceph-mon[27763]: Updating smithi022:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:28:33.419 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:33 smithi022 ceph-mon[27763]: Updating smithi064:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:28:33.419 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:33 smithi022 ceph-mon[27763]: osdmap e65: 8 total, 8 up, 8 in 2024-09-21T09:28:33.419 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:33 smithi022 ceph-mon[27763]: Updating smithi022:/var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:28:33.419 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:33 smithi022 ceph-mon[27763]: Updating smithi064:/var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:28:33.419 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:33 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:28:33.419 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:33 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:28:33.419 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:33 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:28:33.419 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:33 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:28:33.419 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:33 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:28:33.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:33 smithi064 ceph-mon[31124]: Adjusting osd_memory_target on smithi022 to 2993M 2024-09-21T09:28:33.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:33 smithi064 ceph-mon[31124]: Updating smithi022:/etc/ceph/ceph.conf 2024-09-21T09:28:33.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:33 smithi064 ceph-mon[31124]: Updating smithi064:/etc/ceph/ceph.conf 2024-09-21T09:28:33.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:33 smithi064 ceph-mon[31124]: Updating smithi022:/var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:28:33.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:33 smithi064 ceph-mon[31124]: Updating smithi064:/var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:28:33.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:33 smithi064 ceph-mon[31124]: Updating smithi022:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:28:33.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:33 smithi064 ceph-mon[31124]: Updating smithi064:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:28:33.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:33 smithi064 ceph-mon[31124]: osdmap e65: 8 total, 8 up, 8 in 2024-09-21T09:28:33.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:33 smithi064 ceph-mon[31124]: Updating smithi022:/var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:28:33.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:33 smithi064 ceph-mon[31124]: Updating smithi064:/var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:28:33.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:33 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:28:33.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:33 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:28:33.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:33 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:28:33.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:33 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:28:33.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:33 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:28:34.036 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid d65e801e-77fa-11ef-baf6-efdc52797490 -- ceph config log 1 --format=json 2024-09-21T09:28:34.295 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:28:34.307 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:34 smithi022 ceph-mon[39525]: Deploying daemon alertmanager.a on smithi022 2024-09-21T09:28:34.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:34 smithi022 ceph-mon[39525]: pgmap v11: 105 pgs: 8 unknown, 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:28:34.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:34 smithi022 ceph-mon[39525]: osdmap e66: 8 total, 8 up, 8 in 2024-09-21T09:28:34.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:34 smithi022 ceph-mon[39525]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:28:34.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:34 smithi022 ceph-mon[39525]: Cluster is now healthy 2024-09-21T09:28:34.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:34 smithi022 ceph-mon[27763]: Deploying daemon alertmanager.a on smithi022 2024-09-21T09:28:34.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:34 smithi022 ceph-mon[27763]: pgmap v11: 105 pgs: 8 unknown, 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:28:34.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:34 smithi022 ceph-mon[27763]: osdmap e66: 8 total, 8 up, 8 in 2024-09-21T09:28:34.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:34 smithi022 ceph-mon[27763]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:28:34.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:34 smithi022 ceph-mon[27763]: Cluster is now healthy 2024-09-21T09:28:34.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:34 smithi064 ceph-mon[31124]: Deploying daemon alertmanager.a on smithi022 2024-09-21T09:28:34.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:34 smithi064 ceph-mon[31124]: pgmap v11: 105 pgs: 8 unknown, 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:28:34.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:34 smithi064 ceph-mon[31124]: osdmap e66: 8 total, 8 up, 8 in 2024-09-21T09:28:34.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:34 smithi064 ceph-mon[31124]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:28:34.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:34 smithi064 ceph-mon[31124]: Cluster is now healthy 2024-09-21T09:28:36.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:36 smithi064 ceph-mon[31124]: pgmap v13: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 36 KiB/s rd, 206 B/s wr, 14 op/s 2024-09-21T09:28:36.505 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:36 smithi022 ceph-mon[27763]: pgmap v13: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 36 KiB/s rd, 206 B/s wr, 14 op/s 2024-09-21T09:28:36.506 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:36 smithi022 ceph-mon[39525]: pgmap v13: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 36 KiB/s rd, 206 B/s wr, 14 op/s 2024-09-21T09:28:37.480 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-09-21T09:28:38.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:38 smithi022 ceph-mon[27763]: pgmap v14: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 170 B/s wr, 12 op/s 2024-09-21T09:28:38.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:38 smithi022 ceph-mon[27763]: from='client.? 172.21.15.22:0/2405853525' entity='client.admin' cmd=[{"prefix": "config log", "num": 1, "format": "json"}]: dispatch 2024-09-21T09:28:38.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:38 smithi022 ceph-mon[39525]: pgmap v14: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 170 B/s wr, 12 op/s 2024-09-21T09:28:38.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:38 smithi022 ceph-mon[39525]: from='client.? 172.21.15.22:0/2405853525' entity='client.admin' cmd=[{"prefix": "config log", "num": 1, "format": "json"}]: dispatch 2024-09-21T09:28:38.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:38 smithi064 ceph-mon[31124]: pgmap v14: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 170 B/s wr, 12 op/s 2024-09-21T09:28:38.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:38 smithi064 ceph-mon[31124]: from='client.? 172.21.15.22:0/2405853525' entity='client.admin' cmd=[{"prefix": "config log", "num": 1, "format": "json"}]: dispatch 2024-09-21T09:28:39.489 INFO:teuthology.orchestra.run.smithi022.stdout:[{"version":20,"timestamp":"2024-09-21T09:28:31.500184+0000","name":"","changes":[{"name":"osd/host:smithi022/osd_memory_target","previous_value":"4212220313","new_value":"3138478489"}]}] 2024-09-21T09:28:39.490 INFO:tasks.ceph_manager:config epoch is 20 2024-09-21T09:28:39.490 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-09-21T09:28:39.490 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-09-21T09:28:39.490 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid d65e801e-77fa-11ef-baf6-efdc52797490 -- ceph mgr dump --format=json 2024-09-21T09:28:40.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:40 smithi064 ceph-mon[31124]: pgmap v15: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 264 B/s wr, 9 op/s 2024-09-21T09:28:40.494 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:28:40.557 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:40 smithi022 ceph-mon[39525]: pgmap v15: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 264 B/s wr, 9 op/s 2024-09-21T09:28:40.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:40 smithi022 ceph-mon[27763]: pgmap v15: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 264 B/s wr, 9 op/s 2024-09-21T09:28:40.941 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:28:40 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[33213]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:28:40] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-21T09:28:41.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:41 smithi064 ceph-mon[31124]: osdmap e67: 8 total, 8 up, 8 in 2024-09-21T09:28:41.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:41 smithi064 ceph-mon[31124]: pgmap v17: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 22 KiB/s rd, 255 B/s wr, 9 op/s 2024-09-21T09:28:41.758 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:41 smithi022 ceph-mon[27763]: osdmap e67: 8 total, 8 up, 8 in 2024-09-21T09:28:41.758 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:41 smithi022 ceph-mon[27763]: pgmap v17: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 22 KiB/s rd, 255 B/s wr, 9 op/s 2024-09-21T09:28:41.758 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:41 smithi022 ceph-mon[39525]: osdmap e67: 8 total, 8 up, 8 in 2024-09-21T09:28:41.759 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:41 smithi022 ceph-mon[39525]: pgmap v17: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 22 KiB/s rd, 255 B/s wr, 9 op/s 2024-09-21T09:28:41.759 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 21 09:28:41 smithi022 systemd[1]: Starting Ceph alertmanager.a for d65e801e-77fa-11ef-baf6-efdc52797490... 2024-09-21T09:28:42.690 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:42 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:28:42.690 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:42 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:28:42.691 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 21 09:28:42 smithi022 podman[74393]: 2024-09-21 09:28:42.389105058 +0000 UTC m=+0.246107332 image pull c8568f914cd25b2062c44e9f79f9c18da6e3b85fe0c47a12a2191c61426c2b19 quay.io/prometheus/alertmanager:v0.25.0 2024-09-21T09:28:42.691 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 21 09:28:42 smithi022 podman[74393]: 2024-09-21 09:28:42.494140231 +0000 UTC m=+0.351142516 volume create a13068a112d60b89d689283688ab7ec52792b2b573137f1b11936ca7897314cd 2024-09-21T09:28:42.691 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 21 09:28:42 smithi022 podman[74393]: 2024-09-21 09:28:42.577919254 +0000 UTC m=+0.434921536 container create c881f909bded02b97ee633e51746e57baeb514c5d105aa0731d63c13c19ae9e5 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-21T09:28:42.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:42 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:28:42.996 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-09-21T09:28:43.308 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 21 09:28:43 smithi022 podman[74393]: 2024-09-21 09:28:43.098561363 +0000 UTC m=+0.955563612 container init c881f909bded02b97ee633e51746e57baeb514c5d105aa0731d63c13c19ae9e5 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-21T09:28:43.308 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 21 09:28:43 smithi022 podman[74393]: 2024-09-21 09:28:43.103604368 +0000 UTC m=+0.960606607 container start c881f909bded02b97ee633e51746e57baeb514c5d105aa0731d63c13c19ae9e5 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-21T09:28:43.308 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 21 09:28:43 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-alertmanager-a[74454]: ts=2024-09-21T09:28:43.115Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-09-21T09:28:43.308 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 21 09:28:43 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-alertmanager-a[74454]: ts=2024-09-21T09:28:43.115Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-09-21T09:28:43.308 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 21 09:28:43 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-alertmanager-a[74454]: ts=2024-09-21T09:28:43.117Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.22 port=9094 2024-09-21T09:28:43.308 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 21 09:28:43 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-alertmanager-a[74454]: ts=2024-09-21T09:28:43.118Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-09-21T09:28:43.308 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 21 09:28:43 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-alertmanager-a[74454]: ts=2024-09-21T09:28:43.141Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-21T09:28:43.308 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 21 09:28:43 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-alertmanager-a[74454]: ts=2024-09-21T09:28:43.142Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-21T09:28:43.309 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 21 09:28:43 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-alertmanager-a[74454]: ts=2024-09-21T09:28:43.144Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-09-21T09:28:43.309 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 21 09:28:43 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-alertmanager-a[74454]: ts=2024-09-21T09:28:43.144Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-09-21T09:28:43.309 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 21 09:28:43 smithi022 bash[74393]: c881f909bded02b97ee633e51746e57baeb514c5d105aa0731d63c13c19ae9e5 2024-09-21T09:28:43.309 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 21 09:28:43 smithi022 systemd[1]: Started Ceph alertmanager.a for d65e801e-77fa-11ef-baf6-efdc52797490. 2024-09-21T09:28:43.624 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:43 smithi022 ceph-mon[27763]: pgmap v18: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 105 B/s wr, 0 op/s 2024-09-21T09:28:43.624 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:43 smithi022 ceph-mon[27763]: from='client.? 172.21.15.22:0/2981853985' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-21T09:28:43.624 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:43 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:28:43.624 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:43 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:28:43.624 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:43 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:28:43.624 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:43 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:28:43.624 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:43 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:28:43.624 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:43 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:28:43.624 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:43 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-21T09:28:43.624 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:43 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:28:43.625 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:43 smithi022 ceph-mon[39525]: pgmap v18: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 105 B/s wr, 0 op/s 2024-09-21T09:28:43.625 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:43 smithi022 ceph-mon[39525]: from='client.? 172.21.15.22:0/2981853985' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-21T09:28:43.625 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:43 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:28:43.625 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:43 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:28:43.625 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:43 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:28:43.625 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:43 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:28:43.625 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:43 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:28:43.625 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:43 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:28:43.625 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:43 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-21T09:28:43.625 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:43 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:28:43.625 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:28:43 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: [21/Sep/2024:09:28:43] ENGINE Bus STOPPING 2024-09-21T09:28:43.626 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:28:43 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: [21/Sep/2024:09:28:43] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-21T09:28:43.626 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:28:43 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: [21/Sep/2024:09:28:43] ENGINE Bus STOPPED 2024-09-21T09:28:43.626 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:28:43 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: [21/Sep/2024:09:28:43] ENGINE Bus STARTING 2024-09-21T09:28:43.626 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:28:43 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: [21/Sep/2024:09:28:43] ENGINE Serving on http://:::9283 2024-09-21T09:28:43.626 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:28:43 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: [21/Sep/2024:09:28:43] ENGINE Bus STARTED 2024-09-21T09:28:43.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:43 smithi064 ceph-mon[31124]: pgmap v18: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 105 B/s wr, 0 op/s 2024-09-21T09:28:43.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:43 smithi064 ceph-mon[31124]: from='client.? 172.21.15.22:0/2981853985' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-21T09:28:43.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:43 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:28:43.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:43 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:28:43.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:43 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:28:43.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:43 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:28:43.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:43 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:28:43.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:43 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:28:43.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:43 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-21T09:28:43.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:43 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:28:44.310 INFO:teuthology.orchestra.run.smithi022.stdout:{"epoch":20,"flags":0,"active_gid":14541,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6800","nonce":3088744553},{"type":"v1","addr":"172.21.15.22:6801","nonce":3088744553}]},"active_addr":"172.21.15.22:6801/3088744553","active_change":"2024-09-21T09:28:26.777321+0000","active_mgr_features":4540138320759226367,"available":true,"standbys":[{"gid":24415,"name":"x","mgr_features":4540138320759226367,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.22:8443/","prometheus":"http://172.21.15.22:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":60,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.22:0","nonce":438209926}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.22:0","nonce":2111825743}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.22:0","nonce":3644531628}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.22:0","nonce":4075962746}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.22:0","nonce":700573682}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.22:0","nonce":130938820}]}]} 2024-09-21T09:28:44.315 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-09-21T09:28:44.315 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-09-21T09:28:44.315 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid d65e801e-77fa-11ef-baf6-efdc52797490 -- ceph osd dump --format=json 2024-09-21T09:28:44.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:44 smithi022 ceph-mon[39525]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-21T09:28:44.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:44 smithi022 ceph-mon[39525]: Deploying daemon grafana.a on smithi064 2024-09-21T09:28:44.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:44 smithi022 ceph-mon[27763]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-21T09:28:44.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:44 smithi022 ceph-mon[27763]: Deploying daemon grafana.a on smithi064 2024-09-21T09:28:44.576 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:28:44.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:44 smithi064 ceph-mon[31124]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-21T09:28:44.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:44 smithi064 ceph-mon[31124]: Deploying daemon grafana.a on smithi064 2024-09-21T09:28:45.308 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 21 09:28:45 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-alertmanager-a[74454]: ts=2024-09-21T09:28:45.119Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000534026s 2024-09-21T09:28:45.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:45 smithi064 ceph-mon[31124]: pgmap v19: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 102 B/s wr, 0 op/s 2024-09-21T09:28:45.787 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:45 smithi022 ceph-mon[39525]: pgmap v19: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 102 B/s wr, 0 op/s 2024-09-21T09:28:45.788 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:45 smithi022 ceph-mon[27763]: pgmap v19: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 102 B/s wr, 0 op/s 2024-09-21T09:28:46.525 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-09-21T09:28:46.525 INFO:teuthology.orchestra.run.smithi022.stdout:{"epoch":67,"fsid":"d65e801e-77fa-11ef-baf6-efdc52797490","created":"2024-09-21T09:22:04.474917+0000","modified":"2024-09-21T09:28:40.372343+0000","last_up_change":"2024-09-21T09:27:06.441515+0000","last_in_change":"2024-09-21T09:26:49.376072+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-21T09:25:07.405775+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-09-21T09:27:31.151960+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"57","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-09-21T09:27:34.558875+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"57","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}}},{"pool":4,"pool_name":"rbd","create_time":"2024-09-21T09:28:27.564651+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"66","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":66,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"a1fbb74d-ffae-4d5f-81ab-9677d258a95c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6802","nonce":3576301437},{"type":"v1","addr":"172.21.15.22:6803","nonce":3576301437}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6804","nonce":3576301437},{"type":"v1","addr":"172.21.15.22:6805","nonce":3576301437}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6808","nonce":3576301437},{"type":"v1","addr":"172.21.15.22:6809","nonce":3576301437}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6806","nonce":3576301437},{"type":"v1","addr":"172.21.15.22:6807","nonce":3576301437}]},"public_addr":"172.21.15.22:6803/3576301437","cluster_addr":"172.21.15.22:6805/3576301437","heartbeat_back_addr":"172.21.15.22:6809/3576301437","heartbeat_front_addr":"172.21.15.22:6807/3576301437","state":["exists","up"]},{"osd":1,"uuid":"fa4c9385-ac93-4983-92b4-f23f0a3731db","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6810","nonce":3562679765},{"type":"v1","addr":"172.21.15.22:6811","nonce":3562679765}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6812","nonce":3562679765},{"type":"v1","addr":"172.21.15.22:6813","nonce":3562679765}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6816","nonce":3562679765},{"type":"v1","addr":"172.21.15.22:6817","nonce":3562679765}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6814","nonce":3562679765},{"type":"v1","addr":"172.21.15.22:6815","nonce":3562679765}]},"public_addr":"172.21.15.22:6811/3562679765","cluster_addr":"172.21.15.22:6813/3562679765","heartbeat_back_addr":"172.21.15.22:6817/3562679765","heartbeat_front_addr":"172.21.15.22:6815/3562679765","state":["exists","up"]},{"osd":2,"uuid":"1c27d056-2218-47dd-8f13-bc1c13888483","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6818","nonce":695486895},{"type":"v1","addr":"172.21.15.22:6819","nonce":695486895}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6820","nonce":695486895},{"type":"v1","addr":"172.21.15.22:6821","nonce":695486895}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6824","nonce":695486895},{"type":"v1","addr":"172.21.15.22:6825","nonce":695486895}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6822","nonce":695486895},{"type":"v1","addr":"172.21.15.22:6823","nonce":695486895}]},"public_addr":"172.21.15.22:6819/695486895","cluster_addr":"172.21.15.22:6821/695486895","heartbeat_back_addr":"172.21.15.22:6825/695486895","heartbeat_front_addr":"172.21.15.22:6823/695486895","state":["exists","up"]},{"osd":3,"uuid":"8668a232-df4a-46ba-b97d-3ed2f6ddeab2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6826","nonce":1394296125},{"type":"v1","addr":"172.21.15.22:6827","nonce":1394296125}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6828","nonce":1394296125},{"type":"v1","addr":"172.21.15.22:6829","nonce":1394296125}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6832","nonce":1394296125},{"type":"v1","addr":"172.21.15.22:6833","nonce":1394296125}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6830","nonce":1394296125},{"type":"v1","addr":"172.21.15.22:6831","nonce":1394296125}]},"public_addr":"172.21.15.22:6827/1394296125","cluster_addr":"172.21.15.22:6829/1394296125","heartbeat_back_addr":"172.21.15.22:6833/1394296125","heartbeat_front_addr":"172.21.15.22:6831/1394296125","state":["exists","up"]},{"osd":4,"uuid":"44dd8dcc-dbf7-41f7-9ec2-e66922e6c521","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2736095169},{"type":"v1","addr":"172.21.15.64:6801","nonce":2736095169}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6802","nonce":2736095169},{"type":"v1","addr":"172.21.15.64:6803","nonce":2736095169}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6806","nonce":2736095169},{"type":"v1","addr":"172.21.15.64:6807","nonce":2736095169}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":2736095169},{"type":"v1","addr":"172.21.15.64:6805","nonce":2736095169}]},"public_addr":"172.21.15.64:6801/2736095169","cluster_addr":"172.21.15.64:6803/2736095169","heartbeat_back_addr":"172.21.15.64:6807/2736095169","heartbeat_front_addr":"172.21.15.64:6805/2736095169","state":["exists","up"]},{"osd":5,"uuid":"a920292e-6301-4b67-b2a1-30f4788f8fd3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":2673197529},{"type":"v1","addr":"172.21.15.64:6809","nonce":2673197529}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2673197529},{"type":"v1","addr":"172.21.15.64:6811","nonce":2673197529}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2673197529},{"type":"v1","addr":"172.21.15.64:6815","nonce":2673197529}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6812","nonce":2673197529},{"type":"v1","addr":"172.21.15.64:6813","nonce":2673197529}]},"public_addr":"172.21.15.64:6809/2673197529","cluster_addr":"172.21.15.64:6811/2673197529","heartbeat_back_addr":"172.21.15.64:6815/2673197529","heartbeat_front_addr":"172.21.15.64:6813/2673197529","state":["exists","up"]},{"osd":6,"uuid":"ed01fe25-9b7d-48b9-b054-e054280e5826","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1977044182},{"type":"v1","addr":"172.21.15.64:6817","nonce":1977044182}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1977044182},{"type":"v1","addr":"172.21.15.64:6819","nonce":1977044182}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1977044182},{"type":"v1","addr":"172.21.15.64:6823","nonce":1977044182}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1977044182},{"type":"v1","addr":"172.21.15.64:6821","nonce":1977044182}]},"public_addr":"172.21.15.64:6817/1977044182","cluster_addr":"172.21.15.64:6819/1977044182","heartbeat_back_addr":"172.21.15.64:6823/1977044182","heartbeat_front_addr":"172.21.15.64:6821/1977044182","state":["exists","up"]},{"osd":7,"uuid":"a9433be2-fcc7-46f5-91ee-13fbe71bf2f7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":4274465032},{"type":"v1","addr":"172.21.15.64:6825","nonce":4274465032}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":4274465032},{"type":"v1","addr":"172.21.15.64:6827","nonce":4274465032}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":4274465032},{"type":"v1","addr":"172.21.15.64:6831","nonce":4274465032}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":4274465032},{"type":"v1","addr":"172.21.15.64:6829","nonce":4274465032}]},"public_addr":"172.21.15.64:6825/4274465032","cluster_addr":"172.21.15.64:6827/4274465032","heartbeat_back_addr":"172.21.15.64:6831/4274465032","heartbeat_front_addr":"172.21.15.64:6829/4274465032","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:24:15.825542+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:24:40.497779+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:25:04.918091+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:25:30.663725+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:25:53.521949+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:26:17.565302+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:26:40.876016+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:27:04.436407+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.3","mappings":[{"from":7,"to":4}]},{"pgid":"3.a","mappings":[{"from":4,"to":3}]},{"pgid":"3.b","mappings":[{"from":0,"to":2}]},{"pgid":"3.17","mappings":[{"from":0,"to":2}]},{"pgid":"3.22","mappings":[{"from":0,"to":1}]},{"pgid":"3.24","mappings":[{"from":4,"to":2}]},{"pgid":"3.37","mappings":[{"from":5,"to":1}]}],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.22:6801/2706373818":"2024-09-22T09:28:26.777111+0000","172.21.15.22:0/3249188488":"2024-09-22T09:28:26.777111+0000","172.21.15.22:6800/2983422368":"2024-09-22T09:22:26.473640+0000","172.21.15.22:6801/2983422368":"2024-09-22T09:22:26.473640+0000","172.21.15.22:6800/2706373818":"2024-09-22T09:28:26.777111+0000","172.21.15.22:0/1022854075":"2024-09-22T09:22:26.473640+0000","172.21.15.22:0/3766882712":"2024-09-22T09:22:50.910682+0000","172.21.15.22:0/1728251114":"2024-09-22T09:22:26.473640+0000","172.21.15.22:0/2669480478":"2024-09-22T09:22:50.910682+0000","172.21.15.22:0/1467628521":"2024-09-22T09:22:26.473640+0000","172.21.15.22:0/4200462415":"2024-09-22T09:22:50.910682+0000","172.21.15.22:6800/2821000087":"2024-09-22T09:22:50.910682+0000","172.21.15.22:6801/2821000087":"2024-09-22T09:22:50.910682+0000","172.21.15.22:0/4254041618":"2024-09-22T09:28:26.777111+0000","172.21.15.22:0/1998705954":"2024-09-22T09:28:26.777111+0000","172.21.15.22:0/1937461032":"2024-09-22T09:28:26.777111+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[{"pool":4,"snaps":[{"begin":2,"length":1}]}],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-21T09:28:46.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:46 smithi022 ceph-mon[27763]: from='client.? 172.21.15.22:0/48967931' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T09:28:46.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:46 smithi022 ceph-mon[39525]: from='client.? 172.21.15.22:0/48967931' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T09:28:46.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:46 smithi064 ceph-mon[31124]: from='client.? 172.21.15.22:0/48967931' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T09:28:47.176 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-09-21T09:28:47.177 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid d65e801e-77fa-11ef-baf6-efdc52797490 -- ceph osd dump --format=json 2024-09-21T09:28:47.453 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:28:48.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:47 smithi064 ceph-mon[31124]: pgmap v20: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 102 B/s wr, 0 op/s 2024-09-21T09:28:48.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:47 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:28:48.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:47 smithi022 ceph-mon[27763]: pgmap v20: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 102 B/s wr, 0 op/s 2024-09-21T09:28:48.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:47 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:28:48.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:47 smithi022 ceph-mon[39525]: pgmap v20: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 102 B/s wr, 0 op/s 2024-09-21T09:28:48.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:47 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:28:49.343 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-09-21T09:28:49.343 INFO:teuthology.orchestra.run.smithi022.stdout:{"epoch":67,"fsid":"d65e801e-77fa-11ef-baf6-efdc52797490","created":"2024-09-21T09:22:04.474917+0000","modified":"2024-09-21T09:28:40.372343+0000","last_up_change":"2024-09-21T09:27:06.441515+0000","last_in_change":"2024-09-21T09:26:49.376072+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-21T09:25:07.405775+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-09-21T09:27:31.151960+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"57","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-09-21T09:27:34.558875+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"57","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}}},{"pool":4,"pool_name":"rbd","create_time":"2024-09-21T09:28:27.564651+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"66","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":66,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"a1fbb74d-ffae-4d5f-81ab-9677d258a95c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6802","nonce":3576301437},{"type":"v1","addr":"172.21.15.22:6803","nonce":3576301437}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6804","nonce":3576301437},{"type":"v1","addr":"172.21.15.22:6805","nonce":3576301437}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6808","nonce":3576301437},{"type":"v1","addr":"172.21.15.22:6809","nonce":3576301437}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6806","nonce":3576301437},{"type":"v1","addr":"172.21.15.22:6807","nonce":3576301437}]},"public_addr":"172.21.15.22:6803/3576301437","cluster_addr":"172.21.15.22:6805/3576301437","heartbeat_back_addr":"172.21.15.22:6809/3576301437","heartbeat_front_addr":"172.21.15.22:6807/3576301437","state":["exists","up"]},{"osd":1,"uuid":"fa4c9385-ac93-4983-92b4-f23f0a3731db","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6810","nonce":3562679765},{"type":"v1","addr":"172.21.15.22:6811","nonce":3562679765}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6812","nonce":3562679765},{"type":"v1","addr":"172.21.15.22:6813","nonce":3562679765}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6816","nonce":3562679765},{"type":"v1","addr":"172.21.15.22:6817","nonce":3562679765}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6814","nonce":3562679765},{"type":"v1","addr":"172.21.15.22:6815","nonce":3562679765}]},"public_addr":"172.21.15.22:6811/3562679765","cluster_addr":"172.21.15.22:6813/3562679765","heartbeat_back_addr":"172.21.15.22:6817/3562679765","heartbeat_front_addr":"172.21.15.22:6815/3562679765","state":["exists","up"]},{"osd":2,"uuid":"1c27d056-2218-47dd-8f13-bc1c13888483","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6818","nonce":695486895},{"type":"v1","addr":"172.21.15.22:6819","nonce":695486895}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6820","nonce":695486895},{"type":"v1","addr":"172.21.15.22:6821","nonce":695486895}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6824","nonce":695486895},{"type":"v1","addr":"172.21.15.22:6825","nonce":695486895}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6822","nonce":695486895},{"type":"v1","addr":"172.21.15.22:6823","nonce":695486895}]},"public_addr":"172.21.15.22:6819/695486895","cluster_addr":"172.21.15.22:6821/695486895","heartbeat_back_addr":"172.21.15.22:6825/695486895","heartbeat_front_addr":"172.21.15.22:6823/695486895","state":["exists","up"]},{"osd":3,"uuid":"8668a232-df4a-46ba-b97d-3ed2f6ddeab2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6826","nonce":1394296125},{"type":"v1","addr":"172.21.15.22:6827","nonce":1394296125}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6828","nonce":1394296125},{"type":"v1","addr":"172.21.15.22:6829","nonce":1394296125}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6832","nonce":1394296125},{"type":"v1","addr":"172.21.15.22:6833","nonce":1394296125}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6830","nonce":1394296125},{"type":"v1","addr":"172.21.15.22:6831","nonce":1394296125}]},"public_addr":"172.21.15.22:6827/1394296125","cluster_addr":"172.21.15.22:6829/1394296125","heartbeat_back_addr":"172.21.15.22:6833/1394296125","heartbeat_front_addr":"172.21.15.22:6831/1394296125","state":["exists","up"]},{"osd":4,"uuid":"44dd8dcc-dbf7-41f7-9ec2-e66922e6c521","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2736095169},{"type":"v1","addr":"172.21.15.64:6801","nonce":2736095169}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6802","nonce":2736095169},{"type":"v1","addr":"172.21.15.64:6803","nonce":2736095169}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6806","nonce":2736095169},{"type":"v1","addr":"172.21.15.64:6807","nonce":2736095169}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":2736095169},{"type":"v1","addr":"172.21.15.64:6805","nonce":2736095169}]},"public_addr":"172.21.15.64:6801/2736095169","cluster_addr":"172.21.15.64:6803/2736095169","heartbeat_back_addr":"172.21.15.64:6807/2736095169","heartbeat_front_addr":"172.21.15.64:6805/2736095169","state":["exists","up"]},{"osd":5,"uuid":"a920292e-6301-4b67-b2a1-30f4788f8fd3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":2673197529},{"type":"v1","addr":"172.21.15.64:6809","nonce":2673197529}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2673197529},{"type":"v1","addr":"172.21.15.64:6811","nonce":2673197529}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2673197529},{"type":"v1","addr":"172.21.15.64:6815","nonce":2673197529}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6812","nonce":2673197529},{"type":"v1","addr":"172.21.15.64:6813","nonce":2673197529}]},"public_addr":"172.21.15.64:6809/2673197529","cluster_addr":"172.21.15.64:6811/2673197529","heartbeat_back_addr":"172.21.15.64:6815/2673197529","heartbeat_front_addr":"172.21.15.64:6813/2673197529","state":["exists","up"]},{"osd":6,"uuid":"ed01fe25-9b7d-48b9-b054-e054280e5826","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1977044182},{"type":"v1","addr":"172.21.15.64:6817","nonce":1977044182}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1977044182},{"type":"v1","addr":"172.21.15.64:6819","nonce":1977044182}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1977044182},{"type":"v1","addr":"172.21.15.64:6823","nonce":1977044182}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1977044182},{"type":"v1","addr":"172.21.15.64:6821","nonce":1977044182}]},"public_addr":"172.21.15.64:6817/1977044182","cluster_addr":"172.21.15.64:6819/1977044182","heartbeat_back_addr":"172.21.15.64:6823/1977044182","heartbeat_front_addr":"172.21.15.64:6821/1977044182","state":["exists","up"]},{"osd":7,"uuid":"a9433be2-fcc7-46f5-91ee-13fbe71bf2f7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":4274465032},{"type":"v1","addr":"172.21.15.64:6825","nonce":4274465032}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":4274465032},{"type":"v1","addr":"172.21.15.64:6827","nonce":4274465032}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":4274465032},{"type":"v1","addr":"172.21.15.64:6831","nonce":4274465032}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":4274465032},{"type":"v1","addr":"172.21.15.64:6829","nonce":4274465032}]},"public_addr":"172.21.15.64:6825/4274465032","cluster_addr":"172.21.15.64:6827/4274465032","heartbeat_back_addr":"172.21.15.64:6831/4274465032","heartbeat_front_addr":"172.21.15.64:6829/4274465032","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:24:15.825542+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:24:40.497779+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:25:04.918091+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:25:30.663725+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:25:53.521949+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:26:17.565302+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:26:40.876016+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:27:04.436407+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.3","mappings":[{"from":7,"to":4}]},{"pgid":"3.a","mappings":[{"from":4,"to":3}]},{"pgid":"3.b","mappings":[{"from":0,"to":2}]},{"pgid":"3.17","mappings":[{"from":0,"to":2}]},{"pgid":"3.22","mappings":[{"from":0,"to":1}]},{"pgid":"3.24","mappings":[{"from":4,"to":2}]},{"pgid":"3.37","mappings":[{"from":5,"to":1}]}],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.22:6801/2706373818":"2024-09-22T09:28:26.777111+0000","172.21.15.22:0/3249188488":"2024-09-22T09:28:26.777111+0000","172.21.15.22:6800/2983422368":"2024-09-22T09:22:26.473640+0000","172.21.15.22:6801/2983422368":"2024-09-22T09:22:26.473640+0000","172.21.15.22:6800/2706373818":"2024-09-22T09:28:26.777111+0000","172.21.15.22:0/1022854075":"2024-09-22T09:22:26.473640+0000","172.21.15.22:0/3766882712":"2024-09-22T09:22:50.910682+0000","172.21.15.22:0/1728251114":"2024-09-22T09:22:26.473640+0000","172.21.15.22:0/2669480478":"2024-09-22T09:22:50.910682+0000","172.21.15.22:0/1467628521":"2024-09-22T09:22:26.473640+0000","172.21.15.22:0/4200462415":"2024-09-22T09:22:50.910682+0000","172.21.15.22:6800/2821000087":"2024-09-22T09:22:50.910682+0000","172.21.15.22:6801/2821000087":"2024-09-22T09:22:50.910682+0000","172.21.15.22:0/4254041618":"2024-09-22T09:28:26.777111+0000","172.21.15.22:0/1998705954":"2024-09-22T09:28:26.777111+0000","172.21.15.22:0/1937461032":"2024-09-22T09:28:26.777111+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[{"pool":4,"snaps":[{"begin":2,"length":1}]}],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-21T09:28:49.998 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid d65e801e-77fa-11ef-baf6-efdc52797490 -- ceph tell osd.0 flush_pg_stats 2024-09-21T09:28:49.998 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid d65e801e-77fa-11ef-baf6-efdc52797490 -- ceph tell osd.1 flush_pg_stats 2024-09-21T09:28:49.999 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid d65e801e-77fa-11ef-baf6-efdc52797490 -- ceph tell osd.2 flush_pg_stats 2024-09-21T09:28:49.999 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid d65e801e-77fa-11ef-baf6-efdc52797490 -- ceph tell osd.3 flush_pg_stats 2024-09-21T09:28:49.999 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid d65e801e-77fa-11ef-baf6-efdc52797490 -- ceph tell osd.4 flush_pg_stats 2024-09-21T09:28:50.000 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid d65e801e-77fa-11ef-baf6-efdc52797490 -- ceph tell osd.5 flush_pg_stats 2024-09-21T09:28:50.000 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid d65e801e-77fa-11ef-baf6-efdc52797490 -- ceph tell osd.6 flush_pg_stats 2024-09-21T09:28:50.001 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid d65e801e-77fa-11ef-baf6-efdc52797490 -- ceph tell osd.7 flush_pg_stats 2024-09-21T09:28:50.041 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:49 smithi022 ceph-mon[39525]: pgmap v21: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:28:50.042 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:49 smithi022 ceph-mon[39525]: from='client.? 172.21.15.22:0/1878855741' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T09:28:50.042 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:49 smithi022 ceph-mon[27763]: pgmap v21: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:28:50.042 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:49 smithi022 ceph-mon[27763]: from='client.? 172.21.15.22:0/1878855741' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T09:28:50.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:49 smithi064 ceph-mon[31124]: pgmap v21: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:28:50.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:49 smithi064 ceph-mon[31124]: from='client.? 172.21.15.22:0/1878855741' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T09:28:50.535 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:28:50.535 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:28:50.535 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:28:50.536 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:28:50.537 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:28:50.539 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:28:50.540 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:28:50.540 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:28:50.941 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:28:50 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[33213]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:28:50] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-21T09:28:52.000 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:51 smithi022 ceph-mon[39525]: pgmap v22: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:28:52.000 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:51 smithi022 ceph-mon[27763]: pgmap v22: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:28:52.147 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:51 smithi064 ceph-mon[31124]: pgmap v22: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:28:53.374 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 21 09:28:53 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-alertmanager-a[74454]: ts=2024-09-21T09:28:53.120Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.001906446s 2024-09-21T09:28:54.092 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:53 smithi064 ceph-mon[31124]: pgmap v23: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:28:54.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:53 smithi022 ceph-mon[39525]: pgmap v23: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:28:54.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:53 smithi022 ceph-mon[27763]: pgmap v23: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:28:55.100 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 podman[63776]: 2024-09-21 09:28:55.009696984 +0000 UTC m=+0.123113953 container create ffde1f806d0b42f9721d265e4c699a0d9778f0f82cb7ea98bcee27db7f6ec187 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a, io.openshift.expose-services=, maintainer=Paul Cuzner , vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, version=8.5, vcs-type=git, io.buildah.version=1.32.0, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, name=ubi8, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, release=239.1651231664, com.redhat.component=ubi8-container, vendor=Red Hat, Inc., io.k8s.display-name=Red Hat Universal Base Image 8, description=Ceph Grafana Container, io.openshift.tags=base rhel8, architecture=x86_64, build-date=2022-04-29T11:48:11.055897, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-09-21T09:28:55.473 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 podman[63776]: 2024-09-21 09:28:55.220399101 +0000 UTC m=+0.333816028 container init ffde1f806d0b42f9721d265e4c699a0d9778f0f82cb7ea98bcee27db7f6ec187 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a, io.openshift.tags=base rhel8, io.openshift.expose-services=, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, distribution-scope=public, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.32.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Grafana Container configured for Ceph mgr/dashboard integration, release=239.1651231664, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, version=8.5, description=Ceph Grafana Container, name=ubi8, com.redhat.component=ubi8-container, io.k8s.display-name=Red Hat Universal Base Image 8, vcs-type=git, build-date=2022-04-29T11:48:11.055897, vendor=Red Hat, Inc., maintainer=Paul Cuzner , architecture=x86_64) 2024-09-21T09:28:55.473 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 podman[63776]: 2024-09-21 09:28:55.225372182 +0000 UTC m=+0.338789108 container start ffde1f806d0b42f9721d265e4c699a0d9778f0f82cb7ea98bcee27db7f6ec187 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a, io.openshift.expose-services=, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.k8s.display-name=Red Hat Universal Base Image 8, vcs-type=git, vendor=Red Hat, Inc., version=8.5, io.buildah.version=1.32.0, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, build-date=2022-04-29T11:48:11.055897, io.openshift.tags=base rhel8, summary=Grafana Container configured for Ceph mgr/dashboard integration, maintainer=Paul Cuzner , release=239.1651231664, com.redhat.component=ubi8-container, distribution-scope=public, description=Ceph Grafana Container, name=ubi8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-09-21T09:28:55.473 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 bash[63776]: ffde1f806d0b42f9721d265e4c699a0d9778f0f82cb7ea98bcee27db7f6ec187 2024-09-21T09:28:55.474 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 systemd[1]: Started Ceph grafana.a for d65e801e-77fa-11ef-baf6-efdc52797490. 2024-09-21T09:28:55.474 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=settings t=2024-09-21T09:28:55.399459876Z level=info msg="Starting Grafana" version=9.4.7 commit=4add91f03d branch=HEAD compiled=2023-03-16T23:56:52Z 2024-09-21T09:28:55.474 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=settings t=2024-09-21T09:28:55.399648672Z level=warn msg="\"sentry\" frontend logging provider is deprecated and will be removed in the next major version. Use \"grafana\" provider instead." 2024-09-21T09:28:55.474 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=settings t=2024-09-21T09:28:55.399669948Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2024-09-21T09:28:55.474 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=settings t=2024-09-21T09:28:55.399678244Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2024-09-21T09:28:55.475 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=settings t=2024-09-21T09:28:55.399685175Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2024-09-21T09:28:55.475 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=settings t=2024-09-21T09:28:55.399691831Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2024-09-21T09:28:55.475 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=settings t=2024-09-21T09:28:55.399698335Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-09-21T09:28:55.475 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=settings t=2024-09-21T09:28:55.39970558Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-09-21T09:28:55.475 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=settings t=2024-09-21T09:28:55.399712479Z level=info msg="Path Home" path=/usr/share/grafana 2024-09-21T09:28:55.475 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=settings t=2024-09-21T09:28:55.399718914Z level=info msg="Path Data" path=/var/lib/grafana 2024-09-21T09:28:55.475 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=settings t=2024-09-21T09:28:55.399725771Z level=info msg="Path Logs" path=/var/log/grafana 2024-09-21T09:28:55.475 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=settings t=2024-09-21T09:28:55.39973197Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2024-09-21T09:28:55.475 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=settings t=2024-09-21T09:28:55.399738448Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2024-09-21T09:28:55.475 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=settings t=2024-09-21T09:28:55.399744999Z level=info msg="App mode production" 2024-09-21T09:28:55.475 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=sqlstore t=2024-09-21T09:28:55.399805047Z level=info msg="Connecting to DB" dbtype=sqlite3 2024-09-21T09:28:55.476 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=sqlstore t=2024-09-21T09:28:55.399825706Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-09-21T09:28:55.476 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.400768012Z level=info msg="Starting DB migrations" 2024-09-21T09:28:55.476 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.400847278Z level=info msg="Executing migration" id="create migration_log table" 2024-09-21T09:28:55.476 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.401571097Z level=info msg="Executing migration" id="create user table" 2024-09-21T09:28:55.476 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.402041774Z level=info msg="Executing migration" id="add unique index user.login" 2024-09-21T09:28:55.476 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.402579207Z level=info msg="Executing migration" id="add unique index user.email" 2024-09-21T09:28:55.476 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.403063622Z level=info msg="Executing migration" id="drop index UQE_user_login - v1" 2024-09-21T09:28:55.476 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.403734789Z level=info msg="Executing migration" id="drop index UQE_user_email - v1" 2024-09-21T09:28:55.476 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.404142971Z level=info msg="Executing migration" id="Rename table user to user_v1 - v1" 2024-09-21T09:28:55.476 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.404883151Z level=info msg="Executing migration" id="create user table v2" 2024-09-21T09:28:55.476 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.405284269Z level=info msg="Executing migration" id="create index UQE_user_login - v2" 2024-09-21T09:28:55.476 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.405675419Z level=info msg="Executing migration" id="create index UQE_user_email - v2" 2024-09-21T09:28:55.476 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.406106713Z level=info msg="Executing migration" id="copy data_source v1 to v2" 2024-09-21T09:28:55.476 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.406432965Z level=info msg="Executing migration" id="Drop old table user_v1" 2024-09-21T09:28:55.477 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.406815139Z level=info msg="Executing migration" id="Add column help_flags1 to user table" 2024-09-21T09:28:55.477 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.40724093Z level=info msg="Executing migration" id="Update user table charset" 2024-09-21T09:28:55.477 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.407520713Z level=info msg="Executing migration" id="Add last_seen_at column to user" 2024-09-21T09:28:55.477 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.407968439Z level=info msg="Executing migration" id="Add missing user data" 2024-09-21T09:28:55.477 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.408300688Z level=info msg="Executing migration" id="Add is_disabled column to user" 2024-09-21T09:28:55.477 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.408740359Z level=info msg="Executing migration" id="Add index user.login/user.email" 2024-09-21T09:28:55.477 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.409154598Z level=info msg="Executing migration" id="Add is_service_account column to user" 2024-09-21T09:28:55.477 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.409570708Z level=info msg="Executing migration" id="Update is_service_account column to nullable" 2024-09-21T09:28:55.477 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.410703077Z level=info msg="Executing migration" id="create temp user table v1-7" 2024-09-21T09:28:55.477 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.411137276Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v1-7" 2024-09-21T09:28:55.477 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.41156944Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v1-7" 2024-09-21T09:28:55.477 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.412014102Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v1-7" 2024-09-21T09:28:55.477 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.412830153Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v1-7" 2024-09-21T09:28:55.477 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.413853078Z level=info msg="Executing migration" id="Update temp_user table charset" 2024-09-21T09:28:55.478 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.414526455Z level=info msg="Executing migration" id="drop index IDX_temp_user_email - v1" 2024-09-21T09:28:55.478 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.414961945Z level=info msg="Executing migration" id="drop index IDX_temp_user_org_id - v1" 2024-09-21T09:28:55.478 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.415884803Z level=info msg="Executing migration" id="drop index IDX_temp_user_code - v1" 2024-09-21T09:28:55.478 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.41654641Z level=info msg="Executing migration" id="drop index IDX_temp_user_status - v1" 2024-09-21T09:28:55.478 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.417001696Z level=info msg="Executing migration" id="Rename table temp_user to temp_user_tmp_qwerty - v1" 2024-09-21T09:28:55.478 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.417843844Z level=info msg="Executing migration" id="create temp_user v2" 2024-09-21T09:28:55.478 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.418299703Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v2" 2024-09-21T09:28:55.478 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.418741354Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v2" 2024-09-21T09:28:55.478 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.41959715Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v2" 2024-09-21T09:28:55.478 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.420471341Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v2" 2024-09-21T09:28:55.478 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.423433202Z level=info msg="Executing migration" id="copy temp_user v1 to v2" 2024-09-21T09:28:55.478 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.423952611Z level=info msg="Executing migration" id="drop temp_user_tmp_qwerty" 2024-09-21T09:28:55.478 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.426927697Z level=info msg="Executing migration" id="Set created for temp users that will otherwise prematurely expire" 2024-09-21T09:28:55.478 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.427358027Z level=info msg="Executing migration" id="create star table" 2024-09-21T09:28:55.479 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.429712752Z level=info msg="Executing migration" id="add unique index star.user_id_dashboard_id" 2024-09-21T09:28:55.479 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.432583679Z level=info msg="Executing migration" id="create org table v1" 2024-09-21T09:28:55.479 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.435438512Z level=info msg="Executing migration" id="create index UQE_org_name - v1" 2024-09-21T09:28:55.479 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.438247633Z level=info msg="Executing migration" id="create org_user table v1" 2024-09-21T09:28:55.479 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.441005984Z level=info msg="Executing migration" id="create index IDX_org_user_org_id - v1" 2024-09-21T09:28:55.479 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.443755236Z level=info msg="Executing migration" id="create index UQE_org_user_org_id_user_id - v1" 2024-09-21T09:28:55.479 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.446483906Z level=info msg="Executing migration" id="create index IDX_org_user_user_id - v1" 2024-09-21T09:28:55.479 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.446951932Z level=info msg="Executing migration" id="Update org table charset" 2024-09-21T09:28:55.479 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.449780501Z level=info msg="Executing migration" id="Update org_user table charset" 2024-09-21T09:28:55.479 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.452574744Z level=info msg="Executing migration" id="Migrate all Read Only Viewers to Viewers" 2024-09-21T09:28:55.479 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.455550407Z level=info msg="Executing migration" id="create dashboard table" 2024-09-21T09:28:55.479 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.458287862Z level=info msg="Executing migration" id="add index dashboard.account_id" 2024-09-21T09:28:55.479 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.460796396Z level=info msg="Executing migration" id="add unique index dashboard_account_id_slug" 2024-09-21T09:28:55.479 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.463775955Z level=info msg="Executing migration" id="create dashboard_tag table" 2024-09-21T09:28:55.480 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.466593582Z level=info msg="Executing migration" id="add unique index dashboard_tag.dasboard_id_term" 2024-09-21T09:28:55.480 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.469431224Z level=info msg="Executing migration" id="drop index UQE_dashboard_tag_dashboard_id_term - v1" 2024-09-21T09:28:55.480 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.472163257Z level=info msg="Executing migration" id="Rename table dashboard to dashboard_v1 - v1" 2024-09-21T09:28:55.480 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.473321146Z level=info msg="Executing migration" id="create dashboard v2" 2024-09-21T09:28:55.724 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.474185615Z level=info msg="Executing migration" id="create index IDX_dashboard_org_id - v2" 2024-09-21T09:28:55.724 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.475041824Z level=info msg="Executing migration" id="create index UQE_dashboard_org_id_slug - v2" 2024-09-21T09:28:55.725 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.475928689Z level=info msg="Executing migration" id="copy dashboard v1 to v2" 2024-09-21T09:28:55.725 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.47680412Z level=info msg="Executing migration" id="drop table dashboard_v1" 2024-09-21T09:28:55.725 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.477695317Z level=info msg="Executing migration" id="alter dashboard.data to mediumtext v1" 2024-09-21T09:28:55.725 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.478570392Z level=info msg="Executing migration" id="Add column updated_by in dashboard - v2" 2024-09-21T09:28:55.725 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.479480861Z level=info msg="Executing migration" id="Add column created_by in dashboard - v2" 2024-09-21T09:28:55.725 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.48036846Z level=info msg="Executing migration" id="Add column gnetId in dashboard" 2024-09-21T09:28:55.725 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.483452507Z level=info msg="Executing migration" id="Add index for gnetId in dashboard" 2024-09-21T09:28:55.725 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.486296917Z level=info msg="Executing migration" id="Add column plugin_id in dashboard" 2024-09-21T09:28:55.725 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.489257729Z level=info msg="Executing migration" id="Add index for plugin_id in dashboard" 2024-09-21T09:28:55.725 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.492078612Z level=info msg="Executing migration" id="Add index for dashboard_id in dashboard_tag" 2024-09-21T09:28:55.725 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.494850571Z level=info msg="Executing migration" id="Update dashboard table charset" 2024-09-21T09:28:55.726 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.497712507Z level=info msg="Executing migration" id="Update dashboard_tag table charset" 2024-09-21T09:28:55.726 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.500515482Z level=info msg="Executing migration" id="Add column folder_id in dashboard" 2024-09-21T09:28:55.726 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.50340726Z level=info msg="Executing migration" id="Add column isFolder in dashboard" 2024-09-21T09:28:55.726 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.506254717Z level=info msg="Executing migration" id="Add column has_acl in dashboard" 2024-09-21T09:28:55.726 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.50926796Z level=info msg="Executing migration" id="Add column uid in dashboard" 2024-09-21T09:28:55.726 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.512120406Z level=info msg="Executing migration" id="Update uid column values in dashboard" 2024-09-21T09:28:55.726 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.515342209Z level=info msg="Executing migration" id="Add unique index dashboard_org_id_uid" 2024-09-21T09:28:55.726 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.517944173Z level=info msg="Executing migration" id="Remove unique index org_id_slug" 2024-09-21T09:28:55.726 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.520797871Z level=info msg="Executing migration" id="Update dashboard title length" 2024-09-21T09:28:55.726 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.523569069Z level=info msg="Executing migration" id="Add unique index for dashboard_org_id_title_folder_id" 2024-09-21T09:28:55.726 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.526628255Z level=info msg="Executing migration" id="create dashboard_provisioning" 2024-09-21T09:28:55.726 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.529414023Z level=info msg="Executing migration" id="Rename table dashboard_provisioning to dashboard_provisioning_tmp_qwerty - v1" 2024-09-21T09:28:55.726 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.532158172Z level=info msg="Executing migration" id="create dashboard_provisioning v2" 2024-09-21T09:28:55.727 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.53480935Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id - v2" 2024-09-21T09:28:55.727 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.537328169Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id_name - v2" 2024-09-21T09:28:55.727 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.53986476Z level=info msg="Executing migration" id="copy dashboard_provisioning v1 to v2" 2024-09-21T09:28:55.727 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.542678729Z level=info msg="Executing migration" id="drop dashboard_provisioning_tmp_qwerty" 2024-09-21T09:28:55.727 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.545290694Z level=info msg="Executing migration" id="Add check_sum column" 2024-09-21T09:28:55.727 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.547833646Z level=info msg="Executing migration" id="Add index for dashboard_title" 2024-09-21T09:28:55.727 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.55044481Z level=info msg="Executing migration" id="delete tags for deleted dashboards" 2024-09-21T09:28:55.727 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.552925278Z level=info msg="Executing migration" id="delete stars for deleted dashboards" 2024-09-21T09:28:55.727 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.555660047Z level=info msg="Executing migration" id="Add index for dashboard_is_folder" 2024-09-21T09:28:55.727 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.558342554Z level=info msg="Executing migration" id="Add isPublic for dashboard" 2024-09-21T09:28:55.727 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.561057217Z level=info msg="Executing migration" id="create data_source table" 2024-09-21T09:28:55.727 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.563587912Z level=info msg="Executing migration" id="add index data_source.account_id" 2024-09-21T09:28:55.729 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.566267653Z level=info msg="Executing migration" id="add unique index data_source.account_id_name" 2024-09-21T09:28:55.729 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.568975604Z level=info msg="Executing migration" id="drop index IDX_data_source_account_id - v1" 2024-09-21T09:28:55.729 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.571738791Z level=info msg="Executing migration" id="drop index UQE_data_source_account_id_name - v1" 2024-09-21T09:28:55.729 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.574307195Z level=info msg="Executing migration" id="Rename table data_source to data_source_v1 - v1" 2024-09-21T09:28:55.729 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.577006698Z level=info msg="Executing migration" id="create data_source table v2" 2024-09-21T09:28:55.729 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.577938332Z level=info msg="Executing migration" id="create index IDX_data_source_org_id - v2" 2024-09-21T09:28:55.729 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.578691431Z level=info msg="Executing migration" id="create index UQE_data_source_org_id_name - v2" 2024-09-21T09:28:55.729 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.579677709Z level=info msg="Executing migration" id="Drop old table data_source_v1 #2" 2024-09-21T09:28:55.729 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.580517601Z level=info msg="Executing migration" id="Add column with_credentials" 2024-09-21T09:28:55.729 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.581259104Z level=info msg="Executing migration" id="Add secure json data column" 2024-09-21T09:28:55.730 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.582282605Z level=info msg="Executing migration" id="Update data_source table charset" 2024-09-21T09:28:55.730 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.582998032Z level=info msg="Executing migration" id="Update initial version to 1" 2024-09-21T09:28:55.730 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.583826567Z level=info msg="Executing migration" id="Add read_only data column" 2024-09-21T09:28:55.730 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.584681319Z level=info msg="Executing migration" id="Migrate logging ds to loki ds" 2024-09-21T09:28:55.730 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.585685058Z level=info msg="Executing migration" id="Update json_data with nulls" 2024-09-21T09:28:55.730 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.586522316Z level=info msg="Executing migration" id="Add uid column" 2024-09-21T09:28:55.730 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.587277838Z level=info msg="Executing migration" id="Update uid value" 2024-09-21T09:28:55.730 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.588255787Z level=info msg="Executing migration" id="Add unique index datasource_org_id_uid" 2024-09-21T09:28:55.730 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.588926647Z level=info msg="Executing migration" id="add unique index datasource_org_id_is_default" 2024-09-21T09:28:55.730 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.589353712Z level=info msg="Executing migration" id="create api_key table" 2024-09-21T09:28:55.730 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.590216926Z level=info msg="Executing migration" id="add index api_key.account_id" 2024-09-21T09:28:55.730 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.591069626Z level=info msg="Executing migration" id="add index api_key.key" 2024-09-21T09:28:55.730 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.591955388Z level=info msg="Executing migration" id="add index api_key.account_id_name" 2024-09-21T09:28:55.731 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.592919819Z level=info msg="Executing migration" id="drop index IDX_api_key_account_id - v1" 2024-09-21T09:28:55.731 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.593665024Z level=info msg="Executing migration" id="drop index UQE_api_key_key - v1" 2024-09-21T09:28:55.731 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.594072848Z level=info msg="Executing migration" id="drop index UQE_api_key_account_id_name - v1" 2024-09-21T09:28:55.731 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.594932769Z level=info msg="Executing migration" id="Rename table api_key to api_key_v1 - v1" 2024-09-21T09:28:55.731 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.596001549Z level=info msg="Executing migration" id="create api_key table v2" 2024-09-21T09:28:55.731 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.596402212Z level=info msg="Executing migration" id="create index IDX_api_key_org_id - v2" 2024-09-21T09:28:55.731 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.597274919Z level=info msg="Executing migration" id="create index UQE_api_key_key - v2" 2024-09-21T09:28:55.731 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.597711133Z level=info msg="Executing migration" id="create index UQE_api_key_org_id_name - v2" 2024-09-21T09:28:55.731 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.598743044Z level=info msg="Executing migration" id="copy api_key v1 to v2" 2024-09-21T09:28:55.731 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.599458875Z level=info msg="Executing migration" id="Drop old table api_key_v1" 2024-09-21T09:28:55.731 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.600487222Z level=info msg="Executing migration" id="Update api_key table charset" 2024-09-21T09:28:55.731 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.600885834Z level=info msg="Executing migration" id="Add expires to api_key table" 2024-09-21T09:28:55.731 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.601849093Z level=info msg="Executing migration" id="Add service account foreign key" 2024-09-21T09:28:55.731 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.602613396Z level=info msg="Executing migration" id="set service account foreign key to nil if 0" 2024-09-21T09:28:55.732 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.602937543Z level=info msg="Executing migration" id="Add last_used_at to api_key table" 2024-09-21T09:28:55.732 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.603764064Z level=info msg="Executing migration" id="Add is_revoked column to api_key table" 2024-09-21T09:28:55.732 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.604323317Z level=info msg="Executing migration" id="create dashboard_snapshot table v4" 2024-09-21T09:28:55.732 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.607395473Z level=info msg="Executing migration" id="drop table dashboard_snapshot_v4 #1" 2024-09-21T09:28:55.732 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.61018547Z level=info msg="Executing migration" id="create dashboard_snapshot table v5 #2" 2024-09-21T09:28:55.732 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.613043161Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_key - v5" 2024-09-21T09:28:55.732 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.616084438Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_delete_key - v5" 2024-09-21T09:28:55.732 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.616505926Z level=info msg="Executing migration" id="create index IDX_dashboard_snapshot_user_id - v5" 2024-09-21T09:28:55.732 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.619329406Z level=info msg="Executing migration" id="alter dashboard_snapshot to mediumtext v2" 2024-09-21T09:28:55.732 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.622446647Z level=info msg="Executing migration" id="Update dashboard_snapshot table charset" 2024-09-21T09:28:55.732 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.625283668Z level=info msg="Executing migration" id="Add column external_delete_url to dashboard_snapshots table" 2024-09-21T09:28:55.732 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.625867998Z level=info msg="Executing migration" id="Add encrypted dashboard json column" 2024-09-21T09:28:55.732 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.628807782Z level=info msg="Executing migration" id="Change dashboard_encrypted column to MEDIUMBLOB" 2024-09-21T09:28:55.732 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.631694104Z level=info msg="Executing migration" id="create quota table v1" 2024-09-21T09:28:55.733 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.634588663Z level=info msg="Executing migration" id="create index UQE_quota_org_id_user_id_target - v1" 2024-09-21T09:28:55.733 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.635072585Z level=info msg="Executing migration" id="Update quota table charset" 2024-09-21T09:28:55.733 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.637867301Z level=info msg="Executing migration" id="create plugin_setting table" 2024-09-21T09:28:55.733 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.638364779Z level=info msg="Executing migration" id="create index UQE_plugin_setting_org_id_plugin_id - v1" 2024-09-21T09:28:55.733 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.64077926Z level=info msg="Executing migration" id="Add column plugin_version to plugin_settings" 2024-09-21T09:28:55.733 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.643648616Z level=info msg="Executing migration" id="Update plugin_setting table charset" 2024-09-21T09:28:55.733 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.64398328Z level=info msg="Executing migration" id="create session table" 2024-09-21T09:28:55.733 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.646716879Z level=info msg="Executing migration" id="Drop old table playlist table" 2024-09-21T09:28:55.733 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.649331425Z level=info msg="Executing migration" id="Drop old table playlist_item table" 2024-09-21T09:28:55.733 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.652105379Z level=info msg="Executing migration" id="create playlist table v2" 2024-09-21T09:28:55.733 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.652551697Z level=info msg="Executing migration" id="create playlist item table v2" 2024-09-21T09:28:55.733 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.655248921Z level=info msg="Executing migration" id="Update playlist table charset" 2024-09-21T09:28:55.733 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.658008183Z level=info msg="Executing migration" id="Update playlist_item table charset" 2024-09-21T09:28:55.733 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.660520943Z level=info msg="Executing migration" id="drop preferences table v2" 2024-09-21T09:28:55.734 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.660857226Z level=info msg="Executing migration" id="drop preferences table v3" 2024-09-21T09:28:55.734 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.663511815Z level=info msg="Executing migration" id="create preferences table v3" 2024-09-21T09:28:55.734 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.663949168Z level=info msg="Executing migration" id="Update preferences table charset" 2024-09-21T09:28:55.734 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.666339324Z level=info msg="Executing migration" id="Add column team_id in preferences" 2024-09-21T09:28:55.734 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.667173057Z level=info msg="Executing migration" id="Update team_id column values in preferences" 2024-09-21T09:28:55.734 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.668928901Z level=info msg="Executing migration" id="Add column week_start in preferences" 2024-09-21T09:28:55.734 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.669541162Z level=info msg="Executing migration" id="Add column preferences.json_data" 2024-09-21T09:28:55.734 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.67220131Z level=info msg="Executing migration" id="alter preferences.json_data to mediumtext v1" 2024-09-21T09:28:55.734 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.674939287Z level=info msg="Executing migration" id="Add preferences index org_id" 2024-09-21T09:28:55.734 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.677523639Z level=info msg="Executing migration" id="Add preferences index user_id" 2024-09-21T09:28:55.734 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.678482208Z level=info msg="Executing migration" id="create alert table v1" 2024-09-21T09:28:55.734 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.679186311Z level=info msg="Executing migration" id="add index alert org_id & id " 2024-09-21T09:28:55.734 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.680183253Z level=info msg="Executing migration" id="add index alert state" 2024-09-21T09:28:55.735 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.680910628Z level=info msg="Executing migration" id="add index alert dashboard_id" 2024-09-21T09:28:55.735 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.681350682Z level=info msg="Executing migration" id="Create alert_rule_tag table v1" 2024-09-21T09:28:55.735 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.682345052Z level=info msg="Executing migration" id="Add unique index alert_rule_tag.alert_id_tag_id" 2024-09-21T09:28:55.735 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.683048609Z level=info msg="Executing migration" id="drop index UQE_alert_rule_tag_alert_id_tag_id - v1" 2024-09-21T09:28:55.735 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.684052391Z level=info msg="Executing migration" id="Rename table alert_rule_tag to alert_rule_tag_v1 - v1" 2024-09-21T09:28:55.735 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.685660653Z level=info msg="Executing migration" id="Create alert_rule_tag table v2" 2024-09-21T09:28:55.735 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.686494523Z level=info msg="Executing migration" id="create index UQE_alert_rule_tag_alert_id_tag_id - Add unique index alert_rule_tag.alert_id_tag_id V2" 2024-09-21T09:28:55.735 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.68734117Z level=info msg="Executing migration" id="copy alert_rule_tag v1 to v2" 2024-09-21T09:28:55.735 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.688273262Z level=info msg="Executing migration" id="drop table alert_rule_tag_v1" 2024-09-21T09:28:55.735 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.688799969Z level=info msg="Executing migration" id="create alert_notification table v1" 2024-09-21T09:28:55.735 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.689215752Z level=info msg="Executing migration" id="Add column is_default" 2024-09-21T09:28:55.735 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.690078721Z level=info msg="Executing migration" id="Add column frequency" 2024-09-21T09:28:55.735 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.69107092Z level=info msg="Executing migration" id="Add column send_reminder" 2024-09-21T09:28:55.735 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.691878388Z level=info msg="Executing migration" id="Add column disable_resolve_message" 2024-09-21T09:28:55.736 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.692518514Z level=info msg="Executing migration" id="add index alert_notification org_id & name" 2024-09-21T09:28:55.736 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.693399634Z level=info msg="Executing migration" id="Update alert table charset" 2024-09-21T09:28:55.736 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.69442286Z level=info msg="Executing migration" id="Update alert_notification table charset" 2024-09-21T09:28:55.736 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.695144786Z level=info msg="Executing migration" id="create notification_journal table v1" 2024-09-21T09:28:55.736 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.695567827Z level=info msg="Executing migration" id="add index notification_journal org_id & alert_id & notifier_id" 2024-09-21T09:28:55.736 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.69602119Z level=info msg="Executing migration" id="drop alert_notification_journal" 2024-09-21T09:28:55.736 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.696406188Z level=info msg="Executing migration" id="create alert_notification_state table v1" 2024-09-21T09:28:55.736 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.696844802Z level=info msg="Executing migration" id="add index alert_notification_state org_id & alert_id & notifier_id" 2024-09-21T09:28:55.736 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.699486526Z level=info msg="Executing migration" id="Add for to alert table" 2024-09-21T09:28:55.736 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.700362146Z level=info msg="Executing migration" id="Add column uid in alert_notification" 2024-09-21T09:28:55.736 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.703250416Z level=info msg="Executing migration" id="Update uid column values in alert_notification" 2024-09-21T09:28:55.737 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.70607948Z level=info msg="Executing migration" id="Add unique index alert_notification_org_id_uid" 2024-09-21T09:28:55.737 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.708714174Z level=info msg="Executing migration" id="Remove unique index org_id_name" 2024-09-21T09:28:55.737 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.711427127Z level=info msg="Executing migration" id="Add column secure_settings in alert_notification" 2024-09-21T09:28:55.737 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.713977819Z level=info msg="Executing migration" id="alter alert.settings to mediumtext" 2024-09-21T09:28:55.737 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.716653886Z level=info msg="Executing migration" id="Add non-unique index alert_notification_state_alert_id" 2024-09-21T09:28:55.737 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.71944704Z level=info msg="Executing migration" id="Add non-unique index alert_rule_tag_alert_id" 2024-09-21T09:28:55.737 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.719938528Z level=info msg="Executing migration" id="Drop old annotation table v4" 2024-09-21T09:28:55.737 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.722779286Z level=info msg="Executing migration" id="create annotation table v5" 2024-09-21T09:28:55.737 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.725329701Z level=info msg="Executing migration" id="add index annotation 0 v3" 2024-09-21T09:28:55.975 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-mon[31124]: pgmap v24: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:28:55.975 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:28:55.975 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:28:55.976 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:28:55.976 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:28:55.976 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.728039774Z level=info msg="Executing migration" id="add index annotation 1 v3" 2024-09-21T09:28:55.976 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.728502624Z level=info msg="Executing migration" id="add index annotation 2 v3" 2024-09-21T09:28:55.976 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.731218473Z level=info msg="Executing migration" id="add index annotation 3 v3" 2024-09-21T09:28:55.976 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.733822956Z level=info msg="Executing migration" id="add index annotation 4 v3" 2024-09-21T09:28:55.976 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.736485836Z level=info msg="Executing migration" id="Update annotation table charset" 2024-09-21T09:28:55.977 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.736816304Z level=info msg="Executing migration" id="Add column region_id to annotation table" 2024-09-21T09:28:55.977 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.739464555Z level=info msg="Executing migration" id="Drop category_id index" 2024-09-21T09:28:55.977 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.740088544Z level=info msg="Executing migration" id="Add column tags to annotation table" 2024-09-21T09:28:55.977 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.74196645Z level=info msg="Executing migration" id="Create annotation_tag table v2" 2024-09-21T09:28:55.977 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.74288195Z level=info msg="Executing migration" id="Add unique index annotation_tag.annotation_id_tag_id" 2024-09-21T09:28:55.977 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.743341646Z level=info msg="Executing migration" id="drop index UQE_annotation_tag_annotation_id_tag_id - v2" 2024-09-21T09:28:55.977 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.744294058Z level=info msg="Executing migration" id="Rename table annotation_tag to annotation_tag_v2 - v2" 2024-09-21T09:28:55.977 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.74619573Z level=info msg="Executing migration" id="Create annotation_tag table v3" 2024-09-21T09:28:55.977 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.747093096Z level=info msg="Executing migration" id="create index UQE_annotation_tag_annotation_id_tag_id - Add unique index annotation_tag.annotation_id_tag_id V3" 2024-09-21T09:28:55.977 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.747589969Z level=info msg="Executing migration" id="copy annotation_tag v2 to v3" 2024-09-21T09:28:55.977 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.748381311Z level=info msg="Executing migration" id="drop table annotation_tag_v2" 2024-09-21T09:28:55.977 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.749249275Z level=info msg="Executing migration" id="Update alert annotations and set TEXT to empty" 2024-09-21T09:28:55.977 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.750083823Z level=info msg="Executing migration" id="Add created time to annotation table" 2024-09-21T09:28:55.978 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.750850808Z level=info msg="Executing migration" id="Add updated time to annotation table" 2024-09-21T09:28:55.978 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.751616037Z level=info msg="Executing migration" id="Add index for created in annotation table" 2024-09-21T09:28:55.978 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.752484412Z level=info msg="Executing migration" id="Add index for updated in annotation table" 2024-09-21T09:28:55.978 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.753357657Z level=info msg="Executing migration" id="Convert existing annotations from seconds to milliseconds" 2024-09-21T09:28:55.978 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.75373612Z level=info msg="Executing migration" id="Add epoch_end column" 2024-09-21T09:28:55.978 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.754548075Z level=info msg="Executing migration" id="Add index for epoch_end" 2024-09-21T09:28:55.978 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.755529885Z level=info msg="Executing migration" id="Make epoch_end the same as epoch" 2024-09-21T09:28:55.978 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.756281413Z level=info msg="Executing migration" id="Move region to single row" 2024-09-21T09:28:55.978 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.756657594Z level=info msg="Executing migration" id="Remove index org_id_epoch from annotation table" 2024-09-21T09:28:55.978 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.757114418Z level=info msg="Executing migration" id="Remove index org_id_dashboard_id_panel_id_epoch from annotation table" 2024-09-21T09:28:55.978 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.757615499Z level=info msg="Executing migration" id="Add index for org_id_dashboard_id_epoch_end_epoch on annotation table" 2024-09-21T09:28:55.978 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.758097922Z level=info msg="Executing migration" id="Add index for org_id_epoch_end_epoch on annotation table" 2024-09-21T09:28:55.979 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.758539321Z level=info msg="Executing migration" id="Remove index org_id_epoch_epoch_end from annotation table" 2024-09-21T09:28:55.979 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.759000311Z level=info msg="Executing migration" id="Add index for alert_id on annotation table" 2024-09-21T09:28:55.979 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.759438663Z level=info msg="Executing migration" id="Increase tags column to length 4096" 2024-09-21T09:28:55.979 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.759796405Z level=info msg="Executing migration" id="create test_data table" 2024-09-21T09:28:55.979 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.760240048Z level=info msg="Executing migration" id="create dashboard_version table v1" 2024-09-21T09:28:55.979 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.760713935Z level=info msg="Executing migration" id="add index dashboard_version.dashboard_id" 2024-09-21T09:28:55.979 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.761549548Z level=info msg="Executing migration" id="add unique index dashboard_version.dashboard_id and dashboard_version.version" 2024-09-21T09:28:55.979 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.762390364Z level=info msg="Executing migration" id="Set dashboard version to 1 where 0" 2024-09-21T09:28:55.979 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.762787992Z level=info msg="Executing migration" id="save existing dashboard data in dashboard_version table v1" 2024-09-21T09:28:55.979 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.764322061Z level=info msg="Executing migration" id="alter dashboard_version.data to mediumtext v1" 2024-09-21T09:28:55.979 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.765058063Z level=info msg="Executing migration" id="create team table" 2024-09-21T09:28:55.979 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.766075045Z level=info msg="Executing migration" id="add index team.org_id" 2024-09-21T09:28:55.979 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.766675492Z level=info msg="Executing migration" id="add unique index team_org_id_name" 2024-09-21T09:28:55.979 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.767739101Z level=info msg="Executing migration" id="create team member table" 2024-09-21T09:28:55.980 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.770424888Z level=info msg="Executing migration" id="add index team_member.org_id" 2024-09-21T09:28:55.980 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.773262975Z level=info msg="Executing migration" id="add unique index team_member_org_id_team_id_user_id" 2024-09-21T09:28:55.980 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.773750072Z level=info msg="Executing migration" id="add index team_member.team_id" 2024-09-21T09:28:55.980 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.776589249Z level=info msg="Executing migration" id="Add column email to team table" 2024-09-21T09:28:55.980 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.779195138Z level=info msg="Executing migration" id="Add column external to team_member table" 2024-09-21T09:28:55.980 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.782035654Z level=info msg="Executing migration" id="Add column permission to team_member table" 2024-09-21T09:28:55.980 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.782805067Z level=info msg="Executing migration" id="create dashboard acl table" 2024-09-21T09:28:55.980 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.78586327Z level=info msg="Executing migration" id="add index dashboard_acl_dashboard_id" 2024-09-21T09:28:55.980 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.788547213Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_user_id" 2024-09-21T09:28:55.980 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.789057526Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_team_id" 2024-09-21T09:28:55.980 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.791294545Z level=info msg="Executing migration" id="add index dashboard_acl_user_id" 2024-09-21T09:28:55.980 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.794259279Z level=info msg="Executing migration" id="add index dashboard_acl_team_id" 2024-09-21T09:28:55.981 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.796706479Z level=info msg="Executing migration" id="add index dashboard_acl_org_id_role" 2024-09-21T09:28:55.981 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.797189778Z level=info msg="Executing migration" id="add index dashboard_permission" 2024-09-21T09:28:55.981 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.797659412Z level=info msg="Executing migration" id="save default acl rules in dashboard_acl table" 2024-09-21T09:28:55.982 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.798146777Z level=info msg="Executing migration" id="delete acl rules for deleted dashboards and folders" 2024-09-21T09:28:55.982 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.798462101Z level=info msg="Executing migration" id="create tag table" 2024-09-21T09:28:55.982 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.798917221Z level=info msg="Executing migration" id="add index tag.key_value" 2024-09-21T09:28:55.982 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.799417217Z level=info msg="Executing migration" id="create login attempt table" 2024-09-21T09:28:55.982 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.799850454Z level=info msg="Executing migration" id="add index login_attempt.username" 2024-09-21T09:28:55.982 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.800329152Z level=info msg="Executing migration" id="drop index IDX_login_attempt_username - v1" 2024-09-21T09:28:55.982 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.80081486Z level=info msg="Executing migration" id="Rename table login_attempt to login_attempt_tmp_qwerty - v1" 2024-09-21T09:28:55.982 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.802830023Z level=info msg="Executing migration" id="create login_attempt v2" 2024-09-21T09:28:55.982 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.803267118Z level=info msg="Executing migration" id="create index IDX_login_attempt_username - v2" 2024-09-21T09:28:55.982 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.803746913Z level=info msg="Executing migration" id="copy login_attempt v1 to v2" 2024-09-21T09:28:55.982 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.804121709Z level=info msg="Executing migration" id="drop login_attempt_tmp_qwerty" 2024-09-21T09:28:55.982 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.804497985Z level=info msg="Executing migration" id="create user auth table" 2024-09-21T09:28:55.983 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.80495172Z level=info msg="Executing migration" id="create index IDX_user_auth_auth_module_auth_id - v1" 2024-09-21T09:28:55.983 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.805427914Z level=info msg="Executing migration" id="alter user_auth.auth_id to length 190" 2024-09-21T09:28:55.983 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.805777234Z level=info msg="Executing migration" id="Add OAuth access token to user_auth" 2024-09-21T09:28:55.983 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.806571463Z level=info msg="Executing migration" id="Add OAuth refresh token to user_auth" 2024-09-21T09:28:55.983 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.807360773Z level=info msg="Executing migration" id="Add OAuth token type to user_auth" 2024-09-21T09:28:55.983 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.808156116Z level=info msg="Executing migration" id="Add OAuth expiry to user_auth" 2024-09-21T09:28:55.983 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.808978605Z level=info msg="Executing migration" id="Add index to user_id column in user_auth" 2024-09-21T09:28:55.983 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.809469412Z level=info msg="Executing migration" id="Add OAuth ID token to user_auth" 2024-09-21T09:28:55.983 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.810269829Z level=info msg="Executing migration" id="create server_lock table" 2024-09-21T09:28:55.983 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.810714326Z level=info msg="Executing migration" id="add index server_lock.operation_uid" 2024-09-21T09:28:55.983 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.811200053Z level=info msg="Executing migration" id="create user auth token table" 2024-09-21T09:28:55.983 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.811680233Z level=info msg="Executing migration" id="add unique index user_auth_token.auth_token" 2024-09-21T09:28:55.983 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.81216362Z level=info msg="Executing migration" id="add unique index user_auth_token.prev_auth_token" 2024-09-21T09:28:55.983 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.812655534Z level=info msg="Executing migration" id="add index user_auth_token.user_id" 2024-09-21T09:28:55.984 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.813118074Z level=info msg="Executing migration" id="Add revoked_at to the user auth token" 2024-09-21T09:28:55.984 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.813922679Z level=info msg="Executing migration" id="create cache_data table" 2024-09-21T09:28:55.984 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.814377511Z level=info msg="Executing migration" id="add unique index cache_data.cache_key" 2024-09-21T09:28:55.984 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.814853105Z level=info msg="Executing migration" id="create short_url table v1" 2024-09-21T09:28:55.984 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.815315417Z level=info msg="Executing migration" id="add index short_url.org_id-uid" 2024-09-21T09:28:55.984 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.818139588Z level=info msg="Executing migration" id="alter table short_url alter column created_by type to bigint" 2024-09-21T09:28:55.984 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.818694211Z level=info msg="Executing migration" id="delete alert_definition table" 2024-09-21T09:28:55.984 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.819035034Z level=info msg="Executing migration" id="recreate alert_definition table" 2024-09-21T09:28:55.984 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.819523973Z level=info msg="Executing migration" id="add index in alert_definition on org_id and title columns" 2024-09-21T09:28:55.984 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.822414404Z level=info msg="Executing migration" id="add index in alert_definition on org_id and uid columns" 2024-09-21T09:28:55.984 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.824981173Z level=info msg="Executing migration" id="alter alert_definition table data column to mediumtext in mysql" 2024-09-21T09:28:55.984 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.827848073Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and title columns" 2024-09-21T09:28:55.984 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.828326084Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and uid columns" 2024-09-21T09:28:55.985 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.831274099Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and title columns" 2024-09-21T09:28:55.985 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.83414859Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and uid columns" 2024-09-21T09:28:55.985 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.836970413Z level=info msg="Executing migration" id="Add column paused in alert_definition" 2024-09-21T09:28:55.985 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.839649823Z level=info msg="Executing migration" id="drop alert_definition table" 2024-09-21T09:28:55.985 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.84234221Z level=info msg="Executing migration" id="delete alert_definition_version table" 2024-09-21T09:28:55.985 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.844921673Z level=info msg="Executing migration" id="recreate alert_definition_version table" 2024-09-21T09:28:55.985 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.847493164Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_id and version columns" 2024-09-21T09:28:55.985 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.850319234Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_uid and version columns" 2024-09-21T09:28:55.985 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.852914543Z level=info msg="Executing migration" id="alter alert_definition_version table data column to mediumtext in mysql" 2024-09-21T09:28:55.985 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.855678348Z level=info msg="Executing migration" id="drop alert_definition_version table" 2024-09-21T09:28:55.985 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.858418223Z level=info msg="Executing migration" id="create alert_instance table" 2024-09-21T09:28:55.985 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.860904907Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, def_uid and current_state columns" 2024-09-21T09:28:55.985 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.863605542Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, current_state columns" 2024-09-21T09:28:55.985 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.86412848Z level=info msg="Executing migration" id="add column current_state_end to alert_instance" 2024-09-21T09:28:55.986 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.866780774Z level=info msg="Executing migration" id="remove index def_org_id, def_uid, current_state on alert_instance" 2024-09-21T09:28:55.986 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.869390799Z level=info msg="Executing migration" id="remove index def_org_id, current_state on alert_instance" 2024-09-21T09:28:55.986 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.869878979Z level=info msg="Executing migration" id="rename def_org_id to rule_org_id in alert_instance" 2024-09-21T09:28:55.986 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.873450137Z level=info msg="Executing migration" id="rename def_uid to rule_uid in alert_instance" 2024-09-21T09:28:55.986 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.876948967Z level=info msg="Executing migration" id="add index rule_org_id, rule_uid, current_state on alert_instance" 2024-09-21T09:28:55.986 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.877436696Z level=info msg="Executing migration" id="add index rule_org_id, current_state on alert_instance" 2024-09-21T09:28:55.986 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.878395325Z level=info msg="Executing migration" id="add current_reason column related to current_state" 2024-09-21T09:28:55.986 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.879460108Z level=info msg="Executing migration" id="create alert_rule table" 2024-09-21T09:28:55.986 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.8803398Z level=info msg="Executing migration" id="add index in alert_rule on org_id and title columns" 2024-09-21T09:28:55.986 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.881149244Z level=info msg="Executing migration" id="add index in alert_rule on org_id and uid columns" 2024-09-21T09:28:55.986 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.88198012Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespace_uid, group_uid columns" 2024-09-21T09:28:55.986 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.882871597Z level=info msg="Executing migration" id="alter alert_rule table data column to mediumtext in mysql" 2024-09-21T09:28:55.986 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.88377312Z level=info msg="Executing migration" id="add column for to alert_rule" 2024-09-21T09:28:55.986 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.88463708Z level=info msg="Executing migration" id="add column annotations to alert_rule" 2024-09-21T09:28:55.987 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.885493752Z level=info msg="Executing migration" id="add column labels to alert_rule" 2024-09-21T09:28:55.987 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.886367756Z level=info msg="Executing migration" id="remove unique index from alert_rule on org_id, title columns" 2024-09-21T09:28:55.987 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.887211237Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespase_uid and title columns" 2024-09-21T09:28:55.987 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.887703062Z level=info msg="Executing migration" id="add dashboard_uid column to alert_rule" 2024-09-21T09:28:55.987 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.888587107Z level=info msg="Executing migration" id="add panel_id column to alert_rule" 2024-09-21T09:28:55.987 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.889446323Z level=info msg="Executing migration" id="add index in alert_rule on org_id, dashboard_uid and panel_id columns" 2024-09-21T09:28:55.987 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.890320405Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule" 2024-09-21T09:28:55.987 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.891230929Z level=info msg="Executing migration" id="add is_paused column to alert_rule table" 2024-09-21T09:28:55.987 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.89210668Z level=info msg="Executing migration" id="fix is_paused column for alert_rule table" 2024-09-21T09:28:55.987 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.893028432Z level=info msg="Executing migration" id="create alert_rule_version table" 2024-09-21T09:28:55.987 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.893922611Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_uid and version columns" 2024-09-21T09:28:55.987 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.894413759Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_namespace_uid and rule_group columns" 2024-09-21T09:28:55.987 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.897393983Z level=info msg="Executing migration" id="alter alert_rule_version table data column to mediumtext in mysql" 2024-09-21T09:28:55.988 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.900181745Z level=info msg="Executing migration" id="add column for to alert_rule_version" 2024-09-21T09:28:55.988 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.903046592Z level=info msg="Executing migration" id="add column annotations to alert_rule_version" 2024-09-21T09:28:55.988 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.904046728Z level=info msg="Executing migration" id="add column labels to alert_rule_version" 2024-09-21T09:28:55.988 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.906942315Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule_version" 2024-09-21T09:28:55.988 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.9098827Z level=info msg="Executing migration" id="add is_paused column to alert_rule_versions table" 2024-09-21T09:28:55.988 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.912865837Z level=info msg="Executing migration" id="fix is_paused column for alert_rule_version table" 2024-09-21T09:28:55.988 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.913216529Z level=info msg="Executing migration" id=create_alert_configuration_table 2024-09-21T09:28:55.988 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.916239332Z level=info msg="Executing migration" id="Add column default in alert_configuration" 2024-09-21T09:28:55.988 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.919050133Z level=info msg="Executing migration" id="alert alert_configuration alertmanager_configuration column from TEXT to MEDIUMTEXT if mysql" 2024-09-21T09:28:55.988 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.922039341Z level=info msg="Executing migration" id="add column org_id in alert_configuration" 2024-09-21T09:28:55.988 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.922970474Z level=info msg="Executing migration" id="add index in alert_configuration table on org_id column" 2024-09-21T09:28:55.988 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.925902252Z level=info msg="Executing migration" id="add configuration_hash column to alert_configuration" 2024-09-21T09:28:55.989 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.928605792Z level=info msg="Executing migration" id=create_ngalert_configuration_table 2024-09-21T09:28:55.989 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.931531249Z level=info msg="Executing migration" id="add index in ngalert_configuration on org_id column" 2024-09-21T09:28:55.989 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.932018889Z level=info msg="Executing migration" id="add column send_alerts_to in ngalert_configuration" 2024-09-21T09:28:55.989 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.934928474Z level=info msg="Executing migration" id="create provenance_type table" 2024-09-21T09:28:55.989 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.937502576Z level=info msg="Executing migration" id="add index to uniquify (record_key, record_type, org_id) columns" 2024-09-21T09:28:55.989 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.940264373Z level=info msg="Executing migration" id="create alert_image table" 2024-09-21T09:28:55.989 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.943114471Z level=info msg="Executing migration" id="add unique index on token to alert_image table" 2024-09-21T09:28:55.989 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.945559008Z level=info msg="Executing migration" id="support longer URLs in alert_image table" 2024-09-21T09:28:55.989 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.948339327Z level=info msg="Executing migration" id=create_alert_configuration_history_table 2024-09-21T09:28:55.989 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.950915109Z level=info msg="Executing migration" id="drop non-unique orgID index on alert_configuration" 2024-09-21T09:28:55.989 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.951478634Z level=info msg="Executing migration" id="drop unique orgID index on alert_configuration if exists" 2024-09-21T09:28:55.989 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.95157319Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop unique orgID index on alert_configuration if exists" 2024-09-21T09:28:55.989 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.954180955Z level=info msg="Executing migration" id="extract alertmanager configuration history to separate table" 2024-09-21T09:28:55.989 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.956839673Z level=info msg="Executing migration" id="add unique index on orgID to alert_configuration" 2024-09-21T09:28:55.990 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.959321271Z level=info msg="Executing migration" id="clear migration entry \"remove unified alerting data\"" 2024-09-21T09:28:55.990 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.959465095Z level=info msg="Executing migration" id="move dashboard alerts to unified alerting" 2024-09-21T09:28:55.990 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.959605661Z level=info msg="alerts found to migrate" alerts=0 2024-09-21T09:28:55.990 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.960041659Z level=info msg="Executing migration" id="create library_element table v1" 2024-09-21T09:28:55.990 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.962760256Z level=info msg="Executing migration" id="add index library_element org_id-folder_id-name-kind" 2024-09-21T09:28:55.990 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.965579385Z level=info msg="Executing migration" id="create library_element_connection table v1" 2024-09-21T09:28:55.990 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.968428036Z level=info msg="Executing migration" id="add index library_element_connection element_id-kind-connection_id" 2024-09-21T09:28:55.990 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.968935771Z level=info msg="Executing migration" id="add unique index library_element org_id_uid" 2024-09-21T09:28:55.990 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.971596318Z level=info msg="Executing migration" id="increase max description length to 2048" 2024-09-21T09:28:55.990 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.97429203Z level=info msg="Executing migration" id="clone move dashboard alerts to unified alerting" 2024-09-21T09:28:55.990 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.976837676Z level=info msg="Executing migration" id="create data_keys table" 2024-09-21T09:28:55.990 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.97731611Z level=info msg="Executing migration" id="create secrets table" 2024-09-21T09:28:55.990 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.978156444Z level=info msg="Executing migration" id="rename data_keys name column to id" 2024-09-21T09:28:56.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:55 smithi022 ceph-mon[27763]: pgmap v24: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:28:56.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:55 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:28:56.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:55 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:28:56.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:55 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:28:56.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:55 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:28:56.059 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:55 smithi022 ceph-mon[39525]: pgmap v24: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:28:56.059 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:55 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:28:56.059 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:55 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:28:56.059 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:55 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:28:56.059 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:55 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:28:56.229 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.982741575Z level=info msg="Executing migration" id="add name column into data_keys" 2024-09-21T09:28:56.229 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.983664312Z level=info msg="Executing migration" id="copy data_keys id column values into name" 2024-09-21T09:28:56.229 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.984035212Z level=info msg="Executing migration" id="rename data_keys name column to label" 2024-09-21T09:28:56.229 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.98840265Z level=info msg="Executing migration" id="rename data_keys id column back to name" 2024-09-21T09:28:56.229 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.992710505Z level=info msg="Executing migration" id="create kv_store table v1" 2024-09-21T09:28:56.229 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.99357456Z level=info msg="Executing migration" id="add index kv_store.org_id-namespace-key" 2024-09-21T09:28:56.229 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.994069189Z level=info msg="Executing migration" id="update dashboard_uid and panel_id from existing annotations" 2024-09-21T09:28:56.229 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.99487662Z level=info msg="Executing migration" id="create permission table" 2024-09-21T09:28:56.230 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.995298201Z level=info msg="Executing migration" id="add unique index permission.role_id" 2024-09-21T09:28:56.230 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.995778478Z level=info msg="Executing migration" id="add unique index role_id_action_scope" 2024-09-21T09:28:56.230 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.9966699Z level=info msg="Executing migration" id="create role table" 2024-09-21T09:28:56.230 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.997091528Z level=info msg="Executing migration" id="add column display_name" 2024-09-21T09:28:56.230 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.998035815Z level=info msg="Executing migration" id="add column group_name" 2024-09-21T09:28:56.230 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.99898718Z level=info msg="Executing migration" id="add index role.org_id" 2024-09-21T09:28:56.230 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:55.999843613Z level=info msg="Executing migration" id="add unique index role_org_id_name" 2024-09-21T09:28:56.230 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:56.000316478Z level=info msg="Executing migration" id="add index role_org_id_uid" 2024-09-21T09:28:56.230 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:56.001221688Z level=info msg="Executing migration" id="create team role table" 2024-09-21T09:28:56.230 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:56.002114438Z level=info msg="Executing migration" id="add index team_role.org_id" 2024-09-21T09:28:56.230 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:56.002947642Z level=info msg="Executing migration" id="add unique index team_role_org_id_team_id_role_id" 2024-09-21T09:28:56.230 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:56.003415211Z level=info msg="Executing migration" id="add index team_role.team_id" 2024-09-21T09:28:56.230 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:56.004448941Z level=info msg="Executing migration" id="create user role table" 2024-09-21T09:28:56.231 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:56.005142036Z level=info msg="Executing migration" id="add index user_role.org_id" 2024-09-21T09:28:56.231 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:56.006041661Z level=info msg="Executing migration" id="add unique index user_role_org_id_user_id_role_id" 2024-09-21T09:28:56.231 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:56.00648567Z level=info msg="Executing migration" id="add index user_role.user_id" 2024-09-21T09:28:56.231 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:56.007539757Z level=info msg="Executing migration" id="create builtin role table" 2024-09-21T09:28:56.231 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:56.00821524Z level=info msg="Executing migration" id="add index builtin_role.role_id" 2024-09-21T09:28:56.231 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:56.009114859Z level=info msg="Executing migration" id="add index builtin_role.name" 2024-09-21T09:28:56.231 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:56.009584261Z level=info msg="Executing migration" id="Add column org_id to builtin_role table" 2024-09-21T09:28:56.231 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:56.010562887Z level=info msg="Executing migration" id="add index builtin_role.org_id" 2024-09-21T09:28:56.231 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:56.011445975Z level=info msg="Executing migration" id="add unique index builtin_role_org_id_role_id_role" 2024-09-21T09:28:56.231 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:56.012330278Z level=info msg="Executing migration" id="Remove unique index role_org_id_uid" 2024-09-21T09:28:56.231 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:56.012774174Z level=info msg="Executing migration" id="add unique index role.uid" 2024-09-21T09:28:56.231 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:56.013664623Z level=info msg="Executing migration" id="create seed assignment table" 2024-09-21T09:28:56.231 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:56.016719782Z level=info msg="Executing migration" id="add unique index builtin_role_role_name" 2024-09-21T09:28:56.231 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:56.019390692Z level=info msg="Executing migration" id="add column hidden to role table" 2024-09-21T09:28:56.232 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:56.022375364Z level=info msg="Executing migration" id="create query_history table v1" 2024-09-21T09:28:56.232 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:56.025368287Z level=info msg="Executing migration" id="add index query_history.org_id-created_by-datasource_uid" 2024-09-21T09:28:56.232 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:56.028259142Z level=info msg="Executing migration" id="alter table query_history alter column created_by type to bigint" 2024-09-21T09:28:56.232 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:56.031058981Z level=info msg="Executing migration" id="teams permissions migration" 2024-09-21T09:28:56.233 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:56.031483857Z level=info msg="Executing migration" id="dashboard permissions" 2024-09-21T09:28:56.233 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:56.034642734Z level=info msg="Executing migration" id="dashboard permissions uid scopes" 2024-09-21T09:28:56.233 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:56.037513085Z level=info msg="Executing migration" id="drop managed folder create actions" 2024-09-21T09:28:56.233 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:56.040492766Z level=info msg="Executing migration" id="alerting notification permissions" 2024-09-21T09:28:56.233 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:56.040870439Z level=info msg="Executing migration" id="create query_history_star table v1" 2024-09-21T09:28:56.233 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:56.043803553Z level=info msg="Executing migration" id="add index query_history.user_id-query_uid" 2024-09-21T09:28:56.233 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:56.046678914Z level=info msg="Executing migration" id="add column org_id in query_history_star" 2024-09-21T09:28:56.233 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:56.049430158Z level=info msg="Executing migration" id="alter table query_history_star_mig column user_id type to bigint" 2024-09-21T09:28:56.233 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:56.049751521Z level=info msg="Executing migration" id="create correlation table v1" 2024-09-21T09:28:56.233 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:56.052667276Z level=info msg="Executing migration" id="add index correlations.uid" 2024-09-21T09:28:56.233 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:56.055487755Z level=info msg="Executing migration" id="add index correlations.source_uid" 2024-09-21T09:28:56.233 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:56.058224312Z level=info msg="Executing migration" id="add correlation config column" 2024-09-21T09:28:56.233 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:56.059326128Z level=info msg="Executing migration" id="create entity_events table" 2024-09-21T09:28:56.234 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:56.062144259Z level=info msg="Executing migration" id="create dashboard public config v1" 2024-09-21T09:28:56.234 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:56.064840705Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v1" 2024-09-21T09:28:56.234 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:56.065008241Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index UQE_dashboard_public_config_uid - v1" 2024-09-21T09:28:56.234 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:56.067473146Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-09-21T09:28:56.234 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:56.067661108Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-09-21T09:28:56.234 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:56.067947382Z level=info msg="Executing migration" id="Drop old dashboard public config table" 2024-09-21T09:28:56.234 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:56.070615568Z level=info msg="Executing migration" id="recreate dashboard public config v1" 2024-09-21T09:28:56.234 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:56.0710531Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v1" 2024-09-21T09:28:56.234 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:56.073256023Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-09-21T09:28:56.234 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:56.075915528Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v2" 2024-09-21T09:28:56.234 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:56.076387307Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-09-21T09:28:56.234 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:56.079208677Z level=info msg="Executing migration" id="Drop public config table" 2024-09-21T09:28:56.234 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:56.082081848Z level=info msg="Executing migration" id="Recreate dashboard public config v2" 2024-09-21T09:28:56.234 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:56.084605107Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v2" 2024-09-21T09:28:56.235 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:56.085074758Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-09-21T09:28:56.235 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:56.087699682Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_access_token - v2" 2024-09-21T09:28:56.235 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:56.088180718Z level=info msg="Executing migration" id="Rename table dashboard_public_config to dashboard_public - v2" 2024-09-21T09:28:56.235 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:56.091251188Z level=info msg="Executing migration" id="add annotations_enabled column" 2024-09-21T09:28:56.235 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:56.092296413Z level=info msg="Executing migration" id="add time_selection_enabled column" 2024-09-21T09:28:56.235 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:56.093340979Z level=info msg="Executing migration" id="delete orphaned public dashboards" 2024-09-21T09:28:56.235 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:56.0942795Z level=info msg="Executing migration" id="add share column" 2024-09-21T09:28:56.235 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:56.09549254Z level=info msg="Executing migration" id="create default alerting folders" 2024-09-21T09:28:56.235 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:56.096287425Z level=info msg="Executing migration" id="create file table" 2024-09-21T09:28:56.236 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:56.096744625Z level=info msg="Executing migration" id="file table idx: path natural pk" 2024-09-21T09:28:56.236 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:56.097599102Z level=info msg="Executing migration" id="file table idx: parent_folder_path_hash fast folder retrieval" 2024-09-21T09:28:56.236 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:56.098436693Z level=info msg="Executing migration" id="create file_meta table" 2024-09-21T09:28:56.236 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:56.09884932Z level=info msg="Executing migration" id="file table idx: path key" 2024-09-21T09:28:56.236 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:56.09930781Z level=info msg="Executing migration" id="set path collation in file table" 2024-09-21T09:28:56.236 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:56.100138397Z level=info msg="Executing migration" id="managed permissions migration" 2024-09-21T09:28:56.236 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:56.100530958Z level=info msg="Executing migration" id="managed folder permissions alert actions migration" 2024-09-21T09:28:56.236 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:56.101055556Z level=info msg="Executing migration" id="RBAC action name migrator" 2024-09-21T09:28:56.236 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:56.101896401Z level=info msg="Executing migration" id="Add UID column to playlist" 2024-09-21T09:28:56.236 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:56.102959279Z level=info msg="Executing migration" id="Update uid column values in playlist" 2024-09-21T09:28:56.236 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:56.103289264Z level=info msg="Executing migration" id="Add index for uid in playlist" 2024-09-21T09:28:56.236 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:56.104161721Z level=info msg="Executing migration" id="update group index for alert rules" 2024-09-21T09:28:56.237 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:56.105001984Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated migration" 2024-09-21T09:28:56.237 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:56.105828809Z level=info msg="Executing migration" id="admin only folder/dashboard permission" 2024-09-21T09:28:56.237 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:56.106236553Z level=info msg="Executing migration" id="add action column to seed_assignment" 2024-09-21T09:28:56.237 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:56.107305378Z level=info msg="Executing migration" id="add scope column to seed_assignment" 2024-09-21T09:28:56.237 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:56.108362048Z level=info msg="Executing migration" id="remove unique index builtin_role_role_name before nullable update" 2024-09-21T09:28:56.237 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:56.10932747Z level=info msg="Executing migration" id="update seed_assignment role_name column to nullable" 2024-09-21T09:28:56.237 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:56.119408068Z level=info msg="Executing migration" id="add unique index builtin_role_name back" 2024-09-21T09:28:56.237 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:56.120259302Z level=info msg="Executing migration" id="add unique index builtin_role_action_scope" 2024-09-21T09:28:56.237 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:56.121204065Z level=info msg="Executing migration" id="add primary key to seed_assigment" 2024-09-21T09:28:56.237 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:56.124810451Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated fixed migration" 2024-09-21T09:28:56.237 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:56.12514244Z level=info msg="Executing migration" id="migrate external alertmanagers to datsourcse" 2024-09-21T09:28:56.237 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:56.126007069Z level=info msg="Executing migration" id="create folder table" 2024-09-21T09:28:56.237 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:56.126440946Z level=info msg="Executing migration" id="Add index for parent_uid" 2024-09-21T09:28:56.237 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:56.129020519Z level=info msg="Executing migration" id="Add unique index for folder.uid and folder.org_id" 2024-09-21T09:28:56.238 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:56.131800091Z level=info msg="Executing migration" id="Update folder title length" 2024-09-21T09:28:56.238 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:56.132114999Z level=info msg="Executing migration" id="Add unique index for folder.title and folder.parent_uid" 2024-09-21T09:28:56.238 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=migrator t=2024-09-21T09:28:56.134851348Z level=info msg="migrations completed" performed=485 skipped=0 duration=734.024308ms 2024-09-21T09:28:56.238 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=sqlstore t=2024-09-21T09:28:56.135246466Z level=info msg="Created default organization" 2024-09-21T09:28:56.238 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=plugin.loader t=2024-09-21T09:28:56.198291784Z level=info msg="Plugin registered" pluginID=input 2024-09-21T09:28:56.238 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=plugin.loader t=2024-09-21T09:28:56.227655736Z level=info msg="Plugin registered" pluginID=grafana-piechart-panel 2024-09-21T09:28:56.238 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=plugin.loader t=2024-09-21T09:28:56.227678794Z level=info msg="Plugin registered" pluginID=vonage-status-panel 2024-09-21T09:28:56.238 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=secrets t=2024-09-21T09:28:56.227725514Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2024-09-21T09:28:56.238 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=query_data t=2024-09-21T09:28:56.229602757Z level=info msg="Query Service initialization" 2024-09-21T09:28:56.691 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=live.push_http t=2024-09-21T09:28:56.232088777Z level=info msg="Live Push Gateway initialization" 2024-09-21T09:28:57.010 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:56 smithi022 ceph-mon[39525]: Deploying daemon node-exporter.a on smithi022 2024-09-21T09:28:57.010 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:56 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:28:57.010 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:56 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:28:57.010 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:56 smithi022 ceph-mon[27763]: Deploying daemon node-exporter.a on smithi022 2024-09-21T09:28:57.010 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:56 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:28:57.010 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:56 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:28:57.138 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:56 smithi064 ceph-mon[31124]: Deploying daemon node-exporter.a on smithi022 2024-09-21T09:28:57.138 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:56 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:28:57.138 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:56 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:28:57.138 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=infra.usagestats.collector t=2024-09-21T09:28:56.996777388Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2024-09-21T09:28:57.139 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=provisioning.datasources t=2024-09-21T09:28:56.997089574Z level=warn msg="[Deprecated] the datasource provisioning config is outdated. please upgrade" filename=/etc/grafana/provisioning/datasources/ceph-dashboard.yml 2024-09-21T09:28:57.139 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:57 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=provisioning.datasources t=2024-09-21T09:28:57.029791717Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2024-09-21T09:28:57.139 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:57 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=provisioning.datasources t=2024-09-21T09:28:57.061941746Z level=info msg="inserting datasource from configuration " name=Loki uid=P8E80F9AEF21F6940 2024-09-21T09:28:57.139 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:57 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=provisioning.alerting t=2024-09-21T09:28:57.084029666Z level=info msg="starting to provision alerting" 2024-09-21T09:28:57.139 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:57 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=provisioning.alerting t=2024-09-21T09:28:57.084050594Z level=info msg="finished to provision alerting" 2024-09-21T09:28:57.139 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:57 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=grafanaStorageLogger t=2024-09-21T09:28:57.084135834Z level=info msg="storage starting" 2024-09-21T09:28:57.139 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:57 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=ngalert.state.manager t=2024-09-21T09:28:57.08438829Z level=info msg="Warming state cache for startup" 2024-09-21T09:28:57.139 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:57 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=http.server t=2024-09-21T09:28:57.087378908Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2024-09-21T09:28:57.139 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:57 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=ngalert.state.manager t=2024-09-21T09:28:57.094668715Z level=info msg="State cache has been initialized" states=0 duration=10.278115ms 2024-09-21T09:28:57.139 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:57 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=sqlstore.transactions t=2024-09-21T09:28:57.094662004Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-09-21T09:28:57.139 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:57 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=ngalert.multiorg.alertmanager t=2024-09-21T09:28:57.094747596Z level=info msg="starting MultiOrg Alertmanager" 2024-09-21T09:28:57.139 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:57 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=ticker t=2024-09-21T09:28:57.094763573Z level=info msg=starting first_tick=2024-09-21T09:29:00Z 2024-09-21T09:28:57.139 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:57 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=sqlstore.transactions t=2024-09-21T09:28:57.136985546Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-09-21T09:28:57.441 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:57 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=sqlstore.transactions t=2024-09-21T09:28:57.178295652Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-09-21T09:28:57.441 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:28:57 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[63790]: logger=sqlstore.transactions t=2024-09-21T09:28:57.189641474Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=1 code="database is locked" 2024-09-21T09:28:58.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:57 smithi022 ceph-mon[39525]: pgmap v25: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:28:58.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:57 smithi022 ceph-mon[27763]: pgmap v25: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:28:58.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:57 smithi064 ceph-mon[31124]: pgmap v25: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:28:59.981 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:28:59 smithi022 ceph-mon[39525]: pgmap v26: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:28:59.981 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:28:59 smithi022 ceph-mon[27763]: pgmap v26: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:29:00.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:28:59 smithi064 ceph-mon[31124]: pgmap v26: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:29:00.941 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:29:00 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[33213]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:29:00] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-21T09:29:02.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:01 smithi022 ceph-mon[39525]: pgmap v27: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:29:02.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:01 smithi022 ceph-mon[27763]: pgmap v27: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:29:02.177 INFO:teuthology.orchestra.run.smithi022.stdout:34359738425 2024-09-21T09:29:02.177 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid d65e801e-77fa-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.0 2024-09-21T09:29:02.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:01 smithi064 ceph-mon[31124]: pgmap v27: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:29:04.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:03 smithi022 ceph-mon[39525]: pgmap v28: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:29:04.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:03 smithi022 ceph-mon[27763]: pgmap v28: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:29:04.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:03 smithi064 ceph-mon[31124]: pgmap v28: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:29:04.600 INFO:teuthology.orchestra.run.smithi022.stdout:180388626461 2024-09-21T09:29:04.601 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid d65e801e-77fa-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.6 2024-09-21T09:29:06.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:05 smithi064 ceph-mon[31124]: pgmap v29: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:29:06.197 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:05 smithi022 ceph-mon[27763]: pgmap v29: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:29:06.197 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:05 smithi022 ceph-mon[39525]: pgmap v29: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:29:06.198 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:29:05 smithi022 bash[76622]: Writing manifest to image destination 2024-09-21T09:29:06.557 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:29:06 smithi022 podman[76622]: 2024-09-21 09:29:06.378210125 +0000 UTC m=+3.698807032 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-09-21T09:29:06.723 INFO:teuthology.orchestra.run.smithi022.stdout:128849018919 2024-09-21T09:29:06.723 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid d65e801e-77fa-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.4 2024-09-21T09:29:07.037 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:29:06 smithi022 podman[76622]: 2024-09-21 09:29:06.832143574 +0000 UTC m=+4.152740472 container create 7cf4c99ebf3aed86db0ef63d6e9b50874848ebdc9f47803f29d61f6b3cecc372 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-21T09:29:07.308 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:29:07 smithi022 podman[76622]: 2024-09-21 09:29:07.126913264 +0000 UTC m=+4.447510169 container init 7cf4c99ebf3aed86db0ef63d6e9b50874848ebdc9f47803f29d61f6b3cecc372 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-21T09:29:07.308 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:29:07 smithi022 podman[76622]: 2024-09-21 09:29:07.132174043 +0000 UTC m=+4.452770944 container start 7cf4c99ebf3aed86db0ef63d6e9b50874848ebdc9f47803f29d61f6b3cecc372 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-21T09:29:07.308 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:29:07 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a[76952]: ts=2024-09-21T09:29:07.136Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-09-21T09:29:07.308 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:29:07 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a[76952]: ts=2024-09-21T09:29:07.136Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-09-21T09:29:07.308 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:29:07 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a[76952]: ts=2024-09-21T09:29:07.136Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-09-21T09:29:07.309 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:29:07 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a[76952]: ts=2024-09-21T09:29:07.136Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-09-21T09:29:07.309 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:29:07 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a[76952]: ts=2024-09-21T09:29:07.137Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-09-21T09:29:07.309 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:29:07 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a[76952]: ts=2024-09-21T09:29:07.137Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-09-21T09:29:07.309 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:29:07 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a[76952]: ts=2024-09-21T09:29:07.137Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-09-21T09:29:07.309 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:29:07 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a[76952]: ts=2024-09-21T09:29:07.137Z caller=node_exporter.go:117 level=info collector=arp 2024-09-21T09:29:07.309 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:29:07 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a[76952]: ts=2024-09-21T09:29:07.137Z caller=node_exporter.go:117 level=info collector=bcache 2024-09-21T09:29:07.309 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:29:07 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a[76952]: ts=2024-09-21T09:29:07.137Z caller=node_exporter.go:117 level=info collector=bonding 2024-09-21T09:29:07.309 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:29:07 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a[76952]: ts=2024-09-21T09:29:07.137Z caller=node_exporter.go:117 level=info collector=btrfs 2024-09-21T09:29:07.309 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:29:07 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a[76952]: ts=2024-09-21T09:29:07.137Z caller=node_exporter.go:117 level=info collector=conntrack 2024-09-21T09:29:07.309 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:29:07 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a[76952]: ts=2024-09-21T09:29:07.137Z caller=node_exporter.go:117 level=info collector=cpu 2024-09-21T09:29:07.309 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:29:07 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a[76952]: ts=2024-09-21T09:29:07.137Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-09-21T09:29:07.310 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:29:07 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a[76952]: ts=2024-09-21T09:29:07.137Z caller=node_exporter.go:117 level=info collector=diskstats 2024-09-21T09:29:07.310 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:29:07 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a[76952]: ts=2024-09-21T09:29:07.137Z caller=node_exporter.go:117 level=info collector=dmi 2024-09-21T09:29:07.310 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:29:07 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a[76952]: ts=2024-09-21T09:29:07.137Z caller=node_exporter.go:117 level=info collector=edac 2024-09-21T09:29:07.310 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:29:07 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a[76952]: ts=2024-09-21T09:29:07.137Z caller=node_exporter.go:117 level=info collector=entropy 2024-09-21T09:29:07.310 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:29:07 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a[76952]: ts=2024-09-21T09:29:07.137Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-09-21T09:29:07.310 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:29:07 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a[76952]: ts=2024-09-21T09:29:07.137Z caller=node_exporter.go:117 level=info collector=filefd 2024-09-21T09:29:07.310 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:29:07 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a[76952]: ts=2024-09-21T09:29:07.137Z caller=node_exporter.go:117 level=info collector=filesystem 2024-09-21T09:29:07.310 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:29:07 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a[76952]: ts=2024-09-21T09:29:07.137Z caller=node_exporter.go:117 level=info collector=hwmon 2024-09-21T09:29:07.310 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:29:07 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a[76952]: ts=2024-09-21T09:29:07.137Z caller=node_exporter.go:117 level=info collector=infiniband 2024-09-21T09:29:07.311 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:29:07 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a[76952]: ts=2024-09-21T09:29:07.137Z caller=node_exporter.go:117 level=info collector=ipvs 2024-09-21T09:29:07.311 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:29:07 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a[76952]: ts=2024-09-21T09:29:07.137Z caller=node_exporter.go:117 level=info collector=loadavg 2024-09-21T09:29:07.311 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:29:07 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a[76952]: ts=2024-09-21T09:29:07.137Z caller=node_exporter.go:117 level=info collector=mdadm 2024-09-21T09:29:07.311 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:29:07 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a[76952]: ts=2024-09-21T09:29:07.137Z caller=node_exporter.go:117 level=info collector=meminfo 2024-09-21T09:29:07.311 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:29:07 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a[76952]: ts=2024-09-21T09:29:07.137Z caller=node_exporter.go:117 level=info collector=netclass 2024-09-21T09:29:07.311 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:29:07 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a[76952]: ts=2024-09-21T09:29:07.137Z caller=node_exporter.go:117 level=info collector=netdev 2024-09-21T09:29:07.311 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:29:07 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a[76952]: ts=2024-09-21T09:29:07.137Z caller=node_exporter.go:117 level=info collector=netstat 2024-09-21T09:29:07.311 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:29:07 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a[76952]: ts=2024-09-21T09:29:07.137Z caller=node_exporter.go:117 level=info collector=nfs 2024-09-21T09:29:07.311 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:29:07 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a[76952]: ts=2024-09-21T09:29:07.137Z caller=node_exporter.go:117 level=info collector=nfsd 2024-09-21T09:29:07.311 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:29:07 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a[76952]: ts=2024-09-21T09:29:07.137Z caller=node_exporter.go:117 level=info collector=nvme 2024-09-21T09:29:07.311 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:29:07 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a[76952]: ts=2024-09-21T09:29:07.137Z caller=node_exporter.go:117 level=info collector=os 2024-09-21T09:29:07.311 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:29:07 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a[76952]: ts=2024-09-21T09:29:07.137Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-09-21T09:29:07.311 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:29:07 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a[76952]: ts=2024-09-21T09:29:07.137Z caller=node_exporter.go:117 level=info collector=pressure 2024-09-21T09:29:07.312 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:29:07 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a[76952]: ts=2024-09-21T09:29:07.137Z caller=node_exporter.go:117 level=info collector=rapl 2024-09-21T09:29:07.312 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:29:07 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a[76952]: ts=2024-09-21T09:29:07.137Z caller=node_exporter.go:117 level=info collector=schedstat 2024-09-21T09:29:07.312 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:29:07 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a[76952]: ts=2024-09-21T09:29:07.137Z caller=node_exporter.go:117 level=info collector=selinux 2024-09-21T09:29:07.312 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:29:07 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a[76952]: ts=2024-09-21T09:29:07.137Z caller=node_exporter.go:117 level=info collector=sockstat 2024-09-21T09:29:07.312 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:29:07 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a[76952]: ts=2024-09-21T09:29:07.137Z caller=node_exporter.go:117 level=info collector=softnet 2024-09-21T09:29:07.312 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:29:07 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a[76952]: ts=2024-09-21T09:29:07.137Z caller=node_exporter.go:117 level=info collector=stat 2024-09-21T09:29:07.312 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:29:07 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a[76952]: ts=2024-09-21T09:29:07.137Z caller=node_exporter.go:117 level=info collector=tapestats 2024-09-21T09:29:07.312 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:29:07 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a[76952]: ts=2024-09-21T09:29:07.137Z caller=node_exporter.go:117 level=info collector=textfile 2024-09-21T09:29:07.312 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:29:07 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a[76952]: ts=2024-09-21T09:29:07.137Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-09-21T09:29:07.312 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:29:07 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a[76952]: ts=2024-09-21T09:29:07.137Z caller=node_exporter.go:117 level=info collector=time 2024-09-21T09:29:07.312 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:29:07 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a[76952]: ts=2024-09-21T09:29:07.137Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-09-21T09:29:07.312 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:29:07 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a[76952]: ts=2024-09-21T09:29:07.137Z caller=node_exporter.go:117 level=info collector=uname 2024-09-21T09:29:07.312 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:29:07 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a[76952]: ts=2024-09-21T09:29:07.137Z caller=node_exporter.go:117 level=info collector=vmstat 2024-09-21T09:29:07.312 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:29:07 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a[76952]: ts=2024-09-21T09:29:07.137Z caller=node_exporter.go:117 level=info collector=xfs 2024-09-21T09:29:07.313 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:29:07 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a[76952]: ts=2024-09-21T09:29:07.137Z caller=node_exporter.go:117 level=info collector=zfs 2024-09-21T09:29:07.313 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:29:07 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a[76952]: ts=2024-09-21T09:29:07.138Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-09-21T09:29:07.313 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:29:07 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a[76952]: ts=2024-09-21T09:29:07.138Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-09-21T09:29:07.313 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:29:07 smithi022 bash[76622]: 7cf4c99ebf3aed86db0ef63d6e9b50874848ebdc9f47803f29d61f6b3cecc372 2024-09-21T09:29:07.313 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:29:07 smithi022 systemd[1]: Started Ceph node-exporter.a for d65e801e-77fa-11ef-baf6-efdc52797490. 2024-09-21T09:29:07.685 INFO:teuthology.orchestra.run.smithi022.stdout:206158430234 2024-09-21T09:29:07.685 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid d65e801e-77fa-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.7 2024-09-21T09:29:08.017 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:07 smithi022 ceph-mon[39525]: pgmap v30: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:29:08.017 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:07 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:29:08.017 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:07 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:29:08.017 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:07 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:29:08.017 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:07 smithi022 ceph-mon[27763]: pgmap v30: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:29:08.017 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:07 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:29:08.017 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:07 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:29:08.017 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:07 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:29:08.041 INFO:teuthology.orchestra.run.smithi022.stdout:77309411377 2024-09-21T09:29:08.041 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid d65e801e-77fa-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.2 2024-09-21T09:29:08.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:07 smithi064 ceph-mon[31124]: pgmap v30: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:29:08.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:07 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:29:08.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:07 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:29:08.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:07 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:29:08.192 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:29:07 smithi064 systemd[1]: Starting Ceph node-exporter.b for d65e801e-77fa-11ef-baf6-efdc52797490... 2024-09-21T09:29:08.561 INFO:teuthology.orchestra.run.smithi022.stdout:154618822692 2024-09-21T09:29:08.561 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid d65e801e-77fa-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.5 2024-09-21T09:29:08.577 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:29:08 smithi064 bash[64071]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2024-09-21T09:29:08.577 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:29:08 smithi064 bash[64071]: Getting image source signatures 2024-09-21T09:29:08.883 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:29:08 smithi064 bash[64071]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2024-09-21T09:29:08.883 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:29:08 smithi064 bash[64071]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2024-09-21T09:29:08.883 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:29:08 smithi064 bash[64071]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2024-09-21T09:29:09.001 INFO:teuthology.orchestra.run.smithi022.stdout:107374182444 2024-09-21T09:29:09.002 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid d65e801e-77fa-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.3 2024-09-21T09:29:09.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:08 smithi064 ceph-mon[31124]: Deploying daemon node-exporter.b on smithi064 2024-09-21T09:29:09.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:08 smithi022 ceph-mon[39525]: Deploying daemon node-exporter.b on smithi064 2024-09-21T09:29:09.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:08 smithi022 ceph-mon[27763]: Deploying daemon node-exporter.b on smithi064 2024-09-21T09:29:09.423 INFO:teuthology.orchestra.run.smithi022.stdout:55834574903 2024-09-21T09:29:09.424 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid d65e801e-77fa-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.1 2024-09-21T09:29:09.503 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:29:09.505 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:29:09.505 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:29:09.526 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:29:09.526 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:29:09.537 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:29:09.537 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:29:10.049 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:09 smithi022 ceph-mon[39525]: pgmap v31: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:29:10.049 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:09 smithi022 ceph-mon[27763]: pgmap v31: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:29:10.176 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:09 smithi064 ceph-mon[31124]: pgmap v31: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:29:10.441 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:29:10 smithi064 bash[64071]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2024-09-21T09:29:10.442 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:29:10 smithi064 bash[64071]: Writing manifest to image destination 2024-09-21T09:29:10.756 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:29:10 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[33213]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:29:10] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-21T09:29:10.757 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:29:10 smithi064 podman[64071]: 2024-09-21 09:29:10.556409421 +0000 UTC m=+2.460481996 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-09-21T09:29:10.757 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:29:10 smithi064 podman[64071]: 2024-09-21 09:29:10.654950845 +0000 UTC m=+2.559023448 container create 643a1e1aec614f5d11706830fdcc0b7d62c379d649e64324299fa4475b85f6db (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-21T09:29:11.192 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:29:10 smithi064 podman[64071]: 2024-09-21 09:29:10.874297673 +0000 UTC m=+2.778370241 container init 643a1e1aec614f5d11706830fdcc0b7d62c379d649e64324299fa4475b85f6db (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-21T09:29:11.192 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:29:10 smithi064 podman[64071]: 2024-09-21 09:29:10.880786568 +0000 UTC m=+2.784859144 container start 643a1e1aec614f5d11706830fdcc0b7d62c379d649e64324299fa4475b85f6db (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-21T09:29:11.192 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:29:10 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b[64144]: ts=2024-09-21T09:29:10.884Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-09-21T09:29:11.192 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:29:10 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b[64144]: ts=2024-09-21T09:29:10.884Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-09-21T09:29:11.192 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:29:10 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b[64144]: ts=2024-09-21T09:29:10.885Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-09-21T09:29:11.192 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:29:10 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b[64144]: ts=2024-09-21T09:29:10.885Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-09-21T09:29:11.192 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:29:10 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b[64144]: ts=2024-09-21T09:29:10.885Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-09-21T09:29:11.192 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:29:10 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b[64144]: ts=2024-09-21T09:29:10.885Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-09-21T09:29:11.192 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:29:10 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b[64144]: ts=2024-09-21T09:29:10.886Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-09-21T09:29:11.192 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:29:10 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b[64144]: ts=2024-09-21T09:29:10.886Z caller=node_exporter.go:117 level=info collector=arp 2024-09-21T09:29:11.193 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:29:10 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b[64144]: ts=2024-09-21T09:29:10.886Z caller=node_exporter.go:117 level=info collector=bcache 2024-09-21T09:29:11.193 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:29:10 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b[64144]: ts=2024-09-21T09:29:10.886Z caller=node_exporter.go:117 level=info collector=bonding 2024-09-21T09:29:11.193 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:29:10 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b[64144]: ts=2024-09-21T09:29:10.886Z caller=node_exporter.go:117 level=info collector=btrfs 2024-09-21T09:29:11.193 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:29:10 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b[64144]: ts=2024-09-21T09:29:10.886Z caller=node_exporter.go:117 level=info collector=conntrack 2024-09-21T09:29:11.193 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:29:10 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b[64144]: ts=2024-09-21T09:29:10.886Z caller=node_exporter.go:117 level=info collector=cpu 2024-09-21T09:29:11.193 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:29:10 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b[64144]: ts=2024-09-21T09:29:10.886Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-09-21T09:29:11.193 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:29:10 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b[64144]: ts=2024-09-21T09:29:10.886Z caller=node_exporter.go:117 level=info collector=diskstats 2024-09-21T09:29:11.193 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:29:10 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b[64144]: ts=2024-09-21T09:29:10.886Z caller=node_exporter.go:117 level=info collector=dmi 2024-09-21T09:29:11.194 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:29:10 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b[64144]: ts=2024-09-21T09:29:10.886Z caller=node_exporter.go:117 level=info collector=edac 2024-09-21T09:29:11.194 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:29:10 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b[64144]: ts=2024-09-21T09:29:10.886Z caller=node_exporter.go:117 level=info collector=entropy 2024-09-21T09:29:11.194 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:29:10 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b[64144]: ts=2024-09-21T09:29:10.886Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-09-21T09:29:11.194 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:29:10 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b[64144]: ts=2024-09-21T09:29:10.886Z caller=node_exporter.go:117 level=info collector=filefd 2024-09-21T09:29:11.194 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:29:10 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b[64144]: ts=2024-09-21T09:29:10.886Z caller=node_exporter.go:117 level=info collector=filesystem 2024-09-21T09:29:11.194 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:29:10 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b[64144]: ts=2024-09-21T09:29:10.886Z caller=node_exporter.go:117 level=info collector=hwmon 2024-09-21T09:29:11.194 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:29:10 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b[64144]: ts=2024-09-21T09:29:10.886Z caller=node_exporter.go:117 level=info collector=infiniband 2024-09-21T09:29:11.194 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:29:10 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b[64144]: ts=2024-09-21T09:29:10.886Z caller=node_exporter.go:117 level=info collector=ipvs 2024-09-21T09:29:11.194 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:29:10 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b[64144]: ts=2024-09-21T09:29:10.886Z caller=node_exporter.go:117 level=info collector=loadavg 2024-09-21T09:29:11.194 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:29:10 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b[64144]: ts=2024-09-21T09:29:10.886Z caller=node_exporter.go:117 level=info collector=mdadm 2024-09-21T09:29:11.194 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:29:10 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b[64144]: ts=2024-09-21T09:29:10.886Z caller=node_exporter.go:117 level=info collector=meminfo 2024-09-21T09:29:11.194 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:29:10 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b[64144]: ts=2024-09-21T09:29:10.886Z caller=node_exporter.go:117 level=info collector=netclass 2024-09-21T09:29:11.195 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:29:10 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b[64144]: ts=2024-09-21T09:29:10.886Z caller=node_exporter.go:117 level=info collector=netdev 2024-09-21T09:29:11.195 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:29:10 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b[64144]: ts=2024-09-21T09:29:10.886Z caller=node_exporter.go:117 level=info collector=netstat 2024-09-21T09:29:11.195 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:29:10 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b[64144]: ts=2024-09-21T09:29:10.886Z caller=node_exporter.go:117 level=info collector=nfs 2024-09-21T09:29:11.195 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:29:10 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b[64144]: ts=2024-09-21T09:29:10.886Z caller=node_exporter.go:117 level=info collector=nfsd 2024-09-21T09:29:11.195 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:29:10 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b[64144]: ts=2024-09-21T09:29:10.886Z caller=node_exporter.go:117 level=info collector=nvme 2024-09-21T09:29:11.195 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:29:10 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b[64144]: ts=2024-09-21T09:29:10.886Z caller=node_exporter.go:117 level=info collector=os 2024-09-21T09:29:11.195 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:29:10 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b[64144]: ts=2024-09-21T09:29:10.886Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-09-21T09:29:11.195 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:29:10 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b[64144]: ts=2024-09-21T09:29:10.886Z caller=node_exporter.go:117 level=info collector=pressure 2024-09-21T09:29:11.195 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:29:10 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b[64144]: ts=2024-09-21T09:29:10.886Z caller=node_exporter.go:117 level=info collector=rapl 2024-09-21T09:29:11.195 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:29:10 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b[64144]: ts=2024-09-21T09:29:10.886Z caller=node_exporter.go:117 level=info collector=schedstat 2024-09-21T09:29:11.195 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:29:10 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b[64144]: ts=2024-09-21T09:29:10.886Z caller=node_exporter.go:117 level=info collector=selinux 2024-09-21T09:29:11.195 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:29:10 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b[64144]: ts=2024-09-21T09:29:10.886Z caller=node_exporter.go:117 level=info collector=sockstat 2024-09-21T09:29:11.195 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:29:10 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b[64144]: ts=2024-09-21T09:29:10.886Z caller=node_exporter.go:117 level=info collector=softnet 2024-09-21T09:29:11.196 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:29:10 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b[64144]: ts=2024-09-21T09:29:10.886Z caller=node_exporter.go:117 level=info collector=stat 2024-09-21T09:29:11.196 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:29:10 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b[64144]: ts=2024-09-21T09:29:10.886Z caller=node_exporter.go:117 level=info collector=tapestats 2024-09-21T09:29:11.196 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:29:10 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b[64144]: ts=2024-09-21T09:29:10.886Z caller=node_exporter.go:117 level=info collector=textfile 2024-09-21T09:29:11.196 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:29:10 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b[64144]: ts=2024-09-21T09:29:10.886Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-09-21T09:29:11.196 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:29:10 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b[64144]: ts=2024-09-21T09:29:10.886Z caller=node_exporter.go:117 level=info collector=time 2024-09-21T09:29:11.196 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:29:10 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b[64144]: ts=2024-09-21T09:29:10.886Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-09-21T09:29:11.196 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:29:10 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b[64144]: ts=2024-09-21T09:29:10.886Z caller=node_exporter.go:117 level=info collector=uname 2024-09-21T09:29:11.196 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:29:10 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b[64144]: ts=2024-09-21T09:29:10.886Z caller=node_exporter.go:117 level=info collector=vmstat 2024-09-21T09:29:11.196 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:29:10 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b[64144]: ts=2024-09-21T09:29:10.886Z caller=node_exporter.go:117 level=info collector=xfs 2024-09-21T09:29:11.196 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:29:10 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b[64144]: ts=2024-09-21T09:29:10.886Z caller=node_exporter.go:117 level=info collector=zfs 2024-09-21T09:29:11.196 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:29:10 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b[64144]: ts=2024-09-21T09:29:10.886Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-09-21T09:29:11.196 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:29:10 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b[64144]: ts=2024-09-21T09:29:10.886Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-09-21T09:29:11.196 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:29:10 smithi064 bash[64071]: 643a1e1aec614f5d11706830fdcc0b7d62c379d649e64324299fa4475b85f6db 2024-09-21T09:29:11.196 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:29:10 smithi064 systemd[1]: Started Ceph node-exporter.b for d65e801e-77fa-11ef-baf6-efdc52797490. 2024-09-21T09:29:12.228 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:12 smithi022 ceph-mon[39525]: pgmap v32: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:29:12.228 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:12 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:29:12.229 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:12 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:29:12.229 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:12 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:29:12.229 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:12 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:29:12.229 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:12 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:29:12.229 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:12 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:29:12.229 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:12 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:29:12.229 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:12 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:29:12.229 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:12 smithi022 ceph-mon[27763]: pgmap v32: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:29:12.229 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:12 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:29:12.230 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:12 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:29:12.230 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:12 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:29:12.230 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:12 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:29:12.230 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:12 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:29:12.230 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:12 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:29:12.230 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:12 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:29:12.230 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:12 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:29:12.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:12 smithi064 ceph-mon[31124]: pgmap v32: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:29:12.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:12 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:29:12.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:12 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:29:12.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:12 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:29:12.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:12 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:29:12.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:12 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:29:12.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:12 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:29:12.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:12 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:29:12.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:12 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:29:14.240 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:14 smithi022 ceph-mon[39525]: pgmap v33: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:29:14.241 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:14 smithi022 ceph-mon[27763]: pgmap v33: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:29:14.397 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:14 smithi064 ceph-mon[31124]: pgmap v33: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:29:15.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:15 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:29:15.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:15 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:29:15.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:15 smithi022 ceph-mon[27763]: pgmap v34: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:29:15.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:15 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:29:15.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:15 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:29:15.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:15 smithi022 ceph-mon[39525]: pgmap v34: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:29:16.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:15 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:29:16.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:15 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:29:16.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:15 smithi064 ceph-mon[31124]: pgmap v34: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:29:17.258 INFO:teuthology.orchestra.run.smithi022.stdout:128849018922 2024-09-21T09:29:18.144 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:17 smithi022 ceph-mon[39525]: pgmap v35: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:29:18.144 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:17 smithi022 ceph-mon[39525]: from='client.? 172.21.15.22:0/2908522980' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-21T09:29:18.144 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:17 smithi022 ceph-mon[39525]: from='client.? 172.21.15.22:0/3347359185' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-21T09:29:18.144 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:17 smithi022 ceph-mon[27763]: pgmap v35: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:29:18.144 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:17 smithi022 ceph-mon[27763]: from='client.? 172.21.15.22:0/2908522980' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-21T09:29:18.144 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:17 smithi022 ceph-mon[27763]: from='client.? 172.21.15.22:0/3347359185' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-21T09:29:18.169 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849018919 got 128849018922 for osd.4 2024-09-21T09:29:18.170 DEBUG:teuthology.parallel:result is None 2024-09-21T09:29:18.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:17 smithi064 ceph-mon[31124]: pgmap v35: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:29:18.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:17 smithi064 ceph-mon[31124]: from='client.? 172.21.15.22:0/2908522980' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-21T09:29:18.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:17 smithi064 ceph-mon[31124]: from='client.? 172.21.15.22:0/3347359185' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-21T09:29:18.522 INFO:teuthology.orchestra.run.smithi022.stdout:77309411380 2024-09-21T09:29:19.121 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:18 smithi022 ceph-mon[39525]: from='client.? 172.21.15.22:0/1689889668' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-21T09:29:19.121 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:18 smithi022 ceph-mon[27763]: from='client.? 172.21.15.22:0/1689889668' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-21T09:29:19.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:18 smithi064 ceph-mon[31124]: from='client.? 172.21.15.22:0/1689889668' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-21T09:29:19.639 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411377 got 77309411380 for osd.2 2024-09-21T09:29:19.639 DEBUG:teuthology.parallel:result is None 2024-09-21T09:29:19.721 INFO:teuthology.orchestra.run.smithi022.stdout:180388626464 2024-09-21T09:29:19.981 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:19 smithi022 ceph-mon[39525]: pgmap v36: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:29:19.981 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:19 smithi022 ceph-mon[39525]: from='client.? 172.21.15.22:0/127780089' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-21T09:29:19.981 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:19 smithi022 ceph-mon[27763]: pgmap v36: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:29:19.981 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:19 smithi022 ceph-mon[27763]: from='client.? 172.21.15.22:0/127780089' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-21T09:29:20.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:19 smithi064 ceph-mon[31124]: pgmap v36: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:29:20.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:19 smithi064 ceph-mon[31124]: from='client.? 172.21.15.22:0/127780089' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-21T09:29:20.539 INFO:tasks.cephadm.ceph_manager.ceph:need seq 180388626461 got 180388626464 for osd.6 2024-09-21T09:29:20.539 DEBUG:teuthology.parallel:result is None 2024-09-21T09:29:20.942 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:29:20 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[33213]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:29:20] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-21T09:29:21.292 INFO:teuthology.orchestra.run.smithi022.stdout:154618822695 2024-09-21T09:29:21.642 INFO:teuthology.orchestra.run.smithi022.stdout:34359738430 2024-09-21T09:29:21.988 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:21 smithi022 ceph-mon[39525]: pgmap v37: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:29:21.988 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:21 smithi022 ceph-mon[39525]: from='client.? 172.21.15.22:0/3441080282' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-21T09:29:21.989 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:21 smithi022 ceph-mon[39525]: from='client.? 172.21.15.22:0/1423943956' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-21T09:29:21.989 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:21 smithi022 ceph-mon[27763]: pgmap v37: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:29:21.989 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:21 smithi022 ceph-mon[27763]: from='client.? 172.21.15.22:0/3441080282' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-21T09:29:21.989 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:21 smithi022 ceph-mon[27763]: from='client.? 172.21.15.22:0/1423943956' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-21T09:29:22.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:21 smithi064 ceph-mon[31124]: pgmap v37: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:29:22.297 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:21 smithi064 ceph-mon[31124]: from='client.? 172.21.15.22:0/3441080282' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-21T09:29:22.297 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:21 smithi064 ceph-mon[31124]: from='client.? 172.21.15.22:0/1423943956' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-21T09:29:22.365 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738425 got 34359738430 for osd.0 2024-09-21T09:29:22.365 DEBUG:teuthology.parallel:result is None 2024-09-21T09:29:22.715 INFO:tasks.cephadm.ceph_manager.ceph:need seq 154618822692 got 154618822695 for osd.5 2024-09-21T09:29:22.715 DEBUG:teuthology.parallel:result is None 2024-09-21T09:29:22.825 INFO:teuthology.orchestra.run.smithi022.stdout:107374182448 2024-09-21T09:29:22.990 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:29:23.421 INFO:tasks.cephadm.ceph_manager.ceph:need seq 107374182444 got 107374182448 for osd.3 2024-09-21T09:29:23.421 DEBUG:teuthology.parallel:result is None 2024-09-21T09:29:23.916 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:23 smithi022 ceph-mon[27763]: pgmap v38: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:29:23.917 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:23 smithi022 ceph-mon[39525]: pgmap v38: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:29:24.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:23 smithi064 ceph-mon[31124]: pgmap v38: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:29:26.057 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:25 smithi022 ceph-mon[39525]: pgmap v39: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:29:26.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:25 smithi022 ceph-mon[27763]: pgmap v39: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:29:26.151 INFO:teuthology.orchestra.run.smithi022.stdout:206158430238 2024-09-21T09:29:26.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:25 smithi064 ceph-mon[31124]: pgmap v39: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:29:26.788 INFO:teuthology.orchestra.run.smithi022.stdout:55834574907 2024-09-21T09:29:27.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:26 smithi022 ceph-mon[39525]: from='client.? 172.21.15.22:0/612201765' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-21T09:29:27.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:26 smithi022 ceph-mon[39525]: from='client.? 172.21.15.22:0/58239189' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-21T09:29:27.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:26 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:29:27.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:26 smithi022 ceph-mon[27763]: from='client.? 172.21.15.22:0/612201765' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-21T09:29:27.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:26 smithi022 ceph-mon[27763]: from='client.? 172.21.15.22:0/58239189' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-21T09:29:27.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:26 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:29:27.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:26 smithi064 ceph-mon[31124]: from='client.? 172.21.15.22:0/612201765' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-21T09:29:27.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:26 smithi064 ceph-mon[31124]: from='client.? 172.21.15.22:0/58239189' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-21T09:29:27.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:26 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:29:27.467 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574903 got 55834574907 for osd.1 2024-09-21T09:29:27.468 DEBUG:teuthology.parallel:result is None 2024-09-21T09:29:27.956 INFO:tasks.cephadm.ceph_manager.ceph:need seq 206158430234 got 206158430238 for osd.7 2024-09-21T09:29:27.956 DEBUG:teuthology.parallel:result is None 2024-09-21T09:29:27.956 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-09-21T09:29:27.956 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid d65e801e-77fa-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T09:29:28.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:27 smithi064 ceph-mon[31124]: pgmap v40: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:29:28.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:27 smithi022 ceph-mon[39525]: pgmap v40: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:29:28.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:27 smithi022 ceph-mon[27763]: pgmap v40: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:29:28.874 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:29:30.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:29 smithi064 ceph-mon[31124]: pgmap v41: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:29:30.307 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:29 smithi022 ceph-mon[39525]: pgmap v41: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:29:30.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:29 smithi022 ceph-mon[27763]: pgmap v41: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:29:30.941 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:29:30 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[33213]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:29:30] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-21T09:29:31.397 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:31 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:29:31.397 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:31 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:29:31.397 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:31 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:29:31.397 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:31 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:29:31.397 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:31 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:29:31.397 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:31 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:29:31.398 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:31 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:29:31.398 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:31 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:29:31.398 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:31 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:29:31.398 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:31 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:29:31.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:31 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:29:31.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:31 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:29:31.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:31 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:29:31.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:31 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:29:31.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:31 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:29:32.220 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-09-21T09:29:32.221 INFO:teuthology.orchestra.run.smithi022.stderr:dumped all 2024-09-21T09:29:32.551 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:32 smithi022 ceph-mon[39525]: Reconfiguring alertmanager.a (dependencies changed)... 2024-09-21T09:29:32.551 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:32 smithi022 ceph-mon[39525]: Reconfiguring daemon alertmanager.a on smithi022 2024-09-21T09:29:32.551 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:32 smithi022 ceph-mon[39525]: pgmap v42: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:29:32.552 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:32 smithi022 ceph-mon[27763]: Reconfiguring alertmanager.a (dependencies changed)... 2024-09-21T09:29:32.552 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:32 smithi022 ceph-mon[27763]: Reconfiguring daemon alertmanager.a on smithi022 2024-09-21T09:29:32.552 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:32 smithi022 ceph-mon[27763]: pgmap v42: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:29:32.593 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:32 smithi064 ceph-mon[31124]: Reconfiguring alertmanager.a (dependencies changed)... 2024-09-21T09:29:32.593 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:32 smithi064 ceph-mon[31124]: Reconfiguring daemon alertmanager.a on smithi022 2024-09-21T09:29:32.593 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:32 smithi064 ceph-mon[31124]: pgmap v42: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:29:33.367 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:33 smithi022 ceph-mon[39525]: from='client.14721 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:29:33.368 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:33 smithi022 ceph-mon[27763]: from='client.14721 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:29:33.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:33 smithi064 ceph-mon[31124]: from='client.14721 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:29:34.444 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:34 smithi022 ceph-mon[39525]: pgmap v43: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:29:34.445 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 21 09:29:34 smithi022 systemd[1]: Stopping Ceph alertmanager.a for d65e801e-77fa-11ef-baf6-efdc52797490... 2024-09-21T09:29:34.445 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:34 smithi022 ceph-mon[27763]: pgmap v43: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:29:34.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:34 smithi064 ceph-mon[31124]: pgmap v43: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:29:34.765 INFO:teuthology.orchestra.run.smithi022.stdout:{"pg_ready":true,"pg_map":{"version":42,"stamp":"2024-09-21T09:29:30.803505+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":593515,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":126,"num_read_kb":231,"num_write":163,"num_write_kb":1646,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":124,"ondisk_log_size":124,"up":315,"acting":315,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":315,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2386056,"kb_used_data":7208,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747575160,"statfs":{"total":767960285184,"available":765516963840,"internally_reserved":0,"allocated":7380992,"data_stored":4259117,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002735"},"pg_stats":[{"pgid":"3.3f","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.359261+0000","last_change":"2024-09-21T09:27:35.635695+0000","last_active":"2024-09-21T09:28:33.359261+0000","last_peered":"2024-09-21T09:28:33.359261+0000","last_clean":"2024-09-21T09:28:33.359261+0000","last_became_active":"2024-09-21T09:27:35.635255+0000","last_became_peered":"2024-09-21T09:27:35.635255+0000","last_unstale":"2024-09-21T09:28:33.359261+0000","last_undegraded":"2024-09-21T09:28:33.359261+0000","last_fullsized":"2024-09-21T09:28:33.359261+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:15:50.156743+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3e","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.062366+0000","last_change":"2024-09-21T09:27:35.636115+0000","last_active":"2024-09-21T09:28:33.062366+0000","last_peered":"2024-09-21T09:28:33.062366+0000","last_clean":"2024-09-21T09:28:33.062366+0000","last_became_active":"2024-09-21T09:27:35.635778+0000","last_became_peered":"2024-09-21T09:27:35.635778+0000","last_unstale":"2024-09-21T09:28:33.062366+0000","last_undegraded":"2024-09-21T09:28:33.062366+0000","last_fullsized":"2024-09-21T09:28:33.062366+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:36:35.607057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3d","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.141182+0000","last_change":"2024-09-21T09:27:35.640689+0000","last_active":"2024-09-21T09:28:33.141182+0000","last_peered":"2024-09-21T09:28:33.141182+0000","last_clean":"2024-09-21T09:28:33.141182+0000","last_became_active":"2024-09-21T09:27:35.640415+0000","last_became_peered":"2024-09-21T09:27:35.640415+0000","last_unstale":"2024-09-21T09:28:33.141182+0000","last_undegraded":"2024-09-21T09:28:33.141182+0000","last_fullsized":"2024-09-21T09:28:33.141182+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:12:22.314199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3c","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.066315+0000","last_change":"2024-09-21T09:27:35.644536+0000","last_active":"2024-09-21T09:28:33.066315+0000","last_peered":"2024-09-21T09:28:33.066315+0000","last_clean":"2024-09-21T09:28:33.066315+0000","last_became_active":"2024-09-21T09:27:35.644246+0000","last_became_peered":"2024-09-21T09:27:35.644246+0000","last_unstale":"2024-09-21T09:28:33.066315+0000","last_undegraded":"2024-09-21T09:28:33.066315+0000","last_fullsized":"2024-09-21T09:28:33.066315+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:31:37.886172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3b","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.358990+0000","last_change":"2024-09-21T09:27:35.642779+0000","last_active":"2024-09-21T09:28:33.358990+0000","last_peered":"2024-09-21T09:28:33.358990+0000","last_clean":"2024-09-21T09:28:33.358990+0000","last_became_active":"2024-09-21T09:27:35.642487+0000","last_became_peered":"2024-09-21T09:27:35.642487+0000","last_unstale":"2024-09-21T09:28:33.358990+0000","last_undegraded":"2024-09-21T09:28:33.358990+0000","last_fullsized":"2024-09-21T09:28:33.358990+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:17:29.631740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3a","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.066399+0000","last_change":"2024-09-21T09:27:35.651016+0000","last_active":"2024-09-21T09:28:33.066399+0000","last_peered":"2024-09-21T09:28:33.066399+0000","last_clean":"2024-09-21T09:28:33.066399+0000","last_became_active":"2024-09-21T09:27:35.650742+0000","last_became_peered":"2024-09-21T09:27:35.650742+0000","last_unstale":"2024-09-21T09:28:33.066399+0000","last_undegraded":"2024-09-21T09:28:33.066399+0000","last_fullsized":"2024-09-21T09:28:33.066399+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:11:11.493990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.39","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.358835+0000","last_change":"2024-09-21T09:27:35.647235+0000","last_active":"2024-09-21T09:28:33.358835+0000","last_peered":"2024-09-21T09:28:33.358835+0000","last_clean":"2024-09-21T09:28:33.358835+0000","last_became_active":"2024-09-21T09:27:35.646890+0000","last_became_peered":"2024-09-21T09:27:35.646890+0000","last_unstale":"2024-09-21T09:28:33.358835+0000","last_undegraded":"2024-09-21T09:28:33.358835+0000","last_fullsized":"2024-09-21T09:28:33.358835+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:21:45.995387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.38","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.124976+0000","last_change":"2024-09-21T09:27:35.638161+0000","last_active":"2024-09-21T09:28:33.124976+0000","last_peered":"2024-09-21T09:28:33.124976+0000","last_clean":"2024-09-21T09:28:33.124976+0000","last_became_active":"2024-09-21T09:27:35.637751+0000","last_became_peered":"2024-09-21T09:27:35.637751+0000","last_unstale":"2024-09-21T09:28:33.124976+0000","last_undegraded":"2024-09-21T09:28:33.124976+0000","last_fullsized":"2024-09-21T09:28:33.124976+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:07:40.751242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.37","version":"0'0","reported_seq":31,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.359502+0000","last_change":"2024-09-21T09:27:52.079760+0000","last_active":"2024-09-21T09:28:33.359502+0000","last_peered":"2024-09-21T09:28:33.359502+0000","last_clean":"2024-09-21T09:28:33.359502+0000","last_became_active":"2024-09-21T09:27:52.079352+0000","last_became_peered":"2024-09-21T09:27:52.079352+0000","last_unstale":"2024-09-21T09:28:33.359502+0000","last_undegraded":"2024-09-21T09:28:33.359502+0000","last_fullsized":"2024-09-21T09:28:33.359502+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:57:57.194876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.36","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.484301+0000","last_change":"2024-09-21T09:27:35.636094+0000","last_active":"2024-09-21T09:28:33.484301+0000","last_peered":"2024-09-21T09:28:33.484301+0000","last_clean":"2024-09-21T09:28:33.484301+0000","last_became_active":"2024-09-21T09:27:35.635770+0000","last_became_peered":"2024-09-21T09:27:35.635770+0000","last_unstale":"2024-09-21T09:28:33.484301+0000","last_undegraded":"2024-09-21T09:28:33.484301+0000","last_fullsized":"2024-09-21T09:28:33.484301+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:41:52.909549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.35","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.484371+0000","last_change":"2024-09-21T09:27:35.640571+0000","last_active":"2024-09-21T09:28:33.484371+0000","last_peered":"2024-09-21T09:28:33.484371+0000","last_clean":"2024-09-21T09:28:33.484371+0000","last_became_active":"2024-09-21T09:27:35.640253+0000","last_became_peered":"2024-09-21T09:27:35.640253+0000","last_unstale":"2024-09-21T09:28:33.484371+0000","last_undegraded":"2024-09-21T09:28:33.484371+0000","last_fullsized":"2024-09-21T09:28:33.484371+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:40:44.721446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.34","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.062533+0000","last_change":"2024-09-21T09:27:35.639471+0000","last_active":"2024-09-21T09:28:33.062533+0000","last_peered":"2024-09-21T09:28:33.062533+0000","last_clean":"2024-09-21T09:28:33.062533+0000","last_became_active":"2024-09-21T09:27:35.639056+0000","last_became_peered":"2024-09-21T09:27:35.639056+0000","last_unstale":"2024-09-21T09:28:33.062533+0000","last_undegraded":"2024-09-21T09:28:33.062533+0000","last_fullsized":"2024-09-21T09:28:33.062533+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:16:15.724781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.33","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.125281+0000","last_change":"2024-09-21T09:27:35.637331+0000","last_active":"2024-09-21T09:28:33.125281+0000","last_peered":"2024-09-21T09:28:33.125281+0000","last_clean":"2024-09-21T09:28:33.125281+0000","last_became_active":"2024-09-21T09:27:35.637068+0000","last_became_peered":"2024-09-21T09:27:35.637068+0000","last_unstale":"2024-09-21T09:28:33.125281+0000","last_undegraded":"2024-09-21T09:28:33.125281+0000","last_fullsized":"2024-09-21T09:28:33.125281+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:24:52.176104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.32","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.140751+0000","last_change":"2024-09-21T09:27:35.645565+0000","last_active":"2024-09-21T09:28:33.140751+0000","last_peered":"2024-09-21T09:28:33.140751+0000","last_clean":"2024-09-21T09:28:33.140751+0000","last_became_active":"2024-09-21T09:27:35.644114+0000","last_became_peered":"2024-09-21T09:27:35.644114+0000","last_unstale":"2024-09-21T09:28:33.140751+0000","last_undegraded":"2024-09-21T09:28:33.140751+0000","last_fullsized":"2024-09-21T09:28:33.140751+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:27:18.008605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.31","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.062608+0000","last_change":"2024-09-21T09:27:35.634448+0000","last_active":"2024-09-21T09:28:33.062608+0000","last_peered":"2024-09-21T09:28:33.062608+0000","last_clean":"2024-09-21T09:28:33.062608+0000","last_became_active":"2024-09-21T09:27:35.634089+0000","last_became_peered":"2024-09-21T09:27:35.634089+0000","last_unstale":"2024-09-21T09:28:33.062608+0000","last_undegraded":"2024-09-21T09:28:33.062608+0000","last_fullsized":"2024-09-21T09:28:33.062608+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:14:14.726492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.30","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.066102+0000","last_change":"2024-09-21T09:27:35.651997+0000","last_active":"2024-09-21T09:28:33.066102+0000","last_peered":"2024-09-21T09:28:33.066102+0000","last_clean":"2024-09-21T09:28:33.066102+0000","last_became_active":"2024-09-21T09:27:35.651698+0000","last_became_peered":"2024-09-21T09:27:35.651698+0000","last_unstale":"2024-09-21T09:28:33.066102+0000","last_undegraded":"2024-09-21T09:28:33.066102+0000","last_fullsized":"2024-09-21T09:28:33.066102+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:43:39.574788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2f","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.066495+0000","last_change":"2024-09-21T09:27:35.646363+0000","last_active":"2024-09-21T09:28:33.066495+0000","last_peered":"2024-09-21T09:28:33.066495+0000","last_clean":"2024-09-21T09:28:33.066495+0000","last_became_active":"2024-09-21T09:27:35.646084+0000","last_became_peered":"2024-09-21T09:27:35.646084+0000","last_unstale":"2024-09-21T09:28:33.066495+0000","last_undegraded":"2024-09-21T09:28:33.066495+0000","last_fullsized":"2024-09-21T09:28:33.066495+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:35:38.711926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2e","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.061977+0000","last_change":"2024-09-21T09:27:35.636697+0000","last_active":"2024-09-21T09:28:33.061977+0000","last_peered":"2024-09-21T09:28:33.061977+0000","last_clean":"2024-09-21T09:28:33.061977+0000","last_became_active":"2024-09-21T09:27:35.636438+0000","last_became_peered":"2024-09-21T09:27:35.636438+0000","last_unstale":"2024-09-21T09:28:33.061977+0000","last_undegraded":"2024-09-21T09:28:33.061977+0000","last_fullsized":"2024-09-21T09:28:33.061977+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:55:44.287544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,7],"acting":[2,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2d","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.359395+0000","last_change":"2024-09-21T09:27:35.636110+0000","last_active":"2024-09-21T09:28:33.359395+0000","last_peered":"2024-09-21T09:28:33.359395+0000","last_clean":"2024-09-21T09:28:33.359395+0000","last_became_active":"2024-09-21T09:27:35.635807+0000","last_became_peered":"2024-09-21T09:27:35.635807+0000","last_unstale":"2024-09-21T09:28:33.359395+0000","last_undegraded":"2024-09-21T09:28:33.359395+0000","last_fullsized":"2024-09-21T09:28:33.359395+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:22:43.942811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2c","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.484182+0000","last_change":"2024-09-21T09:27:35.650351+0000","last_active":"2024-09-21T09:28:33.484182+0000","last_peered":"2024-09-21T09:28:33.484182+0000","last_clean":"2024-09-21T09:28:33.484182+0000","last_became_active":"2024-09-21T09:27:35.649905+0000","last_became_peered":"2024-09-21T09:27:35.649905+0000","last_unstale":"2024-09-21T09:28:33.484182+0000","last_undegraded":"2024-09-21T09:28:33.484182+0000","last_fullsized":"2024-09-21T09:28:33.484182+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:12:31.390666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2b","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.141020+0000","last_change":"2024-09-21T09:27:35.643345+0000","last_active":"2024-09-21T09:28:33.141020+0000","last_peered":"2024-09-21T09:28:33.141020+0000","last_clean":"2024-09-21T09:28:33.141020+0000","last_became_active":"2024-09-21T09:27:35.642408+0000","last_became_peered":"2024-09-21T09:27:35.642408+0000","last_unstale":"2024-09-21T09:28:33.141020+0000","last_undegraded":"2024-09-21T09:28:33.141020+0000","last_fullsized":"2024-09-21T09:28:33.141020+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:34:59.698334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2a","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.066015+0000","last_change":"2024-09-21T09:27:35.645032+0000","last_active":"2024-09-21T09:28:33.066015+0000","last_peered":"2024-09-21T09:28:33.066015+0000","last_clean":"2024-09-21T09:28:33.066015+0000","last_became_active":"2024-09-21T09:27:35.644322+0000","last_became_peered":"2024-09-21T09:27:35.644322+0000","last_unstale":"2024-09-21T09:28:33.066015+0000","last_undegraded":"2024-09-21T09:28:33.066015+0000","last_fullsized":"2024-09-21T09:28:33.066015+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:10:14.511772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.29","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.125084+0000","last_change":"2024-09-21T09:27:35.638305+0000","last_active":"2024-09-21T09:28:33.125084+0000","last_peered":"2024-09-21T09:28:33.125084+0000","last_clean":"2024-09-21T09:28:33.125084+0000","last_became_active":"2024-09-21T09:27:35.638036+0000","last_became_peered":"2024-09-21T09:27:35.638036+0000","last_unstale":"2024-09-21T09:28:33.125084+0000","last_undegraded":"2024-09-21T09:28:33.125084+0000","last_fullsized":"2024-09-21T09:28:33.125084+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:38:54.627840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.28","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.066181+0000","last_change":"2024-09-21T09:27:35.637277+0000","last_active":"2024-09-21T09:28:33.066181+0000","last_peered":"2024-09-21T09:28:33.066181+0000","last_clean":"2024-09-21T09:28:33.066181+0000","last_became_active":"2024-09-21T09:27:35.636541+0000","last_became_peered":"2024-09-21T09:27:35.636541+0000","last_unstale":"2024-09-21T09:28:33.066181+0000","last_undegraded":"2024-09-21T09:28:33.066181+0000","last_fullsized":"2024-09-21T09:28:33.066181+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:45:47.090531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.27","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.125608+0000","last_change":"2024-09-21T09:27:35.638946+0000","last_active":"2024-09-21T09:28:33.125608+0000","last_peered":"2024-09-21T09:28:33.125608+0000","last_clean":"2024-09-21T09:28:33.125608+0000","last_became_active":"2024-09-21T09:27:35.638663+0000","last_became_peered":"2024-09-21T09:27:35.638663+0000","last_unstale":"2024-09-21T09:28:33.125608+0000","last_undegraded":"2024-09-21T09:28:33.125608+0000","last_fullsized":"2024-09-21T09:28:33.125608+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:49:57.591531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.26","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.484321+0000","last_change":"2024-09-21T09:27:35.638679+0000","last_active":"2024-09-21T09:28:33.484321+0000","last_peered":"2024-09-21T09:28:33.484321+0000","last_clean":"2024-09-21T09:28:33.484321+0000","last_became_active":"2024-09-21T09:27:35.638383+0000","last_became_peered":"2024-09-21T09:27:35.638383+0000","last_unstale":"2024-09-21T09:28:33.484321+0000","last_undegraded":"2024-09-21T09:28:33.484321+0000","last_fullsized":"2024-09-21T09:28:33.484321+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:16:11.663503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,4],"acting":[6,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.25","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.061992+0000","last_change":"2024-09-21T09:27:35.632894+0000","last_active":"2024-09-21T09:28:33.061992+0000","last_peered":"2024-09-21T09:28:33.061992+0000","last_clean":"2024-09-21T09:28:33.061992+0000","last_became_active":"2024-09-21T09:27:35.632569+0000","last_became_peered":"2024-09-21T09:27:35.632569+0000","last_unstale":"2024-09-21T09:28:33.061992+0000","last_undegraded":"2024-09-21T09:28:33.061992+0000","last_fullsized":"2024-09-21T09:28:33.061992+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:25:22.833535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.24","version":"0'0","reported_seq":31,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.125399+0000","last_change":"2024-09-21T09:27:52.079786+0000","last_active":"2024-09-21T09:28:33.125399+0000","last_peered":"2024-09-21T09:28:33.125399+0000","last_clean":"2024-09-21T09:28:33.125399+0000","last_became_active":"2024-09-21T09:27:52.079395+0000","last_became_peered":"2024-09-21T09:27:52.079395+0000","last_unstale":"2024-09-21T09:28:33.125399+0000","last_undegraded":"2024-09-21T09:28:33.125399+0000","last_fullsized":"2024-09-21T09:28:33.125399+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:17:07.388900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.23","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.141121+0000","last_change":"2024-09-21T09:27:35.634785+0000","last_active":"2024-09-21T09:28:33.141121+0000","last_peered":"2024-09-21T09:28:33.141121+0000","last_clean":"2024-09-21T09:28:33.141121+0000","last_became_active":"2024-09-21T09:27:35.634065+0000","last_became_peered":"2024-09-21T09:27:35.634065+0000","last_unstale":"2024-09-21T09:28:33.141121+0000","last_undegraded":"2024-09-21T09:28:33.141121+0000","last_fullsized":"2024-09-21T09:28:33.141121+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:54:21.482132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.22","version":"0'0","reported_seq":31,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.066111+0000","last_change":"2024-09-21T09:27:52.049243+0000","last_active":"2024-09-21T09:28:33.066111+0000","last_peered":"2024-09-21T09:28:33.066111+0000","last_clean":"2024-09-21T09:28:33.066111+0000","last_became_active":"2024-09-21T09:27:52.048885+0000","last_became_peered":"2024-09-21T09:27:52.048885+0000","last_unstale":"2024-09-21T09:28:33.066111+0000","last_undegraded":"2024-09-21T09:28:33.066111+0000","last_fullsized":"2024-09-21T09:28:33.066111+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:44:22.262414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.21","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.061749+0000","last_change":"2024-09-21T09:27:35.636222+0000","last_active":"2024-09-21T09:28:33.061749+0000","last_peered":"2024-09-21T09:28:33.061749+0000","last_clean":"2024-09-21T09:28:33.061749+0000","last_became_active":"2024-09-21T09:27:35.635935+0000","last_became_peered":"2024-09-21T09:27:35.635935+0000","last_unstale":"2024-09-21T09:28:33.061749+0000","last_undegraded":"2024-09-21T09:28:33.061749+0000","last_fullsized":"2024-09-21T09:28:33.061749+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:05:50.402224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.20","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.483839+0000","last_change":"2024-09-21T09:27:35.641740+0000","last_active":"2024-09-21T09:28:33.483839+0000","last_peered":"2024-09-21T09:28:33.483839+0000","last_clean":"2024-09-21T09:28:33.483839+0000","last_became_active":"2024-09-21T09:27:35.641409+0000","last_became_peered":"2024-09-21T09:27:35.641409+0000","last_unstale":"2024-09-21T09:28:33.483839+0000","last_undegraded":"2024-09-21T09:28:33.483839+0000","last_fullsized":"2024-09-21T09:28:33.483839+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:59:30.487730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1a","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.062273+0000","last_change":"2024-09-21T09:27:35.629525+0000","last_active":"2024-09-21T09:28:33.062273+0000","last_peered":"2024-09-21T09:28:33.062273+0000","last_clean":"2024-09-21T09:28:33.062273+0000","last_became_active":"2024-09-21T09:27:35.629167+0000","last_became_peered":"2024-09-21T09:27:35.629167+0000","last_unstale":"2024-09-21T09:28:33.062273+0000","last_undegraded":"2024-09-21T09:28:33.062273+0000","last_fullsized":"2024-09-21T09:28:33.062273+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:06:24.323825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1b","version":"57'1","reported_seq":25,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.125391+0000","last_change":"2024-09-21T09:27:32.265235+0000","last_active":"2024-09-21T09:28:33.125391+0000","last_peered":"2024-09-21T09:28:33.125391+0000","last_clean":"2024-09-21T09:28:33.125391+0000","last_became_active":"2024-09-21T09:27:32.264963+0000","last_became_peered":"2024-09-21T09:27:32.264963+0000","last_unstale":"2024-09-21T09:28:33.125391+0000","last_undegraded":"2024-09-21T09:28:33.125391+0000","last_fullsized":"2024-09-21T09:28:33.125391+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:09:03.927341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.141127+0000","last_change":"2024-09-21T09:27:35.642154+0000","last_active":"2024-09-21T09:28:33.141127+0000","last_peered":"2024-09-21T09:28:33.141127+0000","last_clean":"2024-09-21T09:28:33.141127+0000","last_became_active":"2024-09-21T09:27:35.641801+0000","last_became_peered":"2024-09-21T09:27:35.641801+0000","last_unstale":"2024-09-21T09:28:33.141127+0000","last_undegraded":"2024-09-21T09:28:33.141127+0000","last_fullsized":"2024-09-21T09:28:33.141127+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:50:45.075506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1a","version":"57'1","reported_seq":25,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.484044+0000","last_change":"2024-09-21T09:27:32.269358+0000","last_active":"2024-09-21T09:28:33.484044+0000","last_peered":"2024-09-21T09:28:33.484044+0000","last_clean":"2024-09-21T09:28:33.484044+0000","last_became_active":"2024-09-21T09:27:32.269026+0000","last_became_peered":"2024-09-21T09:27:32.269026+0000","last_unstale":"2024-09-21T09:28:33.484044+0000","last_undegraded":"2024-09-21T09:28:33.484044+0000","last_fullsized":"2024-09-21T09:28:33.484044+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:59:17.916370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.19","version":"0'0","reported_seq":24,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.125195+0000","last_change":"2024-09-21T09:27:32.264601+0000","last_active":"2024-09-21T09:28:33.125195+0000","last_peered":"2024-09-21T09:28:33.125195+0000","last_clean":"2024-09-21T09:28:33.125195+0000","last_became_active":"2024-09-21T09:27:32.264381+0000","last_became_peered":"2024-09-21T09:27:32.264381+0000","last_unstale":"2024-09-21T09:28:33.125195+0000","last_undegraded":"2024-09-21T09:28:33.125195+0000","last_fullsized":"2024-09-21T09:28:33.125195+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:35:57.463156+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.125067+0000","last_change":"2024-09-21T09:27:35.634475+0000","last_active":"2024-09-21T09:28:33.125067+0000","last_peered":"2024-09-21T09:28:33.125067+0000","last_clean":"2024-09-21T09:28:33.125067+0000","last_became_active":"2024-09-21T09:27:35.634224+0000","last_became_peered":"2024-09-21T09:27:35.634224+0000","last_unstale":"2024-09-21T09:28:33.125067+0000","last_undegraded":"2024-09-21T09:28:33.125067+0000","last_fullsized":"2024-09-21T09:28:33.125067+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:38:21.241590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.18","version":"0'0","reported_seq":24,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.066263+0000","last_change":"2024-09-21T09:27:32.267805+0000","last_active":"2024-09-21T09:28:33.066263+0000","last_peered":"2024-09-21T09:28:33.066263+0000","last_clean":"2024-09-21T09:28:33.066263+0000","last_became_active":"2024-09-21T09:27:32.267484+0000","last_became_peered":"2024-09-21T09:27:32.267484+0000","last_unstale":"2024-09-21T09:28:33.066263+0000","last_undegraded":"2024-09-21T09:28:33.066263+0000","last_fullsized":"2024-09-21T09:28:33.066263+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:00:43.105362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,7],"acting":[5,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.062486+0000","last_change":"2024-09-21T09:27:35.631350+0000","last_active":"2024-09-21T09:28:33.062486+0000","last_peered":"2024-09-21T09:28:33.062486+0000","last_clean":"2024-09-21T09:28:33.062486+0000","last_became_active":"2024-09-21T09:27:35.630723+0000","last_became_peered":"2024-09-21T09:27:35.630723+0000","last_unstale":"2024-09-21T09:28:33.062486+0000","last_undegraded":"2024-09-21T09:28:33.062486+0000","last_fullsized":"2024-09-21T09:28:33.062486+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:46:32.135037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.17","version":"0'0","reported_seq":24,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.484390+0000","last_change":"2024-09-21T09:27:32.266307+0000","last_active":"2024-09-21T09:28:33.484390+0000","last_peered":"2024-09-21T09:28:33.484390+0000","last_clean":"2024-09-21T09:28:33.484390+0000","last_became_active":"2024-09-21T09:27:32.265995+0000","last_became_peered":"2024-09-21T09:27:32.265995+0000","last_unstale":"2024-09-21T09:28:33.484390+0000","last_undegraded":"2024-09-21T09:28:33.484390+0000","last_fullsized":"2024-09-21T09:28:33.484390+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:24:28.036534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.066174+0000","last_change":"2024-09-21T09:27:35.647952+0000","last_active":"2024-09-21T09:28:33.066174+0000","last_peered":"2024-09-21T09:28:33.066174+0000","last_clean":"2024-09-21T09:28:33.066174+0000","last_became_active":"2024-09-21T09:27:35.647649+0000","last_became_peered":"2024-09-21T09:27:35.647649+0000","last_unstale":"2024-09-21T09:28:33.066174+0000","last_undegraded":"2024-09-21T09:28:33.066174+0000","last_fullsized":"2024-09-21T09:28:33.066174+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:12:29.850835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.16","version":"0'0","reported_seq":24,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.066325+0000","last_change":"2024-09-21T09:27:32.269137+0000","last_active":"2024-09-21T09:28:33.066325+0000","last_peered":"2024-09-21T09:28:33.066325+0000","last_clean":"2024-09-21T09:28:33.066325+0000","last_became_active":"2024-09-21T09:27:32.268832+0000","last_became_peered":"2024-09-21T09:27:32.268832+0000","last_unstale":"2024-09-21T09:28:33.066325+0000","last_undegraded":"2024-09-21T09:28:33.066325+0000","last_fullsized":"2024-09-21T09:28:33.066325+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:51:58.668390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":18,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.062462+0000","last_change":"2024-09-21T09:27:52.050121+0000","last_active":"2024-09-21T09:28:33.062462+0000","last_peered":"2024-09-21T09:28:33.062462+0000","last_clean":"2024-09-21T09:28:33.062462+0000","last_became_active":"2024-09-21T09:27:52.049549+0000","last_became_peered":"2024-09-21T09:27:52.049549+0000","last_unstale":"2024-09-21T09:28:33.062462+0000","last_undegraded":"2024-09-21T09:28:33.062462+0000","last_fullsized":"2024-09-21T09:28:33.062462+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:25:27.303500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.15","version":"57'3","reported_seq":27,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.062137+0000","last_change":"2024-09-21T09:27:32.270145+0000","last_active":"2024-09-21T09:28:33.062137+0000","last_peered":"2024-09-21T09:28:33.062137+0000","last_clean":"2024-09-21T09:28:33.062137+0000","last_became_active":"2024-09-21T09:27:32.269808+0000","last_became_peered":"2024-09-21T09:27:32.269808+0000","last_unstale":"2024-09-21T09:28:33.062137+0000","last_undegraded":"2024-09-21T09:28:33.062137+0000","last_fullsized":"2024-09-21T09:28:33.062137+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:35:53.397343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.062595+0000","last_change":"2024-09-21T09:27:35.640314+0000","last_active":"2024-09-21T09:28:33.062595+0000","last_peered":"2024-09-21T09:28:33.062595+0000","last_clean":"2024-09-21T09:28:33.062595+0000","last_became_active":"2024-09-21T09:27:35.639976+0000","last_became_peered":"2024-09-21T09:27:35.639976+0000","last_unstale":"2024-09-21T09:28:33.062595+0000","last_undegraded":"2024-09-21T09:28:33.062595+0000","last_fullsized":"2024-09-21T09:28:33.062595+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:55:21.229874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.14","version":"60'9","reported_seq":33,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.484214+0000","last_change":"2024-09-21T09:27:32.266499+0000","last_active":"2024-09-21T09:28:33.484214+0000","last_peered":"2024-09-21T09:28:33.484214+0000","last_clean":"2024-09-21T09:28:33.484214+0000","last_became_active":"2024-09-21T09:27:32.265712+0000","last_became_peered":"2024-09-21T09:27:32.265712+0000","last_unstale":"2024-09-21T09:28:33.484214+0000","last_undegraded":"2024-09-21T09:28:33.484214+0000","last_fullsized":"2024-09-21T09:28:33.484214+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:19:17.014990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2342,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.359220+0000","last_change":"2024-09-21T09:27:35.641626+0000","last_active":"2024-09-21T09:28:33.359220+0000","last_peered":"2024-09-21T09:28:33.359220+0000","last_clean":"2024-09-21T09:28:33.359220+0000","last_became_active":"2024-09-21T09:27:35.641272+0000","last_became_peered":"2024-09-21T09:27:35.641272+0000","last_unstale":"2024-09-21T09:28:33.359220+0000","last_undegraded":"2024-09-21T09:28:33.359220+0000","last_fullsized":"2024-09-21T09:28:33.359220+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:15:05.108072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.13","version":"57'2","reported_seq":26,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.141220+0000","last_change":"2024-09-21T09:27:32.267412+0000","last_active":"2024-09-21T09:28:33.141220+0000","last_peered":"2024-09-21T09:28:33.141220+0000","last_clean":"2024-09-21T09:28:33.141220+0000","last_became_active":"2024-09-21T09:27:32.267108+0000","last_became_peered":"2024-09-21T09:27:32.267108+0000","last_unstale":"2024-09-21T09:28:33.141220+0000","last_undegraded":"2024-09-21T09:28:33.141220+0000","last_fullsized":"2024-09-21T09:28:33.141220+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:54:11.228702+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.140829+0000","last_change":"2024-09-21T09:27:35.641572+0000","last_active":"2024-09-21T09:28:33.140829+0000","last_peered":"2024-09-21T09:28:33.140829+0000","last_clean":"2024-09-21T09:28:33.140829+0000","last_became_active":"2024-09-21T09:27:35.641185+0000","last_became_peered":"2024-09-21T09:27:35.641185+0000","last_unstale":"2024-09-21T09:28:33.140829+0000","last_undegraded":"2024-09-21T09:28:33.140829+0000","last_fullsized":"2024-09-21T09:28:33.140829+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:52:30.993491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.12","version":"0'0","reported_seq":24,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.066197+0000","last_change":"2024-09-21T09:27:32.267644+0000","last_active":"2024-09-21T09:28:33.066197+0000","last_peered":"2024-09-21T09:28:33.066197+0000","last_clean":"2024-09-21T09:28:33.066197+0000","last_became_active":"2024-09-21T09:27:32.267232+0000","last_became_peered":"2024-09-21T09:27:32.267232+0000","last_unstale":"2024-09-21T09:28:33.066197+0000","last_undegraded":"2024-09-21T09:28:33.066197+0000","last_fullsized":"2024-09-21T09:28:33.066197+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:14:19.578564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.358995+0000","last_change":"2024-09-21T09:27:35.644217+0000","last_active":"2024-09-21T09:28:33.358995+0000","last_peered":"2024-09-21T09:28:33.358995+0000","last_clean":"2024-09-21T09:28:33.358995+0000","last_became_active":"2024-09-21T09:27:35.643681+0000","last_became_peered":"2024-09-21T09:27:35.643681+0000","last_unstale":"2024-09-21T09:28:33.358995+0000","last_undegraded":"2024-09-21T09:28:33.358995+0000","last_fullsized":"2024-09-21T09:28:33.358995+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:07:58.418951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.11","version":"0'0","reported_seq":24,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.483959+0000","last_change":"2024-09-21T09:27:32.270501+0000","last_active":"2024-09-21T09:28:33.483959+0000","last_peered":"2024-09-21T09:28:33.483959+0000","last_clean":"2024-09-21T09:28:33.483959+0000","last_became_active":"2024-09-21T09:27:32.269808+0000","last_became_peered":"2024-09-21T09:27:32.269808+0000","last_unstale":"2024-09-21T09:28:33.483959+0000","last_undegraded":"2024-09-21T09:28:33.483959+0000","last_fullsized":"2024-09-21T09:28:33.483959+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:28:03.705631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.10","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.483855+0000","last_change":"2024-09-21T09:27:35.641508+0000","last_active":"2024-09-21T09:28:33.483855+0000","last_peered":"2024-09-21T09:28:33.483855+0000","last_clean":"2024-09-21T09:28:33.483855+0000","last_became_active":"2024-09-21T09:27:35.640996+0000","last_became_peered":"2024-09-21T09:27:35.640996+0000","last_unstale":"2024-09-21T09:28:33.483855+0000","last_undegraded":"2024-09-21T09:28:33.483855+0000","last_fullsized":"2024-09-21T09:28:33.483855+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:02:24.674295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.10","version":"57'8","reported_seq":32,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.061786+0000","last_change":"2024-09-21T09:27:32.264620+0000","last_active":"2024-09-21T09:28:33.061786+0000","last_peered":"2024-09-21T09:28:33.061786+0000","last_clean":"2024-09-21T09:28:33.061786+0000","last_became_active":"2024-09-21T09:27:32.261324+0000","last_became_peered":"2024-09-21T09:27:32.261324+0000","last_unstale":"2024-09-21T09:28:33.061786+0000","last_undegraded":"2024-09-21T09:28:33.061786+0000","last_fullsized":"2024-09-21T09:28:33.061786+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:11:04.933873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.358860+0000","last_change":"2024-09-21T09:27:35.641576+0000","last_active":"2024-09-21T09:28:33.358860+0000","last_peered":"2024-09-21T09:28:33.358860+0000","last_clean":"2024-09-21T09:28:33.358860+0000","last_became_active":"2024-09-21T09:27:35.641261+0000","last_became_peered":"2024-09-21T09:27:35.641261+0000","last_unstale":"2024-09-21T09:28:33.358860+0000","last_undegraded":"2024-09-21T09:28:33.358860+0000","last_fullsized":"2024-09-21T09:28:33.358860+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:59:48.868788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.f","version":"57'1","reported_seq":25,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.062869+0000","last_change":"2024-09-21T09:27:32.270055+0000","last_active":"2024-09-21T09:28:33.062869+0000","last_peered":"2024-09-21T09:28:33.062869+0000","last_clean":"2024-09-21T09:28:33.062869+0000","last_became_active":"2024-09-21T09:27:32.269748+0000","last_became_peered":"2024-09-21T09:27:32.269748+0000","last_unstale":"2024-09-21T09:28:33.062869+0000","last_undegraded":"2024-09-21T09:28:33.062869+0000","last_fullsized":"2024-09-21T09:28:33.062869+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:50:54.853319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.359401+0000","last_change":"2024-09-21T09:27:35.641790+0000","last_active":"2024-09-21T09:28:33.359401+0000","last_peered":"2024-09-21T09:28:33.359401+0000","last_clean":"2024-09-21T09:28:33.359401+0000","last_became_active":"2024-09-21T09:27:35.641348+0000","last_became_peered":"2024-09-21T09:27:35.641348+0000","last_unstale":"2024-09-21T09:28:33.359401+0000","last_undegraded":"2024-09-21T09:28:33.359401+0000","last_fullsized":"2024-09-21T09:28:33.359401+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:25:10.635353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.e","version":"57'2","reported_seq":26,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.062124+0000","last_change":"2024-09-21T09:27:32.264276+0000","last_active":"2024-09-21T09:28:33.062124+0000","last_peered":"2024-09-21T09:28:33.062124+0000","last_clean":"2024-09-21T09:28:33.062124+0000","last_became_active":"2024-09-21T09:27:32.263984+0000","last_became_peered":"2024-09-21T09:27:32.263984+0000","last_unstale":"2024-09-21T09:28:33.062124+0000","last_undegraded":"2024-09-21T09:28:33.062124+0000","last_fullsized":"2024-09-21T09:28:33.062124+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:18:50.045457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.359338+0000","last_change":"2024-09-21T09:27:35.649353+0000","last_active":"2024-09-21T09:28:33.359338+0000","last_peered":"2024-09-21T09:28:33.359338+0000","last_clean":"2024-09-21T09:28:33.359338+0000","last_became_active":"2024-09-21T09:27:35.649033+0000","last_became_peered":"2024-09-21T09:27:35.649033+0000","last_unstale":"2024-09-21T09:28:33.359338+0000","last_undegraded":"2024-09-21T09:28:33.359338+0000","last_fullsized":"2024-09-21T09:28:33.359338+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:13:12.611019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.d","version":"57'1","reported_seq":25,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.062217+0000","last_change":"2024-09-21T09:27:32.260033+0000","last_active":"2024-09-21T09:28:33.062217+0000","last_peered":"2024-09-21T09:28:33.062217+0000","last_clean":"2024-09-21T09:28:33.062217+0000","last_became_active":"2024-09-21T09:27:32.259209+0000","last_became_peered":"2024-09-21T09:27:32.259209+0000","last_unstale":"2024-09-21T09:28:33.062217+0000","last_undegraded":"2024-09-21T09:28:33.062217+0000","last_fullsized":"2024-09-21T09:28:33.062217+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:16:09.835879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.066397+0000","last_change":"2024-09-21T09:27:35.639076+0000","last_active":"2024-09-21T09:28:33.066397+0000","last_peered":"2024-09-21T09:28:33.066397+0000","last_clean":"2024-09-21T09:28:33.066397+0000","last_became_active":"2024-09-21T09:27:35.638802+0000","last_became_peered":"2024-09-21T09:27:35.638802+0000","last_unstale":"2024-09-21T09:28:33.066397+0000","last_undegraded":"2024-09-21T09:28:33.066397+0000","last_fullsized":"2024-09-21T09:28:33.066397+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:36:18.524320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.c","version":"0'0","reported_seq":24,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.062156+0000","last_change":"2024-09-21T09:27:32.270060+0000","last_active":"2024-09-21T09:28:33.062156+0000","last_peered":"2024-09-21T09:28:33.062156+0000","last_clean":"2024-09-21T09:28:33.062156+0000","last_became_active":"2024-09-21T09:27:32.269717+0000","last_became_peered":"2024-09-21T09:27:32.269717+0000","last_unstale":"2024-09-21T09:28:33.062156+0000","last_undegraded":"2024-09-21T09:28:33.062156+0000","last_fullsized":"2024-09-21T09:28:33.062156+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:59:27.725031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.359314+0000","last_change":"2024-09-21T09:27:35.640059+0000","last_active":"2024-09-21T09:28:33.359314+0000","last_peered":"2024-09-21T09:28:33.359314+0000","last_clean":"2024-09-21T09:28:33.359314+0000","last_became_active":"2024-09-21T09:27:35.639746+0000","last_became_peered":"2024-09-21T09:27:35.639746+0000","last_unstale":"2024-09-21T09:28:33.359314+0000","last_undegraded":"2024-09-21T09:28:33.359314+0000","last_fullsized":"2024-09-21T09:28:33.359314+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:33:15.402410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":19,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-21T09:28:41.371073+0000","last_change":"2024-09-21T09:28:33.484488+0000","last_active":"2024-09-21T09:28:41.371073+0000","last_peered":"2024-09-21T09:28:41.371073+0000","last_clean":"2024-09-21T09:28:41.371073+0000","last_became_active":"2024-09-21T09:28:28.844168+0000","last_became_peered":"2024-09-21T09:28:28.844168+0000","last_unstale":"2024-09-21T09:28:41.371073+0000","last_undegraded":"2024-09-21T09:28:41.371073+0000","last_fullsized":"2024-09-21T09:28:41.371073+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:28:27.830318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:28:27.830318+0000","last_clean_scrub_stamp":"2024-09-21T09:28:27.830318+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:07:12.698450+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030427299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"62'79","reported_seq":58,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.359151+0000","last_change":"2024-09-21T09:27:08.460023+0000","last_active":"2024-09-21T09:28:33.359151+0000","last_peered":"2024-09-21T09:28:33.359151+0000","last_clean":"2024-09-21T09:28:33.359151+0000","last_became_active":"2024-09-21T09:27:08.454770+0000","last_became_peered":"2024-09-21T09:27:08.454770+0000","last_unstale":"2024-09-21T09:28:33.359151+0000","last_undegraded":"2024-09-21T09:28:33.359151+0000","last_fullsized":"2024-09-21T09:28:33.359151+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:25:07.646659+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:25:07.646659+0000","last_clean_scrub_stamp":"2024-09-21T09:25:07.646659+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:08:45.658466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":231,"num_write":113,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":34,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.066105+0000","last_change":"2024-09-21T09:27:52.081088+0000","last_active":"2024-09-21T09:28:33.066105+0000","last_peered":"2024-09-21T09:28:33.066105+0000","last_clean":"2024-09-21T09:28:33.066105+0000","last_became_active":"2024-09-21T09:27:52.080680+0000","last_became_peered":"2024-09-21T09:27:52.080680+0000","last_unstale":"2024-09-21T09:28:33.066105+0000","last_undegraded":"2024-09-21T09:28:33.066105+0000","last_fullsized":"2024-09-21T09:28:33.066105+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:01:04.317396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.125141+0000","last_change":"2024-09-21T09:27:35.636200+0000","last_active":"2024-09-21T09:28:33.125141+0000","last_peered":"2024-09-21T09:28:33.125141+0000","last_clean":"2024-09-21T09:28:33.125141+0000","last_became_active":"2024-09-21T09:27:35.635939+0000","last_became_peered":"2024-09-21T09:27:35.635939+0000","last_unstale":"2024-09-21T09:28:33.125141+0000","last_undegraded":"2024-09-21T09:28:33.125141+0000","last_fullsized":"2024-09-21T09:28:33.125141+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:02:06.551561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":19,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-21T09:28:40.923121+0000","last_change":"2024-09-21T09:28:33.141556+0000","last_active":"2024-09-21T09:28:40.923121+0000","last_peered":"2024-09-21T09:28:40.923121+0000","last_clean":"2024-09-21T09:28:40.923121+0000","last_became_active":"2024-09-21T09:28:28.846894+0000","last_became_peered":"2024-09-21T09:28:28.846894+0000","last_unstale":"2024-09-21T09:28:40.923121+0000","last_undegraded":"2024-09-21T09:28:40.923121+0000","last_fullsized":"2024-09-21T09:28:40.923121+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:28:27.830318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:28:27.830318+0000","last_clean_scrub_stamp":"2024-09-21T09:28:27.830318+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:54:49.861215+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000276941,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":24,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.359086+0000","last_change":"2024-09-21T09:27:32.268500+0000","last_active":"2024-09-21T09:28:33.359086+0000","last_peered":"2024-09-21T09:28:33.359086+0000","last_clean":"2024-09-21T09:28:33.359086+0000","last_became_active":"2024-09-21T09:27:32.268026+0000","last_became_peered":"2024-09-21T09:27:32.268026+0000","last_unstale":"2024-09-21T09:28:33.359086+0000","last_undegraded":"2024-09-21T09:28:33.359086+0000","last_fullsized":"2024-09-21T09:28:33.359086+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:44:43.463552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.140711+0000","last_change":"2024-09-21T09:27:35.634025+0000","last_active":"2024-09-21T09:28:33.140711+0000","last_peered":"2024-09-21T09:28:33.140711+0000","last_clean":"2024-09-21T09:28:33.140711+0000","last_became_active":"2024-09-21T09:27:35.633746+0000","last_became_peered":"2024-09-21T09:27:35.633746+0000","last_unstale":"2024-09-21T09:28:33.140711+0000","last_undegraded":"2024-09-21T09:28:33.140711+0000","last_fullsized":"2024-09-21T09:28:33.140711+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:18:40.511292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":19,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-21T09:28:40.375561+0000","last_change":"2024-09-21T09:28:33.062917+0000","last_active":"2024-09-21T09:28:40.375561+0000","last_peered":"2024-09-21T09:28:40.375561+0000","last_clean":"2024-09-21T09:28:40.375561+0000","last_became_active":"2024-09-21T09:28:28.924106+0000","last_became_peered":"2024-09-21T09:28:28.924106+0000","last_unstale":"2024-09-21T09:28:40.375561+0000","last_undegraded":"2024-09-21T09:28:40.375561+0000","last_fullsized":"2024-09-21T09:28:40.375561+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:28:27.830318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:28:27.830318+0000","last_clean_scrub_stamp":"2024-09-21T09:28:27.830318+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:40:32.351197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00058044900000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":24,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.061847+0000","last_change":"2024-09-21T09:27:32.262645+0000","last_active":"2024-09-21T09:28:33.061847+0000","last_peered":"2024-09-21T09:28:33.061847+0000","last_clean":"2024-09-21T09:28:33.061847+0000","last_became_active":"2024-09-21T09:27:32.262373+0000","last_became_peered":"2024-09-21T09:27:32.262373+0000","last_unstale":"2024-09-21T09:28:33.061847+0000","last_undegraded":"2024-09-21T09:28:33.061847+0000","last_fullsized":"2024-09-21T09:28:33.061847+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:37:23.684561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.061597+0000","last_change":"2024-09-21T09:27:35.636105+0000","last_active":"2024-09-21T09:28:33.061597+0000","last_peered":"2024-09-21T09:28:33.061597+0000","last_clean":"2024-09-21T09:28:33.061597+0000","last_became_active":"2024-09-21T09:27:35.635805+0000","last_became_peered":"2024-09-21T09:27:35.635805+0000","last_unstale":"2024-09-21T09:28:33.061597+0000","last_undegraded":"2024-09-21T09:28:33.061597+0000","last_fullsized":"2024-09-21T09:28:33.061597+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:22:53.711130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":19,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-21T09:28:40.376066+0000","last_change":"2024-09-21T09:28:33.062726+0000","last_active":"2024-09-21T09:28:40.376066+0000","last_peered":"2024-09-21T09:28:40.376066+0000","last_clean":"2024-09-21T09:28:40.376066+0000","last_became_active":"2024-09-21T09:28:28.844698+0000","last_became_peered":"2024-09-21T09:28:28.844698+0000","last_unstale":"2024-09-21T09:28:40.376066+0000","last_undegraded":"2024-09-21T09:28:40.376066+0000","last_fullsized":"2024-09-21T09:28:40.376066+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:28:27.830318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:28:27.830318+0000","last_clean_scrub_stamp":"2024-09-21T09:28:27.830318+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:24:39.284345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00072506199999999995,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":24,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.066308+0000","last_change":"2024-09-21T09:27:32.265210+0000","last_active":"2024-09-21T09:28:33.066308+0000","last_peered":"2024-09-21T09:28:33.066308+0000","last_clean":"2024-09-21T09:28:33.066308+0000","last_became_active":"2024-09-21T09:27:32.264740+0000","last_became_peered":"2024-09-21T09:27:32.264740+0000","last_unstale":"2024-09-21T09:28:33.066308+0000","last_undegraded":"2024-09-21T09:28:33.066308+0000","last_fullsized":"2024-09-21T09:28:33.066308+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:14:01.491316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.062810+0000","last_change":"2024-09-21T09:27:35.638582+0000","last_active":"2024-09-21T09:28:33.062810+0000","last_peered":"2024-09-21T09:28:33.062810+0000","last_clean":"2024-09-21T09:28:33.062810+0000","last_became_active":"2024-09-21T09:27:35.638269+0000","last_became_peered":"2024-09-21T09:27:35.638269+0000","last_unstale":"2024-09-21T09:28:33.062810+0000","last_undegraded":"2024-09-21T09:28:33.062810+0000","last_fullsized":"2024-09-21T09:28:33.062810+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:52:20.849743+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.2","version":"66'2","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-21T09:28:40.375501+0000","last_change":"2024-09-21T09:28:33.063343+0000","last_active":"2024-09-21T09:28:40.375501+0000","last_peered":"2024-09-21T09:28:40.375501+0000","last_clean":"2024-09-21T09:28:40.375501+0000","last_became_active":"2024-09-21T09:28:28.924195+0000","last_became_peered":"2024-09-21T09:28:28.924195+0000","last_unstale":"2024-09-21T09:28:40.375501+0000","last_undegraded":"2024-09-21T09:28:40.375501+0000","last_fullsized":"2024-09-21T09:28:40.375501+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:28:27.830318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:28:27.830318+0000","last_clean_scrub_stamp":"2024-09-21T09:28:27.830318+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:33:35.119949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00080024800000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":24,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.062271+0000","last_change":"2024-09-21T09:27:32.264482+0000","last_active":"2024-09-21T09:28:33.062271+0000","last_peered":"2024-09-21T09:28:33.062271+0000","last_clean":"2024-09-21T09:28:33.062271+0000","last_became_active":"2024-09-21T09:27:32.264201+0000","last_became_peered":"2024-09-21T09:27:32.264201+0000","last_unstale":"2024-09-21T09:28:33.062271+0000","last_undegraded":"2024-09-21T09:28:33.062271+0000","last_fullsized":"2024-09-21T09:28:33.062271+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:37:15.565839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.066122+0000","last_change":"2024-09-21T09:27:35.634935+0000","last_active":"2024-09-21T09:28:33.066122+0000","last_peered":"2024-09-21T09:28:33.066122+0000","last_clean":"2024-09-21T09:28:33.066122+0000","last_became_active":"2024-09-21T09:27:35.634594+0000","last_became_peered":"2024-09-21T09:27:35.634594+0000","last_unstale":"2024-09-21T09:28:33.066122+0000","last_undegraded":"2024-09-21T09:28:33.066122+0000","last_fullsized":"2024-09-21T09:28:33.066122+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:53:03.606929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"64'1","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-21T09:28:40.922745+0000","last_change":"2024-09-21T09:28:33.141355+0000","last_active":"2024-09-21T09:28:40.922745+0000","last_peered":"2024-09-21T09:28:40.922745+0000","last_clean":"2024-09-21T09:28:40.922745+0000","last_became_active":"2024-09-21T09:28:28.923619+0000","last_became_peered":"2024-09-21T09:28:28.923619+0000","last_unstale":"2024-09-21T09:28:40.922745+0000","last_undegraded":"2024-09-21T09:28:40.922745+0000","last_fullsized":"2024-09-21T09:28:40.922745+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:28:27.830318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:28:27.830318+0000","last_clean_scrub_stamp":"2024-09-21T09:28:27.830318+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:40:50.990669+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00041745699999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":24,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.359440+0000","last_change":"2024-09-21T09:27:32.274427+0000","last_active":"2024-09-21T09:28:33.359440+0000","last_peered":"2024-09-21T09:28:33.359440+0000","last_clean":"2024-09-21T09:28:33.359440+0000","last_became_active":"2024-09-21T09:27:32.274058+0000","last_became_peered":"2024-09-21T09:27:32.274058+0000","last_unstale":"2024-09-21T09:28:33.359440+0000","last_undegraded":"2024-09-21T09:28:33.359440+0000","last_fullsized":"2024-09-21T09:28:33.359440+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:09:38.714482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.062188+0000","last_change":"2024-09-21T09:27:35.643081+0000","last_active":"2024-09-21T09:28:33.062188+0000","last_peered":"2024-09-21T09:28:33.062188+0000","last_clean":"2024-09-21T09:28:33.062188+0000","last_became_active":"2024-09-21T09:27:35.642792+0000","last_became_peered":"2024-09-21T09:27:35.642792+0000","last_unstale":"2024-09-21T09:28:33.062188+0000","last_undegraded":"2024-09-21T09:28:33.062188+0000","last_fullsized":"2024-09-21T09:28:33.062188+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:05:03.157102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":19,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-21T09:28:40.484507+0000","last_change":"2024-09-21T09:28:33.125263+0000","last_active":"2024-09-21T09:28:40.484507+0000","last_peered":"2024-09-21T09:28:40.484507+0000","last_clean":"2024-09-21T09:28:40.484507+0000","last_became_active":"2024-09-21T09:28:28.845163+0000","last_became_peered":"2024-09-21T09:28:28.845163+0000","last_unstale":"2024-09-21T09:28:40.484507+0000","last_undegraded":"2024-09-21T09:28:40.484507+0000","last_fullsized":"2024-09-21T09:28:40.484507+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:28:27.830318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:28:27.830318+0000","last_clean_scrub_stamp":"2024-09-21T09:28:27.830318+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:56:39.868147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034591199999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"57'1","reported_seq":25,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.062459+0000","last_change":"2024-09-21T09:27:32.265476+0000","last_active":"2024-09-21T09:28:33.062459+0000","last_peered":"2024-09-21T09:28:33.062459+0000","last_clean":"2024-09-21T09:28:33.062459+0000","last_became_active":"2024-09-21T09:27:32.265171+0000","last_became_peered":"2024-09-21T09:27:32.265171+0000","last_unstale":"2024-09-21T09:28:33.062459+0000","last_undegraded":"2024-09-21T09:28:33.062459+0000","last_fullsized":"2024-09-21T09:28:33.062459+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:42:48.279357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.125628+0000","last_change":"2024-09-21T09:27:35.629856+0000","last_active":"2024-09-21T09:28:33.125628+0000","last_peered":"2024-09-21T09:28:33.125628+0000","last_clean":"2024-09-21T09:28:33.125628+0000","last_became_active":"2024-09-21T09:27:35.629345+0000","last_became_peered":"2024-09-21T09:27:35.629345+0000","last_unstale":"2024-09-21T09:28:33.125628+0000","last_undegraded":"2024-09-21T09:28:33.125628+0000","last_fullsized":"2024-09-21T09:28:33.125628+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:30:14.361278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":19,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-21T09:28:40.749241+0000","last_change":"2024-09-21T09:28:33.062764+0000","last_active":"2024-09-21T09:28:40.749241+0000","last_peered":"2024-09-21T09:28:40.749241+0000","last_clean":"2024-09-21T09:28:40.749241+0000","last_became_active":"2024-09-21T09:28:28.924207+0000","last_became_peered":"2024-09-21T09:28:28.924207+0000","last_unstale":"2024-09-21T09:28:40.749241+0000","last_undegraded":"2024-09-21T09:28:40.749241+0000","last_fullsized":"2024-09-21T09:28:40.749241+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:28:27.830318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:28:27.830318+0000","last_clean_scrub_stamp":"2024-09-21T09:28:27.830318+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:08:48.406053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055597199999999995,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"57'2","reported_seq":26,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.484453+0000","last_change":"2024-09-21T09:27:32.264763+0000","last_active":"2024-09-21T09:28:33.484453+0000","last_peered":"2024-09-21T09:28:33.484453+0000","last_clean":"2024-09-21T09:28:33.484453+0000","last_became_active":"2024-09-21T09:27:32.264444+0000","last_became_peered":"2024-09-21T09:27:32.264444+0000","last_unstale":"2024-09-21T09:28:33.484453+0000","last_undegraded":"2024-09-21T09:28:33.484453+0000","last_fullsized":"2024-09-21T09:28:33.484453+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:01:52.871773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.141267+0000","last_change":"2024-09-21T09:27:35.637964+0000","last_active":"2024-09-21T09:28:33.141267+0000","last_peered":"2024-09-21T09:28:33.141267+0000","last_clean":"2024-09-21T09:28:33.141267+0000","last_became_active":"2024-09-21T09:27:35.634680+0000","last_became_peered":"2024-09-21T09:27:35.634680+0000","last_unstale":"2024-09-21T09:28:33.141267+0000","last_undegraded":"2024-09-21T09:28:33.141267+0000","last_fullsized":"2024-09-21T09:28:33.141267+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:50:36.642394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.8","version":"0'0","reported_seq":24,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.359011+0000","last_change":"2024-09-21T09:27:32.270810+0000","last_active":"2024-09-21T09:28:33.359011+0000","last_peered":"2024-09-21T09:28:33.359011+0000","last_clean":"2024-09-21T09:28:33.359011+0000","last_became_active":"2024-09-21T09:27:32.268720+0000","last_became_peered":"2024-09-21T09:27:32.268720+0000","last_unstale":"2024-09-21T09:28:33.359011+0000","last_undegraded":"2024-09-21T09:28:33.359011+0000","last_fullsized":"2024-09-21T09:28:33.359011+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:58:54.820070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.062395+0000","last_change":"2024-09-21T09:27:35.637942+0000","last_active":"2024-09-21T09:28:33.062395+0000","last_peered":"2024-09-21T09:28:33.062395+0000","last_clean":"2024-09-21T09:28:33.062395+0000","last_became_active":"2024-09-21T09:27:35.637064+0000","last_became_peered":"2024-09-21T09:27:35.637064+0000","last_unstale":"2024-09-21T09:28:33.062395+0000","last_undegraded":"2024-09-21T09:28:33.062395+0000","last_fullsized":"2024-09-21T09:28:33.062395+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:16:53.911245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.9","version":"0'0","reported_seq":24,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.062682+0000","last_change":"2024-09-21T09:27:32.262848+0000","last_active":"2024-09-21T09:28:33.062682+0000","last_peered":"2024-09-21T09:28:33.062682+0000","last_clean":"2024-09-21T09:28:33.062682+0000","last_became_active":"2024-09-21T09:27:32.262555+0000","last_became_peered":"2024-09-21T09:27:32.262555+0000","last_unstale":"2024-09-21T09:28:33.062682+0000","last_undegraded":"2024-09-21T09:28:33.062682+0000","last_fullsized":"2024-09-21T09:28:33.062682+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:27:02.670310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.125068+0000","last_change":"2024-09-21T09:27:35.625899+0000","last_active":"2024-09-21T09:28:33.125068+0000","last_peered":"2024-09-21T09:28:33.125068+0000","last_clean":"2024-09-21T09:28:33.125068+0000","last_became_active":"2024-09-21T09:27:35.625619+0000","last_became_peered":"2024-09-21T09:27:35.625619+0000","last_unstale":"2024-09-21T09:28:33.125068+0000","last_undegraded":"2024-09-21T09:28:33.125068+0000","last_fullsized":"2024-09-21T09:28:33.125068+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:38:48.577776+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.a","version":"0'0","reported_seq":24,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.063179+0000","last_change":"2024-09-21T09:27:32.264667+0000","last_active":"2024-09-21T09:28:33.063179+0000","last_peered":"2024-09-21T09:28:33.063179+0000","last_clean":"2024-09-21T09:28:33.063179+0000","last_became_active":"2024-09-21T09:27:32.264353+0000","last_became_peered":"2024-09-21T09:27:32.264353+0000","last_unstale":"2024-09-21T09:28:33.063179+0000","last_undegraded":"2024-09-21T09:28:33.063179+0000","last_fullsized":"2024-09-21T09:28:33.063179+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:21:53.938525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"0'0","reported_seq":31,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.125251+0000","last_change":"2024-09-21T09:27:52.081164+0000","last_active":"2024-09-21T09:28:33.125251+0000","last_peered":"2024-09-21T09:28:33.125251+0000","last_clean":"2024-09-21T09:28:33.125251+0000","last_became_active":"2024-09-21T09:27:52.080726+0000","last_became_peered":"2024-09-21T09:27:52.080726+0000","last_unstale":"2024-09-21T09:28:33.125251+0000","last_undegraded":"2024-09-21T09:28:33.125251+0000","last_fullsized":"2024-09-21T09:28:33.125251+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:37:01.620726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.b","version":"57'2","reported_seq":26,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.359154+0000","last_change":"2024-09-21T09:27:32.274404+0000","last_active":"2024-09-21T09:28:33.359154+0000","last_peered":"2024-09-21T09:28:33.359154+0000","last_clean":"2024-09-21T09:28:33.359154+0000","last_became_active":"2024-09-21T09:27:32.274109+0000","last_became_peered":"2024-09-21T09:27:32.274109+0000","last_unstale":"2024-09-21T09:28:33.359154+0000","last_undegraded":"2024-09-21T09:28:33.359154+0000","last_fullsized":"2024-09-21T09:28:33.359154+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:24:06.607203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":31,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.483953+0000","last_change":"2024-09-21T09:27:52.050465+0000","last_active":"2024-09-21T09:28:33.483953+0000","last_peered":"2024-09-21T09:28:33.483953+0000","last_clean":"2024-09-21T09:28:33.483953+0000","last_became_active":"2024-09-21T09:27:52.050052+0000","last_became_peered":"2024-09-21T09:27:52.050052+0000","last_unstale":"2024-09-21T09:28:33.483953+0000","last_undegraded":"2024-09-21T09:28:33.483953+0000","last_fullsized":"2024-09-21T09:28:33.483953+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:58:35.370217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1d","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.066042+0000","last_change":"2024-09-21T09:27:35.637946+0000","last_active":"2024-09-21T09:28:33.066042+0000","last_peered":"2024-09-21T09:28:33.066042+0000","last_clean":"2024-09-21T09:28:33.066042+0000","last_became_active":"2024-09-21T09:27:35.637664+0000","last_became_peered":"2024-09-21T09:27:35.637664+0000","last_unstale":"2024-09-21T09:28:33.066042+0000","last_undegraded":"2024-09-21T09:28:33.066042+0000","last_fullsized":"2024-09-21T09:28:33.066042+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:22:17.260192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1c","version":"57'2","reported_seq":26,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.359224+0000","last_change":"2024-09-21T09:27:32.268336+0000","last_active":"2024-09-21T09:28:33.359224+0000","last_peered":"2024-09-21T09:28:33.359224+0000","last_clean":"2024-09-21T09:28:33.359224+0000","last_became_active":"2024-09-21T09:27:32.268038+0000","last_became_peered":"2024-09-21T09:27:32.268038+0000","last_unstale":"2024-09-21T09:28:33.359224+0000","last_undegraded":"2024-09-21T09:28:33.359224+0000","last_fullsized":"2024-09-21T09:28:33.359224+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:57:13.991593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1c","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.066636+0000","last_change":"2024-09-21T09:27:35.642780+0000","last_active":"2024-09-21T09:28:33.066636+0000","last_peered":"2024-09-21T09:28:33.066636+0000","last_clean":"2024-09-21T09:28:33.066636+0000","last_became_active":"2024-09-21T09:27:35.642487+0000","last_became_peered":"2024-09-21T09:27:35.642487+0000","last_unstale":"2024-09-21T09:28:33.066636+0000","last_undegraded":"2024-09-21T09:28:33.066636+0000","last_fullsized":"2024-09-21T09:28:33.066636+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:28:54.864609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1d","version":"57'2","reported_seq":26,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.359353+0000","last_change":"2024-09-21T09:27:32.270398+0000","last_active":"2024-09-21T09:28:33.359353+0000","last_peered":"2024-09-21T09:28:33.359353+0000","last_clean":"2024-09-21T09:28:33.359353+0000","last_became_active":"2024-09-21T09:27:32.269469+0000","last_became_peered":"2024-09-21T09:27:32.269469+0000","last_unstale":"2024-09-21T09:28:33.359353+0000","last_undegraded":"2024-09-21T09:28:33.359353+0000","last_fullsized":"2024-09-21T09:28:33.359353+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:11:17.381607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1f","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.141385+0000","last_change":"2024-09-21T09:27:35.646662+0000","last_active":"2024-09-21T09:28:33.141385+0000","last_peered":"2024-09-21T09:28:33.141385+0000","last_clean":"2024-09-21T09:28:33.141385+0000","last_became_active":"2024-09-21T09:27:35.646141+0000","last_became_peered":"2024-09-21T09:27:35.646141+0000","last_unstale":"2024-09-21T09:28:33.141385+0000","last_undegraded":"2024-09-21T09:28:33.141385+0000","last_fullsized":"2024-09-21T09:28:33.141385+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:22:15.834922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1e","version":"57'1","reported_seq":25,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.125593+0000","last_change":"2024-09-21T09:27:32.270481+0000","last_active":"2024-09-21T09:28:33.125593+0000","last_peered":"2024-09-21T09:28:33.125593+0000","last_clean":"2024-09-21T09:28:33.125593+0000","last_became_active":"2024-09-21T09:27:32.269813+0000","last_became_peered":"2024-09-21T09:27:32.269813+0000","last_unstale":"2024-09-21T09:28:33.125593+0000","last_undegraded":"2024-09-21T09:28:33.125593+0000","last_fullsized":"2024-09-21T09:28:33.125593+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:28:41.266917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.125527+0000","last_change":"2024-09-21T09:27:35.636905+0000","last_active":"2024-09-21T09:28:33.125527+0000","last_peered":"2024-09-21T09:28:33.125527+0000","last_clean":"2024-09-21T09:28:33.125527+0000","last_became_active":"2024-09-21T09:27:35.636625+0000","last_became_peered":"2024-09-21T09:27:35.636625+0000","last_unstale":"2024-09-21T09:28:33.125527+0000","last_undegraded":"2024-09-21T09:28:33.125527+0000","last_fullsized":"2024-09-21T09:28:33.125527+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:19:39.022799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1f","version":"57'4","reported_seq":28,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.141529+0000","last_change":"2024-09-21T09:27:32.267938+0000","last_active":"2024-09-21T09:28:33.141529+0000","last_peered":"2024-09-21T09:28:33.141529+0000","last_clean":"2024-09-21T09:28:33.141529+0000","last_became_active":"2024-09-21T09:27:32.267583+0000","last_became_peered":"2024-09-21T09:27:32.267583+0000","last_unstale":"2024-09-21T09:28:33.141529+0000","last_undegraded":"2024-09-21T09:28:33.141529+0000","last_fullsized":"2024-09-21T09:28:33.141529+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:35:00.591469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":192,"acting":192,"num_store_stats":8},{"poolid":2,"num_pg":32,"stat_sum":{"num_bytes":3128,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":48,"num_write_kb":16,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":9384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":42,"ondisk_log_size":42,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":231,"num_write":113,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":79,"ondisk_log_size":79,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430239,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298616,"kb_used_data":1260,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446536,"statfs":{"total":95995035648,"available":95689252864,"internally_reserved":0,"allocated":1290240,"data_stored":900939,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:29:09 2024","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":0.54900000000000004,"15min":0.55700000000000005},"min":{"1min":0.32700000000000001,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.63100000000000001,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.69499999999999995},{"interface":"front","average":{"1min":0.52800000000000002,"5min":0.54100000000000004,"15min":0.54300000000000004},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.69799999999999995,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.42399999999999999}]},{"osd":1,"last update":"Sat Sep 21 09:29:09 2024","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":0.52800000000000002,"15min":0.53300000000000003},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.66900000000000004,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.45500000000000002},{"interface":"front","average":{"1min":0.51700000000000002,"5min":0.55000000000000004,"15min":0.55600000000000005},"min":{"1min":0.36699999999999999,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.67100000000000004,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.54900000000000004}]},{"osd":2,"last update":"Sat Sep 21 09:29:09 2024","interfaces":[{"interface":"back","average":{"1min":0.47899999999999998,"5min":0.51600000000000001,"15min":0.52200000000000002},"min":{"1min":0.32400000000000001,"5min":0.307,"15min":0.307},"max":{"1min":0.66200000000000003,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.54100000000000004},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.56499999999999995,"15min":0.57099999999999995},"min":{"1min":0.378,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.68600000000000005,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.64700000000000002}]},{"osd":3,"last update":"Sat Sep 21 09:29:09 2024","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.52800000000000002,"15min":0.53000000000000003},"min":{"1min":0.372,"5min":0.315,"15min":0.315},"max":{"1min":0.70099999999999996,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.48299999999999998},{"interface":"front","average":{"1min":0.49099999999999999,"5min":0.56799999999999995,"15min":0.58099999999999996},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.69599999999999995,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.58299999999999996}]},{"osd":4,"last update":"Sat Sep 21 09:29:09 2024","interfaces":[{"interface":"back","average":{"1min":0.45300000000000001,"5min":0.48499999999999999,"15min":0.48999999999999999},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.61799999999999999,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.50900000000000001},{"interface":"front","average":{"1min":0.433,"5min":0.49099999999999999,"15min":0.501},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.69899999999999995,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.60999999999999999}]},{"osd":5,"last update":"Sat Sep 21 09:29:09 2024","interfaces":[{"interface":"back","average":{"1min":0.46999999999999997,"5min":0.48799999999999999,"15min":0.49099999999999999},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":0.68100000000000005,"5min":0.71499999999999997,"15min":0.71499999999999997},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.51700000000000002,"5min":0.55500000000000005,"15min":0.56100000000000005},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.67100000000000004,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.67200000000000004}]},{"osd":6,"last update":"Sat Sep 21 09:29:09 2024","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.59599999999999997,"15min":0.60599999999999998},"min":{"1min":0.38200000000000001,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.71399999999999997,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.57599999999999996},{"interface":"front","average":{"1min":0.51300000000000001,"5min":0.57699999999999996,"15min":0.58799999999999997},"min":{"1min":0.371,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.72599999999999998,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.56899999999999995}]}]},{"osd":6,"up_from":42,"seq":180388626467,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298616,"kb_used_data":1260,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446536,"statfs":{"total":95995035648,"available":95689252864,"internally_reserved":0,"allocated":1290240,"data_stored":902374,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:28:45 2024","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.53700000000000003,"15min":0.53900000000000003},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.24199999999999999},{"interface":"front","average":{"1min":0.51000000000000001,"5min":0.53000000000000003,"15min":0.53300000000000003},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.752,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.29399999999999998}]},{"osd":1,"last update":"Sat Sep 21 09:28:45 2024","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.56399999999999995,"15min":0.57099999999999995},"min":{"1min":0.38500000000000001,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.70199999999999996,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.32100000000000001},{"interface":"front","average":{"1min":0.49399999999999999,"5min":0.55600000000000005,"15min":0.56599999999999995},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.71599999999999997,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.35099999999999998}]},{"osd":2,"last update":"Sat Sep 21 09:28:45 2024","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.54100000000000004,"15min":0.54200000000000004},"min":{"1min":0.377,"5min":0.37,"15min":0.37},"max":{"1min":0.68200000000000005,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.433},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.55000000000000004,"15min":0.55400000000000005},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.79600000000000004,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.47799999999999998}]},{"osd":3,"last update":"Sat Sep 21 09:28:45 2024","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.56899999999999995,"15min":0.57699999999999996},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.66100000000000003,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.40000000000000002},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.56499999999999995,"15min":0.57199999999999995},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.754,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.5}]},{"osd":4,"last update":"Sat Sep 21 09:28:45 2024","interfaces":[{"interface":"back","average":{"1min":0.42399999999999999,"5min":0.41199999999999998,"15min":0.40999999999999998},"min":{"1min":0.311,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.55800000000000005,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":0.375},{"interface":"front","average":{"1min":0.41199999999999998,"5min":0.432,"15min":0.436},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.625,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.45900000000000002}]},{"osd":5,"last update":"Sat Sep 21 09:28:45 2024","interfaces":[{"interface":"back","average":{"1min":0.41499999999999998,"5min":0.44700000000000001,"15min":0.45200000000000001},"min":{"1min":0.27400000000000002,"5min":0.255,"15min":0.255},"max":{"1min":0.55100000000000005,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.53700000000000003},{"interface":"front","average":{"1min":0.41699999999999998,"5min":0.47299999999999998,"15min":0.48199999999999998},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.60499999999999998,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.56899999999999995}]},{"osd":7,"last update":"Sat Sep 21 09:29:11 2024","interfaces":[{"interface":"back","average":{"1min":0.49399999999999999,"5min":0.52100000000000002,"15min":0.52500000000000002},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.66300000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.52000000000000002},{"interface":"front","average":{"1min":0.498,"5min":0.52800000000000002,"15min":0.53300000000000003},"min":{"1min":0.33100000000000002,"5min":0.309,"15min":0.309},"max":{"1min":0.76200000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.55300000000000005}]}]},{"osd":5,"up_from":36,"seq":154618822697,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298040,"kb_used_data":684,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447112,"statfs":{"total":95995035648,"available":95689842688,"internally_reserved":0,"allocated":700416,"data_stored":311800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:29:27 2024","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.52000000000000002,"15min":0.51700000000000002},"min":{"1min":0.371,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.69299999999999995,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.51000000000000001},{"interface":"front","average":{"1min":0.505,"5min":0.52800000000000002,"15min":0.52200000000000002},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.69899999999999995,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.64800000000000002}]},{"osd":1,"last update":"Sat Sep 21 09:29:27 2024","interfaces":[{"interface":"back","average":{"1min":0.52900000000000003,"5min":0.53100000000000003,"15min":0.52700000000000002},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.70699999999999996,"5min":0.749,"15min":0.749},"last":0.66800000000000004},{"interface":"front","average":{"1min":0.502,"5min":0.53900000000000003,"15min":0.53900000000000003},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.68300000000000005,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":0.54500000000000004}]},{"osd":2,"last update":"Sat Sep 21 09:29:27 2024","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.54000000000000004,"15min":0.54600000000000004},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.73999999999999999,"5min":0.75,"15min":0.75},"last":0.55900000000000005},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.52600000000000002,"15min":0.51200000000000001},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.73999999999999999,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.52800000000000002}]},{"osd":3,"last update":"Sat Sep 21 09:29:27 2024","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.52700000000000002,"15min":0.52100000000000002},"min":{"1min":0.439,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.60299999999999998,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.60299999999999998},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.55600000000000005,"15min":0.55200000000000005},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.755,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.57599999999999996}]},{"osd":4,"last update":"Sat Sep 21 09:29:27 2024","interfaces":[{"interface":"back","average":{"1min":0.46899999999999997,"5min":0.41399999999999998,"15min":0.38900000000000001},"min":{"1min":0.32100000000000001,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.69299999999999995,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.46300000000000002},{"interface":"front","average":{"1min":0.46899999999999997,"5min":0.40300000000000002,"15min":0.377},"min":{"1min":0.33100000000000002,"5min":0.247,"15min":0.247},"max":{"1min":0.69199999999999995,"5min":0.70099999999999996,"15min":0.70099999999999996},"last":0.41499999999999998}]},{"osd":6,"last update":"Sat Sep 21 09:28:45 2024","interfaces":[{"interface":"back","average":{"1min":0.46700000000000003,"5min":0.42699999999999999,"15min":0.42099999999999999},"min":{"1min":0.33800000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.69399999999999995,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.68100000000000005},{"interface":"front","average":{"1min":0.5,"5min":0.45600000000000002,"15min":0.44800000000000001},"min":{"1min":0.36099999999999999,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.71199999999999997,"5min":0.71199999999999997,"15min":0.71199999999999997},"last":0.442}]},{"osd":7,"last update":"Sat Sep 21 09:29:13 2024","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.51700000000000002,"15min":0.51800000000000002},"min":{"1min":0.373,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.76400000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.63300000000000001},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.54200000000000004,"15min":0.54400000000000004},"min":{"1min":0.40100000000000002,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.80800000000000005,"5min":0.877,"15min":0.877},"last":0.62}]}]},{"osd":4,"up_from":30,"seq":128849018925,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298048,"kb_used_data":692,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447104,"statfs":{"total":95995035648,"available":95689834496,"internally_reserved":0,"allocated":708608,"data_stored":310694,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:28:58 2024","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.54800000000000004,"15min":0.55200000000000005},"min":{"1min":0.33400000000000002,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.78400000000000003,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.48299999999999998},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.55900000000000005,"15min":0.56100000000000005},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.73599999999999999,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.61699999999999999}]},{"osd":1,"last update":"Sat Sep 21 09:28:58 2024","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.57599999999999996,"15min":0.57999999999999996},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.57099999999999995},{"interface":"front","average":{"1min":0.505,"5min":0.53900000000000003,"15min":0.53800000000000003},"min":{"1min":0.38200000000000001,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.68700000000000006,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.55600000000000005}]},{"osd":2,"last update":"Sat Sep 21 09:28:58 2024","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.55200000000000005,"15min":0.56100000000000005},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.76100000000000001,"5min":0.91200000000000003,"15min":0.91200000000000003},"last":0.498},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.55100000000000005,"15min":0.54100000000000004},"min":{"1min":0.39900000000000002,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.72899999999999998,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.52100000000000002}]},{"osd":3,"last update":"Sat Sep 21 09:28:58 2024","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.56499999999999995,"15min":0.56299999999999994},"min":{"1min":0.40300000000000002,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.70699999999999996,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.63500000000000001},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.56599999999999995,"15min":0.56399999999999995},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.90300000000000002,"5min":0.90300000000000002,"15min":0.90300000000000002},"last":0.54200000000000004}]},{"osd":5,"last update":"Sat Sep 21 09:28:26 2024","interfaces":[{"interface":"back","average":{"1min":0.45200000000000001,"5min":0.38500000000000001,"15min":0.374},"min":{"1min":0.32600000000000001,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.69099999999999995,"5min":0.69099999999999995,"15min":0.69099999999999995},"last":0.39300000000000002},{"interface":"front","average":{"1min":0.50800000000000001,"5min":0.42099999999999999,"15min":0.40699999999999997},"min":{"1min":0.32900000000000001,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.82699999999999996,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.45700000000000002}]},{"osd":6,"last update":"Sat Sep 21 09:28:46 2024","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.46800000000000003,"15min":0.45900000000000002},"min":{"1min":0.307,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":1.3440000000000001,"5min":1.3440000000000001,"15min":1.3440000000000001},"last":0.41399999999999998},{"interface":"front","average":{"1min":0.49099999999999999,"5min":0.47099999999999997,"15min":0.46800000000000003},"min":{"1min":0.33700000000000002,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.73099999999999998,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":0.433}]},{"osd":7,"last update":"Sat Sep 21 09:29:12 2024","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.55300000000000005,"15min":0.54900000000000004},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.84999999999999998,"5min":0.86399999999999999,"15min":0.86399999999999999},"last":0.58899999999999997},{"interface":"front","average":{"1min":0.61699999999999999,"5min":0.57699999999999996,"15min":0.57099999999999995},"min":{"1min":0.35699999999999998,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":1.3799999999999999,"5min":1.3799999999999999,"15min":1.3799999999999999},"last":0.60299999999999998}]}]},{"osd":3,"up_from":25,"seq":107374182450,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298048,"kb_used_data":692,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447104,"statfs":{"total":95995035648,"available":95689834496,"internally_reserved":0,"allocated":708608,"data_stored":312425,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:28:40 2024","interfaces":[{"interface":"back","average":{"1min":0.376,"5min":0.36899999999999999,"15min":0.36499999999999999},"min":{"1min":0.27700000000000002,"5min":0.25,"15min":0.25},"max":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"last":0.28199999999999997},{"interface":"front","average":{"1min":0.371,"5min":0.39100000000000001,"15min":0.39000000000000001},"min":{"1min":0.25,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.56399999999999995,"5min":0.60799999999999998,"15min":0.60799999999999998},"last":0.30099999999999999}]},{"osd":1,"last update":"Sat Sep 21 09:28:40 2024","interfaces":[{"interface":"back","average":{"1min":0.40200000000000002,"5min":0.42799999999999999,"15min":0.42999999999999999},"min":{"1min":0.32100000000000001,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.54400000000000004,"5min":0.63900000000000001,"15min":0.63900000000000001},"last":0.34799999999999998},{"interface":"front","average":{"1min":0.38800000000000001,"5min":0.40699999999999997,"15min":0.41299999999999998},"min":{"1min":0.22900000000000001,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":0.59699999999999998,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.249}]},{"osd":2,"last update":"Sat Sep 21 09:28:40 2024","interfaces":[{"interface":"back","average":{"1min":0.41699999999999998,"5min":0.438,"15min":0.443},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.52600000000000002,"5min":0.625,"15min":0.625},"last":0.32200000000000001},{"interface":"front","average":{"1min":0.40500000000000003,"5min":0.41799999999999998,"15min":0.41999999999999998},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":0.56399999999999995,"5min":0.64300000000000002,"15min":0.64300000000000002},"last":0.33500000000000002}]},{"osd":4,"last update":"Sat Sep 21 09:28:58 2024","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.55800000000000005,"15min":0.54600000000000004},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.71599999999999997,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.48999999999999999},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.55600000000000005,"15min":0.54400000000000004},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.78500000000000003,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.60299999999999998}]},{"osd":5,"last update":"Sat Sep 21 09:29:26 2024","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.57999999999999996,"15min":0.58999999999999997},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.78000000000000003,"5min":0.83299999999999996,"15min":0.83299999999999996},"last":0.48199999999999998},{"interface":"front","average":{"1min":0.57899999999999996,"5min":0.59699999999999998,"15min":0.60299999999999998},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.86399999999999999,"5min":0.86399999999999999,"15min":0.86399999999999999},"last":0.53100000000000003}]},{"osd":6,"last update":"Sat Sep 21 09:28:48 2024","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.60699999999999998,"15min":0.61099999999999999},"min":{"1min":0.378,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.77500000000000002,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.503},{"interface":"front","average":{"1min":0.58199999999999996,"5min":0.61199999999999999,"15min":0.61699999999999999},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.76300000000000001,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.56799999999999995}]},{"osd":7,"last update":"Sat Sep 21 09:29:10 2024","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.59999999999999998,"15min":0.60599999999999998},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.61199999999999999},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.59199999999999997,"15min":0.59799999999999998},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.75600000000000001,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.52100000000000002}]}]},{"osd":2,"up_from":18,"seq":77309411383,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298028,"kb_used_data":672,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447124,"statfs":{"total":95995035648,"available":95689854976,"internally_reserved":0,"allocated":688128,"data_stored":309979,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:29:16 2024","interfaces":[{"interface":"back","average":{"1min":0.35799999999999998,"5min":0.35999999999999999,"15min":0.36499999999999999},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"last":0.38},{"interface":"front","average":{"1min":0.36199999999999999,"5min":0.38700000000000001,"15min":0.39200000000000002},"min":{"1min":0.245,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.68600000000000005,"5min":0.68600000000000005,"15min":0.68600000000000005},"last":0.23899999999999999}]},{"osd":1,"last update":"Sat Sep 21 09:29:16 2024","interfaces":[{"interface":"back","average":{"1min":0.35199999999999998,"5min":0.371,"15min":0.36799999999999999},"min":{"1min":0.214,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":0.50900000000000001,"5min":0.65200000000000002,"15min":0.65200000000000002},"last":0.32900000000000001},{"interface":"front","average":{"1min":0.39200000000000002,"5min":0.41999999999999998,"15min":0.41399999999999998},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":0.56699999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"last":0.36799999999999999}]},{"osd":3,"last update":"Sat Sep 21 09:28:37 2024","interfaces":[{"interface":"back","average":{"1min":0.378,"5min":0.42199999999999999,"15min":0.442},"min":{"1min":0.27400000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.54100000000000004,"5min":0.61599999999999999,"15min":0.61599999999999999},"last":0.34899999999999998},{"interface":"front","average":{"1min":0.41299999999999998,"5min":0.45500000000000002,"15min":0.46700000000000003},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":0.51400000000000001,"5min":0.65200000000000002,"15min":0.65200000000000002},"last":0.28499999999999998}]},{"osd":4,"last update":"Sat Sep 21 09:28:59 2024","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.56999999999999995,"15min":0.56799999999999995},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.80800000000000005,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.54600000000000004},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.57999999999999996,"15min":0.58699999999999997},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.78400000000000003,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.47699999999999998}]},{"osd":5,"last update":"Sat Sep 21 09:29:26 2024","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.58399999999999996,"15min":0.59099999999999997},"min":{"1min":0.38200000000000001,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.79500000000000004,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.61399999999999999},{"interface":"front","average":{"1min":0.54500000000000004,"5min":0.58199999999999996,"15min":0.59199999999999997},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.84799999999999998,"5min":0.875,"15min":0.875},"last":0.52100000000000002}]},{"osd":6,"last update":"Sat Sep 21 09:28:44 2024","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":0.58199999999999996,"15min":0.58099999999999996},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.84199999999999997,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":0.59199999999999997},{"interface":"front","average":{"1min":0.60799999999999998,"5min":0.61599999999999999,"15min":0.61799999999999999},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.89800000000000002,"5min":0.89800000000000002,"15min":0.89800000000000002},"last":0.495}]},{"osd":7,"last update":"Sat Sep 21 09:29:10 2024","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.59399999999999997,"15min":0.59899999999999998},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.82699999999999996,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.59099999999999997,"15min":0.59199999999999997},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.79600000000000004,"5min":0.88700000000000001,"15min":0.88700000000000001},"last":0.53200000000000003}]}]},{"osd":1,"up_from":13,"seq":55834574908,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298040,"kb_used_data":684,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447112,"statfs":{"total":95995035648,"available":95689842688,"internally_reserved":0,"allocated":700416,"data_stored":310158,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:28:46 2024","interfaces":[{"interface":"back","average":{"1min":0.38100000000000001,"5min":0.37,"15min":0.34000000000000002},"min":{"1min":0.219,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":0.56200000000000006,"5min":0.61399999999999999,"15min":0.61399999999999999},"last":0.40899999999999997},{"interface":"front","average":{"1min":0.41699999999999998,"5min":0.40300000000000002,"15min":0.373},"min":{"1min":0.24399999999999999,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.55000000000000004,"5min":0.63,"15min":0.63},"last":0.42399999999999999}]},{"osd":2,"last update":"Sat Sep 21 09:29:18 2024","interfaces":[{"interface":"back","average":{"1min":0.38400000000000001,"5min":0.38300000000000001,"15min":0.36599999999999999},"min":{"1min":0.27900000000000003,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":0.57599999999999996,"5min":0.65200000000000002,"15min":0.65200000000000002},"last":0.443},{"interface":"front","average":{"1min":0.35199999999999998,"5min":0.378,"15min":0.372},"min":{"1min":0.26100000000000001,"5min":0.245,"15min":0.245},"max":{"1min":0.48499999999999999,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.375}]},{"osd":3,"last update":"Sat Sep 21 09:28:37 2024","interfaces":[{"interface":"back","average":{"1min":0.376,"5min":0.40999999999999998,"15min":0.41999999999999998},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.56299999999999994,"5min":0.58799999999999997,"15min":0.58799999999999997},"last":0.39300000000000002},{"interface":"front","average":{"1min":0.41299999999999998,"5min":0.45800000000000002,"15min":0.46700000000000003},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.60699999999999998,"5min":0.76800000000000002,"15min":0.76800000000000002},"last":0.46800000000000003}]},{"osd":4,"last update":"Sat Sep 21 09:29:00 2024","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.59199999999999997,"15min":0.60399999999999998},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.78100000000000003,"5min":0.97599999999999998,"15min":0.97599999999999998},"last":0.79500000000000004},{"interface":"front","average":{"1min":0.52100000000000002,"5min":0.56699999999999995,"15min":0.57999999999999996},"min":{"1min":0.374,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.73299999999999998,"5min":0.95699999999999996,"15min":0.95699999999999996},"last":0.63500000000000001}]},{"osd":5,"last update":"Sat Sep 21 09:29:24 2024","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.61699999999999999,"15min":0.63700000000000001},"min":{"1min":0.40799999999999997,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.71699999999999997,"5min":1.014,"15min":1.014},"last":0.70799999999999996},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.57299999999999995,"15min":0.57799999999999996},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.73199999999999998,"5min":0.93500000000000005,"15min":0.93500000000000005},"last":0.60899999999999999}]},{"osd":6,"last update":"Sat Sep 21 09:28:46 2024","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.56499999999999995,"15min":0.56699999999999995},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.76300000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.78600000000000003},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.59899999999999998,"15min":0.59999999999999998},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.83899999999999997,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":0.64400000000000002}]},{"osd":7,"last update":"Sat Sep 21 09:29:10 2024","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.58099999999999996,"15min":0.58299999999999996},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.76100000000000001,"5min":0.88800000000000001,"15min":0.88800000000000001},"last":0.67900000000000005},{"interface":"front","average":{"1min":0.58199999999999996,"5min":0.59599999999999997,"15min":0.59799999999999998},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":0.71899999999999997,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.71999999999999997}]}]},{"osd":0,"up_from":8,"seq":34359738432,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298620,"kb_used_data":1264,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446532,"statfs":{"total":95995035648,"available":95689248768,"internally_reserved":0,"allocated":1294336,"data_stored":900748,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 21 09:28:52 2024","interfaces":[{"interface":"back","average":{"1min":0.41299999999999998,"5min":0.34200000000000003,"15min":0.317},"min":{"1min":0.19700000000000001,"5min":0.185,"15min":0.185},"max":{"1min":1.3009999999999999,"5min":1.3009999999999999,"15min":1.3009999999999999},"last":0.251},{"interface":"front","average":{"1min":0.38,"5min":0.35199999999999998,"15min":0.33600000000000002},"min":{"1min":0.25900000000000001,"5min":0.185,"15min":0.185},"max":{"1min":0.61299999999999999,"5min":0.625,"15min":0.625},"last":0.23400000000000001}]},{"osd":2,"last update":"Sat Sep 21 09:29:14 2024","interfaces":[{"interface":"back","average":{"1min":0.40799999999999997,"5min":0.39000000000000001,"15min":0.375},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":1.2709999999999999,"5min":1.2709999999999999,"15min":1.2709999999999999},"last":0.31900000000000001},{"interface":"front","average":{"1min":0.45400000000000001,"5min":0.432,"15min":0.42299999999999999},"min":{"1min":0.32400000000000001,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.999,"5min":0.999,"15min":0.999},"last":0.30199999999999999}]},{"osd":3,"last update":"Sat Sep 21 09:28:39 2024","interfaces":[{"interface":"back","average":{"1min":0.436,"5min":0.42499999999999999,"15min":0.42899999999999999},"min":{"1min":0.28100000000000003,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":1.0309999999999999,"5min":1.0309999999999999,"15min":1.0309999999999999},"last":0.27500000000000002},{"interface":"front","average":{"1min":0.45400000000000001,"5min":0.433,"15min":0.43099999999999999},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":1.2849999999999999,"5min":1.2849999999999999,"15min":1.2849999999999999},"last":0.28899999999999998}]},{"osd":4,"last update":"Sat Sep 21 09:28:57 2024","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":0.54600000000000004,"15min":0.52300000000000002},"min":{"1min":0.439,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.85399999999999998,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.45900000000000002},{"interface":"front","average":{"1min":0.64500000000000002,"5min":0.59299999999999997,"15min":0.57299999999999995},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":1.2549999999999999,"5min":1.2549999999999999,"15min":1.2549999999999999},"last":0.46800000000000003}]},{"osd":5,"last update":"Sat Sep 21 09:29:26 2024","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.56100000000000005,"15min":0.56100000000000005},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.74399999999999999,"5min":1.177,"15min":1.177},"last":0.41099999999999998},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.58299999999999996,"15min":0.57899999999999996},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.745,"5min":1.0549999999999999,"15min":1.0549999999999999},"last":0.51100000000000001}]},{"osd":6,"last update":"Sat Sep 21 09:28:46 2024","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.59599999999999997,"15min":0.59199999999999997},"min":{"1min":0.39700000000000002,"5min":0.371,"15min":0.371},"max":{"1min":1.073,"5min":1.073,"15min":1.073},"last":0.38500000000000001},{"interface":"front","average":{"1min":0.59799999999999998,"5min":0.57299999999999995,"15min":0.56899999999999995},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.497}]},{"osd":7,"last update":"Sat Sep 21 09:29:08 2024","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":0.625,"15min":0.625},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":1.2270000000000001,"5min":1.2270000000000001,"15min":1.2270000000000001},"last":0.44},{"interface":"front","average":{"1min":0.65900000000000003,"5min":0.628,"15min":0.623},"min":{"1min":0.40699999999999997,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":1.21,"5min":1.21,"15min":1.21},"last":0.47899999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":716,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":2446,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":2432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":2342,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T09:29:34.771 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid d65e801e-77fa-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T09:29:35.307 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 21 09:29:35 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-alertmanager-a[74454]: ts=2024-09-21T09:29:35.030Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-09-21T09:29:35.308 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 21 09:29:35 smithi022 podman[79554]: 2024-09-21 09:29:35.051597687 +0000 UTC m=+0.130434528 container died c881f909bded02b97ee633e51746e57baeb514c5d105aa0731d63c13c19ae9e5 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-21T09:29:35.681 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 21 09:29:35 smithi022 podman[79554]: 2024-09-21 09:29:35.487296011 +0000 UTC m=+0.566132851 container cleanup c881f909bded02b97ee633e51746e57baeb514c5d105aa0731d63c13c19ae9e5 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-21T09:29:35.682 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 21 09:29:35 smithi022 bash[79554]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490-alertmanager-a 2024-09-21T09:29:35.870 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:29:35.994 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 21 09:29:35 smithi022 podman[79597]: 2024-09-21 09:29:35.68062257 +0000 UTC m=+0.623288556 container remove c881f909bded02b97ee633e51746e57baeb514c5d105aa0731d63c13c19ae9e5 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-21T09:29:35.994 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 21 09:29:35 smithi022 podman[79597]: 2024-09-21 09:29:35.764552573 +0000 UTC m=+0.707218559 volume remove a13068a112d60b89d689283688ab7ec52792b2b573137f1b11936ca7897314cd 2024-09-21T09:29:36.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:36 smithi022 ceph-mon[39525]: pgmap v44: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:29:36.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:36 smithi022 ceph-mon[27763]: pgmap v44: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:29:36.559 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 21 09:29:36 smithi022 podman[79620]: 2024-09-21 09:29:36.426860655 +0000 UTC m=+0.914936926 container died 034c18f1ae9a07bfd918cd60f2b6367ebb12087a15492c18bf06113b35159af0 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=modest_lamport, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , RELEASE=quincy-fbd1a6a, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.37.2, GIT_BRANCH=HEAD) 2024-09-21T09:29:36.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:36 smithi064 ceph-mon[31124]: pgmap v44: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:29:37.359 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:37 smithi022 ceph-mon[39525]: pgmap v45: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:29:37.359 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:37 smithi022 ceph-mon[27763]: pgmap v45: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:29:37.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:37 smithi064 ceph-mon[31124]: pgmap v45: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:29:37.764 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-09-21T09:29:37.765 INFO:teuthology.orchestra.run.smithi022.stderr:dumped all 2024-09-21T09:29:37.945 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 21 09:29:37 smithi022 systemd[1]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490@alertmanager.a.service: Deactivated successfully. 2024-09-21T09:29:37.945 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 21 09:29:37 smithi022 systemd[1]: Stopped Ceph alertmanager.a for d65e801e-77fa-11ef-baf6-efdc52797490. 2024-09-21T09:29:37.946 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 21 09:29:37 smithi022 systemd[1]: Starting Ceph alertmanager.a for d65e801e-77fa-11ef-baf6-efdc52797490... 2024-09-21T09:29:37.946 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 21 09:29:37 smithi022 podman[79834]: 2024-09-21 09:29:37.738877296 +0000 UTC m=+0.016760034 image pull c8568f914cd25b2062c44e9f79f9c18da6e3b85fe0c47a12a2191c61426c2b19 quay.io/prometheus/alertmanager:v0.25.0 2024-09-21T09:29:37.946 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 21 09:29:37 smithi022 podman[79834]: 2024-09-21 09:29:37.860248919 +0000 UTC m=+0.138131631 volume create 98a53bc2ba3e24c430d2f42d9a6e529c58e5969051c3f11d6ca2c5977e640a1f 2024-09-21T09:29:37.946 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 21 09:29:37 smithi022 podman[79834]: 2024-09-21 09:29:37.944228905 +0000 UTC m=+0.222111659 container create 42a53bb454cd0933ba1b7686cfc05459135bf705c792ef1bc31c2906f3b7f9f1 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-21T09:29:38.532 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:38 smithi022 ceph-mon[39525]: from='client.24632 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:29:38.532 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:38 smithi022 ceph-mon[27763]: from='client.24632 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:29:38.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:38 smithi064 ceph-mon[31124]: from='client.24632 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:29:38.808 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 21 09:29:38 smithi022 podman[79834]: 2024-09-21 09:29:38.531545862 +0000 UTC m=+0.809428593 container init 42a53bb454cd0933ba1b7686cfc05459135bf705c792ef1bc31c2906f3b7f9f1 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-21T09:29:38.808 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 21 09:29:38 smithi022 podman[79834]: 2024-09-21 09:29:38.537061806 +0000 UTC m=+0.814944518 container start 42a53bb454cd0933ba1b7686cfc05459135bf705c792ef1bc31c2906f3b7f9f1 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-21T09:29:38.808 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 21 09:29:38 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-alertmanager-a[79862]: ts=2024-09-21T09:29:38.551Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-09-21T09:29:38.808 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 21 09:29:38 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-alertmanager-a[79862]: ts=2024-09-21T09:29:38.551Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-09-21T09:29:38.808 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 21 09:29:38 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-alertmanager-a[79862]: ts=2024-09-21T09:29:38.552Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.22 port=9094 2024-09-21T09:29:38.808 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 21 09:29:38 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-alertmanager-a[79862]: ts=2024-09-21T09:29:38.553Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-09-21T09:29:38.808 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 21 09:29:38 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-alertmanager-a[79862]: ts=2024-09-21T09:29:38.576Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-21T09:29:38.808 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 21 09:29:38 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-alertmanager-a[79862]: ts=2024-09-21T09:29:38.577Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-21T09:29:38.808 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 21 09:29:38 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-alertmanager-a[79862]: ts=2024-09-21T09:29:38.580Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-09-21T09:29:38.808 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 21 09:29:38 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-alertmanager-a[79862]: ts=2024-09-21T09:29:38.580Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-09-21T09:29:38.808 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 21 09:29:38 smithi022 bash[79834]: 42a53bb454cd0933ba1b7686cfc05459135bf705c792ef1bc31c2906f3b7f9f1 2024-09-21T09:29:38.808 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 21 09:29:38 smithi022 systemd[1]: Started Ceph alertmanager.a for d65e801e-77fa-11ef-baf6-efdc52797490. 2024-09-21T09:29:39.272 INFO:teuthology.orchestra.run.smithi022.stdout:{"pg_ready":true,"pg_map":{"version":45,"stamp":"2024-09-21T09:29:36.804785+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":593515,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":126,"num_read_kb":231,"num_write":163,"num_write_kb":1646,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":124,"ondisk_log_size":124,"up":315,"acting":315,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":315,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2386056,"kb_used_data":7208,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747575160,"statfs":{"total":767960285184,"available":765516963840,"internally_reserved":0,"allocated":7380992,"data_stored":4259117,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002639"},"pg_stats":[{"pgid":"3.3f","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.359261+0000","last_change":"2024-09-21T09:27:35.635695+0000","last_active":"2024-09-21T09:28:33.359261+0000","last_peered":"2024-09-21T09:28:33.359261+0000","last_clean":"2024-09-21T09:28:33.359261+0000","last_became_active":"2024-09-21T09:27:35.635255+0000","last_became_peered":"2024-09-21T09:27:35.635255+0000","last_unstale":"2024-09-21T09:28:33.359261+0000","last_undegraded":"2024-09-21T09:28:33.359261+0000","last_fullsized":"2024-09-21T09:28:33.359261+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:15:50.156743+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3e","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.062366+0000","last_change":"2024-09-21T09:27:35.636115+0000","last_active":"2024-09-21T09:28:33.062366+0000","last_peered":"2024-09-21T09:28:33.062366+0000","last_clean":"2024-09-21T09:28:33.062366+0000","last_became_active":"2024-09-21T09:27:35.635778+0000","last_became_peered":"2024-09-21T09:27:35.635778+0000","last_unstale":"2024-09-21T09:28:33.062366+0000","last_undegraded":"2024-09-21T09:28:33.062366+0000","last_fullsized":"2024-09-21T09:28:33.062366+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:36:35.607057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3d","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.141182+0000","last_change":"2024-09-21T09:27:35.640689+0000","last_active":"2024-09-21T09:28:33.141182+0000","last_peered":"2024-09-21T09:28:33.141182+0000","last_clean":"2024-09-21T09:28:33.141182+0000","last_became_active":"2024-09-21T09:27:35.640415+0000","last_became_peered":"2024-09-21T09:27:35.640415+0000","last_unstale":"2024-09-21T09:28:33.141182+0000","last_undegraded":"2024-09-21T09:28:33.141182+0000","last_fullsized":"2024-09-21T09:28:33.141182+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:12:22.314199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3c","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.066315+0000","last_change":"2024-09-21T09:27:35.644536+0000","last_active":"2024-09-21T09:28:33.066315+0000","last_peered":"2024-09-21T09:28:33.066315+0000","last_clean":"2024-09-21T09:28:33.066315+0000","last_became_active":"2024-09-21T09:27:35.644246+0000","last_became_peered":"2024-09-21T09:27:35.644246+0000","last_unstale":"2024-09-21T09:28:33.066315+0000","last_undegraded":"2024-09-21T09:28:33.066315+0000","last_fullsized":"2024-09-21T09:28:33.066315+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:31:37.886172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3b","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.358990+0000","last_change":"2024-09-21T09:27:35.642779+0000","last_active":"2024-09-21T09:28:33.358990+0000","last_peered":"2024-09-21T09:28:33.358990+0000","last_clean":"2024-09-21T09:28:33.358990+0000","last_became_active":"2024-09-21T09:27:35.642487+0000","last_became_peered":"2024-09-21T09:27:35.642487+0000","last_unstale":"2024-09-21T09:28:33.358990+0000","last_undegraded":"2024-09-21T09:28:33.358990+0000","last_fullsized":"2024-09-21T09:28:33.358990+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:17:29.631740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3a","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.066399+0000","last_change":"2024-09-21T09:27:35.651016+0000","last_active":"2024-09-21T09:28:33.066399+0000","last_peered":"2024-09-21T09:28:33.066399+0000","last_clean":"2024-09-21T09:28:33.066399+0000","last_became_active":"2024-09-21T09:27:35.650742+0000","last_became_peered":"2024-09-21T09:27:35.650742+0000","last_unstale":"2024-09-21T09:28:33.066399+0000","last_undegraded":"2024-09-21T09:28:33.066399+0000","last_fullsized":"2024-09-21T09:28:33.066399+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:11:11.493990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.39","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.358835+0000","last_change":"2024-09-21T09:27:35.647235+0000","last_active":"2024-09-21T09:28:33.358835+0000","last_peered":"2024-09-21T09:28:33.358835+0000","last_clean":"2024-09-21T09:28:33.358835+0000","last_became_active":"2024-09-21T09:27:35.646890+0000","last_became_peered":"2024-09-21T09:27:35.646890+0000","last_unstale":"2024-09-21T09:28:33.358835+0000","last_undegraded":"2024-09-21T09:28:33.358835+0000","last_fullsized":"2024-09-21T09:28:33.358835+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:21:45.995387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.38","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.124976+0000","last_change":"2024-09-21T09:27:35.638161+0000","last_active":"2024-09-21T09:28:33.124976+0000","last_peered":"2024-09-21T09:28:33.124976+0000","last_clean":"2024-09-21T09:28:33.124976+0000","last_became_active":"2024-09-21T09:27:35.637751+0000","last_became_peered":"2024-09-21T09:27:35.637751+0000","last_unstale":"2024-09-21T09:28:33.124976+0000","last_undegraded":"2024-09-21T09:28:33.124976+0000","last_fullsized":"2024-09-21T09:28:33.124976+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:07:40.751242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.37","version":"0'0","reported_seq":31,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.359502+0000","last_change":"2024-09-21T09:27:52.079760+0000","last_active":"2024-09-21T09:28:33.359502+0000","last_peered":"2024-09-21T09:28:33.359502+0000","last_clean":"2024-09-21T09:28:33.359502+0000","last_became_active":"2024-09-21T09:27:52.079352+0000","last_became_peered":"2024-09-21T09:27:52.079352+0000","last_unstale":"2024-09-21T09:28:33.359502+0000","last_undegraded":"2024-09-21T09:28:33.359502+0000","last_fullsized":"2024-09-21T09:28:33.359502+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:57:57.194876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.36","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.484301+0000","last_change":"2024-09-21T09:27:35.636094+0000","last_active":"2024-09-21T09:28:33.484301+0000","last_peered":"2024-09-21T09:28:33.484301+0000","last_clean":"2024-09-21T09:28:33.484301+0000","last_became_active":"2024-09-21T09:27:35.635770+0000","last_became_peered":"2024-09-21T09:27:35.635770+0000","last_unstale":"2024-09-21T09:28:33.484301+0000","last_undegraded":"2024-09-21T09:28:33.484301+0000","last_fullsized":"2024-09-21T09:28:33.484301+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:41:52.909549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.35","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.484371+0000","last_change":"2024-09-21T09:27:35.640571+0000","last_active":"2024-09-21T09:28:33.484371+0000","last_peered":"2024-09-21T09:28:33.484371+0000","last_clean":"2024-09-21T09:28:33.484371+0000","last_became_active":"2024-09-21T09:27:35.640253+0000","last_became_peered":"2024-09-21T09:27:35.640253+0000","last_unstale":"2024-09-21T09:28:33.484371+0000","last_undegraded":"2024-09-21T09:28:33.484371+0000","last_fullsized":"2024-09-21T09:28:33.484371+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:40:44.721446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.34","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.062533+0000","last_change":"2024-09-21T09:27:35.639471+0000","last_active":"2024-09-21T09:28:33.062533+0000","last_peered":"2024-09-21T09:28:33.062533+0000","last_clean":"2024-09-21T09:28:33.062533+0000","last_became_active":"2024-09-21T09:27:35.639056+0000","last_became_peered":"2024-09-21T09:27:35.639056+0000","last_unstale":"2024-09-21T09:28:33.062533+0000","last_undegraded":"2024-09-21T09:28:33.062533+0000","last_fullsized":"2024-09-21T09:28:33.062533+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:16:15.724781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.33","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.125281+0000","last_change":"2024-09-21T09:27:35.637331+0000","last_active":"2024-09-21T09:28:33.125281+0000","last_peered":"2024-09-21T09:28:33.125281+0000","last_clean":"2024-09-21T09:28:33.125281+0000","last_became_active":"2024-09-21T09:27:35.637068+0000","last_became_peered":"2024-09-21T09:27:35.637068+0000","last_unstale":"2024-09-21T09:28:33.125281+0000","last_undegraded":"2024-09-21T09:28:33.125281+0000","last_fullsized":"2024-09-21T09:28:33.125281+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:24:52.176104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.32","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.140751+0000","last_change":"2024-09-21T09:27:35.645565+0000","last_active":"2024-09-21T09:28:33.140751+0000","last_peered":"2024-09-21T09:28:33.140751+0000","last_clean":"2024-09-21T09:28:33.140751+0000","last_became_active":"2024-09-21T09:27:35.644114+0000","last_became_peered":"2024-09-21T09:27:35.644114+0000","last_unstale":"2024-09-21T09:28:33.140751+0000","last_undegraded":"2024-09-21T09:28:33.140751+0000","last_fullsized":"2024-09-21T09:28:33.140751+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:27:18.008605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.31","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.062608+0000","last_change":"2024-09-21T09:27:35.634448+0000","last_active":"2024-09-21T09:28:33.062608+0000","last_peered":"2024-09-21T09:28:33.062608+0000","last_clean":"2024-09-21T09:28:33.062608+0000","last_became_active":"2024-09-21T09:27:35.634089+0000","last_became_peered":"2024-09-21T09:27:35.634089+0000","last_unstale":"2024-09-21T09:28:33.062608+0000","last_undegraded":"2024-09-21T09:28:33.062608+0000","last_fullsized":"2024-09-21T09:28:33.062608+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:14:14.726492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.30","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.066102+0000","last_change":"2024-09-21T09:27:35.651997+0000","last_active":"2024-09-21T09:28:33.066102+0000","last_peered":"2024-09-21T09:28:33.066102+0000","last_clean":"2024-09-21T09:28:33.066102+0000","last_became_active":"2024-09-21T09:27:35.651698+0000","last_became_peered":"2024-09-21T09:27:35.651698+0000","last_unstale":"2024-09-21T09:28:33.066102+0000","last_undegraded":"2024-09-21T09:28:33.066102+0000","last_fullsized":"2024-09-21T09:28:33.066102+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:43:39.574788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2f","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.066495+0000","last_change":"2024-09-21T09:27:35.646363+0000","last_active":"2024-09-21T09:28:33.066495+0000","last_peered":"2024-09-21T09:28:33.066495+0000","last_clean":"2024-09-21T09:28:33.066495+0000","last_became_active":"2024-09-21T09:27:35.646084+0000","last_became_peered":"2024-09-21T09:27:35.646084+0000","last_unstale":"2024-09-21T09:28:33.066495+0000","last_undegraded":"2024-09-21T09:28:33.066495+0000","last_fullsized":"2024-09-21T09:28:33.066495+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:35:38.711926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2e","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.061977+0000","last_change":"2024-09-21T09:27:35.636697+0000","last_active":"2024-09-21T09:28:33.061977+0000","last_peered":"2024-09-21T09:28:33.061977+0000","last_clean":"2024-09-21T09:28:33.061977+0000","last_became_active":"2024-09-21T09:27:35.636438+0000","last_became_peered":"2024-09-21T09:27:35.636438+0000","last_unstale":"2024-09-21T09:28:33.061977+0000","last_undegraded":"2024-09-21T09:28:33.061977+0000","last_fullsized":"2024-09-21T09:28:33.061977+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:55:44.287544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,7],"acting":[2,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2d","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.359395+0000","last_change":"2024-09-21T09:27:35.636110+0000","last_active":"2024-09-21T09:28:33.359395+0000","last_peered":"2024-09-21T09:28:33.359395+0000","last_clean":"2024-09-21T09:28:33.359395+0000","last_became_active":"2024-09-21T09:27:35.635807+0000","last_became_peered":"2024-09-21T09:27:35.635807+0000","last_unstale":"2024-09-21T09:28:33.359395+0000","last_undegraded":"2024-09-21T09:28:33.359395+0000","last_fullsized":"2024-09-21T09:28:33.359395+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:22:43.942811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2c","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.484182+0000","last_change":"2024-09-21T09:27:35.650351+0000","last_active":"2024-09-21T09:28:33.484182+0000","last_peered":"2024-09-21T09:28:33.484182+0000","last_clean":"2024-09-21T09:28:33.484182+0000","last_became_active":"2024-09-21T09:27:35.649905+0000","last_became_peered":"2024-09-21T09:27:35.649905+0000","last_unstale":"2024-09-21T09:28:33.484182+0000","last_undegraded":"2024-09-21T09:28:33.484182+0000","last_fullsized":"2024-09-21T09:28:33.484182+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:12:31.390666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2b","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.141020+0000","last_change":"2024-09-21T09:27:35.643345+0000","last_active":"2024-09-21T09:28:33.141020+0000","last_peered":"2024-09-21T09:28:33.141020+0000","last_clean":"2024-09-21T09:28:33.141020+0000","last_became_active":"2024-09-21T09:27:35.642408+0000","last_became_peered":"2024-09-21T09:27:35.642408+0000","last_unstale":"2024-09-21T09:28:33.141020+0000","last_undegraded":"2024-09-21T09:28:33.141020+0000","last_fullsized":"2024-09-21T09:28:33.141020+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:34:59.698334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2a","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.066015+0000","last_change":"2024-09-21T09:27:35.645032+0000","last_active":"2024-09-21T09:28:33.066015+0000","last_peered":"2024-09-21T09:28:33.066015+0000","last_clean":"2024-09-21T09:28:33.066015+0000","last_became_active":"2024-09-21T09:27:35.644322+0000","last_became_peered":"2024-09-21T09:27:35.644322+0000","last_unstale":"2024-09-21T09:28:33.066015+0000","last_undegraded":"2024-09-21T09:28:33.066015+0000","last_fullsized":"2024-09-21T09:28:33.066015+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:10:14.511772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.29","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.125084+0000","last_change":"2024-09-21T09:27:35.638305+0000","last_active":"2024-09-21T09:28:33.125084+0000","last_peered":"2024-09-21T09:28:33.125084+0000","last_clean":"2024-09-21T09:28:33.125084+0000","last_became_active":"2024-09-21T09:27:35.638036+0000","last_became_peered":"2024-09-21T09:27:35.638036+0000","last_unstale":"2024-09-21T09:28:33.125084+0000","last_undegraded":"2024-09-21T09:28:33.125084+0000","last_fullsized":"2024-09-21T09:28:33.125084+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:38:54.627840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.28","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.066181+0000","last_change":"2024-09-21T09:27:35.637277+0000","last_active":"2024-09-21T09:28:33.066181+0000","last_peered":"2024-09-21T09:28:33.066181+0000","last_clean":"2024-09-21T09:28:33.066181+0000","last_became_active":"2024-09-21T09:27:35.636541+0000","last_became_peered":"2024-09-21T09:27:35.636541+0000","last_unstale":"2024-09-21T09:28:33.066181+0000","last_undegraded":"2024-09-21T09:28:33.066181+0000","last_fullsized":"2024-09-21T09:28:33.066181+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:45:47.090531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.27","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.125608+0000","last_change":"2024-09-21T09:27:35.638946+0000","last_active":"2024-09-21T09:28:33.125608+0000","last_peered":"2024-09-21T09:28:33.125608+0000","last_clean":"2024-09-21T09:28:33.125608+0000","last_became_active":"2024-09-21T09:27:35.638663+0000","last_became_peered":"2024-09-21T09:27:35.638663+0000","last_unstale":"2024-09-21T09:28:33.125608+0000","last_undegraded":"2024-09-21T09:28:33.125608+0000","last_fullsized":"2024-09-21T09:28:33.125608+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:49:57.591531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.26","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.484321+0000","last_change":"2024-09-21T09:27:35.638679+0000","last_active":"2024-09-21T09:28:33.484321+0000","last_peered":"2024-09-21T09:28:33.484321+0000","last_clean":"2024-09-21T09:28:33.484321+0000","last_became_active":"2024-09-21T09:27:35.638383+0000","last_became_peered":"2024-09-21T09:27:35.638383+0000","last_unstale":"2024-09-21T09:28:33.484321+0000","last_undegraded":"2024-09-21T09:28:33.484321+0000","last_fullsized":"2024-09-21T09:28:33.484321+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:16:11.663503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,4],"acting":[6,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.25","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.061992+0000","last_change":"2024-09-21T09:27:35.632894+0000","last_active":"2024-09-21T09:28:33.061992+0000","last_peered":"2024-09-21T09:28:33.061992+0000","last_clean":"2024-09-21T09:28:33.061992+0000","last_became_active":"2024-09-21T09:27:35.632569+0000","last_became_peered":"2024-09-21T09:27:35.632569+0000","last_unstale":"2024-09-21T09:28:33.061992+0000","last_undegraded":"2024-09-21T09:28:33.061992+0000","last_fullsized":"2024-09-21T09:28:33.061992+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:25:22.833535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.24","version":"0'0","reported_seq":31,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.125399+0000","last_change":"2024-09-21T09:27:52.079786+0000","last_active":"2024-09-21T09:28:33.125399+0000","last_peered":"2024-09-21T09:28:33.125399+0000","last_clean":"2024-09-21T09:28:33.125399+0000","last_became_active":"2024-09-21T09:27:52.079395+0000","last_became_peered":"2024-09-21T09:27:52.079395+0000","last_unstale":"2024-09-21T09:28:33.125399+0000","last_undegraded":"2024-09-21T09:28:33.125399+0000","last_fullsized":"2024-09-21T09:28:33.125399+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:17:07.388900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.23","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.141121+0000","last_change":"2024-09-21T09:27:35.634785+0000","last_active":"2024-09-21T09:28:33.141121+0000","last_peered":"2024-09-21T09:28:33.141121+0000","last_clean":"2024-09-21T09:28:33.141121+0000","last_became_active":"2024-09-21T09:27:35.634065+0000","last_became_peered":"2024-09-21T09:27:35.634065+0000","last_unstale":"2024-09-21T09:28:33.141121+0000","last_undegraded":"2024-09-21T09:28:33.141121+0000","last_fullsized":"2024-09-21T09:28:33.141121+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:54:21.482132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.22","version":"0'0","reported_seq":31,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.066111+0000","last_change":"2024-09-21T09:27:52.049243+0000","last_active":"2024-09-21T09:28:33.066111+0000","last_peered":"2024-09-21T09:28:33.066111+0000","last_clean":"2024-09-21T09:28:33.066111+0000","last_became_active":"2024-09-21T09:27:52.048885+0000","last_became_peered":"2024-09-21T09:27:52.048885+0000","last_unstale":"2024-09-21T09:28:33.066111+0000","last_undegraded":"2024-09-21T09:28:33.066111+0000","last_fullsized":"2024-09-21T09:28:33.066111+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:44:22.262414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.21","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.061749+0000","last_change":"2024-09-21T09:27:35.636222+0000","last_active":"2024-09-21T09:28:33.061749+0000","last_peered":"2024-09-21T09:28:33.061749+0000","last_clean":"2024-09-21T09:28:33.061749+0000","last_became_active":"2024-09-21T09:27:35.635935+0000","last_became_peered":"2024-09-21T09:27:35.635935+0000","last_unstale":"2024-09-21T09:28:33.061749+0000","last_undegraded":"2024-09-21T09:28:33.061749+0000","last_fullsized":"2024-09-21T09:28:33.061749+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:05:50.402224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.20","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.483839+0000","last_change":"2024-09-21T09:27:35.641740+0000","last_active":"2024-09-21T09:28:33.483839+0000","last_peered":"2024-09-21T09:28:33.483839+0000","last_clean":"2024-09-21T09:28:33.483839+0000","last_became_active":"2024-09-21T09:27:35.641409+0000","last_became_peered":"2024-09-21T09:27:35.641409+0000","last_unstale":"2024-09-21T09:28:33.483839+0000","last_undegraded":"2024-09-21T09:28:33.483839+0000","last_fullsized":"2024-09-21T09:28:33.483839+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:59:30.487730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1a","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.062273+0000","last_change":"2024-09-21T09:27:35.629525+0000","last_active":"2024-09-21T09:28:33.062273+0000","last_peered":"2024-09-21T09:28:33.062273+0000","last_clean":"2024-09-21T09:28:33.062273+0000","last_became_active":"2024-09-21T09:27:35.629167+0000","last_became_peered":"2024-09-21T09:27:35.629167+0000","last_unstale":"2024-09-21T09:28:33.062273+0000","last_undegraded":"2024-09-21T09:28:33.062273+0000","last_fullsized":"2024-09-21T09:28:33.062273+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:06:24.323825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1b","version":"57'1","reported_seq":25,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.125391+0000","last_change":"2024-09-21T09:27:32.265235+0000","last_active":"2024-09-21T09:28:33.125391+0000","last_peered":"2024-09-21T09:28:33.125391+0000","last_clean":"2024-09-21T09:28:33.125391+0000","last_became_active":"2024-09-21T09:27:32.264963+0000","last_became_peered":"2024-09-21T09:27:32.264963+0000","last_unstale":"2024-09-21T09:28:33.125391+0000","last_undegraded":"2024-09-21T09:28:33.125391+0000","last_fullsized":"2024-09-21T09:28:33.125391+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:09:03.927341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.141127+0000","last_change":"2024-09-21T09:27:35.642154+0000","last_active":"2024-09-21T09:28:33.141127+0000","last_peered":"2024-09-21T09:28:33.141127+0000","last_clean":"2024-09-21T09:28:33.141127+0000","last_became_active":"2024-09-21T09:27:35.641801+0000","last_became_peered":"2024-09-21T09:27:35.641801+0000","last_unstale":"2024-09-21T09:28:33.141127+0000","last_undegraded":"2024-09-21T09:28:33.141127+0000","last_fullsized":"2024-09-21T09:28:33.141127+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:50:45.075506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1a","version":"57'1","reported_seq":25,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.484044+0000","last_change":"2024-09-21T09:27:32.269358+0000","last_active":"2024-09-21T09:28:33.484044+0000","last_peered":"2024-09-21T09:28:33.484044+0000","last_clean":"2024-09-21T09:28:33.484044+0000","last_became_active":"2024-09-21T09:27:32.269026+0000","last_became_peered":"2024-09-21T09:27:32.269026+0000","last_unstale":"2024-09-21T09:28:33.484044+0000","last_undegraded":"2024-09-21T09:28:33.484044+0000","last_fullsized":"2024-09-21T09:28:33.484044+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:59:17.916370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.19","version":"0'0","reported_seq":24,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.125195+0000","last_change":"2024-09-21T09:27:32.264601+0000","last_active":"2024-09-21T09:28:33.125195+0000","last_peered":"2024-09-21T09:28:33.125195+0000","last_clean":"2024-09-21T09:28:33.125195+0000","last_became_active":"2024-09-21T09:27:32.264381+0000","last_became_peered":"2024-09-21T09:27:32.264381+0000","last_unstale":"2024-09-21T09:28:33.125195+0000","last_undegraded":"2024-09-21T09:28:33.125195+0000","last_fullsized":"2024-09-21T09:28:33.125195+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:35:57.463156+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.125067+0000","last_change":"2024-09-21T09:27:35.634475+0000","last_active":"2024-09-21T09:28:33.125067+0000","last_peered":"2024-09-21T09:28:33.125067+0000","last_clean":"2024-09-21T09:28:33.125067+0000","last_became_active":"2024-09-21T09:27:35.634224+0000","last_became_peered":"2024-09-21T09:27:35.634224+0000","last_unstale":"2024-09-21T09:28:33.125067+0000","last_undegraded":"2024-09-21T09:28:33.125067+0000","last_fullsized":"2024-09-21T09:28:33.125067+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:38:21.241590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.18","version":"0'0","reported_seq":24,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.066263+0000","last_change":"2024-09-21T09:27:32.267805+0000","last_active":"2024-09-21T09:28:33.066263+0000","last_peered":"2024-09-21T09:28:33.066263+0000","last_clean":"2024-09-21T09:28:33.066263+0000","last_became_active":"2024-09-21T09:27:32.267484+0000","last_became_peered":"2024-09-21T09:27:32.267484+0000","last_unstale":"2024-09-21T09:28:33.066263+0000","last_undegraded":"2024-09-21T09:28:33.066263+0000","last_fullsized":"2024-09-21T09:28:33.066263+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:00:43.105362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,7],"acting":[5,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.062486+0000","last_change":"2024-09-21T09:27:35.631350+0000","last_active":"2024-09-21T09:28:33.062486+0000","last_peered":"2024-09-21T09:28:33.062486+0000","last_clean":"2024-09-21T09:28:33.062486+0000","last_became_active":"2024-09-21T09:27:35.630723+0000","last_became_peered":"2024-09-21T09:27:35.630723+0000","last_unstale":"2024-09-21T09:28:33.062486+0000","last_undegraded":"2024-09-21T09:28:33.062486+0000","last_fullsized":"2024-09-21T09:28:33.062486+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:46:32.135037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.17","version":"0'0","reported_seq":24,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.484390+0000","last_change":"2024-09-21T09:27:32.266307+0000","last_active":"2024-09-21T09:28:33.484390+0000","last_peered":"2024-09-21T09:28:33.484390+0000","last_clean":"2024-09-21T09:28:33.484390+0000","last_became_active":"2024-09-21T09:27:32.265995+0000","last_became_peered":"2024-09-21T09:27:32.265995+0000","last_unstale":"2024-09-21T09:28:33.484390+0000","last_undegraded":"2024-09-21T09:28:33.484390+0000","last_fullsized":"2024-09-21T09:28:33.484390+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:24:28.036534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.066174+0000","last_change":"2024-09-21T09:27:35.647952+0000","last_active":"2024-09-21T09:28:33.066174+0000","last_peered":"2024-09-21T09:28:33.066174+0000","last_clean":"2024-09-21T09:28:33.066174+0000","last_became_active":"2024-09-21T09:27:35.647649+0000","last_became_peered":"2024-09-21T09:27:35.647649+0000","last_unstale":"2024-09-21T09:28:33.066174+0000","last_undegraded":"2024-09-21T09:28:33.066174+0000","last_fullsized":"2024-09-21T09:28:33.066174+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:12:29.850835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.16","version":"0'0","reported_seq":24,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.066325+0000","last_change":"2024-09-21T09:27:32.269137+0000","last_active":"2024-09-21T09:28:33.066325+0000","last_peered":"2024-09-21T09:28:33.066325+0000","last_clean":"2024-09-21T09:28:33.066325+0000","last_became_active":"2024-09-21T09:27:32.268832+0000","last_became_peered":"2024-09-21T09:27:32.268832+0000","last_unstale":"2024-09-21T09:28:33.066325+0000","last_undegraded":"2024-09-21T09:28:33.066325+0000","last_fullsized":"2024-09-21T09:28:33.066325+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:51:58.668390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":18,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.062462+0000","last_change":"2024-09-21T09:27:52.050121+0000","last_active":"2024-09-21T09:28:33.062462+0000","last_peered":"2024-09-21T09:28:33.062462+0000","last_clean":"2024-09-21T09:28:33.062462+0000","last_became_active":"2024-09-21T09:27:52.049549+0000","last_became_peered":"2024-09-21T09:27:52.049549+0000","last_unstale":"2024-09-21T09:28:33.062462+0000","last_undegraded":"2024-09-21T09:28:33.062462+0000","last_fullsized":"2024-09-21T09:28:33.062462+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:25:27.303500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.15","version":"57'3","reported_seq":27,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.062137+0000","last_change":"2024-09-21T09:27:32.270145+0000","last_active":"2024-09-21T09:28:33.062137+0000","last_peered":"2024-09-21T09:28:33.062137+0000","last_clean":"2024-09-21T09:28:33.062137+0000","last_became_active":"2024-09-21T09:27:32.269808+0000","last_became_peered":"2024-09-21T09:27:32.269808+0000","last_unstale":"2024-09-21T09:28:33.062137+0000","last_undegraded":"2024-09-21T09:28:33.062137+0000","last_fullsized":"2024-09-21T09:28:33.062137+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:35:53.397343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.062595+0000","last_change":"2024-09-21T09:27:35.640314+0000","last_active":"2024-09-21T09:28:33.062595+0000","last_peered":"2024-09-21T09:28:33.062595+0000","last_clean":"2024-09-21T09:28:33.062595+0000","last_became_active":"2024-09-21T09:27:35.639976+0000","last_became_peered":"2024-09-21T09:27:35.639976+0000","last_unstale":"2024-09-21T09:28:33.062595+0000","last_undegraded":"2024-09-21T09:28:33.062595+0000","last_fullsized":"2024-09-21T09:28:33.062595+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:55:21.229874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.14","version":"60'9","reported_seq":33,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.484214+0000","last_change":"2024-09-21T09:27:32.266499+0000","last_active":"2024-09-21T09:28:33.484214+0000","last_peered":"2024-09-21T09:28:33.484214+0000","last_clean":"2024-09-21T09:28:33.484214+0000","last_became_active":"2024-09-21T09:27:32.265712+0000","last_became_peered":"2024-09-21T09:27:32.265712+0000","last_unstale":"2024-09-21T09:28:33.484214+0000","last_undegraded":"2024-09-21T09:28:33.484214+0000","last_fullsized":"2024-09-21T09:28:33.484214+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:19:17.014990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2342,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.359220+0000","last_change":"2024-09-21T09:27:35.641626+0000","last_active":"2024-09-21T09:28:33.359220+0000","last_peered":"2024-09-21T09:28:33.359220+0000","last_clean":"2024-09-21T09:28:33.359220+0000","last_became_active":"2024-09-21T09:27:35.641272+0000","last_became_peered":"2024-09-21T09:27:35.641272+0000","last_unstale":"2024-09-21T09:28:33.359220+0000","last_undegraded":"2024-09-21T09:28:33.359220+0000","last_fullsized":"2024-09-21T09:28:33.359220+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:15:05.108072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.13","version":"57'2","reported_seq":26,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.141220+0000","last_change":"2024-09-21T09:27:32.267412+0000","last_active":"2024-09-21T09:28:33.141220+0000","last_peered":"2024-09-21T09:28:33.141220+0000","last_clean":"2024-09-21T09:28:33.141220+0000","last_became_active":"2024-09-21T09:27:32.267108+0000","last_became_peered":"2024-09-21T09:27:32.267108+0000","last_unstale":"2024-09-21T09:28:33.141220+0000","last_undegraded":"2024-09-21T09:28:33.141220+0000","last_fullsized":"2024-09-21T09:28:33.141220+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:54:11.228702+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.140829+0000","last_change":"2024-09-21T09:27:35.641572+0000","last_active":"2024-09-21T09:28:33.140829+0000","last_peered":"2024-09-21T09:28:33.140829+0000","last_clean":"2024-09-21T09:28:33.140829+0000","last_became_active":"2024-09-21T09:27:35.641185+0000","last_became_peered":"2024-09-21T09:27:35.641185+0000","last_unstale":"2024-09-21T09:28:33.140829+0000","last_undegraded":"2024-09-21T09:28:33.140829+0000","last_fullsized":"2024-09-21T09:28:33.140829+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:52:30.993491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.12","version":"0'0","reported_seq":24,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.066197+0000","last_change":"2024-09-21T09:27:32.267644+0000","last_active":"2024-09-21T09:28:33.066197+0000","last_peered":"2024-09-21T09:28:33.066197+0000","last_clean":"2024-09-21T09:28:33.066197+0000","last_became_active":"2024-09-21T09:27:32.267232+0000","last_became_peered":"2024-09-21T09:27:32.267232+0000","last_unstale":"2024-09-21T09:28:33.066197+0000","last_undegraded":"2024-09-21T09:28:33.066197+0000","last_fullsized":"2024-09-21T09:28:33.066197+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:14:19.578564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.358995+0000","last_change":"2024-09-21T09:27:35.644217+0000","last_active":"2024-09-21T09:28:33.358995+0000","last_peered":"2024-09-21T09:28:33.358995+0000","last_clean":"2024-09-21T09:28:33.358995+0000","last_became_active":"2024-09-21T09:27:35.643681+0000","last_became_peered":"2024-09-21T09:27:35.643681+0000","last_unstale":"2024-09-21T09:28:33.358995+0000","last_undegraded":"2024-09-21T09:28:33.358995+0000","last_fullsized":"2024-09-21T09:28:33.358995+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:07:58.418951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.11","version":"0'0","reported_seq":24,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.483959+0000","last_change":"2024-09-21T09:27:32.270501+0000","last_active":"2024-09-21T09:28:33.483959+0000","last_peered":"2024-09-21T09:28:33.483959+0000","last_clean":"2024-09-21T09:28:33.483959+0000","last_became_active":"2024-09-21T09:27:32.269808+0000","last_became_peered":"2024-09-21T09:27:32.269808+0000","last_unstale":"2024-09-21T09:28:33.483959+0000","last_undegraded":"2024-09-21T09:28:33.483959+0000","last_fullsized":"2024-09-21T09:28:33.483959+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:28:03.705631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.10","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.483855+0000","last_change":"2024-09-21T09:27:35.641508+0000","last_active":"2024-09-21T09:28:33.483855+0000","last_peered":"2024-09-21T09:28:33.483855+0000","last_clean":"2024-09-21T09:28:33.483855+0000","last_became_active":"2024-09-21T09:27:35.640996+0000","last_became_peered":"2024-09-21T09:27:35.640996+0000","last_unstale":"2024-09-21T09:28:33.483855+0000","last_undegraded":"2024-09-21T09:28:33.483855+0000","last_fullsized":"2024-09-21T09:28:33.483855+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:02:24.674295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.10","version":"57'8","reported_seq":32,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.061786+0000","last_change":"2024-09-21T09:27:32.264620+0000","last_active":"2024-09-21T09:28:33.061786+0000","last_peered":"2024-09-21T09:28:33.061786+0000","last_clean":"2024-09-21T09:28:33.061786+0000","last_became_active":"2024-09-21T09:27:32.261324+0000","last_became_peered":"2024-09-21T09:27:32.261324+0000","last_unstale":"2024-09-21T09:28:33.061786+0000","last_undegraded":"2024-09-21T09:28:33.061786+0000","last_fullsized":"2024-09-21T09:28:33.061786+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:11:04.933873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.358860+0000","last_change":"2024-09-21T09:27:35.641576+0000","last_active":"2024-09-21T09:28:33.358860+0000","last_peered":"2024-09-21T09:28:33.358860+0000","last_clean":"2024-09-21T09:28:33.358860+0000","last_became_active":"2024-09-21T09:27:35.641261+0000","last_became_peered":"2024-09-21T09:27:35.641261+0000","last_unstale":"2024-09-21T09:28:33.358860+0000","last_undegraded":"2024-09-21T09:28:33.358860+0000","last_fullsized":"2024-09-21T09:28:33.358860+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:59:48.868788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.f","version":"57'1","reported_seq":25,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.062869+0000","last_change":"2024-09-21T09:27:32.270055+0000","last_active":"2024-09-21T09:28:33.062869+0000","last_peered":"2024-09-21T09:28:33.062869+0000","last_clean":"2024-09-21T09:28:33.062869+0000","last_became_active":"2024-09-21T09:27:32.269748+0000","last_became_peered":"2024-09-21T09:27:32.269748+0000","last_unstale":"2024-09-21T09:28:33.062869+0000","last_undegraded":"2024-09-21T09:28:33.062869+0000","last_fullsized":"2024-09-21T09:28:33.062869+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:50:54.853319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.359401+0000","last_change":"2024-09-21T09:27:35.641790+0000","last_active":"2024-09-21T09:28:33.359401+0000","last_peered":"2024-09-21T09:28:33.359401+0000","last_clean":"2024-09-21T09:28:33.359401+0000","last_became_active":"2024-09-21T09:27:35.641348+0000","last_became_peered":"2024-09-21T09:27:35.641348+0000","last_unstale":"2024-09-21T09:28:33.359401+0000","last_undegraded":"2024-09-21T09:28:33.359401+0000","last_fullsized":"2024-09-21T09:28:33.359401+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:25:10.635353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.e","version":"57'2","reported_seq":26,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.062124+0000","last_change":"2024-09-21T09:27:32.264276+0000","last_active":"2024-09-21T09:28:33.062124+0000","last_peered":"2024-09-21T09:28:33.062124+0000","last_clean":"2024-09-21T09:28:33.062124+0000","last_became_active":"2024-09-21T09:27:32.263984+0000","last_became_peered":"2024-09-21T09:27:32.263984+0000","last_unstale":"2024-09-21T09:28:33.062124+0000","last_undegraded":"2024-09-21T09:28:33.062124+0000","last_fullsized":"2024-09-21T09:28:33.062124+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:18:50.045457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.359338+0000","last_change":"2024-09-21T09:27:35.649353+0000","last_active":"2024-09-21T09:28:33.359338+0000","last_peered":"2024-09-21T09:28:33.359338+0000","last_clean":"2024-09-21T09:28:33.359338+0000","last_became_active":"2024-09-21T09:27:35.649033+0000","last_became_peered":"2024-09-21T09:27:35.649033+0000","last_unstale":"2024-09-21T09:28:33.359338+0000","last_undegraded":"2024-09-21T09:28:33.359338+0000","last_fullsized":"2024-09-21T09:28:33.359338+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:13:12.611019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.d","version":"57'1","reported_seq":25,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.062217+0000","last_change":"2024-09-21T09:27:32.260033+0000","last_active":"2024-09-21T09:28:33.062217+0000","last_peered":"2024-09-21T09:28:33.062217+0000","last_clean":"2024-09-21T09:28:33.062217+0000","last_became_active":"2024-09-21T09:27:32.259209+0000","last_became_peered":"2024-09-21T09:27:32.259209+0000","last_unstale":"2024-09-21T09:28:33.062217+0000","last_undegraded":"2024-09-21T09:28:33.062217+0000","last_fullsized":"2024-09-21T09:28:33.062217+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:16:09.835879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.066397+0000","last_change":"2024-09-21T09:27:35.639076+0000","last_active":"2024-09-21T09:28:33.066397+0000","last_peered":"2024-09-21T09:28:33.066397+0000","last_clean":"2024-09-21T09:28:33.066397+0000","last_became_active":"2024-09-21T09:27:35.638802+0000","last_became_peered":"2024-09-21T09:27:35.638802+0000","last_unstale":"2024-09-21T09:28:33.066397+0000","last_undegraded":"2024-09-21T09:28:33.066397+0000","last_fullsized":"2024-09-21T09:28:33.066397+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:36:18.524320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.c","version":"0'0","reported_seq":24,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.062156+0000","last_change":"2024-09-21T09:27:32.270060+0000","last_active":"2024-09-21T09:28:33.062156+0000","last_peered":"2024-09-21T09:28:33.062156+0000","last_clean":"2024-09-21T09:28:33.062156+0000","last_became_active":"2024-09-21T09:27:32.269717+0000","last_became_peered":"2024-09-21T09:27:32.269717+0000","last_unstale":"2024-09-21T09:28:33.062156+0000","last_undegraded":"2024-09-21T09:28:33.062156+0000","last_fullsized":"2024-09-21T09:28:33.062156+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:59:27.725031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.359314+0000","last_change":"2024-09-21T09:27:35.640059+0000","last_active":"2024-09-21T09:28:33.359314+0000","last_peered":"2024-09-21T09:28:33.359314+0000","last_clean":"2024-09-21T09:28:33.359314+0000","last_became_active":"2024-09-21T09:27:35.639746+0000","last_became_peered":"2024-09-21T09:27:35.639746+0000","last_unstale":"2024-09-21T09:28:33.359314+0000","last_undegraded":"2024-09-21T09:28:33.359314+0000","last_fullsized":"2024-09-21T09:28:33.359314+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:33:15.402410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":19,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-21T09:28:41.371073+0000","last_change":"2024-09-21T09:28:33.484488+0000","last_active":"2024-09-21T09:28:41.371073+0000","last_peered":"2024-09-21T09:28:41.371073+0000","last_clean":"2024-09-21T09:28:41.371073+0000","last_became_active":"2024-09-21T09:28:28.844168+0000","last_became_peered":"2024-09-21T09:28:28.844168+0000","last_unstale":"2024-09-21T09:28:41.371073+0000","last_undegraded":"2024-09-21T09:28:41.371073+0000","last_fullsized":"2024-09-21T09:28:41.371073+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:28:27.830318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:28:27.830318+0000","last_clean_scrub_stamp":"2024-09-21T09:28:27.830318+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:07:12.698450+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030427299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"62'79","reported_seq":58,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.359151+0000","last_change":"2024-09-21T09:27:08.460023+0000","last_active":"2024-09-21T09:28:33.359151+0000","last_peered":"2024-09-21T09:28:33.359151+0000","last_clean":"2024-09-21T09:28:33.359151+0000","last_became_active":"2024-09-21T09:27:08.454770+0000","last_became_peered":"2024-09-21T09:27:08.454770+0000","last_unstale":"2024-09-21T09:28:33.359151+0000","last_undegraded":"2024-09-21T09:28:33.359151+0000","last_fullsized":"2024-09-21T09:28:33.359151+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:25:07.646659+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:25:07.646659+0000","last_clean_scrub_stamp":"2024-09-21T09:25:07.646659+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:08:45.658466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":231,"num_write":113,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":34,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.066105+0000","last_change":"2024-09-21T09:27:52.081088+0000","last_active":"2024-09-21T09:28:33.066105+0000","last_peered":"2024-09-21T09:28:33.066105+0000","last_clean":"2024-09-21T09:28:33.066105+0000","last_became_active":"2024-09-21T09:27:52.080680+0000","last_became_peered":"2024-09-21T09:27:52.080680+0000","last_unstale":"2024-09-21T09:28:33.066105+0000","last_undegraded":"2024-09-21T09:28:33.066105+0000","last_fullsized":"2024-09-21T09:28:33.066105+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:01:04.317396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.125141+0000","last_change":"2024-09-21T09:27:35.636200+0000","last_active":"2024-09-21T09:28:33.125141+0000","last_peered":"2024-09-21T09:28:33.125141+0000","last_clean":"2024-09-21T09:28:33.125141+0000","last_became_active":"2024-09-21T09:27:35.635939+0000","last_became_peered":"2024-09-21T09:27:35.635939+0000","last_unstale":"2024-09-21T09:28:33.125141+0000","last_undegraded":"2024-09-21T09:28:33.125141+0000","last_fullsized":"2024-09-21T09:28:33.125141+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:02:06.551561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":19,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-21T09:28:40.923121+0000","last_change":"2024-09-21T09:28:33.141556+0000","last_active":"2024-09-21T09:28:40.923121+0000","last_peered":"2024-09-21T09:28:40.923121+0000","last_clean":"2024-09-21T09:28:40.923121+0000","last_became_active":"2024-09-21T09:28:28.846894+0000","last_became_peered":"2024-09-21T09:28:28.846894+0000","last_unstale":"2024-09-21T09:28:40.923121+0000","last_undegraded":"2024-09-21T09:28:40.923121+0000","last_fullsized":"2024-09-21T09:28:40.923121+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:28:27.830318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:28:27.830318+0000","last_clean_scrub_stamp":"2024-09-21T09:28:27.830318+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:54:49.861215+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000276941,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":24,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.359086+0000","last_change":"2024-09-21T09:27:32.268500+0000","last_active":"2024-09-21T09:28:33.359086+0000","last_peered":"2024-09-21T09:28:33.359086+0000","last_clean":"2024-09-21T09:28:33.359086+0000","last_became_active":"2024-09-21T09:27:32.268026+0000","last_became_peered":"2024-09-21T09:27:32.268026+0000","last_unstale":"2024-09-21T09:28:33.359086+0000","last_undegraded":"2024-09-21T09:28:33.359086+0000","last_fullsized":"2024-09-21T09:28:33.359086+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:44:43.463552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.140711+0000","last_change":"2024-09-21T09:27:35.634025+0000","last_active":"2024-09-21T09:28:33.140711+0000","last_peered":"2024-09-21T09:28:33.140711+0000","last_clean":"2024-09-21T09:28:33.140711+0000","last_became_active":"2024-09-21T09:27:35.633746+0000","last_became_peered":"2024-09-21T09:27:35.633746+0000","last_unstale":"2024-09-21T09:28:33.140711+0000","last_undegraded":"2024-09-21T09:28:33.140711+0000","last_fullsized":"2024-09-21T09:28:33.140711+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:18:40.511292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":19,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-21T09:28:40.375561+0000","last_change":"2024-09-21T09:28:33.062917+0000","last_active":"2024-09-21T09:28:40.375561+0000","last_peered":"2024-09-21T09:28:40.375561+0000","last_clean":"2024-09-21T09:28:40.375561+0000","last_became_active":"2024-09-21T09:28:28.924106+0000","last_became_peered":"2024-09-21T09:28:28.924106+0000","last_unstale":"2024-09-21T09:28:40.375561+0000","last_undegraded":"2024-09-21T09:28:40.375561+0000","last_fullsized":"2024-09-21T09:28:40.375561+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:28:27.830318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:28:27.830318+0000","last_clean_scrub_stamp":"2024-09-21T09:28:27.830318+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:40:32.351197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00058044900000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":24,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.061847+0000","last_change":"2024-09-21T09:27:32.262645+0000","last_active":"2024-09-21T09:28:33.061847+0000","last_peered":"2024-09-21T09:28:33.061847+0000","last_clean":"2024-09-21T09:28:33.061847+0000","last_became_active":"2024-09-21T09:27:32.262373+0000","last_became_peered":"2024-09-21T09:27:32.262373+0000","last_unstale":"2024-09-21T09:28:33.061847+0000","last_undegraded":"2024-09-21T09:28:33.061847+0000","last_fullsized":"2024-09-21T09:28:33.061847+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:37:23.684561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.061597+0000","last_change":"2024-09-21T09:27:35.636105+0000","last_active":"2024-09-21T09:28:33.061597+0000","last_peered":"2024-09-21T09:28:33.061597+0000","last_clean":"2024-09-21T09:28:33.061597+0000","last_became_active":"2024-09-21T09:27:35.635805+0000","last_became_peered":"2024-09-21T09:27:35.635805+0000","last_unstale":"2024-09-21T09:28:33.061597+0000","last_undegraded":"2024-09-21T09:28:33.061597+0000","last_fullsized":"2024-09-21T09:28:33.061597+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:22:53.711130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":19,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-21T09:28:40.376066+0000","last_change":"2024-09-21T09:28:33.062726+0000","last_active":"2024-09-21T09:28:40.376066+0000","last_peered":"2024-09-21T09:28:40.376066+0000","last_clean":"2024-09-21T09:28:40.376066+0000","last_became_active":"2024-09-21T09:28:28.844698+0000","last_became_peered":"2024-09-21T09:28:28.844698+0000","last_unstale":"2024-09-21T09:28:40.376066+0000","last_undegraded":"2024-09-21T09:28:40.376066+0000","last_fullsized":"2024-09-21T09:28:40.376066+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:28:27.830318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:28:27.830318+0000","last_clean_scrub_stamp":"2024-09-21T09:28:27.830318+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:24:39.284345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00072506199999999995,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":24,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.066308+0000","last_change":"2024-09-21T09:27:32.265210+0000","last_active":"2024-09-21T09:28:33.066308+0000","last_peered":"2024-09-21T09:28:33.066308+0000","last_clean":"2024-09-21T09:28:33.066308+0000","last_became_active":"2024-09-21T09:27:32.264740+0000","last_became_peered":"2024-09-21T09:27:32.264740+0000","last_unstale":"2024-09-21T09:28:33.066308+0000","last_undegraded":"2024-09-21T09:28:33.066308+0000","last_fullsized":"2024-09-21T09:28:33.066308+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:14:01.491316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.062810+0000","last_change":"2024-09-21T09:27:35.638582+0000","last_active":"2024-09-21T09:28:33.062810+0000","last_peered":"2024-09-21T09:28:33.062810+0000","last_clean":"2024-09-21T09:28:33.062810+0000","last_became_active":"2024-09-21T09:27:35.638269+0000","last_became_peered":"2024-09-21T09:27:35.638269+0000","last_unstale":"2024-09-21T09:28:33.062810+0000","last_undegraded":"2024-09-21T09:28:33.062810+0000","last_fullsized":"2024-09-21T09:28:33.062810+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:52:20.849743+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.2","version":"66'2","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-21T09:28:40.375501+0000","last_change":"2024-09-21T09:28:33.063343+0000","last_active":"2024-09-21T09:28:40.375501+0000","last_peered":"2024-09-21T09:28:40.375501+0000","last_clean":"2024-09-21T09:28:40.375501+0000","last_became_active":"2024-09-21T09:28:28.924195+0000","last_became_peered":"2024-09-21T09:28:28.924195+0000","last_unstale":"2024-09-21T09:28:40.375501+0000","last_undegraded":"2024-09-21T09:28:40.375501+0000","last_fullsized":"2024-09-21T09:28:40.375501+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:28:27.830318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:28:27.830318+0000","last_clean_scrub_stamp":"2024-09-21T09:28:27.830318+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:33:35.119949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00080024800000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":24,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.062271+0000","last_change":"2024-09-21T09:27:32.264482+0000","last_active":"2024-09-21T09:28:33.062271+0000","last_peered":"2024-09-21T09:28:33.062271+0000","last_clean":"2024-09-21T09:28:33.062271+0000","last_became_active":"2024-09-21T09:27:32.264201+0000","last_became_peered":"2024-09-21T09:27:32.264201+0000","last_unstale":"2024-09-21T09:28:33.062271+0000","last_undegraded":"2024-09-21T09:28:33.062271+0000","last_fullsized":"2024-09-21T09:28:33.062271+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:37:15.565839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.066122+0000","last_change":"2024-09-21T09:27:35.634935+0000","last_active":"2024-09-21T09:28:33.066122+0000","last_peered":"2024-09-21T09:28:33.066122+0000","last_clean":"2024-09-21T09:28:33.066122+0000","last_became_active":"2024-09-21T09:27:35.634594+0000","last_became_peered":"2024-09-21T09:27:35.634594+0000","last_unstale":"2024-09-21T09:28:33.066122+0000","last_undegraded":"2024-09-21T09:28:33.066122+0000","last_fullsized":"2024-09-21T09:28:33.066122+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:53:03.606929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"64'1","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-21T09:28:40.922745+0000","last_change":"2024-09-21T09:28:33.141355+0000","last_active":"2024-09-21T09:28:40.922745+0000","last_peered":"2024-09-21T09:28:40.922745+0000","last_clean":"2024-09-21T09:28:40.922745+0000","last_became_active":"2024-09-21T09:28:28.923619+0000","last_became_peered":"2024-09-21T09:28:28.923619+0000","last_unstale":"2024-09-21T09:28:40.922745+0000","last_undegraded":"2024-09-21T09:28:40.922745+0000","last_fullsized":"2024-09-21T09:28:40.922745+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:28:27.830318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:28:27.830318+0000","last_clean_scrub_stamp":"2024-09-21T09:28:27.830318+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:40:50.990669+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00041745699999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":24,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.359440+0000","last_change":"2024-09-21T09:27:32.274427+0000","last_active":"2024-09-21T09:28:33.359440+0000","last_peered":"2024-09-21T09:28:33.359440+0000","last_clean":"2024-09-21T09:28:33.359440+0000","last_became_active":"2024-09-21T09:27:32.274058+0000","last_became_peered":"2024-09-21T09:27:32.274058+0000","last_unstale":"2024-09-21T09:28:33.359440+0000","last_undegraded":"2024-09-21T09:28:33.359440+0000","last_fullsized":"2024-09-21T09:28:33.359440+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:09:38.714482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.062188+0000","last_change":"2024-09-21T09:27:35.643081+0000","last_active":"2024-09-21T09:28:33.062188+0000","last_peered":"2024-09-21T09:28:33.062188+0000","last_clean":"2024-09-21T09:28:33.062188+0000","last_became_active":"2024-09-21T09:27:35.642792+0000","last_became_peered":"2024-09-21T09:27:35.642792+0000","last_unstale":"2024-09-21T09:28:33.062188+0000","last_undegraded":"2024-09-21T09:28:33.062188+0000","last_fullsized":"2024-09-21T09:28:33.062188+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:05:03.157102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":19,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-21T09:28:40.484507+0000","last_change":"2024-09-21T09:28:33.125263+0000","last_active":"2024-09-21T09:28:40.484507+0000","last_peered":"2024-09-21T09:28:40.484507+0000","last_clean":"2024-09-21T09:28:40.484507+0000","last_became_active":"2024-09-21T09:28:28.845163+0000","last_became_peered":"2024-09-21T09:28:28.845163+0000","last_unstale":"2024-09-21T09:28:40.484507+0000","last_undegraded":"2024-09-21T09:28:40.484507+0000","last_fullsized":"2024-09-21T09:28:40.484507+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:28:27.830318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:28:27.830318+0000","last_clean_scrub_stamp":"2024-09-21T09:28:27.830318+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:56:39.868147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034591199999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"57'1","reported_seq":25,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.062459+0000","last_change":"2024-09-21T09:27:32.265476+0000","last_active":"2024-09-21T09:28:33.062459+0000","last_peered":"2024-09-21T09:28:33.062459+0000","last_clean":"2024-09-21T09:28:33.062459+0000","last_became_active":"2024-09-21T09:27:32.265171+0000","last_became_peered":"2024-09-21T09:27:32.265171+0000","last_unstale":"2024-09-21T09:28:33.062459+0000","last_undegraded":"2024-09-21T09:28:33.062459+0000","last_fullsized":"2024-09-21T09:28:33.062459+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:42:48.279357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.125628+0000","last_change":"2024-09-21T09:27:35.629856+0000","last_active":"2024-09-21T09:28:33.125628+0000","last_peered":"2024-09-21T09:28:33.125628+0000","last_clean":"2024-09-21T09:28:33.125628+0000","last_became_active":"2024-09-21T09:27:35.629345+0000","last_became_peered":"2024-09-21T09:27:35.629345+0000","last_unstale":"2024-09-21T09:28:33.125628+0000","last_undegraded":"2024-09-21T09:28:33.125628+0000","last_fullsized":"2024-09-21T09:28:33.125628+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:30:14.361278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":19,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-21T09:28:40.749241+0000","last_change":"2024-09-21T09:28:33.062764+0000","last_active":"2024-09-21T09:28:40.749241+0000","last_peered":"2024-09-21T09:28:40.749241+0000","last_clean":"2024-09-21T09:28:40.749241+0000","last_became_active":"2024-09-21T09:28:28.924207+0000","last_became_peered":"2024-09-21T09:28:28.924207+0000","last_unstale":"2024-09-21T09:28:40.749241+0000","last_undegraded":"2024-09-21T09:28:40.749241+0000","last_fullsized":"2024-09-21T09:28:40.749241+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:28:27.830318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:28:27.830318+0000","last_clean_scrub_stamp":"2024-09-21T09:28:27.830318+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:08:48.406053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055597199999999995,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"57'2","reported_seq":26,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.484453+0000","last_change":"2024-09-21T09:27:32.264763+0000","last_active":"2024-09-21T09:28:33.484453+0000","last_peered":"2024-09-21T09:28:33.484453+0000","last_clean":"2024-09-21T09:28:33.484453+0000","last_became_active":"2024-09-21T09:27:32.264444+0000","last_became_peered":"2024-09-21T09:27:32.264444+0000","last_unstale":"2024-09-21T09:28:33.484453+0000","last_undegraded":"2024-09-21T09:28:33.484453+0000","last_fullsized":"2024-09-21T09:28:33.484453+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:01:52.871773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.141267+0000","last_change":"2024-09-21T09:27:35.637964+0000","last_active":"2024-09-21T09:28:33.141267+0000","last_peered":"2024-09-21T09:28:33.141267+0000","last_clean":"2024-09-21T09:28:33.141267+0000","last_became_active":"2024-09-21T09:27:35.634680+0000","last_became_peered":"2024-09-21T09:27:35.634680+0000","last_unstale":"2024-09-21T09:28:33.141267+0000","last_undegraded":"2024-09-21T09:28:33.141267+0000","last_fullsized":"2024-09-21T09:28:33.141267+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:50:36.642394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.8","version":"0'0","reported_seq":24,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.359011+0000","last_change":"2024-09-21T09:27:32.270810+0000","last_active":"2024-09-21T09:28:33.359011+0000","last_peered":"2024-09-21T09:28:33.359011+0000","last_clean":"2024-09-21T09:28:33.359011+0000","last_became_active":"2024-09-21T09:27:32.268720+0000","last_became_peered":"2024-09-21T09:27:32.268720+0000","last_unstale":"2024-09-21T09:28:33.359011+0000","last_undegraded":"2024-09-21T09:28:33.359011+0000","last_fullsized":"2024-09-21T09:28:33.359011+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:58:54.820070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.062395+0000","last_change":"2024-09-21T09:27:35.637942+0000","last_active":"2024-09-21T09:28:33.062395+0000","last_peered":"2024-09-21T09:28:33.062395+0000","last_clean":"2024-09-21T09:28:33.062395+0000","last_became_active":"2024-09-21T09:27:35.637064+0000","last_became_peered":"2024-09-21T09:27:35.637064+0000","last_unstale":"2024-09-21T09:28:33.062395+0000","last_undegraded":"2024-09-21T09:28:33.062395+0000","last_fullsized":"2024-09-21T09:28:33.062395+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:16:53.911245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.9","version":"0'0","reported_seq":24,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.062682+0000","last_change":"2024-09-21T09:27:32.262848+0000","last_active":"2024-09-21T09:28:33.062682+0000","last_peered":"2024-09-21T09:28:33.062682+0000","last_clean":"2024-09-21T09:28:33.062682+0000","last_became_active":"2024-09-21T09:27:32.262555+0000","last_became_peered":"2024-09-21T09:27:32.262555+0000","last_unstale":"2024-09-21T09:28:33.062682+0000","last_undegraded":"2024-09-21T09:28:33.062682+0000","last_fullsized":"2024-09-21T09:28:33.062682+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:27:02.670310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.125068+0000","last_change":"2024-09-21T09:27:35.625899+0000","last_active":"2024-09-21T09:28:33.125068+0000","last_peered":"2024-09-21T09:28:33.125068+0000","last_clean":"2024-09-21T09:28:33.125068+0000","last_became_active":"2024-09-21T09:27:35.625619+0000","last_became_peered":"2024-09-21T09:27:35.625619+0000","last_unstale":"2024-09-21T09:28:33.125068+0000","last_undegraded":"2024-09-21T09:28:33.125068+0000","last_fullsized":"2024-09-21T09:28:33.125068+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:38:48.577776+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.a","version":"0'0","reported_seq":24,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.063179+0000","last_change":"2024-09-21T09:27:32.264667+0000","last_active":"2024-09-21T09:28:33.063179+0000","last_peered":"2024-09-21T09:28:33.063179+0000","last_clean":"2024-09-21T09:28:33.063179+0000","last_became_active":"2024-09-21T09:27:32.264353+0000","last_became_peered":"2024-09-21T09:27:32.264353+0000","last_unstale":"2024-09-21T09:28:33.063179+0000","last_undegraded":"2024-09-21T09:28:33.063179+0000","last_fullsized":"2024-09-21T09:28:33.063179+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:21:53.938525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"0'0","reported_seq":31,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.125251+0000","last_change":"2024-09-21T09:27:52.081164+0000","last_active":"2024-09-21T09:28:33.125251+0000","last_peered":"2024-09-21T09:28:33.125251+0000","last_clean":"2024-09-21T09:28:33.125251+0000","last_became_active":"2024-09-21T09:27:52.080726+0000","last_became_peered":"2024-09-21T09:27:52.080726+0000","last_unstale":"2024-09-21T09:28:33.125251+0000","last_undegraded":"2024-09-21T09:28:33.125251+0000","last_fullsized":"2024-09-21T09:28:33.125251+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:37:01.620726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.b","version":"57'2","reported_seq":26,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.359154+0000","last_change":"2024-09-21T09:27:32.274404+0000","last_active":"2024-09-21T09:28:33.359154+0000","last_peered":"2024-09-21T09:28:33.359154+0000","last_clean":"2024-09-21T09:28:33.359154+0000","last_became_active":"2024-09-21T09:27:32.274109+0000","last_became_peered":"2024-09-21T09:27:32.274109+0000","last_unstale":"2024-09-21T09:28:33.359154+0000","last_undegraded":"2024-09-21T09:28:33.359154+0000","last_fullsized":"2024-09-21T09:28:33.359154+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:24:06.607203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":31,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.483953+0000","last_change":"2024-09-21T09:27:52.050465+0000","last_active":"2024-09-21T09:28:33.483953+0000","last_peered":"2024-09-21T09:28:33.483953+0000","last_clean":"2024-09-21T09:28:33.483953+0000","last_became_active":"2024-09-21T09:27:52.050052+0000","last_became_peered":"2024-09-21T09:27:52.050052+0000","last_unstale":"2024-09-21T09:28:33.483953+0000","last_undegraded":"2024-09-21T09:28:33.483953+0000","last_fullsized":"2024-09-21T09:28:33.483953+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:58:35.370217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1d","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.066042+0000","last_change":"2024-09-21T09:27:35.637946+0000","last_active":"2024-09-21T09:28:33.066042+0000","last_peered":"2024-09-21T09:28:33.066042+0000","last_clean":"2024-09-21T09:28:33.066042+0000","last_became_active":"2024-09-21T09:27:35.637664+0000","last_became_peered":"2024-09-21T09:27:35.637664+0000","last_unstale":"2024-09-21T09:28:33.066042+0000","last_undegraded":"2024-09-21T09:28:33.066042+0000","last_fullsized":"2024-09-21T09:28:33.066042+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:22:17.260192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1c","version":"57'2","reported_seq":26,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.359224+0000","last_change":"2024-09-21T09:27:32.268336+0000","last_active":"2024-09-21T09:28:33.359224+0000","last_peered":"2024-09-21T09:28:33.359224+0000","last_clean":"2024-09-21T09:28:33.359224+0000","last_became_active":"2024-09-21T09:27:32.268038+0000","last_became_peered":"2024-09-21T09:27:32.268038+0000","last_unstale":"2024-09-21T09:28:33.359224+0000","last_undegraded":"2024-09-21T09:28:33.359224+0000","last_fullsized":"2024-09-21T09:28:33.359224+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:57:13.991593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1c","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.066636+0000","last_change":"2024-09-21T09:27:35.642780+0000","last_active":"2024-09-21T09:28:33.066636+0000","last_peered":"2024-09-21T09:28:33.066636+0000","last_clean":"2024-09-21T09:28:33.066636+0000","last_became_active":"2024-09-21T09:27:35.642487+0000","last_became_peered":"2024-09-21T09:27:35.642487+0000","last_unstale":"2024-09-21T09:28:33.066636+0000","last_undegraded":"2024-09-21T09:28:33.066636+0000","last_fullsized":"2024-09-21T09:28:33.066636+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:28:54.864609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1d","version":"57'2","reported_seq":26,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.359353+0000","last_change":"2024-09-21T09:27:32.270398+0000","last_active":"2024-09-21T09:28:33.359353+0000","last_peered":"2024-09-21T09:28:33.359353+0000","last_clean":"2024-09-21T09:28:33.359353+0000","last_became_active":"2024-09-21T09:27:32.269469+0000","last_became_peered":"2024-09-21T09:27:32.269469+0000","last_unstale":"2024-09-21T09:28:33.359353+0000","last_undegraded":"2024-09-21T09:28:33.359353+0000","last_fullsized":"2024-09-21T09:28:33.359353+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:11:17.381607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1f","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.141385+0000","last_change":"2024-09-21T09:27:35.646662+0000","last_active":"2024-09-21T09:28:33.141385+0000","last_peered":"2024-09-21T09:28:33.141385+0000","last_clean":"2024-09-21T09:28:33.141385+0000","last_became_active":"2024-09-21T09:27:35.646141+0000","last_became_peered":"2024-09-21T09:27:35.646141+0000","last_unstale":"2024-09-21T09:28:33.141385+0000","last_undegraded":"2024-09-21T09:28:33.141385+0000","last_fullsized":"2024-09-21T09:28:33.141385+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:22:15.834922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1e","version":"57'1","reported_seq":25,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.125593+0000","last_change":"2024-09-21T09:27:32.270481+0000","last_active":"2024-09-21T09:28:33.125593+0000","last_peered":"2024-09-21T09:28:33.125593+0000","last_clean":"2024-09-21T09:28:33.125593+0000","last_became_active":"2024-09-21T09:27:32.269813+0000","last_became_peered":"2024-09-21T09:27:32.269813+0000","last_unstale":"2024-09-21T09:28:33.125593+0000","last_undegraded":"2024-09-21T09:28:33.125593+0000","last_fullsized":"2024-09-21T09:28:33.125593+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:28:41.266917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.125527+0000","last_change":"2024-09-21T09:27:35.636905+0000","last_active":"2024-09-21T09:28:33.125527+0000","last_peered":"2024-09-21T09:28:33.125527+0000","last_clean":"2024-09-21T09:28:33.125527+0000","last_became_active":"2024-09-21T09:27:35.636625+0000","last_became_peered":"2024-09-21T09:27:35.636625+0000","last_unstale":"2024-09-21T09:28:33.125527+0000","last_undegraded":"2024-09-21T09:28:33.125527+0000","last_fullsized":"2024-09-21T09:28:33.125527+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:19:39.022799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1f","version":"57'4","reported_seq":28,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:28:33.141529+0000","last_change":"2024-09-21T09:27:32.267938+0000","last_active":"2024-09-21T09:28:33.141529+0000","last_peered":"2024-09-21T09:28:33.141529+0000","last_clean":"2024-09-21T09:28:33.141529+0000","last_became_active":"2024-09-21T09:27:32.267583+0000","last_became_peered":"2024-09-21T09:27:32.267583+0000","last_unstale":"2024-09-21T09:28:33.141529+0000","last_undegraded":"2024-09-21T09:28:33.141529+0000","last_fullsized":"2024-09-21T09:28:33.141529+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:35:00.591469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":192,"acting":192,"num_store_stats":8},{"poolid":2,"num_pg":32,"stat_sum":{"num_bytes":3128,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":48,"num_write_kb":16,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":9384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":42,"ondisk_log_size":42,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":231,"num_write":113,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":79,"ondisk_log_size":79,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430240,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298616,"kb_used_data":1260,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446536,"statfs":{"total":95995035648,"available":95689252864,"internally_reserved":0,"allocated":1290240,"data_stored":900939,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:29:09 2024","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":0.54900000000000004,"15min":0.55700000000000005},"min":{"1min":0.32700000000000001,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.63100000000000001,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.495},{"interface":"front","average":{"1min":0.52800000000000002,"5min":0.54100000000000004,"15min":0.54300000000000004},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.69799999999999995,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.73499999999999999}]},{"osd":1,"last update":"Sat Sep 21 09:29:09 2024","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":0.52800000000000002,"15min":0.53300000000000003},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.66900000000000004,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.71599999999999997},{"interface":"front","average":{"1min":0.51700000000000002,"5min":0.55000000000000004,"15min":0.55600000000000005},"min":{"1min":0.36699999999999999,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.67100000000000004,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.60299999999999998}]},{"osd":2,"last update":"Sat Sep 21 09:29:09 2024","interfaces":[{"interface":"back","average":{"1min":0.47899999999999998,"5min":0.51600000000000001,"15min":0.52200000000000002},"min":{"1min":0.32400000000000001,"5min":0.307,"15min":0.307},"max":{"1min":0.66200000000000003,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.70499999999999996},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.56499999999999995,"15min":0.57099999999999995},"min":{"1min":0.378,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.68600000000000005,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.56799999999999995}]},{"osd":3,"last update":"Sat Sep 21 09:29:09 2024","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.52800000000000002,"15min":0.53000000000000003},"min":{"1min":0.372,"5min":0.315,"15min":0.315},"max":{"1min":0.70099999999999996,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.67300000000000004},{"interface":"front","average":{"1min":0.49099999999999999,"5min":0.56799999999999995,"15min":0.58099999999999996},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.69599999999999995,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.65600000000000003}]},{"osd":4,"last update":"Sat Sep 21 09:29:09 2024","interfaces":[{"interface":"back","average":{"1min":0.45300000000000001,"5min":0.48499999999999999,"15min":0.48999999999999999},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.61799999999999999,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.77400000000000002},{"interface":"front","average":{"1min":0.433,"5min":0.49099999999999999,"15min":0.501},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.69899999999999995,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.53100000000000003}]},{"osd":5,"last update":"Sat Sep 21 09:29:09 2024","interfaces":[{"interface":"back","average":{"1min":0.46999999999999997,"5min":0.48799999999999999,"15min":0.49099999999999999},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":0.68100000000000005,"5min":0.71499999999999997,"15min":0.71499999999999997},"last":0.752},{"interface":"front","average":{"1min":0.51700000000000002,"5min":0.55500000000000005,"15min":0.56100000000000005},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.67100000000000004,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.69199999999999995}]},{"osd":6,"last update":"Sat Sep 21 09:29:09 2024","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.59599999999999997,"15min":0.60599999999999998},"min":{"1min":0.38200000000000001,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.71399999999999997,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.79800000000000004},{"interface":"front","average":{"1min":0.51300000000000001,"5min":0.57699999999999996,"15min":0.58799999999999997},"min":{"1min":0.371,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.72599999999999998,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.63300000000000001}]}]},{"osd":6,"up_from":42,"seq":180388626468,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298616,"kb_used_data":1260,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446536,"statfs":{"total":95995035648,"available":95689252864,"internally_reserved":0,"allocated":1290240,"data_stored":902374,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:28:45 2024","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.53700000000000003,"15min":0.53900000000000003},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.60699999999999998},{"interface":"front","average":{"1min":0.51000000000000001,"5min":0.53000000000000003,"15min":0.53300000000000003},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.752,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.53200000000000003}]},{"osd":1,"last update":"Sat Sep 21 09:28:45 2024","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.56399999999999995,"15min":0.57099999999999995},"min":{"1min":0.38500000000000001,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.70199999999999996,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.35699999999999998},{"interface":"front","average":{"1min":0.49399999999999999,"5min":0.55600000000000005,"15min":0.56599999999999995},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.71599999999999997,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.65300000000000002}]},{"osd":2,"last update":"Sat Sep 21 09:28:45 2024","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.54100000000000004,"15min":0.54200000000000004},"min":{"1min":0.377,"5min":0.37,"15min":0.37},"max":{"1min":0.68200000000000005,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.73999999999999999},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.55000000000000004,"15min":0.55400000000000005},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.79600000000000004,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.625}]},{"osd":3,"last update":"Sat Sep 21 09:28:45 2024","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.56899999999999995,"15min":0.57699999999999996},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.66100000000000003,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.46800000000000003},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.56499999999999995,"15min":0.57199999999999995},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.754,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.69999999999999996}]},{"osd":4,"last update":"Sat Sep 21 09:28:45 2024","interfaces":[{"interface":"back","average":{"1min":0.42399999999999999,"5min":0.41199999999999998,"15min":0.40999999999999998},"min":{"1min":0.311,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.55800000000000005,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":0.57499999999999996},{"interface":"front","average":{"1min":0.41199999999999998,"5min":0.432,"15min":0.436},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.625,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.51000000000000001}]},{"osd":5,"last update":"Sat Sep 21 09:28:45 2024","interfaces":[{"interface":"back","average":{"1min":0.41499999999999998,"5min":0.44700000000000001,"15min":0.45200000000000001},"min":{"1min":0.27400000000000002,"5min":0.255,"15min":0.255},"max":{"1min":0.55100000000000005,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.71499999999999997},{"interface":"front","average":{"1min":0.41699999999999998,"5min":0.47299999999999998,"15min":0.48199999999999998},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.60499999999999998,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.56599999999999995}]},{"osd":7,"last update":"Sat Sep 21 09:29:11 2024","interfaces":[{"interface":"back","average":{"1min":0.49399999999999999,"5min":0.52100000000000002,"15min":0.52500000000000002},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.66300000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.66700000000000004},{"interface":"front","average":{"1min":0.498,"5min":0.52800000000000002,"15min":0.53300000000000003},"min":{"1min":0.33100000000000002,"5min":0.309,"15min":0.309},"max":{"1min":0.76200000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.77000000000000002}]}]},{"osd":5,"up_from":36,"seq":154618822698,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298040,"kb_used_data":684,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447112,"statfs":{"total":95995035648,"available":95689842688,"internally_reserved":0,"allocated":700416,"data_stored":311800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:29:27 2024","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.52000000000000002,"15min":0.51700000000000002},"min":{"1min":0.371,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.69299999999999995,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.76600000000000001},{"interface":"front","average":{"1min":0.505,"5min":0.52800000000000002,"15min":0.52200000000000002},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.69899999999999995,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.77800000000000002}]},{"osd":1,"last update":"Sat Sep 21 09:29:27 2024","interfaces":[{"interface":"back","average":{"1min":0.52900000000000003,"5min":0.53100000000000003,"15min":0.52700000000000002},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.70699999999999996,"5min":0.749,"15min":0.749},"last":0.83099999999999996},{"interface":"front","average":{"1min":0.502,"5min":0.53900000000000003,"15min":0.53900000000000003},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.68300000000000005,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":0.74299999999999999}]},{"osd":2,"last update":"Sat Sep 21 09:29:27 2024","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.54000000000000004,"15min":0.54600000000000004},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.73999999999999999,"5min":0.75,"15min":0.75},"last":0.81100000000000005},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.52600000000000002,"15min":0.51200000000000001},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.73999999999999999,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.79400000000000004}]},{"osd":3,"last update":"Sat Sep 21 09:29:27 2024","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.52700000000000002,"15min":0.52100000000000002},"min":{"1min":0.439,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.60299999999999998,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.71099999999999997},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.55600000000000005,"15min":0.55200000000000005},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.755,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.72399999999999998}]},{"osd":4,"last update":"Sat Sep 21 09:29:27 2024","interfaces":[{"interface":"back","average":{"1min":0.46899999999999997,"5min":0.41399999999999998,"15min":0.38900000000000001},"min":{"1min":0.32100000000000001,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.69299999999999995,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.67000000000000004},{"interface":"front","average":{"1min":0.46899999999999997,"5min":0.40300000000000002,"15min":0.377},"min":{"1min":0.33100000000000002,"5min":0.247,"15min":0.247},"max":{"1min":0.69199999999999995,"5min":0.70099999999999996,"15min":0.70099999999999996},"last":0.53700000000000003}]},{"osd":6,"last update":"Sat Sep 21 09:28:45 2024","interfaces":[{"interface":"back","average":{"1min":0.46700000000000003,"5min":0.42699999999999999,"15min":0.42099999999999999},"min":{"1min":0.33800000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.69399999999999995,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.56000000000000005},{"interface":"front","average":{"1min":0.5,"5min":0.45600000000000002,"15min":0.44800000000000001},"min":{"1min":0.36099999999999999,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.71199999999999997,"5min":0.71199999999999997,"15min":0.71199999999999997},"last":0.65100000000000002}]},{"osd":7,"last update":"Sat Sep 21 09:29:13 2024","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.51700000000000002,"15min":0.51800000000000002},"min":{"1min":0.373,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.76400000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.59999999999999998},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.54200000000000004,"15min":0.54400000000000004},"min":{"1min":0.40100000000000002,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.80800000000000005,"5min":0.877,"15min":0.877},"last":0.622}]}]},{"osd":4,"up_from":30,"seq":128849018926,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298048,"kb_used_data":692,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447104,"statfs":{"total":95995035648,"available":95689834496,"internally_reserved":0,"allocated":708608,"data_stored":310694,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:28:58 2024","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.54800000000000004,"15min":0.55200000000000005},"min":{"1min":0.33400000000000002,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.78400000000000003,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.438},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.55900000000000005,"15min":0.56100000000000005},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.73599999999999999,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.58899999999999997}]},{"osd":1,"last update":"Sat Sep 21 09:28:58 2024","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.57599999999999996,"15min":0.57999999999999996},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.57599999999999996},{"interface":"front","average":{"1min":0.505,"5min":0.53900000000000003,"15min":0.53800000000000003},"min":{"1min":0.38200000000000001,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.68700000000000006,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.47499999999999998}]},{"osd":2,"last update":"Sat Sep 21 09:28:58 2024","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.55200000000000005,"15min":0.56100000000000005},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.76100000000000001,"5min":0.91200000000000003,"15min":0.91200000000000003},"last":0.374},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.55100000000000005,"15min":0.54100000000000004},"min":{"1min":0.39900000000000002,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.72899999999999998,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.60399999999999998}]},{"osd":3,"last update":"Sat Sep 21 09:28:58 2024","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.56499999999999995,"15min":0.56299999999999994},"min":{"1min":0.40300000000000002,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.70699999999999996,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.56100000000000005},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.56599999999999995,"15min":0.56399999999999995},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.90300000000000002,"5min":0.90300000000000002,"15min":0.90300000000000002},"last":0.623}]},{"osd":5,"last update":"Sat Sep 21 09:29:31 2024","interfaces":[{"interface":"back","average":{"1min":0.46400000000000002,"5min":0.40400000000000003,"15min":0.38},"min":{"1min":0.32400000000000001,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.68899999999999995,"5min":0.69099999999999995,"15min":0.69099999999999995},"last":0.39700000000000002},{"interface":"front","average":{"1min":0.53000000000000003,"5min":0.44700000000000001,"15min":0.41499999999999998},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.77000000000000002,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.496}]},{"osd":6,"last update":"Sat Sep 21 09:28:46 2024","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.46800000000000003,"15min":0.45900000000000002},"min":{"1min":0.307,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":1.3440000000000001,"5min":1.3440000000000001,"15min":1.3440000000000001},"last":0.52000000000000002},{"interface":"front","average":{"1min":0.49099999999999999,"5min":0.47099999999999997,"15min":0.46800000000000003},"min":{"1min":0.33700000000000002,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.73099999999999998,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":0.53500000000000003}]},{"osd":7,"last update":"Sat Sep 21 09:29:12 2024","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.55300000000000005,"15min":0.54900000000000004},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.84999999999999998,"5min":0.86399999999999999,"15min":0.86399999999999999},"last":0.63500000000000001},{"interface":"front","average":{"1min":0.61699999999999999,"5min":0.57699999999999996,"15min":0.57099999999999995},"min":{"1min":0.35699999999999998,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":1.3799999999999999,"5min":1.3799999999999999,"15min":1.3799999999999999},"last":0.55100000000000005}]}]},{"osd":3,"up_from":25,"seq":107374182451,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298048,"kb_used_data":692,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447104,"statfs":{"total":95995035648,"available":95689834496,"internally_reserved":0,"allocated":708608,"data_stored":312425,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:28:40 2024","interfaces":[{"interface":"back","average":{"1min":0.376,"5min":0.36899999999999999,"15min":0.36499999999999999},"min":{"1min":0.27700000000000002,"5min":0.25,"15min":0.25},"max":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"last":0.46200000000000002},{"interface":"front","average":{"1min":0.371,"5min":0.39100000000000001,"15min":0.39000000000000001},"min":{"1min":0.25,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.56399999999999995,"5min":0.60799999999999998,"15min":0.60799999999999998},"last":0.34899999999999998}]},{"osd":1,"last update":"Sat Sep 21 09:28:40 2024","interfaces":[{"interface":"back","average":{"1min":0.40200000000000002,"5min":0.42799999999999999,"15min":0.42999999999999999},"min":{"1min":0.32100000000000001,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.54400000000000004,"5min":0.63900000000000001,"15min":0.63900000000000001},"last":0.30299999999999999},{"interface":"front","average":{"1min":0.38800000000000001,"5min":0.40699999999999997,"15min":0.41299999999999998},"min":{"1min":0.22900000000000001,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":0.59699999999999998,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.32100000000000001}]},{"osd":2,"last update":"Sat Sep 21 09:28:40 2024","interfaces":[{"interface":"back","average":{"1min":0.41699999999999998,"5min":0.438,"15min":0.443},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.52600000000000002,"5min":0.625,"15min":0.625},"last":0.27300000000000002},{"interface":"front","average":{"1min":0.40500000000000003,"5min":0.41799999999999998,"15min":0.41999999999999998},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":0.56399999999999995,"5min":0.64300000000000002,"15min":0.64300000000000002},"last":0.42299999999999999}]},{"osd":4,"last update":"Sat Sep 21 09:28:58 2024","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.55800000000000005,"15min":0.54600000000000004},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.71599999999999997,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.378},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.55600000000000005,"15min":0.54400000000000004},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.78500000000000003,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.35899999999999999}]},{"osd":5,"last update":"Sat Sep 21 09:29:26 2024","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.57999999999999996,"15min":0.58999999999999997},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.78000000000000003,"5min":0.83299999999999996,"15min":0.83299999999999996},"last":0.499},{"interface":"front","average":{"1min":0.57899999999999996,"5min":0.59699999999999998,"15min":0.60299999999999998},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.86399999999999999,"5min":0.86399999999999999,"15min":0.86399999999999999},"last":0.41099999999999998}]},{"osd":6,"last update":"Sat Sep 21 09:28:48 2024","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.60699999999999998,"15min":0.61099999999999999},"min":{"1min":0.378,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.77500000000000002,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.52400000000000002},{"interface":"front","average":{"1min":0.58199999999999996,"5min":0.61199999999999999,"15min":0.61699999999999999},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.76300000000000001,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.48799999999999999}]},{"osd":7,"last update":"Sat Sep 21 09:29:10 2024","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.59999999999999998,"15min":0.60599999999999998},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.439},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.59199999999999997,"15min":0.59799999999999998},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.75600000000000001,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.47199999999999998}]}]},{"osd":2,"up_from":18,"seq":77309411384,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298028,"kb_used_data":672,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447124,"statfs":{"total":95995035648,"available":95689854976,"internally_reserved":0,"allocated":688128,"data_stored":309979,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:29:16 2024","interfaces":[{"interface":"back","average":{"1min":0.35799999999999998,"5min":0.35999999999999999,"15min":0.36499999999999999},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"last":0.35199999999999998},{"interface":"front","average":{"1min":0.36199999999999999,"5min":0.38700000000000001,"15min":0.39200000000000002},"min":{"1min":0.245,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.68600000000000005,"5min":0.68600000000000005,"15min":0.68600000000000005},"last":0.33400000000000002}]},{"osd":1,"last update":"Sat Sep 21 09:29:16 2024","interfaces":[{"interface":"back","average":{"1min":0.35199999999999998,"5min":0.371,"15min":0.36799999999999999},"min":{"1min":0.214,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":0.50900000000000001,"5min":0.65200000000000002,"15min":0.65200000000000002},"last":0.38},{"interface":"front","average":{"1min":0.39200000000000002,"5min":0.41999999999999998,"15min":0.41399999999999998},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":0.56699999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"last":0.39300000000000002}]},{"osd":3,"last update":"Sat Sep 21 09:28:37 2024","interfaces":[{"interface":"back","average":{"1min":0.378,"5min":0.42199999999999999,"15min":0.442},"min":{"1min":0.27400000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.54100000000000004,"5min":0.61599999999999999,"15min":0.61599999999999999},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.41299999999999998,"5min":0.45500000000000002,"15min":0.46700000000000003},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":0.51400000000000001,"5min":0.65200000000000002,"15min":0.65200000000000002},"last":0.31900000000000001}]},{"osd":4,"last update":"Sat Sep 21 09:28:59 2024","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.56999999999999995,"15min":0.56799999999999995},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.80800000000000005,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.51800000000000002},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.57999999999999996,"15min":0.58699999999999997},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.78400000000000003,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.497}]},{"osd":5,"last update":"Sat Sep 21 09:29:26 2024","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.58399999999999996,"15min":0.59099999999999997},"min":{"1min":0.38200000000000001,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.79500000000000004,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.47299999999999998},{"interface":"front","average":{"1min":0.54500000000000004,"5min":0.58199999999999996,"15min":0.59199999999999997},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.84799999999999998,"5min":0.875,"15min":0.875},"last":0.443}]},{"osd":6,"last update":"Sat Sep 21 09:28:44 2024","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":0.58199999999999996,"15min":0.58099999999999996},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.84199999999999997,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":0.45200000000000001},{"interface":"front","average":{"1min":0.60799999999999998,"5min":0.61599999999999999,"15min":0.61799999999999999},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.89800000000000002,"5min":0.89800000000000002,"15min":0.89800000000000002},"last":0.54400000000000004}]},{"osd":7,"last update":"Sat Sep 21 09:29:10 2024","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.59399999999999997,"15min":0.59899999999999998},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.82699999999999996,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.621},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.59099999999999997,"15min":0.59199999999999997},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.79600000000000004,"5min":0.88700000000000001,"15min":0.88700000000000001},"last":0.52900000000000003}]}]},{"osd":1,"up_from":13,"seq":55834574909,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298040,"kb_used_data":684,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447112,"statfs":{"total":95995035648,"available":95689842688,"internally_reserved":0,"allocated":700416,"data_stored":310158,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:28:46 2024","interfaces":[{"interface":"back","average":{"1min":0.38100000000000001,"5min":0.37,"15min":0.34000000000000002},"min":{"1min":0.219,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":0.56200000000000006,"5min":0.61399999999999999,"15min":0.61399999999999999},"last":0.45800000000000002},{"interface":"front","average":{"1min":0.41699999999999998,"5min":0.40300000000000002,"15min":0.373},"min":{"1min":0.24399999999999999,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.55000000000000004,"5min":0.63,"15min":0.63},"last":0.44400000000000001}]},{"osd":2,"last update":"Sat Sep 21 09:29:18 2024","interfaces":[{"interface":"back","average":{"1min":0.38400000000000001,"5min":0.38300000000000001,"15min":0.36599999999999999},"min":{"1min":0.27900000000000003,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":0.57599999999999996,"5min":0.65200000000000002,"15min":0.65200000000000002},"last":0.379},{"interface":"front","average":{"1min":0.35199999999999998,"5min":0.378,"15min":0.372},"min":{"1min":0.26100000000000001,"5min":0.245,"15min":0.245},"max":{"1min":0.48499999999999999,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.40200000000000002}]},{"osd":3,"last update":"Sat Sep 21 09:28:37 2024","interfaces":[{"interface":"back","average":{"1min":0.376,"5min":0.40999999999999998,"15min":0.41999999999999998},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.56299999999999994,"5min":0.58799999999999997,"15min":0.58799999999999997},"last":0.34799999999999998},{"interface":"front","average":{"1min":0.41299999999999998,"5min":0.45800000000000002,"15min":0.46700000000000003},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.60699999999999998,"5min":0.76800000000000002,"15min":0.76800000000000002},"last":0.53100000000000003}]},{"osd":4,"last update":"Sat Sep 21 09:29:00 2024","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.59199999999999997,"15min":0.60399999999999998},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.78100000000000003,"5min":0.97599999999999998,"15min":0.97599999999999998},"last":0.63500000000000001},{"interface":"front","average":{"1min":0.52100000000000002,"5min":0.56699999999999995,"15min":0.57999999999999996},"min":{"1min":0.374,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.73299999999999998,"5min":0.95699999999999996,"15min":0.95699999999999996},"last":0.54800000000000004}]},{"osd":5,"last update":"Sat Sep 21 09:29:24 2024","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.61699999999999999,"15min":0.63700000000000001},"min":{"1min":0.40799999999999997,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.71699999999999997,"5min":1.014,"15min":1.014},"last":0.66100000000000003},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.57299999999999995,"15min":0.57799999999999996},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.73199999999999998,"5min":0.93500000000000005,"15min":0.93500000000000005},"last":0.56200000000000006}]},{"osd":6,"last update":"Sat Sep 21 09:28:46 2024","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.56499999999999995,"15min":0.56699999999999995},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.76300000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.69199999999999995},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.59899999999999998,"15min":0.59999999999999998},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.83899999999999997,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":0.71299999999999997}]},{"osd":7,"last update":"Sat Sep 21 09:29:10 2024","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.58099999999999996,"15min":0.58299999999999996},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.76100000000000001,"5min":0.88800000000000001,"15min":0.88800000000000001},"last":0.621},{"interface":"front","average":{"1min":0.58199999999999996,"5min":0.59599999999999997,"15min":0.59799999999999998},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":0.71899999999999997,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.64500000000000002}]}]},{"osd":0,"up_from":8,"seq":34359738433,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298620,"kb_used_data":1264,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446532,"statfs":{"total":95995035648,"available":95689248768,"internally_reserved":0,"allocated":1294336,"data_stored":900748,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 21 09:28:52 2024","interfaces":[{"interface":"back","average":{"1min":0.41299999999999998,"5min":0.34200000000000003,"15min":0.317},"min":{"1min":0.19700000000000001,"5min":0.185,"15min":0.185},"max":{"1min":1.3009999999999999,"5min":1.3009999999999999,"15min":1.3009999999999999},"last":0.39700000000000002},{"interface":"front","average":{"1min":0.38,"5min":0.35199999999999998,"15min":0.33600000000000002},"min":{"1min":0.25900000000000001,"5min":0.185,"15min":0.185},"max":{"1min":0.61299999999999999,"5min":0.625,"15min":0.625},"last":0.30199999999999999}]},{"osd":2,"last update":"Sat Sep 21 09:29:14 2024","interfaces":[{"interface":"back","average":{"1min":0.40799999999999997,"5min":0.39000000000000001,"15min":0.375},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":1.2709999999999999,"5min":1.2709999999999999,"15min":1.2709999999999999},"last":0.36499999999999999},{"interface":"front","average":{"1min":0.45400000000000001,"5min":0.432,"15min":0.42299999999999999},"min":{"1min":0.32400000000000001,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.999,"5min":0.999,"15min":0.999},"last":0.38400000000000001}]},{"osd":3,"last update":"Sat Sep 21 09:28:39 2024","interfaces":[{"interface":"back","average":{"1min":0.436,"5min":0.42499999999999999,"15min":0.42899999999999999},"min":{"1min":0.28100000000000003,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":1.0309999999999999,"5min":1.0309999999999999,"15min":1.0309999999999999},"last":0.34699999999999998},{"interface":"front","average":{"1min":0.45400000000000001,"5min":0.433,"15min":0.43099999999999999},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":1.2849999999999999,"5min":1.2849999999999999,"15min":1.2849999999999999},"last":0.32100000000000001}]},{"osd":4,"last update":"Sat Sep 21 09:28:57 2024","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":0.54600000000000004,"15min":0.52300000000000002},"min":{"1min":0.439,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.85399999999999998,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.70699999999999996},{"interface":"front","average":{"1min":0.64500000000000002,"5min":0.59299999999999997,"15min":0.57299999999999995},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":1.2549999999999999,"5min":1.2549999999999999,"15min":1.2549999999999999},"last":0.72599999999999998}]},{"osd":5,"last update":"Sat Sep 21 09:29:26 2024","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.56100000000000005,"15min":0.56100000000000005},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.74399999999999999,"5min":1.177,"15min":1.177},"last":0.54600000000000004},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.58299999999999996,"15min":0.57899999999999996},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.745,"5min":1.0549999999999999,"15min":1.0549999999999999},"last":0.68200000000000005}]},{"osd":6,"last update":"Sat Sep 21 09:28:46 2024","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.59599999999999997,"15min":0.59199999999999997},"min":{"1min":0.39700000000000002,"5min":0.371,"15min":0.371},"max":{"1min":1.073,"5min":1.073,"15min":1.073},"last":0.55600000000000005},{"interface":"front","average":{"1min":0.59799999999999998,"5min":0.57299999999999995,"15min":0.56899999999999995},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.63500000000000001}]},{"osd":7,"last update":"Sat Sep 21 09:29:08 2024","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":0.625,"15min":0.625},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":1.2270000000000001,"5min":1.2270000000000001,"15min":1.2270000000000001},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.65900000000000003,"5min":0.628,"15min":0.623},"min":{"1min":0.40699999999999997,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":1.21,"5min":1.21,"15min":1.21},"last":0.65200000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":716,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":2446,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":2432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":2342,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T09:29:39.277 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-09-21T09:29:39.277 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-09-21T09:29:39.277 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-09-21T09:29:39.277 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid d65e801e-77fa-11ef-baf6-efdc52797490 -- ceph health --format=json 2024-09-21T09:29:39.536 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:29:39.911 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:39 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:29:39.911 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:39 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:29:39.911 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:39 smithi022 ceph-mon[39525]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-21T09:29:39.912 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:39 smithi022 ceph-mon[39525]: Reconfiguring daemon prometheus.a on smithi064 2024-09-21T09:29:39.912 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:39 smithi022 ceph-mon[39525]: pgmap v46: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:29:39.912 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:39 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:29:39.912 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:39 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:29:39.912 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:39 smithi022 ceph-mon[27763]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-21T09:29:39.912 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:39 smithi022 ceph-mon[27763]: Reconfiguring daemon prometheus.a on smithi064 2024-09-21T09:29:39.912 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:39 smithi022 ceph-mon[27763]: pgmap v46: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:29:39.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:39 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:29:39.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:39 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:29:39.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:39 smithi064 ceph-mon[31124]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-21T09:29:39.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:39 smithi064 ceph-mon[31124]: Reconfiguring daemon prometheus.a on smithi064 2024-09-21T09:29:39.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:39 smithi064 ceph-mon[31124]: pgmap v46: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:29:40.630 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:29:40 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[33213]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:29:40] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-21T09:29:40.759 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 21 09:29:40 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-alertmanager-a[79862]: ts=2024-09-21T09:29:40.553Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000643946s 2024-09-21T09:29:41.523 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-09-21T09:29:41.524 INFO:teuthology.orchestra.run.smithi022.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-09-21T09:29:42.117 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:29:41 smithi064 systemd[1]: Stopping Ceph prometheus.a for d65e801e-77fa-11ef-baf6-efdc52797490... 2024-09-21T09:29:42.117 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:41 smithi064 ceph-mon[31124]: pgmap v47: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:29:42.117 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:41 smithi064 ceph-mon[31124]: from='client.? 172.21.15.22:0/2490269858' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-21T09:29:42.117 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:41 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:29:42.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:41 smithi022 ceph-mon[39525]: pgmap v47: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:29:42.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:41 smithi022 ceph-mon[39525]: from='client.? 172.21.15.22:0/2490269858' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-21T09:29:42.176 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:41 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:29:42.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:41 smithi022 ceph-mon[27763]: pgmap v47: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:29:42.176 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:41 smithi022 ceph-mon[27763]: from='client.? 172.21.15.22:0/2490269858' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-21T09:29:42.177 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:41 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:29:42.201 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-09-21T09:29:42.201 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid d65e801e-77fa-11ef-baf6-efdc52797490 -- ceph fs dump --format=json 2024-09-21T09:29:42.416 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:29:42 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[61677]: ts=2024-09-21T09:29:42.115Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-09-21T09:29:42.416 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:29:42 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[61677]: ts=2024-09-21T09:29:42.116Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-09-21T09:29:42.416 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:29:42 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[61677]: ts=2024-09-21T09:29:42.116Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-09-21T09:29:42.416 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:29:42 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[61677]: ts=2024-09-21T09:29:42.116Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-09-21T09:29:42.416 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:29:42 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[61677]: ts=2024-09-21T09:29:42.116Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-09-21T09:29:42.416 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:29:42 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[61677]: ts=2024-09-21T09:29:42.116Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-09-21T09:29:42.416 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:29:42 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[61677]: ts=2024-09-21T09:29:42.116Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-09-21T09:29:42.416 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:29:42 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[61677]: ts=2024-09-21T09:29:42.116Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-09-21T09:29:42.416 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:29:42 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[61677]: ts=2024-09-21T09:29:42.116Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-09-21T09:29:42.416 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:29:42 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[61677]: ts=2024-09-21T09:29:42.116Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-09-21T09:29:42.416 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:29:42 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[61677]: ts=2024-09-21T09:29:42.116Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-09-21T09:29:42.417 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:29:42 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[61677]: ts=2024-09-21T09:29:42.116Z caller=main.go:1147 level=info msg="See you next time!" 2024-09-21T09:29:42.417 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:29:42 smithi064 podman[65045]: 2024-09-21 09:29:42.126736988 +0000 UTC m=+0.107013159 container died 5f305effcaf8002b2da448af66a8a7486ec77a2ddcb5ce39551ad51709f6a622 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:29:42.456 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:29:42.691 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:29:42 smithi064 podman[65045]: 2024-09-21 09:29:42.513347441 +0000 UTC m=+0.493623600 container cleanup 5f305effcaf8002b2da448af66a8a7486ec77a2ddcb5ce39551ad51709f6a622 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:29:42.691 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:29:42 smithi064 bash[65045]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a 2024-09-21T09:29:43.163 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:29:42 smithi064 podman[65061]: 2024-09-21 09:29:42.781707349 +0000 UTC m=+0.653939806 container remove 5f305effcaf8002b2da448af66a8a7486ec77a2ddcb5ce39551ad51709f6a622 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:29:43.163 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:29:42 smithi064 systemd[1]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490@prometheus.a.service: Deactivated successfully. 2024-09-21T09:29:43.163 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:29:42 smithi064 systemd[1]: Stopped Ceph prometheus.a for d65e801e-77fa-11ef-baf6-efdc52797490. 2024-09-21T09:29:43.164 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:29:42 smithi064 systemd[1]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490@prometheus.a.service: Consumed 1.129s CPU time. 2024-09-21T09:29:43.164 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:29:42 smithi064 systemd[1]: Starting Ceph prometheus.a for d65e801e-77fa-11ef-baf6-efdc52797490... 2024-09-21T09:29:43.429 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:29:43 smithi064 podman[65162]: 2024-09-21 09:29:43.066070645 +0000 UTC m=+0.017230405 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-09-21T09:29:43.429 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:29:43 smithi064 podman[65162]: 2024-09-21 09:29:43.165734989 +0000 UTC m=+0.116894766 container create caee9b54c1319453eb55a86c6515efffd6936b629fd2bfe37633a259dfd353a8 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:29:43.429 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:29:43 smithi064 podman[65162]: 2024-09-21 09:29:43.376238097 +0000 UTC m=+0.327397833 container init caee9b54c1319453eb55a86c6515efffd6936b629fd2bfe37633a259dfd353a8 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:29:43.429 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:29:43 smithi064 podman[65162]: 2024-09-21 09:29:43.38100978 +0000 UTC m=+0.332169515 container start caee9b54c1319453eb55a86c6515efffd6936b629fd2bfe37633a259dfd353a8 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:29:43.429 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:29:43 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[65177]: ts=2024-09-21T09:29:43.405Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-09-21T09:29:43.430 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:29:43 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[65177]: ts=2024-09-21T09:29:43.405Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-09-21T09:29:43.430 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:29:43 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[65177]: ts=2024-09-21T09:29:43.405Z caller=main.go:570 level=info host_details="(Linux 5.14.0-508.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024 x86_64 smithi064 (none))" 2024-09-21T09:29:43.430 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:29:43 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[65177]: ts=2024-09-21T09:29:43.405Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-09-21T09:29:43.430 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:29:43 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[65177]: ts=2024-09-21T09:29:43.405Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-21T09:29:43.430 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:29:43 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[65177]: ts=2024-09-21T09:29:43.407Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-09-21T09:29:43.430 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:29:43 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[65177]: ts=2024-09-21T09:29:43.407Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-09-21T09:29:43.430 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:29:43 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[65177]: ts=2024-09-21T09:29:43.408Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-09-21T09:29:43.430 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:29:43 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[65177]: ts=2024-09-21T09:29:43.408Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-09-21T09:29:43.430 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:29:43 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[65177]: ts=2024-09-21T09:29:43.411Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-09-21T09:29:43.430 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:29:43 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[65177]: ts=2024-09-21T09:29:43.411Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.664µs 2024-09-21T09:29:43.430 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:29:43 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[65177]: ts=2024-09-21T09:29:43.411Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-09-21T09:29:43.430 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:29:43 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[65177]: ts=2024-09-21T09:29:43.411Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=1 2024-09-21T09:29:43.430 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:29:43 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[65177]: ts=2024-09-21T09:29:43.411Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=1 2024-09-21T09:29:43.430 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:29:43 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[65177]: ts=2024-09-21T09:29:43.411Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=20.962µs wal_replay_duration=480.588µs wbl_replay_duration=132ns total_replay_duration=520.19µs 2024-09-21T09:29:43.431 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:29:43 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[65177]: ts=2024-09-21T09:29:43.412Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-09-21T09:29:43.431 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:29:43 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[65177]: ts=2024-09-21T09:29:43.412Z caller=main.go:1029 level=info msg="TSDB started" 2024-09-21T09:29:43.431 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:29:43 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[65177]: ts=2024-09-21T09:29:43.412Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-21T09:29:43.431 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:29:43 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[65177]: ts=2024-09-21T09:29:43.427Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=15.432766ms db_storage=714ns remote_storage=1.219µs web_handler=364ns query_engine=563ns scrape=399.754µs scrape_sd=27.27µs notify=34.214µs notify_sd=9.498µs rules=14.732205ms tracing=4.565µs 2024-09-21T09:29:43.431 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:29:43 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[65177]: ts=2024-09-21T09:29:43.427Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-09-21T09:29:43.431 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:29:43 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[65177]: ts=2024-09-21T09:29:43.427Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-09-21T09:29:43.691 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:29:43 smithi064 bash[65162]: caee9b54c1319453eb55a86c6515efffd6936b629fd2bfe37633a259dfd353a8 2024-09-21T09:29:43.691 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:29:43 smithi064 systemd[1]: Started Ceph prometheus.a for d65e801e-77fa-11ef-baf6-efdc52797490. 2024-09-21T09:29:43.779 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:29:43 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: [21/Sep/2024:09:29:43] ENGINE Bus STOPPING 2024-09-21T09:29:43.779 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:29:43 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: [21/Sep/2024:09:29:43] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-21T09:29:43.780 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:29:43 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: [21/Sep/2024:09:29:43] ENGINE Bus STOPPED 2024-09-21T09:29:43.780 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:29:43 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: [21/Sep/2024:09:29:43] ENGINE Bus STARTING 2024-09-21T09:29:44.030 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:29:43 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: [21/Sep/2024:09:29:43] ENGINE Serving on http://:::9283 2024-09-21T09:29:44.030 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:29:43 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: [21/Sep/2024:09:29:43] ENGINE Bus STARTED 2024-09-21T09:29:44.030 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:29:43 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: [21/Sep/2024:09:29:43] ENGINE Bus STOPPING 2024-09-21T09:29:44.030 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:43 smithi022 ceph-mon[27763]: pgmap v48: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:29:44.030 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:43 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:29:44.031 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:43 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:29:44.031 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:43 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-21T09:29:44.031 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:43 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi022.front.sepia.ceph.com:9093"}]: dispatch 2024-09-21T09:29:44.031 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:43 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:29:44.031 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:43 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T09:29:44.031 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:43 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi064.front.sepia.ceph.com:9095"}]: dispatch 2024-09-21T09:29:44.031 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:43 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:29:44.031 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:43 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T09:29:44.031 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:43 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi064.front.sepia.ceph.com:3000"}]: dispatch 2024-09-21T09:29:44.031 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:43 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:29:44.031 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:43 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:29:44.031 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:43 smithi022 ceph-mon[39525]: pgmap v48: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:29:44.032 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:43 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:29:44.032 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:43 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:29:44.032 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:43 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-21T09:29:44.032 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:43 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi022.front.sepia.ceph.com:9093"}]: dispatch 2024-09-21T09:29:44.032 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:43 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:29:44.032 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:43 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T09:29:44.032 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:43 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi064.front.sepia.ceph.com:9095"}]: dispatch 2024-09-21T09:29:44.032 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:43 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:29:44.032 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:43 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T09:29:44.032 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:43 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi064.front.sepia.ceph.com:3000"}]: dispatch 2024-09-21T09:29:44.032 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:43 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:29:44.032 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:43 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:29:44.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:43 smithi064 ceph-mon[31124]: pgmap v48: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:29:44.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:43 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:29:44.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:43 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:29:44.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:43 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-21T09:29:44.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:43 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi022.front.sepia.ceph.com:9093"}]: dispatch 2024-09-21T09:29:44.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:43 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:29:44.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:43 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T09:29:44.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:43 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi064.front.sepia.ceph.com:9095"}]: dispatch 2024-09-21T09:29:44.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:43 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:29:44.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:43 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T09:29:44.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:43 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi064.front.sepia.ceph.com:3000"}]: dispatch 2024-09-21T09:29:44.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:43 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:29:44.193 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:43 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:29:44.306 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:29:44 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: [21/Sep/2024:09:29:44] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-21T09:29:44.307 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:29:44 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: [21/Sep/2024:09:29:44] ENGINE Bus STOPPED 2024-09-21T09:29:44.307 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:29:44 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: [21/Sep/2024:09:29:44] ENGINE Bus STARTING 2024-09-21T09:29:44.320 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-09-21T09:29:44.320 INFO:teuthology.orchestra.run.smithi022.stdout:{"epoch":8,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14424,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.22:6835/3558312740","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6834","nonce":3558312740},{"type":"v1","addr":"172.21.15.22:6835","nonce":3558312740}]},"join_fscid":-1,"export_targets":[],"features":4540138320759226367,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":2}],"filesystems":[{"mdsmap":{"epoch":8,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-09-21T09:27:37.584397+0000","modified":"2024-09-21T09:27:50.029809+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"max_mds":1,"in":[0],"up":{"mds_0":24325},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24325":{"gid":24325,"name":"b","rank":0,"incarnation":4,"state":"up:active","state_seq":5,"addr":"172.21.15.64:6833/3589447551","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6832","nonce":3589447551},{"type":"v1","addr":"172.21.15.64:6833","nonce":3589447551}]},"join_fscid":-1,"export_targets":[],"features":4540138320759226367,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","standby_count_wanted":0},"id":1}]} 2024-09-21T09:29:44.320 INFO:teuthology.orchestra.run.smithi022.stderr:dumped fsmap epoch 8 2024-09-21T09:29:44.558 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:29:44 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: [21/Sep/2024:09:29:44] ENGINE Serving on http://:::9283 2024-09-21T09:29:44.558 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:29:44 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: [21/Sep/2024:09:29:44] ENGINE Bus STARTED 2024-09-21T09:29:44.558 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:29:44 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: [21/Sep/2024:09:29:44] ENGINE Bus STOPPING 2024-09-21T09:29:45.057 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:44 smithi022 ceph-mon[39525]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-21T09:29:45.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:44 smithi022 ceph-mon[39525]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi022.front.sepia.ceph.com:9093"}]: dispatch 2024-09-21T09:29:45.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:44 smithi022 ceph-mon[39525]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T09:29:45.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:44 smithi022 ceph-mon[39525]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi064.front.sepia.ceph.com:9095"}]: dispatch 2024-09-21T09:29:45.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:44 smithi022 ceph-mon[39525]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T09:29:45.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:44 smithi022 ceph-mon[39525]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi064.front.sepia.ceph.com:3000"}]: dispatch 2024-09-21T09:29:45.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:44 smithi022 ceph-mon[39525]: from='client.? 172.21.15.22:0/2834637693' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-21T09:29:45.058 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:29:44 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: [21/Sep/2024:09:29:44] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-21T09:29:45.058 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:29:44 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: [21/Sep/2024:09:29:44] ENGINE Bus STOPPED 2024-09-21T09:29:45.058 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:29:44 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: [21/Sep/2024:09:29:44] ENGINE Bus STARTING 2024-09-21T09:29:45.059 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:29:45 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: [21/Sep/2024:09:29:45] ENGINE Serving on http://:::9283 2024-09-21T09:29:45.059 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:29:45 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: [21/Sep/2024:09:29:45] ENGINE Bus STARTED 2024-09-21T09:29:45.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:44 smithi022 ceph-mon[27763]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-21T09:29:45.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:44 smithi022 ceph-mon[27763]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi022.front.sepia.ceph.com:9093"}]: dispatch 2024-09-21T09:29:45.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:44 smithi022 ceph-mon[27763]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T09:29:45.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:44 smithi022 ceph-mon[27763]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi064.front.sepia.ceph.com:9095"}]: dispatch 2024-09-21T09:29:45.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:44 smithi022 ceph-mon[27763]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T09:29:45.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:44 smithi022 ceph-mon[27763]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi064.front.sepia.ceph.com:3000"}]: dispatch 2024-09-21T09:29:45.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:44 smithi022 ceph-mon[27763]: from='client.? 172.21.15.22:0/2834637693' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-21T09:29:45.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:44 smithi064 ceph-mon[31124]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-21T09:29:45.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:44 smithi064 ceph-mon[31124]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi022.front.sepia.ceph.com:9093"}]: dispatch 2024-09-21T09:29:45.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:44 smithi064 ceph-mon[31124]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T09:29:45.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:44 smithi064 ceph-mon[31124]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi064.front.sepia.ceph.com:9095"}]: dispatch 2024-09-21T09:29:45.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:44 smithi064 ceph-mon[31124]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T09:29:45.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:44 smithi064 ceph-mon[31124]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi064.front.sepia.ceph.com:3000"}]: dispatch 2024-09-21T09:29:45.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:44 smithi064 ceph-mon[31124]: from='client.? 172.21.15.22:0/2834637693' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-21T09:29:45.303 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid d65e801e-77fa-11ef-baf6-efdc52797490 -- ceph fs dump --format=json 2024-09-21T09:29:45.649 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:29:46.025 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:45 smithi022 ceph-mon[39525]: pgmap v49: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:29:46.025 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:45 smithi022 ceph-mon[27763]: pgmap v49: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:29:46.162 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:45 smithi064 ceph-mon[31124]: pgmap v49: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:29:47.508 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-09-21T09:29:47.508 INFO:teuthology.orchestra.run.smithi022.stdout:{"epoch":8,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14424,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.22:6835/3558312740","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6834","nonce":3558312740},{"type":"v1","addr":"172.21.15.22:6835","nonce":3558312740}]},"join_fscid":-1,"export_targets":[],"features":4540138320759226367,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":2}],"filesystems":[{"mdsmap":{"epoch":8,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-09-21T09:27:37.584397+0000","modified":"2024-09-21T09:27:50.029809+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"max_mds":1,"in":[0],"up":{"mds_0":24325},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24325":{"gid":24325,"name":"b","rank":0,"incarnation":4,"state":"up:active","state_seq":5,"addr":"172.21.15.64:6833/3589447551","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6832","nonce":3589447551},{"type":"v1","addr":"172.21.15.64:6833","nonce":3589447551}]},"join_fscid":-1,"export_targets":[],"features":4540138320759226367,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","standby_count_wanted":0},"id":1}]} 2024-09-21T09:29:47.508 INFO:teuthology.orchestra.run.smithi022.stderr:dumped fsmap epoch 8 2024-09-21T09:29:48.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:48 smithi022 ceph-mon[39525]: pgmap v50: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:29:48.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:48 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:29:48.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:48 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:29:48.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:48 smithi022 ceph-mon[39525]: from='client.? 172.21.15.22:0/989867303' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-21T09:29:48.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:48 smithi022 ceph-mon[27763]: pgmap v50: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:29:48.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:48 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:29:48.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:48 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:29:48.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:48 smithi022 ceph-mon[27763]: from='client.? 172.21.15.22:0/989867303' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-21T09:29:48.694 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:48 smithi064 ceph-mon[31124]: pgmap v50: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:29:48.694 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:48 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:29:48.694 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:48 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:29:48.694 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:48 smithi064 ceph-mon[31124]: from='client.? 172.21.15.22:0/989867303' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-21T09:29:48.808 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 21 09:29:48 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-alertmanager-a[79862]: ts=2024-09-21T09:29:48.556Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.003757541s 2024-09-21T09:29:48.912 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid d65e801e-77fa-11ef-baf6-efdc52797490 -- ceph osd dump --format=json 2024-09-21T09:29:49.587 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:29:50.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:50 smithi022 ceph-mon[27763]: pgmap v51: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:29:50.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:50 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:29:50.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:50 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:29:50.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:50 smithi022 ceph-mon[39525]: pgmap v51: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:29:50.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:50 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:29:50.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:50 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:29:50.461 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:50 smithi064 ceph-mon[31124]: pgmap v51: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:29:50.461 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:50 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:29:50.461 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:50 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:29:50.826 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:29:50 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[33213]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:29:50] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-21T09:29:51.351 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-09-21T09:29:51.351 INFO:teuthology.orchestra.run.smithi022.stdout:{"epoch":67,"fsid":"d65e801e-77fa-11ef-baf6-efdc52797490","created":"2024-09-21T09:22:04.474917+0000","modified":"2024-09-21T09:28:40.372343+0000","last_up_change":"2024-09-21T09:27:06.441515+0000","last_in_change":"2024-09-21T09:26:49.376072+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-21T09:25:07.405775+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-09-21T09:27:31.151960+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"57","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-09-21T09:27:34.558875+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"57","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}}},{"pool":4,"pool_name":"rbd","create_time":"2024-09-21T09:28:27.564651+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"66","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":66,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"a1fbb74d-ffae-4d5f-81ab-9677d258a95c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6802","nonce":3576301437},{"type":"v1","addr":"172.21.15.22:6803","nonce":3576301437}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6804","nonce":3576301437},{"type":"v1","addr":"172.21.15.22:6805","nonce":3576301437}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6808","nonce":3576301437},{"type":"v1","addr":"172.21.15.22:6809","nonce":3576301437}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6806","nonce":3576301437},{"type":"v1","addr":"172.21.15.22:6807","nonce":3576301437}]},"public_addr":"172.21.15.22:6803/3576301437","cluster_addr":"172.21.15.22:6805/3576301437","heartbeat_back_addr":"172.21.15.22:6809/3576301437","heartbeat_front_addr":"172.21.15.22:6807/3576301437","state":["exists","up"]},{"osd":1,"uuid":"fa4c9385-ac93-4983-92b4-f23f0a3731db","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6810","nonce":3562679765},{"type":"v1","addr":"172.21.15.22:6811","nonce":3562679765}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6812","nonce":3562679765},{"type":"v1","addr":"172.21.15.22:6813","nonce":3562679765}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6816","nonce":3562679765},{"type":"v1","addr":"172.21.15.22:6817","nonce":3562679765}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6814","nonce":3562679765},{"type":"v1","addr":"172.21.15.22:6815","nonce":3562679765}]},"public_addr":"172.21.15.22:6811/3562679765","cluster_addr":"172.21.15.22:6813/3562679765","heartbeat_back_addr":"172.21.15.22:6817/3562679765","heartbeat_front_addr":"172.21.15.22:6815/3562679765","state":["exists","up"]},{"osd":2,"uuid":"1c27d056-2218-47dd-8f13-bc1c13888483","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6818","nonce":695486895},{"type":"v1","addr":"172.21.15.22:6819","nonce":695486895}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6820","nonce":695486895},{"type":"v1","addr":"172.21.15.22:6821","nonce":695486895}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6824","nonce":695486895},{"type":"v1","addr":"172.21.15.22:6825","nonce":695486895}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6822","nonce":695486895},{"type":"v1","addr":"172.21.15.22:6823","nonce":695486895}]},"public_addr":"172.21.15.22:6819/695486895","cluster_addr":"172.21.15.22:6821/695486895","heartbeat_back_addr":"172.21.15.22:6825/695486895","heartbeat_front_addr":"172.21.15.22:6823/695486895","state":["exists","up"]},{"osd":3,"uuid":"8668a232-df4a-46ba-b97d-3ed2f6ddeab2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6826","nonce":1394296125},{"type":"v1","addr":"172.21.15.22:6827","nonce":1394296125}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6828","nonce":1394296125},{"type":"v1","addr":"172.21.15.22:6829","nonce":1394296125}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6832","nonce":1394296125},{"type":"v1","addr":"172.21.15.22:6833","nonce":1394296125}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6830","nonce":1394296125},{"type":"v1","addr":"172.21.15.22:6831","nonce":1394296125}]},"public_addr":"172.21.15.22:6827/1394296125","cluster_addr":"172.21.15.22:6829/1394296125","heartbeat_back_addr":"172.21.15.22:6833/1394296125","heartbeat_front_addr":"172.21.15.22:6831/1394296125","state":["exists","up"]},{"osd":4,"uuid":"44dd8dcc-dbf7-41f7-9ec2-e66922e6c521","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2736095169},{"type":"v1","addr":"172.21.15.64:6801","nonce":2736095169}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6802","nonce":2736095169},{"type":"v1","addr":"172.21.15.64:6803","nonce":2736095169}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6806","nonce":2736095169},{"type":"v1","addr":"172.21.15.64:6807","nonce":2736095169}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":2736095169},{"type":"v1","addr":"172.21.15.64:6805","nonce":2736095169}]},"public_addr":"172.21.15.64:6801/2736095169","cluster_addr":"172.21.15.64:6803/2736095169","heartbeat_back_addr":"172.21.15.64:6807/2736095169","heartbeat_front_addr":"172.21.15.64:6805/2736095169","state":["exists","up"]},{"osd":5,"uuid":"a920292e-6301-4b67-b2a1-30f4788f8fd3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":2673197529},{"type":"v1","addr":"172.21.15.64:6809","nonce":2673197529}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2673197529},{"type":"v1","addr":"172.21.15.64:6811","nonce":2673197529}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2673197529},{"type":"v1","addr":"172.21.15.64:6815","nonce":2673197529}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6812","nonce":2673197529},{"type":"v1","addr":"172.21.15.64:6813","nonce":2673197529}]},"public_addr":"172.21.15.64:6809/2673197529","cluster_addr":"172.21.15.64:6811/2673197529","heartbeat_back_addr":"172.21.15.64:6815/2673197529","heartbeat_front_addr":"172.21.15.64:6813/2673197529","state":["exists","up"]},{"osd":6,"uuid":"ed01fe25-9b7d-48b9-b054-e054280e5826","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1977044182},{"type":"v1","addr":"172.21.15.64:6817","nonce":1977044182}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1977044182},{"type":"v1","addr":"172.21.15.64:6819","nonce":1977044182}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1977044182},{"type":"v1","addr":"172.21.15.64:6823","nonce":1977044182}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1977044182},{"type":"v1","addr":"172.21.15.64:6821","nonce":1977044182}]},"public_addr":"172.21.15.64:6817/1977044182","cluster_addr":"172.21.15.64:6819/1977044182","heartbeat_back_addr":"172.21.15.64:6823/1977044182","heartbeat_front_addr":"172.21.15.64:6821/1977044182","state":["exists","up"]},{"osd":7,"uuid":"a9433be2-fcc7-46f5-91ee-13fbe71bf2f7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":4274465032},{"type":"v1","addr":"172.21.15.64:6825","nonce":4274465032}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":4274465032},{"type":"v1","addr":"172.21.15.64:6827","nonce":4274465032}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":4274465032},{"type":"v1","addr":"172.21.15.64:6831","nonce":4274465032}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":4274465032},{"type":"v1","addr":"172.21.15.64:6829","nonce":4274465032}]},"public_addr":"172.21.15.64:6825/4274465032","cluster_addr":"172.21.15.64:6827/4274465032","heartbeat_back_addr":"172.21.15.64:6831/4274465032","heartbeat_front_addr":"172.21.15.64:6829/4274465032","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:24:15.825542+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:24:40.497779+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:25:04.918091+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:25:30.663725+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:25:53.521949+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:26:17.565302+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:26:40.876016+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:27:04.436407+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.3","mappings":[{"from":7,"to":4}]},{"pgid":"3.a","mappings":[{"from":4,"to":3}]},{"pgid":"3.b","mappings":[{"from":0,"to":2}]},{"pgid":"3.17","mappings":[{"from":0,"to":2}]},{"pgid":"3.22","mappings":[{"from":0,"to":1}]},{"pgid":"3.24","mappings":[{"from":4,"to":2}]},{"pgid":"3.37","mappings":[{"from":5,"to":1}]}],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.22:6801/2706373818":"2024-09-22T09:28:26.777111+0000","172.21.15.22:0/3249188488":"2024-09-22T09:28:26.777111+0000","172.21.15.22:6800/2983422368":"2024-09-22T09:22:26.473640+0000","172.21.15.22:6801/2983422368":"2024-09-22T09:22:26.473640+0000","172.21.15.22:6800/2706373818":"2024-09-22T09:28:26.777111+0000","172.21.15.22:0/1022854075":"2024-09-22T09:22:26.473640+0000","172.21.15.22:0/3766882712":"2024-09-22T09:22:50.910682+0000","172.21.15.22:0/1728251114":"2024-09-22T09:22:26.473640+0000","172.21.15.22:0/2669480478":"2024-09-22T09:22:50.910682+0000","172.21.15.22:0/1467628521":"2024-09-22T09:22:26.473640+0000","172.21.15.22:0/4200462415":"2024-09-22T09:22:50.910682+0000","172.21.15.22:6800/2821000087":"2024-09-22T09:22:50.910682+0000","172.21.15.22:6801/2821000087":"2024-09-22T09:22:50.910682+0000","172.21.15.22:0/4254041618":"2024-09-22T09:28:26.777111+0000","172.21.15.22:0/1998705954":"2024-09-22T09:28:26.777111+0000","172.21.15.22:0/1937461032":"2024-09-22T09:28:26.777111+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[{"pool":4,"snaps":[{"begin":2,"length":1}]}],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-21T09:29:51.541 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:51 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:29:51.541 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:51 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:29:51.541 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:51 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:29:51.541 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:51 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:29:51.542 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:29:51 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:29:51] "GET /metrics HTTP/1.1" 200 236473 "" "Prometheus/2.43.0" 2024-09-21T09:29:51.542 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:51 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:29:51.542 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:51 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:29:51.542 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:51 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:29:51.542 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:51 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:29:51.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:51 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:29:51.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:51 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:29:51.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:51 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:29:51.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:51 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:29:52.224 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid d65e801e-77fa-11ef-baf6-efdc52797490 -- ceph fs dump --format=json 2024-09-21T09:29:52.472 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:52 smithi064 ceph-mon[31124]: pgmap v52: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:29:52.472 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:52 smithi064 ceph-mon[31124]: from='client.? 172.21.15.22:0/3304947708' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T09:29:52.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:52 smithi022 ceph-mon[39525]: pgmap v52: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:29:52.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:52 smithi022 ceph-mon[39525]: from='client.? 172.21.15.22:0/3304947708' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T09:29:52.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:52 smithi022 ceph-mon[27763]: pgmap v52: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:29:52.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:52 smithi022 ceph-mon[27763]: from='client.? 172.21.15.22:0/3304947708' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T09:29:52.674 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:29:54.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:54 smithi022 ceph-mon[39525]: pgmap v53: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:29:54.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:54 smithi022 ceph-mon[27763]: pgmap v53: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:29:54.594 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-09-21T09:29:54.594 INFO:teuthology.orchestra.run.smithi022.stdout:{"epoch":8,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14424,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.22:6835/3558312740","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6834","nonce":3558312740},{"type":"v1","addr":"172.21.15.22:6835","nonce":3558312740}]},"join_fscid":-1,"export_targets":[],"features":4540138320759226367,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":2}],"filesystems":[{"mdsmap":{"epoch":8,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-09-21T09:27:37.584397+0000","modified":"2024-09-21T09:27:50.029809+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"max_mds":1,"in":[0],"up":{"mds_0":24325},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24325":{"gid":24325,"name":"b","rank":0,"incarnation":4,"state":"up:active","state_seq":5,"addr":"172.21.15.64:6833/3589447551","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6832","nonce":3589447551},{"type":"v1","addr":"172.21.15.64:6833","nonce":3589447551}]},"join_fscid":-1,"export_targets":[],"features":4540138320759226367,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","standby_count_wanted":0},"id":1}]} 2024-09-21T09:29:54.595 INFO:teuthology.orchestra.run.smithi022.stderr:dumped fsmap epoch 8 2024-09-21T09:29:54.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:54 smithi064 ceph-mon[31124]: pgmap v53: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:29:55.305 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 8, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-09-21T09:27:37.584397+0000', 'modified': '2024-09-21T09:27:50.029809+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2'}}, 'max_mds': 1, 'in': [0], 'up': {'mds_0': 24325}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_24325': {'gid': 24325, 'name': 'b', 'rank': 0, 'incarnation': 4, 'state': 'up:active', 'state_seq': 5, 'addr': '172.21.15.64:6833/3589447551', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.64:6832', 'nonce': 3589447551}, {'type': 'v1', 'addr': '172.21.15.64:6833', 'nonce': 3589447551}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540138320759226367, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2'}}}}, 'data_pools': [3], 'metadata_pool': 2, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'standby_count_wanted': 0} 2024-09-21T09:29:55.305 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: 1/1 2024-09-21T09:29:55.305 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid d65e801e-77fa-11ef-baf6-efdc52797490 -- ceph tell mds.b status 2024-09-21T09:29:55.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:55 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:29:55.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:55 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:29:55.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:55 smithi022 ceph-mon[27763]: from='client.? 172.21.15.22:0/3419665507' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-21T09:29:55.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:55 smithi022 ceph-mon[27763]: pgmap v54: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:29:55.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:55 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:29:55.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:55 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:29:55.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:55 smithi022 ceph-mon[39525]: from='client.? 172.21.15.22:0/3419665507' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-21T09:29:55.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:55 smithi022 ceph-mon[39525]: pgmap v54: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:29:55.704 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:29:55.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:55 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:29:55.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:55 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:29:55.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:55 smithi064 ceph-mon[31124]: from='client.? 172.21.15.22:0/3419665507' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-21T09:29:55.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:55 smithi064 ceph-mon[31124]: pgmap v54: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:29:57.926 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:57 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:29:57.926 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:57 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:29:57.926 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:57 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:29:57.926 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:57 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:29:57.926 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:57 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:29:57.926 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:57 smithi022 ceph-mon[27763]: pgmap v55: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:29:57.926 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:57 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:29:57.926 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:57 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:29:57.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:57 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:29:57.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:57 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:29:57.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:57 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:29:57.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:57 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:29:57.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:57 smithi022 ceph-mon[39525]: pgmap v55: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:29:57.927 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:57 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:29:57.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:57 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:29:57.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:57 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:29:57.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:57 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:29:57.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:57 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:29:57.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:57 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:29:57.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:57 smithi064 ceph-mon[31124]: pgmap v55: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:29:57.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:57 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:29:58.126 INFO:teuthology.orchestra.run.smithi022.stderr:2024-09-21T09:29:58.123+0000 7ff48affd640 0 client.14769 ms_handle_reset on v2:172.21.15.64:6832/3589447551 2024-09-21T09:29:58.144 INFO:teuthology.orchestra.run.smithi022.stderr:2024-09-21T09:29:58.142+0000 7ff48affd640 0 client.24680 ms_handle_reset on v2:172.21.15.64:6832/3589447551 2024-09-21T09:29:58.144 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-09-21T09:29:58.144 INFO:teuthology.orchestra.run.smithi022.stdout: "cluster_fsid": "d65e801e-77fa-11ef-baf6-efdc52797490", 2024-09-21T09:29:58.144 INFO:teuthology.orchestra.run.smithi022.stdout: "whoami": 0, 2024-09-21T09:29:58.144 INFO:teuthology.orchestra.run.smithi022.stdout: "id": 24325, 2024-09-21T09:29:58.144 INFO:teuthology.orchestra.run.smithi022.stdout: "want_state": "up:active", 2024-09-21T09:29:58.144 INFO:teuthology.orchestra.run.smithi022.stdout: "state": "up:active", 2024-09-21T09:29:58.144 INFO:teuthology.orchestra.run.smithi022.stdout: "fs_name": "cephfs", 2024-09-21T09:29:58.144 INFO:teuthology.orchestra.run.smithi022.stdout: "rank_uptime": 140.51625586399999, 2024-09-21T09:29:58.145 INFO:teuthology.orchestra.run.smithi022.stdout: "mdsmap_epoch": 8, 2024-09-21T09:29:58.145 INFO:teuthology.orchestra.run.smithi022.stdout: "osdmap_epoch": 67, 2024-09-21T09:29:58.145 INFO:teuthology.orchestra.run.smithi022.stdout: "osdmap_epoch_barrier": 57, 2024-09-21T09:29:58.145 INFO:teuthology.orchestra.run.smithi022.stdout: "uptime": 156.39798995000001 2024-09-21T09:29:58.145 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-09-21T09:29:58.681 INFO:tasks.cephadm:Setup complete, yielding 2024-09-21T09:29:58.681 INFO:teuthology.run_tasks:Running task print... 2024-09-21T09:29:58.690 INFO:teuthology.task.print:**** done end installing quincy cephadm ... 2024-09-21T09:29:58.691 INFO:teuthology.run_tasks:Running task print... 2024-09-21T09:29:58.705 INFO:teuthology.task.print:**** done start cephadm.shell ceph config set mgr... 2024-09-21T09:29:58.705 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-21T09:29:58.714 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi022.front.sepia.ceph.com 2024-09-21T09:29:58.714 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d65e801e-77fa-11ef-baf6-efdc52797490 -- bash -c 'ceph config set mgr mgr/cephadm/use_repo_digest true --force' 2024-09-21T09:29:58.976 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:30:00.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:59 smithi022 ceph-mon[27763]: pgmap v56: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-21T09:30:00.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:29:59 smithi022 ceph-mon[27763]: mgrmap e21: y(active, since 92s), standbys: x 2024-09-21T09:30:00.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:59 smithi022 ceph-mon[39525]: pgmap v56: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-21T09:30:00.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:29:59 smithi022 ceph-mon[39525]: mgrmap e21: y(active, since 92s), standbys: x 2024-09-21T09:30:00.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:59 smithi064 ceph-mon[31124]: pgmap v56: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-21T09:30:00.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:29:59 smithi064 ceph-mon[31124]: mgrmap e21: y(active, since 92s), standbys: x 2024-09-21T09:30:00.865 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:30:00 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[33213]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:30:00] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-21T09:30:01.002 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:30:00 smithi022 ceph-mon[39525]: overall HEALTH_OK 2024-09-21T09:30:01.003 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:30:00 smithi022 ceph-mon[39525]: from='client.? 172.21.15.22:0/1244413651' entity='client.admin' 2024-09-21T09:30:01.003 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:30:00 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:30:01.003 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:30:00 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:30:01.003 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:30:00 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:30:01.003 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:30:00 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:30:01.003 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:30:00 smithi022 ceph-mon[27763]: overall HEALTH_OK 2024-09-21T09:30:01.003 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:30:00 smithi022 ceph-mon[27763]: from='client.? 172.21.15.22:0/1244413651' entity='client.admin' 2024-09-21T09:30:01.004 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:30:00 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:30:01.004 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:30:00 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:30:01.004 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:30:00 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:30:01.004 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:30:00 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:30:01.004 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:30:00 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: [21/Sep/2024:09:30:00] ENGINE Bus STOPPING 2024-09-21T09:30:01.004 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:30:00 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: [21/Sep/2024:09:30:00] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-21T09:30:01.004 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:30:00 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: [21/Sep/2024:09:30:00] ENGINE Bus STOPPED 2024-09-21T09:30:01.004 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:30:00 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: [21/Sep/2024:09:30:00] ENGINE Bus STARTING 2024-09-21T09:30:01.004 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:30:00 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: [21/Sep/2024:09:30:00] ENGINE Serving on http://:::9283 2024-09-21T09:30:01.004 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:30:00 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: [21/Sep/2024:09:30:00] ENGINE Bus STARTED 2024-09-21T09:30:01.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:30:00 smithi064 ceph-mon[31124]: overall HEALTH_OK 2024-09-21T09:30:01.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:30:00 smithi064 ceph-mon[31124]: from='client.? 172.21.15.22:0/1244413651' entity='client.admin' 2024-09-21T09:30:01.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:30:00 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:30:01.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:30:00 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:30:01.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:30:00 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:30:01.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:30:00 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:30:01.408 INFO:teuthology.run_tasks:Running task print... 2024-09-21T09:30:01.417 INFO:teuthology.task.print:**** done cephadm.shell ceph config set mgr... 2024-09-21T09:30:01.417 INFO:teuthology.run_tasks:Running task print... 2024-09-21T09:30:01.523 INFO:teuthology.task.print:**** done start telemetry quincy... 2024-09-21T09:30:01.523 INFO:teuthology.run_tasks:Running task workunit... 2024-09-21T09:30:01.536 INFO:tasks.workunit:Pulling workunits from ref 62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:30:01.536 INFO:tasks.workunit:Making a separate scratch dir for every client... 2024-09-21T09:30:01.536 DEBUG:teuthology.orchestra.run.smithi022:> stat -- /home/ubuntu/cephtest/mnt.0 2024-09-21T09:30:01.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:30:01.556 INFO:teuthology.orchestra.run.smithi022.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2024-09-21T09:30:01.556 DEBUG:teuthology.orchestra.run.smithi022:> mkdir -- /home/ubuntu/cephtest/mnt.0 2024-09-21T09:30:01.557 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:30:01 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:30:01] "GET /metrics HTTP/1.1" 200 236469 "" "Prometheus/2.43.0" 2024-09-21T09:30:01.574 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2024-09-21T09:30:01.575 DEBUG:teuthology.orchestra.run.smithi022:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2024-09-21T09:30:01.632 INFO:tasks.workunit:timeout=3h 2024-09-21T09:30:01.632 INFO:tasks.workunit:cleanup=True 2024-09-21T09:30:01.633 DEBUG:teuthology.orchestra.run.smithi022:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone https://git.ceph.com/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 && git checkout 62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:30:01.692 INFO:tasks.workunit.client.0.smithi022.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2024-09-21T09:30:02.154 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:30:01 smithi022 ceph-mon[39525]: pgmap v57: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-21T09:30:02.155 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:30:01 smithi022 ceph-mon[27763]: pgmap v57: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-21T09:30:02.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:30:01 smithi064 ceph-mon[31124]: pgmap v57: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-21T09:30:04.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:30:03 smithi064 ceph-mon[31124]: pgmap v58: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-21T09:30:04.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:30:03 smithi022 ceph-mon[39525]: pgmap v58: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-21T09:30:04.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:30:03 smithi022 ceph-mon[27763]: pgmap v58: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-21T09:30:06.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:30:05 smithi064 ceph-mon[31124]: pgmap v59: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-21T09:30:06.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:30:05 smithi022 ceph-mon[27763]: pgmap v59: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-21T09:30:06.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:30:05 smithi022 ceph-mon[39525]: pgmap v59: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-21T09:30:08.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:30:07 smithi064 ceph-mon[31124]: pgmap v60: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-21T09:30:08.307 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:30:07 smithi022 ceph-mon[39525]: pgmap v60: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-21T09:30:08.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:30:07 smithi022 ceph-mon[27763]: pgmap v60: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-21T09:30:10.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:30:09 smithi064 ceph-mon[31124]: pgmap v61: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-21T09:30:10.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:30:09 smithi022 ceph-mon[39525]: pgmap v61: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-21T09:30:10.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:30:09 smithi022 ceph-mon[27763]: pgmap v61: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-21T09:30:10.941 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:30:10 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[33213]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:30:10] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-21T09:30:11.558 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:30:11 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:30:11] "GET /metrics HTTP/1.1" 200 236469 "" "Prometheus/2.43.0" 2024-09-21T09:30:12.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:30:11 smithi064 ceph-mon[31124]: pgmap v62: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:30:12.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:30:11 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:30:12.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:30:11 smithi022 ceph-mon[39525]: pgmap v62: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:30:12.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:30:11 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:30:12.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:30:11 smithi022 ceph-mon[27763]: pgmap v62: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:30:12.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:30:11 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:30:14.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:30:13 smithi064 ceph-mon[31124]: pgmap v63: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:30:14.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:30:13 smithi022 ceph-mon[39525]: pgmap v63: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:30:14.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:30:13 smithi022 ceph-mon[27763]: pgmap v63: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:30:16.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:30:15 smithi064 ceph-mon[31124]: pgmap v64: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:30:16.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:30:15 smithi022 ceph-mon[39525]: pgmap v64: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:30:16.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:30:15 smithi022 ceph-mon[27763]: pgmap v64: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:30:18.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:30:17 smithi064 ceph-mon[31124]: pgmap v65: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:30:18.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:30:17 smithi022 ceph-mon[39525]: pgmap v65: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:30:18.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:30:17 smithi022 ceph-mon[27763]: pgmap v65: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:30:20.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:30:19 smithi064 ceph-mon[31124]: pgmap v66: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:30:20.307 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:30:19 smithi022 ceph-mon[39525]: pgmap v66: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:30:20.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:30:19 smithi022 ceph-mon[27763]: pgmap v66: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:30:20.941 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:30:20 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[33213]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:30:20] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-21T09:30:21.557 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:30:21 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:30:21] "GET /metrics HTTP/1.1" 200 236474 "" "Prometheus/2.43.0" 2024-09-21T09:30:22.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:30:21 smithi064 ceph-mon[31124]: pgmap v67: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:30:22.307 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:30:21 smithi022 ceph-mon[39525]: pgmap v67: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:30:22.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:30:21 smithi022 ceph-mon[27763]: pgmap v67: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:30:24.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:30:23 smithi064 ceph-mon[31124]: pgmap v68: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:30:24.307 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:30:23 smithi022 ceph-mon[39525]: pgmap v68: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:30:24.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:30:23 smithi022 ceph-mon[27763]: pgmap v68: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:30:26.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:30:25 smithi064 ceph-mon[31124]: pgmap v69: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:30:26.307 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:30:25 smithi022 ceph-mon[39525]: pgmap v69: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:30:26.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:30:25 smithi022 ceph-mon[27763]: pgmap v69: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:30:27.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:30:26 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:30:27.307 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:30:26 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:30:27.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:30:26 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:30:28.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:30:27 smithi064 ceph-mon[31124]: pgmap v70: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:30:28.307 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:30:27 smithi022 ceph-mon[39525]: pgmap v70: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:30:28.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:30:27 smithi022 ceph-mon[27763]: pgmap v70: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:30:30.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:30:29 smithi064 ceph-mon[31124]: pgmap v71: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:30:30.307 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:30:29 smithi022 ceph-mon[39525]: pgmap v71: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:30:30.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:30:29 smithi022 ceph-mon[27763]: pgmap v71: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:30:30.941 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:30:30 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[33213]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:30:30] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-21T09:30:31.558 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:30:31 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:30:31] "GET /metrics HTTP/1.1" 200 236455 "" "Prometheus/2.43.0" 2024-09-21T09:30:32.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:30:31 smithi064 ceph-mon[31124]: pgmap v72: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:30:32.307 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:30:31 smithi022 ceph-mon[39525]: pgmap v72: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:30:32.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:30:31 smithi022 ceph-mon[27763]: pgmap v72: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:30:34.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:30:33 smithi064 ceph-mon[31124]: pgmap v73: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:30:34.307 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:30:33 smithi022 ceph-mon[39525]: pgmap v73: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:30:34.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:30:33 smithi022 ceph-mon[27763]: pgmap v73: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:30:36.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:30:35 smithi064 ceph-mon[31124]: pgmap v74: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:30:36.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:30:35 smithi022 ceph-mon[39525]: pgmap v74: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:30:36.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:30:35 smithi022 ceph-mon[27763]: pgmap v74: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:30:38.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:30:37 smithi064 ceph-mon[31124]: pgmap v75: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:30:38.307 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:30:37 smithi022 ceph-mon[39525]: pgmap v75: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:30:38.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:30:37 smithi022 ceph-mon[27763]: pgmap v75: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:30:40.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:30:39 smithi064 ceph-mon[31124]: pgmap v76: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:30:40.307 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:30:39 smithi022 ceph-mon[39525]: pgmap v76: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:30:40.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:30:39 smithi022 ceph-mon[27763]: pgmap v76: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:30:40.941 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:30:40 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[33213]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:30:40] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-21T09:30:41.558 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:30:41 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:30:41] "GET /metrics HTTP/1.1" 200 236455 "" "Prometheus/2.43.0" 2024-09-21T09:30:42.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:30:41 smithi064 ceph-mon[31124]: pgmap v77: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:30:42.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:30:41 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:30:42.307 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:30:41 smithi022 ceph-mon[39525]: pgmap v77: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:30:42.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:30:41 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:30:42.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:30:41 smithi022 ceph-mon[27763]: pgmap v77: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:30:42.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:30:41 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:30:44.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:30:43 smithi064 ceph-mon[31124]: pgmap v78: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:30:44.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:30:43 smithi022 ceph-mon[39525]: pgmap v78: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:30:44.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:30:43 smithi022 ceph-mon[27763]: pgmap v78: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:30:46.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:30:45 smithi064 ceph-mon[31124]: pgmap v79: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:30:46.307 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:30:45 smithi022 ceph-mon[39525]: pgmap v79: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:30:46.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:30:45 smithi022 ceph-mon[27763]: pgmap v79: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:30:48.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:30:47 smithi064 ceph-mon[31124]: pgmap v80: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:30:48.307 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:30:47 smithi022 ceph-mon[39525]: pgmap v80: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:30:48.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:30:47 smithi022 ceph-mon[27763]: pgmap v80: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:30:50.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:30:49 smithi064 ceph-mon[31124]: pgmap v81: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:30:50.307 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:30:49 smithi022 ceph-mon[27763]: pgmap v81: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:30:50.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:30:49 smithi022 ceph-mon[39525]: pgmap v81: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:30:50.941 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:30:50 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[33213]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:30:50] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-21T09:30:51.418 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:30:51 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:30:51] "GET /metrics HTTP/1.1" 200 236455 "" "Prometheus/2.43.0" 2024-09-21T09:30:52.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:30:51 smithi064 ceph-mon[31124]: pgmap v82: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:30:52.307 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:30:51 smithi022 ceph-mon[39525]: pgmap v82: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:30:52.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:30:51 smithi022 ceph-mon[27763]: pgmap v82: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:30:54.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:30:53 smithi064 ceph-mon[31124]: pgmap v83: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:30:54.307 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:30:53 smithi022 ceph-mon[39525]: pgmap v83: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:30:54.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:30:53 smithi022 ceph-mon[27763]: pgmap v83: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:30:56.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:30:55 smithi064 ceph-mon[31124]: pgmap v84: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:30:56.307 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:30:55 smithi022 ceph-mon[39525]: pgmap v84: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:30:56.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:30:55 smithi022 ceph-mon[27763]: pgmap v84: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:30:57.307 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:30:56 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:30:57.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:30:56 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:30:57.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:30:56 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:30:58.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:30:57 smithi022 ceph-mon[39525]: pgmap v85: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:30:58.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:30:57 smithi022 ceph-mon[27763]: pgmap v85: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:30:58.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:30:57 smithi064 ceph-mon[31124]: pgmap v85: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:31:00.087 INFO:tasks.workunit.client.0.smithi022.stderr:Updating files: 94% (12243/12896) Updating files: 95% (12252/12896) Updating files: 96% (12381/12896) Updating files: 97% (12510/12896) Updating files: 98% (12639/12896) Updating files: 99% (12768/12896) Updating files: 100% (12896/12896) Updating files: 100% (12896/12896), done. 2024-09-21T09:31:00.189 INFO:tasks.workunit.client.0.smithi022.stderr:Note: switching to '62da54b4b83a96ca9976b7e1d0c8272a564a2208'. 2024-09-21T09:31:00.189 INFO:tasks.workunit.client.0.smithi022.stderr: 2024-09-21T09:31:00.189 INFO:tasks.workunit.client.0.smithi022.stderr:You are in 'detached HEAD' state. You can look around, make experimental 2024-09-21T09:31:00.189 INFO:tasks.workunit.client.0.smithi022.stderr:changes and commit them, and you can discard any commits you make in this 2024-09-21T09:31:00.189 INFO:tasks.workunit.client.0.smithi022.stderr:state without impacting any branches by switching back to a branch. 2024-09-21T09:31:00.189 INFO:tasks.workunit.client.0.smithi022.stderr: 2024-09-21T09:31:00.189 INFO:tasks.workunit.client.0.smithi022.stderr:If you want to create a new branch to retain commits you create, you may 2024-09-21T09:31:00.189 INFO:tasks.workunit.client.0.smithi022.stderr:do so (now or later) by using -c with the switch command. Example: 2024-09-21T09:31:00.189 INFO:tasks.workunit.client.0.smithi022.stderr: 2024-09-21T09:31:00.190 INFO:tasks.workunit.client.0.smithi022.stderr: git switch -c 2024-09-21T09:31:00.190 INFO:tasks.workunit.client.0.smithi022.stderr: 2024-09-21T09:31:00.190 INFO:tasks.workunit.client.0.smithi022.stderr:Or undo this operation with: 2024-09-21T09:31:00.190 INFO:tasks.workunit.client.0.smithi022.stderr: 2024-09-21T09:31:00.190 INFO:tasks.workunit.client.0.smithi022.stderr: git switch - 2024-09-21T09:31:00.190 INFO:tasks.workunit.client.0.smithi022.stderr: 2024-09-21T09:31:00.190 INFO:tasks.workunit.client.0.smithi022.stderr:Turn off this advice by setting config variable advice.detachedHead to false 2024-09-21T09:31:00.190 INFO:tasks.workunit.client.0.smithi022.stderr: 2024-09-21T09:31:00.190 INFO:tasks.workunit.client.0.smithi022.stderr:HEAD is now at 62da54b4b83 Merge pull request #59904 from mcv21/patch-2 2024-09-21T09:31:00.191 DEBUG:teuthology.orchestra.run.smithi022:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2024-09-21T09:31:00.259 INFO:tasks.workunit.client.0.smithi022.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2024-09-21T09:31:00.261 INFO:tasks.workunit.client.0.smithi022.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-09-21T09:31:00.261 INFO:tasks.workunit.client.0.smithi022.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2024-09-21T09:31:00.307 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:30:59 smithi022 ceph-mon[27763]: pgmap v86: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:31:00.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:30:59 smithi022 ceph-mon[39525]: pgmap v86: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:31:00.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:30:59 smithi064 ceph-mon[31124]: pgmap v86: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:31:00.940 INFO:tasks.workunit.client.0.smithi022.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2024-09-21T09:31:00.941 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:31:00 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[33213]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:31:00] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-21T09:31:01.009 INFO:tasks.workunit.client.0.smithi022.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2024-09-21T09:31:01.052 INFO:tasks.workunit.client.0.smithi022.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-09-21T09:31:01.053 INFO:tasks.workunit.client.0.smithi022.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-09-21T09:31:01.053 INFO:tasks.workunit.client.0.smithi022.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2024-09-21T09:31:01.094 INFO:tasks.workunit.client.0.smithi022.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-09-21T09:31:01.098 DEBUG:teuthology.orchestra.run.smithi022:> set -ex 2024-09-21T09:31:01.098 DEBUG:teuthology.orchestra.run.smithi022:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2024-09-21T09:31:01.156 INFO:tasks.workunit:Running workunits matching test_telemetry_quincy.sh on client.0... 2024-09-21T09:31:01.157 INFO:tasks.workunit:Running workunit test_telemetry_quincy.sh... 2024-09-21T09:31:01.157 DEBUG:teuthology.orchestra.run.smithi022:workunit test test_telemetry_quincy.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=62da54b4b83a96ca9976b7e1d0c8272a564a2208 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/test_telemetry_quincy.sh 2024-09-21T09:31:01.259 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:00 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:31:01.260 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:00 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:31:01.269 INFO:tasks.workunit.client.0.smithi022.stderr:+ ceph config set mgr mgr/telemetry/channel_ident true 2024-09-21T09:31:01.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:00 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:31:01.531 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:31:01 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:31:01] "GET /metrics HTTP/1.1" 200 236430 "" "Prometheus/2.43.0" 2024-09-21T09:31:01.531 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:31:01 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: [21/Sep/2024:09:31:01] ENGINE Bus STOPPING 2024-09-21T09:31:01.640 INFO:tasks.workunit.client.0.smithi022.stderr:+ ceph config set mgr mgr/telemetry/organization ceph-qa 2024-09-21T09:31:01.808 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:31:01 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: [21/Sep/2024:09:31:01] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-21T09:31:01.808 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:31:01 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: [21/Sep/2024:09:31:01] ENGINE Bus STOPPED 2024-09-21T09:31:01.808 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:31:01 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: [21/Sep/2024:09:31:01] ENGINE Bus STARTING 2024-09-21T09:31:02.014 INFO:tasks.workunit.client.0.smithi022.stderr:+ ceph config set mgr mgr/telemetry/description 'upgrade test cluster' 2024-09-21T09:31:02.308 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:31:01 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: [21/Sep/2024:09:31:01] ENGINE Serving on http://:::9283 2024-09-21T09:31:02.308 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:31:01 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: [21/Sep/2024:09:31:01] ENGINE Bus STARTED 2024-09-21T09:31:02.308 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:31:01 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: [21/Sep/2024:09:31:01] ENGINE Bus STOPPING 2024-09-21T09:31:02.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:01 smithi022 ceph-mon[39525]: pgmap v87: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:31:02.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:01 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:02.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:01 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:31:02.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:01 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:31:02.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:01 smithi022 ceph-mon[39525]: from='client.? 172.21.15.22:0/2539704360' entity='client.admin' 2024-09-21T09:31:02.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:01 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:31:02.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:01 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:02.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:01 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:31:02.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:01 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:31:02.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:01 smithi022 ceph-mon[39525]: from='client.? 172.21.15.22:0/544379316' entity='client.admin' 2024-09-21T09:31:02.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:01 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:31:02.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:01 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:02.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:01 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:31:02.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:01 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:31:02.310 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:01 smithi022 ceph-mon[27763]: pgmap v87: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:31:02.310 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:01 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:02.310 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:01 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:31:02.310 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:01 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:31:02.310 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:01 smithi022 ceph-mon[27763]: from='client.? 172.21.15.22:0/2539704360' entity='client.admin' 2024-09-21T09:31:02.310 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:01 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:31:02.310 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:01 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:02.311 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:01 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:31:02.311 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:01 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:31:02.311 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:01 smithi022 ceph-mon[27763]: from='client.? 172.21.15.22:0/544379316' entity='client.admin' 2024-09-21T09:31:02.311 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:01 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:31:02.311 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:01 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:02.311 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:01 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:31:02.311 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:01 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:31:02.342 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:01 smithi064 ceph-mon[31124]: pgmap v87: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:31:02.343 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:01 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:02.343 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:01 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:31:02.343 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:01 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:31:02.343 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:01 smithi064 ceph-mon[31124]: from='client.? 172.21.15.22:0/2539704360' entity='client.admin' 2024-09-21T09:31:02.343 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:01 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:31:02.343 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:01 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:02.343 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:01 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:31:02.343 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:01 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:31:02.343 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:01 smithi064 ceph-mon[31124]: from='client.? 172.21.15.22:0/544379316' entity='client.admin' 2024-09-21T09:31:02.343 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:01 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:31:02.343 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:01 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:02.343 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:01 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:31:02.344 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:01 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:31:02.393 INFO:tasks.workunit.client.0.smithi022.stderr:+ ceph telemetry preview 2024-09-21T09:31:02.807 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:31:02 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: [21/Sep/2024:09:31:02] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-21T09:31:02.808 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:31:02 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: [21/Sep/2024:09:31:02] ENGINE Bus STOPPED 2024-09-21T09:31:02.808 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:31:02 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: [21/Sep/2024:09:31:02] ENGINE Bus STARTING 2024-09-21T09:31:02.808 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:31:02 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: [21/Sep/2024:09:31:02] ENGINE Serving on http://:::9283 2024-09-21T09:31:02.808 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:31:02 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: [21/Sep/2024:09:31:02] ENGINE Bus STARTED 2024-09-21T09:31:02.808 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:31:02 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: [21/Sep/2024:09:31:02] ENGINE Bus STOPPING 2024-09-21T09:31:02.857 INFO:tasks.workunit.client.0.smithi022.stdout:{ 2024-09-21T09:31:02.857 INFO:tasks.workunit.client.0.smithi022.stdout: "balancer": { 2024-09-21T09:31:02.857 INFO:tasks.workunit.client.0.smithi022.stdout: "active": true, 2024-09-21T09:31:02.857 INFO:tasks.workunit.client.0.smithi022.stdout: "mode": "" 2024-09-21T09:31:02.857 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:02.857 INFO:tasks.workunit.client.0.smithi022.stdout: "channels": [ 2024-09-21T09:31:02.857 INFO:tasks.workunit.client.0.smithi022.stdout: "basic", 2024-09-21T09:31:02.857 INFO:tasks.workunit.client.0.smithi022.stdout: "crash", 2024-09-21T09:31:02.857 INFO:tasks.workunit.client.0.smithi022.stdout: "device", 2024-09-21T09:31:02.857 INFO:tasks.workunit.client.0.smithi022.stdout: "ident" 2024-09-21T09:31:02.857 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:02.857 INFO:tasks.workunit.client.0.smithi022.stdout: "channels_available": [ 2024-09-21T09:31:02.858 INFO:tasks.workunit.client.0.smithi022.stdout: "basic", 2024-09-21T09:31:02.858 INFO:tasks.workunit.client.0.smithi022.stdout: "ident", 2024-09-21T09:31:02.858 INFO:tasks.workunit.client.0.smithi022.stdout: "crash", 2024-09-21T09:31:02.858 INFO:tasks.workunit.client.0.smithi022.stdout: "device", 2024-09-21T09:31:02.858 INFO:tasks.workunit.client.0.smithi022.stdout: "perf" 2024-09-21T09:31:02.858 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:02.858 INFO:tasks.workunit.client.0.smithi022.stdout: "collections_available": [ 2024-09-21T09:31:02.858 INFO:tasks.workunit.client.0.smithi022.stdout: "basic_base", 2024-09-21T09:31:02.858 INFO:tasks.workunit.client.0.smithi022.stdout: "device_base", 2024-09-21T09:31:02.858 INFO:tasks.workunit.client.0.smithi022.stdout: "crash_base", 2024-09-21T09:31:02.858 INFO:tasks.workunit.client.0.smithi022.stdout: "ident_base", 2024-09-21T09:31:02.858 INFO:tasks.workunit.client.0.smithi022.stdout: "perf_perf", 2024-09-21T09:31:02.858 INFO:tasks.workunit.client.0.smithi022.stdout: "basic_mds_metadata", 2024-09-21T09:31:02.859 INFO:tasks.workunit.client.0.smithi022.stdout: "basic_pool_usage", 2024-09-21T09:31:02.859 INFO:tasks.workunit.client.0.smithi022.stdout: "basic_usage_by_class", 2024-09-21T09:31:02.859 INFO:tasks.workunit.client.0.smithi022.stdout: "basic_rook_v01", 2024-09-21T09:31:02.859 INFO:tasks.workunit.client.0.smithi022.stdout: "perf_memory_metrics", 2024-09-21T09:31:02.859 INFO:tasks.workunit.client.0.smithi022.stdout: "basic_pool_options_bluestore" 2024-09-21T09:31:02.859 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:02.859 INFO:tasks.workunit.client.0.smithi022.stdout: "collections_opted_in": [ 2024-09-21T09:31:02.859 INFO:tasks.workunit.client.0.smithi022.stdout: "basic_base", 2024-09-21T09:31:02.859 INFO:tasks.workunit.client.0.smithi022.stdout: "device_base", 2024-09-21T09:31:02.859 INFO:tasks.workunit.client.0.smithi022.stdout: "crash_base", 2024-09-21T09:31:02.859 INFO:tasks.workunit.client.0.smithi022.stdout: "ident_base", 2024-09-21T09:31:02.859 INFO:tasks.workunit.client.0.smithi022.stdout: "perf_perf", 2024-09-21T09:31:02.859 INFO:tasks.workunit.client.0.smithi022.stdout: "basic_mds_metadata", 2024-09-21T09:31:02.859 INFO:tasks.workunit.client.0.smithi022.stdout: "basic_pool_usage", 2024-09-21T09:31:02.859 INFO:tasks.workunit.client.0.smithi022.stdout: "basic_usage_by_class", 2024-09-21T09:31:02.859 INFO:tasks.workunit.client.0.smithi022.stdout: "basic_rook_v01", 2024-09-21T09:31:02.860 INFO:tasks.workunit.client.0.smithi022.stdout: "perf_memory_metrics", 2024-09-21T09:31:02.860 INFO:tasks.workunit.client.0.smithi022.stdout: "basic_pool_options_bluestore" 2024-09-21T09:31:02.860 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:02.860 INFO:tasks.workunit.client.0.smithi022.stdout: "config": { 2024-09-21T09:31:02.860 INFO:tasks.workunit.client.0.smithi022.stdout: "active_changed": [ 2024-09-21T09:31:02.860 INFO:tasks.workunit.client.0.smithi022.stdout: "auth_allow_insecure_global_id_reclaim", 2024-09-21T09:31:02.860 INFO:tasks.workunit.client.0.smithi022.stdout: "auth_debug", 2024-09-21T09:31:02.860 INFO:tasks.workunit.client.0.smithi022.stdout: "auth_mon_ticket_ttl", 2024-09-21T09:31:02.860 INFO:tasks.workunit.client.0.smithi022.stdout: "auth_service_ticket_ttl", 2024-09-21T09:31:02.860 INFO:tasks.workunit.client.0.smithi022.stdout: "bdev_debug_aio", 2024-09-21T09:31:02.860 INFO:tasks.workunit.client.0.smithi022.stdout: "client_check_pool_perm", 2024-09-21T09:31:02.860 INFO:tasks.workunit.client.0.smithi022.stdout: "client_mount_gid", 2024-09-21T09:31:02.860 INFO:tasks.workunit.client.0.smithi022.stdout: "client_mount_uid", 2024-09-21T09:31:02.860 INFO:tasks.workunit.client.0.smithi022.stdout: "client_quota", 2024-09-21T09:31:02.860 INFO:tasks.workunit.client.0.smithi022.stdout: "container_image", 2024-09-21T09:31:02.860 INFO:tasks.workunit.client.0.smithi022.stdout: "daemonize", 2024-09-21T09:31:02.860 INFO:tasks.workunit.client.0.smithi022.stdout: "debug_asserts_on_shutdown", 2024-09-21T09:31:02.861 INFO:tasks.workunit.client.0.smithi022.stdout: "debug_mgr", 2024-09-21T09:31:02.861 INFO:tasks.workunit.client.0.smithi022.stdout: "debug_mon", 2024-09-21T09:31:02.861 INFO:tasks.workunit.client.0.smithi022.stdout: "debug_ms", 2024-09-21T09:31:02.861 INFO:tasks.workunit.client.0.smithi022.stdout: "debug_osd", 2024-09-21T09:31:02.861 INFO:tasks.workunit.client.0.smithi022.stdout: "debug_paxos", 2024-09-21T09:31:02.861 INFO:tasks.workunit.client.0.smithi022.stdout: "keyring", 2024-09-21T09:31:02.861 INFO:tasks.workunit.client.0.smithi022.stdout: "leveldb_block_size", 2024-09-21T09:31:02.861 INFO:tasks.workunit.client.0.smithi022.stdout: "leveldb_cache_size", 2024-09-21T09:31:02.861 INFO:tasks.workunit.client.0.smithi022.stdout: "leveldb_compression", 2024-09-21T09:31:02.861 INFO:tasks.workunit.client.0.smithi022.stdout: "leveldb_log", 2024-09-21T09:31:02.861 INFO:tasks.workunit.client.0.smithi022.stdout: "leveldb_write_buffer_size", 2024-09-21T09:31:02.861 INFO:tasks.workunit.client.0.smithi022.stdout: "log_to_file", 2024-09-21T09:31:02.861 INFO:tasks.workunit.client.0.smithi022.stdout: "log_to_journald", 2024-09-21T09:31:02.862 INFO:tasks.workunit.client.0.smithi022.stdout: "log_to_stderr", 2024-09-21T09:31:02.862 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_allow_pool_delete", 2024-09-21T09:31:02.862 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_clock_drift_allowed", 2024-09-21T09:31:02.862 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_cluster_log_to_file", 2024-09-21T09:31:02.862 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_cluster_log_to_journald", 2024-09-21T09:31:02.862 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_cluster_log_to_stderr", 2024-09-21T09:31:02.862 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_data_avail_warn", 2024-09-21T09:31:02.862 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_election_default_strategy", 2024-09-21T09:31:02.862 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_host", 2024-09-21T09:31:02.862 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_max_pg_per_osd", 2024-09-21T09:31:02.862 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_mgr_mkfs_grace", 2024-09-21T09:31:02.862 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_osd_prime_pg_temp", 2024-09-21T09:31:02.862 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_osd_reporter_subtree_level", 2024-09-21T09:31:02.863 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_pg_warn_max_object_skew", 2024-09-21T09:31:02.863 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_reweight_min_bytes_per_osd", 2024-09-21T09:31:02.863 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_reweight_min_pgs_per_osd", 2024-09-21T09:31:02.863 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_warn_on_crush_straw_calc_version_zero", 2024-09-21T09:31:02.863 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_warn_on_insecure_global_id_reclaim", 2024-09-21T09:31:02.863 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_warn_on_insecure_global_id_reclaim_allowed", 2024-09-21T09:31:02.863 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_warn_on_legacy_crush_tunables", 2024-09-21T09:31:02.863 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_warn_on_osd_down_out_interval_zero", 2024-09-21T09:31:02.863 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_warn_on_pool_pg_num_not_power_of_two", 2024-09-21T09:31:02.863 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_warn_on_too_few_osds", 2024-09-21T09:31:02.863 INFO:tasks.workunit.client.0.smithi022.stdout: "ms_die_on_bug", 2024-09-21T09:31:02.863 INFO:tasks.workunit.client.0.smithi022.stdout: "ms_die_on_old_message", 2024-09-21T09:31:02.863 INFO:tasks.workunit.client.0.smithi022.stdout: "no_config_file", 2024-09-21T09:31:02.863 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_class_default_list", 2024-09-21T09:31:02.863 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_class_load_list", 2024-09-21T09:31:02.864 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_debug_misdirected_ops", 2024-09-21T09:31:02.864 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_debug_op_order", 2024-09-21T09:31:02.864 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_debug_pg_log_writeout", 2024-09-21T09:31:02.864 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_debug_shutdown", 2024-09-21T09:31:02.864 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_debug_verify_cached_snaps", 2024-09-21T09:31:02.864 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_debug_verify_missing_on_start", 2024-09-21T09:31:02.864 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_debug_verify_stray_on_activate", 2024-09-21T09:31:02.864 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_deep_scrub_update_digest_min_age", 2024-09-21T09:31:02.864 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_mclock_profile", 2024-09-21T09:31:02.864 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_mclock_scheduler_background_best_effort_lim", 2024-09-21T09:31:02.864 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_mclock_scheduler_background_best_effort_res", 2024-09-21T09:31:02.864 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_mclock_scheduler_background_best_effort_wgt", 2024-09-21T09:31:02.864 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_mclock_scheduler_background_recovery_lim", 2024-09-21T09:31:02.864 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_mclock_scheduler_background_recovery_res", 2024-09-21T09:31:02.864 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_mclock_scheduler_background_recovery_wgt", 2024-09-21T09:31:02.865 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_mclock_scheduler_client_lim", 2024-09-21T09:31:02.865 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_mclock_scheduler_client_res", 2024-09-21T09:31:02.865 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_mclock_scheduler_client_wgt", 2024-09-21T09:31:02.865 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_memory_target", 2024-09-21T09:31:02.865 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_memory_target_autotune", 2024-09-21T09:31:02.865 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_op_queue", 2024-09-21T09:31:02.865 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_op_queue_cut_off", 2024-09-21T09:31:02.865 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_pool_default_erasure_code_profile", 2024-09-21T09:31:02.865 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_pool_default_pg_autoscale_mode", 2024-09-21T09:31:02.865 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_recover_clone_overlap", 2024-09-21T09:31:02.865 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_recovery_max_chunk", 2024-09-21T09:31:02.865 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_scrub_load_threshold", 2024-09-21T09:31:02.865 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_scrub_max_interval", 2024-09-21T09:31:02.865 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_shutdown_pgref_assert", 2024-09-21T09:31:02.865 INFO:tasks.workunit.client.0.smithi022.stdout: "public_addr", 2024-09-21T09:31:02.865 INFO:tasks.workunit.client.0.smithi022.stdout: "public_addrv", 2024-09-21T09:31:02.866 INFO:tasks.workunit.client.0.smithi022.stdout: "public_network", 2024-09-21T09:31:02.866 INFO:tasks.workunit.client.0.smithi022.stdout: "rbd_default_features", 2024-09-21T09:31:02.866 INFO:tasks.workunit.client.0.smithi022.stdout: "rbd_qos_exclude_ops", 2024-09-21T09:31:02.866 INFO:tasks.workunit.client.0.smithi022.stdout: "setgroup", 2024-09-21T09:31:02.866 INFO:tasks.workunit.client.0.smithi022.stdout: "setuser" 2024-09-21T09:31:02.866 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:02.866 INFO:tasks.workunit.client.0.smithi022.stdout: "cluster_changed": [ 2024-09-21T09:31:02.866 INFO:tasks.workunit.client.0.smithi022.stdout: "auth_allow_insecure_global_id_reclaim", 2024-09-21T09:31:02.866 INFO:tasks.workunit.client.0.smithi022.stdout: "auth_debug", 2024-09-21T09:31:02.866 INFO:tasks.workunit.client.0.smithi022.stdout: "auth_mon_ticket_ttl", 2024-09-21T09:31:02.866 INFO:tasks.workunit.client.0.smithi022.stdout: "auth_service_ticket_ttl", 2024-09-21T09:31:02.866 INFO:tasks.workunit.client.0.smithi022.stdout: "bdev_debug_aio", 2024-09-21T09:31:02.866 INFO:tasks.workunit.client.0.smithi022.stdout: "container_image", 2024-09-21T09:31:02.866 INFO:tasks.workunit.client.0.smithi022.stdout: "debug_asserts_on_shutdown", 2024-09-21T09:31:02.866 INFO:tasks.workunit.client.0.smithi022.stdout: "debug_mgr", 2024-09-21T09:31:02.867 INFO:tasks.workunit.client.0.smithi022.stdout: "debug_mon", 2024-09-21T09:31:02.867 INFO:tasks.workunit.client.0.smithi022.stdout: "debug_ms", 2024-09-21T09:31:02.867 INFO:tasks.workunit.client.0.smithi022.stdout: "debug_osd", 2024-09-21T09:31:02.867 INFO:tasks.workunit.client.0.smithi022.stdout: "debug_paxos", 2024-09-21T09:31:02.867 INFO:tasks.workunit.client.0.smithi022.stdout: "log_to_file", 2024-09-21T09:31:02.867 INFO:tasks.workunit.client.0.smithi022.stdout: "log_to_journald", 2024-09-21T09:31:02.867 INFO:tasks.workunit.client.0.smithi022.stdout: "log_to_stderr", 2024-09-21T09:31:02.867 INFO:tasks.workunit.client.0.smithi022.stdout: "mds_join_fs", 2024-09-21T09:31:02.867 INFO:tasks.workunit.client.0.smithi022.stdout: "mgr/cephadm/allow_ptrace", 2024-09-21T09:31:02.867 INFO:tasks.workunit.client.0.smithi022.stdout: "mgr/cephadm/container_init", 2024-09-21T09:31:02.867 INFO:tasks.workunit.client.0.smithi022.stdout: "mgr/cephadm/migration_current", 2024-09-21T09:31:02.867 INFO:tasks.workunit.client.0.smithi022.stdout: "mgr/cephadm/use_repo_digest", 2024-09-21T09:31:02.867 INFO:tasks.workunit.client.0.smithi022.stdout: "mgr/dashboard/ALERTMANAGER_API_HOST", 2024-09-21T09:31:02.867 INFO:tasks.workunit.client.0.smithi022.stdout: "mgr/dashboard/GRAFANA_API_SSL_VERIFY", 2024-09-21T09:31:02.868 INFO:tasks.workunit.client.0.smithi022.stdout: "mgr/dashboard/GRAFANA_API_URL", 2024-09-21T09:31:02.868 INFO:tasks.workunit.client.0.smithi022.stdout: "mgr/dashboard/PROMETHEUS_API_HOST", 2024-09-21T09:31:02.868 INFO:tasks.workunit.client.0.smithi022.stdout: "mgr/dashboard/ssl_server_port", 2024-09-21T09:31:02.868 INFO:tasks.workunit.client.0.smithi022.stdout: "mgr/orchestrator/orchestrator", 2024-09-21T09:31:02.868 INFO:tasks.workunit.client.0.smithi022.stdout: "mgr/telemetry/channel_ident", 2024-09-21T09:31:02.868 INFO:tasks.workunit.client.0.smithi022.stdout: "mgr/telemetry/description", 2024-09-21T09:31:02.868 INFO:tasks.workunit.client.0.smithi022.stdout: "mgr/telemetry/organization", 2024-09-21T09:31:02.868 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_allow_pool_delete", 2024-09-21T09:31:02.868 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_clock_drift_allowed", 2024-09-21T09:31:02.868 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_cluster_log_to_file", 2024-09-21T09:31:02.868 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_data_avail_warn", 2024-09-21T09:31:02.868 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_election_default_strategy", 2024-09-21T09:31:02.868 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_max_pg_per_osd", 2024-09-21T09:31:02.868 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_mgr_mkfs_grace", 2024-09-21T09:31:02.868 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_osd_prime_pg_temp", 2024-09-21T09:31:02.869 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_osd_reporter_subtree_level", 2024-09-21T09:31:02.869 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_pg_warn_max_object_skew", 2024-09-21T09:31:02.869 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_reweight_min_bytes_per_osd", 2024-09-21T09:31:02.869 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_reweight_min_pgs_per_osd", 2024-09-21T09:31:02.869 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_warn_on_crush_straw_calc_version_zero", 2024-09-21T09:31:02.869 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_warn_on_insecure_global_id_reclaim", 2024-09-21T09:31:02.869 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_warn_on_insecure_global_id_reclaim_allowed", 2024-09-21T09:31:02.869 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_warn_on_legacy_crush_tunables", 2024-09-21T09:31:02.869 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_warn_on_osd_down_out_interval_zero", 2024-09-21T09:31:02.869 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_warn_on_pool_pg_num_not_power_of_two", 2024-09-21T09:31:02.869 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_warn_on_too_few_osds", 2024-09-21T09:31:02.869 INFO:tasks.workunit.client.0.smithi022.stdout: "ms_die_on_bug", 2024-09-21T09:31:02.869 INFO:tasks.workunit.client.0.smithi022.stdout: "ms_die_on_old_message", 2024-09-21T09:31:02.869 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_class_default_list", 2024-09-21T09:31:02.869 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_class_load_list", 2024-09-21T09:31:02.870 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_debug_misdirected_ops", 2024-09-21T09:31:02.870 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_debug_op_order", 2024-09-21T09:31:02.870 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_debug_pg_log_writeout", 2024-09-21T09:31:02.870 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_debug_shutdown", 2024-09-21T09:31:02.870 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_debug_verify_cached_snaps", 2024-09-21T09:31:02.870 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_debug_verify_missing_on_start", 2024-09-21T09:31:02.870 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_debug_verify_stray_on_activate", 2024-09-21T09:31:02.870 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_deep_scrub_update_digest_min_age", 2024-09-21T09:31:02.870 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_mclock_profile", 2024-09-21T09:31:02.870 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_memory_target", 2024-09-21T09:31:02.870 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_memory_target_autotune", 2024-09-21T09:31:02.870 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_op_queue", 2024-09-21T09:31:02.870 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_op_queue_cut_off", 2024-09-21T09:31:02.870 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_pool_default_erasure_code_profile", 2024-09-21T09:31:02.871 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_pool_default_pg_autoscale_mode", 2024-09-21T09:31:02.871 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_recover_clone_overlap", 2024-09-21T09:31:02.871 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_recovery_max_chunk", 2024-09-21T09:31:02.871 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_scrub_load_threshold", 2024-09-21T09:31:02.871 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_scrub_max_interval", 2024-09-21T09:31:02.871 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_shutdown_pgref_assert", 2024-09-21T09:31:02.871 INFO:tasks.workunit.client.0.smithi022.stdout: "public_network", 2024-09-21T09:31:02.871 INFO:tasks.workunit.client.0.smithi022.stdout: "rgw_cache_enabled", 2024-09-21T09:31:02.871 INFO:tasks.workunit.client.0.smithi022.stdout: "rgw_enable_ops_log", 2024-09-21T09:31:02.871 INFO:tasks.workunit.client.0.smithi022.stdout: "rgw_enable_usage_log" 2024-09-21T09:31:02.871 INFO:tasks.workunit.client.0.smithi022.stdout: ] 2024-09-21T09:31:02.871 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:02.871 INFO:tasks.workunit.client.0.smithi022.stdout: "contact": null, 2024-09-21T09:31:02.871 INFO:tasks.workunit.client.0.smithi022.stdout: "crashes": [], 2024-09-21T09:31:02.871 INFO:tasks.workunit.client.0.smithi022.stdout: "created": "2024-09-21T09:22:01.045656Z", 2024-09-21T09:31:02.872 INFO:tasks.workunit.client.0.smithi022.stdout: "crush": { 2024-09-21T09:31:02.872 INFO:tasks.workunit.client.0.smithi022.stdout: "bucket_algs": { 2024-09-21T09:31:02.872 INFO:tasks.workunit.client.0.smithi022.stdout: "straw2": 3 2024-09-21T09:31:02.872 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:02.872 INFO:tasks.workunit.client.0.smithi022.stdout: "bucket_sizes": { 2024-09-21T09:31:02.872 INFO:tasks.workunit.client.0.smithi022.stdout: "2": 1, 2024-09-21T09:31:02.872 INFO:tasks.workunit.client.0.smithi022.stdout: "4": 2 2024-09-21T09:31:02.872 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:02.872 INFO:tasks.workunit.client.0.smithi022.stdout: "bucket_types": { 2024-09-21T09:31:02.872 INFO:tasks.workunit.client.0.smithi022.stdout: "1": 2, 2024-09-21T09:31:02.872 INFO:tasks.workunit.client.0.smithi022.stdout: "11": 1 2024-09-21T09:31:02.872 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:02.872 INFO:tasks.workunit.client.0.smithi022.stdout: "compat_weight_set": false, 2024-09-21T09:31:02.872 INFO:tasks.workunit.client.0.smithi022.stdout: "device_classes": [ 2024-09-21T09:31:02.872 INFO:tasks.workunit.client.0.smithi022.stdout: 8 2024-09-21T09:31:02.873 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:02.873 INFO:tasks.workunit.client.0.smithi022.stdout: "num_buckets": 6, 2024-09-21T09:31:02.873 INFO:tasks.workunit.client.0.smithi022.stdout: "num_devices": 8, 2024-09-21T09:31:02.873 INFO:tasks.workunit.client.0.smithi022.stdout: "num_rules": 1, 2024-09-21T09:31:02.873 INFO:tasks.workunit.client.0.smithi022.stdout: "num_types": 12, 2024-09-21T09:31:02.873 INFO:tasks.workunit.client.0.smithi022.stdout: "num_weight_sets": 0, 2024-09-21T09:31:02.873 INFO:tasks.workunit.client.0.smithi022.stdout: "tunables": { 2024-09-21T09:31:02.873 INFO:tasks.workunit.client.0.smithi022.stdout: "allowed_bucket_algs": 54, 2024-09-21T09:31:02.873 INFO:tasks.workunit.client.0.smithi022.stdout: "choose_local_fallback_tries": 0, 2024-09-21T09:31:02.873 INFO:tasks.workunit.client.0.smithi022.stdout: "choose_local_tries": 0, 2024-09-21T09:31:02.873 INFO:tasks.workunit.client.0.smithi022.stdout: "choose_total_tries": 50, 2024-09-21T09:31:02.873 INFO:tasks.workunit.client.0.smithi022.stdout: "chooseleaf_descend_once": 1, 2024-09-21T09:31:02.873 INFO:tasks.workunit.client.0.smithi022.stdout: "chooseleaf_stable": 1, 2024-09-21T09:31:02.873 INFO:tasks.workunit.client.0.smithi022.stdout: "chooseleaf_vary_r": 1, 2024-09-21T09:31:02.873 INFO:tasks.workunit.client.0.smithi022.stdout: "has_v2_rules": 0, 2024-09-21T09:31:02.874 INFO:tasks.workunit.client.0.smithi022.stdout: "has_v3_rules": 0, 2024-09-21T09:31:02.874 INFO:tasks.workunit.client.0.smithi022.stdout: "has_v4_buckets": 1, 2024-09-21T09:31:02.874 INFO:tasks.workunit.client.0.smithi022.stdout: "has_v5_rules": 0, 2024-09-21T09:31:02.874 INFO:tasks.workunit.client.0.smithi022.stdout: "legacy_tunables": 0, 2024-09-21T09:31:02.874 INFO:tasks.workunit.client.0.smithi022.stdout: "minimum_required_version": "jewel", 2024-09-21T09:31:02.874 INFO:tasks.workunit.client.0.smithi022.stdout: "optimal_tunables": 1, 2024-09-21T09:31:02.874 INFO:tasks.workunit.client.0.smithi022.stdout: "profile": "jewel", 2024-09-21T09:31:02.874 INFO:tasks.workunit.client.0.smithi022.stdout: "require_feature_tunables": 1, 2024-09-21T09:31:02.874 INFO:tasks.workunit.client.0.smithi022.stdout: "require_feature_tunables2": 1, 2024-09-21T09:31:02.874 INFO:tasks.workunit.client.0.smithi022.stdout: "require_feature_tunables3": 1, 2024-09-21T09:31:02.874 INFO:tasks.workunit.client.0.smithi022.stdout: "require_feature_tunables5": 1, 2024-09-21T09:31:02.874 INFO:tasks.workunit.client.0.smithi022.stdout: "straw_calc_version": 1 2024-09-21T09:31:02.874 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:02.874 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:02.875 INFO:tasks.workunit.client.0.smithi022.stdout: "description": "upgrade test cluster", 2024-09-21T09:31:02.875 INFO:tasks.workunit.client.0.smithi022.stdout: "fs": { 2024-09-21T09:31:02.875 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 1, 2024-09-21T09:31:02.875 INFO:tasks.workunit.client.0.smithi022.stdout: "feature_flags": { 2024-09-21T09:31:02.875 INFO:tasks.workunit.client.0.smithi022.stdout: "enable_multiple": true, 2024-09-21T09:31:02.875 INFO:tasks.workunit.client.0.smithi022.stdout: "ever_enabled_multiple": true 2024-09-21T09:31:02.875 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:02.875 INFO:tasks.workunit.client.0.smithi022.stdout: "filesystems": [ 2024-09-21T09:31:02.875 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:02.875 INFO:tasks.workunit.client.0.smithi022.stdout: "approx_ctime": "2024-09", 2024-09-21T09:31:02.875 INFO:tasks.workunit.client.0.smithi022.stdout: "balancer_enabled": false, 2024-09-21T09:31:02.875 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:02.875 INFO:tasks.workunit.client.0.smithi022.stdout: "cached_caps": 0, 2024-09-21T09:31:02.875 INFO:tasks.workunit.client.0.smithi022.stdout: "cached_dns": 10, 2024-09-21T09:31:02.876 INFO:tasks.workunit.client.0.smithi022.stdout: "cached_inos": 13, 2024-09-21T09:31:02.876 INFO:tasks.workunit.client.0.smithi022.stdout: "cached_subtrees": 2, 2024-09-21T09:31:02.876 INFO:tasks.workunit.client.0.smithi022.stdout: "ever_allowed_features": 0, 2024-09-21T09:31:02.876 INFO:tasks.workunit.client.0.smithi022.stdout: "explicitly_allowed_features": 0, 2024-09-21T09:31:02.876 INFO:tasks.workunit.client.0.smithi022.stdout: "files": 0, 2024-09-21T09:31:02.876 INFO:tasks.workunit.client.0.smithi022.stdout: "max_mds": 1, 2024-09-21T09:31:02.876 INFO:tasks.workunit.client.0.smithi022.stdout: "num_data_pools": 1, 2024-09-21T09:31:02.876 INFO:tasks.workunit.client.0.smithi022.stdout: "num_in": 1, 2024-09-21T09:31:02.876 INFO:tasks.workunit.client.0.smithi022.stdout: "num_mds": 1, 2024-09-21T09:31:02.876 INFO:tasks.workunit.client.0.smithi022.stdout: "num_sessions": 0, 2024-09-21T09:31:02.876 INFO:tasks.workunit.client.0.smithi022.stdout: "num_standby_replay": 0, 2024-09-21T09:31:02.876 INFO:tasks.workunit.client.0.smithi022.stdout: "num_up": 1, 2024-09-21T09:31:02.876 INFO:tasks.workunit.client.0.smithi022.stdout: "snaps": 0, 2024-09-21T09:31:02.876 INFO:tasks.workunit.client.0.smithi022.stdout: "standby_count_wanted": 0 2024-09-21T09:31:02.876 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:02.877 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:02.877 INFO:tasks.workunit.client.0.smithi022.stdout: "num_standby_mds": 1, 2024-09-21T09:31:02.877 INFO:tasks.workunit.client.0.smithi022.stdout: "total_num_mds": 2 2024-09-21T09:31:02.877 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:02.877 INFO:tasks.workunit.client.0.smithi022.stdout: "hosts": { 2024-09-21T09:31:02.877 INFO:tasks.workunit.client.0.smithi022.stdout: "num": 2, 2024-09-21T09:31:02.877 INFO:tasks.workunit.client.0.smithi022.stdout: "num_with_mds": 2, 2024-09-21T09:31:02.877 INFO:tasks.workunit.client.0.smithi022.stdout: "num_with_mgr": 2, 2024-09-21T09:31:02.877 INFO:tasks.workunit.client.0.smithi022.stdout: "num_with_mon": 2, 2024-09-21T09:31:02.877 INFO:tasks.workunit.client.0.smithi022.stdout: "num_with_osd": 2 2024-09-21T09:31:02.877 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:02.877 INFO:tasks.workunit.client.0.smithi022.stdout: "leaderboard": false, 2024-09-21T09:31:02.877 INFO:tasks.workunit.client.0.smithi022.stdout: "license": "sharing-1-0", 2024-09-21T09:31:02.877 INFO:tasks.workunit.client.0.smithi022.stdout: "metadata": { 2024-09-21T09:31:02.877 INFO:tasks.workunit.client.0.smithi022.stdout: "mds": { 2024-09-21T09:31:02.878 INFO:tasks.workunit.client.0.smithi022.stdout: "arch": { 2024-09-21T09:31:02.878 INFO:tasks.workunit.client.0.smithi022.stdout: "x86_64": 2 2024-09-21T09:31:02.878 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:02.878 INFO:tasks.workunit.client.0.smithi022.stdout: "ceph_version": { 2024-09-21T09:31:02.878 INFO:tasks.workunit.client.0.smithi022.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 2 2024-09-21T09:31:02.878 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:02.878 INFO:tasks.workunit.client.0.smithi022.stdout: "cpu": { 2024-09-21T09:31:02.878 INFO:tasks.workunit.client.0.smithi022.stdout: "Intel(R) Xeon(R) CPU E5-1620 v3 @ 3.50GHz": 2 2024-09-21T09:31:02.878 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:02.878 INFO:tasks.workunit.client.0.smithi022.stdout: "distro": { 2024-09-21T09:31:02.878 INFO:tasks.workunit.client.0.smithi022.stdout: "centos": 2 2024-09-21T09:31:02.878 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:02.878 INFO:tasks.workunit.client.0.smithi022.stdout: "distro_description": { 2024-09-21T09:31:02.878 INFO:tasks.workunit.client.0.smithi022.stdout: "CentOS Stream 9": 2 2024-09-21T09:31:02.879 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:02.879 INFO:tasks.workunit.client.0.smithi022.stdout: "kernel_description": { 2024-09-21T09:31:02.879 INFO:tasks.workunit.client.0.smithi022.stdout: "#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024": 2 2024-09-21T09:31:02.879 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:02.879 INFO:tasks.workunit.client.0.smithi022.stdout: "kernel_version": { 2024-09-21T09:31:02.879 INFO:tasks.workunit.client.0.smithi022.stdout: "5.14.0-508.el9.x86_64": 2 2024-09-21T09:31:02.879 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:02.879 INFO:tasks.workunit.client.0.smithi022.stdout: "os": { 2024-09-21T09:31:02.879 INFO:tasks.workunit.client.0.smithi022.stdout: "Linux": 2 2024-09-21T09:31:02.879 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:02.879 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:02.879 INFO:tasks.workunit.client.0.smithi022.stdout: "mon": { 2024-09-21T09:31:02.879 INFO:tasks.workunit.client.0.smithi022.stdout: "arch": { 2024-09-21T09:31:02.880 INFO:tasks.workunit.client.0.smithi022.stdout: "x86_64": 3 2024-09-21T09:31:02.880 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:02.880 INFO:tasks.workunit.client.0.smithi022.stdout: "ceph_version": { 2024-09-21T09:31:02.880 INFO:tasks.workunit.client.0.smithi022.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 3 2024-09-21T09:31:02.880 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:02.880 INFO:tasks.workunit.client.0.smithi022.stdout: "cpu": { 2024-09-21T09:31:02.880 INFO:tasks.workunit.client.0.smithi022.stdout: "Intel(R) Xeon(R) CPU E5-1620 v3 @ 3.50GHz": 3 2024-09-21T09:31:02.880 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:02.880 INFO:tasks.workunit.client.0.smithi022.stdout: "distro": { 2024-09-21T09:31:02.880 INFO:tasks.workunit.client.0.smithi022.stdout: "centos": 3 2024-09-21T09:31:02.880 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:02.880 INFO:tasks.workunit.client.0.smithi022.stdout: "distro_description": { 2024-09-21T09:31:02.880 INFO:tasks.workunit.client.0.smithi022.stdout: "CentOS Stream 9": 3 2024-09-21T09:31:02.880 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:02.880 INFO:tasks.workunit.client.0.smithi022.stdout: "kernel_description": { 2024-09-21T09:31:02.881 INFO:tasks.workunit.client.0.smithi022.stdout: "#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024": 3 2024-09-21T09:31:02.881 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:02.881 INFO:tasks.workunit.client.0.smithi022.stdout: "kernel_version": { 2024-09-21T09:31:02.881 INFO:tasks.workunit.client.0.smithi022.stdout: "5.14.0-508.el9.x86_64": 3 2024-09-21T09:31:02.881 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:02.881 INFO:tasks.workunit.client.0.smithi022.stdout: "os": { 2024-09-21T09:31:02.881 INFO:tasks.workunit.client.0.smithi022.stdout: "Linux": 3 2024-09-21T09:31:02.881 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:02.881 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:02.881 INFO:tasks.workunit.client.0.smithi022.stdout: "osd": { 2024-09-21T09:31:02.881 INFO:tasks.workunit.client.0.smithi022.stdout: "arch": { 2024-09-21T09:31:02.881 INFO:tasks.workunit.client.0.smithi022.stdout: "x86_64": 8 2024-09-21T09:31:02.881 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:02.881 INFO:tasks.workunit.client.0.smithi022.stdout: "ceph_version": { 2024-09-21T09:31:02.882 INFO:tasks.workunit.client.0.smithi022.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 8 2024-09-21T09:31:02.882 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:02.882 INFO:tasks.workunit.client.0.smithi022.stdout: "cpu": { 2024-09-21T09:31:02.882 INFO:tasks.workunit.client.0.smithi022.stdout: "Intel(R) Xeon(R) CPU E5-1620 v3 @ 3.50GHz": 8 2024-09-21T09:31:02.882 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:02.882 INFO:tasks.workunit.client.0.smithi022.stdout: "distro": { 2024-09-21T09:31:02.882 INFO:tasks.workunit.client.0.smithi022.stdout: "centos": 8 2024-09-21T09:31:02.882 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:02.882 INFO:tasks.workunit.client.0.smithi022.stdout: "distro_description": { 2024-09-21T09:31:02.882 INFO:tasks.workunit.client.0.smithi022.stdout: "CentOS Stream 9": 8 2024-09-21T09:31:02.882 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:02.882 INFO:tasks.workunit.client.0.smithi022.stdout: "kernel_description": { 2024-09-21T09:31:02.882 INFO:tasks.workunit.client.0.smithi022.stdout: "#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024": 8 2024-09-21T09:31:02.882 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:02.883 INFO:tasks.workunit.client.0.smithi022.stdout: "kernel_version": { 2024-09-21T09:31:02.883 INFO:tasks.workunit.client.0.smithi022.stdout: "5.14.0-508.el9.x86_64": 8 2024-09-21T09:31:02.883 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:02.883 INFO:tasks.workunit.client.0.smithi022.stdout: "os": { 2024-09-21T09:31:02.883 INFO:tasks.workunit.client.0.smithi022.stdout: "Linux": 8 2024-09-21T09:31:02.883 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:02.883 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_objectstore": { 2024-09-21T09:31:02.883 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore": 8 2024-09-21T09:31:02.883 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:02.883 INFO:tasks.workunit.client.0.smithi022.stdout: "rotational": { 2024-09-21T09:31:02.883 INFO:tasks.workunit.client.0.smithi022.stdout: "0": 8 2024-09-21T09:31:02.883 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:02.883 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:02.883 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:02.883 INFO:tasks.workunit.client.0.smithi022.stdout: "mon": { 2024-09-21T09:31:02.884 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 3, 2024-09-21T09:31:02.884 INFO:tasks.workunit.client.0.smithi022.stdout: "features": { 2024-09-21T09:31:02.884 INFO:tasks.workunit.client.0.smithi022.stdout: "optional": [], 2024-09-21T09:31:02.884 INFO:tasks.workunit.client.0.smithi022.stdout: "persistent": [ 2024-09-21T09:31:02.884 INFO:tasks.workunit.client.0.smithi022.stdout: "kraken", 2024-09-21T09:31:02.884 INFO:tasks.workunit.client.0.smithi022.stdout: "luminous", 2024-09-21T09:31:02.884 INFO:tasks.workunit.client.0.smithi022.stdout: "mimic", 2024-09-21T09:31:02.884 INFO:tasks.workunit.client.0.smithi022.stdout: "osdmap-prune", 2024-09-21T09:31:02.884 INFO:tasks.workunit.client.0.smithi022.stdout: "nautilus", 2024-09-21T09:31:02.884 INFO:tasks.workunit.client.0.smithi022.stdout: "octopus", 2024-09-21T09:31:02.884 INFO:tasks.workunit.client.0.smithi022.stdout: "pacific", 2024-09-21T09:31:02.884 INFO:tasks.workunit.client.0.smithi022.stdout: "elector-pinging", 2024-09-21T09:31:02.884 INFO:tasks.workunit.client.0.smithi022.stdout: "quincy" 2024-09-21T09:31:02.884 INFO:tasks.workunit.client.0.smithi022.stdout: ] 2024-09-21T09:31:02.885 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:02.885 INFO:tasks.workunit.client.0.smithi022.stdout: "ipv4_addr_mons": 6, 2024-09-21T09:31:02.885 INFO:tasks.workunit.client.0.smithi022.stdout: "ipv6_addr_mons": 0, 2024-09-21T09:31:02.885 INFO:tasks.workunit.client.0.smithi022.stdout: "min_mon_release": 17, 2024-09-21T09:31:02.885 INFO:tasks.workunit.client.0.smithi022.stdout: "v1_addr_mons": 3, 2024-09-21T09:31:02.885 INFO:tasks.workunit.client.0.smithi022.stdout: "v2_addr_mons": 3 2024-09-21T09:31:02.885 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:02.885 INFO:tasks.workunit.client.0.smithi022.stdout: "organization": "ceph-qa", 2024-09-21T09:31:02.885 INFO:tasks.workunit.client.0.smithi022.stdout: "osd": { 2024-09-21T09:31:02.885 INFO:tasks.workunit.client.0.smithi022.stdout: "cluster_network": false, 2024-09-21T09:31:02.885 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 8, 2024-09-21T09:31:02.885 INFO:tasks.workunit.client.0.smithi022.stdout: "require_min_compat_client": "luminous", 2024-09-21T09:31:02.885 INFO:tasks.workunit.client.0.smithi022.stdout: "require_osd_release": "quincy" 2024-09-21T09:31:02.885 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:02.885 INFO:tasks.workunit.client.0.smithi022.stdout: "pools": [ 2024-09-21T09:31:02.886 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:02.886 INFO:tasks.workunit.client.0.smithi022.stdout: "application": [ 2024-09-21T09:31:02.886 INFO:tasks.workunit.client.0.smithi022.stdout: "mgr" 2024-09-21T09:31:02.886 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:02.886 INFO:tasks.workunit.client.0.smithi022.stdout: "cache_mode": "none", 2024-09-21T09:31:02.886 INFO:tasks.workunit.client.0.smithi022.stdout: "erasure_code_profile": {}, 2024-09-21T09:31:02.886 INFO:tasks.workunit.client.0.smithi022.stdout: "min_size": 2, 2024-09-21T09:31:02.886 INFO:tasks.workunit.client.0.smithi022.stdout: "options": {}, 2024-09-21T09:31:02.886 INFO:tasks.workunit.client.0.smithi022.stdout: "pg_autoscale_mode": "off", 2024-09-21T09:31:02.886 INFO:tasks.workunit.client.0.smithi022.stdout: "pg_num": 1, 2024-09-21T09:31:02.886 INFO:tasks.workunit.client.0.smithi022.stdout: "pgp_num": 1, 2024-09-21T09:31:02.886 INFO:tasks.workunit.client.0.smithi022.stdout: "pool": 1, 2024-09-21T09:31:02.886 INFO:tasks.workunit.client.0.smithi022.stdout: "size": 3, 2024-09-21T09:31:02.886 INFO:tasks.workunit.client.0.smithi022.stdout: "stats": { 2024-09-21T09:31:02.887 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_raw": 727115966730, 2024-09-21T09:31:02.887 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes_used": 1781760, 2024-09-21T09:31:02.887 INFO:tasks.workunit.client.0.smithi022.stdout: "compress_bytes_used": 0, 2024-09-21T09:31:02.887 INFO:tasks.workunit.client.0.smithi022.stdout: "compress_under_bytes": 0, 2024-09-21T09:31:02.887 INFO:tasks.workunit.client.0.smithi022.stdout: "data_bytes_used": 1781760, 2024-09-21T09:31:02.887 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty": 0, 2024-09-21T09:31:02.887 INFO:tasks.workunit.client.0.smithi022.stdout: "max_avail": 242371985408, 2024-09-21T09:31:02.887 INFO:tasks.workunit.client.0.smithi022.stdout: "objects": 2, 2024-09-21T09:31:02.887 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_bytes_used": 0, 2024-09-21T09:31:02.887 INFO:tasks.workunit.client.0.smithi022.stdout: "percent_used": 2.4504422526661074e-06, 2024-09-21T09:31:02.887 INFO:tasks.workunit.client.0.smithi022.stdout: "quota_bytes": 0, 2024-09-21T09:31:02.887 INFO:tasks.workunit.client.0.smithi022.stdout: "quota_objects": 0, 2024-09-21T09:31:02.887 INFO:tasks.workunit.client.0.smithi022.stdout: "rd": 126, 2024-09-21T09:31:02.887 INFO:tasks.workunit.client.0.smithi022.stdout: "rd_bytes": 236544, 2024-09-21T09:31:02.887 INFO:tasks.workunit.client.0.smithi022.stdout: "stored": 590368, 2024-09-21T09:31:02.888 INFO:tasks.workunit.client.0.smithi022.stdout: "stored_data": 590368, 2024-09-21T09:31:02.888 INFO:tasks.workunit.client.0.smithi022.stdout: "stored_omap": 0, 2024-09-21T09:31:02.888 INFO:tasks.workunit.client.0.smithi022.stdout: "stored_raw": 1771104, 2024-09-21T09:31:02.888 INFO:tasks.workunit.client.0.smithi022.stdout: "wr": 113, 2024-09-21T09:31:02.888 INFO:tasks.workunit.client.0.smithi022.stdout: "wr_bytes": 1667072 2024-09-21T09:31:02.888 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:02.888 INFO:tasks.workunit.client.0.smithi022.stdout: "target_max_bytes": 0, 2024-09-21T09:31:02.888 INFO:tasks.workunit.client.0.smithi022.stdout: "target_max_objects": 0, 2024-09-21T09:31:02.888 INFO:tasks.workunit.client.0.smithi022.stdout: "type": "replicated" 2024-09-21T09:31:02.888 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:02.888 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:02.888 INFO:tasks.workunit.client.0.smithi022.stdout: "application": [ 2024-09-21T09:31:02.888 INFO:tasks.workunit.client.0.smithi022.stdout: "cephfs" 2024-09-21T09:31:02.888 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:02.888 INFO:tasks.workunit.client.0.smithi022.stdout: "cache_mode": "none", 2024-09-21T09:31:02.888 INFO:tasks.workunit.client.0.smithi022.stdout: "erasure_code_profile": {}, 2024-09-21T09:31:02.888 INFO:tasks.workunit.client.0.smithi022.stdout: "min_size": 2, 2024-09-21T09:31:02.889 INFO:tasks.workunit.client.0.smithi022.stdout: "options": {}, 2024-09-21T09:31:02.889 INFO:tasks.workunit.client.0.smithi022.stdout: "pg_autoscale_mode": "off", 2024-09-21T09:31:02.889 INFO:tasks.workunit.client.0.smithi022.stdout: "pg_num": 32, 2024-09-21T09:31:02.889 INFO:tasks.workunit.client.0.smithi022.stdout: "pgp_num": 32, 2024-09-21T09:31:02.889 INFO:tasks.workunit.client.0.smithi022.stdout: "pool": 2, 2024-09-21T09:31:02.889 INFO:tasks.workunit.client.0.smithi022.stdout: "size": 3, 2024-09-21T09:31:02.889 INFO:tasks.workunit.client.0.smithi022.stdout: "stats": { 2024-09-21T09:31:02.889 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_raw": 727115966730, 2024-09-21T09:31:02.889 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes_used": 98304, 2024-09-21T09:31:02.889 INFO:tasks.workunit.client.0.smithi022.stdout: "compress_bytes_used": 0, 2024-09-21T09:31:02.889 INFO:tasks.workunit.client.0.smithi022.stdout: "compress_under_bytes": 0, 2024-09-21T09:31:02.889 INFO:tasks.workunit.client.0.smithi022.stdout: "data_bytes_used": 98304, 2024-09-21T09:31:02.889 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty": 0, 2024-09-21T09:31:02.889 INFO:tasks.workunit.client.0.smithi022.stdout: "max_avail": 242371985408, 2024-09-21T09:31:02.889 INFO:tasks.workunit.client.0.smithi022.stdout: "objects": 22, 2024-09-21T09:31:02.889 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_bytes_used": 0, 2024-09-21T09:31:02.890 INFO:tasks.workunit.client.0.smithi022.stdout: "percent_used": 1.3519712638299097e-07, 2024-09-21T09:31:02.890 INFO:tasks.workunit.client.0.smithi022.stdout: "quota_bytes": 0, 2024-09-21T09:31:02.890 INFO:tasks.workunit.client.0.smithi022.stdout: "quota_objects": 0, 2024-09-21T09:31:02.890 INFO:tasks.workunit.client.0.smithi022.stdout: "rd": 0, 2024-09-21T09:31:02.890 INFO:tasks.workunit.client.0.smithi022.stdout: "rd_bytes": 0, 2024-09-21T09:31:02.890 INFO:tasks.workunit.client.0.smithi022.stdout: "stored": 3410, 2024-09-21T09:31:02.890 INFO:tasks.workunit.client.0.smithi022.stdout: "stored_data": 3410, 2024-09-21T09:31:02.890 INFO:tasks.workunit.client.0.smithi022.stdout: "stored_omap": 0, 2024-09-21T09:31:02.890 INFO:tasks.workunit.client.0.smithi022.stdout: "stored_raw": 10230, 2024-09-21T09:31:02.890 INFO:tasks.workunit.client.0.smithi022.stdout: "wr": 51, 2024-09-21T09:31:02.890 INFO:tasks.workunit.client.0.smithi022.stdout: "wr_bytes": 19456 2024-09-21T09:31:02.890 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:02.890 INFO:tasks.workunit.client.0.smithi022.stdout: "target_max_bytes": 0, 2024-09-21T09:31:02.890 INFO:tasks.workunit.client.0.smithi022.stdout: "target_max_objects": 0, 2024-09-21T09:31:02.890 INFO:tasks.workunit.client.0.smithi022.stdout: "type": "replicated" 2024-09-21T09:31:02.891 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:02.891 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:02.891 INFO:tasks.workunit.client.0.smithi022.stdout: "application": [ 2024-09-21T09:31:02.891 INFO:tasks.workunit.client.0.smithi022.stdout: "cephfs" 2024-09-21T09:31:02.891 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:02.891 INFO:tasks.workunit.client.0.smithi022.stdout: "cache_mode": "none", 2024-09-21T09:31:02.891 INFO:tasks.workunit.client.0.smithi022.stdout: "erasure_code_profile": {}, 2024-09-21T09:31:02.891 INFO:tasks.workunit.client.0.smithi022.stdout: "min_size": 2, 2024-09-21T09:31:02.891 INFO:tasks.workunit.client.0.smithi022.stdout: "options": {}, 2024-09-21T09:31:02.891 INFO:tasks.workunit.client.0.smithi022.stdout: "pg_autoscale_mode": "off", 2024-09-21T09:31:02.891 INFO:tasks.workunit.client.0.smithi022.stdout: "pg_num": 64, 2024-09-21T09:31:02.891 INFO:tasks.workunit.client.0.smithi022.stdout: "pgp_num": 64, 2024-09-21T09:31:02.891 INFO:tasks.workunit.client.0.smithi022.stdout: "pool": 3, 2024-09-21T09:31:02.891 INFO:tasks.workunit.client.0.smithi022.stdout: "size": 3, 2024-09-21T09:31:02.891 INFO:tasks.workunit.client.0.smithi022.stdout: "stats": { 2024-09-21T09:31:02.891 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_raw": 727115966730, 2024-09-21T09:31:02.891 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes_used": 0, 2024-09-21T09:31:02.892 INFO:tasks.workunit.client.0.smithi022.stdout: "compress_bytes_used": 0, 2024-09-21T09:31:02.892 INFO:tasks.workunit.client.0.smithi022.stdout: "compress_under_bytes": 0, 2024-09-21T09:31:02.892 INFO:tasks.workunit.client.0.smithi022.stdout: "data_bytes_used": 0, 2024-09-21T09:31:02.892 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty": 0, 2024-09-21T09:31:02.892 INFO:tasks.workunit.client.0.smithi022.stdout: "max_avail": 242371985408, 2024-09-21T09:31:02.892 INFO:tasks.workunit.client.0.smithi022.stdout: "objects": 0, 2024-09-21T09:31:02.892 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_bytes_used": 0, 2024-09-21T09:31:02.892 INFO:tasks.workunit.client.0.smithi022.stdout: "percent_used": 0.0, 2024-09-21T09:31:02.892 INFO:tasks.workunit.client.0.smithi022.stdout: "quota_bytes": 0, 2024-09-21T09:31:02.892 INFO:tasks.workunit.client.0.smithi022.stdout: "quota_objects": 0, 2024-09-21T09:31:02.892 INFO:tasks.workunit.client.0.smithi022.stdout: "rd": 0, 2024-09-21T09:31:02.892 INFO:tasks.workunit.client.0.smithi022.stdout: "rd_bytes": 0, 2024-09-21T09:31:02.892 INFO:tasks.workunit.client.0.smithi022.stdout: "stored": 0, 2024-09-21T09:31:02.892 INFO:tasks.workunit.client.0.smithi022.stdout: "stored_data": 0, 2024-09-21T09:31:02.892 INFO:tasks.workunit.client.0.smithi022.stdout: "stored_omap": 0, 2024-09-21T09:31:02.892 INFO:tasks.workunit.client.0.smithi022.stdout: "stored_raw": 0, 2024-09-21T09:31:02.893 INFO:tasks.workunit.client.0.smithi022.stdout: "wr": 0, 2024-09-21T09:31:02.893 INFO:tasks.workunit.client.0.smithi022.stdout: "wr_bytes": 0 2024-09-21T09:31:02.893 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:02.893 INFO:tasks.workunit.client.0.smithi022.stdout: "target_max_bytes": 0, 2024-09-21T09:31:02.893 INFO:tasks.workunit.client.0.smithi022.stdout: "target_max_objects": 0, 2024-09-21T09:31:02.893 INFO:tasks.workunit.client.0.smithi022.stdout: "type": "replicated" 2024-09-21T09:31:02.893 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:02.893 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:02.893 INFO:tasks.workunit.client.0.smithi022.stdout: "application": [ 2024-09-21T09:31:02.893 INFO:tasks.workunit.client.0.smithi022.stdout: "rbd" 2024-09-21T09:31:02.893 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:02.893 INFO:tasks.workunit.client.0.smithi022.stdout: "cache_mode": "none", 2024-09-21T09:31:02.893 INFO:tasks.workunit.client.0.smithi022.stdout: "erasure_code_profile": {}, 2024-09-21T09:31:02.893 INFO:tasks.workunit.client.0.smithi022.stdout: "min_size": 2, 2024-09-21T09:31:02.893 INFO:tasks.workunit.client.0.smithi022.stdout: "options": {}, 2024-09-21T09:31:02.894 INFO:tasks.workunit.client.0.smithi022.stdout: "pg_autoscale_mode": "off", 2024-09-21T09:31:02.894 INFO:tasks.workunit.client.0.smithi022.stdout: "pg_num": 8, 2024-09-21T09:31:02.894 INFO:tasks.workunit.client.0.smithi022.stdout: "pgp_num": 8, 2024-09-21T09:31:02.894 INFO:tasks.workunit.client.0.smithi022.stdout: "pool": 4, 2024-09-21T09:31:02.894 INFO:tasks.workunit.client.0.smithi022.stdout: "size": 3, 2024-09-21T09:31:02.894 INFO:tasks.workunit.client.0.smithi022.stdout: "stats": { 2024-09-21T09:31:02.894 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_raw": 727115966730, 2024-09-21T09:31:02.894 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes_used": 12288, 2024-09-21T09:31:02.894 INFO:tasks.workunit.client.0.smithi022.stdout: "compress_bytes_used": 0, 2024-09-21T09:31:02.894 INFO:tasks.workunit.client.0.smithi022.stdout: "compress_under_bytes": 0, 2024-09-21T09:31:02.894 INFO:tasks.workunit.client.0.smithi022.stdout: "data_bytes_used": 12288, 2024-09-21T09:31:02.894 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty": 0, 2024-09-21T09:31:02.894 INFO:tasks.workunit.client.0.smithi022.stdout: "max_avail": 242371985408, 2024-09-21T09:31:02.894 INFO:tasks.workunit.client.0.smithi022.stdout: "objects": 2, 2024-09-21T09:31:02.894 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_bytes_used": 0, 2024-09-21T09:31:02.894 INFO:tasks.workunit.client.0.smithi022.stdout: "percent_used": 1.689964257423071e-08, 2024-09-21T09:31:02.895 INFO:tasks.workunit.client.0.smithi022.stdout: "quota_bytes": 0, 2024-09-21T09:31:02.895 INFO:tasks.workunit.client.0.smithi022.stdout: "quota_objects": 0, 2024-09-21T09:31:02.895 INFO:tasks.workunit.client.0.smithi022.stdout: "rd": 0, 2024-09-21T09:31:02.895 INFO:tasks.workunit.client.0.smithi022.stdout: "rd_bytes": 0, 2024-09-21T09:31:02.895 INFO:tasks.workunit.client.0.smithi022.stdout: "stored": 19, 2024-09-21T09:31:02.895 INFO:tasks.workunit.client.0.smithi022.stdout: "stored_data": 19, 2024-09-21T09:31:02.895 INFO:tasks.workunit.client.0.smithi022.stdout: "stored_omap": 0, 2024-09-21T09:31:02.895 INFO:tasks.workunit.client.0.smithi022.stdout: "stored_raw": 57, 2024-09-21T09:31:02.895 INFO:tasks.workunit.client.0.smithi022.stdout: "wr": 2, 2024-09-21T09:31:02.895 INFO:tasks.workunit.client.0.smithi022.stdout: "wr_bytes": 2048 2024-09-21T09:31:02.895 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:02.895 INFO:tasks.workunit.client.0.smithi022.stdout: "target_max_bytes": 0, 2024-09-21T09:31:02.895 INFO:tasks.workunit.client.0.smithi022.stdout: "target_max_objects": 0, 2024-09-21T09:31:02.895 INFO:tasks.workunit.client.0.smithi022.stdout: "type": "replicated" 2024-09-21T09:31:02.895 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:02.896 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:02.896 INFO:tasks.workunit.client.0.smithi022.stdout: "rbd": { 2024-09-21T09:31:02.896 INFO:tasks.workunit.client.0.smithi022.stdout: "mirroring_by_pool": [ 2024-09-21T09:31:02.896 INFO:tasks.workunit.client.0.smithi022.stdout: false 2024-09-21T09:31:02.896 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:02.896 INFO:tasks.workunit.client.0.smithi022.stdout: "num_images_by_pool": [ 2024-09-21T09:31:02.896 INFO:tasks.workunit.client.0.smithi022.stdout: 0 2024-09-21T09:31:02.896 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:02.896 INFO:tasks.workunit.client.0.smithi022.stdout: "num_pools": 1 2024-09-21T09:31:02.896 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:02.896 INFO:tasks.workunit.client.0.smithi022.stdout: "report_id": "77addb0d-e829-4757-884e-46deb5965591", 2024-09-21T09:31:02.896 INFO:tasks.workunit.client.0.smithi022.stdout: "report_timestamp": "2024-09-21T09:31:02.850146", 2024-09-21T09:31:02.896 INFO:tasks.workunit.client.0.smithi022.stdout: "report_version": 1, 2024-09-21T09:31:02.896 INFO:tasks.workunit.client.0.smithi022.stdout: "rook": { 2024-09-21T09:31:02.897 INFO:tasks.workunit.client.0.smithi022.stdout: "cluster": { 2024-09-21T09:31:02.897 INFO:tasks.workunit.client.0.smithi022.stdout: "external-mode": null, 2024-09-21T09:31:02.897 INFO:tasks.workunit.client.0.smithi022.stdout: "mon": { 2024-09-21T09:31:02.897 INFO:tasks.workunit.client.0.smithi022.stdout: "allow-multiple-per-node": null, 2024-09-21T09:31:02.897 INFO:tasks.workunit.client.0.smithi022.stdout: "count": null, 2024-09-21T09:31:02.897 INFO:tasks.workunit.client.0.smithi022.stdout: "max-id": null, 2024-09-21T09:31:02.897 INFO:tasks.workunit.client.0.smithi022.stdout: "pvc": { 2024-09-21T09:31:02.897 INFO:tasks.workunit.client.0.smithi022.stdout: "enabled": null 2024-09-21T09:31:02.897 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:02.897 INFO:tasks.workunit.client.0.smithi022.stdout: "stretch": { 2024-09-21T09:31:02.897 INFO:tasks.workunit.client.0.smithi022.stdout: "enabled": null 2024-09-21T09:31:02.897 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:02.897 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:02.897 INFO:tasks.workunit.client.0.smithi022.stdout: "network": { 2024-09-21T09:31:02.898 INFO:tasks.workunit.client.0.smithi022.stdout: "provider": null 2024-09-21T09:31:02.898 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:02.898 INFO:tasks.workunit.client.0.smithi022.stdout: "storage": { 2024-09-21T09:31:02.898 INFO:tasks.workunit.client.0.smithi022.stdout: "device-set": { 2024-09-21T09:31:02.898 INFO:tasks.workunit.client.0.smithi022.stdout: "count": { 2024-09-21T09:31:02.898 INFO:tasks.workunit.client.0.smithi022.stdout: "non-portable": null, 2024-09-21T09:31:02.898 INFO:tasks.workunit.client.0.smithi022.stdout: "portable": null, 2024-09-21T09:31:02.898 INFO:tasks.workunit.client.0.smithi022.stdout: "total": null 2024-09-21T09:31:02.898 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:02.898 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:02.898 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:02.898 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:02.898 INFO:tasks.workunit.client.0.smithi022.stdout: "csi": { 2024-09-21T09:31:02.899 INFO:tasks.workunit.client.0.smithi022.stdout: "version": null 2024-09-21T09:31:02.899 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:02.899 INFO:tasks.workunit.client.0.smithi022.stdout: "kubernetes": { 2024-09-21T09:31:02.899 INFO:tasks.workunit.client.0.smithi022.stdout: "version": null 2024-09-21T09:31:02.899 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:02.899 INFO:tasks.workunit.client.0.smithi022.stdout: "node": { 2024-09-21T09:31:02.899 INFO:tasks.workunit.client.0.smithi022.stdout: "count": { 2024-09-21T09:31:02.899 INFO:tasks.workunit.client.0.smithi022.stdout: "kubernetes-total": null, 2024-09-21T09:31:02.899 INFO:tasks.workunit.client.0.smithi022.stdout: "with-ceph-daemons": null, 2024-09-21T09:31:02.899 INFO:tasks.workunit.client.0.smithi022.stdout: "with-csi-cephfs-plugin": null, 2024-09-21T09:31:02.899 INFO:tasks.workunit.client.0.smithi022.stdout: "with-csi-nfs-plugin": null, 2024-09-21T09:31:02.899 INFO:tasks.workunit.client.0.smithi022.stdout: "with-csi-rbd-plugin": null 2024-09-21T09:31:02.899 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:02.899 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:02.900 INFO:tasks.workunit.client.0.smithi022.stdout: "usage": { 2024-09-21T09:31:02.900 INFO:tasks.workunit.client.0.smithi022.stdout: "storage-class": { 2024-09-21T09:31:02.900 INFO:tasks.workunit.client.0.smithi022.stdout: "count": { 2024-09-21T09:31:02.900 INFO:tasks.workunit.client.0.smithi022.stdout: "bucket": null, 2024-09-21T09:31:02.900 INFO:tasks.workunit.client.0.smithi022.stdout: "cephfs": null, 2024-09-21T09:31:02.900 INFO:tasks.workunit.client.0.smithi022.stdout: "nfs": null, 2024-09-21T09:31:02.900 INFO:tasks.workunit.client.0.smithi022.stdout: "rbd": null, 2024-09-21T09:31:02.900 INFO:tasks.workunit.client.0.smithi022.stdout: "total": null 2024-09-21T09:31:02.900 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:02.900 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:02.900 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:02.900 INFO:tasks.workunit.client.0.smithi022.stdout: "version": null 2024-09-21T09:31:02.900 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:02.900 INFO:tasks.workunit.client.0.smithi022.stdout: "services": {}, 2024-09-21T09:31:02.900 INFO:tasks.workunit.client.0.smithi022.stdout: "usage": { 2024-09-21T09:31:02.901 INFO:tasks.workunit.client.0.smithi022.stdout: "pg_num": 105, 2024-09-21T09:31:02.901 INFO:tasks.workunit.client.0.smithi022.stdout: "pools": 4, 2024-09-21T09:31:02.901 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_by_class": { 2024-09-21T09:31:02.901 INFO:tasks.workunit.client.0.smithi022.stdout: "ssd": { 2024-09-21T09:31:02.901 INFO:tasks.workunit.client.0.smithi022.stdout: "total_avail_bytes": 765516976128, 2024-09-21T09:31:02.901 INFO:tasks.workunit.client.0.smithi022.stdout: "total_bytes": 767960285184, 2024-09-21T09:31:02.901 INFO:tasks.workunit.client.0.smithi022.stdout: "total_used_bytes": 2443309056, 2024-09-21T09:31:02.901 INFO:tasks.workunit.client.0.smithi022.stdout: "total_used_raw_bytes": 2443309056, 2024-09-21T09:31:02.901 INFO:tasks.workunit.client.0.smithi022.stdout: "total_used_raw_ratio": 0.003181556472554803 2024-09-21T09:31:02.901 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:02.901 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:02.901 INFO:tasks.workunit.client.0.smithi022.stdout: "total_avail_bytes": 765516976128, 2024-09-21T09:31:02.901 INFO:tasks.workunit.client.0.smithi022.stdout: "total_bytes": 767960285184, 2024-09-21T09:31:02.901 INFO:tasks.workunit.client.0.smithi022.stdout: "total_used_bytes": 2443309056 2024-09-21T09:31:02.902 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:02.902 INFO:tasks.workunit.client.0.smithi022.stdout:} 2024-09-21T09:31:02.902 INFO:tasks.workunit.client.0.smithi022.stdout:Device report is generated separately. To see it run 'ceph telemetry preview-device'. 2024-09-21T09:31:02.902 INFO:tasks.workunit.client.0.smithi022.stderr:+ ceph telemetry preview-device 2024-09-21T09:31:03.267 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:31:02 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: [21/Sep/2024:09:31:02] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-21T09:31:03.267 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:31:02 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: [21/Sep/2024:09:31:02] ENGINE Bus STOPPED 2024-09-21T09:31:03.267 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:31:02 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: [21/Sep/2024:09:31:02] ENGINE Bus STARTING 2024-09-21T09:31:03.267 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:31:03 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: [21/Sep/2024:09:31:03] ENGINE Serving on http://:::9283 2024-09-21T09:31:03.267 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:31:03 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: [21/Sep/2024:09:31:03] ENGINE Bus STARTED 2024-09-21T09:31:03.355 INFO:tasks.workunit.client.0.smithi022.stdout:{ 2024-09-21T09:31:03.355 INFO:tasks.workunit.client.0.smithi022.stdout: "384b39b0-77fc-11ef-b8c0-0cc47a595de4": { 2024-09-21T09:31:03.356 INFO:tasks.workunit.client.0.smithi022.stdout: "INTEL_SSDPED1D480GA_384bca4c-77fc-11ef-b8c0-0cc47a595de4": { 2024-09-21T09:31:03.356 INFO:tasks.workunit.client.0.smithi022.stdout: "20240921-092509": { 2024-09-21T09:31:03.356 INFO:tasks.workunit.client.0.smithi022.stdout: "device": { 2024-09-21T09:31:03.356 INFO:tasks.workunit.client.0.smithi022.stdout: "info_name": "/dev/nvme0n1", 2024-09-21T09:31:03.356 INFO:tasks.workunit.client.0.smithi022.stdout: "name": "/dev/nvme0n1", 2024-09-21T09:31:03.356 INFO:tasks.workunit.client.0.smithi022.stdout: "protocol": "NVMe", 2024-09-21T09:31:03.356 INFO:tasks.workunit.client.0.smithi022.stdout: "type": "nvme" 2024-09-21T09:31:03.356 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.356 INFO:tasks.workunit.client.0.smithi022.stdout: "firmware_version": "E2010325", 2024-09-21T09:31:03.357 INFO:tasks.workunit.client.0.smithi022.stdout: "host_id": "384b39b0-77fc-11ef-b8c0-0cc47a595de4", 2024-09-21T09:31:03.357 INFO:tasks.workunit.client.0.smithi022.stdout: "json_format_version": [ 2024-09-21T09:31:03.357 INFO:tasks.workunit.client.0.smithi022.stdout: 1, 2024-09-21T09:31:03.357 INFO:tasks.workunit.client.0.smithi022.stdout: 0 2024-09-21T09:31:03.357 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:03.357 INFO:tasks.workunit.client.0.smithi022.stdout: "local_time": { 2024-09-21T09:31:03.357 INFO:tasks.workunit.client.0.smithi022.stdout: "asctime": "Sat Sep 21 09:25:09 2024 UTC", 2024-09-21T09:31:03.357 INFO:tasks.workunit.client.0.smithi022.stdout: "time_t": 1726910709 2024-09-21T09:31:03.357 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.357 INFO:tasks.workunit.client.0.smithi022.stdout: "logical_block_size": 512, 2024-09-21T09:31:03.357 INFO:tasks.workunit.client.0.smithi022.stdout: "model_name": "INTEL SSDPED1D480GA", 2024-09-21T09:31:03.358 INFO:tasks.workunit.client.0.smithi022.stdout: "nvme_controller_id": 0, 2024-09-21T09:31:03.358 INFO:tasks.workunit.client.0.smithi022.stdout: "nvme_ieee_oui_identifier": 6083300, 2024-09-21T09:31:03.358 INFO:tasks.workunit.client.0.smithi022.stdout: "nvme_namespaces": [ 2024-09-21T09:31:03.358 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:03.358 INFO:tasks.workunit.client.0.smithi022.stdout: "capacity": { 2024-09-21T09:31:03.358 INFO:tasks.workunit.client.0.smithi022.stdout: "blocks": 937703088, 2024-09-21T09:31:03.358 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 480103981056 2024-09-21T09:31:03.358 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.358 INFO:tasks.workunit.client.0.smithi022.stdout: "formatted_lba_size": 512, 2024-09-21T09:31:03.358 INFO:tasks.workunit.client.0.smithi022.stdout: "id": 1, 2024-09-21T09:31:03.358 INFO:tasks.workunit.client.0.smithi022.stdout: "size": { 2024-09-21T09:31:03.358 INFO:tasks.workunit.client.0.smithi022.stdout: "blocks": 937703088, 2024-09-21T09:31:03.358 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 480103981056 2024-09-21T09:31:03.359 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.359 INFO:tasks.workunit.client.0.smithi022.stdout: "utilization": { 2024-09-21T09:31:03.359 INFO:tasks.workunit.client.0.smithi022.stdout: "blocks": 937703088, 2024-09-21T09:31:03.359 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 480103981056 2024-09-21T09:31:03.359 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:03.359 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:03.359 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:03.359 INFO:tasks.workunit.client.0.smithi022.stdout: "nvme_number_of_namespaces": 1, 2024-09-21T09:31:03.359 INFO:tasks.workunit.client.0.smithi022.stdout: "nvme_pci_vendor": { 2024-09-21T09:31:03.359 INFO:tasks.workunit.client.0.smithi022.stdout: "id": 32902, 2024-09-21T09:31:03.360 INFO:tasks.workunit.client.0.smithi022.stdout: "subsystem_id": 32902 2024-09-21T09:31:03.360 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.360 INFO:tasks.workunit.client.0.smithi022.stdout: "nvme_smart_health_information_add_log": { 2024-09-21T09:31:03.360 INFO:tasks.workunit.client.0.smithi022.stdout: "Device stats": { 2024-09-21T09:31:03.360 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_fw_downgrades": { 2024-09-21T09:31:03.360 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 0, 2024-09-21T09:31:03.360 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 0 2024-09-21T09:31:03.360 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.360 INFO:tasks.workunit.client.0.smithi022.stdout: "crc_error_count": { 2024-09-21T09:31:03.360 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 100, 2024-09-21T09:31:03.361 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 0 2024-09-21T09:31:03.361 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.361 INFO:tasks.workunit.client.0.smithi022.stdout: "end_to_end_error_detection_count": { 2024-09-21T09:31:03.361 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 100, 2024-09-21T09:31:03.361 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 0 2024-09-21T09:31:03.361 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.361 INFO:tasks.workunit.client.0.smithi022.stdout: "erase_fail_count": { 2024-09-21T09:31:03.361 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 0, 2024-09-21T09:31:03.361 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 0 2024-09-21T09:31:03.361 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.361 INFO:tasks.workunit.client.0.smithi022.stdout: "host_bytes_written": { 2024-09-21T09:31:03.361 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 100, 2024-09-21T09:31:03.362 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 6829746 2024-09-21T09:31:03.362 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.362 INFO:tasks.workunit.client.0.smithi022.stdout: "host_ctx_wear_used": { 2024-09-21T09:31:03.362 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 0, 2024-09-21T09:31:03.362 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 0 2024-09-21T09:31:03.362 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.362 INFO:tasks.workunit.client.0.smithi022.stdout: "media_bytes_read": { 2024-09-21T09:31:03.362 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 0, 2024-09-21T09:31:03.362 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 0 2024-09-21T09:31:03.362 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.362 INFO:tasks.workunit.client.0.smithi022.stdout: "nand_bytes_written": { 2024-09-21T09:31:03.362 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 0, 2024-09-21T09:31:03.362 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 0 2024-09-21T09:31:03.362 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.363 INFO:tasks.workunit.client.0.smithi022.stdout: "perf_stat_indicator": { 2024-09-21T09:31:03.363 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 0, 2024-09-21T09:31:03.363 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 0 2024-09-21T09:31:03.363 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.363 INFO:tasks.workunit.client.0.smithi022.stdout: "pll_lock_loss_count": { 2024-09-21T09:31:03.363 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 100, 2024-09-21T09:31:03.363 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 0 2024-09-21T09:31:03.363 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.363 INFO:tasks.workunit.client.0.smithi022.stdout: "program_fail_count": { 2024-09-21T09:31:03.363 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 0, 2024-09-21T09:31:03.363 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 0 2024-09-21T09:31:03.363 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.363 INFO:tasks.workunit.client.0.smithi022.stdout: "re_alloc_sectr_cnt": { 2024-09-21T09:31:03.363 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 0, 2024-09-21T09:31:03.364 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 0 2024-09-21T09:31:03.364 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.364 INFO:tasks.workunit.client.0.smithi022.stdout: "retry_buffer_overflow_count": { 2024-09-21T09:31:03.364 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 100, 2024-09-21T09:31:03.364 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 0 2024-09-21T09:31:03.364 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.364 INFO:tasks.workunit.client.0.smithi022.stdout: "soft_ecc_err_rate": { 2024-09-21T09:31:03.364 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 0, 2024-09-21T09:31:03.364 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 0 2024-09-21T09:31:03.364 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.364 INFO:tasks.workunit.client.0.smithi022.stdout: "thermal_throttle_status": { 2024-09-21T09:31:03.364 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 100, 2024-09-21T09:31:03.364 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": { 2024-09-21T09:31:03.364 INFO:tasks.workunit.client.0.smithi022.stdout: "cnt": 0, 2024-09-21T09:31:03.364 INFO:tasks.workunit.client.0.smithi022.stdout: "pct": 0 2024-09-21T09:31:03.365 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:03.365 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.365 INFO:tasks.workunit.client.0.smithi022.stdout: "timed_workload_host_reads": { 2024-09-21T09:31:03.365 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 100, 2024-09-21T09:31:03.365 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 16 2024-09-21T09:31:03.365 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.365 INFO:tasks.workunit.client.0.smithi022.stdout: "timed_workload_media_wear": { 2024-09-21T09:31:03.365 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 100, 2024-09-21T09:31:03.365 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": "0.000000" 2024-09-21T09:31:03.365 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.365 INFO:tasks.workunit.client.0.smithi022.stdout: "timed_workload_timer": { 2024-09-21T09:31:03.365 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 100, 2024-09-21T09:31:03.365 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 1212022 2024-09-21T09:31:03.366 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.366 INFO:tasks.workunit.client.0.smithi022.stdout: "unexp_power_loss": { 2024-09-21T09:31:03.366 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 0, 2024-09-21T09:31:03.366 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 0 2024-09-21T09:31:03.366 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.366 INFO:tasks.workunit.client.0.smithi022.stdout: "wear_leveling": { 2024-09-21T09:31:03.366 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 0, 2024-09-21T09:31:03.366 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": { 2024-09-21T09:31:03.366 INFO:tasks.workunit.client.0.smithi022.stdout: "avg": 0, 2024-09-21T09:31:03.366 INFO:tasks.workunit.client.0.smithi022.stdout: "max": 0, 2024-09-21T09:31:03.366 INFO:tasks.workunit.client.0.smithi022.stdout: "min": 0 2024-09-21T09:31:03.366 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:03.367 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:03.367 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.367 INFO:tasks.workunit.client.0.smithi022.stdout: "Intel Smart log": "nvme0n1" 2024-09-21T09:31:03.367 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.367 INFO:tasks.workunit.client.0.smithi022.stdout: "nvme_smart_health_information_log": { 2024-09-21T09:31:03.367 INFO:tasks.workunit.client.0.smithi022.stdout: "available_spare": 100, 2024-09-21T09:31:03.367 INFO:tasks.workunit.client.0.smithi022.stdout: "available_spare_threshold": 0, 2024-09-21T09:31:03.367 INFO:tasks.workunit.client.0.smithi022.stdout: "controller_busy_time": 2680, 2024-09-21T09:31:03.367 INFO:tasks.workunit.client.0.smithi022.stdout: "critical_warning": 0, 2024-09-21T09:31:03.367 INFO:tasks.workunit.client.0.smithi022.stdout: "data_units_read": 95367672, 2024-09-21T09:31:03.367 INFO:tasks.workunit.client.0.smithi022.stdout: "data_units_written": 447594268, 2024-09-21T09:31:03.368 INFO:tasks.workunit.client.0.smithi022.stdout: "host_reads": 1403952490, 2024-09-21T09:31:03.368 INFO:tasks.workunit.client.0.smithi022.stdout: "host_writes": 10869660916, 2024-09-21T09:31:03.368 INFO:tasks.workunit.client.0.smithi022.stdout: "media_errors": 0, 2024-09-21T09:31:03.368 INFO:tasks.workunit.client.0.smithi022.stdout: "num_err_log_entries": 0, 2024-09-21T09:31:03.368 INFO:tasks.workunit.client.0.smithi022.stdout: "percentage_used": 0, 2024-09-21T09:31:03.368 INFO:tasks.workunit.client.0.smithi022.stdout: "power_cycles": 16189, 2024-09-21T09:31:03.368 INFO:tasks.workunit.client.0.smithi022.stdout: "power_on_hours": 27263, 2024-09-21T09:31:03.368 INFO:tasks.workunit.client.0.smithi022.stdout: "temperature": 31, 2024-09-21T09:31:03.369 INFO:tasks.workunit.client.0.smithi022.stdout: "unsafe_shutdowns": 5631 2024-09-21T09:31:03.369 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.369 INFO:tasks.workunit.client.0.smithi022.stdout: "nvme_vendor": "intel", 2024-09-21T09:31:03.369 INFO:tasks.workunit.client.0.smithi022.stdout: "nvme_version": { 2024-09-21T09:31:03.369 INFO:tasks.workunit.client.0.smithi022.stdout: "string": "<1.2", 2024-09-21T09:31:03.369 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:03.369 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.369 INFO:tasks.workunit.client.0.smithi022.stdout: "power_cycle_count": 16189, 2024-09-21T09:31:03.369 INFO:tasks.workunit.client.0.smithi022.stdout: "power_on_time": { 2024-09-21T09:31:03.369 INFO:tasks.workunit.client.0.smithi022.stdout: "hours": 27263 2024-09-21T09:31:03.369 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.369 INFO:tasks.workunit.client.0.smithi022.stdout: "serial_number": "deleted", 2024-09-21T09:31:03.369 INFO:tasks.workunit.client.0.smithi022.stdout: "smart_status": { 2024-09-21T09:31:03.369 INFO:tasks.workunit.client.0.smithi022.stdout: "nvme": { 2024-09-21T09:31:03.369 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:03.370 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.370 INFO:tasks.workunit.client.0.smithi022.stdout: "passed": true 2024-09-21T09:31:03.370 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.370 INFO:tasks.workunit.client.0.smithi022.stdout: "smartctl": { 2024-09-21T09:31:03.370 INFO:tasks.workunit.client.0.smithi022.stdout: "argv": [ 2024-09-21T09:31:03.370 INFO:tasks.workunit.client.0.smithi022.stdout: "smartctl", 2024-09-21T09:31:03.370 INFO:tasks.workunit.client.0.smithi022.stdout: "-x", 2024-09-21T09:31:03.370 INFO:tasks.workunit.client.0.smithi022.stdout: "--json=o", 2024-09-21T09:31:03.370 INFO:tasks.workunit.client.0.smithi022.stdout: "/dev/nvme0n1" 2024-09-21T09:31:03.370 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:03.370 INFO:tasks.workunit.client.0.smithi022.stdout: "build_info": "(local build)", 2024-09-21T09:31:03.370 INFO:tasks.workunit.client.0.smithi022.stdout: "exit_status": 0, 2024-09-21T09:31:03.370 INFO:tasks.workunit.client.0.smithi022.stdout: "output": [ 2024-09-21T09:31:03.371 INFO:tasks.workunit.client.0.smithi022.stdout: "smartctl 7.2 2020-12-30 r5155 [x86_64-linux-5.14.0-508.el9.x86_64] (local build)", 2024-09-21T09:31:03.371 INFO:tasks.workunit.client.0.smithi022.stdout: "Copyright (C) 2002-20, Bruce Allen, Christian Franke, www.smartmontools.org", 2024-09-21T09:31:03.371 INFO:tasks.workunit.client.0.smithi022.stdout: "", 2024-09-21T09:31:03.371 INFO:tasks.workunit.client.0.smithi022.stdout: "=== START OF INFORMATION SECTION ===", 2024-09-21T09:31:03.371 INFO:tasks.workunit.client.0.smithi022.stdout: "Model Number: INTEL SSDPED1D480GA", 2024-09-21T09:31:03.371 INFO:tasks.workunit.client.0.smithi022.stdout: "Serial Number: deleted", 2024-09-21T09:31:03.371 INFO:tasks.workunit.client.0.smithi022.stdout: "Firmware Version: E2010325", 2024-09-21T09:31:03.371 INFO:tasks.workunit.client.0.smithi022.stdout: "PCI Vendor/Subsystem ID: 0x8086", 2024-09-21T09:31:03.371 INFO:tasks.workunit.client.0.smithi022.stdout: "IEEE OUI Identifier: 0x5cd2e4", 2024-09-21T09:31:03.371 INFO:tasks.workunit.client.0.smithi022.stdout: "Controller ID: 0", 2024-09-21T09:31:03.371 INFO:tasks.workunit.client.0.smithi022.stdout: "NVMe Version: <1.2", 2024-09-21T09:31:03.372 INFO:tasks.workunit.client.0.smithi022.stdout: "Number of Namespaces: 1", 2024-09-21T09:31:03.372 INFO:tasks.workunit.client.0.smithi022.stdout: "Namespace 1 Size/Capacity: 480,103,981,056 [480 GB]", 2024-09-21T09:31:03.372 INFO:tasks.workunit.client.0.smithi022.stdout: "Namespace 1 Formatted LBA Size: 512", 2024-09-21T09:31:03.372 INFO:tasks.workunit.client.0.smithi022.stdout: "Local Time is: Sat Sep 21 09:25:09 2024 UTC", 2024-09-21T09:31:03.372 INFO:tasks.workunit.client.0.smithi022.stdout: "Firmware Updates (0x02): 1 Slot", 2024-09-21T09:31:03.372 INFO:tasks.workunit.client.0.smithi022.stdout: "Optional Admin Commands (0x0007): Security Format Frmw_DL", 2024-09-21T09:31:03.372 INFO:tasks.workunit.client.0.smithi022.stdout: "Optional NVM Commands (0x0006): Wr_Unc DS_Mngmt", 2024-09-21T09:31:03.372 INFO:tasks.workunit.client.0.smithi022.stdout: "Log Page Attributes (0x02): Cmd_Eff_Lg", 2024-09-21T09:31:03.372 INFO:tasks.workunit.client.0.smithi022.stdout: "Maximum Data Transfer Size: 32 Pages", 2024-09-21T09:31:03.372 INFO:tasks.workunit.client.0.smithi022.stdout: "", 2024-09-21T09:31:03.372 INFO:tasks.workunit.client.0.smithi022.stdout: "Supported Power States", 2024-09-21T09:31:03.372 INFO:tasks.workunit.client.0.smithi022.stdout: "St Op Max Active Idle RL RT WL WT Ent_Lat Ex_Lat", 2024-09-21T09:31:03.372 INFO:tasks.workunit.client.0.smithi022.stdout: " 0 + 18.00W - - 0 0 0 0 0 0", 2024-09-21T09:31:03.372 INFO:tasks.workunit.client.0.smithi022.stdout: "", 2024-09-21T09:31:03.373 INFO:tasks.workunit.client.0.smithi022.stdout: "Supported LBA Sizes (NSID 0x1)", 2024-09-21T09:31:03.373 INFO:tasks.workunit.client.0.smithi022.stdout: "Id Fmt Data Metadt Rel_Perf", 2024-09-21T09:31:03.373 INFO:tasks.workunit.client.0.smithi022.stdout: " 0 + 512 0 2", 2024-09-21T09:31:03.373 INFO:tasks.workunit.client.0.smithi022.stdout: "", 2024-09-21T09:31:03.373 INFO:tasks.workunit.client.0.smithi022.stdout: "=== START OF SMART DATA SECTION ===", 2024-09-21T09:31:03.373 INFO:tasks.workunit.client.0.smithi022.stdout: "SMART overall-health self-assessment test result: PASSED", 2024-09-21T09:31:03.373 INFO:tasks.workunit.client.0.smithi022.stdout: "", 2024-09-21T09:31:03.373 INFO:tasks.workunit.client.0.smithi022.stdout: "SMART/Health Information (NVMe Log 0x02)", 2024-09-21T09:31:03.373 INFO:tasks.workunit.client.0.smithi022.stdout: "Critical Warning: 0x00", 2024-09-21T09:31:03.373 INFO:tasks.workunit.client.0.smithi022.stdout: "Temperature: 31 Celsius", 2024-09-21T09:31:03.373 INFO:tasks.workunit.client.0.smithi022.stdout: "Available Spare: 100%", 2024-09-21T09:31:03.373 INFO:tasks.workunit.client.0.smithi022.stdout: "Available Spare Threshold: 0%", 2024-09-21T09:31:03.373 INFO:tasks.workunit.client.0.smithi022.stdout: "Percentage Used: 0%", 2024-09-21T09:31:03.373 INFO:tasks.workunit.client.0.smithi022.stdout: "Data Units Read: 95,367,672 [48.8 TB]", 2024-09-21T09:31:03.373 INFO:tasks.workunit.client.0.smithi022.stdout: "Data Units Written: 447,594,268 [229 TB]", 2024-09-21T09:31:03.374 INFO:tasks.workunit.client.0.smithi022.stdout: "Host Read Commands: 1,403,952,490", 2024-09-21T09:31:03.374 INFO:tasks.workunit.client.0.smithi022.stdout: "Host Write Commands: 10,869,660,916", 2024-09-21T09:31:03.374 INFO:tasks.workunit.client.0.smithi022.stdout: "Controller Busy Time: 2,680", 2024-09-21T09:31:03.374 INFO:tasks.workunit.client.0.smithi022.stdout: "Power Cycles: 16,189", 2024-09-21T09:31:03.374 INFO:tasks.workunit.client.0.smithi022.stdout: "Power On Hours: 27,263", 2024-09-21T09:31:03.374 INFO:tasks.workunit.client.0.smithi022.stdout: "Unsafe Shutdowns: 5,631", 2024-09-21T09:31:03.374 INFO:tasks.workunit.client.0.smithi022.stdout: "Media and Data Integrity Errors: 0", 2024-09-21T09:31:03.374 INFO:tasks.workunit.client.0.smithi022.stdout: "Error Information Log Entries: 0", 2024-09-21T09:31:03.374 INFO:tasks.workunit.client.0.smithi022.stdout: "", 2024-09-21T09:31:03.374 INFO:tasks.workunit.client.0.smithi022.stdout: "Error Information (NVMe Log 0x01, 16 of 64 entries)", 2024-09-21T09:31:03.374 INFO:tasks.workunit.client.0.smithi022.stdout: "No Errors Logged", 2024-09-21T09:31:03.374 INFO:tasks.workunit.client.0.smithi022.stdout: "" 2024-09-21T09:31:03.375 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:03.375 INFO:tasks.workunit.client.0.smithi022.stdout: "platform_info": "x86_64-linux-5.14.0-508.el9.x86_64", 2024-09-21T09:31:03.375 INFO:tasks.workunit.client.0.smithi022.stdout: "svn_revision": "5155", 2024-09-21T09:31:03.375 INFO:tasks.workunit.client.0.smithi022.stdout: "version": [ 2024-09-21T09:31:03.375 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:03.375 INFO:tasks.workunit.client.0.smithi022.stdout: 2 2024-09-21T09:31:03.375 INFO:tasks.workunit.client.0.smithi022.stdout: ] 2024-09-21T09:31:03.375 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.375 INFO:tasks.workunit.client.0.smithi022.stdout: "temperature": { 2024-09-21T09:31:03.375 INFO:tasks.workunit.client.0.smithi022.stdout: "current": 31 2024-09-21T09:31:03.375 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.375 INFO:tasks.workunit.client.0.smithi022.stdout: "user_capacity": { 2024-09-21T09:31:03.375 INFO:tasks.workunit.client.0.smithi022.stdout: "blocks": 937703088, 2024-09-21T09:31:03.375 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 480103981056 2024-09-21T09:31:03.376 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:03.376 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:03.376 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:03.376 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.376 INFO:tasks.workunit.client.0.smithi022.stdout: "384d1fc8-77fc-11ef-b8c0-0cc47a595de4": { 2024-09-21T09:31:03.376 INFO:tasks.workunit.client.0.smithi022.stdout: "INTEL_SSDPEDMD400G4_384da218-77fc-11ef-b8c0-0cc47a595de4": { 2024-09-21T09:31:03.376 INFO:tasks.workunit.client.0.smithi022.stdout: "20240921-092513": { 2024-09-21T09:31:03.376 INFO:tasks.workunit.client.0.smithi022.stdout: "device": { 2024-09-21T09:31:03.376 INFO:tasks.workunit.client.0.smithi022.stdout: "info_name": "/dev/nvme0n1", 2024-09-21T09:31:03.376 INFO:tasks.workunit.client.0.smithi022.stdout: "name": "/dev/nvme0n1", 2024-09-21T09:31:03.376 INFO:tasks.workunit.client.0.smithi022.stdout: "protocol": "NVMe", 2024-09-21T09:31:03.376 INFO:tasks.workunit.client.0.smithi022.stdout: "type": "nvme" 2024-09-21T09:31:03.376 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.376 INFO:tasks.workunit.client.0.smithi022.stdout: "firmware_version": "8DV101H0", 2024-09-21T09:31:03.377 INFO:tasks.workunit.client.0.smithi022.stdout: "host_id": "384d1fc8-77fc-11ef-b8c0-0cc47a595de4", 2024-09-21T09:31:03.377 INFO:tasks.workunit.client.0.smithi022.stdout: "json_format_version": [ 2024-09-21T09:31:03.377 INFO:tasks.workunit.client.0.smithi022.stdout: 1, 2024-09-21T09:31:03.377 INFO:tasks.workunit.client.0.smithi022.stdout: 0 2024-09-21T09:31:03.377 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:03.377 INFO:tasks.workunit.client.0.smithi022.stdout: "local_time": { 2024-09-21T09:31:03.377 INFO:tasks.workunit.client.0.smithi022.stdout: "asctime": "Sat Sep 21 09:25:12 2024 UTC", 2024-09-21T09:31:03.378 INFO:tasks.workunit.client.0.smithi022.stdout: "time_t": 1726910712 2024-09-21T09:31:03.378 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.378 INFO:tasks.workunit.client.0.smithi022.stdout: "logical_block_size": 512, 2024-09-21T09:31:03.378 INFO:tasks.workunit.client.0.smithi022.stdout: "model_name": "INTEL SSDPEDMD400G4", 2024-09-21T09:31:03.378 INFO:tasks.workunit.client.0.smithi022.stdout: "nvme_controller_id": 0, 2024-09-21T09:31:03.378 INFO:tasks.workunit.client.0.smithi022.stdout: "nvme_ieee_oui_identifier": 6083300, 2024-09-21T09:31:03.378 INFO:tasks.workunit.client.0.smithi022.stdout: "nvme_namespaces": [ 2024-09-21T09:31:03.378 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:03.379 INFO:tasks.workunit.client.0.smithi022.stdout: "capacity": { 2024-09-21T09:31:03.379 INFO:tasks.workunit.client.0.smithi022.stdout: "blocks": 781422768, 2024-09-21T09:31:03.379 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 400088457216 2024-09-21T09:31:03.379 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.379 INFO:tasks.workunit.client.0.smithi022.stdout: "formatted_lba_size": 512, 2024-09-21T09:31:03.379 INFO:tasks.workunit.client.0.smithi022.stdout: "id": 1, 2024-09-21T09:31:03.379 INFO:tasks.workunit.client.0.smithi022.stdout: "size": { 2024-09-21T09:31:03.379 INFO:tasks.workunit.client.0.smithi022.stdout: "blocks": 781422768, 2024-09-21T09:31:03.379 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 400088457216 2024-09-21T09:31:03.380 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.380 INFO:tasks.workunit.client.0.smithi022.stdout: "utilization": { 2024-09-21T09:31:03.380 INFO:tasks.workunit.client.0.smithi022.stdout: "blocks": 781422768, 2024-09-21T09:31:03.380 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 400088457216 2024-09-21T09:31:03.380 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:03.380 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:03.380 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:03.380 INFO:tasks.workunit.client.0.smithi022.stdout: "nvme_number_of_namespaces": 1, 2024-09-21T09:31:03.380 INFO:tasks.workunit.client.0.smithi022.stdout: "nvme_pci_vendor": { 2024-09-21T09:31:03.380 INFO:tasks.workunit.client.0.smithi022.stdout: "id": 32902, 2024-09-21T09:31:03.380 INFO:tasks.workunit.client.0.smithi022.stdout: "subsystem_id": 32902 2024-09-21T09:31:03.381 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.381 INFO:tasks.workunit.client.0.smithi022.stdout: "nvme_smart_health_information_add_log": { 2024-09-21T09:31:03.381 INFO:tasks.workunit.client.0.smithi022.stdout: "Device stats": { 2024-09-21T09:31:03.381 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_fw_downgrades": { 2024-09-21T09:31:03.381 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 0, 2024-09-21T09:31:03.381 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 0 2024-09-21T09:31:03.381 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.381 INFO:tasks.workunit.client.0.smithi022.stdout: "crc_error_count": { 2024-09-21T09:31:03.381 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 100, 2024-09-21T09:31:03.381 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 0 2024-09-21T09:31:03.381 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.382 INFO:tasks.workunit.client.0.smithi022.stdout: "end_to_end_error_detection_count": { 2024-09-21T09:31:03.382 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 100, 2024-09-21T09:31:03.382 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 0 2024-09-21T09:31:03.382 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.382 INFO:tasks.workunit.client.0.smithi022.stdout: "erase_fail_count": { 2024-09-21T09:31:03.382 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 100, 2024-09-21T09:31:03.382 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 0 2024-09-21T09:31:03.382 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.382 INFO:tasks.workunit.client.0.smithi022.stdout: "host_bytes_written": { 2024-09-21T09:31:03.382 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 100, 2024-09-21T09:31:03.382 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 34797061 2024-09-21T09:31:03.383 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.383 INFO:tasks.workunit.client.0.smithi022.stdout: "host_ctx_wear_used": { 2024-09-21T09:31:03.383 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 100, 2024-09-21T09:31:03.383 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 1 2024-09-21T09:31:03.383 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.383 INFO:tasks.workunit.client.0.smithi022.stdout: "media_bytes_read": { 2024-09-21T09:31:03.383 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 0, 2024-09-21T09:31:03.383 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 0 2024-09-21T09:31:03.383 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.383 INFO:tasks.workunit.client.0.smithi022.stdout: "nand_bytes_written": { 2024-09-21T09:31:03.383 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 100, 2024-09-21T09:31:03.383 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 57236814 2024-09-21T09:31:03.383 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.383 INFO:tasks.workunit.client.0.smithi022.stdout: "perf_stat_indicator": { 2024-09-21T09:31:03.383 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 0, 2024-09-21T09:31:03.384 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 0 2024-09-21T09:31:03.384 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.384 INFO:tasks.workunit.client.0.smithi022.stdout: "pll_lock_loss_count": { 2024-09-21T09:31:03.384 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 100, 2024-09-21T09:31:03.384 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 0 2024-09-21T09:31:03.384 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.384 INFO:tasks.workunit.client.0.smithi022.stdout: "program_fail_count": { 2024-09-21T09:31:03.384 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 100, 2024-09-21T09:31:03.384 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 0 2024-09-21T09:31:03.384 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.384 INFO:tasks.workunit.client.0.smithi022.stdout: "re_alloc_sectr_cnt": { 2024-09-21T09:31:03.384 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 0, 2024-09-21T09:31:03.385 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 0 2024-09-21T09:31:03.385 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.385 INFO:tasks.workunit.client.0.smithi022.stdout: "retry_buffer_overflow_count": { 2024-09-21T09:31:03.385 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 100, 2024-09-21T09:31:03.385 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 0 2024-09-21T09:31:03.385 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.385 INFO:tasks.workunit.client.0.smithi022.stdout: "soft_ecc_err_rate": { 2024-09-21T09:31:03.385 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 0, 2024-09-21T09:31:03.385 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 0 2024-09-21T09:31:03.385 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.385 INFO:tasks.workunit.client.0.smithi022.stdout: "thermal_throttle_status": { 2024-09-21T09:31:03.385 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 100, 2024-09-21T09:31:03.385 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": { 2024-09-21T09:31:03.386 INFO:tasks.workunit.client.0.smithi022.stdout: "cnt": 0, 2024-09-21T09:31:03.386 INFO:tasks.workunit.client.0.smithi022.stdout: "pct": 0 2024-09-21T09:31:03.386 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:03.386 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.386 INFO:tasks.workunit.client.0.smithi022.stdout: "timed_workload_host_reads": { 2024-09-21T09:31:03.386 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 100, 2024-09-21T09:31:03.386 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 65535 2024-09-21T09:31:03.386 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.386 INFO:tasks.workunit.client.0.smithi022.stdout: "timed_workload_media_wear": { 2024-09-21T09:31:03.386 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 100, 2024-09-21T09:31:03.386 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": "63.999023" 2024-09-21T09:31:03.386 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.387 INFO:tasks.workunit.client.0.smithi022.stdout: "timed_workload_timer": { 2024-09-21T09:31:03.387 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 100, 2024-09-21T09:31:03.387 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 65535 2024-09-21T09:31:03.387 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.387 INFO:tasks.workunit.client.0.smithi022.stdout: "unexp_power_loss": { 2024-09-21T09:31:03.387 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 0, 2024-09-21T09:31:03.387 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 0 2024-09-21T09:31:03.387 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.388 INFO:tasks.workunit.client.0.smithi022.stdout: "wear_leveling": { 2024-09-21T09:31:03.388 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 88, 2024-09-21T09:31:03.388 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": { 2024-09-21T09:31:03.388 INFO:tasks.workunit.client.0.smithi022.stdout: "avg": 3844, 2024-09-21T09:31:03.388 INFO:tasks.workunit.client.0.smithi022.stdout: "max": 3860, 2024-09-21T09:31:03.388 INFO:tasks.workunit.client.0.smithi022.stdout: "min": 3828 2024-09-21T09:31:03.388 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:03.388 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:03.388 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.388 INFO:tasks.workunit.client.0.smithi022.stdout: "Intel Smart log": "nvme0n1" 2024-09-21T09:31:03.388 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.388 INFO:tasks.workunit.client.0.smithi022.stdout: "nvme_smart_health_information_log": { 2024-09-21T09:31:03.389 INFO:tasks.workunit.client.0.smithi022.stdout: "available_spare": 100, 2024-09-21T09:31:03.389 INFO:tasks.workunit.client.0.smithi022.stdout: "available_spare_threshold": 10, 2024-09-21T09:31:03.389 INFO:tasks.workunit.client.0.smithi022.stdout: "controller_busy_time": 39, 2024-09-21T09:31:03.389 INFO:tasks.workunit.client.0.smithi022.stdout: "critical_warning": 0, 2024-09-21T09:31:03.389 INFO:tasks.workunit.client.0.smithi022.stdout: "data_units_read": 467647523, 2024-09-21T09:31:03.389 INFO:tasks.workunit.client.0.smithi022.stdout: "data_units_written": 2280460231, 2024-09-21T09:31:03.389 INFO:tasks.workunit.client.0.smithi022.stdout: "host_reads": 8064859300, 2024-09-21T09:31:03.389 INFO:tasks.workunit.client.0.smithi022.stdout: "host_writes": 41665631302, 2024-09-21T09:31:03.390 INFO:tasks.workunit.client.0.smithi022.stdout: "media_errors": 0, 2024-09-21T09:31:03.390 INFO:tasks.workunit.client.0.smithi022.stdout: "num_err_log_entries": 0, 2024-09-21T09:31:03.390 INFO:tasks.workunit.client.0.smithi022.stdout: "percentage_used": 12, 2024-09-21T09:31:03.390 INFO:tasks.workunit.client.0.smithi022.stdout: "power_cycles": 55086, 2024-09-21T09:31:03.390 INFO:tasks.workunit.client.0.smithi022.stdout: "power_on_hours": 60974, 2024-09-21T09:31:03.390 INFO:tasks.workunit.client.0.smithi022.stdout: "temperature": 22, 2024-09-21T09:31:03.390 INFO:tasks.workunit.client.0.smithi022.stdout: "unsafe_shutdowns": 2616 2024-09-21T09:31:03.390 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.390 INFO:tasks.workunit.client.0.smithi022.stdout: "nvme_vendor": "intel", 2024-09-21T09:31:03.391 INFO:tasks.workunit.client.0.smithi022.stdout: "nvme_version": { 2024-09-21T09:31:03.391 INFO:tasks.workunit.client.0.smithi022.stdout: "string": "<1.2", 2024-09-21T09:31:03.391 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:03.391 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.391 INFO:tasks.workunit.client.0.smithi022.stdout: "power_cycle_count": 55086, 2024-09-21T09:31:03.391 INFO:tasks.workunit.client.0.smithi022.stdout: "power_on_time": { 2024-09-21T09:31:03.391 INFO:tasks.workunit.client.0.smithi022.stdout: "hours": 60974 2024-09-21T09:31:03.391 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.392 INFO:tasks.workunit.client.0.smithi022.stdout: "serial_number": "deleted", 2024-09-21T09:31:03.392 INFO:tasks.workunit.client.0.smithi022.stdout: "smart_status": { 2024-09-21T09:31:03.392 INFO:tasks.workunit.client.0.smithi022.stdout: "nvme": { 2024-09-21T09:31:03.392 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:03.392 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.392 INFO:tasks.workunit.client.0.smithi022.stdout: "passed": true 2024-09-21T09:31:03.392 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.393 INFO:tasks.workunit.client.0.smithi022.stdout: "smartctl": { 2024-09-21T09:31:03.393 INFO:tasks.workunit.client.0.smithi022.stdout: "argv": [ 2024-09-21T09:31:03.393 INFO:tasks.workunit.client.0.smithi022.stdout: "smartctl", 2024-09-21T09:31:03.393 INFO:tasks.workunit.client.0.smithi022.stdout: "-x", 2024-09-21T09:31:03.393 INFO:tasks.workunit.client.0.smithi022.stdout: "--json=o", 2024-09-21T09:31:03.393 INFO:tasks.workunit.client.0.smithi022.stdout: "/dev/nvme0n1" 2024-09-21T09:31:03.393 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:03.394 INFO:tasks.workunit.client.0.smithi022.stdout: "build_info": "(local build)", 2024-09-21T09:31:03.394 INFO:tasks.workunit.client.0.smithi022.stdout: "exit_status": 0, 2024-09-21T09:31:03.394 INFO:tasks.workunit.client.0.smithi022.stdout: "output": [ 2024-09-21T09:31:03.394 INFO:tasks.workunit.client.0.smithi022.stdout: "smartctl 7.2 2020-12-30 r5155 [x86_64-linux-5.14.0-508.el9.x86_64] (local build)", 2024-09-21T09:31:03.394 INFO:tasks.workunit.client.0.smithi022.stdout: "Copyright (C) 2002-20, Bruce Allen, Christian Franke, www.smartmontools.org", 2024-09-21T09:31:03.394 INFO:tasks.workunit.client.0.smithi022.stdout: "", 2024-09-21T09:31:03.395 INFO:tasks.workunit.client.0.smithi022.stdout: "=== START OF INFORMATION SECTION ===", 2024-09-21T09:31:03.395 INFO:tasks.workunit.client.0.smithi022.stdout: "Model Number: INTEL SSDPEDMD400G4", 2024-09-21T09:31:03.395 INFO:tasks.workunit.client.0.smithi022.stdout: "Serial Number: deleted", 2024-09-21T09:31:03.395 INFO:tasks.workunit.client.0.smithi022.stdout: "Firmware Version: 8DV101H0", 2024-09-21T09:31:03.395 INFO:tasks.workunit.client.0.smithi022.stdout: "PCI Vendor/Subsystem ID: 0x8086", 2024-09-21T09:31:03.395 INFO:tasks.workunit.client.0.smithi022.stdout: "IEEE OUI Identifier: 0x5cd2e4", 2024-09-21T09:31:03.395 INFO:tasks.workunit.client.0.smithi022.stdout: "Controller ID: 0", 2024-09-21T09:31:03.396 INFO:tasks.workunit.client.0.smithi022.stdout: "NVMe Version: <1.2", 2024-09-21T09:31:03.396 INFO:tasks.workunit.client.0.smithi022.stdout: "Number of Namespaces: 1", 2024-09-21T09:31:03.396 INFO:tasks.workunit.client.0.smithi022.stdout: "Namespace 1 Size/Capacity: 400,088,457,216 [400 GB]", 2024-09-21T09:31:03.396 INFO:tasks.workunit.client.0.smithi022.stdout: "Namespace 1 Formatted LBA Size: 512", 2024-09-21T09:31:03.396 INFO:tasks.workunit.client.0.smithi022.stdout: "Local Time is: Sat Sep 21 09:25:12 2024 UTC", 2024-09-21T09:31:03.396 INFO:tasks.workunit.client.0.smithi022.stdout: "Firmware Updates (0x02): 1 Slot", 2024-09-21T09:31:03.397 INFO:tasks.workunit.client.0.smithi022.stdout: "Optional Admin Commands (0x0006): Format Frmw_DL", 2024-09-21T09:31:03.397 INFO:tasks.workunit.client.0.smithi022.stdout: "Optional NVM Commands (0x0006): Wr_Unc DS_Mngmt", 2024-09-21T09:31:03.397 INFO:tasks.workunit.client.0.smithi022.stdout: "Log Page Attributes (0x02): Cmd_Eff_Lg", 2024-09-21T09:31:03.397 INFO:tasks.workunit.client.0.smithi022.stdout: "Maximum Data Transfer Size: 32 Pages", 2024-09-21T09:31:03.397 INFO:tasks.workunit.client.0.smithi022.stdout: "", 2024-09-21T09:31:03.397 INFO:tasks.workunit.client.0.smithi022.stdout: "Supported Power States", 2024-09-21T09:31:03.397 INFO:tasks.workunit.client.0.smithi022.stdout: "St Op Max Active Idle RL RT WL WT Ent_Lat Ex_Lat", 2024-09-21T09:31:03.398 INFO:tasks.workunit.client.0.smithi022.stdout: " 0 + 25.00W - - 0 0 0 0 0 0", 2024-09-21T09:31:03.398 INFO:tasks.workunit.client.0.smithi022.stdout: "", 2024-09-21T09:31:03.398 INFO:tasks.workunit.client.0.smithi022.stdout: "Supported LBA Sizes (NSID 0x1)", 2024-09-21T09:31:03.398 INFO:tasks.workunit.client.0.smithi022.stdout: "Id Fmt Data Metadt Rel_Perf", 2024-09-21T09:31:03.398 INFO:tasks.workunit.client.0.smithi022.stdout: " 0 + 512 0 2", 2024-09-21T09:31:03.398 INFO:tasks.workunit.client.0.smithi022.stdout: " 1 - 512 8 2", 2024-09-21T09:31:03.399 INFO:tasks.workunit.client.0.smithi022.stdout: " 2 - 512 16 2", 2024-09-21T09:31:03.399 INFO:tasks.workunit.client.0.smithi022.stdout: " 3 - 4096 0 0", 2024-09-21T09:31:03.399 INFO:tasks.workunit.client.0.smithi022.stdout: " 4 - 4096 8 0", 2024-09-21T09:31:03.399 INFO:tasks.workunit.client.0.smithi022.stdout: " 5 - 4096 64 0", 2024-09-21T09:31:03.399 INFO:tasks.workunit.client.0.smithi022.stdout: " 6 - 4096 128 0", 2024-09-21T09:31:03.399 INFO:tasks.workunit.client.0.smithi022.stdout: "", 2024-09-21T09:31:03.400 INFO:tasks.workunit.client.0.smithi022.stdout: "=== START OF SMART DATA SECTION ===", 2024-09-21T09:31:03.400 INFO:tasks.workunit.client.0.smithi022.stdout: "SMART overall-health self-assessment test result: PASSED", 2024-09-21T09:31:03.400 INFO:tasks.workunit.client.0.smithi022.stdout: "", 2024-09-21T09:31:03.400 INFO:tasks.workunit.client.0.smithi022.stdout: "SMART/Health Information (NVMe Log 0x02)", 2024-09-21T09:31:03.400 INFO:tasks.workunit.client.0.smithi022.stdout: "Critical Warning: 0x00", 2024-09-21T09:31:03.400 INFO:tasks.workunit.client.0.smithi022.stdout: "Temperature: 22 Celsius", 2024-09-21T09:31:03.400 INFO:tasks.workunit.client.0.smithi022.stdout: "Available Spare: 100%", 2024-09-21T09:31:03.401 INFO:tasks.workunit.client.0.smithi022.stdout: "Available Spare Threshold: 10%", 2024-09-21T09:31:03.401 INFO:tasks.workunit.client.0.smithi022.stdout: "Percentage Used: 12%", 2024-09-21T09:31:03.401 INFO:tasks.workunit.client.0.smithi022.stdout: "Data Units Read: 467,647,523 [239 TB]", 2024-09-21T09:31:03.401 INFO:tasks.workunit.client.0.smithi022.stdout: "Data Units Written: 2,280,460,231 [1.16 PB]", 2024-09-21T09:31:03.401 INFO:tasks.workunit.client.0.smithi022.stdout: "Host Read Commands: 8,064,859,300", 2024-09-21T09:31:03.401 INFO:tasks.workunit.client.0.smithi022.stdout: "Host Write Commands: 41,665,631,302", 2024-09-21T09:31:03.402 INFO:tasks.workunit.client.0.smithi022.stdout: "Controller Busy Time: 39", 2024-09-21T09:31:03.402 INFO:tasks.workunit.client.0.smithi022.stdout: "Power Cycles: 55,086", 2024-09-21T09:31:03.402 INFO:tasks.workunit.client.0.smithi022.stdout: "Power On Hours: 60,974", 2024-09-21T09:31:03.402 INFO:tasks.workunit.client.0.smithi022.stdout: "Unsafe Shutdowns: 2,616", 2024-09-21T09:31:03.402 INFO:tasks.workunit.client.0.smithi022.stdout: "Media and Data Integrity Errors: 0", 2024-09-21T09:31:03.402 INFO:tasks.workunit.client.0.smithi022.stdout: "Error Information Log Entries: 0", 2024-09-21T09:31:03.403 INFO:tasks.workunit.client.0.smithi022.stdout: "", 2024-09-21T09:31:03.403 INFO:tasks.workunit.client.0.smithi022.stdout: "Error Information (NVMe Log 0x01, 16 of 64 entries)", 2024-09-21T09:31:03.403 INFO:tasks.workunit.client.0.smithi022.stdout: "No Errors Logged", 2024-09-21T09:31:03.403 INFO:tasks.workunit.client.0.smithi022.stdout: "" 2024-09-21T09:31:03.403 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:03.403 INFO:tasks.workunit.client.0.smithi022.stdout: "platform_info": "x86_64-linux-5.14.0-508.el9.x86_64", 2024-09-21T09:31:03.404 INFO:tasks.workunit.client.0.smithi022.stdout: "svn_revision": "5155", 2024-09-21T09:31:03.404 INFO:tasks.workunit.client.0.smithi022.stdout: "version": [ 2024-09-21T09:31:03.404 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:03.404 INFO:tasks.workunit.client.0.smithi022.stdout: 2 2024-09-21T09:31:03.404 INFO:tasks.workunit.client.0.smithi022.stdout: ] 2024-09-21T09:31:03.404 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.405 INFO:tasks.workunit.client.0.smithi022.stdout: "temperature": { 2024-09-21T09:31:03.405 INFO:tasks.workunit.client.0.smithi022.stdout: "current": 22 2024-09-21T09:31:03.405 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.405 INFO:tasks.workunit.client.0.smithi022.stdout: "user_capacity": { 2024-09-21T09:31:03.405 INFO:tasks.workunit.client.0.smithi022.stdout: "blocks": 781422768, 2024-09-21T09:31:03.405 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 400088457216 2024-09-21T09:31:03.405 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:03.406 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:03.406 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:03.406 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:03.406 INFO:tasks.workunit.client.0.smithi022.stdout:} 2024-09-21T09:31:03.407 INFO:tasks.workunit.client.0.smithi022.stderr:+ ceph telemetry preview-all 2024-09-21T09:31:03.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:03 smithi022 ceph-mon[39525]: from='client.? 172.21.15.22:0/2645615157' entity='client.admin' 2024-09-21T09:31:03.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:03 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:31:03.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:03 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:03.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:03 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:31:03.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:03 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:31:03.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:03 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:31:03.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:03 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-21T09:31:03.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:03 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:31:03.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:03 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-21T09:31:03.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:03 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:31:03.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:03 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:31:03.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:03 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:31:03.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:03 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:31:03.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:03 smithi022 ceph-mon[27763]: from='client.? 172.21.15.22:0/2645615157' entity='client.admin' 2024-09-21T09:31:03.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:03 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:31:03.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:03 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:03.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:03 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:31:03.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:03 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:31:03.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:03 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:31:03.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:03 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-21T09:31:03.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:03 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:31:03.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:03 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-21T09:31:03.560 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:03 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:31:03.560 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:03 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:31:03.560 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:03 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:31:03.560 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:03 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:31:03.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:03 smithi064 ceph-mon[31124]: from='client.? 172.21.15.22:0/2645615157' entity='client.admin' 2024-09-21T09:31:03.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:03 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:31:03.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:03 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:03.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:03 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:31:03.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:03 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:31:03.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:03 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:31:03.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:03 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-21T09:31:03.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:03 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:31:03.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:03 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-21T09:31:03.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:03 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:31:03.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:03 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:31:03.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:03 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:31:03.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:03 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:31:03.848 INFO:tasks.workunit.client.0.smithi022.stdout:{ 2024-09-21T09:31:03.848 INFO:tasks.workunit.client.0.smithi022.stdout: "device_report": { 2024-09-21T09:31:03.848 INFO:tasks.workunit.client.0.smithi022.stdout: "384b39b0-77fc-11ef-b8c0-0cc47a595de4": { 2024-09-21T09:31:03.848 INFO:tasks.workunit.client.0.smithi022.stdout: "INTEL_SSDPED1D480GA_384bca4c-77fc-11ef-b8c0-0cc47a595de4": { 2024-09-21T09:31:03.848 INFO:tasks.workunit.client.0.smithi022.stdout: "20240921-092509": { 2024-09-21T09:31:03.848 INFO:tasks.workunit.client.0.smithi022.stdout: "device": { 2024-09-21T09:31:03.848 INFO:tasks.workunit.client.0.smithi022.stdout: "info_name": "/dev/nvme0n1", 2024-09-21T09:31:03.849 INFO:tasks.workunit.client.0.smithi022.stdout: "name": "/dev/nvme0n1", 2024-09-21T09:31:03.849 INFO:tasks.workunit.client.0.smithi022.stdout: "protocol": "NVMe", 2024-09-21T09:31:03.849 INFO:tasks.workunit.client.0.smithi022.stdout: "type": "nvme" 2024-09-21T09:31:03.849 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.849 INFO:tasks.workunit.client.0.smithi022.stdout: "firmware_version": "E2010325", 2024-09-21T09:31:03.849 INFO:tasks.workunit.client.0.smithi022.stdout: "host_id": "384b39b0-77fc-11ef-b8c0-0cc47a595de4", 2024-09-21T09:31:03.849 INFO:tasks.workunit.client.0.smithi022.stdout: "json_format_version": [ 2024-09-21T09:31:03.849 INFO:tasks.workunit.client.0.smithi022.stdout: 1, 2024-09-21T09:31:03.849 INFO:tasks.workunit.client.0.smithi022.stdout: 0 2024-09-21T09:31:03.849 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:03.849 INFO:tasks.workunit.client.0.smithi022.stdout: "local_time": { 2024-09-21T09:31:03.849 INFO:tasks.workunit.client.0.smithi022.stdout: "asctime": "Sat Sep 21 09:25:09 2024 UTC", 2024-09-21T09:31:03.849 INFO:tasks.workunit.client.0.smithi022.stdout: "time_t": 1726910709 2024-09-21T09:31:03.849 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.849 INFO:tasks.workunit.client.0.smithi022.stdout: "logical_block_size": 512, 2024-09-21T09:31:03.850 INFO:tasks.workunit.client.0.smithi022.stdout: "model_name": "INTEL SSDPED1D480GA", 2024-09-21T09:31:03.850 INFO:tasks.workunit.client.0.smithi022.stdout: "nvme_controller_id": 0, 2024-09-21T09:31:03.850 INFO:tasks.workunit.client.0.smithi022.stdout: "nvme_ieee_oui_identifier": 6083300, 2024-09-21T09:31:03.850 INFO:tasks.workunit.client.0.smithi022.stdout: "nvme_namespaces": [ 2024-09-21T09:31:03.850 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:03.850 INFO:tasks.workunit.client.0.smithi022.stdout: "capacity": { 2024-09-21T09:31:03.850 INFO:tasks.workunit.client.0.smithi022.stdout: "blocks": 937703088, 2024-09-21T09:31:03.850 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 480103981056 2024-09-21T09:31:03.850 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.850 INFO:tasks.workunit.client.0.smithi022.stdout: "formatted_lba_size": 512, 2024-09-21T09:31:03.850 INFO:tasks.workunit.client.0.smithi022.stdout: "id": 1, 2024-09-21T09:31:03.850 INFO:tasks.workunit.client.0.smithi022.stdout: "size": { 2024-09-21T09:31:03.850 INFO:tasks.workunit.client.0.smithi022.stdout: "blocks": 937703088, 2024-09-21T09:31:03.850 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 480103981056 2024-09-21T09:31:03.851 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.851 INFO:tasks.workunit.client.0.smithi022.stdout: "utilization": { 2024-09-21T09:31:03.851 INFO:tasks.workunit.client.0.smithi022.stdout: "blocks": 937703088, 2024-09-21T09:31:03.851 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 480103981056 2024-09-21T09:31:03.851 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:03.851 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:03.851 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:03.851 INFO:tasks.workunit.client.0.smithi022.stdout: "nvme_number_of_namespaces": 1, 2024-09-21T09:31:03.851 INFO:tasks.workunit.client.0.smithi022.stdout: "nvme_pci_vendor": { 2024-09-21T09:31:03.851 INFO:tasks.workunit.client.0.smithi022.stdout: "id": 32902, 2024-09-21T09:31:03.851 INFO:tasks.workunit.client.0.smithi022.stdout: "subsystem_id": 32902 2024-09-21T09:31:03.851 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.851 INFO:tasks.workunit.client.0.smithi022.stdout: "nvme_smart_health_information_add_log": { 2024-09-21T09:31:03.851 INFO:tasks.workunit.client.0.smithi022.stdout: "Device stats": { 2024-09-21T09:31:03.852 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_fw_downgrades": { 2024-09-21T09:31:03.852 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 0, 2024-09-21T09:31:03.852 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 0 2024-09-21T09:31:03.852 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.852 INFO:tasks.workunit.client.0.smithi022.stdout: "crc_error_count": { 2024-09-21T09:31:03.852 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 100, 2024-09-21T09:31:03.852 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 0 2024-09-21T09:31:03.852 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.852 INFO:tasks.workunit.client.0.smithi022.stdout: "end_to_end_error_detection_count": { 2024-09-21T09:31:03.852 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 100, 2024-09-21T09:31:03.852 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 0 2024-09-21T09:31:03.852 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.852 INFO:tasks.workunit.client.0.smithi022.stdout: "erase_fail_count": { 2024-09-21T09:31:03.852 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 0, 2024-09-21T09:31:03.853 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 0 2024-09-21T09:31:03.853 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.853 INFO:tasks.workunit.client.0.smithi022.stdout: "host_bytes_written": { 2024-09-21T09:31:03.853 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 100, 2024-09-21T09:31:03.853 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 6829746 2024-09-21T09:31:03.853 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.853 INFO:tasks.workunit.client.0.smithi022.stdout: "host_ctx_wear_used": { 2024-09-21T09:31:03.853 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 0, 2024-09-21T09:31:03.853 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 0 2024-09-21T09:31:03.853 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.853 INFO:tasks.workunit.client.0.smithi022.stdout: "media_bytes_read": { 2024-09-21T09:31:03.853 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 0, 2024-09-21T09:31:03.853 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 0 2024-09-21T09:31:03.853 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.854 INFO:tasks.workunit.client.0.smithi022.stdout: "nand_bytes_written": { 2024-09-21T09:31:03.854 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 0, 2024-09-21T09:31:03.854 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 0 2024-09-21T09:31:03.854 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.854 INFO:tasks.workunit.client.0.smithi022.stdout: "perf_stat_indicator": { 2024-09-21T09:31:03.854 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 0, 2024-09-21T09:31:03.854 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 0 2024-09-21T09:31:03.854 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.854 INFO:tasks.workunit.client.0.smithi022.stdout: "pll_lock_loss_count": { 2024-09-21T09:31:03.854 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 100, 2024-09-21T09:31:03.854 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 0 2024-09-21T09:31:03.854 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.854 INFO:tasks.workunit.client.0.smithi022.stdout: "program_fail_count": { 2024-09-21T09:31:03.854 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 0, 2024-09-21T09:31:03.855 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 0 2024-09-21T09:31:03.855 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.855 INFO:tasks.workunit.client.0.smithi022.stdout: "re_alloc_sectr_cnt": { 2024-09-21T09:31:03.855 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 0, 2024-09-21T09:31:03.855 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 0 2024-09-21T09:31:03.855 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.855 INFO:tasks.workunit.client.0.smithi022.stdout: "retry_buffer_overflow_count": { 2024-09-21T09:31:03.855 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 100, 2024-09-21T09:31:03.855 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 0 2024-09-21T09:31:03.855 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.855 INFO:tasks.workunit.client.0.smithi022.stdout: "soft_ecc_err_rate": { 2024-09-21T09:31:03.855 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 0, 2024-09-21T09:31:03.855 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 0 2024-09-21T09:31:03.855 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.855 INFO:tasks.workunit.client.0.smithi022.stdout: "thermal_throttle_status": { 2024-09-21T09:31:03.856 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 100, 2024-09-21T09:31:03.856 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": { 2024-09-21T09:31:03.856 INFO:tasks.workunit.client.0.smithi022.stdout: "cnt": 0, 2024-09-21T09:31:03.856 INFO:tasks.workunit.client.0.smithi022.stdout: "pct": 0 2024-09-21T09:31:03.856 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:03.856 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.856 INFO:tasks.workunit.client.0.smithi022.stdout: "timed_workload_host_reads": { 2024-09-21T09:31:03.856 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 100, 2024-09-21T09:31:03.856 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 16 2024-09-21T09:31:03.856 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.856 INFO:tasks.workunit.client.0.smithi022.stdout: "timed_workload_media_wear": { 2024-09-21T09:31:03.856 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 100, 2024-09-21T09:31:03.856 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": "0.000000" 2024-09-21T09:31:03.856 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.856 INFO:tasks.workunit.client.0.smithi022.stdout: "timed_workload_timer": { 2024-09-21T09:31:03.857 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 100, 2024-09-21T09:31:03.857 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 1212022 2024-09-21T09:31:03.857 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.857 INFO:tasks.workunit.client.0.smithi022.stdout: "unexp_power_loss": { 2024-09-21T09:31:03.857 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 0, 2024-09-21T09:31:03.857 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 0 2024-09-21T09:31:03.857 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.857 INFO:tasks.workunit.client.0.smithi022.stdout: "wear_leveling": { 2024-09-21T09:31:03.857 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 0, 2024-09-21T09:31:03.857 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": { 2024-09-21T09:31:03.857 INFO:tasks.workunit.client.0.smithi022.stdout: "avg": 0, 2024-09-21T09:31:03.857 INFO:tasks.workunit.client.0.smithi022.stdout: "max": 0, 2024-09-21T09:31:03.857 INFO:tasks.workunit.client.0.smithi022.stdout: "min": 0 2024-09-21T09:31:03.857 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:03.857 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:03.857 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.858 INFO:tasks.workunit.client.0.smithi022.stdout: "Intel Smart log": "nvme0n1" 2024-09-21T09:31:03.858 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.858 INFO:tasks.workunit.client.0.smithi022.stdout: "nvme_smart_health_information_log": { 2024-09-21T09:31:03.858 INFO:tasks.workunit.client.0.smithi022.stdout: "available_spare": 100, 2024-09-21T09:31:03.858 INFO:tasks.workunit.client.0.smithi022.stdout: "available_spare_threshold": 0, 2024-09-21T09:31:03.858 INFO:tasks.workunit.client.0.smithi022.stdout: "controller_busy_time": 2680, 2024-09-21T09:31:03.858 INFO:tasks.workunit.client.0.smithi022.stdout: "critical_warning": 0, 2024-09-21T09:31:03.858 INFO:tasks.workunit.client.0.smithi022.stdout: "data_units_read": 95367672, 2024-09-21T09:31:03.858 INFO:tasks.workunit.client.0.smithi022.stdout: "data_units_written": 447594268, 2024-09-21T09:31:03.858 INFO:tasks.workunit.client.0.smithi022.stdout: "host_reads": 1403952490, 2024-09-21T09:31:03.858 INFO:tasks.workunit.client.0.smithi022.stdout: "host_writes": 10869660916, 2024-09-21T09:31:03.858 INFO:tasks.workunit.client.0.smithi022.stdout: "media_errors": 0, 2024-09-21T09:31:03.858 INFO:tasks.workunit.client.0.smithi022.stdout: "num_err_log_entries": 0, 2024-09-21T09:31:03.858 INFO:tasks.workunit.client.0.smithi022.stdout: "percentage_used": 0, 2024-09-21T09:31:03.858 INFO:tasks.workunit.client.0.smithi022.stdout: "power_cycles": 16189, 2024-09-21T09:31:03.859 INFO:tasks.workunit.client.0.smithi022.stdout: "power_on_hours": 27263, 2024-09-21T09:31:03.859 INFO:tasks.workunit.client.0.smithi022.stdout: "temperature": 31, 2024-09-21T09:31:03.859 INFO:tasks.workunit.client.0.smithi022.stdout: "unsafe_shutdowns": 5631 2024-09-21T09:31:03.859 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.859 INFO:tasks.workunit.client.0.smithi022.stdout: "nvme_vendor": "intel", 2024-09-21T09:31:03.859 INFO:tasks.workunit.client.0.smithi022.stdout: "nvme_version": { 2024-09-21T09:31:03.859 INFO:tasks.workunit.client.0.smithi022.stdout: "string": "<1.2", 2024-09-21T09:31:03.859 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:03.859 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.859 INFO:tasks.workunit.client.0.smithi022.stdout: "power_cycle_count": 16189, 2024-09-21T09:31:03.859 INFO:tasks.workunit.client.0.smithi022.stdout: "power_on_time": { 2024-09-21T09:31:03.860 INFO:tasks.workunit.client.0.smithi022.stdout: "hours": 27263 2024-09-21T09:31:03.860 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.860 INFO:tasks.workunit.client.0.smithi022.stdout: "serial_number": "deleted", 2024-09-21T09:31:03.860 INFO:tasks.workunit.client.0.smithi022.stdout: "smart_status": { 2024-09-21T09:31:03.860 INFO:tasks.workunit.client.0.smithi022.stdout: "nvme": { 2024-09-21T09:31:03.860 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:03.860 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.860 INFO:tasks.workunit.client.0.smithi022.stdout: "passed": true 2024-09-21T09:31:03.860 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.860 INFO:tasks.workunit.client.0.smithi022.stdout: "smartctl": { 2024-09-21T09:31:03.860 INFO:tasks.workunit.client.0.smithi022.stdout: "argv": [ 2024-09-21T09:31:03.860 INFO:tasks.workunit.client.0.smithi022.stdout: "smartctl", 2024-09-21T09:31:03.860 INFO:tasks.workunit.client.0.smithi022.stdout: "-x", 2024-09-21T09:31:03.860 INFO:tasks.workunit.client.0.smithi022.stdout: "--json=o", 2024-09-21T09:31:03.861 INFO:tasks.workunit.client.0.smithi022.stdout: "/dev/nvme0n1" 2024-09-21T09:31:03.861 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:03.861 INFO:tasks.workunit.client.0.smithi022.stdout: "build_info": "(local build)", 2024-09-21T09:31:03.861 INFO:tasks.workunit.client.0.smithi022.stdout: "exit_status": 0, 2024-09-21T09:31:03.861 INFO:tasks.workunit.client.0.smithi022.stdout: "output": [ 2024-09-21T09:31:03.861 INFO:tasks.workunit.client.0.smithi022.stdout: "smartctl 7.2 2020-12-30 r5155 [x86_64-linux-5.14.0-508.el9.x86_64] (local build)", 2024-09-21T09:31:03.861 INFO:tasks.workunit.client.0.smithi022.stdout: "Copyright (C) 2002-20, Bruce Allen, Christian Franke, www.smartmontools.org", 2024-09-21T09:31:03.861 INFO:tasks.workunit.client.0.smithi022.stdout: "", 2024-09-21T09:31:03.861 INFO:tasks.workunit.client.0.smithi022.stdout: "=== START OF INFORMATION SECTION ===", 2024-09-21T09:31:03.861 INFO:tasks.workunit.client.0.smithi022.stdout: "Model Number: INTEL SSDPED1D480GA", 2024-09-21T09:31:03.861 INFO:tasks.workunit.client.0.smithi022.stdout: "Serial Number: deleted", 2024-09-21T09:31:03.861 INFO:tasks.workunit.client.0.smithi022.stdout: "Firmware Version: E2010325", 2024-09-21T09:31:03.861 INFO:tasks.workunit.client.0.smithi022.stdout: "PCI Vendor/Subsystem ID: 0x8086", 2024-09-21T09:31:03.861 INFO:tasks.workunit.client.0.smithi022.stdout: "IEEE OUI Identifier: 0x5cd2e4", 2024-09-21T09:31:03.862 INFO:tasks.workunit.client.0.smithi022.stdout: "Controller ID: 0", 2024-09-21T09:31:03.862 INFO:tasks.workunit.client.0.smithi022.stdout: "NVMe Version: <1.2", 2024-09-21T09:31:03.862 INFO:tasks.workunit.client.0.smithi022.stdout: "Number of Namespaces: 1", 2024-09-21T09:31:03.862 INFO:tasks.workunit.client.0.smithi022.stdout: "Namespace 1 Size/Capacity: 480,103,981,056 [480 GB]", 2024-09-21T09:31:03.862 INFO:tasks.workunit.client.0.smithi022.stdout: "Namespace 1 Formatted LBA Size: 512", 2024-09-21T09:31:03.862 INFO:tasks.workunit.client.0.smithi022.stdout: "Local Time is: Sat Sep 21 09:25:09 2024 UTC", 2024-09-21T09:31:03.862 INFO:tasks.workunit.client.0.smithi022.stdout: "Firmware Updates (0x02): 1 Slot", 2024-09-21T09:31:03.862 INFO:tasks.workunit.client.0.smithi022.stdout: "Optional Admin Commands (0x0007): Security Format Frmw_DL", 2024-09-21T09:31:03.862 INFO:tasks.workunit.client.0.smithi022.stdout: "Optional NVM Commands (0x0006): Wr_Unc DS_Mngmt", 2024-09-21T09:31:03.862 INFO:tasks.workunit.client.0.smithi022.stdout: "Log Page Attributes (0x02): Cmd_Eff_Lg", 2024-09-21T09:31:03.862 INFO:tasks.workunit.client.0.smithi022.stdout: "Maximum Data Transfer Size: 32 Pages", 2024-09-21T09:31:03.862 INFO:tasks.workunit.client.0.smithi022.stdout: "", 2024-09-21T09:31:03.862 INFO:tasks.workunit.client.0.smithi022.stdout: "Supported Power States", 2024-09-21T09:31:03.862 INFO:tasks.workunit.client.0.smithi022.stdout: "St Op Max Active Idle RL RT WL WT Ent_Lat Ex_Lat", 2024-09-21T09:31:03.863 INFO:tasks.workunit.client.0.smithi022.stdout: " 0 + 18.00W - - 0 0 0 0 0 0", 2024-09-21T09:31:03.863 INFO:tasks.workunit.client.0.smithi022.stdout: "", 2024-09-21T09:31:03.863 INFO:tasks.workunit.client.0.smithi022.stdout: "Supported LBA Sizes (NSID 0x1)", 2024-09-21T09:31:03.863 INFO:tasks.workunit.client.0.smithi022.stdout: "Id Fmt Data Metadt Rel_Perf", 2024-09-21T09:31:03.863 INFO:tasks.workunit.client.0.smithi022.stdout: " 0 + 512 0 2", 2024-09-21T09:31:03.863 INFO:tasks.workunit.client.0.smithi022.stdout: "", 2024-09-21T09:31:03.863 INFO:tasks.workunit.client.0.smithi022.stdout: "=== START OF SMART DATA SECTION ===", 2024-09-21T09:31:03.863 INFO:tasks.workunit.client.0.smithi022.stdout: "SMART overall-health self-assessment test result: PASSED", 2024-09-21T09:31:03.863 INFO:tasks.workunit.client.0.smithi022.stdout: "", 2024-09-21T09:31:03.863 INFO:tasks.workunit.client.0.smithi022.stdout: "SMART/Health Information (NVMe Log 0x02)", 2024-09-21T09:31:03.863 INFO:tasks.workunit.client.0.smithi022.stdout: "Critical Warning: 0x00", 2024-09-21T09:31:03.863 INFO:tasks.workunit.client.0.smithi022.stdout: "Temperature: 31 Celsius", 2024-09-21T09:31:03.863 INFO:tasks.workunit.client.0.smithi022.stdout: "Available Spare: 100%", 2024-09-21T09:31:03.864 INFO:tasks.workunit.client.0.smithi022.stdout: "Available Spare Threshold: 0%", 2024-09-21T09:31:03.864 INFO:tasks.workunit.client.0.smithi022.stdout: "Percentage Used: 0%", 2024-09-21T09:31:03.864 INFO:tasks.workunit.client.0.smithi022.stdout: "Data Units Read: 95,367,672 [48.8 TB]", 2024-09-21T09:31:03.864 INFO:tasks.workunit.client.0.smithi022.stdout: "Data Units Written: 447,594,268 [229 TB]", 2024-09-21T09:31:03.864 INFO:tasks.workunit.client.0.smithi022.stdout: "Host Read Commands: 1,403,952,490", 2024-09-21T09:31:03.864 INFO:tasks.workunit.client.0.smithi022.stdout: "Host Write Commands: 10,869,660,916", 2024-09-21T09:31:03.864 INFO:tasks.workunit.client.0.smithi022.stdout: "Controller Busy Time: 2,680", 2024-09-21T09:31:03.864 INFO:tasks.workunit.client.0.smithi022.stdout: "Power Cycles: 16,189", 2024-09-21T09:31:03.864 INFO:tasks.workunit.client.0.smithi022.stdout: "Power On Hours: 27,263", 2024-09-21T09:31:03.864 INFO:tasks.workunit.client.0.smithi022.stdout: "Unsafe Shutdowns: 5,631", 2024-09-21T09:31:03.864 INFO:tasks.workunit.client.0.smithi022.stdout: "Media and Data Integrity Errors: 0", 2024-09-21T09:31:03.864 INFO:tasks.workunit.client.0.smithi022.stdout: "Error Information Log Entries: 0", 2024-09-21T09:31:03.865 INFO:tasks.workunit.client.0.smithi022.stdout: "", 2024-09-21T09:31:03.865 INFO:tasks.workunit.client.0.smithi022.stdout: "Error Information (NVMe Log 0x01, 16 of 64 entries)", 2024-09-21T09:31:03.865 INFO:tasks.workunit.client.0.smithi022.stdout: "No Errors Logged", 2024-09-21T09:31:03.865 INFO:tasks.workunit.client.0.smithi022.stdout: "" 2024-09-21T09:31:03.865 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:03.865 INFO:tasks.workunit.client.0.smithi022.stdout: "platform_info": "x86_64-linux-5.14.0-508.el9.x86_64", 2024-09-21T09:31:03.865 INFO:tasks.workunit.client.0.smithi022.stdout: "svn_revision": "5155", 2024-09-21T09:31:03.865 INFO:tasks.workunit.client.0.smithi022.stdout: "version": [ 2024-09-21T09:31:03.865 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:03.865 INFO:tasks.workunit.client.0.smithi022.stdout: 2 2024-09-21T09:31:03.865 INFO:tasks.workunit.client.0.smithi022.stdout: ] 2024-09-21T09:31:03.865 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.865 INFO:tasks.workunit.client.0.smithi022.stdout: "temperature": { 2024-09-21T09:31:03.866 INFO:tasks.workunit.client.0.smithi022.stdout: "current": 31 2024-09-21T09:31:03.866 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.866 INFO:tasks.workunit.client.0.smithi022.stdout: "user_capacity": { 2024-09-21T09:31:03.866 INFO:tasks.workunit.client.0.smithi022.stdout: "blocks": 937703088, 2024-09-21T09:31:03.866 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 480103981056 2024-09-21T09:31:03.866 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:03.866 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:03.866 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:03.866 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.866 INFO:tasks.workunit.client.0.smithi022.stdout: "384d1fc8-77fc-11ef-b8c0-0cc47a595de4": { 2024-09-21T09:31:03.866 INFO:tasks.workunit.client.0.smithi022.stdout: "INTEL_SSDPEDMD400G4_384da218-77fc-11ef-b8c0-0cc47a595de4": { 2024-09-21T09:31:03.866 INFO:tasks.workunit.client.0.smithi022.stdout: "20240921-092513": { 2024-09-21T09:31:03.866 INFO:tasks.workunit.client.0.smithi022.stdout: "device": { 2024-09-21T09:31:03.866 INFO:tasks.workunit.client.0.smithi022.stdout: "info_name": "/dev/nvme0n1", 2024-09-21T09:31:03.866 INFO:tasks.workunit.client.0.smithi022.stdout: "name": "/dev/nvme0n1", 2024-09-21T09:31:03.867 INFO:tasks.workunit.client.0.smithi022.stdout: "protocol": "NVMe", 2024-09-21T09:31:03.867 INFO:tasks.workunit.client.0.smithi022.stdout: "type": "nvme" 2024-09-21T09:31:03.867 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.867 INFO:tasks.workunit.client.0.smithi022.stdout: "firmware_version": "8DV101H0", 2024-09-21T09:31:03.867 INFO:tasks.workunit.client.0.smithi022.stdout: "host_id": "384d1fc8-77fc-11ef-b8c0-0cc47a595de4", 2024-09-21T09:31:03.867 INFO:tasks.workunit.client.0.smithi022.stdout: "json_format_version": [ 2024-09-21T09:31:03.867 INFO:tasks.workunit.client.0.smithi022.stdout: 1, 2024-09-21T09:31:03.867 INFO:tasks.workunit.client.0.smithi022.stdout: 0 2024-09-21T09:31:03.867 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:03.867 INFO:tasks.workunit.client.0.smithi022.stdout: "local_time": { 2024-09-21T09:31:03.867 INFO:tasks.workunit.client.0.smithi022.stdout: "asctime": "Sat Sep 21 09:25:12 2024 UTC", 2024-09-21T09:31:03.867 INFO:tasks.workunit.client.0.smithi022.stdout: "time_t": 1726910712 2024-09-21T09:31:03.867 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.867 INFO:tasks.workunit.client.0.smithi022.stdout: "logical_block_size": 512, 2024-09-21T09:31:03.868 INFO:tasks.workunit.client.0.smithi022.stdout: "model_name": "INTEL SSDPEDMD400G4", 2024-09-21T09:31:03.868 INFO:tasks.workunit.client.0.smithi022.stdout: "nvme_controller_id": 0, 2024-09-21T09:31:03.868 INFO:tasks.workunit.client.0.smithi022.stdout: "nvme_ieee_oui_identifier": 6083300, 2024-09-21T09:31:03.868 INFO:tasks.workunit.client.0.smithi022.stdout: "nvme_namespaces": [ 2024-09-21T09:31:03.868 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:03.868 INFO:tasks.workunit.client.0.smithi022.stdout: "capacity": { 2024-09-21T09:31:03.868 INFO:tasks.workunit.client.0.smithi022.stdout: "blocks": 781422768, 2024-09-21T09:31:03.868 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 400088457216 2024-09-21T09:31:03.868 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.868 INFO:tasks.workunit.client.0.smithi022.stdout: "formatted_lba_size": 512, 2024-09-21T09:31:03.868 INFO:tasks.workunit.client.0.smithi022.stdout: "id": 1, 2024-09-21T09:31:03.868 INFO:tasks.workunit.client.0.smithi022.stdout: "size": { 2024-09-21T09:31:03.868 INFO:tasks.workunit.client.0.smithi022.stdout: "blocks": 781422768, 2024-09-21T09:31:03.868 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 400088457216 2024-09-21T09:31:03.868 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.869 INFO:tasks.workunit.client.0.smithi022.stdout: "utilization": { 2024-09-21T09:31:03.869 INFO:tasks.workunit.client.0.smithi022.stdout: "blocks": 781422768, 2024-09-21T09:31:03.869 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 400088457216 2024-09-21T09:31:03.869 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:03.869 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:03.869 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:03.869 INFO:tasks.workunit.client.0.smithi022.stdout: "nvme_number_of_namespaces": 1, 2024-09-21T09:31:03.869 INFO:tasks.workunit.client.0.smithi022.stdout: "nvme_pci_vendor": { 2024-09-21T09:31:03.869 INFO:tasks.workunit.client.0.smithi022.stdout: "id": 32902, 2024-09-21T09:31:03.869 INFO:tasks.workunit.client.0.smithi022.stdout: "subsystem_id": 32902 2024-09-21T09:31:03.869 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.869 INFO:tasks.workunit.client.0.smithi022.stdout: "nvme_smart_health_information_add_log": { 2024-09-21T09:31:03.869 INFO:tasks.workunit.client.0.smithi022.stdout: "Device stats": { 2024-09-21T09:31:03.869 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_fw_downgrades": { 2024-09-21T09:31:03.869 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 0, 2024-09-21T09:31:03.870 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 0 2024-09-21T09:31:03.870 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.870 INFO:tasks.workunit.client.0.smithi022.stdout: "crc_error_count": { 2024-09-21T09:31:03.870 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 100, 2024-09-21T09:31:03.870 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 0 2024-09-21T09:31:03.870 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.870 INFO:tasks.workunit.client.0.smithi022.stdout: "end_to_end_error_detection_count": { 2024-09-21T09:31:03.870 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 100, 2024-09-21T09:31:03.870 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 0 2024-09-21T09:31:03.870 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.870 INFO:tasks.workunit.client.0.smithi022.stdout: "erase_fail_count": { 2024-09-21T09:31:03.871 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 100, 2024-09-21T09:31:03.871 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 0 2024-09-21T09:31:03.871 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.871 INFO:tasks.workunit.client.0.smithi022.stdout: "host_bytes_written": { 2024-09-21T09:31:03.871 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 100, 2024-09-21T09:31:03.871 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 34797061 2024-09-21T09:31:03.871 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.871 INFO:tasks.workunit.client.0.smithi022.stdout: "host_ctx_wear_used": { 2024-09-21T09:31:03.871 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 100, 2024-09-21T09:31:03.871 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 1 2024-09-21T09:31:03.871 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.871 INFO:tasks.workunit.client.0.smithi022.stdout: "media_bytes_read": { 2024-09-21T09:31:03.871 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 0, 2024-09-21T09:31:03.871 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 0 2024-09-21T09:31:03.872 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.872 INFO:tasks.workunit.client.0.smithi022.stdout: "nand_bytes_written": { 2024-09-21T09:31:03.872 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 100, 2024-09-21T09:31:03.872 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 57236814 2024-09-21T09:31:03.872 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.872 INFO:tasks.workunit.client.0.smithi022.stdout: "perf_stat_indicator": { 2024-09-21T09:31:03.872 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 0, 2024-09-21T09:31:03.872 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 0 2024-09-21T09:31:03.872 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.872 INFO:tasks.workunit.client.0.smithi022.stdout: "pll_lock_loss_count": { 2024-09-21T09:31:03.872 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 100, 2024-09-21T09:31:03.872 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 0 2024-09-21T09:31:03.872 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.872 INFO:tasks.workunit.client.0.smithi022.stdout: "program_fail_count": { 2024-09-21T09:31:03.872 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 100, 2024-09-21T09:31:03.873 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 0 2024-09-21T09:31:03.873 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.873 INFO:tasks.workunit.client.0.smithi022.stdout: "re_alloc_sectr_cnt": { 2024-09-21T09:31:03.873 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 0, 2024-09-21T09:31:03.873 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 0 2024-09-21T09:31:03.873 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.873 INFO:tasks.workunit.client.0.smithi022.stdout: "retry_buffer_overflow_count": { 2024-09-21T09:31:03.873 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 100, 2024-09-21T09:31:03.873 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 0 2024-09-21T09:31:03.873 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.873 INFO:tasks.workunit.client.0.smithi022.stdout: "soft_ecc_err_rate": { 2024-09-21T09:31:03.874 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 0, 2024-09-21T09:31:03.874 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 0 2024-09-21T09:31:03.874 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.874 INFO:tasks.workunit.client.0.smithi022.stdout: "thermal_throttle_status": { 2024-09-21T09:31:03.874 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 100, 2024-09-21T09:31:03.874 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": { 2024-09-21T09:31:03.874 INFO:tasks.workunit.client.0.smithi022.stdout: "cnt": 0, 2024-09-21T09:31:03.874 INFO:tasks.workunit.client.0.smithi022.stdout: "pct": 0 2024-09-21T09:31:03.874 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:03.874 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.874 INFO:tasks.workunit.client.0.smithi022.stdout: "timed_workload_host_reads": { 2024-09-21T09:31:03.874 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 100, 2024-09-21T09:31:03.874 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 65535 2024-09-21T09:31:03.874 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.874 INFO:tasks.workunit.client.0.smithi022.stdout: "timed_workload_media_wear": { 2024-09-21T09:31:03.875 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 100, 2024-09-21T09:31:03.875 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": "63.999023" 2024-09-21T09:31:03.875 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.875 INFO:tasks.workunit.client.0.smithi022.stdout: "timed_workload_timer": { 2024-09-21T09:31:03.875 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 100, 2024-09-21T09:31:03.875 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 65535 2024-09-21T09:31:03.875 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.875 INFO:tasks.workunit.client.0.smithi022.stdout: "unexp_power_loss": { 2024-09-21T09:31:03.875 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 0, 2024-09-21T09:31:03.875 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 0 2024-09-21T09:31:03.875 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.875 INFO:tasks.workunit.client.0.smithi022.stdout: "wear_leveling": { 2024-09-21T09:31:03.875 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 88, 2024-09-21T09:31:03.875 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": { 2024-09-21T09:31:03.875 INFO:tasks.workunit.client.0.smithi022.stdout: "avg": 3844, 2024-09-21T09:31:03.876 INFO:tasks.workunit.client.0.smithi022.stdout: "max": 3860, 2024-09-21T09:31:03.876 INFO:tasks.workunit.client.0.smithi022.stdout: "min": 3828 2024-09-21T09:31:03.876 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:03.876 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:03.876 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.876 INFO:tasks.workunit.client.0.smithi022.stdout: "Intel Smart log": "nvme0n1" 2024-09-21T09:31:03.876 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.876 INFO:tasks.workunit.client.0.smithi022.stdout: "nvme_smart_health_information_log": { 2024-09-21T09:31:03.876 INFO:tasks.workunit.client.0.smithi022.stdout: "available_spare": 100, 2024-09-21T09:31:03.876 INFO:tasks.workunit.client.0.smithi022.stdout: "available_spare_threshold": 10, 2024-09-21T09:31:03.876 INFO:tasks.workunit.client.0.smithi022.stdout: "controller_busy_time": 39, 2024-09-21T09:31:03.876 INFO:tasks.workunit.client.0.smithi022.stdout: "critical_warning": 0, 2024-09-21T09:31:03.876 INFO:tasks.workunit.client.0.smithi022.stdout: "data_units_read": 467647523, 2024-09-21T09:31:03.876 INFO:tasks.workunit.client.0.smithi022.stdout: "data_units_written": 2280460231, 2024-09-21T09:31:03.876 INFO:tasks.workunit.client.0.smithi022.stdout: "host_reads": 8064859300, 2024-09-21T09:31:03.877 INFO:tasks.workunit.client.0.smithi022.stdout: "host_writes": 41665631302, 2024-09-21T09:31:03.877 INFO:tasks.workunit.client.0.smithi022.stdout: "media_errors": 0, 2024-09-21T09:31:03.877 INFO:tasks.workunit.client.0.smithi022.stdout: "num_err_log_entries": 0, 2024-09-21T09:31:03.877 INFO:tasks.workunit.client.0.smithi022.stdout: "percentage_used": 12, 2024-09-21T09:31:03.877 INFO:tasks.workunit.client.0.smithi022.stdout: "power_cycles": 55086, 2024-09-21T09:31:03.877 INFO:tasks.workunit.client.0.smithi022.stdout: "power_on_hours": 60974, 2024-09-21T09:31:03.877 INFO:tasks.workunit.client.0.smithi022.stdout: "temperature": 22, 2024-09-21T09:31:03.877 INFO:tasks.workunit.client.0.smithi022.stdout: "unsafe_shutdowns": 2616 2024-09-21T09:31:03.877 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.877 INFO:tasks.workunit.client.0.smithi022.stdout: "nvme_vendor": "intel", 2024-09-21T09:31:03.877 INFO:tasks.workunit.client.0.smithi022.stdout: "nvme_version": { 2024-09-21T09:31:03.877 INFO:tasks.workunit.client.0.smithi022.stdout: "string": "<1.2", 2024-09-21T09:31:03.877 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:03.877 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.878 INFO:tasks.workunit.client.0.smithi022.stdout: "power_cycle_count": 55086, 2024-09-21T09:31:03.878 INFO:tasks.workunit.client.0.smithi022.stdout: "power_on_time": { 2024-09-21T09:31:03.878 INFO:tasks.workunit.client.0.smithi022.stdout: "hours": 60974 2024-09-21T09:31:03.878 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.878 INFO:tasks.workunit.client.0.smithi022.stdout: "serial_number": "deleted", 2024-09-21T09:31:03.878 INFO:tasks.workunit.client.0.smithi022.stdout: "smart_status": { 2024-09-21T09:31:03.878 INFO:tasks.workunit.client.0.smithi022.stdout: "nvme": { 2024-09-21T09:31:03.878 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:03.878 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.878 INFO:tasks.workunit.client.0.smithi022.stdout: "passed": true 2024-09-21T09:31:03.878 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.878 INFO:tasks.workunit.client.0.smithi022.stdout: "smartctl": { 2024-09-21T09:31:03.878 INFO:tasks.workunit.client.0.smithi022.stdout: "argv": [ 2024-09-21T09:31:03.878 INFO:tasks.workunit.client.0.smithi022.stdout: "smartctl", 2024-09-21T09:31:03.878 INFO:tasks.workunit.client.0.smithi022.stdout: "-x", 2024-09-21T09:31:03.879 INFO:tasks.workunit.client.0.smithi022.stdout: "--json=o", 2024-09-21T09:31:03.879 INFO:tasks.workunit.client.0.smithi022.stdout: "/dev/nvme0n1" 2024-09-21T09:31:03.879 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:03.879 INFO:tasks.workunit.client.0.smithi022.stdout: "build_info": "(local build)", 2024-09-21T09:31:03.879 INFO:tasks.workunit.client.0.smithi022.stdout: "exit_status": 0, 2024-09-21T09:31:03.879 INFO:tasks.workunit.client.0.smithi022.stdout: "output": [ 2024-09-21T09:31:03.879 INFO:tasks.workunit.client.0.smithi022.stdout: "smartctl 7.2 2020-12-30 r5155 [x86_64-linux-5.14.0-508.el9.x86_64] (local build)", 2024-09-21T09:31:03.879 INFO:tasks.workunit.client.0.smithi022.stdout: "Copyright (C) 2002-20, Bruce Allen, Christian Franke, www.smartmontools.org", 2024-09-21T09:31:03.879 INFO:tasks.workunit.client.0.smithi022.stdout: "", 2024-09-21T09:31:03.879 INFO:tasks.workunit.client.0.smithi022.stdout: "=== START OF INFORMATION SECTION ===", 2024-09-21T09:31:03.879 INFO:tasks.workunit.client.0.smithi022.stdout: "Model Number: INTEL SSDPEDMD400G4", 2024-09-21T09:31:03.879 INFO:tasks.workunit.client.0.smithi022.stdout: "Serial Number: deleted", 2024-09-21T09:31:03.880 INFO:tasks.workunit.client.0.smithi022.stdout: "Firmware Version: 8DV101H0", 2024-09-21T09:31:03.880 INFO:tasks.workunit.client.0.smithi022.stdout: "PCI Vendor/Subsystem ID: 0x8086", 2024-09-21T09:31:03.880 INFO:tasks.workunit.client.0.smithi022.stdout: "IEEE OUI Identifier: 0x5cd2e4", 2024-09-21T09:31:03.880 INFO:tasks.workunit.client.0.smithi022.stdout: "Controller ID: 0", 2024-09-21T09:31:03.880 INFO:tasks.workunit.client.0.smithi022.stdout: "NVMe Version: <1.2", 2024-09-21T09:31:03.880 INFO:tasks.workunit.client.0.smithi022.stdout: "Number of Namespaces: 1", 2024-09-21T09:31:03.880 INFO:tasks.workunit.client.0.smithi022.stdout: "Namespace 1 Size/Capacity: 400,088,457,216 [400 GB]", 2024-09-21T09:31:03.880 INFO:tasks.workunit.client.0.smithi022.stdout: "Namespace 1 Formatted LBA Size: 512", 2024-09-21T09:31:03.880 INFO:tasks.workunit.client.0.smithi022.stdout: "Local Time is: Sat Sep 21 09:25:12 2024 UTC", 2024-09-21T09:31:03.880 INFO:tasks.workunit.client.0.smithi022.stdout: "Firmware Updates (0x02): 1 Slot", 2024-09-21T09:31:03.880 INFO:tasks.workunit.client.0.smithi022.stdout: "Optional Admin Commands (0x0006): Format Frmw_DL", 2024-09-21T09:31:03.880 INFO:tasks.workunit.client.0.smithi022.stdout: "Optional NVM Commands (0x0006): Wr_Unc DS_Mngmt", 2024-09-21T09:31:03.880 INFO:tasks.workunit.client.0.smithi022.stdout: "Log Page Attributes (0x02): Cmd_Eff_Lg", 2024-09-21T09:31:03.880 INFO:tasks.workunit.client.0.smithi022.stdout: "Maximum Data Transfer Size: 32 Pages", 2024-09-21T09:31:03.880 INFO:tasks.workunit.client.0.smithi022.stdout: "", 2024-09-21T09:31:03.881 INFO:tasks.workunit.client.0.smithi022.stdout: "Supported Power States", 2024-09-21T09:31:03.881 INFO:tasks.workunit.client.0.smithi022.stdout: "St Op Max Active Idle RL RT WL WT Ent_Lat Ex_Lat", 2024-09-21T09:31:03.881 INFO:tasks.workunit.client.0.smithi022.stdout: " 0 + 25.00W - - 0 0 0 0 0 0", 2024-09-21T09:31:03.881 INFO:tasks.workunit.client.0.smithi022.stdout: "", 2024-09-21T09:31:03.881 INFO:tasks.workunit.client.0.smithi022.stdout: "Supported LBA Sizes (NSID 0x1)", 2024-09-21T09:31:03.881 INFO:tasks.workunit.client.0.smithi022.stdout: "Id Fmt Data Metadt Rel_Perf", 2024-09-21T09:31:03.881 INFO:tasks.workunit.client.0.smithi022.stdout: " 0 + 512 0 2", 2024-09-21T09:31:03.881 INFO:tasks.workunit.client.0.smithi022.stdout: " 1 - 512 8 2", 2024-09-21T09:31:03.881 INFO:tasks.workunit.client.0.smithi022.stdout: " 2 - 512 16 2", 2024-09-21T09:31:03.881 INFO:tasks.workunit.client.0.smithi022.stdout: " 3 - 4096 0 0", 2024-09-21T09:31:03.881 INFO:tasks.workunit.client.0.smithi022.stdout: " 4 - 4096 8 0", 2024-09-21T09:31:03.881 INFO:tasks.workunit.client.0.smithi022.stdout: " 5 - 4096 64 0", 2024-09-21T09:31:03.881 INFO:tasks.workunit.client.0.smithi022.stdout: " 6 - 4096 128 0", 2024-09-21T09:31:03.882 INFO:tasks.workunit.client.0.smithi022.stdout: "", 2024-09-21T09:31:03.882 INFO:tasks.workunit.client.0.smithi022.stdout: "=== START OF SMART DATA SECTION ===", 2024-09-21T09:31:03.882 INFO:tasks.workunit.client.0.smithi022.stdout: "SMART overall-health self-assessment test result: PASSED", 2024-09-21T09:31:03.882 INFO:tasks.workunit.client.0.smithi022.stdout: "", 2024-09-21T09:31:03.882 INFO:tasks.workunit.client.0.smithi022.stdout: "SMART/Health Information (NVMe Log 0x02)", 2024-09-21T09:31:03.882 INFO:tasks.workunit.client.0.smithi022.stdout: "Critical Warning: 0x00", 2024-09-21T09:31:03.882 INFO:tasks.workunit.client.0.smithi022.stdout: "Temperature: 22 Celsius", 2024-09-21T09:31:03.882 INFO:tasks.workunit.client.0.smithi022.stdout: "Available Spare: 100%", 2024-09-21T09:31:03.882 INFO:tasks.workunit.client.0.smithi022.stdout: "Available Spare Threshold: 10%", 2024-09-21T09:31:03.882 INFO:tasks.workunit.client.0.smithi022.stdout: "Percentage Used: 12%", 2024-09-21T09:31:03.882 INFO:tasks.workunit.client.0.smithi022.stdout: "Data Units Read: 467,647,523 [239 TB]", 2024-09-21T09:31:03.882 INFO:tasks.workunit.client.0.smithi022.stdout: "Data Units Written: 2,280,460,231 [1.16 PB]", 2024-09-21T09:31:03.882 INFO:tasks.workunit.client.0.smithi022.stdout: "Host Read Commands: 8,064,859,300", 2024-09-21T09:31:03.882 INFO:tasks.workunit.client.0.smithi022.stdout: "Host Write Commands: 41,665,631,302", 2024-09-21T09:31:03.883 INFO:tasks.workunit.client.0.smithi022.stdout: "Controller Busy Time: 39", 2024-09-21T09:31:03.883 INFO:tasks.workunit.client.0.smithi022.stdout: "Power Cycles: 55,086", 2024-09-21T09:31:03.883 INFO:tasks.workunit.client.0.smithi022.stdout: "Power On Hours: 60,974", 2024-09-21T09:31:03.883 INFO:tasks.workunit.client.0.smithi022.stdout: "Unsafe Shutdowns: 2,616", 2024-09-21T09:31:03.883 INFO:tasks.workunit.client.0.smithi022.stdout: "Media and Data Integrity Errors: 0", 2024-09-21T09:31:03.883 INFO:tasks.workunit.client.0.smithi022.stdout: "Error Information Log Entries: 0", 2024-09-21T09:31:03.883 INFO:tasks.workunit.client.0.smithi022.stdout: "", 2024-09-21T09:31:03.883 INFO:tasks.workunit.client.0.smithi022.stdout: "Error Information (NVMe Log 0x01, 16 of 64 entries)", 2024-09-21T09:31:03.883 INFO:tasks.workunit.client.0.smithi022.stdout: "No Errors Logged", 2024-09-21T09:31:03.883 INFO:tasks.workunit.client.0.smithi022.stdout: "" 2024-09-21T09:31:03.883 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:03.883 INFO:tasks.workunit.client.0.smithi022.stdout: "platform_info": "x86_64-linux-5.14.0-508.el9.x86_64", 2024-09-21T09:31:03.883 INFO:tasks.workunit.client.0.smithi022.stdout: "svn_revision": "5155", 2024-09-21T09:31:03.883 INFO:tasks.workunit.client.0.smithi022.stdout: "version": [ 2024-09-21T09:31:03.883 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:03.884 INFO:tasks.workunit.client.0.smithi022.stdout: 2 2024-09-21T09:31:03.884 INFO:tasks.workunit.client.0.smithi022.stdout: ] 2024-09-21T09:31:03.884 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.884 INFO:tasks.workunit.client.0.smithi022.stdout: "temperature": { 2024-09-21T09:31:03.884 INFO:tasks.workunit.client.0.smithi022.stdout: "current": 22 2024-09-21T09:31:03.884 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.884 INFO:tasks.workunit.client.0.smithi022.stdout: "user_capacity": { 2024-09-21T09:31:03.884 INFO:tasks.workunit.client.0.smithi022.stdout: "blocks": 781422768, 2024-09-21T09:31:03.884 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 400088457216 2024-09-21T09:31:03.884 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:03.884 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:03.884 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:03.884 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:03.885 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.885 INFO:tasks.workunit.client.0.smithi022.stdout: "report": { 2024-09-21T09:31:03.885 INFO:tasks.workunit.client.0.smithi022.stdout: "balancer": { 2024-09-21T09:31:03.885 INFO:tasks.workunit.client.0.smithi022.stdout: "active": true, 2024-09-21T09:31:03.885 INFO:tasks.workunit.client.0.smithi022.stdout: "mode": "" 2024-09-21T09:31:03.885 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.885 INFO:tasks.workunit.client.0.smithi022.stdout: "channels": [ 2024-09-21T09:31:03.885 INFO:tasks.workunit.client.0.smithi022.stdout: "basic", 2024-09-21T09:31:03.885 INFO:tasks.workunit.client.0.smithi022.stdout: "crash", 2024-09-21T09:31:03.885 INFO:tasks.workunit.client.0.smithi022.stdout: "device", 2024-09-21T09:31:03.885 INFO:tasks.workunit.client.0.smithi022.stdout: "ident" 2024-09-21T09:31:03.886 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:03.886 INFO:tasks.workunit.client.0.smithi022.stdout: "channels_available": [ 2024-09-21T09:31:03.886 INFO:tasks.workunit.client.0.smithi022.stdout: "basic", 2024-09-21T09:31:03.886 INFO:tasks.workunit.client.0.smithi022.stdout: "ident", 2024-09-21T09:31:03.886 INFO:tasks.workunit.client.0.smithi022.stdout: "crash", 2024-09-21T09:31:03.886 INFO:tasks.workunit.client.0.smithi022.stdout: "device", 2024-09-21T09:31:03.886 INFO:tasks.workunit.client.0.smithi022.stdout: "perf" 2024-09-21T09:31:03.886 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:03.886 INFO:tasks.workunit.client.0.smithi022.stdout: "collections_available": [ 2024-09-21T09:31:03.886 INFO:tasks.workunit.client.0.smithi022.stdout: "basic_base", 2024-09-21T09:31:03.886 INFO:tasks.workunit.client.0.smithi022.stdout: "device_base", 2024-09-21T09:31:03.886 INFO:tasks.workunit.client.0.smithi022.stdout: "crash_base", 2024-09-21T09:31:03.887 INFO:tasks.workunit.client.0.smithi022.stdout: "ident_base", 2024-09-21T09:31:03.887 INFO:tasks.workunit.client.0.smithi022.stdout: "perf_perf", 2024-09-21T09:31:03.887 INFO:tasks.workunit.client.0.smithi022.stdout: "basic_mds_metadata", 2024-09-21T09:31:03.887 INFO:tasks.workunit.client.0.smithi022.stdout: "basic_pool_usage", 2024-09-21T09:31:03.887 INFO:tasks.workunit.client.0.smithi022.stdout: "basic_usage_by_class", 2024-09-21T09:31:03.887 INFO:tasks.workunit.client.0.smithi022.stdout: "basic_rook_v01", 2024-09-21T09:31:03.887 INFO:tasks.workunit.client.0.smithi022.stdout: "perf_memory_metrics", 2024-09-21T09:31:03.887 INFO:tasks.workunit.client.0.smithi022.stdout: "basic_pool_options_bluestore" 2024-09-21T09:31:03.887 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:03.887 INFO:tasks.workunit.client.0.smithi022.stdout: "collections_opted_in": [ 2024-09-21T09:31:03.887 INFO:tasks.workunit.client.0.smithi022.stdout: "basic_base", 2024-09-21T09:31:03.887 INFO:tasks.workunit.client.0.smithi022.stdout: "device_base", 2024-09-21T09:31:03.887 INFO:tasks.workunit.client.0.smithi022.stdout: "crash_base", 2024-09-21T09:31:03.887 INFO:tasks.workunit.client.0.smithi022.stdout: "ident_base", 2024-09-21T09:31:03.887 INFO:tasks.workunit.client.0.smithi022.stdout: "perf_perf", 2024-09-21T09:31:03.888 INFO:tasks.workunit.client.0.smithi022.stdout: "basic_mds_metadata", 2024-09-21T09:31:03.888 INFO:tasks.workunit.client.0.smithi022.stdout: "basic_pool_usage", 2024-09-21T09:31:03.888 INFO:tasks.workunit.client.0.smithi022.stdout: "basic_usage_by_class", 2024-09-21T09:31:03.888 INFO:tasks.workunit.client.0.smithi022.stdout: "basic_rook_v01", 2024-09-21T09:31:03.888 INFO:tasks.workunit.client.0.smithi022.stdout: "perf_memory_metrics", 2024-09-21T09:31:03.888 INFO:tasks.workunit.client.0.smithi022.stdout: "basic_pool_options_bluestore" 2024-09-21T09:31:03.888 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:03.888 INFO:tasks.workunit.client.0.smithi022.stdout: "config": { 2024-09-21T09:31:03.888 INFO:tasks.workunit.client.0.smithi022.stdout: "active_changed": [ 2024-09-21T09:31:03.888 INFO:tasks.workunit.client.0.smithi022.stdout: "auth_allow_insecure_global_id_reclaim", 2024-09-21T09:31:03.888 INFO:tasks.workunit.client.0.smithi022.stdout: "auth_debug", 2024-09-21T09:31:03.888 INFO:tasks.workunit.client.0.smithi022.stdout: "auth_mon_ticket_ttl", 2024-09-21T09:31:03.888 INFO:tasks.workunit.client.0.smithi022.stdout: "auth_service_ticket_ttl", 2024-09-21T09:31:03.889 INFO:tasks.workunit.client.0.smithi022.stdout: "bdev_debug_aio", 2024-09-21T09:31:03.889 INFO:tasks.workunit.client.0.smithi022.stdout: "client_check_pool_perm", 2024-09-21T09:31:03.889 INFO:tasks.workunit.client.0.smithi022.stdout: "client_mount_gid", 2024-09-21T09:31:03.889 INFO:tasks.workunit.client.0.smithi022.stdout: "client_mount_uid", 2024-09-21T09:31:03.889 INFO:tasks.workunit.client.0.smithi022.stdout: "client_quota", 2024-09-21T09:31:03.889 INFO:tasks.workunit.client.0.smithi022.stdout: "container_image", 2024-09-21T09:31:03.889 INFO:tasks.workunit.client.0.smithi022.stdout: "daemonize", 2024-09-21T09:31:03.889 INFO:tasks.workunit.client.0.smithi022.stdout: "debug_asserts_on_shutdown", 2024-09-21T09:31:03.889 INFO:tasks.workunit.client.0.smithi022.stdout: "debug_mgr", 2024-09-21T09:31:03.889 INFO:tasks.workunit.client.0.smithi022.stdout: "debug_mon", 2024-09-21T09:31:03.889 INFO:tasks.workunit.client.0.smithi022.stdout: "debug_ms", 2024-09-21T09:31:03.889 INFO:tasks.workunit.client.0.smithi022.stdout: "debug_osd", 2024-09-21T09:31:03.889 INFO:tasks.workunit.client.0.smithi022.stdout: "debug_paxos", 2024-09-21T09:31:03.890 INFO:tasks.workunit.client.0.smithi022.stdout: "keyring", 2024-09-21T09:31:03.890 INFO:tasks.workunit.client.0.smithi022.stdout: "leveldb_block_size", 2024-09-21T09:31:03.890 INFO:tasks.workunit.client.0.smithi022.stdout: "leveldb_cache_size", 2024-09-21T09:31:03.890 INFO:tasks.workunit.client.0.smithi022.stdout: "leveldb_compression", 2024-09-21T09:31:03.890 INFO:tasks.workunit.client.0.smithi022.stdout: "leveldb_log", 2024-09-21T09:31:03.890 INFO:tasks.workunit.client.0.smithi022.stdout: "leveldb_write_buffer_size", 2024-09-21T09:31:03.890 INFO:tasks.workunit.client.0.smithi022.stdout: "log_to_file", 2024-09-21T09:31:03.890 INFO:tasks.workunit.client.0.smithi022.stdout: "log_to_journald", 2024-09-21T09:31:03.890 INFO:tasks.workunit.client.0.smithi022.stdout: "log_to_stderr", 2024-09-21T09:31:03.890 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_allow_pool_delete", 2024-09-21T09:31:03.890 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_clock_drift_allowed", 2024-09-21T09:31:03.890 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_cluster_log_to_file", 2024-09-21T09:31:03.890 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_cluster_log_to_journald", 2024-09-21T09:31:03.890 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_cluster_log_to_stderr", 2024-09-21T09:31:03.891 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_data_avail_warn", 2024-09-21T09:31:03.891 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_election_default_strategy", 2024-09-21T09:31:03.891 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_host", 2024-09-21T09:31:03.891 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_max_pg_per_osd", 2024-09-21T09:31:03.891 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_mgr_mkfs_grace", 2024-09-21T09:31:03.891 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_osd_prime_pg_temp", 2024-09-21T09:31:03.891 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_osd_reporter_subtree_level", 2024-09-21T09:31:03.891 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_pg_warn_max_object_skew", 2024-09-21T09:31:03.891 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_reweight_min_bytes_per_osd", 2024-09-21T09:31:03.891 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_reweight_min_pgs_per_osd", 2024-09-21T09:31:03.891 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_warn_on_crush_straw_calc_version_zero", 2024-09-21T09:31:03.891 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_warn_on_insecure_global_id_reclaim", 2024-09-21T09:31:03.891 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_warn_on_insecure_global_id_reclaim_allowed", 2024-09-21T09:31:03.891 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_warn_on_legacy_crush_tunables", 2024-09-21T09:31:03.891 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_warn_on_osd_down_out_interval_zero", 2024-09-21T09:31:03.891 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_warn_on_pool_pg_num_not_power_of_two", 2024-09-21T09:31:03.892 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_warn_on_too_few_osds", 2024-09-21T09:31:03.892 INFO:tasks.workunit.client.0.smithi022.stdout: "ms_die_on_bug", 2024-09-21T09:31:03.892 INFO:tasks.workunit.client.0.smithi022.stdout: "ms_die_on_old_message", 2024-09-21T09:31:03.892 INFO:tasks.workunit.client.0.smithi022.stdout: "no_config_file", 2024-09-21T09:31:03.892 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_class_default_list", 2024-09-21T09:31:03.892 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_class_load_list", 2024-09-21T09:31:03.892 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_debug_misdirected_ops", 2024-09-21T09:31:03.892 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_debug_op_order", 2024-09-21T09:31:03.892 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_debug_pg_log_writeout", 2024-09-21T09:31:03.892 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_debug_shutdown", 2024-09-21T09:31:03.892 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_debug_verify_cached_snaps", 2024-09-21T09:31:03.892 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_debug_verify_missing_on_start", 2024-09-21T09:31:03.893 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_debug_verify_stray_on_activate", 2024-09-21T09:31:03.893 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_deep_scrub_update_digest_min_age", 2024-09-21T09:31:03.893 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_mclock_profile", 2024-09-21T09:31:03.893 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_mclock_scheduler_background_best_effort_lim", 2024-09-21T09:31:03.893 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_mclock_scheduler_background_best_effort_res", 2024-09-21T09:31:03.893 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_mclock_scheduler_background_best_effort_wgt", 2024-09-21T09:31:03.893 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_mclock_scheduler_background_recovery_lim", 2024-09-21T09:31:03.893 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_mclock_scheduler_background_recovery_res", 2024-09-21T09:31:03.893 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_mclock_scheduler_background_recovery_wgt", 2024-09-21T09:31:03.893 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_mclock_scheduler_client_lim", 2024-09-21T09:31:03.893 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_mclock_scheduler_client_res", 2024-09-21T09:31:03.893 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_mclock_scheduler_client_wgt", 2024-09-21T09:31:03.893 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_memory_target", 2024-09-21T09:31:03.893 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_memory_target_autotune", 2024-09-21T09:31:03.894 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_op_queue", 2024-09-21T09:31:03.894 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_op_queue_cut_off", 2024-09-21T09:31:03.894 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_pool_default_erasure_code_profile", 2024-09-21T09:31:03.894 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_pool_default_pg_autoscale_mode", 2024-09-21T09:31:03.894 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_recover_clone_overlap", 2024-09-21T09:31:03.894 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_recovery_max_chunk", 2024-09-21T09:31:03.894 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_scrub_load_threshold", 2024-09-21T09:31:03.894 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_scrub_max_interval", 2024-09-21T09:31:03.894 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_shutdown_pgref_assert", 2024-09-21T09:31:03.894 INFO:tasks.workunit.client.0.smithi022.stdout: "public_addr", 2024-09-21T09:31:03.894 INFO:tasks.workunit.client.0.smithi022.stdout: "public_addrv", 2024-09-21T09:31:03.894 INFO:tasks.workunit.client.0.smithi022.stdout: "public_network", 2024-09-21T09:31:03.894 INFO:tasks.workunit.client.0.smithi022.stdout: "rbd_default_features", 2024-09-21T09:31:03.894 INFO:tasks.workunit.client.0.smithi022.stdout: "rbd_qos_exclude_ops", 2024-09-21T09:31:03.895 INFO:tasks.workunit.client.0.smithi022.stdout: "setgroup", 2024-09-21T09:31:03.895 INFO:tasks.workunit.client.0.smithi022.stdout: "setuser" 2024-09-21T09:31:03.895 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:03.895 INFO:tasks.workunit.client.0.smithi022.stdout: "cluster_changed": [ 2024-09-21T09:31:03.895 INFO:tasks.workunit.client.0.smithi022.stdout: "auth_allow_insecure_global_id_reclaim", 2024-09-21T09:31:03.895 INFO:tasks.workunit.client.0.smithi022.stdout: "auth_debug", 2024-09-21T09:31:03.895 INFO:tasks.workunit.client.0.smithi022.stdout: "auth_mon_ticket_ttl", 2024-09-21T09:31:03.895 INFO:tasks.workunit.client.0.smithi022.stdout: "auth_service_ticket_ttl", 2024-09-21T09:31:03.895 INFO:tasks.workunit.client.0.smithi022.stdout: "bdev_debug_aio", 2024-09-21T09:31:03.895 INFO:tasks.workunit.client.0.smithi022.stdout: "container_image", 2024-09-21T09:31:03.895 INFO:tasks.workunit.client.0.smithi022.stdout: "debug_asserts_on_shutdown", 2024-09-21T09:31:03.895 INFO:tasks.workunit.client.0.smithi022.stdout: "debug_mgr", 2024-09-21T09:31:03.895 INFO:tasks.workunit.client.0.smithi022.stdout: "debug_mon", 2024-09-21T09:31:03.895 INFO:tasks.workunit.client.0.smithi022.stdout: "debug_ms", 2024-09-21T09:31:03.895 INFO:tasks.workunit.client.0.smithi022.stdout: "debug_osd", 2024-09-21T09:31:03.896 INFO:tasks.workunit.client.0.smithi022.stdout: "debug_paxos", 2024-09-21T09:31:03.896 INFO:tasks.workunit.client.0.smithi022.stdout: "log_to_file", 2024-09-21T09:31:03.896 INFO:tasks.workunit.client.0.smithi022.stdout: "log_to_journald", 2024-09-21T09:31:03.896 INFO:tasks.workunit.client.0.smithi022.stdout: "log_to_stderr", 2024-09-21T09:31:03.896 INFO:tasks.workunit.client.0.smithi022.stdout: "mds_join_fs", 2024-09-21T09:31:03.896 INFO:tasks.workunit.client.0.smithi022.stdout: "mgr/cephadm/allow_ptrace", 2024-09-21T09:31:03.896 INFO:tasks.workunit.client.0.smithi022.stdout: "mgr/cephadm/container_init", 2024-09-21T09:31:03.896 INFO:tasks.workunit.client.0.smithi022.stdout: "mgr/cephadm/migration_current", 2024-09-21T09:31:03.896 INFO:tasks.workunit.client.0.smithi022.stdout: "mgr/cephadm/use_repo_digest", 2024-09-21T09:31:03.896 INFO:tasks.workunit.client.0.smithi022.stdout: "mgr/dashboard/ALERTMANAGER_API_HOST", 2024-09-21T09:31:03.896 INFO:tasks.workunit.client.0.smithi022.stdout: "mgr/dashboard/GRAFANA_API_SSL_VERIFY", 2024-09-21T09:31:03.896 INFO:tasks.workunit.client.0.smithi022.stdout: "mgr/dashboard/GRAFANA_API_URL", 2024-09-21T09:31:03.896 INFO:tasks.workunit.client.0.smithi022.stdout: "mgr/dashboard/PROMETHEUS_API_HOST", 2024-09-21T09:31:03.896 INFO:tasks.workunit.client.0.smithi022.stdout: "mgr/dashboard/ssl_server_port", 2024-09-21T09:31:03.896 INFO:tasks.workunit.client.0.smithi022.stdout: "mgr/orchestrator/orchestrator", 2024-09-21T09:31:03.896 INFO:tasks.workunit.client.0.smithi022.stdout: "mgr/telemetry/channel_ident", 2024-09-21T09:31:03.897 INFO:tasks.workunit.client.0.smithi022.stdout: "mgr/telemetry/description", 2024-09-21T09:31:03.897 INFO:tasks.workunit.client.0.smithi022.stdout: "mgr/telemetry/organization", 2024-09-21T09:31:03.897 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_allow_pool_delete", 2024-09-21T09:31:03.897 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_clock_drift_allowed", 2024-09-21T09:31:03.897 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_cluster_log_to_file", 2024-09-21T09:31:03.897 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_data_avail_warn", 2024-09-21T09:31:03.897 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_election_default_strategy", 2024-09-21T09:31:03.897 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_max_pg_per_osd", 2024-09-21T09:31:03.897 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_mgr_mkfs_grace", 2024-09-21T09:31:03.897 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_osd_prime_pg_temp", 2024-09-21T09:31:03.897 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_osd_reporter_subtree_level", 2024-09-21T09:31:03.897 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_pg_warn_max_object_skew", 2024-09-21T09:31:03.897 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_reweight_min_bytes_per_osd", 2024-09-21T09:31:03.897 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_reweight_min_pgs_per_osd", 2024-09-21T09:31:03.898 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_warn_on_crush_straw_calc_version_zero", 2024-09-21T09:31:03.898 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_warn_on_insecure_global_id_reclaim", 2024-09-21T09:31:03.898 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_warn_on_insecure_global_id_reclaim_allowed", 2024-09-21T09:31:03.898 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_warn_on_legacy_crush_tunables", 2024-09-21T09:31:03.898 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_warn_on_osd_down_out_interval_zero", 2024-09-21T09:31:03.898 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_warn_on_pool_pg_num_not_power_of_two", 2024-09-21T09:31:03.898 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_warn_on_too_few_osds", 2024-09-21T09:31:03.898 INFO:tasks.workunit.client.0.smithi022.stdout: "ms_die_on_bug", 2024-09-21T09:31:03.898 INFO:tasks.workunit.client.0.smithi022.stdout: "ms_die_on_old_message", 2024-09-21T09:31:03.898 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_class_default_list", 2024-09-21T09:31:03.898 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_class_load_list", 2024-09-21T09:31:03.898 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_debug_misdirected_ops", 2024-09-21T09:31:03.899 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_debug_op_order", 2024-09-21T09:31:03.899 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_debug_pg_log_writeout", 2024-09-21T09:31:03.899 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_debug_shutdown", 2024-09-21T09:31:03.899 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_debug_verify_cached_snaps", 2024-09-21T09:31:03.899 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_debug_verify_missing_on_start", 2024-09-21T09:31:03.899 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_debug_verify_stray_on_activate", 2024-09-21T09:31:03.899 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_deep_scrub_update_digest_min_age", 2024-09-21T09:31:03.899 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_mclock_profile", 2024-09-21T09:31:03.899 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_memory_target", 2024-09-21T09:31:03.899 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_memory_target_autotune", 2024-09-21T09:31:03.899 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_op_queue", 2024-09-21T09:31:03.900 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_op_queue_cut_off", 2024-09-21T09:31:03.900 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_pool_default_erasure_code_profile", 2024-09-21T09:31:03.900 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_pool_default_pg_autoscale_mode", 2024-09-21T09:31:03.900 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_recover_clone_overlap", 2024-09-21T09:31:03.900 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_recovery_max_chunk", 2024-09-21T09:31:03.900 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_scrub_load_threshold", 2024-09-21T09:31:03.900 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_scrub_max_interval", 2024-09-21T09:31:03.900 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_shutdown_pgref_assert", 2024-09-21T09:31:03.900 INFO:tasks.workunit.client.0.smithi022.stdout: "public_network", 2024-09-21T09:31:03.900 INFO:tasks.workunit.client.0.smithi022.stdout: "rgw_cache_enabled", 2024-09-21T09:31:03.900 INFO:tasks.workunit.client.0.smithi022.stdout: "rgw_enable_ops_log", 2024-09-21T09:31:03.901 INFO:tasks.workunit.client.0.smithi022.stdout: "rgw_enable_usage_log" 2024-09-21T09:31:03.901 INFO:tasks.workunit.client.0.smithi022.stdout: ] 2024-09-21T09:31:03.901 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.901 INFO:tasks.workunit.client.0.smithi022.stdout: "contact": null, 2024-09-21T09:31:03.901 INFO:tasks.workunit.client.0.smithi022.stdout: "crashes": [], 2024-09-21T09:31:03.901 INFO:tasks.workunit.client.0.smithi022.stdout: "created": "2024-09-21T09:22:01.045656Z", 2024-09-21T09:31:03.901 INFO:tasks.workunit.client.0.smithi022.stdout: "crush": { 2024-09-21T09:31:03.901 INFO:tasks.workunit.client.0.smithi022.stdout: "bucket_algs": { 2024-09-21T09:31:03.901 INFO:tasks.workunit.client.0.smithi022.stdout: "straw2": 3 2024-09-21T09:31:03.901 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.901 INFO:tasks.workunit.client.0.smithi022.stdout: "bucket_sizes": { 2024-09-21T09:31:03.901 INFO:tasks.workunit.client.0.smithi022.stdout: "2": 1, 2024-09-21T09:31:03.901 INFO:tasks.workunit.client.0.smithi022.stdout: "4": 2 2024-09-21T09:31:03.901 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.902 INFO:tasks.workunit.client.0.smithi022.stdout: "bucket_types": { 2024-09-21T09:31:03.902 INFO:tasks.workunit.client.0.smithi022.stdout: "1": 2, 2024-09-21T09:31:03.902 INFO:tasks.workunit.client.0.smithi022.stdout: "11": 1 2024-09-21T09:31:03.902 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.902 INFO:tasks.workunit.client.0.smithi022.stdout: "compat_weight_set": false, 2024-09-21T09:31:03.902 INFO:tasks.workunit.client.0.smithi022.stdout: "device_classes": [ 2024-09-21T09:31:03.902 INFO:tasks.workunit.client.0.smithi022.stdout: 8 2024-09-21T09:31:03.902 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:03.902 INFO:tasks.workunit.client.0.smithi022.stdout: "num_buckets": 6, 2024-09-21T09:31:03.902 INFO:tasks.workunit.client.0.smithi022.stdout: "num_devices": 8, 2024-09-21T09:31:03.902 INFO:tasks.workunit.client.0.smithi022.stdout: "num_rules": 1, 2024-09-21T09:31:03.902 INFO:tasks.workunit.client.0.smithi022.stdout: "num_types": 12, 2024-09-21T09:31:03.902 INFO:tasks.workunit.client.0.smithi022.stdout: "num_weight_sets": 0, 2024-09-21T09:31:03.902 INFO:tasks.workunit.client.0.smithi022.stdout: "tunables": { 2024-09-21T09:31:03.903 INFO:tasks.workunit.client.0.smithi022.stdout: "allowed_bucket_algs": 54, 2024-09-21T09:31:03.903 INFO:tasks.workunit.client.0.smithi022.stdout: "choose_local_fallback_tries": 0, 2024-09-21T09:31:03.903 INFO:tasks.workunit.client.0.smithi022.stdout: "choose_local_tries": 0, 2024-09-21T09:31:03.903 INFO:tasks.workunit.client.0.smithi022.stdout: "choose_total_tries": 50, 2024-09-21T09:31:03.903 INFO:tasks.workunit.client.0.smithi022.stdout: "chooseleaf_descend_once": 1, 2024-09-21T09:31:03.903 INFO:tasks.workunit.client.0.smithi022.stdout: "chooseleaf_stable": 1, 2024-09-21T09:31:03.903 INFO:tasks.workunit.client.0.smithi022.stdout: "chooseleaf_vary_r": 1, 2024-09-21T09:31:03.903 INFO:tasks.workunit.client.0.smithi022.stdout: "has_v2_rules": 0, 2024-09-21T09:31:03.903 INFO:tasks.workunit.client.0.smithi022.stdout: "has_v3_rules": 0, 2024-09-21T09:31:03.903 INFO:tasks.workunit.client.0.smithi022.stdout: "has_v4_buckets": 1, 2024-09-21T09:31:03.903 INFO:tasks.workunit.client.0.smithi022.stdout: "has_v5_rules": 0, 2024-09-21T09:31:03.903 INFO:tasks.workunit.client.0.smithi022.stdout: "legacy_tunables": 0, 2024-09-21T09:31:03.903 INFO:tasks.workunit.client.0.smithi022.stdout: "minimum_required_version": "jewel", 2024-09-21T09:31:03.903 INFO:tasks.workunit.client.0.smithi022.stdout: "optimal_tunables": 1, 2024-09-21T09:31:03.904 INFO:tasks.workunit.client.0.smithi022.stdout: "profile": "jewel", 2024-09-21T09:31:03.904 INFO:tasks.workunit.client.0.smithi022.stdout: "require_feature_tunables": 1, 2024-09-21T09:31:03.904 INFO:tasks.workunit.client.0.smithi022.stdout: "require_feature_tunables2": 1, 2024-09-21T09:31:03.904 INFO:tasks.workunit.client.0.smithi022.stdout: "require_feature_tunables3": 1, 2024-09-21T09:31:03.904 INFO:tasks.workunit.client.0.smithi022.stdout: "require_feature_tunables5": 1, 2024-09-21T09:31:03.904 INFO:tasks.workunit.client.0.smithi022.stdout: "straw_calc_version": 1 2024-09-21T09:31:03.904 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:03.904 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.904 INFO:tasks.workunit.client.0.smithi022.stdout: "description": "upgrade test cluster", 2024-09-21T09:31:03.904 INFO:tasks.workunit.client.0.smithi022.stdout: "fs": { 2024-09-21T09:31:03.904 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 1, 2024-09-21T09:31:03.904 INFO:tasks.workunit.client.0.smithi022.stdout: "feature_flags": { 2024-09-21T09:31:03.904 INFO:tasks.workunit.client.0.smithi022.stdout: "enable_multiple": true, 2024-09-21T09:31:03.904 INFO:tasks.workunit.client.0.smithi022.stdout: "ever_enabled_multiple": true 2024-09-21T09:31:03.904 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.905 INFO:tasks.workunit.client.0.smithi022.stdout: "filesystems": [ 2024-09-21T09:31:03.905 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:03.905 INFO:tasks.workunit.client.0.smithi022.stdout: "approx_ctime": "2024-09", 2024-09-21T09:31:03.905 INFO:tasks.workunit.client.0.smithi022.stdout: "balancer_enabled": false, 2024-09-21T09:31:03.905 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:03.905 INFO:tasks.workunit.client.0.smithi022.stdout: "cached_caps": 0, 2024-09-21T09:31:03.905 INFO:tasks.workunit.client.0.smithi022.stdout: "cached_dns": 10, 2024-09-21T09:31:03.905 INFO:tasks.workunit.client.0.smithi022.stdout: "cached_inos": 13, 2024-09-21T09:31:03.906 INFO:tasks.workunit.client.0.smithi022.stdout: "cached_subtrees": 2, 2024-09-21T09:31:03.906 INFO:tasks.workunit.client.0.smithi022.stdout: "ever_allowed_features": 0, 2024-09-21T09:31:03.906 INFO:tasks.workunit.client.0.smithi022.stdout: "explicitly_allowed_features": 0, 2024-09-21T09:31:03.906 INFO:tasks.workunit.client.0.smithi022.stdout: "files": 0, 2024-09-21T09:31:03.906 INFO:tasks.workunit.client.0.smithi022.stdout: "max_mds": 1, 2024-09-21T09:31:03.906 INFO:tasks.workunit.client.0.smithi022.stdout: "num_data_pools": 1, 2024-09-21T09:31:03.906 INFO:tasks.workunit.client.0.smithi022.stdout: "num_in": 1, 2024-09-21T09:31:03.906 INFO:tasks.workunit.client.0.smithi022.stdout: "num_mds": 1, 2024-09-21T09:31:03.906 INFO:tasks.workunit.client.0.smithi022.stdout: "num_sessions": 0, 2024-09-21T09:31:03.906 INFO:tasks.workunit.client.0.smithi022.stdout: "num_standby_replay": 0, 2024-09-21T09:31:03.906 INFO:tasks.workunit.client.0.smithi022.stdout: "num_up": 1, 2024-09-21T09:31:03.906 INFO:tasks.workunit.client.0.smithi022.stdout: "snaps": 0, 2024-09-21T09:31:03.906 INFO:tasks.workunit.client.0.smithi022.stdout: "standby_count_wanted": 0 2024-09-21T09:31:03.907 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:03.907 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:03.907 INFO:tasks.workunit.client.0.smithi022.stdout: "num_standby_mds": 1, 2024-09-21T09:31:03.907 INFO:tasks.workunit.client.0.smithi022.stdout: "total_num_mds": 2 2024-09-21T09:31:03.907 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.907 INFO:tasks.workunit.client.0.smithi022.stdout: "hosts": { 2024-09-21T09:31:03.907 INFO:tasks.workunit.client.0.smithi022.stdout: "num": 2, 2024-09-21T09:31:03.907 INFO:tasks.workunit.client.0.smithi022.stdout: "num_with_mds": 2, 2024-09-21T09:31:03.907 INFO:tasks.workunit.client.0.smithi022.stdout: "num_with_mgr": 2, 2024-09-21T09:31:03.907 INFO:tasks.workunit.client.0.smithi022.stdout: "num_with_mon": 2, 2024-09-21T09:31:03.908 INFO:tasks.workunit.client.0.smithi022.stdout: "num_with_osd": 2 2024-09-21T09:31:03.908 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.908 INFO:tasks.workunit.client.0.smithi022.stdout: "leaderboard": false, 2024-09-21T09:31:03.908 INFO:tasks.workunit.client.0.smithi022.stdout: "license": "sharing-1-0", 2024-09-21T09:31:03.908 INFO:tasks.workunit.client.0.smithi022.stdout: "metadata": { 2024-09-21T09:31:03.908 INFO:tasks.workunit.client.0.smithi022.stdout: "mds": { 2024-09-21T09:31:03.908 INFO:tasks.workunit.client.0.smithi022.stdout: "arch": { 2024-09-21T09:31:03.908 INFO:tasks.workunit.client.0.smithi022.stdout: "x86_64": 2 2024-09-21T09:31:03.908 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.908 INFO:tasks.workunit.client.0.smithi022.stdout: "ceph_version": { 2024-09-21T09:31:03.908 INFO:tasks.workunit.client.0.smithi022.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 2 2024-09-21T09:31:03.908 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.908 INFO:tasks.workunit.client.0.smithi022.stdout: "cpu": { 2024-09-21T09:31:03.909 INFO:tasks.workunit.client.0.smithi022.stdout: "Intel(R) Xeon(R) CPU E5-1620 v3 @ 3.50GHz": 2 2024-09-21T09:31:03.909 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.909 INFO:tasks.workunit.client.0.smithi022.stdout: "distro": { 2024-09-21T09:31:03.909 INFO:tasks.workunit.client.0.smithi022.stdout: "centos": 2 2024-09-21T09:31:03.909 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.909 INFO:tasks.workunit.client.0.smithi022.stdout: "distro_description": { 2024-09-21T09:31:03.909 INFO:tasks.workunit.client.0.smithi022.stdout: "CentOS Stream 9": 2 2024-09-21T09:31:03.909 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.909 INFO:tasks.workunit.client.0.smithi022.stdout: "kernel_description": { 2024-09-21T09:31:03.909 INFO:tasks.workunit.client.0.smithi022.stdout: "#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024": 2 2024-09-21T09:31:03.909 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.910 INFO:tasks.workunit.client.0.smithi022.stdout: "kernel_version": { 2024-09-21T09:31:03.910 INFO:tasks.workunit.client.0.smithi022.stdout: "5.14.0-508.el9.x86_64": 2 2024-09-21T09:31:03.910 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.910 INFO:tasks.workunit.client.0.smithi022.stdout: "os": { 2024-09-21T09:31:03.910 INFO:tasks.workunit.client.0.smithi022.stdout: "Linux": 2 2024-09-21T09:31:03.910 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:03.910 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.910 INFO:tasks.workunit.client.0.smithi022.stdout: "mon": { 2024-09-21T09:31:03.910 INFO:tasks.workunit.client.0.smithi022.stdout: "arch": { 2024-09-21T09:31:03.910 INFO:tasks.workunit.client.0.smithi022.stdout: "x86_64": 3 2024-09-21T09:31:03.910 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.910 INFO:tasks.workunit.client.0.smithi022.stdout: "ceph_version": { 2024-09-21T09:31:03.910 INFO:tasks.workunit.client.0.smithi022.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 3 2024-09-21T09:31:03.911 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.911 INFO:tasks.workunit.client.0.smithi022.stdout: "cpu": { 2024-09-21T09:31:03.911 INFO:tasks.workunit.client.0.smithi022.stdout: "Intel(R) Xeon(R) CPU E5-1620 v3 @ 3.50GHz": 3 2024-09-21T09:31:03.911 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.911 INFO:tasks.workunit.client.0.smithi022.stdout: "distro": { 2024-09-21T09:31:03.911 INFO:tasks.workunit.client.0.smithi022.stdout: "centos": 3 2024-09-21T09:31:03.911 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.911 INFO:tasks.workunit.client.0.smithi022.stdout: "distro_description": { 2024-09-21T09:31:03.911 INFO:tasks.workunit.client.0.smithi022.stdout: "CentOS Stream 9": 3 2024-09-21T09:31:03.911 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.911 INFO:tasks.workunit.client.0.smithi022.stdout: "kernel_description": { 2024-09-21T09:31:03.912 INFO:tasks.workunit.client.0.smithi022.stdout: "#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024": 3 2024-09-21T09:31:03.912 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.912 INFO:tasks.workunit.client.0.smithi022.stdout: "kernel_version": { 2024-09-21T09:31:03.912 INFO:tasks.workunit.client.0.smithi022.stdout: "5.14.0-508.el9.x86_64": 3 2024-09-21T09:31:03.912 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.912 INFO:tasks.workunit.client.0.smithi022.stdout: "os": { 2024-09-21T09:31:03.912 INFO:tasks.workunit.client.0.smithi022.stdout: "Linux": 3 2024-09-21T09:31:03.912 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:03.912 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.912 INFO:tasks.workunit.client.0.smithi022.stdout: "osd": { 2024-09-21T09:31:03.912 INFO:tasks.workunit.client.0.smithi022.stdout: "arch": { 2024-09-21T09:31:03.912 INFO:tasks.workunit.client.0.smithi022.stdout: "x86_64": 8 2024-09-21T09:31:03.912 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.912 INFO:tasks.workunit.client.0.smithi022.stdout: "ceph_version": { 2024-09-21T09:31:03.912 INFO:tasks.workunit.client.0.smithi022.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 8 2024-09-21T09:31:03.913 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.913 INFO:tasks.workunit.client.0.smithi022.stdout: "cpu": { 2024-09-21T09:31:03.913 INFO:tasks.workunit.client.0.smithi022.stdout: "Intel(R) Xeon(R) CPU E5-1620 v3 @ 3.50GHz": 8 2024-09-21T09:31:03.913 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.913 INFO:tasks.workunit.client.0.smithi022.stdout: "distro": { 2024-09-21T09:31:03.913 INFO:tasks.workunit.client.0.smithi022.stdout: "centos": 8 2024-09-21T09:31:03.913 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.913 INFO:tasks.workunit.client.0.smithi022.stdout: "distro_description": { 2024-09-21T09:31:03.913 INFO:tasks.workunit.client.0.smithi022.stdout: "CentOS Stream 9": 8 2024-09-21T09:31:03.913 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.913 INFO:tasks.workunit.client.0.smithi022.stdout: "kernel_description": { 2024-09-21T09:31:03.913 INFO:tasks.workunit.client.0.smithi022.stdout: "#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024": 8 2024-09-21T09:31:03.913 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.913 INFO:tasks.workunit.client.0.smithi022.stdout: "kernel_version": { 2024-09-21T09:31:03.914 INFO:tasks.workunit.client.0.smithi022.stdout: "5.14.0-508.el9.x86_64": 8 2024-09-21T09:31:03.914 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.914 INFO:tasks.workunit.client.0.smithi022.stdout: "os": { 2024-09-21T09:31:03.914 INFO:tasks.workunit.client.0.smithi022.stdout: "Linux": 8 2024-09-21T09:31:03.914 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.914 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_objectstore": { 2024-09-21T09:31:03.914 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore": 8 2024-09-21T09:31:03.914 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.914 INFO:tasks.workunit.client.0.smithi022.stdout: "rotational": { 2024-09-21T09:31:03.914 INFO:tasks.workunit.client.0.smithi022.stdout: "0": 8 2024-09-21T09:31:03.914 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:03.914 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:03.914 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.915 INFO:tasks.workunit.client.0.smithi022.stdout: "mon": { 2024-09-21T09:31:03.915 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 3, 2024-09-21T09:31:03.915 INFO:tasks.workunit.client.0.smithi022.stdout: "features": { 2024-09-21T09:31:03.915 INFO:tasks.workunit.client.0.smithi022.stdout: "optional": [], 2024-09-21T09:31:03.915 INFO:tasks.workunit.client.0.smithi022.stdout: "persistent": [ 2024-09-21T09:31:03.915 INFO:tasks.workunit.client.0.smithi022.stdout: "kraken", 2024-09-21T09:31:03.915 INFO:tasks.workunit.client.0.smithi022.stdout: "luminous", 2024-09-21T09:31:03.915 INFO:tasks.workunit.client.0.smithi022.stdout: "mimic", 2024-09-21T09:31:03.915 INFO:tasks.workunit.client.0.smithi022.stdout: "osdmap-prune", 2024-09-21T09:31:03.915 INFO:tasks.workunit.client.0.smithi022.stdout: "nautilus", 2024-09-21T09:31:03.915 INFO:tasks.workunit.client.0.smithi022.stdout: "octopus", 2024-09-21T09:31:03.915 INFO:tasks.workunit.client.0.smithi022.stdout: "pacific", 2024-09-21T09:31:03.915 INFO:tasks.workunit.client.0.smithi022.stdout: "elector-pinging", 2024-09-21T09:31:03.916 INFO:tasks.workunit.client.0.smithi022.stdout: "quincy" 2024-09-21T09:31:03.916 INFO:tasks.workunit.client.0.smithi022.stdout: ] 2024-09-21T09:31:03.916 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.916 INFO:tasks.workunit.client.0.smithi022.stdout: "ipv4_addr_mons": 6, 2024-09-21T09:31:03.916 INFO:tasks.workunit.client.0.smithi022.stdout: "ipv6_addr_mons": 0, 2024-09-21T09:31:03.916 INFO:tasks.workunit.client.0.smithi022.stdout: "min_mon_release": 17, 2024-09-21T09:31:03.916 INFO:tasks.workunit.client.0.smithi022.stdout: "v1_addr_mons": 3, 2024-09-21T09:31:03.916 INFO:tasks.workunit.client.0.smithi022.stdout: "v2_addr_mons": 3 2024-09-21T09:31:03.916 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.916 INFO:tasks.workunit.client.0.smithi022.stdout: "organization": "ceph-qa", 2024-09-21T09:31:03.916 INFO:tasks.workunit.client.0.smithi022.stdout: "osd": { 2024-09-21T09:31:03.916 INFO:tasks.workunit.client.0.smithi022.stdout: "cluster_network": false, 2024-09-21T09:31:03.916 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 8, 2024-09-21T09:31:03.916 INFO:tasks.workunit.client.0.smithi022.stdout: "require_min_compat_client": "luminous", 2024-09-21T09:31:03.917 INFO:tasks.workunit.client.0.smithi022.stdout: "require_osd_release": "quincy" 2024-09-21T09:31:03.917 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.917 INFO:tasks.workunit.client.0.smithi022.stdout: "pools": [ 2024-09-21T09:31:03.917 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:03.917 INFO:tasks.workunit.client.0.smithi022.stdout: "application": [ 2024-09-21T09:31:03.917 INFO:tasks.workunit.client.0.smithi022.stdout: "mgr" 2024-09-21T09:31:03.917 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:03.917 INFO:tasks.workunit.client.0.smithi022.stdout: "cache_mode": "none", 2024-09-21T09:31:03.917 INFO:tasks.workunit.client.0.smithi022.stdout: "erasure_code_profile": {}, 2024-09-21T09:31:03.917 INFO:tasks.workunit.client.0.smithi022.stdout: "min_size": 2, 2024-09-21T09:31:03.917 INFO:tasks.workunit.client.0.smithi022.stdout: "options": {}, 2024-09-21T09:31:03.917 INFO:tasks.workunit.client.0.smithi022.stdout: "pg_autoscale_mode": "off", 2024-09-21T09:31:03.917 INFO:tasks.workunit.client.0.smithi022.stdout: "pg_num": 1, 2024-09-21T09:31:03.917 INFO:tasks.workunit.client.0.smithi022.stdout: "pgp_num": 1, 2024-09-21T09:31:03.917 INFO:tasks.workunit.client.0.smithi022.stdout: "pool": 1, 2024-09-21T09:31:03.918 INFO:tasks.workunit.client.0.smithi022.stdout: "size": 3, 2024-09-21T09:31:03.918 INFO:tasks.workunit.client.0.smithi022.stdout: "stats": { 2024-09-21T09:31:03.918 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_raw": 727115966730, 2024-09-21T09:31:03.918 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes_used": 1781760, 2024-09-21T09:31:03.918 INFO:tasks.workunit.client.0.smithi022.stdout: "compress_bytes_used": 0, 2024-09-21T09:31:03.918 INFO:tasks.workunit.client.0.smithi022.stdout: "compress_under_bytes": 0, 2024-09-21T09:31:03.918 INFO:tasks.workunit.client.0.smithi022.stdout: "data_bytes_used": 1781760, 2024-09-21T09:31:03.918 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty": 0, 2024-09-21T09:31:03.918 INFO:tasks.workunit.client.0.smithi022.stdout: "max_avail": 242371985408, 2024-09-21T09:31:03.918 INFO:tasks.workunit.client.0.smithi022.stdout: "objects": 2, 2024-09-21T09:31:03.918 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_bytes_used": 0, 2024-09-21T09:31:03.919 INFO:tasks.workunit.client.0.smithi022.stdout: "percent_used": 2.4504422526661074e-06, 2024-09-21T09:31:03.919 INFO:tasks.workunit.client.0.smithi022.stdout: "quota_bytes": 0, 2024-09-21T09:31:03.919 INFO:tasks.workunit.client.0.smithi022.stdout: "quota_objects": 0, 2024-09-21T09:31:03.919 INFO:tasks.workunit.client.0.smithi022.stdout: "rd": 126, 2024-09-21T09:31:03.919 INFO:tasks.workunit.client.0.smithi022.stdout: "rd_bytes": 236544, 2024-09-21T09:31:03.919 INFO:tasks.workunit.client.0.smithi022.stdout: "stored": 590368, 2024-09-21T09:31:03.919 INFO:tasks.workunit.client.0.smithi022.stdout: "stored_data": 590368, 2024-09-21T09:31:03.919 INFO:tasks.workunit.client.0.smithi022.stdout: "stored_omap": 0, 2024-09-21T09:31:03.919 INFO:tasks.workunit.client.0.smithi022.stdout: "stored_raw": 1771104, 2024-09-21T09:31:03.919 INFO:tasks.workunit.client.0.smithi022.stdout: "wr": 113, 2024-09-21T09:31:03.919 INFO:tasks.workunit.client.0.smithi022.stdout: "wr_bytes": 1667072 2024-09-21T09:31:03.919 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.919 INFO:tasks.workunit.client.0.smithi022.stdout: "target_max_bytes": 0, 2024-09-21T09:31:03.919 INFO:tasks.workunit.client.0.smithi022.stdout: "target_max_objects": 0, 2024-09-21T09:31:03.919 INFO:tasks.workunit.client.0.smithi022.stdout: "type": "replicated" 2024-09-21T09:31:03.920 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.920 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:03.920 INFO:tasks.workunit.client.0.smithi022.stdout: "application": [ 2024-09-21T09:31:03.920 INFO:tasks.workunit.client.0.smithi022.stdout: "cephfs" 2024-09-21T09:31:03.920 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:03.920 INFO:tasks.workunit.client.0.smithi022.stdout: "cache_mode": "none", 2024-09-21T09:31:03.920 INFO:tasks.workunit.client.0.smithi022.stdout: "erasure_code_profile": {}, 2024-09-21T09:31:03.920 INFO:tasks.workunit.client.0.smithi022.stdout: "min_size": 2, 2024-09-21T09:31:03.920 INFO:tasks.workunit.client.0.smithi022.stdout: "options": {}, 2024-09-21T09:31:03.920 INFO:tasks.workunit.client.0.smithi022.stdout: "pg_autoscale_mode": "off", 2024-09-21T09:31:03.920 INFO:tasks.workunit.client.0.smithi022.stdout: "pg_num": 32, 2024-09-21T09:31:03.920 INFO:tasks.workunit.client.0.smithi022.stdout: "pgp_num": 32, 2024-09-21T09:31:03.920 INFO:tasks.workunit.client.0.smithi022.stdout: "pool": 2, 2024-09-21T09:31:03.920 INFO:tasks.workunit.client.0.smithi022.stdout: "size": 3, 2024-09-21T09:31:03.921 INFO:tasks.workunit.client.0.smithi022.stdout: "stats": { 2024-09-21T09:31:03.921 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_raw": 727115966730, 2024-09-21T09:31:03.921 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes_used": 98304, 2024-09-21T09:31:03.921 INFO:tasks.workunit.client.0.smithi022.stdout: "compress_bytes_used": 0, 2024-09-21T09:31:03.921 INFO:tasks.workunit.client.0.smithi022.stdout: "compress_under_bytes": 0, 2024-09-21T09:31:03.921 INFO:tasks.workunit.client.0.smithi022.stdout: "data_bytes_used": 98304, 2024-09-21T09:31:03.921 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty": 0, 2024-09-21T09:31:03.921 INFO:tasks.workunit.client.0.smithi022.stdout: "max_avail": 242371985408, 2024-09-21T09:31:03.921 INFO:tasks.workunit.client.0.smithi022.stdout: "objects": 22, 2024-09-21T09:31:03.921 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_bytes_used": 0, 2024-09-21T09:31:03.921 INFO:tasks.workunit.client.0.smithi022.stdout: "percent_used": 1.3519712638299097e-07, 2024-09-21T09:31:03.921 INFO:tasks.workunit.client.0.smithi022.stdout: "quota_bytes": 0, 2024-09-21T09:31:03.921 INFO:tasks.workunit.client.0.smithi022.stdout: "quota_objects": 0, 2024-09-21T09:31:03.921 INFO:tasks.workunit.client.0.smithi022.stdout: "rd": 0, 2024-09-21T09:31:03.922 INFO:tasks.workunit.client.0.smithi022.stdout: "rd_bytes": 0, 2024-09-21T09:31:03.922 INFO:tasks.workunit.client.0.smithi022.stdout: "stored": 3410, 2024-09-21T09:31:03.922 INFO:tasks.workunit.client.0.smithi022.stdout: "stored_data": 3410, 2024-09-21T09:31:03.922 INFO:tasks.workunit.client.0.smithi022.stdout: "stored_omap": 0, 2024-09-21T09:31:03.922 INFO:tasks.workunit.client.0.smithi022.stdout: "stored_raw": 10230, 2024-09-21T09:31:03.922 INFO:tasks.workunit.client.0.smithi022.stdout: "wr": 51, 2024-09-21T09:31:03.922 INFO:tasks.workunit.client.0.smithi022.stdout: "wr_bytes": 19456 2024-09-21T09:31:03.922 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.922 INFO:tasks.workunit.client.0.smithi022.stdout: "target_max_bytes": 0, 2024-09-21T09:31:03.922 INFO:tasks.workunit.client.0.smithi022.stdout: "target_max_objects": 0, 2024-09-21T09:31:03.922 INFO:tasks.workunit.client.0.smithi022.stdout: "type": "replicated" 2024-09-21T09:31:03.922 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.922 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:03.923 INFO:tasks.workunit.client.0.smithi022.stdout: "application": [ 2024-09-21T09:31:03.923 INFO:tasks.workunit.client.0.smithi022.stdout: "cephfs" 2024-09-21T09:31:03.923 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:03.923 INFO:tasks.workunit.client.0.smithi022.stdout: "cache_mode": "none", 2024-09-21T09:31:03.923 INFO:tasks.workunit.client.0.smithi022.stdout: "erasure_code_profile": {}, 2024-09-21T09:31:03.923 INFO:tasks.workunit.client.0.smithi022.stdout: "min_size": 2, 2024-09-21T09:31:03.923 INFO:tasks.workunit.client.0.smithi022.stdout: "options": {}, 2024-09-21T09:31:03.923 INFO:tasks.workunit.client.0.smithi022.stdout: "pg_autoscale_mode": "off", 2024-09-21T09:31:03.923 INFO:tasks.workunit.client.0.smithi022.stdout: "pg_num": 64, 2024-09-21T09:31:03.923 INFO:tasks.workunit.client.0.smithi022.stdout: "pgp_num": 64, 2024-09-21T09:31:03.923 INFO:tasks.workunit.client.0.smithi022.stdout: "pool": 3, 2024-09-21T09:31:03.923 INFO:tasks.workunit.client.0.smithi022.stdout: "size": 3, 2024-09-21T09:31:03.923 INFO:tasks.workunit.client.0.smithi022.stdout: "stats": { 2024-09-21T09:31:03.923 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_raw": 727115966730, 2024-09-21T09:31:03.924 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes_used": 0, 2024-09-21T09:31:03.924 INFO:tasks.workunit.client.0.smithi022.stdout: "compress_bytes_used": 0, 2024-09-21T09:31:03.924 INFO:tasks.workunit.client.0.smithi022.stdout: "compress_under_bytes": 0, 2024-09-21T09:31:03.924 INFO:tasks.workunit.client.0.smithi022.stdout: "data_bytes_used": 0, 2024-09-21T09:31:03.924 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty": 0, 2024-09-21T09:31:03.924 INFO:tasks.workunit.client.0.smithi022.stdout: "max_avail": 242371985408, 2024-09-21T09:31:03.924 INFO:tasks.workunit.client.0.smithi022.stdout: "objects": 0, 2024-09-21T09:31:03.924 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_bytes_used": 0, 2024-09-21T09:31:03.924 INFO:tasks.workunit.client.0.smithi022.stdout: "percent_used": 0.0, 2024-09-21T09:31:03.924 INFO:tasks.workunit.client.0.smithi022.stdout: "quota_bytes": 0, 2024-09-21T09:31:03.924 INFO:tasks.workunit.client.0.smithi022.stdout: "quota_objects": 0, 2024-09-21T09:31:03.924 INFO:tasks.workunit.client.0.smithi022.stdout: "rd": 0, 2024-09-21T09:31:03.924 INFO:tasks.workunit.client.0.smithi022.stdout: "rd_bytes": 0, 2024-09-21T09:31:03.924 INFO:tasks.workunit.client.0.smithi022.stdout: "stored": 0, 2024-09-21T09:31:03.925 INFO:tasks.workunit.client.0.smithi022.stdout: "stored_data": 0, 2024-09-21T09:31:03.925 INFO:tasks.workunit.client.0.smithi022.stdout: "stored_omap": 0, 2024-09-21T09:31:03.925 INFO:tasks.workunit.client.0.smithi022.stdout: "stored_raw": 0, 2024-09-21T09:31:03.925 INFO:tasks.workunit.client.0.smithi022.stdout: "wr": 0, 2024-09-21T09:31:03.925 INFO:tasks.workunit.client.0.smithi022.stdout: "wr_bytes": 0 2024-09-21T09:31:03.925 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.925 INFO:tasks.workunit.client.0.smithi022.stdout: "target_max_bytes": 0, 2024-09-21T09:31:03.925 INFO:tasks.workunit.client.0.smithi022.stdout: "target_max_objects": 0, 2024-09-21T09:31:03.925 INFO:tasks.workunit.client.0.smithi022.stdout: "type": "replicated" 2024-09-21T09:31:03.925 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.925 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:03.925 INFO:tasks.workunit.client.0.smithi022.stdout: "application": [ 2024-09-21T09:31:03.925 INFO:tasks.workunit.client.0.smithi022.stdout: "rbd" 2024-09-21T09:31:03.925 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:03.926 INFO:tasks.workunit.client.0.smithi022.stdout: "cache_mode": "none", 2024-09-21T09:31:03.926 INFO:tasks.workunit.client.0.smithi022.stdout: "erasure_code_profile": {}, 2024-09-21T09:31:03.926 INFO:tasks.workunit.client.0.smithi022.stdout: "min_size": 2, 2024-09-21T09:31:03.926 INFO:tasks.workunit.client.0.smithi022.stdout: "options": {}, 2024-09-21T09:31:03.926 INFO:tasks.workunit.client.0.smithi022.stdout: "pg_autoscale_mode": "off", 2024-09-21T09:31:03.926 INFO:tasks.workunit.client.0.smithi022.stdout: "pg_num": 8, 2024-09-21T09:31:03.926 INFO:tasks.workunit.client.0.smithi022.stdout: "pgp_num": 8, 2024-09-21T09:31:03.926 INFO:tasks.workunit.client.0.smithi022.stdout: "pool": 4, 2024-09-21T09:31:03.926 INFO:tasks.workunit.client.0.smithi022.stdout: "size": 3, 2024-09-21T09:31:03.926 INFO:tasks.workunit.client.0.smithi022.stdout: "stats": { 2024-09-21T09:31:03.926 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_raw": 727115966730, 2024-09-21T09:31:03.926 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes_used": 12288, 2024-09-21T09:31:03.926 INFO:tasks.workunit.client.0.smithi022.stdout: "compress_bytes_used": 0, 2024-09-21T09:31:03.927 INFO:tasks.workunit.client.0.smithi022.stdout: "compress_under_bytes": 0, 2024-09-21T09:31:03.927 INFO:tasks.workunit.client.0.smithi022.stdout: "data_bytes_used": 12288, 2024-09-21T09:31:03.927 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty": 0, 2024-09-21T09:31:03.927 INFO:tasks.workunit.client.0.smithi022.stdout: "max_avail": 242371985408, 2024-09-21T09:31:03.927 INFO:tasks.workunit.client.0.smithi022.stdout: "objects": 2, 2024-09-21T09:31:03.927 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_bytes_used": 0, 2024-09-21T09:31:03.927 INFO:tasks.workunit.client.0.smithi022.stdout: "percent_used": 1.689964257423071e-08, 2024-09-21T09:31:03.927 INFO:tasks.workunit.client.0.smithi022.stdout: "quota_bytes": 0, 2024-09-21T09:31:03.927 INFO:tasks.workunit.client.0.smithi022.stdout: "quota_objects": 0, 2024-09-21T09:31:03.927 INFO:tasks.workunit.client.0.smithi022.stdout: "rd": 0, 2024-09-21T09:31:03.928 INFO:tasks.workunit.client.0.smithi022.stdout: "rd_bytes": 0, 2024-09-21T09:31:03.928 INFO:tasks.workunit.client.0.smithi022.stdout: "stored": 19, 2024-09-21T09:31:03.928 INFO:tasks.workunit.client.0.smithi022.stdout: "stored_data": 19, 2024-09-21T09:31:03.928 INFO:tasks.workunit.client.0.smithi022.stdout: "stored_omap": 0, 2024-09-21T09:31:03.928 INFO:tasks.workunit.client.0.smithi022.stdout: "stored_raw": 57, 2024-09-21T09:31:03.928 INFO:tasks.workunit.client.0.smithi022.stdout: "wr": 2, 2024-09-21T09:31:03.928 INFO:tasks.workunit.client.0.smithi022.stdout: "wr_bytes": 2048 2024-09-21T09:31:03.928 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.928 INFO:tasks.workunit.client.0.smithi022.stdout: "target_max_bytes": 0, 2024-09-21T09:31:03.928 INFO:tasks.workunit.client.0.smithi022.stdout: "target_max_objects": 0, 2024-09-21T09:31:03.928 INFO:tasks.workunit.client.0.smithi022.stdout: "type": "replicated" 2024-09-21T09:31:03.928 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:03.929 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:03.929 INFO:tasks.workunit.client.0.smithi022.stdout: "rbd": { 2024-09-21T09:31:03.929 INFO:tasks.workunit.client.0.smithi022.stdout: "mirroring_by_pool": [ 2024-09-21T09:31:03.929 INFO:tasks.workunit.client.0.smithi022.stdout: false 2024-09-21T09:31:03.929 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:03.929 INFO:tasks.workunit.client.0.smithi022.stdout: "num_images_by_pool": [ 2024-09-21T09:31:03.929 INFO:tasks.workunit.client.0.smithi022.stdout: 0 2024-09-21T09:31:03.929 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:03.929 INFO:tasks.workunit.client.0.smithi022.stdout: "num_pools": 1 2024-09-21T09:31:03.929 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.929 INFO:tasks.workunit.client.0.smithi022.stdout: "report_id": "77addb0d-e829-4757-884e-46deb5965591", 2024-09-21T09:31:03.929 INFO:tasks.workunit.client.0.smithi022.stdout: "report_timestamp": "2024-09-21T09:31:03.829133", 2024-09-21T09:31:03.929 INFO:tasks.workunit.client.0.smithi022.stdout: "report_version": 1, 2024-09-21T09:31:03.929 INFO:tasks.workunit.client.0.smithi022.stdout: "rook": { 2024-09-21T09:31:03.930 INFO:tasks.workunit.client.0.smithi022.stdout: "cluster": { 2024-09-21T09:31:03.930 INFO:tasks.workunit.client.0.smithi022.stdout: "external-mode": null, 2024-09-21T09:31:03.930 INFO:tasks.workunit.client.0.smithi022.stdout: "mon": { 2024-09-21T09:31:03.930 INFO:tasks.workunit.client.0.smithi022.stdout: "allow-multiple-per-node": null, 2024-09-21T09:31:03.930 INFO:tasks.workunit.client.0.smithi022.stdout: "count": null, 2024-09-21T09:31:03.930 INFO:tasks.workunit.client.0.smithi022.stdout: "max-id": null, 2024-09-21T09:31:03.930 INFO:tasks.workunit.client.0.smithi022.stdout: "pvc": { 2024-09-21T09:31:03.930 INFO:tasks.workunit.client.0.smithi022.stdout: "enabled": null 2024-09-21T09:31:03.930 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.930 INFO:tasks.workunit.client.0.smithi022.stdout: "stretch": { 2024-09-21T09:31:03.930 INFO:tasks.workunit.client.0.smithi022.stdout: "enabled": null 2024-09-21T09:31:03.930 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:03.930 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.931 INFO:tasks.workunit.client.0.smithi022.stdout: "network": { 2024-09-21T09:31:03.931 INFO:tasks.workunit.client.0.smithi022.stdout: "provider": null 2024-09-21T09:31:03.931 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.931 INFO:tasks.workunit.client.0.smithi022.stdout: "storage": { 2024-09-21T09:31:03.931 INFO:tasks.workunit.client.0.smithi022.stdout: "device-set": { 2024-09-21T09:31:03.931 INFO:tasks.workunit.client.0.smithi022.stdout: "count": { 2024-09-21T09:31:03.931 INFO:tasks.workunit.client.0.smithi022.stdout: "non-portable": null, 2024-09-21T09:31:03.931 INFO:tasks.workunit.client.0.smithi022.stdout: "portable": null, 2024-09-21T09:31:03.931 INFO:tasks.workunit.client.0.smithi022.stdout: "total": null 2024-09-21T09:31:03.931 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:03.931 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:03.931 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:03.931 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.931 INFO:tasks.workunit.client.0.smithi022.stdout: "csi": { 2024-09-21T09:31:03.931 INFO:tasks.workunit.client.0.smithi022.stdout: "version": null 2024-09-21T09:31:03.932 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.932 INFO:tasks.workunit.client.0.smithi022.stdout: "kubernetes": { 2024-09-21T09:31:03.932 INFO:tasks.workunit.client.0.smithi022.stdout: "version": null 2024-09-21T09:31:03.932 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.932 INFO:tasks.workunit.client.0.smithi022.stdout: "node": { 2024-09-21T09:31:03.932 INFO:tasks.workunit.client.0.smithi022.stdout: "count": { 2024-09-21T09:31:03.932 INFO:tasks.workunit.client.0.smithi022.stdout: "kubernetes-total": null, 2024-09-21T09:31:03.932 INFO:tasks.workunit.client.0.smithi022.stdout: "with-ceph-daemons": null, 2024-09-21T09:31:03.932 INFO:tasks.workunit.client.0.smithi022.stdout: "with-csi-cephfs-plugin": null, 2024-09-21T09:31:03.932 INFO:tasks.workunit.client.0.smithi022.stdout: "with-csi-nfs-plugin": null, 2024-09-21T09:31:03.932 INFO:tasks.workunit.client.0.smithi022.stdout: "with-csi-rbd-plugin": null 2024-09-21T09:31:03.932 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:03.932 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.933 INFO:tasks.workunit.client.0.smithi022.stdout: "usage": { 2024-09-21T09:31:03.933 INFO:tasks.workunit.client.0.smithi022.stdout: "storage-class": { 2024-09-21T09:31:03.933 INFO:tasks.workunit.client.0.smithi022.stdout: "count": { 2024-09-21T09:31:03.933 INFO:tasks.workunit.client.0.smithi022.stdout: "bucket": null, 2024-09-21T09:31:03.933 INFO:tasks.workunit.client.0.smithi022.stdout: "cephfs": null, 2024-09-21T09:31:03.933 INFO:tasks.workunit.client.0.smithi022.stdout: "nfs": null, 2024-09-21T09:31:03.933 INFO:tasks.workunit.client.0.smithi022.stdout: "rbd": null, 2024-09-21T09:31:03.933 INFO:tasks.workunit.client.0.smithi022.stdout: "total": null 2024-09-21T09:31:03.933 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:03.933 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:03.933 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.933 INFO:tasks.workunit.client.0.smithi022.stdout: "version": null 2024-09-21T09:31:03.933 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.933 INFO:tasks.workunit.client.0.smithi022.stdout: "services": {}, 2024-09-21T09:31:03.933 INFO:tasks.workunit.client.0.smithi022.stdout: "usage": { 2024-09-21T09:31:03.934 INFO:tasks.workunit.client.0.smithi022.stdout: "pg_num": 105, 2024-09-21T09:31:03.934 INFO:tasks.workunit.client.0.smithi022.stdout: "pools": 4, 2024-09-21T09:31:03.934 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_by_class": { 2024-09-21T09:31:03.934 INFO:tasks.workunit.client.0.smithi022.stdout: "ssd": { 2024-09-21T09:31:03.934 INFO:tasks.workunit.client.0.smithi022.stdout: "total_avail_bytes": 765516976128, 2024-09-21T09:31:03.934 INFO:tasks.workunit.client.0.smithi022.stdout: "total_bytes": 767960285184, 2024-09-21T09:31:03.934 INFO:tasks.workunit.client.0.smithi022.stdout: "total_used_bytes": 2443309056, 2024-09-21T09:31:03.934 INFO:tasks.workunit.client.0.smithi022.stdout: "total_used_raw_bytes": 2443309056, 2024-09-21T09:31:03.934 INFO:tasks.workunit.client.0.smithi022.stdout: "total_used_raw_ratio": 0.003181556472554803 2024-09-21T09:31:03.934 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:03.934 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:03.934 INFO:tasks.workunit.client.0.smithi022.stdout: "total_avail_bytes": 765516976128, 2024-09-21T09:31:03.934 INFO:tasks.workunit.client.0.smithi022.stdout: "total_bytes": 767960285184, 2024-09-21T09:31:03.934 INFO:tasks.workunit.client.0.smithi022.stdout: "total_used_bytes": 2443309056 2024-09-21T09:31:03.935 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:03.935 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:03.935 INFO:tasks.workunit.client.0.smithi022.stdout:} 2024-09-21T09:31:03.935 INFO:tasks.workunit.client.0.smithi022.stderr:++ ceph telemetry collection ls 2024-09-21T09:31:04.318 INFO:tasks.workunit.client.0.smithi022.stderr:+ COLLECTIONS='New collections are available: 2024-09-21T09:31:04.318 INFO:tasks.workunit.client.0.smithi022.stderr:['\''basic_base'\'', '\''basic_mds_metadata'\'', '\''basic_pool_options_bluestore'\'', '\''basic_pool_usage'\'', '\''basic_rook_v01'\'', '\''basic_usage_by_class'\'', '\''crash_base'\'', '\''device_base'\'', '\''ident_base'\'', '\''perf_memory_metrics'\'', '\''perf_perf'\''] 2024-09-21T09:31:04.318 INFO:tasks.workunit.client.0.smithi022.stderr:Run `ceph telemetry on` to opt-in to these collections. 2024-09-21T09:31:04.318 INFO:tasks.workunit.client.0.smithi022.stderr: 2024-09-21T09:31:04.319 INFO:tasks.workunit.client.0.smithi022.stderr:NAME STATUS DESC 2024-09-21T09:31:04.319 INFO:tasks.workunit.client.0.smithi022.stderr:basic_base NOT REPORTING: NOT OPTED-IN Basic information about the cluster (capacity, number and type of daemons, version, etc.) 2024-09-21T09:31:04.319 INFO:tasks.workunit.client.0.smithi022.stderr:basic_mds_metadata NOT REPORTING: NOT OPTED-IN MDS metadata 2024-09-21T09:31:04.319 INFO:tasks.workunit.client.0.smithi022.stderr:basic_pool_options_bluestore NOT REPORTING: NOT OPTED-IN Per-pool bluestore config options 2024-09-21T09:31:04.319 INFO:tasks.workunit.client.0.smithi022.stderr:basic_pool_usage NOT REPORTING: NOT OPTED-IN Default pool application and usage statistics 2024-09-21T09:31:04.319 INFO:tasks.workunit.client.0.smithi022.stderr:basic_rook_v01 NOT REPORTING: NOT OPTED-IN Basic Rook deployment data 2024-09-21T09:31:04.319 INFO:tasks.workunit.client.0.smithi022.stderr:basic_usage_by_class NOT REPORTING: NOT OPTED-IN Default device class usage statistics 2024-09-21T09:31:04.319 INFO:tasks.workunit.client.0.smithi022.stderr:crash_base NOT REPORTING: NOT OPTED-IN Information about daemon crashes (daemon type and version, backtrace, etc.) 2024-09-21T09:31:04.319 INFO:tasks.workunit.client.0.smithi022.stderr:device_base NOT REPORTING: NOT OPTED-IN Information about device health metrics 2024-09-21T09:31:04.319 INFO:tasks.workunit.client.0.smithi022.stderr:ident_base NOT REPORTING: NOT OPTED-IN User-provided identifying information about the cluster 2024-09-21T09:31:04.319 INFO:tasks.workunit.client.0.smithi022.stderr:perf_memory_metrics NOT REPORTING: NOT OPTED-IN, CHANNEL perf IS OFF Heap stats and mempools for mon and mds 2024-09-21T09:31:04.319 INFO:tasks.workunit.client.0.smithi022.stderr:perf_perf NOT REPORTING: NOT OPTED-IN, CHANNEL perf IS OFF Information about performance counters of the cluster ' 2024-09-21T09:31:04.319 INFO:tasks.workunit.client.0.smithi022.stderr:+ NEW_COLLECTIONS=("perf_perf" "basic_mds_metadata" "basic_pool_usage" "basic_rook_v01" "perf_memory_metrics" "basic_pool_options_bluestore") 2024-09-21T09:31:04.320 INFO:tasks.workunit.client.0.smithi022.stderr:+ for col in ${NEW_COLLECTIONS[@]} 2024-09-21T09:31:04.320 INFO:tasks.workunit.client.0.smithi022.stderr:+ [[ New collections are available: 2024-09-21T09:31:04.320 INFO:tasks.workunit.client.0.smithi022.stderr:['basic_base', 'basic_mds_metadata', 'basic_pool_options_bluestore', 'basic_pool_usage', 'basic_rook_v01', 'basic_usage_by_class', 'crash_base', 'device_base', 'ident_base', 'perf_memory_metrics', 'perf_perf'] 2024-09-21T09:31:04.320 INFO:tasks.workunit.client.0.smithi022.stderr:Run `ceph telemetry on` to opt-in to these collections. 2024-09-21T09:31:04.320 INFO:tasks.workunit.client.0.smithi022.stderr: 2024-09-21T09:31:04.320 INFO:tasks.workunit.client.0.smithi022.stderr:NAME STATUS DESC 2024-09-21T09:31:04.320 INFO:tasks.workunit.client.0.smithi022.stderr:basic_base NOT REPORTING: NOT OPTED-IN Basic information about the cluster (capacity, number and type of daemons, version, etc.) 2024-09-21T09:31:04.320 INFO:tasks.workunit.client.0.smithi022.stderr:basic_mds_metadata NOT REPORTING: NOT OPTED-IN MDS metadata 2024-09-21T09:31:04.320 INFO:tasks.workunit.client.0.smithi022.stderr:basic_pool_options_bluestore NOT REPORTING: NOT OPTED-IN Per-pool bluestore config options 2024-09-21T09:31:04.320 INFO:tasks.workunit.client.0.smithi022.stderr:basic_pool_usage NOT REPORTING: NOT OPTED-IN Default pool application and usage statistics 2024-09-21T09:31:04.320 INFO:tasks.workunit.client.0.smithi022.stderr:basic_rook_v01 NOT REPORTING: NOT OPTED-IN Basic Rook deployment data 2024-09-21T09:31:04.320 INFO:tasks.workunit.client.0.smithi022.stderr:basic_usage_by_class NOT REPORTING: NOT OPTED-IN Default device class usage statistics 2024-09-21T09:31:04.321 INFO:tasks.workunit.client.0.smithi022.stderr:crash_base NOT REPORTING: NOT OPTED-IN Information about daemon crashes (daemon type and version, backtrace, etc.) 2024-09-21T09:31:04.321 INFO:tasks.workunit.client.0.smithi022.stderr:device_base NOT REPORTING: NOT OPTED-IN Information about device health metrics 2024-09-21T09:31:04.321 INFO:tasks.workunit.client.0.smithi022.stderr:ident_base NOT REPORTING: NOT OPTED-IN User-provided identifying information about the cluster 2024-09-21T09:31:04.321 INFO:tasks.workunit.client.0.smithi022.stderr:perf_memory_metrics NOT REPORTING: NOT OPTED-IN, CHANNEL perf IS OFF Heap stats and mempools for mon and mds 2024-09-21T09:31:04.321 INFO:tasks.workunit.client.0.smithi022.stderr:perf_perf NOT REPORTING: NOT OPTED-IN, CHANNEL perf IS OFF Information about performance counters of the cluster == *perf_perf* ]] 2024-09-21T09:31:04.321 INFO:tasks.workunit.client.0.smithi022.stderr:+ for col in ${NEW_COLLECTIONS[@]} 2024-09-21T09:31:04.321 INFO:tasks.workunit.client.0.smithi022.stderr:+ [[ New collections are available: 2024-09-21T09:31:04.321 INFO:tasks.workunit.client.0.smithi022.stderr:['basic_base', 'basic_mds_metadata', 'basic_pool_options_bluestore', 'basic_pool_usage', 'basic_rook_v01', 'basic_usage_by_class', 'crash_base', 'device_base', 'ident_base', 'perf_memory_metrics', 'perf_perf'] 2024-09-21T09:31:04.321 INFO:tasks.workunit.client.0.smithi022.stderr:Run `ceph telemetry on` to opt-in to these collections. 2024-09-21T09:31:04.321 INFO:tasks.workunit.client.0.smithi022.stderr: 2024-09-21T09:31:04.321 INFO:tasks.workunit.client.0.smithi022.stderr:NAME STATUS DESC 2024-09-21T09:31:04.321 INFO:tasks.workunit.client.0.smithi022.stderr:basic_base NOT REPORTING: NOT OPTED-IN Basic information about the cluster (capacity, number and type of daemons, version, etc.) 2024-09-21T09:31:04.321 INFO:tasks.workunit.client.0.smithi022.stderr:basic_mds_metadata NOT REPORTING: NOT OPTED-IN MDS metadata 2024-09-21T09:31:04.322 INFO:tasks.workunit.client.0.smithi022.stderr:basic_pool_options_bluestore NOT REPORTING: NOT OPTED-IN Per-pool bluestore config options 2024-09-21T09:31:04.322 INFO:tasks.workunit.client.0.smithi022.stderr:basic_pool_usage NOT REPORTING: NOT OPTED-IN Default pool application and usage statistics 2024-09-21T09:31:04.322 INFO:tasks.workunit.client.0.smithi022.stderr:basic_rook_v01 NOT REPORTING: NOT OPTED-IN Basic Rook deployment data 2024-09-21T09:31:04.322 INFO:tasks.workunit.client.0.smithi022.stderr:basic_usage_by_class NOT REPORTING: NOT OPTED-IN Default device class usage statistics 2024-09-21T09:31:04.322 INFO:tasks.workunit.client.0.smithi022.stderr:crash_base NOT REPORTING: NOT OPTED-IN Information about daemon crashes (daemon type and version, backtrace, etc.) 2024-09-21T09:31:04.322 INFO:tasks.workunit.client.0.smithi022.stderr:device_base NOT REPORTING: NOT OPTED-IN Information about device health metrics 2024-09-21T09:31:04.322 INFO:tasks.workunit.client.0.smithi022.stderr:ident_base NOT REPORTING: NOT OPTED-IN User-provided identifying information about the cluster 2024-09-21T09:31:04.322 INFO:tasks.workunit.client.0.smithi022.stderr:perf_memory_metrics NOT REPORTING: NOT OPTED-IN, CHANNEL perf IS OFF Heap stats and mempools for mon and mds 2024-09-21T09:31:04.322 INFO:tasks.workunit.client.0.smithi022.stderr:perf_perf NOT REPORTING: NOT OPTED-IN, CHANNEL perf IS OFF Information about performance counters of the cluster == *basic_mds_metadata* ]] 2024-09-21T09:31:04.322 INFO:tasks.workunit.client.0.smithi022.stderr:+ for col in ${NEW_COLLECTIONS[@]} 2024-09-21T09:31:04.322 INFO:tasks.workunit.client.0.smithi022.stderr:+ [[ New collections are available: 2024-09-21T09:31:04.322 INFO:tasks.workunit.client.0.smithi022.stderr:['basic_base', 'basic_mds_metadata', 'basic_pool_options_bluestore', 'basic_pool_usage', 'basic_rook_v01', 'basic_usage_by_class', 'crash_base', 'device_base', 'ident_base', 'perf_memory_metrics', 'perf_perf'] 2024-09-21T09:31:04.322 INFO:tasks.workunit.client.0.smithi022.stderr:Run `ceph telemetry on` to opt-in to these collections. 2024-09-21T09:31:04.322 INFO:tasks.workunit.client.0.smithi022.stderr: 2024-09-21T09:31:04.323 INFO:tasks.workunit.client.0.smithi022.stderr:NAME STATUS DESC 2024-09-21T09:31:04.323 INFO:tasks.workunit.client.0.smithi022.stderr:basic_base NOT REPORTING: NOT OPTED-IN Basic information about the cluster (capacity, number and type of daemons, version, etc.) 2024-09-21T09:31:04.323 INFO:tasks.workunit.client.0.smithi022.stderr:basic_mds_metadata NOT REPORTING: NOT OPTED-IN MDS metadata 2024-09-21T09:31:04.323 INFO:tasks.workunit.client.0.smithi022.stderr:basic_pool_options_bluestore NOT REPORTING: NOT OPTED-IN Per-pool bluestore config options 2024-09-21T09:31:04.323 INFO:tasks.workunit.client.0.smithi022.stderr:basic_pool_usage NOT REPORTING: NOT OPTED-IN Default pool application and usage statistics 2024-09-21T09:31:04.323 INFO:tasks.workunit.client.0.smithi022.stderr:basic_rook_v01 NOT REPORTING: NOT OPTED-IN Basic Rook deployment data 2024-09-21T09:31:04.323 INFO:tasks.workunit.client.0.smithi022.stderr:basic_usage_by_class NOT REPORTING: NOT OPTED-IN Default device class usage statistics 2024-09-21T09:31:04.323 INFO:tasks.workunit.client.0.smithi022.stderr:crash_base NOT REPORTING: NOT OPTED-IN Information about daemon crashes (daemon type and version, backtrace, etc.) 2024-09-21T09:31:04.323 INFO:tasks.workunit.client.0.smithi022.stderr:device_base NOT REPORTING: NOT OPTED-IN Information about device health metrics 2024-09-21T09:31:04.323 INFO:tasks.workunit.client.0.smithi022.stderr:ident_base NOT REPORTING: NOT OPTED-IN User-provided identifying information about the cluster 2024-09-21T09:31:04.323 INFO:tasks.workunit.client.0.smithi022.stderr:perf_memory_metrics NOT REPORTING: NOT OPTED-IN, CHANNEL perf IS OFF Heap stats and mempools for mon and mds 2024-09-21T09:31:04.323 INFO:tasks.workunit.client.0.smithi022.stderr:perf_perf NOT REPORTING: NOT OPTED-IN, CHANNEL perf IS OFF Information about performance counters of the cluster == *basic_pool_usage* ]] 2024-09-21T09:31:04.323 INFO:tasks.workunit.client.0.smithi022.stderr:+ for col in ${NEW_COLLECTIONS[@]} 2024-09-21T09:31:04.324 INFO:tasks.workunit.client.0.smithi022.stderr:+ [[ New collections are available: 2024-09-21T09:31:04.324 INFO:tasks.workunit.client.0.smithi022.stderr:['basic_base', 'basic_mds_metadata', 'basic_pool_options_bluestore', 'basic_pool_usage', 'basic_rook_v01', 'basic_usage_by_class', 'crash_base', 'device_base', 'ident_base', 'perf_memory_metrics', 'perf_perf'] 2024-09-21T09:31:04.324 INFO:tasks.workunit.client.0.smithi022.stderr:Run `ceph telemetry on` to opt-in to these collections. 2024-09-21T09:31:04.324 INFO:tasks.workunit.client.0.smithi022.stderr: 2024-09-21T09:31:04.324 INFO:tasks.workunit.client.0.smithi022.stderr:NAME STATUS DESC 2024-09-21T09:31:04.324 INFO:tasks.workunit.client.0.smithi022.stderr:basic_base NOT REPORTING: NOT OPTED-IN Basic information about the cluster (capacity, number and type of daemons, version, etc.) 2024-09-21T09:31:04.324 INFO:tasks.workunit.client.0.smithi022.stderr:basic_mds_metadata NOT REPORTING: NOT OPTED-IN MDS metadata 2024-09-21T09:31:04.324 INFO:tasks.workunit.client.0.smithi022.stderr:basic_pool_options_bluestore NOT REPORTING: NOT OPTED-IN Per-pool bluestore config options 2024-09-21T09:31:04.324 INFO:tasks.workunit.client.0.smithi022.stderr:basic_pool_usage NOT REPORTING: NOT OPTED-IN Default pool application and usage statistics 2024-09-21T09:31:04.324 INFO:tasks.workunit.client.0.smithi022.stderr:basic_rook_v01 NOT REPORTING: NOT OPTED-IN Basic Rook deployment data 2024-09-21T09:31:04.324 INFO:tasks.workunit.client.0.smithi022.stderr:basic_usage_by_class NOT REPORTING: NOT OPTED-IN Default device class usage statistics 2024-09-21T09:31:04.325 INFO:tasks.workunit.client.0.smithi022.stderr:crash_base NOT REPORTING: NOT OPTED-IN Information about daemon crashes (daemon type and version, backtrace, etc.) 2024-09-21T09:31:04.325 INFO:tasks.workunit.client.0.smithi022.stderr:device_base NOT REPORTING: NOT OPTED-IN Information about device health metrics 2024-09-21T09:31:04.325 INFO:tasks.workunit.client.0.smithi022.stderr:ident_base NOT REPORTING: NOT OPTED-IN User-provided identifying information about the cluster 2024-09-21T09:31:04.325 INFO:tasks.workunit.client.0.smithi022.stderr:perf_memory_metrics NOT REPORTING: NOT OPTED-IN, CHANNEL perf IS OFF Heap stats and mempools for mon and mds 2024-09-21T09:31:04.325 INFO:tasks.workunit.client.0.smithi022.stderr:perf_perf NOT REPORTING: NOT OPTED-IN, CHANNEL perf IS OFF Information about performance counters of the cluster == *basic_rook_v01* ]] 2024-09-21T09:31:04.325 INFO:tasks.workunit.client.0.smithi022.stderr:+ for col in ${NEW_COLLECTIONS[@]} 2024-09-21T09:31:04.325 INFO:tasks.workunit.client.0.smithi022.stderr:+ [[ New collections are available: 2024-09-21T09:31:04.325 INFO:tasks.workunit.client.0.smithi022.stderr:['basic_base', 'basic_mds_metadata', 'basic_pool_options_bluestore', 'basic_pool_usage', 'basic_rook_v01', 'basic_usage_by_class', 'crash_base', 'device_base', 'ident_base', 'perf_memory_metrics', 'perf_perf'] 2024-09-21T09:31:04.325 INFO:tasks.workunit.client.0.smithi022.stderr:Run `ceph telemetry on` to opt-in to these collections. 2024-09-21T09:31:04.325 INFO:tasks.workunit.client.0.smithi022.stderr: 2024-09-21T09:31:04.325 INFO:tasks.workunit.client.0.smithi022.stderr:NAME STATUS DESC 2024-09-21T09:31:04.325 INFO:tasks.workunit.client.0.smithi022.stderr:basic_base NOT REPORTING: NOT OPTED-IN Basic information about the cluster (capacity, number and type of daemons, version, etc.) 2024-09-21T09:31:04.325 INFO:tasks.workunit.client.0.smithi022.stderr:basic_mds_metadata NOT REPORTING: NOT OPTED-IN MDS metadata 2024-09-21T09:31:04.325 INFO:tasks.workunit.client.0.smithi022.stderr:basic_pool_options_bluestore NOT REPORTING: NOT OPTED-IN Per-pool bluestore config options 2024-09-21T09:31:04.326 INFO:tasks.workunit.client.0.smithi022.stderr:basic_pool_usage NOT REPORTING: NOT OPTED-IN Default pool application and usage statistics 2024-09-21T09:31:04.326 INFO:tasks.workunit.client.0.smithi022.stderr:basic_rook_v01 NOT REPORTING: NOT OPTED-IN Basic Rook deployment data 2024-09-21T09:31:04.326 INFO:tasks.workunit.client.0.smithi022.stderr:basic_usage_by_class NOT REPORTING: NOT OPTED-IN Default device class usage statistics 2024-09-21T09:31:04.326 INFO:tasks.workunit.client.0.smithi022.stderr:crash_base NOT REPORTING: NOT OPTED-IN Information about daemon crashes (daemon type and version, backtrace, etc.) 2024-09-21T09:31:04.326 INFO:tasks.workunit.client.0.smithi022.stderr:device_base NOT REPORTING: NOT OPTED-IN Information about device health metrics 2024-09-21T09:31:04.326 INFO:tasks.workunit.client.0.smithi022.stderr:ident_base NOT REPORTING: NOT OPTED-IN User-provided identifying information about the cluster 2024-09-21T09:31:04.326 INFO:tasks.workunit.client.0.smithi022.stderr:perf_memory_metrics NOT REPORTING: NOT OPTED-IN, CHANNEL perf IS OFF Heap stats and mempools for mon and mds 2024-09-21T09:31:04.326 INFO:tasks.workunit.client.0.smithi022.stderr:perf_perf NOT REPORTING: NOT OPTED-IN, CHANNEL perf IS OFF Information about performance counters of the cluster == *perf_memory_metrics* ]] 2024-09-21T09:31:04.326 INFO:tasks.workunit.client.0.smithi022.stderr:+ for col in ${NEW_COLLECTIONS[@]} 2024-09-21T09:31:04.326 INFO:tasks.workunit.client.0.smithi022.stderr:+ [[ New collections are available: 2024-09-21T09:31:04.326 INFO:tasks.workunit.client.0.smithi022.stderr:['basic_base', 'basic_mds_metadata', 'basic_pool_options_bluestore', 'basic_pool_usage', 'basic_rook_v01', 'basic_usage_by_class', 'crash_base', 'device_base', 'ident_base', 'perf_memory_metrics', 'perf_perf'] 2024-09-21T09:31:04.326 INFO:tasks.workunit.client.0.smithi022.stderr:Run `ceph telemetry on` to opt-in to these collections. 2024-09-21T09:31:04.326 INFO:tasks.workunit.client.0.smithi022.stderr: 2024-09-21T09:31:04.326 INFO:tasks.workunit.client.0.smithi022.stderr:NAME STATUS DESC 2024-09-21T09:31:04.327 INFO:tasks.workunit.client.0.smithi022.stderr:basic_base NOT REPORTING: NOT OPTED-IN Basic information about the cluster (capacity, number and type of daemons, version, etc.) 2024-09-21T09:31:04.327 INFO:tasks.workunit.client.0.smithi022.stderr:basic_mds_metadata NOT REPORTING: NOT OPTED-IN MDS metadata 2024-09-21T09:31:04.327 INFO:tasks.workunit.client.0.smithi022.stderr:basic_pool_options_bluestore NOT REPORTING: NOT OPTED-IN Per-pool bluestore config options 2024-09-21T09:31:04.327 INFO:tasks.workunit.client.0.smithi022.stderr:basic_pool_usage NOT REPORTING: NOT OPTED-IN Default pool application and usage statistics 2024-09-21T09:31:04.327 INFO:tasks.workunit.client.0.smithi022.stderr:basic_rook_v01 NOT REPORTING: NOT OPTED-IN Basic Rook deployment data 2024-09-21T09:31:04.327 INFO:tasks.workunit.client.0.smithi022.stderr:basic_usage_by_class NOT REPORTING: NOT OPTED-IN Default device class usage statistics 2024-09-21T09:31:04.327 INFO:tasks.workunit.client.0.smithi022.stderr:crash_base NOT REPORTING: NOT OPTED-IN Information about daemon crashes (daemon type and version, backtrace, etc.) 2024-09-21T09:31:04.327 INFO:tasks.workunit.client.0.smithi022.stderr:device_base NOT REPORTING: NOT OPTED-IN Information about device health metrics 2024-09-21T09:31:04.327 INFO:tasks.workunit.client.0.smithi022.stderr:ident_base NOT REPORTING: NOT OPTED-IN User-provided identifying information about the cluster 2024-09-21T09:31:04.327 INFO:tasks.workunit.client.0.smithi022.stderr:perf_memory_metrics NOT REPORTING: NOT OPTED-IN, CHANNEL perf IS OFF Heap stats and mempools for mon and mds 2024-09-21T09:31:04.327 INFO:tasks.workunit.client.0.smithi022.stderr:perf_perf NOT REPORTING: NOT OPTED-IN, CHANNEL perf IS OFF Information about performance counters of the cluster == *basic_pool_options_bluestore* ]] 2024-09-21T09:31:04.327 INFO:tasks.workunit.client.0.smithi022.stderr:+ ceph telemetry on --license sharing-1-0 2024-09-21T09:31:04.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:04 smithi022 ceph-mon[27763]: from='client.14805 -' entity='client.admin' cmd=[{"prefix": "telemetry preview", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:31:04.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:04 smithi022 ceph-mon[27763]: pgmap v88: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:31:04.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:04 smithi022 ceph-mon[27763]: from='client.14805 -' entity='client.admin' cmd=[{"prefix": "telemetry preview", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:31:04.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:04 smithi022 ceph-mon[27763]: from='client.14817 -' entity='client.admin' cmd=[{"prefix": "telemetry preview-device", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:31:04.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:04 smithi022 ceph-mon[27763]: mgrmap e22: y(active, since 2m), standbys: x 2024-09-21T09:31:04.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:04 smithi022 ceph-mon[27763]: from='client.14817 -' entity='client.admin' cmd=[{"prefix": "telemetry preview-device", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:31:04.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:04 smithi022 ceph-mon[27763]: from='client.24728 -' entity='client.admin' cmd=[{"prefix": "telemetry preview-all", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:31:04.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:04 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:31:04.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:04 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-21T09:31:04.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:04 smithi022 ceph-mon[27763]: from='client.24728 -' entity='client.admin' cmd=[{"prefix": "telemetry preview-all", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:31:04.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:04 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:31:04.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:04 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-21T09:31:04.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:04 smithi022 ceph-mon[27763]: from='client.14829 -' entity='client.admin' cmd=[{"prefix": "telemetry collection ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:31:04.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:04 smithi022 ceph-mon[39525]: from='client.14805 -' entity='client.admin' cmd=[{"prefix": "telemetry preview", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:31:04.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:04 smithi022 ceph-mon[39525]: pgmap v88: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:31:04.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:04 smithi022 ceph-mon[39525]: from='client.14805 -' entity='client.admin' cmd=[{"prefix": "telemetry preview", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:31:04.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:04 smithi022 ceph-mon[39525]: from='client.14817 -' entity='client.admin' cmd=[{"prefix": "telemetry preview-device", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:31:04.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:04 smithi022 ceph-mon[39525]: mgrmap e22: y(active, since 2m), standbys: x 2024-09-21T09:31:04.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:04 smithi022 ceph-mon[39525]: from='client.14817 -' entity='client.admin' cmd=[{"prefix": "telemetry preview-device", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:31:04.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:04 smithi022 ceph-mon[39525]: from='client.24728 -' entity='client.admin' cmd=[{"prefix": "telemetry preview-all", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:31:04.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:04 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:31:04.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:04 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-21T09:31:04.560 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:04 smithi022 ceph-mon[39525]: from='client.24728 -' entity='client.admin' cmd=[{"prefix": "telemetry preview-all", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:31:04.560 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:04 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:31:04.560 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:04 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-21T09:31:04.560 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:04 smithi022 ceph-mon[39525]: from='client.14829 -' entity='client.admin' cmd=[{"prefix": "telemetry collection ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:31:04.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:04 smithi064 ceph-mon[31124]: from='client.14805 -' entity='client.admin' cmd=[{"prefix": "telemetry preview", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:31:04.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:04 smithi064 ceph-mon[31124]: pgmap v88: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:31:04.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:04 smithi064 ceph-mon[31124]: from='client.14805 -' entity='client.admin' cmd=[{"prefix": "telemetry preview", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:31:04.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:04 smithi064 ceph-mon[31124]: from='client.14817 -' entity='client.admin' cmd=[{"prefix": "telemetry preview-device", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:31:04.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:04 smithi064 ceph-mon[31124]: mgrmap e22: y(active, since 2m), standbys: x 2024-09-21T09:31:04.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:04 smithi064 ceph-mon[31124]: from='client.14817 -' entity='client.admin' cmd=[{"prefix": "telemetry preview-device", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:31:04.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:04 smithi064 ceph-mon[31124]: from='client.24728 -' entity='client.admin' cmd=[{"prefix": "telemetry preview-all", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:31:04.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:04 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:31:04.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:04 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-21T09:31:04.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:04 smithi064 ceph-mon[31124]: from='client.24728 -' entity='client.admin' cmd=[{"prefix": "telemetry preview-all", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:31:04.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:04 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:31:04.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:04 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-21T09:31:04.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:04 smithi064 ceph-mon[31124]: from='client.14829 -' entity='client.admin' cmd=[{"prefix": "telemetry collection ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:31:04.806 INFO:tasks.workunit.client.0.smithi022.stdout:Telemetry is on. 2024-09-21T09:31:04.806 INFO:tasks.workunit.client.0.smithi022.stdout:Some channels are disabled, please enable with: 2024-09-21T09:31:04.806 INFO:tasks.workunit.client.0.smithi022.stdout:`ceph telemetry enable channel perf` 2024-09-21T09:31:04.817 INFO:tasks.workunit.client.0.smithi022.stderr:+ ceph telemetry enable channel perf 2024-09-21T09:31:05.057 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:31:04 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: [21/Sep/2024:09:31:04] ENGINE Bus STOPPING 2024-09-21T09:31:05.271 INFO:tasks.workunit.client.0.smithi022.stdout:channel_perf is enabled 2024-09-21T09:31:05.282 INFO:tasks.workunit.client.0.smithi022.stderr:++ ceph config get mgr mgr/telemetry/last_opt_revision 2024-09-21T09:31:05.558 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:31:05 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: [21/Sep/2024:09:31:05] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-21T09:31:05.558 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:31:05 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: [21/Sep/2024:09:31:05] ENGINE Bus STOPPED 2024-09-21T09:31:05.558 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:31:05 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: [21/Sep/2024:09:31:05] ENGINE Bus STARTING 2024-09-21T09:31:05.558 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:31:05 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: [21/Sep/2024:09:31:05] ENGINE Serving on http://:::9283 2024-09-21T09:31:05.558 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:31:05 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: [21/Sep/2024:09:31:05] ENGINE Bus STARTED 2024-09-21T09:31:05.558 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:31:05 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: [21/Sep/2024:09:31:05] ENGINE Bus STOPPING 2024-09-21T09:31:05.644 INFO:tasks.workunit.client.0.smithi022.stderr:+ LAST_OPT_REVISION=1 2024-09-21T09:31:05.644 INFO:tasks.workunit.client.0.smithi022.stderr:+ '[' 1 -ne 1 ']' 2024-09-21T09:31:05.644 INFO:tasks.workunit.client.0.smithi022.stderr:+ ceph telemetry show 2024-09-21T09:31:05.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:05 smithi064 ceph-mon[31124]: from='client.14829 -' entity='client.admin' cmd=[{"prefix": "telemetry collection ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:31:05.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:05 smithi064 ceph-mon[31124]: from='client.24740 -' entity='client.admin' cmd=[{"prefix": "telemetry on", "license": "sharing-1-0", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:31:05.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:05 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:31:05.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:05 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:31:05.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:05 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:31:05.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:05 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:31:05.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:05 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:31:05.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:05 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:05.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:05 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:31:05.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:05 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-21T09:31:05.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:05 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:31:05.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:05 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:31:05.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:05 smithi064 ceph-mon[31124]: from='client.24740 -' entity='client.admin' cmd=[{"prefix": "telemetry on", "license": "sharing-1-0", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:31:05.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:05 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:31:05.943 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:05 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:31:05.943 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:05 smithi064 ceph-mon[31124]: pgmap v89: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:31:05.943 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:05 smithi064 ceph-mon[31124]: from='client.14841 -' entity='client.admin' cmd=[{"prefix": "telemetry enable channel", "channels": ["perf"], "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:31:05.943 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:05 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:31:05.943 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:05 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:31:05.943 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:05 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:05.943 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:05 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:31:05.943 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:05 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:31:05.943 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:05 smithi064 ceph-mon[31124]: from='client.? 172.21.15.22:0/1027081936' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/telemetry/last_opt_revision"}]: dispatch 2024-09-21T09:31:06.058 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:31:05 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: [21/Sep/2024:09:31:05] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-21T09:31:06.058 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:31:05 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: [21/Sep/2024:09:31:05] ENGINE Bus STOPPED 2024-09-21T09:31:06.058 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:31:05 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: [21/Sep/2024:09:31:05] ENGINE Bus STARTING 2024-09-21T09:31:06.058 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:31:05 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: [21/Sep/2024:09:31:05] ENGINE Serving on http://:::9283 2024-09-21T09:31:06.058 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:31:05 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: [21/Sep/2024:09:31:05] ENGINE Bus STARTED 2024-09-21T09:31:06.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:05 smithi022 ceph-mon[27763]: from='client.14829 -' entity='client.admin' cmd=[{"prefix": "telemetry collection ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:31:06.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:05 smithi022 ceph-mon[27763]: from='client.24740 -' entity='client.admin' cmd=[{"prefix": "telemetry on", "license": "sharing-1-0", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:31:06.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:05 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:31:06.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:05 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:31:06.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:05 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:31:06.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:05 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:31:06.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:05 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:31:06.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:05 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:06.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:05 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:31:06.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:05 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-21T09:31:06.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:05 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:31:06.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:05 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:31:06.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:05 smithi022 ceph-mon[27763]: from='client.24740 -' entity='client.admin' cmd=[{"prefix": "telemetry on", "license": "sharing-1-0", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:31:06.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:05 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:31:06.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:05 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:31:06.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:05 smithi022 ceph-mon[27763]: pgmap v89: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:31:06.060 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:05 smithi022 ceph-mon[27763]: from='client.14841 -' entity='client.admin' cmd=[{"prefix": "telemetry enable channel", "channels": ["perf"], "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:31:06.060 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:05 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:31:06.060 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:05 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:31:06.060 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:05 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:06.060 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:05 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:31:06.060 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:05 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:31:06.060 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:05 smithi022 ceph-mon[27763]: from='client.? 172.21.15.22:0/1027081936' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/telemetry/last_opt_revision"}]: dispatch 2024-09-21T09:31:06.060 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:05 smithi022 ceph-mon[39525]: from='client.14829 -' entity='client.admin' cmd=[{"prefix": "telemetry collection ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:31:06.060 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:05 smithi022 ceph-mon[39525]: from='client.24740 -' entity='client.admin' cmd=[{"prefix": "telemetry on", "license": "sharing-1-0", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:31:06.060 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:05 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:31:06.060 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:05 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:31:06.060 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:05 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:31:06.061 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:05 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:31:06.061 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:05 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:31:06.061 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:05 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:06.061 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:05 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:31:06.061 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:05 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-21T09:31:06.061 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:05 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:31:06.061 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:05 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:31:06.061 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:05 smithi022 ceph-mon[39525]: from='client.24740 -' entity='client.admin' cmd=[{"prefix": "telemetry on", "license": "sharing-1-0", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:31:06.061 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:05 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:31:06.061 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:05 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:31:06.061 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:05 smithi022 ceph-mon[39525]: pgmap v89: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:31:06.061 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:05 smithi022 ceph-mon[39525]: from='client.14841 -' entity='client.admin' cmd=[{"prefix": "telemetry enable channel", "channels": ["perf"], "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:31:06.061 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:05 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:31:06.061 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:05 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:31:06.061 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:05 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:06.062 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:05 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:31:06.062 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:05 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:31:06.062 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:05 smithi022 ceph-mon[39525]: from='client.? 172.21.15.22:0/1027081936' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/telemetry/last_opt_revision"}]: dispatch 2024-09-21T09:31:06.307 INFO:tasks.workunit.client.0.smithi022.stdout:{ 2024-09-21T09:31:06.307 INFO:tasks.workunit.client.0.smithi022.stdout: "balancer": { 2024-09-21T09:31:06.308 INFO:tasks.workunit.client.0.smithi022.stdout: "active": true, 2024-09-21T09:31:06.308 INFO:tasks.workunit.client.0.smithi022.stdout: "mode": "" 2024-09-21T09:31:06.308 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.308 INFO:tasks.workunit.client.0.smithi022.stdout: "channels": [ 2024-09-21T09:31:06.308 INFO:tasks.workunit.client.0.smithi022.stdout: "basic", 2024-09-21T09:31:06.308 INFO:tasks.workunit.client.0.smithi022.stdout: "crash", 2024-09-21T09:31:06.308 INFO:tasks.workunit.client.0.smithi022.stdout: "device", 2024-09-21T09:31:06.308 INFO:tasks.workunit.client.0.smithi022.stdout: "ident", 2024-09-21T09:31:06.308 INFO:tasks.workunit.client.0.smithi022.stdout: "perf" 2024-09-21T09:31:06.308 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:06.308 INFO:tasks.workunit.client.0.smithi022.stdout: "channels_available": [ 2024-09-21T09:31:06.308 INFO:tasks.workunit.client.0.smithi022.stdout: "basic", 2024-09-21T09:31:06.309 INFO:tasks.workunit.client.0.smithi022.stdout: "ident", 2024-09-21T09:31:06.309 INFO:tasks.workunit.client.0.smithi022.stdout: "crash", 2024-09-21T09:31:06.309 INFO:tasks.workunit.client.0.smithi022.stdout: "device", 2024-09-21T09:31:06.309 INFO:tasks.workunit.client.0.smithi022.stdout: "perf" 2024-09-21T09:31:06.309 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:06.309 INFO:tasks.workunit.client.0.smithi022.stdout: "collections_available": [ 2024-09-21T09:31:06.309 INFO:tasks.workunit.client.0.smithi022.stdout: "basic_base", 2024-09-21T09:31:06.309 INFO:tasks.workunit.client.0.smithi022.stdout: "device_base", 2024-09-21T09:31:06.309 INFO:tasks.workunit.client.0.smithi022.stdout: "crash_base", 2024-09-21T09:31:06.309 INFO:tasks.workunit.client.0.smithi022.stdout: "ident_base", 2024-09-21T09:31:06.309 INFO:tasks.workunit.client.0.smithi022.stdout: "perf_perf", 2024-09-21T09:31:06.309 INFO:tasks.workunit.client.0.smithi022.stdout: "basic_mds_metadata", 2024-09-21T09:31:06.309 INFO:tasks.workunit.client.0.smithi022.stdout: "basic_pool_usage", 2024-09-21T09:31:06.310 INFO:tasks.workunit.client.0.smithi022.stdout: "basic_usage_by_class", 2024-09-21T09:31:06.310 INFO:tasks.workunit.client.0.smithi022.stdout: "basic_rook_v01", 2024-09-21T09:31:06.310 INFO:tasks.workunit.client.0.smithi022.stdout: "perf_memory_metrics", 2024-09-21T09:31:06.310 INFO:tasks.workunit.client.0.smithi022.stdout: "basic_pool_options_bluestore" 2024-09-21T09:31:06.310 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:06.310 INFO:tasks.workunit.client.0.smithi022.stdout: "collections_opted_in": [ 2024-09-21T09:31:06.310 INFO:tasks.workunit.client.0.smithi022.stdout: "basic_base", 2024-09-21T09:31:06.310 INFO:tasks.workunit.client.0.smithi022.stdout: "device_base", 2024-09-21T09:31:06.310 INFO:tasks.workunit.client.0.smithi022.stdout: "crash_base", 2024-09-21T09:31:06.310 INFO:tasks.workunit.client.0.smithi022.stdout: "ident_base", 2024-09-21T09:31:06.310 INFO:tasks.workunit.client.0.smithi022.stdout: "perf_perf", 2024-09-21T09:31:06.310 INFO:tasks.workunit.client.0.smithi022.stdout: "basic_mds_metadata", 2024-09-21T09:31:06.310 INFO:tasks.workunit.client.0.smithi022.stdout: "basic_pool_usage", 2024-09-21T09:31:06.311 INFO:tasks.workunit.client.0.smithi022.stdout: "basic_usage_by_class", 2024-09-21T09:31:06.311 INFO:tasks.workunit.client.0.smithi022.stdout: "basic_rook_v01", 2024-09-21T09:31:06.311 INFO:tasks.workunit.client.0.smithi022.stdout: "perf_memory_metrics", 2024-09-21T09:31:06.311 INFO:tasks.workunit.client.0.smithi022.stdout: "basic_pool_options_bluestore" 2024-09-21T09:31:06.311 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:06.311 INFO:tasks.workunit.client.0.smithi022.stdout: "config": { 2024-09-21T09:31:06.311 INFO:tasks.workunit.client.0.smithi022.stdout: "active_changed": [ 2024-09-21T09:31:06.311 INFO:tasks.workunit.client.0.smithi022.stdout: "auth_allow_insecure_global_id_reclaim", 2024-09-21T09:31:06.311 INFO:tasks.workunit.client.0.smithi022.stdout: "auth_debug", 2024-09-21T09:31:06.311 INFO:tasks.workunit.client.0.smithi022.stdout: "auth_mon_ticket_ttl", 2024-09-21T09:31:06.311 INFO:tasks.workunit.client.0.smithi022.stdout: "auth_service_ticket_ttl", 2024-09-21T09:31:06.311 INFO:tasks.workunit.client.0.smithi022.stdout: "bdev_debug_aio", 2024-09-21T09:31:06.311 INFO:tasks.workunit.client.0.smithi022.stdout: "client_check_pool_perm", 2024-09-21T09:31:06.312 INFO:tasks.workunit.client.0.smithi022.stdout: "client_mount_gid", 2024-09-21T09:31:06.312 INFO:tasks.workunit.client.0.smithi022.stdout: "client_mount_uid", 2024-09-21T09:31:06.312 INFO:tasks.workunit.client.0.smithi022.stdout: "client_quota", 2024-09-21T09:31:06.312 INFO:tasks.workunit.client.0.smithi022.stdout: "container_image", 2024-09-21T09:31:06.312 INFO:tasks.workunit.client.0.smithi022.stdout: "daemonize", 2024-09-21T09:31:06.312 INFO:tasks.workunit.client.0.smithi022.stdout: "debug_asserts_on_shutdown", 2024-09-21T09:31:06.312 INFO:tasks.workunit.client.0.smithi022.stdout: "debug_mgr", 2024-09-21T09:31:06.312 INFO:tasks.workunit.client.0.smithi022.stdout: "debug_mon", 2024-09-21T09:31:06.312 INFO:tasks.workunit.client.0.smithi022.stdout: "debug_ms", 2024-09-21T09:31:06.312 INFO:tasks.workunit.client.0.smithi022.stdout: "debug_osd", 2024-09-21T09:31:06.312 INFO:tasks.workunit.client.0.smithi022.stdout: "debug_paxos", 2024-09-21T09:31:06.312 INFO:tasks.workunit.client.0.smithi022.stdout: "keyring", 2024-09-21T09:31:06.313 INFO:tasks.workunit.client.0.smithi022.stdout: "leveldb_block_size", 2024-09-21T09:31:06.313 INFO:tasks.workunit.client.0.smithi022.stdout: "leveldb_cache_size", 2024-09-21T09:31:06.313 INFO:tasks.workunit.client.0.smithi022.stdout: "leveldb_compression", 2024-09-21T09:31:06.313 INFO:tasks.workunit.client.0.smithi022.stdout: "leveldb_log", 2024-09-21T09:31:06.313 INFO:tasks.workunit.client.0.smithi022.stdout: "leveldb_write_buffer_size", 2024-09-21T09:31:06.313 INFO:tasks.workunit.client.0.smithi022.stdout: "log_to_file", 2024-09-21T09:31:06.313 INFO:tasks.workunit.client.0.smithi022.stdout: "log_to_journald", 2024-09-21T09:31:06.313 INFO:tasks.workunit.client.0.smithi022.stdout: "log_to_stderr", 2024-09-21T09:31:06.313 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_allow_pool_delete", 2024-09-21T09:31:06.313 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_clock_drift_allowed", 2024-09-21T09:31:06.313 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_cluster_log_to_file", 2024-09-21T09:31:06.313 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_cluster_log_to_journald", 2024-09-21T09:31:06.313 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_cluster_log_to_stderr", 2024-09-21T09:31:06.313 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_data_avail_warn", 2024-09-21T09:31:06.313 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_election_default_strategy", 2024-09-21T09:31:06.314 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_host", 2024-09-21T09:31:06.314 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_max_pg_per_osd", 2024-09-21T09:31:06.314 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_mgr_mkfs_grace", 2024-09-21T09:31:06.314 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_osd_prime_pg_temp", 2024-09-21T09:31:06.314 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_osd_reporter_subtree_level", 2024-09-21T09:31:06.314 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_pg_warn_max_object_skew", 2024-09-21T09:31:06.314 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_reweight_min_bytes_per_osd", 2024-09-21T09:31:06.314 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_reweight_min_pgs_per_osd", 2024-09-21T09:31:06.314 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_warn_on_crush_straw_calc_version_zero", 2024-09-21T09:31:06.314 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_warn_on_insecure_global_id_reclaim", 2024-09-21T09:31:06.314 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_warn_on_insecure_global_id_reclaim_allowed", 2024-09-21T09:31:06.314 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_warn_on_legacy_crush_tunables", 2024-09-21T09:31:06.314 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_warn_on_osd_down_out_interval_zero", 2024-09-21T09:31:06.314 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_warn_on_pool_pg_num_not_power_of_two", 2024-09-21T09:31:06.315 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_warn_on_too_few_osds", 2024-09-21T09:31:06.315 INFO:tasks.workunit.client.0.smithi022.stdout: "ms_die_on_bug", 2024-09-21T09:31:06.315 INFO:tasks.workunit.client.0.smithi022.stdout: "ms_die_on_old_message", 2024-09-21T09:31:06.315 INFO:tasks.workunit.client.0.smithi022.stdout: "no_config_file", 2024-09-21T09:31:06.315 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_class_default_list", 2024-09-21T09:31:06.315 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_class_load_list", 2024-09-21T09:31:06.315 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_debug_misdirected_ops", 2024-09-21T09:31:06.315 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_debug_op_order", 2024-09-21T09:31:06.315 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_debug_pg_log_writeout", 2024-09-21T09:31:06.315 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_debug_shutdown", 2024-09-21T09:31:06.315 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_debug_verify_cached_snaps", 2024-09-21T09:31:06.315 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_debug_verify_missing_on_start", 2024-09-21T09:31:06.315 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_debug_verify_stray_on_activate", 2024-09-21T09:31:06.316 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_deep_scrub_update_digest_min_age", 2024-09-21T09:31:06.316 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_mclock_profile", 2024-09-21T09:31:06.316 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_mclock_scheduler_background_best_effort_lim", 2024-09-21T09:31:06.316 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_mclock_scheduler_background_best_effort_res", 2024-09-21T09:31:06.316 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_mclock_scheduler_background_best_effort_wgt", 2024-09-21T09:31:06.316 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_mclock_scheduler_background_recovery_lim", 2024-09-21T09:31:06.316 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_mclock_scheduler_background_recovery_res", 2024-09-21T09:31:06.316 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_mclock_scheduler_background_recovery_wgt", 2024-09-21T09:31:06.316 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_mclock_scheduler_client_lim", 2024-09-21T09:31:06.316 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_mclock_scheduler_client_res", 2024-09-21T09:31:06.316 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_mclock_scheduler_client_wgt", 2024-09-21T09:31:06.316 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_memory_target", 2024-09-21T09:31:06.316 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_memory_target_autotune", 2024-09-21T09:31:06.316 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_op_queue", 2024-09-21T09:31:06.317 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_op_queue_cut_off", 2024-09-21T09:31:06.317 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_pool_default_erasure_code_profile", 2024-09-21T09:31:06.317 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_pool_default_pg_autoscale_mode", 2024-09-21T09:31:06.317 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_recover_clone_overlap", 2024-09-21T09:31:06.317 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_recovery_max_chunk", 2024-09-21T09:31:06.317 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_scrub_load_threshold", 2024-09-21T09:31:06.317 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_scrub_max_interval", 2024-09-21T09:31:06.317 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_shutdown_pgref_assert", 2024-09-21T09:31:06.317 INFO:tasks.workunit.client.0.smithi022.stdout: "public_addr", 2024-09-21T09:31:06.317 INFO:tasks.workunit.client.0.smithi022.stdout: "public_addrv", 2024-09-21T09:31:06.317 INFO:tasks.workunit.client.0.smithi022.stdout: "public_network", 2024-09-21T09:31:06.317 INFO:tasks.workunit.client.0.smithi022.stdout: "rbd_default_features", 2024-09-21T09:31:06.317 INFO:tasks.workunit.client.0.smithi022.stdout: "rbd_qos_exclude_ops", 2024-09-21T09:31:06.317 INFO:tasks.workunit.client.0.smithi022.stdout: "setgroup", 2024-09-21T09:31:06.318 INFO:tasks.workunit.client.0.smithi022.stdout: "setuser" 2024-09-21T09:31:06.318 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:06.318 INFO:tasks.workunit.client.0.smithi022.stdout: "cluster_changed": [ 2024-09-21T09:31:06.318 INFO:tasks.workunit.client.0.smithi022.stdout: "auth_allow_insecure_global_id_reclaim", 2024-09-21T09:31:06.318 INFO:tasks.workunit.client.0.smithi022.stdout: "auth_debug", 2024-09-21T09:31:06.318 INFO:tasks.workunit.client.0.smithi022.stdout: "auth_mon_ticket_ttl", 2024-09-21T09:31:06.318 INFO:tasks.workunit.client.0.smithi022.stdout: "auth_service_ticket_ttl", 2024-09-21T09:31:06.318 INFO:tasks.workunit.client.0.smithi022.stdout: "bdev_debug_aio", 2024-09-21T09:31:06.318 INFO:tasks.workunit.client.0.smithi022.stdout: "container_image", 2024-09-21T09:31:06.318 INFO:tasks.workunit.client.0.smithi022.stdout: "debug_asserts_on_shutdown", 2024-09-21T09:31:06.318 INFO:tasks.workunit.client.0.smithi022.stdout: "debug_mgr", 2024-09-21T09:31:06.318 INFO:tasks.workunit.client.0.smithi022.stdout: "debug_mon", 2024-09-21T09:31:06.318 INFO:tasks.workunit.client.0.smithi022.stdout: "debug_ms", 2024-09-21T09:31:06.319 INFO:tasks.workunit.client.0.smithi022.stdout: "debug_osd", 2024-09-21T09:31:06.319 INFO:tasks.workunit.client.0.smithi022.stdout: "debug_paxos", 2024-09-21T09:31:06.319 INFO:tasks.workunit.client.0.smithi022.stdout: "log_to_file", 2024-09-21T09:31:06.319 INFO:tasks.workunit.client.0.smithi022.stdout: "log_to_journald", 2024-09-21T09:31:06.319 INFO:tasks.workunit.client.0.smithi022.stdout: "log_to_stderr", 2024-09-21T09:31:06.319 INFO:tasks.workunit.client.0.smithi022.stdout: "mds_join_fs", 2024-09-21T09:31:06.319 INFO:tasks.workunit.client.0.smithi022.stdout: "mgr/cephadm/allow_ptrace", 2024-09-21T09:31:06.319 INFO:tasks.workunit.client.0.smithi022.stdout: "mgr/cephadm/container_init", 2024-09-21T09:31:06.319 INFO:tasks.workunit.client.0.smithi022.stdout: "mgr/cephadm/migration_current", 2024-09-21T09:31:06.319 INFO:tasks.workunit.client.0.smithi022.stdout: "mgr/cephadm/use_repo_digest", 2024-09-21T09:31:06.319 INFO:tasks.workunit.client.0.smithi022.stdout: "mgr/dashboard/ALERTMANAGER_API_HOST", 2024-09-21T09:31:06.319 INFO:tasks.workunit.client.0.smithi022.stdout: "mgr/dashboard/GRAFANA_API_SSL_VERIFY", 2024-09-21T09:31:06.319 INFO:tasks.workunit.client.0.smithi022.stdout: "mgr/dashboard/GRAFANA_API_URL", 2024-09-21T09:31:06.319 INFO:tasks.workunit.client.0.smithi022.stdout: "mgr/dashboard/PROMETHEUS_API_HOST", 2024-09-21T09:31:06.319 INFO:tasks.workunit.client.0.smithi022.stdout: "mgr/dashboard/ssl_server_port", 2024-09-21T09:31:06.320 INFO:tasks.workunit.client.0.smithi022.stdout: "mgr/orchestrator/orchestrator", 2024-09-21T09:31:06.320 INFO:tasks.workunit.client.0.smithi022.stdout: "mgr/telemetry/channel_ident", 2024-09-21T09:31:06.320 INFO:tasks.workunit.client.0.smithi022.stdout: "mgr/telemetry/channel_perf", 2024-09-21T09:31:06.320 INFO:tasks.workunit.client.0.smithi022.stdout: "mgr/telemetry/description", 2024-09-21T09:31:06.320 INFO:tasks.workunit.client.0.smithi022.stdout: "mgr/telemetry/enabled", 2024-09-21T09:31:06.320 INFO:tasks.workunit.client.0.smithi022.stdout: "mgr/telemetry/organization", 2024-09-21T09:31:06.320 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_allow_pool_delete", 2024-09-21T09:31:06.320 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_clock_drift_allowed", 2024-09-21T09:31:06.320 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_cluster_log_to_file", 2024-09-21T09:31:06.320 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_data_avail_warn", 2024-09-21T09:31:06.320 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_election_default_strategy", 2024-09-21T09:31:06.320 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_max_pg_per_osd", 2024-09-21T09:31:06.320 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_mgr_mkfs_grace", 2024-09-21T09:31:06.320 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_osd_prime_pg_temp", 2024-09-21T09:31:06.320 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_osd_reporter_subtree_level", 2024-09-21T09:31:06.321 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_pg_warn_max_object_skew", 2024-09-21T09:31:06.321 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_reweight_min_bytes_per_osd", 2024-09-21T09:31:06.321 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_reweight_min_pgs_per_osd", 2024-09-21T09:31:06.321 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_warn_on_crush_straw_calc_version_zero", 2024-09-21T09:31:06.321 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_warn_on_insecure_global_id_reclaim", 2024-09-21T09:31:06.321 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_warn_on_insecure_global_id_reclaim_allowed", 2024-09-21T09:31:06.321 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_warn_on_legacy_crush_tunables", 2024-09-21T09:31:06.321 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_warn_on_osd_down_out_interval_zero", 2024-09-21T09:31:06.321 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_warn_on_pool_pg_num_not_power_of_two", 2024-09-21T09:31:06.321 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_warn_on_too_few_osds", 2024-09-21T09:31:06.321 INFO:tasks.workunit.client.0.smithi022.stdout: "ms_die_on_bug", 2024-09-21T09:31:06.321 INFO:tasks.workunit.client.0.smithi022.stdout: "ms_die_on_old_message", 2024-09-21T09:31:06.321 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_class_default_list", 2024-09-21T09:31:06.321 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_class_load_list", 2024-09-21T09:31:06.321 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_debug_misdirected_ops", 2024-09-21T09:31:06.322 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_debug_op_order", 2024-09-21T09:31:06.322 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_debug_pg_log_writeout", 2024-09-21T09:31:06.322 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_debug_shutdown", 2024-09-21T09:31:06.322 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_debug_verify_cached_snaps", 2024-09-21T09:31:06.322 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_debug_verify_missing_on_start", 2024-09-21T09:31:06.322 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_debug_verify_stray_on_activate", 2024-09-21T09:31:06.322 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_deep_scrub_update_digest_min_age", 2024-09-21T09:31:06.322 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_mclock_profile", 2024-09-21T09:31:06.322 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_memory_target", 2024-09-21T09:31:06.322 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_memory_target_autotune", 2024-09-21T09:31:06.322 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_op_queue", 2024-09-21T09:31:06.322 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_op_queue_cut_off", 2024-09-21T09:31:06.323 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_pool_default_erasure_code_profile", 2024-09-21T09:31:06.323 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_pool_default_pg_autoscale_mode", 2024-09-21T09:31:06.323 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_recover_clone_overlap", 2024-09-21T09:31:06.323 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_recovery_max_chunk", 2024-09-21T09:31:06.323 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_scrub_load_threshold", 2024-09-21T09:31:06.323 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_scrub_max_interval", 2024-09-21T09:31:06.323 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_shutdown_pgref_assert", 2024-09-21T09:31:06.323 INFO:tasks.workunit.client.0.smithi022.stdout: "public_network", 2024-09-21T09:31:06.323 INFO:tasks.workunit.client.0.smithi022.stdout: "rgw_cache_enabled", 2024-09-21T09:31:06.323 INFO:tasks.workunit.client.0.smithi022.stdout: "rgw_enable_ops_log", 2024-09-21T09:31:06.323 INFO:tasks.workunit.client.0.smithi022.stdout: "rgw_enable_usage_log" 2024-09-21T09:31:06.323 INFO:tasks.workunit.client.0.smithi022.stdout: ] 2024-09-21T09:31:06.323 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.323 INFO:tasks.workunit.client.0.smithi022.stdout: "contact": null, 2024-09-21T09:31:06.323 INFO:tasks.workunit.client.0.smithi022.stdout: "crashes": [], 2024-09-21T09:31:06.324 INFO:tasks.workunit.client.0.smithi022.stdout: "created": "2024-09-21T09:22:01.045656Z", 2024-09-21T09:31:06.324 INFO:tasks.workunit.client.0.smithi022.stdout: "crush": { 2024-09-21T09:31:06.324 INFO:tasks.workunit.client.0.smithi022.stdout: "bucket_algs": { 2024-09-21T09:31:06.324 INFO:tasks.workunit.client.0.smithi022.stdout: "straw2": 3 2024-09-21T09:31:06.324 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.324 INFO:tasks.workunit.client.0.smithi022.stdout: "bucket_sizes": { 2024-09-21T09:31:06.324 INFO:tasks.workunit.client.0.smithi022.stdout: "2": 1, 2024-09-21T09:31:06.324 INFO:tasks.workunit.client.0.smithi022.stdout: "4": 2 2024-09-21T09:31:06.324 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.324 INFO:tasks.workunit.client.0.smithi022.stdout: "bucket_types": { 2024-09-21T09:31:06.324 INFO:tasks.workunit.client.0.smithi022.stdout: "1": 2, 2024-09-21T09:31:06.324 INFO:tasks.workunit.client.0.smithi022.stdout: "11": 1 2024-09-21T09:31:06.324 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.324 INFO:tasks.workunit.client.0.smithi022.stdout: "compat_weight_set": false, 2024-09-21T09:31:06.325 INFO:tasks.workunit.client.0.smithi022.stdout: "device_classes": [ 2024-09-21T09:31:06.325 INFO:tasks.workunit.client.0.smithi022.stdout: 8 2024-09-21T09:31:06.325 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:06.325 INFO:tasks.workunit.client.0.smithi022.stdout: "num_buckets": 6, 2024-09-21T09:31:06.325 INFO:tasks.workunit.client.0.smithi022.stdout: "num_devices": 8, 2024-09-21T09:31:06.325 INFO:tasks.workunit.client.0.smithi022.stdout: "num_rules": 1, 2024-09-21T09:31:06.325 INFO:tasks.workunit.client.0.smithi022.stdout: "num_types": 12, 2024-09-21T09:31:06.325 INFO:tasks.workunit.client.0.smithi022.stdout: "num_weight_sets": 0, 2024-09-21T09:31:06.325 INFO:tasks.workunit.client.0.smithi022.stdout: "tunables": { 2024-09-21T09:31:06.325 INFO:tasks.workunit.client.0.smithi022.stdout: "allowed_bucket_algs": 54, 2024-09-21T09:31:06.325 INFO:tasks.workunit.client.0.smithi022.stdout: "choose_local_fallback_tries": 0, 2024-09-21T09:31:06.325 INFO:tasks.workunit.client.0.smithi022.stdout: "choose_local_tries": 0, 2024-09-21T09:31:06.325 INFO:tasks.workunit.client.0.smithi022.stdout: "choose_total_tries": 50, 2024-09-21T09:31:06.326 INFO:tasks.workunit.client.0.smithi022.stdout: "chooseleaf_descend_once": 1, 2024-09-21T09:31:06.326 INFO:tasks.workunit.client.0.smithi022.stdout: "chooseleaf_stable": 1, 2024-09-21T09:31:06.326 INFO:tasks.workunit.client.0.smithi022.stdout: "chooseleaf_vary_r": 1, 2024-09-21T09:31:06.326 INFO:tasks.workunit.client.0.smithi022.stdout: "has_v2_rules": 0, 2024-09-21T09:31:06.326 INFO:tasks.workunit.client.0.smithi022.stdout: "has_v3_rules": 0, 2024-09-21T09:31:06.326 INFO:tasks.workunit.client.0.smithi022.stdout: "has_v4_buckets": 1, 2024-09-21T09:31:06.326 INFO:tasks.workunit.client.0.smithi022.stdout: "has_v5_rules": 0, 2024-09-21T09:31:06.326 INFO:tasks.workunit.client.0.smithi022.stdout: "legacy_tunables": 0, 2024-09-21T09:31:06.326 INFO:tasks.workunit.client.0.smithi022.stdout: "minimum_required_version": "jewel", 2024-09-21T09:31:06.326 INFO:tasks.workunit.client.0.smithi022.stdout: "optimal_tunables": 1, 2024-09-21T09:31:06.326 INFO:tasks.workunit.client.0.smithi022.stdout: "profile": "jewel", 2024-09-21T09:31:06.326 INFO:tasks.workunit.client.0.smithi022.stdout: "require_feature_tunables": 1, 2024-09-21T09:31:06.326 INFO:tasks.workunit.client.0.smithi022.stdout: "require_feature_tunables2": 1, 2024-09-21T09:31:06.326 INFO:tasks.workunit.client.0.smithi022.stdout: "require_feature_tunables3": 1, 2024-09-21T09:31:06.327 INFO:tasks.workunit.client.0.smithi022.stdout: "require_feature_tunables5": 1, 2024-09-21T09:31:06.327 INFO:tasks.workunit.client.0.smithi022.stdout: "straw_calc_version": 1 2024-09-21T09:31:06.327 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.327 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.327 INFO:tasks.workunit.client.0.smithi022.stdout: "description": "upgrade test cluster", 2024-09-21T09:31:06.327 INFO:tasks.workunit.client.0.smithi022.stdout: "fs": { 2024-09-21T09:31:06.327 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 1, 2024-09-21T09:31:06.327 INFO:tasks.workunit.client.0.smithi022.stdout: "feature_flags": { 2024-09-21T09:31:06.327 INFO:tasks.workunit.client.0.smithi022.stdout: "enable_multiple": true, 2024-09-21T09:31:06.327 INFO:tasks.workunit.client.0.smithi022.stdout: "ever_enabled_multiple": true 2024-09-21T09:31:06.327 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.327 INFO:tasks.workunit.client.0.smithi022.stdout: "filesystems": [ 2024-09-21T09:31:06.327 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:06.327 INFO:tasks.workunit.client.0.smithi022.stdout: "approx_ctime": "2024-09", 2024-09-21T09:31:06.328 INFO:tasks.workunit.client.0.smithi022.stdout: "balancer_enabled": false, 2024-09-21T09:31:06.328 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.328 INFO:tasks.workunit.client.0.smithi022.stdout: "cached_caps": 0, 2024-09-21T09:31:06.328 INFO:tasks.workunit.client.0.smithi022.stdout: "cached_dns": 10, 2024-09-21T09:31:06.328 INFO:tasks.workunit.client.0.smithi022.stdout: "cached_inos": 13, 2024-09-21T09:31:06.328 INFO:tasks.workunit.client.0.smithi022.stdout: "cached_subtrees": 2, 2024-09-21T09:31:06.328 INFO:tasks.workunit.client.0.smithi022.stdout: "ever_allowed_features": 0, 2024-09-21T09:31:06.328 INFO:tasks.workunit.client.0.smithi022.stdout: "explicitly_allowed_features": 0, 2024-09-21T09:31:06.328 INFO:tasks.workunit.client.0.smithi022.stdout: "files": 0, 2024-09-21T09:31:06.328 INFO:tasks.workunit.client.0.smithi022.stdout: "max_mds": 1, 2024-09-21T09:31:06.328 INFO:tasks.workunit.client.0.smithi022.stdout: "num_data_pools": 1, 2024-09-21T09:31:06.328 INFO:tasks.workunit.client.0.smithi022.stdout: "num_in": 1, 2024-09-21T09:31:06.328 INFO:tasks.workunit.client.0.smithi022.stdout: "num_mds": 1, 2024-09-21T09:31:06.328 INFO:tasks.workunit.client.0.smithi022.stdout: "num_sessions": 0, 2024-09-21T09:31:06.328 INFO:tasks.workunit.client.0.smithi022.stdout: "num_standby_replay": 0, 2024-09-21T09:31:06.328 INFO:tasks.workunit.client.0.smithi022.stdout: "num_up": 1, 2024-09-21T09:31:06.329 INFO:tasks.workunit.client.0.smithi022.stdout: "snaps": 0, 2024-09-21T09:31:06.329 INFO:tasks.workunit.client.0.smithi022.stdout: "standby_count_wanted": 0 2024-09-21T09:31:06.329 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.329 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:06.329 INFO:tasks.workunit.client.0.smithi022.stdout: "num_standby_mds": 1, 2024-09-21T09:31:06.329 INFO:tasks.workunit.client.0.smithi022.stdout: "total_num_mds": 2 2024-09-21T09:31:06.329 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.329 INFO:tasks.workunit.client.0.smithi022.stdout: "heap_stats": { 2024-09-21T09:31:06.329 INFO:tasks.workunit.client.0.smithi022.stdout: "mds": { 2024-09-21T09:31:06.329 INFO:tasks.workunit.client.0.smithi022.stdout: "mds.aa5e831b31d98e3163af5edfeb921f9af32463c5": { 2024-09-21T09:31:06.329 INFO:tasks.workunit.client.0.smithi022.stdout: "actual_memory_used": 20578304, 2024-09-21T09:31:06.329 INFO:tasks.workunit.client.0.smithi022.stdout: "central_cache_freelist": 401648, 2024-09-21T09:31:06.329 INFO:tasks.workunit.client.0.smithi022.stdout: "malloc_metadata": 2621440, 2024-09-21T09:31:06.329 INFO:tasks.workunit.client.0.smithi022.stdout: "page_heap_freelist": 0, 2024-09-21T09:31:06.329 INFO:tasks.workunit.client.0.smithi022.stdout: "released_to_os": 606208, 2024-09-21T09:31:06.330 INFO:tasks.workunit.client.0.smithi022.stdout: "spans_in_use": 433, 2024-09-21T09:31:06.330 INFO:tasks.workunit.client.0.smithi022.stdout: "tcmalloc_page_size": 8192, 2024-09-21T09:31:06.330 INFO:tasks.workunit.client.0.smithi022.stdout: "thread_cache_freelists": 1384024, 2024-09-21T09:31:06.330 INFO:tasks.workunit.client.0.smithi022.stdout: "thread_heaps_in_use": 20, 2024-09-21T09:31:06.330 INFO:tasks.workunit.client.0.smithi022.stdout: "transfer_cache_freelist": 1067520, 2024-09-21T09:31:06.330 INFO:tasks.workunit.client.0.smithi022.stdout: "use_by_application": 15103672, 2024-09-21T09:31:06.330 INFO:tasks.workunit.client.0.smithi022.stdout: "virtual_address_space_used": 21184512 2024-09-21T09:31:06.330 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.330 INFO:tasks.workunit.client.0.smithi022.stdout: "mds.f85c107c3119ea450f9f41ee1907af1822a71ba3": { 2024-09-21T09:31:06.330 INFO:tasks.workunit.client.0.smithi022.stdout: "actual_memory_used": 20135936, 2024-09-21T09:31:06.330 INFO:tasks.workunit.client.0.smithi022.stdout: "central_cache_freelist": 417648, 2024-09-21T09:31:06.330 INFO:tasks.workunit.client.0.smithi022.stdout: "malloc_metadata": 2621440, 2024-09-21T09:31:06.330 INFO:tasks.workunit.client.0.smithi022.stdout: "page_heap_freelist": 729088, 2024-09-21T09:31:06.331 INFO:tasks.workunit.client.0.smithi022.stdout: "released_to_os": 0, 2024-09-21T09:31:06.331 INFO:tasks.workunit.client.0.smithi022.stdout: "spans_in_use": 352, 2024-09-21T09:31:06.331 INFO:tasks.workunit.client.0.smithi022.stdout: "tcmalloc_page_size": 8192, 2024-09-21T09:31:06.331 INFO:tasks.workunit.client.0.smithi022.stdout: "thread_cache_freelists": 1122544, 2024-09-21T09:31:06.331 INFO:tasks.workunit.client.0.smithi022.stdout: "thread_heaps_in_use": 12, 2024-09-21T09:31:06.331 INFO:tasks.workunit.client.0.smithi022.stdout: "transfer_cache_freelist": 549376, 2024-09-21T09:31:06.331 INFO:tasks.workunit.client.0.smithi022.stdout: "use_by_application": 14695840, 2024-09-21T09:31:06.331 INFO:tasks.workunit.client.0.smithi022.stdout: "virtual_address_space_used": 20135936 2024-09-21T09:31:06.331 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.331 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.331 INFO:tasks.workunit.client.0.smithi022.stdout: "mon": { 2024-09-21T09:31:06.331 INFO:tasks.workunit.client.0.smithi022.stdout: "mon.5349066dea191aa6e7969482c5114baabfb76f71": { 2024-09-21T09:31:06.331 INFO:tasks.workunit.client.0.smithi022.stdout: "actual_memory_used": 37126144, 2024-09-21T09:31:06.331 INFO:tasks.workunit.client.0.smithi022.stdout: "central_cache_freelist": 1490736, 2024-09-21T09:31:06.332 INFO:tasks.workunit.client.0.smithi022.stdout: "malloc_metadata": 2621440, 2024-09-21T09:31:06.332 INFO:tasks.workunit.client.0.smithi022.stdout: "page_heap_freelist": 0, 2024-09-21T09:31:06.332 INFO:tasks.workunit.client.0.smithi022.stdout: "released_to_os": 14573568, 2024-09-21T09:31:06.332 INFO:tasks.workunit.client.0.smithi022.stdout: "spans_in_use": 1136, 2024-09-21T09:31:06.332 INFO:tasks.workunit.client.0.smithi022.stdout: "tcmalloc_page_size": 8192, 2024-09-21T09:31:06.332 INFO:tasks.workunit.client.0.smithi022.stdout: "thread_cache_freelists": 7420376, 2024-09-21T09:31:06.332 INFO:tasks.workunit.client.0.smithi022.stdout: "thread_heaps_in_use": 22, 2024-09-21T09:31:06.332 INFO:tasks.workunit.client.0.smithi022.stdout: "transfer_cache_freelist": 1487104, 2024-09-21T09:31:06.332 INFO:tasks.workunit.client.0.smithi022.stdout: "use_by_application": 24106488, 2024-09-21T09:31:06.332 INFO:tasks.workunit.client.0.smithi022.stdout: "virtual_address_space_used": 51699712 2024-09-21T09:31:06.332 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.332 INFO:tasks.workunit.client.0.smithi022.stdout: "mon.aa5e831b31d98e3163af5edfeb921f9af32463c5": { 2024-09-21T09:31:06.332 INFO:tasks.workunit.client.0.smithi022.stdout: "actual_memory_used": 35815424, 2024-09-21T09:31:06.332 INFO:tasks.workunit.client.0.smithi022.stdout: "central_cache_freelist": 1420664, 2024-09-21T09:31:06.333 INFO:tasks.workunit.client.0.smithi022.stdout: "malloc_metadata": 2621440, 2024-09-21T09:31:06.333 INFO:tasks.workunit.client.0.smithi022.stdout: "page_heap_freelist": 0, 2024-09-21T09:31:06.333 INFO:tasks.workunit.client.0.smithi022.stdout: "released_to_os": 15892480, 2024-09-21T09:31:06.333 INFO:tasks.workunit.client.0.smithi022.stdout: "spans_in_use": 1098, 2024-09-21T09:31:06.333 INFO:tasks.workunit.client.0.smithi022.stdout: "tcmalloc_page_size": 8192, 2024-09-21T09:31:06.333 INFO:tasks.workunit.client.0.smithi022.stdout: "thread_cache_freelists": 6957376, 2024-09-21T09:31:06.333 INFO:tasks.workunit.client.0.smithi022.stdout: "thread_heaps_in_use": 21, 2024-09-21T09:31:06.333 INFO:tasks.workunit.client.0.smithi022.stdout: "transfer_cache_freelist": 1256448, 2024-09-21T09:31:06.333 INFO:tasks.workunit.client.0.smithi022.stdout: "use_by_application": 23559496, 2024-09-21T09:31:06.333 INFO:tasks.workunit.client.0.smithi022.stdout: "virtual_address_space_used": 51707904 2024-09-21T09:31:06.333 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.333 INFO:tasks.workunit.client.0.smithi022.stdout: "mon.f85c107c3119ea450f9f41ee1907af1822a71ba3": { 2024-09-21T09:31:06.333 INFO:tasks.workunit.client.0.smithi022.stdout: "actual_memory_used": 44515328, 2024-09-21T09:31:06.333 INFO:tasks.workunit.client.0.smithi022.stdout: "central_cache_freelist": 1603128, 2024-09-21T09:31:06.334 INFO:tasks.workunit.client.0.smithi022.stdout: "malloc_metadata": 2621440, 2024-09-21T09:31:06.334 INFO:tasks.workunit.client.0.smithi022.stdout: "page_heap_freelist": 0, 2024-09-21T09:31:06.334 INFO:tasks.workunit.client.0.smithi022.stdout: "released_to_os": 7086080, 2024-09-21T09:31:06.334 INFO:tasks.workunit.client.0.smithi022.stdout: "spans_in_use": 1759, 2024-09-21T09:31:06.334 INFO:tasks.workunit.client.0.smithi022.stdout: "tcmalloc_page_size": 8192, 2024-09-21T09:31:06.334 INFO:tasks.workunit.client.0.smithi022.stdout: "thread_cache_freelists": 9792136, 2024-09-21T09:31:06.334 INFO:tasks.workunit.client.0.smithi022.stdout: "thread_heaps_in_use": 22, 2024-09-21T09:31:06.334 INFO:tasks.workunit.client.0.smithi022.stdout: "transfer_cache_freelist": 2818816, 2024-09-21T09:31:06.334 INFO:tasks.workunit.client.0.smithi022.stdout: "use_by_application": 27679808, 2024-09-21T09:31:06.334 INFO:tasks.workunit.client.0.smithi022.stdout: "virtual_address_space_used": 51601408 2024-09-21T09:31:06.334 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.334 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.334 INFO:tasks.workunit.client.0.smithi022.stdout: "osd": { 2024-09-21T09:31:06.334 INFO:tasks.workunit.client.0.smithi022.stdout: "osd.0": { 2024-09-21T09:31:06.335 INFO:tasks.workunit.client.0.smithi022.stdout: "actual_memory_used": 273334272, 2024-09-21T09:31:06.335 INFO:tasks.workunit.client.0.smithi022.stdout: "central_cache_freelist": 769368, 2024-09-21T09:31:06.335 INFO:tasks.workunit.client.0.smithi022.stdout: "malloc_metadata": 2752512, 2024-09-21T09:31:06.335 INFO:tasks.workunit.client.0.smithi022.stdout: "page_heap_freelist": 0, 2024-09-21T09:31:06.335 INFO:tasks.workunit.client.0.smithi022.stdout: "released_to_os": 327680, 2024-09-21T09:31:06.335 INFO:tasks.workunit.client.0.smithi022.stdout: "spans_in_use": 1355, 2024-09-21T09:31:06.335 INFO:tasks.workunit.client.0.smithi022.stdout: "tcmalloc_page_size": 8192, 2024-09-21T09:31:06.335 INFO:tasks.workunit.client.0.smithi022.stdout: "thread_cache_freelists": 5397728, 2024-09-21T09:31:06.335 INFO:tasks.workunit.client.0.smithi022.stdout: "thread_heaps_in_use": 46, 2024-09-21T09:31:06.335 INFO:tasks.workunit.client.0.smithi022.stdout: "transfer_cache_freelist": 1971968, 2024-09-21T09:31:06.335 INFO:tasks.workunit.client.0.smithi022.stdout: "use_by_application": 262442696, 2024-09-21T09:31:06.335 INFO:tasks.workunit.client.0.smithi022.stdout: "virtual_address_space_used": 273661952 2024-09-21T09:31:06.335 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.335 INFO:tasks.workunit.client.0.smithi022.stdout: "osd.1": { 2024-09-21T09:31:06.335 INFO:tasks.workunit.client.0.smithi022.stdout: "actual_memory_used": 272809984, 2024-09-21T09:31:06.336 INFO:tasks.workunit.client.0.smithi022.stdout: "central_cache_freelist": 620088, 2024-09-21T09:31:06.336 INFO:tasks.workunit.client.0.smithi022.stdout: "malloc_metadata": 2752512, 2024-09-21T09:31:06.336 INFO:tasks.workunit.client.0.smithi022.stdout: "page_heap_freelist": 0, 2024-09-21T09:31:06.336 INFO:tasks.workunit.client.0.smithi022.stdout: "released_to_os": 917504, 2024-09-21T09:31:06.336 INFO:tasks.workunit.client.0.smithi022.stdout: "spans_in_use": 1320, 2024-09-21T09:31:06.336 INFO:tasks.workunit.client.0.smithi022.stdout: "tcmalloc_page_size": 8192, 2024-09-21T09:31:06.336 INFO:tasks.workunit.client.0.smithi022.stdout: "thread_cache_freelists": 5394160, 2024-09-21T09:31:06.336 INFO:tasks.workunit.client.0.smithi022.stdout: "thread_heaps_in_use": 46, 2024-09-21T09:31:06.336 INFO:tasks.workunit.client.0.smithi022.stdout: "transfer_cache_freelist": 1928704, 2024-09-21T09:31:06.336 INFO:tasks.workunit.client.0.smithi022.stdout: "use_by_application": 262114520, 2024-09-21T09:31:06.336 INFO:tasks.workunit.client.0.smithi022.stdout: "virtual_address_space_used": 273727488 2024-09-21T09:31:06.336 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.336 INFO:tasks.workunit.client.0.smithi022.stdout: "osd.2": { 2024-09-21T09:31:06.336 INFO:tasks.workunit.client.0.smithi022.stdout: "actual_memory_used": 272048128, 2024-09-21T09:31:06.337 INFO:tasks.workunit.client.0.smithi022.stdout: "central_cache_freelist": 798544, 2024-09-21T09:31:06.337 INFO:tasks.workunit.client.0.smithi022.stdout: "malloc_metadata": 2752512, 2024-09-21T09:31:06.337 INFO:tasks.workunit.client.0.smithi022.stdout: "page_heap_freelist": 0, 2024-09-21T09:31:06.337 INFO:tasks.workunit.client.0.smithi022.stdout: "released_to_os": 843776, 2024-09-21T09:31:06.337 INFO:tasks.workunit.client.0.smithi022.stdout: "spans_in_use": 1251, 2024-09-21T09:31:06.337 INFO:tasks.workunit.client.0.smithi022.stdout: "tcmalloc_page_size": 8192, 2024-09-21T09:31:06.337 INFO:tasks.workunit.client.0.smithi022.stdout: "thread_cache_freelists": 4637464, 2024-09-21T09:31:06.337 INFO:tasks.workunit.client.0.smithi022.stdout: "thread_heaps_in_use": 47, 2024-09-21T09:31:06.337 INFO:tasks.workunit.client.0.smithi022.stdout: "transfer_cache_freelist": 2226176, 2024-09-21T09:31:06.337 INFO:tasks.workunit.client.0.smithi022.stdout: "use_by_application": 261633432, 2024-09-21T09:31:06.337 INFO:tasks.workunit.client.0.smithi022.stdout: "virtual_address_space_used": 272891904 2024-09-21T09:31:06.337 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.337 INFO:tasks.workunit.client.0.smithi022.stdout: "osd.3": { 2024-09-21T09:31:06.337 INFO:tasks.workunit.client.0.smithi022.stdout: "actual_memory_used": 271597568, 2024-09-21T09:31:06.337 INFO:tasks.workunit.client.0.smithi022.stdout: "central_cache_freelist": 640040, 2024-09-21T09:31:06.338 INFO:tasks.workunit.client.0.smithi022.stdout: "malloc_metadata": 2752512, 2024-09-21T09:31:06.338 INFO:tasks.workunit.client.0.smithi022.stdout: "page_heap_freelist": 0, 2024-09-21T09:31:06.338 INFO:tasks.workunit.client.0.smithi022.stdout: "released_to_os": 32768, 2024-09-21T09:31:06.338 INFO:tasks.workunit.client.0.smithi022.stdout: "spans_in_use": 1246, 2024-09-21T09:31:06.338 INFO:tasks.workunit.client.0.smithi022.stdout: "tcmalloc_page_size": 8192, 2024-09-21T09:31:06.338 INFO:tasks.workunit.client.0.smithi022.stdout: "thread_cache_freelists": 4656264, 2024-09-21T09:31:06.338 INFO:tasks.workunit.client.0.smithi022.stdout: "thread_heaps_in_use": 46, 2024-09-21T09:31:06.338 INFO:tasks.workunit.client.0.smithi022.stdout: "transfer_cache_freelist": 1735424, 2024-09-21T09:31:06.338 INFO:tasks.workunit.client.0.smithi022.stdout: "use_by_application": 261813328, 2024-09-21T09:31:06.338 INFO:tasks.workunit.client.0.smithi022.stdout: "virtual_address_space_used": 271630336 2024-09-21T09:31:06.338 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.338 INFO:tasks.workunit.client.0.smithi022.stdout: "osd.4": { 2024-09-21T09:31:06.338 INFO:tasks.workunit.client.0.smithi022.stdout: "actual_memory_used": 272105472, 2024-09-21T09:31:06.338 INFO:tasks.workunit.client.0.smithi022.stdout: "central_cache_freelist": 726400, 2024-09-21T09:31:06.338 INFO:tasks.workunit.client.0.smithi022.stdout: "malloc_metadata": 2752512, 2024-09-21T09:31:06.339 INFO:tasks.workunit.client.0.smithi022.stdout: "page_heap_freelist": 0, 2024-09-21T09:31:06.339 INFO:tasks.workunit.client.0.smithi022.stdout: "released_to_os": 507904, 2024-09-21T09:31:06.339 INFO:tasks.workunit.client.0.smithi022.stdout: "spans_in_use": 1259, 2024-09-21T09:31:06.339 INFO:tasks.workunit.client.0.smithi022.stdout: "tcmalloc_page_size": 8192, 2024-09-21T09:31:06.339 INFO:tasks.workunit.client.0.smithi022.stdout: "thread_cache_freelists": 4612168, 2024-09-21T09:31:06.339 INFO:tasks.workunit.client.0.smithi022.stdout: "thread_heaps_in_use": 46, 2024-09-21T09:31:06.339 INFO:tasks.workunit.client.0.smithi022.stdout: "transfer_cache_freelist": 2231808, 2024-09-21T09:31:06.339 INFO:tasks.workunit.client.0.smithi022.stdout: "use_by_application": 261782584, 2024-09-21T09:31:06.339 INFO:tasks.workunit.client.0.smithi022.stdout: "virtual_address_space_used": 272613376 2024-09-21T09:31:06.339 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.339 INFO:tasks.workunit.client.0.smithi022.stdout: "osd.5": { 2024-09-21T09:31:06.339 INFO:tasks.workunit.client.0.smithi022.stdout: "actual_memory_used": 272949248, 2024-09-21T09:31:06.339 INFO:tasks.workunit.client.0.smithi022.stdout: "central_cache_freelist": 841080, 2024-09-21T09:31:06.340 INFO:tasks.workunit.client.0.smithi022.stdout: "malloc_metadata": 2752512, 2024-09-21T09:31:06.340 INFO:tasks.workunit.client.0.smithi022.stdout: "page_heap_freelist": 0, 2024-09-21T09:31:06.340 INFO:tasks.workunit.client.0.smithi022.stdout: "released_to_os": 778240, 2024-09-21T09:31:06.340 INFO:tasks.workunit.client.0.smithi022.stdout: "spans_in_use": 1264, 2024-09-21T09:31:06.340 INFO:tasks.workunit.client.0.smithi022.stdout: "tcmalloc_page_size": 8192, 2024-09-21T09:31:06.340 INFO:tasks.workunit.client.0.smithi022.stdout: "thread_cache_freelists": 5126952, 2024-09-21T09:31:06.340 INFO:tasks.workunit.client.0.smithi022.stdout: "thread_heaps_in_use": 46, 2024-09-21T09:31:06.340 INFO:tasks.workunit.client.0.smithi022.stdout: "transfer_cache_freelist": 1959680, 2024-09-21T09:31:06.340 INFO:tasks.workunit.client.0.smithi022.stdout: "use_by_application": 262269024, 2024-09-21T09:31:06.340 INFO:tasks.workunit.client.0.smithi022.stdout: "virtual_address_space_used": 273727488 2024-09-21T09:31:06.340 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.340 INFO:tasks.workunit.client.0.smithi022.stdout: "osd.6": { 2024-09-21T09:31:06.340 INFO:tasks.workunit.client.0.smithi022.stdout: "actual_memory_used": 271695872, 2024-09-21T09:31:06.340 INFO:tasks.workunit.client.0.smithi022.stdout: "central_cache_freelist": 741424, 2024-09-21T09:31:06.340 INFO:tasks.workunit.client.0.smithi022.stdout: "malloc_metadata": 2752512, 2024-09-21T09:31:06.341 INFO:tasks.workunit.client.0.smithi022.stdout: "page_heap_freelist": 24576, 2024-09-21T09:31:06.341 INFO:tasks.workunit.client.0.smithi022.stdout: "released_to_os": 147456, 2024-09-21T09:31:06.341 INFO:tasks.workunit.client.0.smithi022.stdout: "spans_in_use": 1240, 2024-09-21T09:31:06.341 INFO:tasks.workunit.client.0.smithi022.stdout: "tcmalloc_page_size": 8192, 2024-09-21T09:31:06.341 INFO:tasks.workunit.client.0.smithi022.stdout: "thread_cache_freelists": 4653664, 2024-09-21T09:31:06.341 INFO:tasks.workunit.client.0.smithi022.stdout: "thread_heaps_in_use": 46, 2024-09-21T09:31:06.341 INFO:tasks.workunit.client.0.smithi022.stdout: "transfer_cache_freelist": 1986560, 2024-09-21T09:31:06.341 INFO:tasks.workunit.client.0.smithi022.stdout: "use_by_application": 261537136, 2024-09-21T09:31:06.341 INFO:tasks.workunit.client.0.smithi022.stdout: "virtual_address_space_used": 271843328 2024-09-21T09:31:06.341 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.341 INFO:tasks.workunit.client.0.smithi022.stdout: "osd.7": { 2024-09-21T09:31:06.341 INFO:tasks.workunit.client.0.smithi022.stdout: "actual_memory_used": 272916480, 2024-09-21T09:31:06.341 INFO:tasks.workunit.client.0.smithi022.stdout: "central_cache_freelist": 1015904, 2024-09-21T09:31:06.341 INFO:tasks.workunit.client.0.smithi022.stdout: "malloc_metadata": 2752512, 2024-09-21T09:31:06.341 INFO:tasks.workunit.client.0.smithi022.stdout: "page_heap_freelist": 24576, 2024-09-21T09:31:06.342 INFO:tasks.workunit.client.0.smithi022.stdout: "released_to_os": 811008, 2024-09-21T09:31:06.342 INFO:tasks.workunit.client.0.smithi022.stdout: "spans_in_use": 1293, 2024-09-21T09:31:06.342 INFO:tasks.workunit.client.0.smithi022.stdout: "tcmalloc_page_size": 8192, 2024-09-21T09:31:06.342 INFO:tasks.workunit.client.0.smithi022.stdout: "thread_cache_freelists": 4784248, 2024-09-21T09:31:06.342 INFO:tasks.workunit.client.0.smithi022.stdout: "thread_heaps_in_use": 45, 2024-09-21T09:31:06.342 INFO:tasks.workunit.client.0.smithi022.stdout: "transfer_cache_freelist": 2045440, 2024-09-21T09:31:06.342 INFO:tasks.workunit.client.0.smithi022.stdout: "use_by_application": 262293800, 2024-09-21T09:31:06.342 INFO:tasks.workunit.client.0.smithi022.stdout: "virtual_address_space_used": 273727488 2024-09-21T09:31:06.342 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.342 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.342 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.342 INFO:tasks.workunit.client.0.smithi022.stdout: "hosts": { 2024-09-21T09:31:06.342 INFO:tasks.workunit.client.0.smithi022.stdout: "num": 2, 2024-09-21T09:31:06.342 INFO:tasks.workunit.client.0.smithi022.stdout: "num_with_mds": 2, 2024-09-21T09:31:06.343 INFO:tasks.workunit.client.0.smithi022.stdout: "num_with_mgr": 2, 2024-09-21T09:31:06.343 INFO:tasks.workunit.client.0.smithi022.stdout: "num_with_mon": 2, 2024-09-21T09:31:06.343 INFO:tasks.workunit.client.0.smithi022.stdout: "num_with_osd": 2 2024-09-21T09:31:06.343 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.343 INFO:tasks.workunit.client.0.smithi022.stdout: "io_rate": { 2024-09-21T09:31:06.343 INFO:tasks.workunit.client.0.smithi022.stdout: "pg_stats_delta": { 2024-09-21T09:31:06.343 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": 0, 2024-09-21T09:31:06.343 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:06.343 INFO:tasks.workunit.client.0.smithi022.stdout: "num_store_stats": 0, 2024-09-21T09:31:06.343 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:06.343 INFO:tasks.workunit.client.0.smithi022.stdout: "stamp_delta": "12.002616", 2024-09-21T09:31:06.343 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:06.343 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:06.343 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:06.343 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:06.344 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:06.344 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:06.344 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:06.344 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:06.344 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:06.344 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:06.344 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:06.344 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:06.344 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:06.344 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:06.344 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:06.344 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:06.344 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:06.344 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:06.345 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:06.345 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:06.345 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:06.345 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:06.345 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:06.345 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:06.345 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:06.345 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:06.345 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:06.345 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:06.345 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:06.345 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:06.345 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:06.346 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:06.346 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:06.346 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:06.346 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:06.346 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:06.346 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:06.346 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:06.346 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:06.346 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:06.346 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:06.346 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.346 INFO:tasks.workunit.client.0.smithi022.stdout: "store_stats": { 2024-09-21T09:31:06.346 INFO:tasks.workunit.client.0.smithi022.stdout: "allocated": 0, 2024-09-21T09:31:06.346 INFO:tasks.workunit.client.0.smithi022.stdout: "available": 0, 2024-09-21T09:31:06.347 INFO:tasks.workunit.client.0.smithi022.stdout: "data_compressed": 0, 2024-09-21T09:31:06.347 INFO:tasks.workunit.client.0.smithi022.stdout: "data_compressed_allocated": 0, 2024-09-21T09:31:06.347 INFO:tasks.workunit.client.0.smithi022.stdout: "data_compressed_original": 0, 2024-09-21T09:31:06.347 INFO:tasks.workunit.client.0.smithi022.stdout: "data_stored": 0, 2024-09-21T09:31:06.347 INFO:tasks.workunit.client.0.smithi022.stdout: "internal_metadata": 0, 2024-09-21T09:31:06.347 INFO:tasks.workunit.client.0.smithi022.stdout: "internally_reserved": 0, 2024-09-21T09:31:06.347 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_allocated": 0, 2024-09-21T09:31:06.347 INFO:tasks.workunit.client.0.smithi022.stdout: "total": 0 2024-09-21T09:31:06.347 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.347 INFO:tasks.workunit.client.0.smithi022.stdout: "up": 0 2024-09-21T09:31:06.347 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.347 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.347 INFO:tasks.workunit.client.0.smithi022.stdout: "leaderboard": false, 2024-09-21T09:31:06.348 INFO:tasks.workunit.client.0.smithi022.stdout: "license": "sharing-1-0", 2024-09-21T09:31:06.348 INFO:tasks.workunit.client.0.smithi022.stdout: "mempool": { 2024-09-21T09:31:06.348 INFO:tasks.workunit.client.0.smithi022.stdout: "mds": { 2024-09-21T09:31:06.348 INFO:tasks.workunit.client.0.smithi022.stdout: "mds.aa5e831b31d98e3163af5edfeb921f9af32463c5": { 2024-09-21T09:31:06.348 INFO:tasks.workunit.client.0.smithi022.stdout: "bloom_filter": { 2024-09-21T09:31:06.348 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.348 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.348 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.348 INFO:tasks.workunit.client.0.smithi022.stdout: "bluefs": { 2024-09-21T09:31:06.348 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.348 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.348 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.348 INFO:tasks.workunit.client.0.smithi022.stdout: "bluefs_file_reader": { 2024-09-21T09:31:06.348 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.348 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.349 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.349 INFO:tasks.workunit.client.0.smithi022.stdout: "bluefs_file_writer": { 2024-09-21T09:31:06.349 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.349 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.349 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.349 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_Blob": { 2024-09-21T09:31:06.349 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.349 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.349 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.349 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_Buffer": { 2024-09-21T09:31:06.349 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.349 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.349 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.349 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_Extent": { 2024-09-21T09:31:06.349 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.350 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.350 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.350 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_SharedBlob": { 2024-09-21T09:31:06.350 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.350 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.350 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.350 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_alloc": { 2024-09-21T09:31:06.350 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.350 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.350 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.350 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_cache_data": { 2024-09-21T09:31:06.350 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.350 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.350 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.351 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_cache_meta": { 2024-09-21T09:31:06.351 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.351 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.351 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.351 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_cache_onode": { 2024-09-21T09:31:06.351 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.351 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.351 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.351 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_cache_other": { 2024-09-21T09:31:06.351 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.351 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.351 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.351 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_fsck": { 2024-09-21T09:31:06.351 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.351 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.352 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.352 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_inline_bl": { 2024-09-21T09:31:06.352 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.352 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.352 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.352 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_txc": { 2024-09-21T09:31:06.352 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.352 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.352 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.352 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_writing": { 2024-09-21T09:31:06.352 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.352 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.352 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.352 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_writing_deferred": { 2024-09-21T09:31:06.353 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.353 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.353 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.353 INFO:tasks.workunit.client.0.smithi022.stdout: "buffer_anon": { 2024-09-21T09:31:06.353 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 222471, 2024-09-21T09:31:06.353 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 72 2024-09-21T09:31:06.353 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.353 INFO:tasks.workunit.client.0.smithi022.stdout: "buffer_meta": { 2024-09-21T09:31:06.353 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.353 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.353 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.353 INFO:tasks.workunit.client.0.smithi022.stdout: "mds_co": { 2024-09-21T09:31:06.353 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 50488, 2024-09-21T09:31:06.353 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 208 2024-09-21T09:31:06.354 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.354 INFO:tasks.workunit.client.0.smithi022.stdout: "osd": { 2024-09-21T09:31:06.354 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.354 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.354 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.354 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_mapbl": { 2024-09-21T09:31:06.354 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.354 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.354 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.354 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_pglog": { 2024-09-21T09:31:06.354 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.354 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.354 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.354 INFO:tasks.workunit.client.0.smithi022.stdout: "osdmap": { 2024-09-21T09:31:06.355 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 8136, 2024-09-21T09:31:06.355 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 139 2024-09-21T09:31:06.355 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.355 INFO:tasks.workunit.client.0.smithi022.stdout: "osdmap_mapping": { 2024-09-21T09:31:06.355 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.355 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.355 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.355 INFO:tasks.workunit.client.0.smithi022.stdout: "pgmap": { 2024-09-21T09:31:06.355 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.355 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.355 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.355 INFO:tasks.workunit.client.0.smithi022.stdout: "unittest_1": { 2024-09-21T09:31:06.355 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.355 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.355 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.356 INFO:tasks.workunit.client.0.smithi022.stdout: "unittest_2": { 2024-09-21T09:31:06.356 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.356 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.356 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.356 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.356 INFO:tasks.workunit.client.0.smithi022.stdout: "mds.f85c107c3119ea450f9f41ee1907af1822a71ba3": { 2024-09-21T09:31:06.356 INFO:tasks.workunit.client.0.smithi022.stdout: "bloom_filter": { 2024-09-21T09:31:06.356 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.356 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.356 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.356 INFO:tasks.workunit.client.0.smithi022.stdout: "bluefs": { 2024-09-21T09:31:06.356 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.356 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.356 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.356 INFO:tasks.workunit.client.0.smithi022.stdout: "bluefs_file_reader": { 2024-09-21T09:31:06.357 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.357 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.357 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.357 INFO:tasks.workunit.client.0.smithi022.stdout: "bluefs_file_writer": { 2024-09-21T09:31:06.357 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.357 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.357 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.357 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_Blob": { 2024-09-21T09:31:06.357 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.357 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.357 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.357 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_Buffer": { 2024-09-21T09:31:06.357 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.358 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.358 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.358 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_Extent": { 2024-09-21T09:31:06.358 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.358 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.358 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.358 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_SharedBlob": { 2024-09-21T09:31:06.358 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.358 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.358 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.358 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_alloc": { 2024-09-21T09:31:06.358 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.358 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.358 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.359 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_cache_data": { 2024-09-21T09:31:06.359 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.359 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.359 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.359 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_cache_meta": { 2024-09-21T09:31:06.359 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.359 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.359 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.359 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_cache_onode": { 2024-09-21T09:31:06.359 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.359 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.359 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.359 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_cache_other": { 2024-09-21T09:31:06.359 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.360 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.360 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.360 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_fsck": { 2024-09-21T09:31:06.360 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.360 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.360 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.360 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_inline_bl": { 2024-09-21T09:31:06.360 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.360 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.360 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.360 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_txc": { 2024-09-21T09:31:06.360 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.360 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.360 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.360 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_writing": { 2024-09-21T09:31:06.361 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.361 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.361 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.361 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_writing_deferred": { 2024-09-21T09:31:06.361 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.361 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.361 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.361 INFO:tasks.workunit.client.0.smithi022.stdout: "buffer_anon": { 2024-09-21T09:31:06.361 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 149097, 2024-09-21T09:31:06.361 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 33 2024-09-21T09:31:06.361 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.361 INFO:tasks.workunit.client.0.smithi022.stdout: "buffer_meta": { 2024-09-21T09:31:06.361 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.361 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.362 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.362 INFO:tasks.workunit.client.0.smithi022.stdout: "mds_co": { 2024-09-21T09:31:06.362 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 856, 2024-09-21T09:31:06.362 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 2 2024-09-21T09:31:06.362 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.362 INFO:tasks.workunit.client.0.smithi022.stdout: "osd": { 2024-09-21T09:31:06.362 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.362 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.362 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.362 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_mapbl": { 2024-09-21T09:31:06.362 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.362 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.362 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.362 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_pglog": { 2024-09-21T09:31:06.363 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.363 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.363 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.363 INFO:tasks.workunit.client.0.smithi022.stdout: "osdmap": { 2024-09-21T09:31:06.363 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.363 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.363 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.363 INFO:tasks.workunit.client.0.smithi022.stdout: "osdmap_mapping": { 2024-09-21T09:31:06.363 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.363 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.363 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.363 INFO:tasks.workunit.client.0.smithi022.stdout: "pgmap": { 2024-09-21T09:31:06.363 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.363 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.363 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.364 INFO:tasks.workunit.client.0.smithi022.stdout: "unittest_1": { 2024-09-21T09:31:06.364 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.364 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.364 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.364 INFO:tasks.workunit.client.0.smithi022.stdout: "unittest_2": { 2024-09-21T09:31:06.364 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.364 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.364 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.364 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.364 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.364 INFO:tasks.workunit.client.0.smithi022.stdout: "mon": { 2024-09-21T09:31:06.364 INFO:tasks.workunit.client.0.smithi022.stdout: "mon.5349066dea191aa6e7969482c5114baabfb76f71": { 2024-09-21T09:31:06.364 INFO:tasks.workunit.client.0.smithi022.stdout: "bloom_filter": { 2024-09-21T09:31:06.364 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.364 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.365 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.365 INFO:tasks.workunit.client.0.smithi022.stdout: "bluefs": { 2024-09-21T09:31:06.365 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.365 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.365 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.365 INFO:tasks.workunit.client.0.smithi022.stdout: "bluefs_file_reader": { 2024-09-21T09:31:06.365 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.365 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.365 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.365 INFO:tasks.workunit.client.0.smithi022.stdout: "bluefs_file_writer": { 2024-09-21T09:31:06.365 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.365 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.365 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.365 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_Blob": { 2024-09-21T09:31:06.365 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.366 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.366 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.366 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_Buffer": { 2024-09-21T09:31:06.366 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.366 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.366 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.366 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_Extent": { 2024-09-21T09:31:06.366 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.366 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.366 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.366 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_SharedBlob": { 2024-09-21T09:31:06.366 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.366 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.366 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.366 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_alloc": { 2024-09-21T09:31:06.367 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.367 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.367 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.367 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_cache_data": { 2024-09-21T09:31:06.367 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.367 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.367 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.367 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_cache_meta": { 2024-09-21T09:31:06.367 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.367 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.367 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.367 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_cache_onode": { 2024-09-21T09:31:06.367 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.367 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.367 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.368 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_cache_other": { 2024-09-21T09:31:06.368 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.368 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.368 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.368 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_fsck": { 2024-09-21T09:31:06.368 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.368 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.368 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.368 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_inline_bl": { 2024-09-21T09:31:06.368 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.368 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.368 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.368 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_txc": { 2024-09-21T09:31:06.368 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.369 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.369 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.369 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_writing": { 2024-09-21T09:31:06.369 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.369 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.369 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.369 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_writing_deferred": { 2024-09-21T09:31:06.369 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.369 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.369 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.369 INFO:tasks.workunit.client.0.smithi022.stdout: "buffer_anon": { 2024-09-21T09:31:06.370 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 1309296, 2024-09-21T09:31:06.370 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 420 2024-09-21T09:31:06.370 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.370 INFO:tasks.workunit.client.0.smithi022.stdout: "buffer_meta": { 2024-09-21T09:31:06.370 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.370 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.370 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.370 INFO:tasks.workunit.client.0.smithi022.stdout: "mds_co": { 2024-09-21T09:31:06.370 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.370 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.370 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.370 INFO:tasks.workunit.client.0.smithi022.stdout: "osd": { 2024-09-21T09:31:06.371 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.371 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.371 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.371 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_mapbl": { 2024-09-21T09:31:06.371 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.371 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.371 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.371 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_pglog": { 2024-09-21T09:31:06.371 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.371 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.371 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.371 INFO:tasks.workunit.client.0.smithi022.stdout: "osdmap": { 2024-09-21T09:31:06.371 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 6784, 2024-09-21T09:31:06.372 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 138 2024-09-21T09:31:06.372 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.372 INFO:tasks.workunit.client.0.smithi022.stdout: "osdmap_mapping": { 2024-09-21T09:31:06.372 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 12744, 2024-09-21T09:31:06.372 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 1546 2024-09-21T09:31:06.372 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.372 INFO:tasks.workunit.client.0.smithi022.stdout: "pgmap": { 2024-09-21T09:31:06.372 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 5312, 2024-09-21T09:31:06.372 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 103 2024-09-21T09:31:06.372 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.372 INFO:tasks.workunit.client.0.smithi022.stdout: "unittest_1": { 2024-09-21T09:31:06.372 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.372 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.372 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.373 INFO:tasks.workunit.client.0.smithi022.stdout: "unittest_2": { 2024-09-21T09:31:06.373 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.373 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.373 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.373 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.373 INFO:tasks.workunit.client.0.smithi022.stdout: "mon.aa5e831b31d98e3163af5edfeb921f9af32463c5": { 2024-09-21T09:31:06.373 INFO:tasks.workunit.client.0.smithi022.stdout: "bloom_filter": { 2024-09-21T09:31:06.373 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.373 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.373 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.373 INFO:tasks.workunit.client.0.smithi022.stdout: "bluefs": { 2024-09-21T09:31:06.373 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.373 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.373 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.374 INFO:tasks.workunit.client.0.smithi022.stdout: "bluefs_file_reader": { 2024-09-21T09:31:06.374 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.374 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.374 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.374 INFO:tasks.workunit.client.0.smithi022.stdout: "bluefs_file_writer": { 2024-09-21T09:31:06.374 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.374 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.374 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.374 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_Blob": { 2024-09-21T09:31:06.374 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.374 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.374 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.374 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_Buffer": { 2024-09-21T09:31:06.375 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.375 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.375 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.375 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_Extent": { 2024-09-21T09:31:06.375 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.375 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.375 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.375 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_SharedBlob": { 2024-09-21T09:31:06.375 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.375 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.375 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.375 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_alloc": { 2024-09-21T09:31:06.376 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.376 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.376 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.376 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_cache_data": { 2024-09-21T09:31:06.376 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.376 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.376 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.376 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_cache_meta": { 2024-09-21T09:31:06.376 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.376 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.376 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.376 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_cache_onode": { 2024-09-21T09:31:06.376 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.376 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.376 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.377 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_cache_other": { 2024-09-21T09:31:06.377 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.377 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.377 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.377 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_fsck": { 2024-09-21T09:31:06.377 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.377 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.377 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.377 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_inline_bl": { 2024-09-21T09:31:06.377 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.378 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.378 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.378 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_txc": { 2024-09-21T09:31:06.378 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.378 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.378 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.378 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_writing": { 2024-09-21T09:31:06.378 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.378 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.378 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.378 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_writing_deferred": { 2024-09-21T09:31:06.378 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.378 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.379 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.379 INFO:tasks.workunit.client.0.smithi022.stdout: "buffer_anon": { 2024-09-21T09:31:06.379 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 1087814, 2024-09-21T09:31:06.379 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 347 2024-09-21T09:31:06.379 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.379 INFO:tasks.workunit.client.0.smithi022.stdout: "buffer_meta": { 2024-09-21T09:31:06.379 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.379 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.379 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.380 INFO:tasks.workunit.client.0.smithi022.stdout: "mds_co": { 2024-09-21T09:31:06.380 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.380 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.380 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.380 INFO:tasks.workunit.client.0.smithi022.stdout: "osd": { 2024-09-21T09:31:06.380 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.380 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.380 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.380 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_mapbl": { 2024-09-21T09:31:06.380 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.380 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.380 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.380 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_pglog": { 2024-09-21T09:31:06.381 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.381 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.381 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.381 INFO:tasks.workunit.client.0.smithi022.stdout: "osdmap": { 2024-09-21T09:31:06.381 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 6784, 2024-09-21T09:31:06.381 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 138 2024-09-21T09:31:06.381 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.381 INFO:tasks.workunit.client.0.smithi022.stdout: "osdmap_mapping": { 2024-09-21T09:31:06.381 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 12744, 2024-09-21T09:31:06.381 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 1546 2024-09-21T09:31:06.381 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.381 INFO:tasks.workunit.client.0.smithi022.stdout: "pgmap": { 2024-09-21T09:31:06.381 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 5312, 2024-09-21T09:31:06.382 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 103 2024-09-21T09:31:06.382 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.382 INFO:tasks.workunit.client.0.smithi022.stdout: "unittest_1": { 2024-09-21T09:31:06.382 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.382 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.382 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.382 INFO:tasks.workunit.client.0.smithi022.stdout: "unittest_2": { 2024-09-21T09:31:06.382 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.382 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.382 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.383 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.383 INFO:tasks.workunit.client.0.smithi022.stdout: "mon.f85c107c3119ea450f9f41ee1907af1822a71ba3": { 2024-09-21T09:31:06.383 INFO:tasks.workunit.client.0.smithi022.stdout: "bloom_filter": { 2024-09-21T09:31:06.383 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.383 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.383 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.383 INFO:tasks.workunit.client.0.smithi022.stdout: "bluefs": { 2024-09-21T09:31:06.383 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.383 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.383 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.383 INFO:tasks.workunit.client.0.smithi022.stdout: "bluefs_file_reader": { 2024-09-21T09:31:06.383 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.384 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.384 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.384 INFO:tasks.workunit.client.0.smithi022.stdout: "bluefs_file_writer": { 2024-09-21T09:31:06.384 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.384 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.384 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.384 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_Blob": { 2024-09-21T09:31:06.384 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.384 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.384 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.384 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_Buffer": { 2024-09-21T09:31:06.384 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.384 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.385 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.385 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_Extent": { 2024-09-21T09:31:06.385 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.385 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.385 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.385 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_SharedBlob": { 2024-09-21T09:31:06.385 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.385 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.385 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.385 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_alloc": { 2024-09-21T09:31:06.385 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.385 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.386 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.386 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_cache_data": { 2024-09-21T09:31:06.386 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.386 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.386 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.386 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_cache_meta": { 2024-09-21T09:31:06.386 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.386 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.386 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.386 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_cache_onode": { 2024-09-21T09:31:06.386 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.386 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.386 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.387 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_cache_other": { 2024-09-21T09:31:06.387 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.387 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.387 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.387 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_fsck": { 2024-09-21T09:31:06.387 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.387 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.387 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.387 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_inline_bl": { 2024-09-21T09:31:06.387 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.388 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.388 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.388 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_txc": { 2024-09-21T09:31:06.388 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.388 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.388 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.388 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_writing": { 2024-09-21T09:31:06.388 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.388 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.388 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.388 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_writing_deferred": { 2024-09-21T09:31:06.388 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.388 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.389 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.389 INFO:tasks.workunit.client.0.smithi022.stdout: "buffer_anon": { 2024-09-21T09:31:06.389 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 1131104, 2024-09-21T09:31:06.389 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 348 2024-09-21T09:31:06.389 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.389 INFO:tasks.workunit.client.0.smithi022.stdout: "buffer_meta": { 2024-09-21T09:31:06.389 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 352, 2024-09-21T09:31:06.389 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 4 2024-09-21T09:31:06.389 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.389 INFO:tasks.workunit.client.0.smithi022.stdout: "mds_co": { 2024-09-21T09:31:06.389 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.389 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.390 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.390 INFO:tasks.workunit.client.0.smithi022.stdout: "osd": { 2024-09-21T09:31:06.390 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.390 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.390 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.390 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_mapbl": { 2024-09-21T09:31:06.390 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.390 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.390 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.390 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_pglog": { 2024-09-21T09:31:06.390 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.390 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.390 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.390 INFO:tasks.workunit.client.0.smithi022.stdout: "osdmap": { 2024-09-21T09:31:06.391 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 6784, 2024-09-21T09:31:06.391 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 138 2024-09-21T09:31:06.391 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.391 INFO:tasks.workunit.client.0.smithi022.stdout: "osdmap_mapping": { 2024-09-21T09:31:06.391 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 12744, 2024-09-21T09:31:06.391 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 1546 2024-09-21T09:31:06.391 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.391 INFO:tasks.workunit.client.0.smithi022.stdout: "pgmap": { 2024-09-21T09:31:06.391 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 10624, 2024-09-21T09:31:06.391 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 206 2024-09-21T09:31:06.391 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.391 INFO:tasks.workunit.client.0.smithi022.stdout: "unittest_1": { 2024-09-21T09:31:06.391 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.391 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.392 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.392 INFO:tasks.workunit.client.0.smithi022.stdout: "unittest_2": { 2024-09-21T09:31:06.392 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.392 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.392 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.392 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.392 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.392 INFO:tasks.workunit.client.0.smithi022.stdout: "osd": { 2024-09-21T09:31:06.392 INFO:tasks.workunit.client.0.smithi022.stdout: "osd.0": { 2024-09-21T09:31:06.392 INFO:tasks.workunit.client.0.smithi022.stdout: "bloom_filter": { 2024-09-21T09:31:06.392 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.392 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.393 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.393 INFO:tasks.workunit.client.0.smithi022.stdout: "bluefs": { 2024-09-21T09:31:06.393 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 6304, 2024-09-21T09:31:06.393 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 115 2024-09-21T09:31:06.393 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.393 INFO:tasks.workunit.client.0.smithi022.stdout: "bluefs_file_reader": { 2024-09-21T09:31:06.393 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 16896, 2024-09-21T09:31:06.393 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 8 2024-09-21T09:31:06.393 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.393 INFO:tasks.workunit.client.0.smithi022.stdout: "bluefs_file_writer": { 2024-09-21T09:31:06.393 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 576, 2024-09-21T09:31:06.393 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 3 2024-09-21T09:31:06.393 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.393 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_Blob": { 2024-09-21T09:31:06.393 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 20272, 2024-09-21T09:31:06.394 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 181 2024-09-21T09:31:06.394 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.394 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_Buffer": { 2024-09-21T09:31:06.394 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 224, 2024-09-21T09:31:06.394 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 2 2024-09-21T09:31:06.394 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.394 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_Extent": { 2024-09-21T09:31:06.394 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 8688, 2024-09-21T09:31:06.394 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 181 2024-09-21T09:31:06.394 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.394 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_SharedBlob": { 2024-09-21T09:31:06.394 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 20272, 2024-09-21T09:31:06.395 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 181 2024-09-21T09:31:06.395 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.395 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_alloc": { 2024-09-21T09:31:06.395 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 5440, 2024-09-21T09:31:06.395 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 68 2024-09-21T09:31:06.395 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.395 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_cache_data": { 2024-09-21T09:31:06.395 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 8192, 2024-09-21T09:31:06.395 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 2 2024-09-21T09:31:06.395 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.395 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_cache_meta": { 2024-09-21T09:31:06.395 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 53743, 2024-09-21T09:31:06.396 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 9742 2024-09-21T09:31:06.396 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.396 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_cache_onode": { 2024-09-21T09:31:06.396 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 133280, 2024-09-21T09:31:06.396 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 196 2024-09-21T09:31:06.396 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.396 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_cache_other": { 2024-09-21T09:31:06.396 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 10952, 2024-09-21T09:31:06.396 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 706 2024-09-21T09:31:06.396 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.396 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_fsck": { 2024-09-21T09:31:06.396 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.397 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.397 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.397 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_inline_bl": { 2024-09-21T09:31:06.397 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 31156, 2024-09-21T09:31:06.397 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 196 2024-09-21T09:31:06.397 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.397 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_txc": { 2024-09-21T09:31:06.397 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.398 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.398 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.398 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_writing": { 2024-09-21T09:31:06.398 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 380503, 2024-09-21T09:31:06.398 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 82 2024-09-21T09:31:06.398 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.398 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_writing_deferred": { 2024-09-21T09:31:06.398 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.398 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.398 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.398 INFO:tasks.workunit.client.0.smithi022.stdout: "buffer_anon": { 2024-09-21T09:31:06.398 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 2442382, 2024-09-21T09:31:06.398 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 196 2024-09-21T09:31:06.399 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.399 INFO:tasks.workunit.client.0.smithi022.stdout: "buffer_meta": { 2024-09-21T09:31:06.399 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 5544, 2024-09-21T09:31:06.399 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 63 2024-09-21T09:31:06.399 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.399 INFO:tasks.workunit.client.0.smithi022.stdout: "mds_co": { 2024-09-21T09:31:06.399 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.399 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.399 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.399 INFO:tasks.workunit.client.0.smithi022.stdout: "osd": { 2024-09-21T09:31:06.400 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 509696, 2024-09-21T09:31:06.400 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 44 2024-09-21T09:31:06.400 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.400 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_mapbl": { 2024-09-21T09:31:06.400 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.400 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.400 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.400 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_pglog": { 2024-09-21T09:31:06.400 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 63544, 2024-09-21T09:31:06.400 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 127 2024-09-21T09:31:06.400 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.401 INFO:tasks.workunit.client.0.smithi022.stdout: "osdmap": { 2024-09-21T09:31:06.401 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 236944, 2024-09-21T09:31:06.401 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 3552 2024-09-21T09:31:06.401 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.401 INFO:tasks.workunit.client.0.smithi022.stdout: "osdmap_mapping": { 2024-09-21T09:31:06.401 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.401 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.401 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.401 INFO:tasks.workunit.client.0.smithi022.stdout: "pgmap": { 2024-09-21T09:31:06.402 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.402 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.402 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.402 INFO:tasks.workunit.client.0.smithi022.stdout: "unittest_1": { 2024-09-21T09:31:06.402 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.402 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.402 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.402 INFO:tasks.workunit.client.0.smithi022.stdout: "unittest_2": { 2024-09-21T09:31:06.402 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.402 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.402 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.402 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.403 INFO:tasks.workunit.client.0.smithi022.stdout: "osd.1": { 2024-09-21T09:31:06.403 INFO:tasks.workunit.client.0.smithi022.stdout: "bloom_filter": { 2024-09-21T09:31:06.403 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.403 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.403 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.403 INFO:tasks.workunit.client.0.smithi022.stdout: "bluefs": { 2024-09-21T09:31:06.403 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 6304, 2024-09-21T09:31:06.403 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 115 2024-09-21T09:31:06.403 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.403 INFO:tasks.workunit.client.0.smithi022.stdout: "bluefs_file_reader": { 2024-09-21T09:31:06.403 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 16896, 2024-09-21T09:31:06.403 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 8 2024-09-21T09:31:06.404 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.404 INFO:tasks.workunit.client.0.smithi022.stdout: "bluefs_file_writer": { 2024-09-21T09:31:06.404 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 576, 2024-09-21T09:31:06.404 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 3 2024-09-21T09:31:06.404 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.404 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_Blob": { 2024-09-21T09:31:06.404 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 19152, 2024-09-21T09:31:06.404 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 171 2024-09-21T09:31:06.404 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.404 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_Buffer": { 2024-09-21T09:31:06.404 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 784, 2024-09-21T09:31:06.404 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 7 2024-09-21T09:31:06.405 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.405 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_Extent": { 2024-09-21T09:31:06.405 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 8208, 2024-09-21T09:31:06.405 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 171 2024-09-21T09:31:06.405 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.405 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_SharedBlob": { 2024-09-21T09:31:06.405 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 19152, 2024-09-21T09:31:06.405 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 171 2024-09-21T09:31:06.405 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.405 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_alloc": { 2024-09-21T09:31:06.405 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 4880, 2024-09-21T09:31:06.405 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 61 2024-09-21T09:31:06.406 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.406 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_cache_data": { 2024-09-21T09:31:06.406 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 8192, 2024-09-21T09:31:06.406 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 2 2024-09-21T09:31:06.406 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.406 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_cache_meta": { 2024-09-21T09:31:06.406 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 48508, 2024-09-21T09:31:06.406 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 8876 2024-09-21T09:31:06.406 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.406 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_cache_onode": { 2024-09-21T09:31:06.407 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 125800, 2024-09-21T09:31:06.407 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 185 2024-09-21T09:31:06.407 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.407 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_cache_other": { 2024-09-21T09:31:06.407 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 9172, 2024-09-21T09:31:06.407 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 537 2024-09-21T09:31:06.407 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.407 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_fsck": { 2024-09-21T09:31:06.407 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.407 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.407 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.408 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_inline_bl": { 2024-09-21T09:31:06.408 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 27400, 2024-09-21T09:31:06.408 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 185 2024-09-21T09:31:06.408 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.408 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_txc": { 2024-09-21T09:31:06.408 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 736, 2024-09-21T09:31:06.408 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 1 2024-09-21T09:31:06.408 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.408 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_writing": { 2024-09-21T09:31:06.408 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 447438, 2024-09-21T09:31:06.408 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 98 2024-09-21T09:31:06.408 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.408 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_writing_deferred": { 2024-09-21T09:31:06.409 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 12096, 2024-09-21T09:31:06.409 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 3 2024-09-21T09:31:06.409 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.409 INFO:tasks.workunit.client.0.smithi022.stdout: "buffer_anon": { 2024-09-21T09:31:06.409 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 2403228, 2024-09-21T09:31:06.409 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 178 2024-09-21T09:31:06.409 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.409 INFO:tasks.workunit.client.0.smithi022.stdout: "buffer_meta": { 2024-09-21T09:31:06.409 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 4840, 2024-09-21T09:31:06.409 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 55 2024-09-21T09:31:06.409 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.409 INFO:tasks.workunit.client.0.smithi022.stdout: "mds_co": { 2024-09-21T09:31:06.409 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.410 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.410 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.410 INFO:tasks.workunit.client.0.smithi022.stdout: "osd": { 2024-09-21T09:31:06.410 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 440192, 2024-09-21T09:31:06.410 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 38 2024-09-21T09:31:06.410 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.410 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_mapbl": { 2024-09-21T09:31:06.410 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.410 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.410 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.410 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_pglog": { 2024-09-21T09:31:06.410 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 8528, 2024-09-21T09:31:06.410 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 22 2024-09-21T09:31:06.410 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.411 INFO:tasks.workunit.client.0.smithi022.stdout: "osdmap": { 2024-09-21T09:31:06.411 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 236176, 2024-09-21T09:31:06.411 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 3516 2024-09-21T09:31:06.411 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.411 INFO:tasks.workunit.client.0.smithi022.stdout: "osdmap_mapping": { 2024-09-21T09:31:06.411 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.411 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.411 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.411 INFO:tasks.workunit.client.0.smithi022.stdout: "pgmap": { 2024-09-21T09:31:06.411 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.411 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.411 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.411 INFO:tasks.workunit.client.0.smithi022.stdout: "unittest_1": { 2024-09-21T09:31:06.411 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.412 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.412 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.412 INFO:tasks.workunit.client.0.smithi022.stdout: "unittest_2": { 2024-09-21T09:31:06.412 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.412 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.412 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.412 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.412 INFO:tasks.workunit.client.0.smithi022.stdout: "osd.2": { 2024-09-21T09:31:06.412 INFO:tasks.workunit.client.0.smithi022.stdout: "bloom_filter": { 2024-09-21T09:31:06.412 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.412 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.412 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.412 INFO:tasks.workunit.client.0.smithi022.stdout: "bluefs": { 2024-09-21T09:31:06.412 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 6304, 2024-09-21T09:31:06.412 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 115 2024-09-21T09:31:06.413 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.413 INFO:tasks.workunit.client.0.smithi022.stdout: "bluefs_file_reader": { 2024-09-21T09:31:06.413 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 16896, 2024-09-21T09:31:06.413 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 8 2024-09-21T09:31:06.413 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.413 INFO:tasks.workunit.client.0.smithi022.stdout: "bluefs_file_writer": { 2024-09-21T09:31:06.413 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 576, 2024-09-21T09:31:06.413 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 3 2024-09-21T09:31:06.413 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.413 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_Blob": { 2024-09-21T09:31:06.413 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 18816, 2024-09-21T09:31:06.413 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 168 2024-09-21T09:31:06.414 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.414 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_Buffer": { 2024-09-21T09:31:06.414 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 784, 2024-09-21T09:31:06.414 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 7 2024-09-21T09:31:06.414 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.414 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_Extent": { 2024-09-21T09:31:06.414 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 8064, 2024-09-21T09:31:06.414 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 168 2024-09-21T09:31:06.414 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.414 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_SharedBlob": { 2024-09-21T09:31:06.414 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 18816, 2024-09-21T09:31:06.414 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 168 2024-09-21T09:31:06.415 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.415 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_alloc": { 2024-09-21T09:31:06.415 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 4320, 2024-09-21T09:31:06.415 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 54 2024-09-21T09:31:06.415 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.415 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_cache_data": { 2024-09-21T09:31:06.415 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 8192, 2024-09-21T09:31:06.415 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 2 2024-09-21T09:31:06.415 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.415 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_cache_meta": { 2024-09-21T09:31:06.415 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 45210, 2024-09-21T09:31:06.415 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 8397 2024-09-21T09:31:06.416 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.416 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_cache_onode": { 2024-09-21T09:31:06.416 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 119000, 2024-09-21T09:31:06.416 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 175 2024-09-21T09:31:06.416 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.416 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_cache_other": { 2024-09-21T09:31:06.416 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 8216, 2024-09-21T09:31:06.416 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 523 2024-09-21T09:31:06.416 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.416 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_fsck": { 2024-09-21T09:31:06.416 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.416 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.416 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.417 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_inline_bl": { 2024-09-21T09:31:06.417 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 25344, 2024-09-21T09:31:06.417 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 175 2024-09-21T09:31:06.417 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.417 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_txc": { 2024-09-21T09:31:06.417 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 736, 2024-09-21T09:31:06.417 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 1 2024-09-21T09:31:06.417 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.417 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_writing": { 2024-09-21T09:31:06.417 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 369614, 2024-09-21T09:31:06.417 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 79 2024-09-21T09:31:06.417 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.418 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_writing_deferred": { 2024-09-21T09:31:06.418 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 12096, 2024-09-21T09:31:06.418 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 3 2024-09-21T09:31:06.418 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.418 INFO:tasks.workunit.client.0.smithi022.stdout: "buffer_anon": { 2024-09-21T09:31:06.418 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 2414434, 2024-09-21T09:31:06.418 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 179 2024-09-21T09:31:06.418 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.418 INFO:tasks.workunit.client.0.smithi022.stdout: "buffer_meta": { 2024-09-21T09:31:06.418 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 4224, 2024-09-21T09:31:06.418 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 48 2024-09-21T09:31:06.418 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.418 INFO:tasks.workunit.client.0.smithi022.stdout: "mds_co": { 2024-09-21T09:31:06.419 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.419 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.419 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.419 INFO:tasks.workunit.client.0.smithi022.stdout: "osd": { 2024-09-21T09:31:06.419 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 347520, 2024-09-21T09:31:06.419 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 30 2024-09-21T09:31:06.419 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.419 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_mapbl": { 2024-09-21T09:31:06.419 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.419 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.419 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.419 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_pglog": { 2024-09-21T09:31:06.419 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 8320, 2024-09-21T09:31:06.419 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 24 2024-09-21T09:31:06.420 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.420 INFO:tasks.workunit.client.0.smithi022.stdout: "osdmap": { 2024-09-21T09:31:06.420 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 236688, 2024-09-21T09:31:06.420 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 3540 2024-09-21T09:31:06.420 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.420 INFO:tasks.workunit.client.0.smithi022.stdout: "osdmap_mapping": { 2024-09-21T09:31:06.420 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.420 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.420 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.420 INFO:tasks.workunit.client.0.smithi022.stdout: "pgmap": { 2024-09-21T09:31:06.420 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.420 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.420 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.421 INFO:tasks.workunit.client.0.smithi022.stdout: "unittest_1": { 2024-09-21T09:31:06.421 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.421 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.421 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.421 INFO:tasks.workunit.client.0.smithi022.stdout: "unittest_2": { 2024-09-21T09:31:06.421 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.421 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.421 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.421 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.421 INFO:tasks.workunit.client.0.smithi022.stdout: "osd.3": { 2024-09-21T09:31:06.421 INFO:tasks.workunit.client.0.smithi022.stdout: "bloom_filter": { 2024-09-21T09:31:06.421 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.422 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.422 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.422 INFO:tasks.workunit.client.0.smithi022.stdout: "bluefs": { 2024-09-21T09:31:06.422 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 6304, 2024-09-21T09:31:06.422 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 115 2024-09-21T09:31:06.422 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.422 INFO:tasks.workunit.client.0.smithi022.stdout: "bluefs_file_reader": { 2024-09-21T09:31:06.422 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 16896, 2024-09-21T09:31:06.422 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 8 2024-09-21T09:31:06.422 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.422 INFO:tasks.workunit.client.0.smithi022.stdout: "bluefs_file_writer": { 2024-09-21T09:31:06.423 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 576, 2024-09-21T09:31:06.423 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 3 2024-09-21T09:31:06.423 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.423 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_Blob": { 2024-09-21T09:31:06.423 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 19376, 2024-09-21T09:31:06.423 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 173 2024-09-21T09:31:06.423 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.423 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_Buffer": { 2024-09-21T09:31:06.423 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 336, 2024-09-21T09:31:06.423 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 3 2024-09-21T09:31:06.423 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.423 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_Extent": { 2024-09-21T09:31:06.424 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 8304, 2024-09-21T09:31:06.424 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 173 2024-09-21T09:31:06.424 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.424 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_SharedBlob": { 2024-09-21T09:31:06.424 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 19376, 2024-09-21T09:31:06.424 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 173 2024-09-21T09:31:06.424 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.424 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_alloc": { 2024-09-21T09:31:06.424 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 3920, 2024-09-21T09:31:06.424 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 49 2024-09-21T09:31:06.424 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.424 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_cache_data": { 2024-09-21T09:31:06.424 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 4096, 2024-09-21T09:31:06.424 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 1 2024-09-21T09:31:06.425 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.425 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_cache_meta": { 2024-09-21T09:31:06.425 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 47228, 2024-09-21T09:31:06.425 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 8774 2024-09-21T09:31:06.425 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.425 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_cache_onode": { 2024-09-21T09:31:06.425 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 124440, 2024-09-21T09:31:06.425 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 183 2024-09-21T09:31:06.425 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.425 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_cache_other": { 2024-09-21T09:31:06.425 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 9132, 2024-09-21T09:31:06.425 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 542 2024-09-21T09:31:06.425 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.426 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_fsck": { 2024-09-21T09:31:06.426 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.426 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.426 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.426 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_inline_bl": { 2024-09-21T09:31:06.426 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 26552, 2024-09-21T09:31:06.426 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 183 2024-09-21T09:31:06.426 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.426 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_txc": { 2024-09-21T09:31:06.426 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 2208, 2024-09-21T09:31:06.426 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 3 2024-09-21T09:31:06.427 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.427 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_writing": { 2024-09-21T09:31:06.427 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 283359, 2024-09-21T09:31:06.427 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 58 2024-09-21T09:31:06.427 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.427 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_writing_deferred": { 2024-09-21T09:31:06.427 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 30637, 2024-09-21T09:31:06.427 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 9 2024-09-21T09:31:06.427 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.427 INFO:tasks.workunit.client.0.smithi022.stdout: "buffer_anon": { 2024-09-21T09:31:06.427 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 2445344, 2024-09-21T09:31:06.428 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 189 2024-09-21T09:31:06.428 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.428 INFO:tasks.workunit.client.0.smithi022.stdout: "buffer_meta": { 2024-09-21T09:31:06.428 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 4840, 2024-09-21T09:31:06.428 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 55 2024-09-21T09:31:06.428 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.428 INFO:tasks.workunit.client.0.smithi022.stdout: "mds_co": { 2024-09-21T09:31:06.428 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.428 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.428 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.428 INFO:tasks.workunit.client.0.smithi022.stdout: "osd": { 2024-09-21T09:31:06.428 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 428608, 2024-09-21T09:31:06.428 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 37 2024-09-21T09:31:06.428 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.429 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_mapbl": { 2024-09-21T09:31:06.429 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.429 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.429 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.429 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_pglog": { 2024-09-21T09:31:06.429 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 10400, 2024-09-21T09:31:06.429 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 20 2024-09-21T09:31:06.429 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.429 INFO:tasks.workunit.client.0.smithi022.stdout: "osdmap": { 2024-09-21T09:31:06.429 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 236176, 2024-09-21T09:31:06.429 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 3516 2024-09-21T09:31:06.429 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.429 INFO:tasks.workunit.client.0.smithi022.stdout: "osdmap_mapping": { 2024-09-21T09:31:06.430 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.430 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.430 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.430 INFO:tasks.workunit.client.0.smithi022.stdout: "pgmap": { 2024-09-21T09:31:06.430 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.430 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.430 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.430 INFO:tasks.workunit.client.0.smithi022.stdout: "unittest_1": { 2024-09-21T09:31:06.430 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.430 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.430 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.430 INFO:tasks.workunit.client.0.smithi022.stdout: "unittest_2": { 2024-09-21T09:31:06.430 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.430 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.431 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.431 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.431 INFO:tasks.workunit.client.0.smithi022.stdout: "osd.4": { 2024-09-21T09:31:06.431 INFO:tasks.workunit.client.0.smithi022.stdout: "bloom_filter": { 2024-09-21T09:31:06.431 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.431 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.431 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.431 INFO:tasks.workunit.client.0.smithi022.stdout: "bluefs": { 2024-09-21T09:31:06.431 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 6304, 2024-09-21T09:31:06.431 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 115 2024-09-21T09:31:06.431 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.431 INFO:tasks.workunit.client.0.smithi022.stdout: "bluefs_file_reader": { 2024-09-21T09:31:06.431 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 16896, 2024-09-21T09:31:06.431 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 8 2024-09-21T09:31:06.432 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.432 INFO:tasks.workunit.client.0.smithi022.stdout: "bluefs_file_writer": { 2024-09-21T09:31:06.432 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 576, 2024-09-21T09:31:06.432 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 3 2024-09-21T09:31:06.432 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.432 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_Blob": { 2024-09-21T09:31:06.432 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 19376, 2024-09-21T09:31:06.432 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 173 2024-09-21T09:31:06.432 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.432 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_Buffer": { 2024-09-21T09:31:06.432 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 784, 2024-09-21T09:31:06.432 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 7 2024-09-21T09:31:06.432 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.432 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_Extent": { 2024-09-21T09:31:06.433 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 8304, 2024-09-21T09:31:06.433 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 173 2024-09-21T09:31:06.433 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.433 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_SharedBlob": { 2024-09-21T09:31:06.433 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 19376, 2024-09-21T09:31:06.433 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 173 2024-09-21T09:31:06.433 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.433 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_alloc": { 2024-09-21T09:31:06.433 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 3440, 2024-09-21T09:31:06.433 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 43 2024-09-21T09:31:06.433 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.433 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_cache_data": { 2024-09-21T09:31:06.433 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 15086, 2024-09-21T09:31:06.433 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 3 2024-09-21T09:31:06.433 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.434 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_cache_meta": { 2024-09-21T09:31:06.434 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 48736, 2024-09-21T09:31:06.434 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 8972 2024-09-21T09:31:06.434 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.434 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_cache_onode": { 2024-09-21T09:31:06.434 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 127160, 2024-09-21T09:31:06.434 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 187 2024-09-21T09:31:06.434 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.434 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_cache_other": { 2024-09-21T09:31:06.434 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 9436, 2024-09-21T09:31:06.434 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 543 2024-09-21T09:31:06.434 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.434 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_fsck": { 2024-09-21T09:31:06.434 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.435 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.435 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.435 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_inline_bl": { 2024-09-21T09:31:06.435 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 27576, 2024-09-21T09:31:06.435 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 187 2024-09-21T09:31:06.435 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.435 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_txc": { 2024-09-21T09:31:06.435 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 736, 2024-09-21T09:31:06.435 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 1 2024-09-21T09:31:06.435 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.435 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_writing": { 2024-09-21T09:31:06.435 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 898068, 2024-09-21T09:31:06.435 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 66 2024-09-21T09:31:06.435 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.436 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_writing_deferred": { 2024-09-21T09:31:06.436 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 12096, 2024-09-21T09:31:06.436 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 3 2024-09-21T09:31:06.436 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.436 INFO:tasks.workunit.client.0.smithi022.stdout: "buffer_anon": { 2024-09-21T09:31:06.436 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 2405354, 2024-09-21T09:31:06.436 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 169 2024-09-21T09:31:06.436 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.436 INFO:tasks.workunit.client.0.smithi022.stdout: "buffer_meta": { 2024-09-21T09:31:06.436 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 5016, 2024-09-21T09:31:06.436 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 57 2024-09-21T09:31:06.436 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.436 INFO:tasks.workunit.client.0.smithi022.stdout: "mds_co": { 2024-09-21T09:31:06.436 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.436 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.437 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.437 INFO:tasks.workunit.client.0.smithi022.stdout: "osd": { 2024-09-21T09:31:06.437 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 463360, 2024-09-21T09:31:06.437 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 40 2024-09-21T09:31:06.437 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.437 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_mapbl": { 2024-09-21T09:31:06.437 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.437 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.437 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.437 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_pglog": { 2024-09-21T09:31:06.437 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 8944, 2024-09-21T09:31:06.437 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 18 2024-09-21T09:31:06.437 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.437 INFO:tasks.workunit.client.0.smithi022.stdout: "osdmap": { 2024-09-21T09:31:06.437 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 236432, 2024-09-21T09:31:06.438 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 3528 2024-09-21T09:31:06.438 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.438 INFO:tasks.workunit.client.0.smithi022.stdout: "osdmap_mapping": { 2024-09-21T09:31:06.438 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.438 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.438 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.438 INFO:tasks.workunit.client.0.smithi022.stdout: "pgmap": { 2024-09-21T09:31:06.438 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.438 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.438 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.438 INFO:tasks.workunit.client.0.smithi022.stdout: "unittest_1": { 2024-09-21T09:31:06.438 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.438 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.438 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.439 INFO:tasks.workunit.client.0.smithi022.stdout: "unittest_2": { 2024-09-21T09:31:06.439 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.439 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.439 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.439 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.439 INFO:tasks.workunit.client.0.smithi022.stdout: "osd.5": { 2024-09-21T09:31:06.439 INFO:tasks.workunit.client.0.smithi022.stdout: "bloom_filter": { 2024-09-21T09:31:06.439 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.439 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.439 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.439 INFO:tasks.workunit.client.0.smithi022.stdout: "bluefs": { 2024-09-21T09:31:06.439 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 6304, 2024-09-21T09:31:06.439 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 115 2024-09-21T09:31:06.439 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.440 INFO:tasks.workunit.client.0.smithi022.stdout: "bluefs_file_reader": { 2024-09-21T09:31:06.440 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 16896, 2024-09-21T09:31:06.440 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 8 2024-09-21T09:31:06.440 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.440 INFO:tasks.workunit.client.0.smithi022.stdout: "bluefs_file_writer": { 2024-09-21T09:31:06.440 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 576, 2024-09-21T09:31:06.440 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 3 2024-09-21T09:31:06.440 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.440 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_Blob": { 2024-09-21T09:31:06.440 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 19152, 2024-09-21T09:31:06.440 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 171 2024-09-21T09:31:06.440 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.440 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_Buffer": { 2024-09-21T09:31:06.440 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 224, 2024-09-21T09:31:06.441 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 2 2024-09-21T09:31:06.441 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.441 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_Extent": { 2024-09-21T09:31:06.441 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 8208, 2024-09-21T09:31:06.441 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 171 2024-09-21T09:31:06.441 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.441 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_SharedBlob": { 2024-09-21T09:31:06.441 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 19152, 2024-09-21T09:31:06.441 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 171 2024-09-21T09:31:06.441 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.441 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_alloc": { 2024-09-21T09:31:06.441 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 3040, 2024-09-21T09:31:06.441 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 38 2024-09-21T09:31:06.441 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.441 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_cache_data": { 2024-09-21T09:31:06.442 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.442 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.442 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.442 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_cache_meta": { 2024-09-21T09:31:06.442 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 49086, 2024-09-21T09:31:06.442 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 8963 2024-09-21T09:31:06.442 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.442 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_cache_onode": { 2024-09-21T09:31:06.442 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 127160, 2024-09-21T09:31:06.442 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 187 2024-09-21T09:31:06.442 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.442 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_cache_other": { 2024-09-21T09:31:06.442 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 9540, 2024-09-21T09:31:06.442 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 542 2024-09-21T09:31:06.443 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.443 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_fsck": { 2024-09-21T09:31:06.443 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.443 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.443 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.443 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_inline_bl": { 2024-09-21T09:31:06.443 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 27912, 2024-09-21T09:31:06.443 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 187 2024-09-21T09:31:06.443 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.443 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_txc": { 2024-09-21T09:31:06.443 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 2208, 2024-09-21T09:31:06.443 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 3 2024-09-21T09:31:06.443 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.443 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_writing": { 2024-09-21T09:31:06.443 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 830116, 2024-09-21T09:31:06.444 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 49 2024-09-21T09:31:06.444 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.444 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_writing_deferred": { 2024-09-21T09:31:06.444 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 30637, 2024-09-21T09:31:06.444 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 9 2024-09-21T09:31:06.444 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.444 INFO:tasks.workunit.client.0.smithi022.stdout: "buffer_anon": { 2024-09-21T09:31:06.444 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 2394760, 2024-09-21T09:31:06.444 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 164 2024-09-21T09:31:06.444 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.444 INFO:tasks.workunit.client.0.smithi022.stdout: "buffer_meta": { 2024-09-21T09:31:06.444 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 5456, 2024-09-21T09:31:06.444 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 62 2024-09-21T09:31:06.444 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.444 INFO:tasks.workunit.client.0.smithi022.stdout: "mds_co": { 2024-09-21T09:31:06.445 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.445 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.445 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.445 INFO:tasks.workunit.client.0.smithi022.stdout: "osd": { 2024-09-21T09:31:06.445 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 474944, 2024-09-21T09:31:06.445 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 41 2024-09-21T09:31:06.445 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.445 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_mapbl": { 2024-09-21T09:31:06.445 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.445 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.445 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.445 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_pglog": { 2024-09-21T09:31:06.445 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 13000, 2024-09-21T09:31:06.445 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 25 2024-09-21T09:31:06.446 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.446 INFO:tasks.workunit.client.0.smithi022.stdout: "osdmap": { 2024-09-21T09:31:06.446 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 236944, 2024-09-21T09:31:06.446 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 3552 2024-09-21T09:31:06.446 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.446 INFO:tasks.workunit.client.0.smithi022.stdout: "osdmap_mapping": { 2024-09-21T09:31:06.446 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.446 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.446 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.446 INFO:tasks.workunit.client.0.smithi022.stdout: "pgmap": { 2024-09-21T09:31:06.446 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.446 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.447 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.447 INFO:tasks.workunit.client.0.smithi022.stdout: "unittest_1": { 2024-09-21T09:31:06.447 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.447 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.447 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.447 INFO:tasks.workunit.client.0.smithi022.stdout: "unittest_2": { 2024-09-21T09:31:06.447 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.447 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.447 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.447 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.447 INFO:tasks.workunit.client.0.smithi022.stdout: "osd.6": { 2024-09-21T09:31:06.447 INFO:tasks.workunit.client.0.smithi022.stdout: "bloom_filter": { 2024-09-21T09:31:06.447 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.448 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.448 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.448 INFO:tasks.workunit.client.0.smithi022.stdout: "bluefs": { 2024-09-21T09:31:06.448 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 6304, 2024-09-21T09:31:06.448 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 115 2024-09-21T09:31:06.448 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.448 INFO:tasks.workunit.client.0.smithi022.stdout: "bluefs_file_reader": { 2024-09-21T09:31:06.448 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 16896, 2024-09-21T09:31:06.448 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 8 2024-09-21T09:31:06.448 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.448 INFO:tasks.workunit.client.0.smithi022.stdout: "bluefs_file_writer": { 2024-09-21T09:31:06.448 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 576, 2024-09-21T09:31:06.448 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 3 2024-09-21T09:31:06.449 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.449 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_Blob": { 2024-09-21T09:31:06.449 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 20160, 2024-09-21T09:31:06.449 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 180 2024-09-21T09:31:06.449 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.449 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_Buffer": { 2024-09-21T09:31:06.449 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 224, 2024-09-21T09:31:06.449 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 2 2024-09-21T09:31:06.449 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.449 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_Extent": { 2024-09-21T09:31:06.449 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 8640, 2024-09-21T09:31:06.450 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 180 2024-09-21T09:31:06.450 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.450 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_SharedBlob": { 2024-09-21T09:31:06.450 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 20160, 2024-09-21T09:31:06.450 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 180 2024-09-21T09:31:06.450 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.450 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_alloc": { 2024-09-21T09:31:06.450 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 2720, 2024-09-21T09:31:06.450 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 34 2024-09-21T09:31:06.450 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.450 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_cache_data": { 2024-09-21T09:31:06.450 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 15086, 2024-09-21T09:31:06.450 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 3 2024-09-21T09:31:06.451 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.451 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_cache_meta": { 2024-09-21T09:31:06.451 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 48793, 2024-09-21T09:31:06.451 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 9206 2024-09-21T09:31:06.451 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.451 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_cache_onode": { 2024-09-21T09:31:06.451 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 125800, 2024-09-21T09:31:06.451 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 185 2024-09-21T09:31:06.451 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.451 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_cache_other": { 2024-09-21T09:31:06.451 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 10564, 2024-09-21T09:31:06.451 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 699 2024-09-21T09:31:06.451 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.451 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_fsck": { 2024-09-21T09:31:06.451 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.452 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.452 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.452 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_inline_bl": { 2024-09-21T09:31:06.452 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 28428, 2024-09-21T09:31:06.452 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 185 2024-09-21T09:31:06.452 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.452 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_txc": { 2024-09-21T09:31:06.452 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.452 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.452 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.452 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_writing": { 2024-09-21T09:31:06.452 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 214192, 2024-09-21T09:31:06.452 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 42 2024-09-21T09:31:06.453 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.453 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_writing_deferred": { 2024-09-21T09:31:06.453 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.453 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.453 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.453 INFO:tasks.workunit.client.0.smithi022.stdout: "buffer_anon": { 2024-09-21T09:31:06.453 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 2396044, 2024-09-21T09:31:06.453 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 163 2024-09-21T09:31:06.453 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.453 INFO:tasks.workunit.client.0.smithi022.stdout: "buffer_meta": { 2024-09-21T09:31:06.453 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 4664, 2024-09-21T09:31:06.453 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 53 2024-09-21T09:31:06.453 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.453 INFO:tasks.workunit.client.0.smithi022.stdout: "mds_co": { 2024-09-21T09:31:06.454 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.454 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.454 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.454 INFO:tasks.workunit.client.0.smithi022.stdout: "osd": { 2024-09-21T09:31:06.454 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 451776, 2024-09-21T09:31:06.454 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 39 2024-09-21T09:31:06.454 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.454 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_mapbl": { 2024-09-21T09:31:06.454 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.454 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.454 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.454 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_pglog": { 2024-09-21T09:31:06.454 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 62712, 2024-09-21T09:31:06.454 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 131 2024-09-21T09:31:06.455 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.455 INFO:tasks.workunit.client.0.smithi022.stdout: "osdmap": { 2024-09-21T09:31:06.455 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 236944, 2024-09-21T09:31:06.455 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 3552 2024-09-21T09:31:06.455 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.455 INFO:tasks.workunit.client.0.smithi022.stdout: "osdmap_mapping": { 2024-09-21T09:31:06.455 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.455 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.455 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.455 INFO:tasks.workunit.client.0.smithi022.stdout: "pgmap": { 2024-09-21T09:31:06.455 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.455 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.455 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.455 INFO:tasks.workunit.client.0.smithi022.stdout: "unittest_1": { 2024-09-21T09:31:06.456 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.456 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.456 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.456 INFO:tasks.workunit.client.0.smithi022.stdout: "unittest_2": { 2024-09-21T09:31:06.456 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.456 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.456 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.456 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.456 INFO:tasks.workunit.client.0.smithi022.stdout: "osd.7": { 2024-09-21T09:31:06.456 INFO:tasks.workunit.client.0.smithi022.stdout: "bloom_filter": { 2024-09-21T09:31:06.456 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.456 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.456 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.456 INFO:tasks.workunit.client.0.smithi022.stdout: "bluefs": { 2024-09-21T09:31:06.456 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 6304, 2024-09-21T09:31:06.457 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 115 2024-09-21T09:31:06.457 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.457 INFO:tasks.workunit.client.0.smithi022.stdout: "bluefs_file_reader": { 2024-09-21T09:31:06.457 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 16896, 2024-09-21T09:31:06.457 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 8 2024-09-21T09:31:06.457 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.457 INFO:tasks.workunit.client.0.smithi022.stdout: "bluefs_file_writer": { 2024-09-21T09:31:06.457 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 576, 2024-09-21T09:31:06.457 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 3 2024-09-21T09:31:06.457 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.457 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_Blob": { 2024-09-21T09:31:06.457 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 20160, 2024-09-21T09:31:06.457 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 180 2024-09-21T09:31:06.457 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.458 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_Buffer": { 2024-09-21T09:31:06.458 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 1232, 2024-09-21T09:31:06.458 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 11 2024-09-21T09:31:06.458 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.458 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_Extent": { 2024-09-21T09:31:06.458 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 8640, 2024-09-21T09:31:06.458 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 180 2024-09-21T09:31:06.458 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.458 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_SharedBlob": { 2024-09-21T09:31:06.458 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 20160, 2024-09-21T09:31:06.458 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 180 2024-09-21T09:31:06.458 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.458 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_alloc": { 2024-09-21T09:31:06.458 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 2240, 2024-09-21T09:31:06.459 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 28 2024-09-21T09:31:06.459 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.459 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_cache_data": { 2024-09-21T09:31:06.459 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 286720, 2024-09-21T09:31:06.459 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 11 2024-09-21T09:31:06.459 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.459 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_cache_meta": { 2024-09-21T09:31:06.459 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 52758, 2024-09-21T09:31:06.459 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 9697 2024-09-21T09:31:06.459 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.459 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_cache_onode": { 2024-09-21T09:31:06.459 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 132600, 2024-09-21T09:31:06.459 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 195 2024-09-21T09:31:06.459 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.460 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_cache_other": { 2024-09-21T09:31:06.460 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 11348, 2024-09-21T09:31:06.460 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 706 2024-09-21T09:31:06.460 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.460 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_fsck": { 2024-09-21T09:31:06.460 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.460 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.460 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.460 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_inline_bl": { 2024-09-21T09:31:06.460 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 30988, 2024-09-21T09:31:06.460 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 195 2024-09-21T09:31:06.460 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.460 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_txc": { 2024-09-21T09:31:06.460 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.461 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.461 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.461 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_writing": { 2024-09-21T09:31:06.461 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 448030, 2024-09-21T09:31:06.461 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 36 2024-09-21T09:31:06.461 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.461 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_writing_deferred": { 2024-09-21T09:31:06.461 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.461 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.461 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.461 INFO:tasks.workunit.client.0.smithi022.stdout: "buffer_anon": { 2024-09-21T09:31:06.461 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 2428010, 2024-09-21T09:31:06.461 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 218 2024-09-21T09:31:06.461 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.462 INFO:tasks.workunit.client.0.smithi022.stdout: "buffer_meta": { 2024-09-21T09:31:06.462 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 6424, 2024-09-21T09:31:06.462 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 73 2024-09-21T09:31:06.462 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.462 INFO:tasks.workunit.client.0.smithi022.stdout: "mds_co": { 2024-09-21T09:31:06.462 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.462 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.462 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.462 INFO:tasks.workunit.client.0.smithi022.stdout: "osd": { 2024-09-21T09:31:06.462 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 532864, 2024-09-21T09:31:06.462 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 46 2024-09-21T09:31:06.462 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.462 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_mapbl": { 2024-09-21T09:31:06.463 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.463 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.463 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.463 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_pglog": { 2024-09-21T09:31:06.463 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 64064, 2024-09-21T09:31:06.463 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 148 2024-09-21T09:31:06.463 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.463 INFO:tasks.workunit.client.0.smithi022.stdout: "osdmap": { 2024-09-21T09:31:06.463 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 236176, 2024-09-21T09:31:06.463 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 3516 2024-09-21T09:31:06.463 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.463 INFO:tasks.workunit.client.0.smithi022.stdout: "osdmap_mapping": { 2024-09-21T09:31:06.463 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.463 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.464 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.464 INFO:tasks.workunit.client.0.smithi022.stdout: "pgmap": { 2024-09-21T09:31:06.464 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.464 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.464 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.464 INFO:tasks.workunit.client.0.smithi022.stdout: "unittest_1": { 2024-09-21T09:31:06.464 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.464 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.464 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.464 INFO:tasks.workunit.client.0.smithi022.stdout: "unittest_2": { 2024-09-21T09:31:06.464 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:06.464 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:06.464 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.464 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.465 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.465 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.465 INFO:tasks.workunit.client.0.smithi022.stdout: "metadata": { 2024-09-21T09:31:06.465 INFO:tasks.workunit.client.0.smithi022.stdout: "mds": { 2024-09-21T09:31:06.465 INFO:tasks.workunit.client.0.smithi022.stdout: "arch": { 2024-09-21T09:31:06.465 INFO:tasks.workunit.client.0.smithi022.stdout: "x86_64": 2 2024-09-21T09:31:06.465 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.465 INFO:tasks.workunit.client.0.smithi022.stdout: "ceph_version": { 2024-09-21T09:31:06.465 INFO:tasks.workunit.client.0.smithi022.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 2 2024-09-21T09:31:06.465 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.465 INFO:tasks.workunit.client.0.smithi022.stdout: "cpu": { 2024-09-21T09:31:06.465 INFO:tasks.workunit.client.0.smithi022.stdout: "Intel(R) Xeon(R) CPU E5-1620 v3 @ 3.50GHz": 2 2024-09-21T09:31:06.465 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.465 INFO:tasks.workunit.client.0.smithi022.stdout: "distro": { 2024-09-21T09:31:06.465 INFO:tasks.workunit.client.0.smithi022.stdout: "centos": 2 2024-09-21T09:31:06.466 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.466 INFO:tasks.workunit.client.0.smithi022.stdout: "distro_description": { 2024-09-21T09:31:06.466 INFO:tasks.workunit.client.0.smithi022.stdout: "CentOS Stream 9": 2 2024-09-21T09:31:06.466 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.466 INFO:tasks.workunit.client.0.smithi022.stdout: "kernel_description": { 2024-09-21T09:31:06.466 INFO:tasks.workunit.client.0.smithi022.stdout: "#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024": 2 2024-09-21T09:31:06.466 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.466 INFO:tasks.workunit.client.0.smithi022.stdout: "kernel_version": { 2024-09-21T09:31:06.466 INFO:tasks.workunit.client.0.smithi022.stdout: "5.14.0-508.el9.x86_64": 2 2024-09-21T09:31:06.466 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.466 INFO:tasks.workunit.client.0.smithi022.stdout: "os": { 2024-09-21T09:31:06.466 INFO:tasks.workunit.client.0.smithi022.stdout: "Linux": 2 2024-09-21T09:31:06.466 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.467 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.467 INFO:tasks.workunit.client.0.smithi022.stdout: "mon": { 2024-09-21T09:31:06.467 INFO:tasks.workunit.client.0.smithi022.stdout: "arch": { 2024-09-21T09:31:06.467 INFO:tasks.workunit.client.0.smithi022.stdout: "x86_64": 3 2024-09-21T09:31:06.467 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.467 INFO:tasks.workunit.client.0.smithi022.stdout: "ceph_version": { 2024-09-21T09:31:06.467 INFO:tasks.workunit.client.0.smithi022.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 3 2024-09-21T09:31:06.467 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.467 INFO:tasks.workunit.client.0.smithi022.stdout: "cpu": { 2024-09-21T09:31:06.467 INFO:tasks.workunit.client.0.smithi022.stdout: "Intel(R) Xeon(R) CPU E5-1620 v3 @ 3.50GHz": 3 2024-09-21T09:31:06.467 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.467 INFO:tasks.workunit.client.0.smithi022.stdout: "distro": { 2024-09-21T09:31:06.467 INFO:tasks.workunit.client.0.smithi022.stdout: "centos": 3 2024-09-21T09:31:06.467 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.467 INFO:tasks.workunit.client.0.smithi022.stdout: "distro_description": { 2024-09-21T09:31:06.468 INFO:tasks.workunit.client.0.smithi022.stdout: "CentOS Stream 9": 3 2024-09-21T09:31:06.468 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.468 INFO:tasks.workunit.client.0.smithi022.stdout: "kernel_description": { 2024-09-21T09:31:06.468 INFO:tasks.workunit.client.0.smithi022.stdout: "#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024": 3 2024-09-21T09:31:06.468 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.468 INFO:tasks.workunit.client.0.smithi022.stdout: "kernel_version": { 2024-09-21T09:31:06.468 INFO:tasks.workunit.client.0.smithi022.stdout: "5.14.0-508.el9.x86_64": 3 2024-09-21T09:31:06.468 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.468 INFO:tasks.workunit.client.0.smithi022.stdout: "os": { 2024-09-21T09:31:06.468 INFO:tasks.workunit.client.0.smithi022.stdout: "Linux": 3 2024-09-21T09:31:06.468 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.468 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.468 INFO:tasks.workunit.client.0.smithi022.stdout: "osd": { 2024-09-21T09:31:06.468 INFO:tasks.workunit.client.0.smithi022.stdout: "arch": { 2024-09-21T09:31:06.469 INFO:tasks.workunit.client.0.smithi022.stdout: "x86_64": 8 2024-09-21T09:31:06.469 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.469 INFO:tasks.workunit.client.0.smithi022.stdout: "ceph_version": { 2024-09-21T09:31:06.469 INFO:tasks.workunit.client.0.smithi022.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 8 2024-09-21T09:31:06.469 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.469 INFO:tasks.workunit.client.0.smithi022.stdout: "cpu": { 2024-09-21T09:31:06.469 INFO:tasks.workunit.client.0.smithi022.stdout: "Intel(R) Xeon(R) CPU E5-1620 v3 @ 3.50GHz": 8 2024-09-21T09:31:06.469 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.469 INFO:tasks.workunit.client.0.smithi022.stdout: "distro": { 2024-09-21T09:31:06.469 INFO:tasks.workunit.client.0.smithi022.stdout: "centos": 8 2024-09-21T09:31:06.469 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.469 INFO:tasks.workunit.client.0.smithi022.stdout: "distro_description": { 2024-09-21T09:31:06.469 INFO:tasks.workunit.client.0.smithi022.stdout: "CentOS Stream 9": 8 2024-09-21T09:31:06.469 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.470 INFO:tasks.workunit.client.0.smithi022.stdout: "kernel_description": { 2024-09-21T09:31:06.470 INFO:tasks.workunit.client.0.smithi022.stdout: "#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024": 8 2024-09-21T09:31:06.470 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.470 INFO:tasks.workunit.client.0.smithi022.stdout: "kernel_version": { 2024-09-21T09:31:06.470 INFO:tasks.workunit.client.0.smithi022.stdout: "5.14.0-508.el9.x86_64": 8 2024-09-21T09:31:06.470 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.470 INFO:tasks.workunit.client.0.smithi022.stdout: "os": { 2024-09-21T09:31:06.470 INFO:tasks.workunit.client.0.smithi022.stdout: "Linux": 8 2024-09-21T09:31:06.470 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.470 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_objectstore": { 2024-09-21T09:31:06.470 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore": 8 2024-09-21T09:31:06.470 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.470 INFO:tasks.workunit.client.0.smithi022.stdout: "rotational": { 2024-09-21T09:31:06.470 INFO:tasks.workunit.client.0.smithi022.stdout: "0": 8 2024-09-21T09:31:06.471 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.471 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.471 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.471 INFO:tasks.workunit.client.0.smithi022.stdout: "mon": { 2024-09-21T09:31:06.471 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 3, 2024-09-21T09:31:06.471 INFO:tasks.workunit.client.0.smithi022.stdout: "features": { 2024-09-21T09:31:06.471 INFO:tasks.workunit.client.0.smithi022.stdout: "optional": [], 2024-09-21T09:31:06.471 INFO:tasks.workunit.client.0.smithi022.stdout: "persistent": [ 2024-09-21T09:31:06.471 INFO:tasks.workunit.client.0.smithi022.stdout: "kraken", 2024-09-21T09:31:06.471 INFO:tasks.workunit.client.0.smithi022.stdout: "luminous", 2024-09-21T09:31:06.471 INFO:tasks.workunit.client.0.smithi022.stdout: "mimic", 2024-09-21T09:31:06.471 INFO:tasks.workunit.client.0.smithi022.stdout: "osdmap-prune", 2024-09-21T09:31:06.471 INFO:tasks.workunit.client.0.smithi022.stdout: "nautilus", 2024-09-21T09:31:06.471 INFO:tasks.workunit.client.0.smithi022.stdout: "octopus", 2024-09-21T09:31:06.471 INFO:tasks.workunit.client.0.smithi022.stdout: "pacific", 2024-09-21T09:31:06.472 INFO:tasks.workunit.client.0.smithi022.stdout: "elector-pinging", 2024-09-21T09:31:06.472 INFO:tasks.workunit.client.0.smithi022.stdout: "quincy" 2024-09-21T09:31:06.472 INFO:tasks.workunit.client.0.smithi022.stdout: ] 2024-09-21T09:31:06.472 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.472 INFO:tasks.workunit.client.0.smithi022.stdout: "ipv4_addr_mons": 6, 2024-09-21T09:31:06.472 INFO:tasks.workunit.client.0.smithi022.stdout: "ipv6_addr_mons": 0, 2024-09-21T09:31:06.472 INFO:tasks.workunit.client.0.smithi022.stdout: "min_mon_release": 17, 2024-09-21T09:31:06.472 INFO:tasks.workunit.client.0.smithi022.stdout: "v1_addr_mons": 3, 2024-09-21T09:31:06.472 INFO:tasks.workunit.client.0.smithi022.stdout: "v2_addr_mons": 3 2024-09-21T09:31:06.472 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.472 INFO:tasks.workunit.client.0.smithi022.stdout: "organization": "ceph-qa", 2024-09-21T09:31:06.472 INFO:tasks.workunit.client.0.smithi022.stdout: "osd": { 2024-09-21T09:31:06.472 INFO:tasks.workunit.client.0.smithi022.stdout: "cluster_network": false, 2024-09-21T09:31:06.472 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 8, 2024-09-21T09:31:06.473 INFO:tasks.workunit.client.0.smithi022.stdout: "require_min_compat_client": "luminous", 2024-09-21T09:31:06.473 INFO:tasks.workunit.client.0.smithi022.stdout: "require_osd_release": "quincy" 2024-09-21T09:31:06.473 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.473 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_perf_histograms": [ 2024-09-21T09:31:06.473 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:06.473 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-21T09:31:06.473 INFO:tasks.workunit.client.0.smithi022.stdout: "axes": [ 2024-09-21T09:31:06.473 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:06.473 INFO:tasks.workunit.client.0.smithi022.stdout: "buckets": 32, 2024-09-21T09:31:06.473 INFO:tasks.workunit.client.0.smithi022.stdout: "min": 0, 2024-09-21T09:31:06.473 INFO:tasks.workunit.client.0.smithi022.stdout: "name": "Latency (usec)", 2024-09-21T09:31:06.473 INFO:tasks.workunit.client.0.smithi022.stdout: "quant_size": 100000, 2024-09-21T09:31:06.473 INFO:tasks.workunit.client.0.smithi022.stdout: "ranges": [ 2024-09-21T09:31:06.473 INFO:tasks.workunit.client.0.smithi022.stdout: "[None, -1]", 2024-09-21T09:31:06.474 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 99999]", 2024-09-21T09:31:06.474 INFO:tasks.workunit.client.0.smithi022.stdout: "[100000, 199999]", 2024-09-21T09:31:06.474 INFO:tasks.workunit.client.0.smithi022.stdout: "[200000, 399999]", 2024-09-21T09:31:06.474 INFO:tasks.workunit.client.0.smithi022.stdout: "[400000, 799999]", 2024-09-21T09:31:06.474 INFO:tasks.workunit.client.0.smithi022.stdout: "[800000, 1599999]", 2024-09-21T09:31:06.474 INFO:tasks.workunit.client.0.smithi022.stdout: "[1600000, 3199999]", 2024-09-21T09:31:06.474 INFO:tasks.workunit.client.0.smithi022.stdout: "[3200000, 6399999]", 2024-09-21T09:31:06.474 INFO:tasks.workunit.client.0.smithi022.stdout: "[6400000, 12799999]", 2024-09-21T09:31:06.474 INFO:tasks.workunit.client.0.smithi022.stdout: "[12800000, 25599999]", 2024-09-21T09:31:06.474 INFO:tasks.workunit.client.0.smithi022.stdout: "[25600000, 51199999]", 2024-09-21T09:31:06.474 INFO:tasks.workunit.client.0.smithi022.stdout: "[51200000, 102399999]", 2024-09-21T09:31:06.474 INFO:tasks.workunit.client.0.smithi022.stdout: "[102400000, 204799999]", 2024-09-21T09:31:06.475 INFO:tasks.workunit.client.0.smithi022.stdout: "[204800000, 409599999]", 2024-09-21T09:31:06.475 INFO:tasks.workunit.client.0.smithi022.stdout: "[409600000, 819199999]", 2024-09-21T09:31:06.475 INFO:tasks.workunit.client.0.smithi022.stdout: "[819200000, 1638399999]", 2024-09-21T09:31:06.475 INFO:tasks.workunit.client.0.smithi022.stdout: "[1638400000, 3276799999]", 2024-09-21T09:31:06.475 INFO:tasks.workunit.client.0.smithi022.stdout: "[3276800000, 6553599999]", 2024-09-21T09:31:06.475 INFO:tasks.workunit.client.0.smithi022.stdout: "[6553600000, 13107199999]", 2024-09-21T09:31:06.475 INFO:tasks.workunit.client.0.smithi022.stdout: "[13107200000, 26214399999]", 2024-09-21T09:31:06.475 INFO:tasks.workunit.client.0.smithi022.stdout: "[26214400000, 52428799999]", 2024-09-21T09:31:06.475 INFO:tasks.workunit.client.0.smithi022.stdout: "[52428800000, 104857599999]", 2024-09-21T09:31:06.475 INFO:tasks.workunit.client.0.smithi022.stdout: "[104857600000, 209715199999]", 2024-09-21T09:31:06.475 INFO:tasks.workunit.client.0.smithi022.stdout: "[209715200000, 419430399999]", 2024-09-21T09:31:06.475 INFO:tasks.workunit.client.0.smithi022.stdout: "[419430400000, 838860799999]", 2024-09-21T09:31:06.475 INFO:tasks.workunit.client.0.smithi022.stdout: "[838860800000, 1677721599999]", 2024-09-21T09:31:06.475 INFO:tasks.workunit.client.0.smithi022.stdout: "[1677721600000, 3355443199999]", 2024-09-21T09:31:06.476 INFO:tasks.workunit.client.0.smithi022.stdout: "[3355443200000, 6710886399999]", 2024-09-21T09:31:06.476 INFO:tasks.workunit.client.0.smithi022.stdout: "[6710886400000, 13421772799999]", 2024-09-21T09:31:06.476 INFO:tasks.workunit.client.0.smithi022.stdout: "[13421772800000, 26843545599999]", 2024-09-21T09:31:06.476 INFO:tasks.workunit.client.0.smithi022.stdout: "[26843545600000, 53687091199999]", 2024-09-21T09:31:06.476 INFO:tasks.workunit.client.0.smithi022.stdout: "[53687091200000, None]" 2024-09-21T09:31:06.476 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:06.476 INFO:tasks.workunit.client.0.smithi022.stdout: "scale_type": "log2" 2024-09-21T09:31:06.476 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.476 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:06.476 INFO:tasks.workunit.client.0.smithi022.stdout: "buckets": 32, 2024-09-21T09:31:06.476 INFO:tasks.workunit.client.0.smithi022.stdout: "min": 0, 2024-09-21T09:31:06.476 INFO:tasks.workunit.client.0.smithi022.stdout: "name": "Request size (bytes)", 2024-09-21T09:31:06.476 INFO:tasks.workunit.client.0.smithi022.stdout: "quant_size": 512, 2024-09-21T09:31:06.476 INFO:tasks.workunit.client.0.smithi022.stdout: "ranges": [ 2024-09-21T09:31:06.476 INFO:tasks.workunit.client.0.smithi022.stdout: "[None, -1]", 2024-09-21T09:31:06.477 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 511]", 2024-09-21T09:31:06.477 INFO:tasks.workunit.client.0.smithi022.stdout: "[512, 1023]", 2024-09-21T09:31:06.477 INFO:tasks.workunit.client.0.smithi022.stdout: "[1024, 2047]", 2024-09-21T09:31:06.477 INFO:tasks.workunit.client.0.smithi022.stdout: "[2048, 4095]", 2024-09-21T09:31:06.477 INFO:tasks.workunit.client.0.smithi022.stdout: "[4096, 8191]", 2024-09-21T09:31:06.477 INFO:tasks.workunit.client.0.smithi022.stdout: "[8192, 16383]", 2024-09-21T09:31:06.477 INFO:tasks.workunit.client.0.smithi022.stdout: "[16384, 32767]", 2024-09-21T09:31:06.477 INFO:tasks.workunit.client.0.smithi022.stdout: "[32768, 65535]", 2024-09-21T09:31:06.477 INFO:tasks.workunit.client.0.smithi022.stdout: "[65536, 131071]", 2024-09-21T09:31:06.477 INFO:tasks.workunit.client.0.smithi022.stdout: "[131072, 262143]", 2024-09-21T09:31:06.477 INFO:tasks.workunit.client.0.smithi022.stdout: "[262144, 524287]", 2024-09-21T09:31:06.477 INFO:tasks.workunit.client.0.smithi022.stdout: "[524288, 1048575]", 2024-09-21T09:31:06.477 INFO:tasks.workunit.client.0.smithi022.stdout: "[1048576, 2097151]", 2024-09-21T09:31:06.478 INFO:tasks.workunit.client.0.smithi022.stdout: "[2097152, 4194303]", 2024-09-21T09:31:06.478 INFO:tasks.workunit.client.0.smithi022.stdout: "[4194304, 8388607]", 2024-09-21T09:31:06.478 INFO:tasks.workunit.client.0.smithi022.stdout: "[8388608, 16777215]", 2024-09-21T09:31:06.478 INFO:tasks.workunit.client.0.smithi022.stdout: "[16777216, 33554431]", 2024-09-21T09:31:06.478 INFO:tasks.workunit.client.0.smithi022.stdout: "[33554432, 67108863]", 2024-09-21T09:31:06.478 INFO:tasks.workunit.client.0.smithi022.stdout: "[67108864, 134217727]", 2024-09-21T09:31:06.478 INFO:tasks.workunit.client.0.smithi022.stdout: "[134217728, 268435455]", 2024-09-21T09:31:06.478 INFO:tasks.workunit.client.0.smithi022.stdout: "[268435456, 536870911]", 2024-09-21T09:31:06.478 INFO:tasks.workunit.client.0.smithi022.stdout: "[536870912, 1073741823]", 2024-09-21T09:31:06.478 INFO:tasks.workunit.client.0.smithi022.stdout: "[1073741824, 2147483647]", 2024-09-21T09:31:06.478 INFO:tasks.workunit.client.0.smithi022.stdout: "[2147483648, 4294967295]", 2024-09-21T09:31:06.478 INFO:tasks.workunit.client.0.smithi022.stdout: "[4294967296, 8589934591]", 2024-09-21T09:31:06.479 INFO:tasks.workunit.client.0.smithi022.stdout: "[8589934592, 17179869183]", 2024-09-21T09:31:06.479 INFO:tasks.workunit.client.0.smithi022.stdout: "[17179869184, 34359738367]", 2024-09-21T09:31:06.479 INFO:tasks.workunit.client.0.smithi022.stdout: "[34359738368, 68719476735]", 2024-09-21T09:31:06.479 INFO:tasks.workunit.client.0.smithi022.stdout: "[68719476736, 137438953471]", 2024-09-21T09:31:06.479 INFO:tasks.workunit.client.0.smithi022.stdout: "[137438953472, 274877906943]", 2024-09-21T09:31:06.479 INFO:tasks.workunit.client.0.smithi022.stdout: "[274877906944, None]" 2024-09-21T09:31:06.479 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:06.479 INFO:tasks.workunit.client.0.smithi022.stdout: "scale_type": "log2" 2024-09-21T09:31:06.479 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.479 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:06.479 INFO:tasks.workunit.client.0.smithi022.stdout: "osds": [ 2024-09-21T09:31:06.479 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:06.480 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_id": 0, 2024-09-21T09:31:06.480 INFO:tasks.workunit.client.0.smithi022.stdout: "values": [ 2024-09-21T09:31:06.480 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.480 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.480 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.480 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.480 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.480 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.480 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.480 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.480 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.480 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.480 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.481 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.481 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.481 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.481 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.481 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.481 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.481 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.481 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.481 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.481 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.481 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.481 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.481 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.481 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.481 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.482 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.482 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.482 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.482 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.482 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.482 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:31:06.482 INFO:tasks.workunit.client.0.smithi022.stdout: ] 2024-09-21T09:31:06.482 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.482 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:06.482 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_id": 1, 2024-09-21T09:31:06.482 INFO:tasks.workunit.client.0.smithi022.stdout: "values": [ 2024-09-21T09:31:06.482 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.482 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.483 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.483 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.483 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.483 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.483 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.483 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.483 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.483 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.483 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.483 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.483 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.483 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.483 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.483 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.483 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.484 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.484 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.484 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.484 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.484 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.484 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.484 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.484 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.484 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.484 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.484 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.484 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.484 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.485 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.485 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:31:06.485 INFO:tasks.workunit.client.0.smithi022.stdout: ] 2024-09-21T09:31:06.485 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.485 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:06.485 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_id": 2, 2024-09-21T09:31:06.485 INFO:tasks.workunit.client.0.smithi022.stdout: "values": [ 2024-09-21T09:31:06.485 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.485 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.485 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.485 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.485 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.485 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.485 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.486 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.486 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.486 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.486 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.486 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.486 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.486 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.486 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.486 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.486 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.486 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.486 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.486 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.487 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.487 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.487 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.487 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.487 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.487 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.487 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.487 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.487 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.487 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.487 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.487 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:31:06.487 INFO:tasks.workunit.client.0.smithi022.stdout: ] 2024-09-21T09:31:06.488 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.488 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:06.488 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_id": 3, 2024-09-21T09:31:06.488 INFO:tasks.workunit.client.0.smithi022.stdout: "values": [ 2024-09-21T09:31:06.488 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.488 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.488 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.488 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.488 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.488 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.488 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.488 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.489 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.489 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.489 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.489 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.489 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.489 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.489 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.489 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.489 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.489 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.489 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.490 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.490 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.490 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.490 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.490 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.490 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.490 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.490 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.490 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.491 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.491 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.491 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.491 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:31:06.491 INFO:tasks.workunit.client.0.smithi022.stdout: ] 2024-09-21T09:31:06.491 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.491 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:06.491 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_id": 4, 2024-09-21T09:31:06.491 INFO:tasks.workunit.client.0.smithi022.stdout: "values": [ 2024-09-21T09:31:06.491 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.491 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.491 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.492 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.492 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.492 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.492 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.492 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.492 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.492 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.492 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.492 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.492 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.492 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.492 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.493 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.493 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.493 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.493 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.493 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.493 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.493 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.493 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.493 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.493 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.493 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.493 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.494 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.494 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.494 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.494 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.494 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:31:06.494 INFO:tasks.workunit.client.0.smithi022.stdout: ] 2024-09-21T09:31:06.494 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.494 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:06.494 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_id": 5, 2024-09-21T09:31:06.494 INFO:tasks.workunit.client.0.smithi022.stdout: "values": [ 2024-09-21T09:31:06.494 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.494 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.495 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.495 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.495 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.495 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.495 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.495 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.495 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.495 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.495 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.495 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.495 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.495 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.495 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.495 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.495 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.496 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.496 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.496 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.496 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.496 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.496 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.496 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.496 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.496 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.496 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.497 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.497 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.497 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.497 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.497 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:31:06.497 INFO:tasks.workunit.client.0.smithi022.stdout: ] 2024-09-21T09:31:06.497 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.497 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:06.497 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_id": 6, 2024-09-21T09:31:06.497 INFO:tasks.workunit.client.0.smithi022.stdout: "values": [ 2024-09-21T09:31:06.497 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.498 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.498 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.498 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.498 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.498 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.498 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.498 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.498 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.498 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.499 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.499 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.499 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.499 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.499 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.499 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.499 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.499 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.499 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.499 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.499 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.499 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.499 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.499 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.500 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.500 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.500 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.500 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.500 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.500 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.500 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.500 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:31:06.500 INFO:tasks.workunit.client.0.smithi022.stdout: ] 2024-09-21T09:31:06.500 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.501 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:06.501 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_id": 7, 2024-09-21T09:31:06.501 INFO:tasks.workunit.client.0.smithi022.stdout: "values": [ 2024-09-21T09:31:06.501 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.501 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.501 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.501 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 8, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.501 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.501 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.501 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.501 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.502 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.502 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.502 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.502 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.502 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.502 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.502 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.502 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.503 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.503 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.503 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.503 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.503 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.503 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.503 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.503 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.503 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.503 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.504 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.504 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.504 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.504 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.504 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.504 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:31:06.504 INFO:tasks.workunit.client.0.smithi022.stdout: ] 2024-09-21T09:31:06.504 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.504 INFO:tasks.workunit.client.0.smithi022.stdout: ] 2024-09-21T09:31:06.504 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.504 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-21T09:31:06.504 INFO:tasks.workunit.client.0.smithi022.stdout: "axes": [ 2024-09-21T09:31:06.504 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:06.505 INFO:tasks.workunit.client.0.smithi022.stdout: "buckets": 32, 2024-09-21T09:31:06.505 INFO:tasks.workunit.client.0.smithi022.stdout: "min": 0, 2024-09-21T09:31:06.505 INFO:tasks.workunit.client.0.smithi022.stdout: "name": "Latency (usec)", 2024-09-21T09:31:06.505 INFO:tasks.workunit.client.0.smithi022.stdout: "quant_size": 100000, 2024-09-21T09:31:06.505 INFO:tasks.workunit.client.0.smithi022.stdout: "ranges": [ 2024-09-21T09:31:06.505 INFO:tasks.workunit.client.0.smithi022.stdout: "[None, -1]", 2024-09-21T09:31:06.505 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 99999]", 2024-09-21T09:31:06.505 INFO:tasks.workunit.client.0.smithi022.stdout: "[100000, 199999]", 2024-09-21T09:31:06.505 INFO:tasks.workunit.client.0.smithi022.stdout: "[200000, 399999]", 2024-09-21T09:31:06.505 INFO:tasks.workunit.client.0.smithi022.stdout: "[400000, 799999]", 2024-09-21T09:31:06.505 INFO:tasks.workunit.client.0.smithi022.stdout: "[800000, 1599999]", 2024-09-21T09:31:06.505 INFO:tasks.workunit.client.0.smithi022.stdout: "[1600000, 3199999]", 2024-09-21T09:31:06.505 INFO:tasks.workunit.client.0.smithi022.stdout: "[3200000, 6399999]", 2024-09-21T09:31:06.506 INFO:tasks.workunit.client.0.smithi022.stdout: "[6400000, 12799999]", 2024-09-21T09:31:06.506 INFO:tasks.workunit.client.0.smithi022.stdout: "[12800000, 25599999]", 2024-09-21T09:31:06.506 INFO:tasks.workunit.client.0.smithi022.stdout: "[25600000, 51199999]", 2024-09-21T09:31:06.506 INFO:tasks.workunit.client.0.smithi022.stdout: "[51200000, 102399999]", 2024-09-21T09:31:06.506 INFO:tasks.workunit.client.0.smithi022.stdout: "[102400000, 204799999]", 2024-09-21T09:31:06.506 INFO:tasks.workunit.client.0.smithi022.stdout: "[204800000, 409599999]", 2024-09-21T09:31:06.506 INFO:tasks.workunit.client.0.smithi022.stdout: "[409600000, 819199999]", 2024-09-21T09:31:06.506 INFO:tasks.workunit.client.0.smithi022.stdout: "[819200000, 1638399999]", 2024-09-21T09:31:06.506 INFO:tasks.workunit.client.0.smithi022.stdout: "[1638400000, 3276799999]", 2024-09-21T09:31:06.506 INFO:tasks.workunit.client.0.smithi022.stdout: "[3276800000, 6553599999]", 2024-09-21T09:31:06.506 INFO:tasks.workunit.client.0.smithi022.stdout: "[6553600000, 13107199999]", 2024-09-21T09:31:06.506 INFO:tasks.workunit.client.0.smithi022.stdout: "[13107200000, 26214399999]", 2024-09-21T09:31:06.506 INFO:tasks.workunit.client.0.smithi022.stdout: "[26214400000, 52428799999]", 2024-09-21T09:31:06.506 INFO:tasks.workunit.client.0.smithi022.stdout: "[52428800000, 104857599999]", 2024-09-21T09:31:06.507 INFO:tasks.workunit.client.0.smithi022.stdout: "[104857600000, 209715199999]", 2024-09-21T09:31:06.507 INFO:tasks.workunit.client.0.smithi022.stdout: "[209715200000, 419430399999]", 2024-09-21T09:31:06.507 INFO:tasks.workunit.client.0.smithi022.stdout: "[419430400000, 838860799999]", 2024-09-21T09:31:06.507 INFO:tasks.workunit.client.0.smithi022.stdout: "[838860800000, 1677721599999]", 2024-09-21T09:31:06.507 INFO:tasks.workunit.client.0.smithi022.stdout: "[1677721600000, 3355443199999]", 2024-09-21T09:31:06.507 INFO:tasks.workunit.client.0.smithi022.stdout: "[3355443200000, 6710886399999]", 2024-09-21T09:31:06.507 INFO:tasks.workunit.client.0.smithi022.stdout: "[6710886400000, 13421772799999]", 2024-09-21T09:31:06.507 INFO:tasks.workunit.client.0.smithi022.stdout: "[13421772800000, 26843545599999]", 2024-09-21T09:31:06.507 INFO:tasks.workunit.client.0.smithi022.stdout: "[26843545600000, 53687091199999]", 2024-09-21T09:31:06.507 INFO:tasks.workunit.client.0.smithi022.stdout: "[53687091200000, None]" 2024-09-21T09:31:06.507 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:06.507 INFO:tasks.workunit.client.0.smithi022.stdout: "scale_type": "log2" 2024-09-21T09:31:06.507 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.507 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:06.508 INFO:tasks.workunit.client.0.smithi022.stdout: "buckets": 32, 2024-09-21T09:31:06.508 INFO:tasks.workunit.client.0.smithi022.stdout: "min": 0, 2024-09-21T09:31:06.508 INFO:tasks.workunit.client.0.smithi022.stdout: "name": "Request size (bytes)", 2024-09-21T09:31:06.508 INFO:tasks.workunit.client.0.smithi022.stdout: "quant_size": 512, 2024-09-21T09:31:06.508 INFO:tasks.workunit.client.0.smithi022.stdout: "ranges": [ 2024-09-21T09:31:06.508 INFO:tasks.workunit.client.0.smithi022.stdout: "[None, -1]", 2024-09-21T09:31:06.508 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 511]", 2024-09-21T09:31:06.508 INFO:tasks.workunit.client.0.smithi022.stdout: "[512, 1023]", 2024-09-21T09:31:06.508 INFO:tasks.workunit.client.0.smithi022.stdout: "[1024, 2047]", 2024-09-21T09:31:06.508 INFO:tasks.workunit.client.0.smithi022.stdout: "[2048, 4095]", 2024-09-21T09:31:06.508 INFO:tasks.workunit.client.0.smithi022.stdout: "[4096, 8191]", 2024-09-21T09:31:06.508 INFO:tasks.workunit.client.0.smithi022.stdout: "[8192, 16383]", 2024-09-21T09:31:06.508 INFO:tasks.workunit.client.0.smithi022.stdout: "[16384, 32767]", 2024-09-21T09:31:06.508 INFO:tasks.workunit.client.0.smithi022.stdout: "[32768, 65535]", 2024-09-21T09:31:06.509 INFO:tasks.workunit.client.0.smithi022.stdout: "[65536, 131071]", 2024-09-21T09:31:06.509 INFO:tasks.workunit.client.0.smithi022.stdout: "[131072, 262143]", 2024-09-21T09:31:06.509 INFO:tasks.workunit.client.0.smithi022.stdout: "[262144, 524287]", 2024-09-21T09:31:06.509 INFO:tasks.workunit.client.0.smithi022.stdout: "[524288, 1048575]", 2024-09-21T09:31:06.509 INFO:tasks.workunit.client.0.smithi022.stdout: "[1048576, 2097151]", 2024-09-21T09:31:06.509 INFO:tasks.workunit.client.0.smithi022.stdout: "[2097152, 4194303]", 2024-09-21T09:31:06.509 INFO:tasks.workunit.client.0.smithi022.stdout: "[4194304, 8388607]", 2024-09-21T09:31:06.509 INFO:tasks.workunit.client.0.smithi022.stdout: "[8388608, 16777215]", 2024-09-21T09:31:06.509 INFO:tasks.workunit.client.0.smithi022.stdout: "[16777216, 33554431]", 2024-09-21T09:31:06.509 INFO:tasks.workunit.client.0.smithi022.stdout: "[33554432, 67108863]", 2024-09-21T09:31:06.509 INFO:tasks.workunit.client.0.smithi022.stdout: "[67108864, 134217727]", 2024-09-21T09:31:06.509 INFO:tasks.workunit.client.0.smithi022.stdout: "[134217728, 268435455]", 2024-09-21T09:31:06.509 INFO:tasks.workunit.client.0.smithi022.stdout: "[268435456, 536870911]", 2024-09-21T09:31:06.509 INFO:tasks.workunit.client.0.smithi022.stdout: "[536870912, 1073741823]", 2024-09-21T09:31:06.510 INFO:tasks.workunit.client.0.smithi022.stdout: "[1073741824, 2147483647]", 2024-09-21T09:31:06.510 INFO:tasks.workunit.client.0.smithi022.stdout: "[2147483648, 4294967295]", 2024-09-21T09:31:06.510 INFO:tasks.workunit.client.0.smithi022.stdout: "[4294967296, 8589934591]", 2024-09-21T09:31:06.510 INFO:tasks.workunit.client.0.smithi022.stdout: "[8589934592, 17179869183]", 2024-09-21T09:31:06.510 INFO:tasks.workunit.client.0.smithi022.stdout: "[17179869184, 34359738367]", 2024-09-21T09:31:06.510 INFO:tasks.workunit.client.0.smithi022.stdout: "[34359738368, 68719476735]", 2024-09-21T09:31:06.510 INFO:tasks.workunit.client.0.smithi022.stdout: "[68719476736, 137438953471]", 2024-09-21T09:31:06.510 INFO:tasks.workunit.client.0.smithi022.stdout: "[137438953472, 274877906943]", 2024-09-21T09:31:06.510 INFO:tasks.workunit.client.0.smithi022.stdout: "[274877906944, None]" 2024-09-21T09:31:06.510 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:06.510 INFO:tasks.workunit.client.0.smithi022.stdout: "scale_type": "log2" 2024-09-21T09:31:06.511 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.511 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:06.511 INFO:tasks.workunit.client.0.smithi022.stdout: "osds": [ 2024-09-21T09:31:06.511 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:06.511 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_id": 0, 2024-09-21T09:31:06.511 INFO:tasks.workunit.client.0.smithi022.stdout: "values": [ 2024-09-21T09:31:06.511 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.511 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.511 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.511 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.511 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.512 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.512 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.512 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.512 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.512 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.512 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.512 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.512 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.512 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.512 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.512 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.512 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.513 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.513 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.513 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.513 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.513 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.513 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.513 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.513 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.514 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.514 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.514 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.514 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.514 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.514 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.514 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:31:06.514 INFO:tasks.workunit.client.0.smithi022.stdout: ] 2024-09-21T09:31:06.514 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.514 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:06.514 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_id": 1, 2024-09-21T09:31:06.514 INFO:tasks.workunit.client.0.smithi022.stdout: "values": [ 2024-09-21T09:31:06.515 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.515 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.515 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.515 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.515 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.515 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.515 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.515 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.515 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.515 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.516 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.516 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.516 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.516 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.516 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.516 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.516 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.516 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.516 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.516 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.516 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.516 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.516 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.517 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.517 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.517 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.517 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.517 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.517 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.517 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.517 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.517 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:31:06.517 INFO:tasks.workunit.client.0.smithi022.stdout: ] 2024-09-21T09:31:06.517 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.517 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:06.517 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_id": 2, 2024-09-21T09:31:06.518 INFO:tasks.workunit.client.0.smithi022.stdout: "values": [ 2024-09-21T09:31:06.518 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.518 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.518 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.518 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.518 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.518 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.518 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.518 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.518 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.518 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.518 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.518 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.518 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.519 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.519 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.519 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.519 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.519 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.519 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.519 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.519 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.519 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.519 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.519 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.519 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.520 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.520 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.520 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.520 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.520 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.520 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.520 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:31:06.520 INFO:tasks.workunit.client.0.smithi022.stdout: ] 2024-09-21T09:31:06.520 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.520 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:06.520 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_id": 3, 2024-09-21T09:31:06.520 INFO:tasks.workunit.client.0.smithi022.stdout: "values": [ 2024-09-21T09:31:06.520 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.521 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.521 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.521 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.521 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.521 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.521 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.521 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.521 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.521 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.521 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.521 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.521 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.521 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.522 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.522 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.522 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.522 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.522 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.522 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.522 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.522 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.522 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.522 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.522 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.522 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.523 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.523 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.523 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.523 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.523 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.523 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:31:06.523 INFO:tasks.workunit.client.0.smithi022.stdout: ] 2024-09-21T09:31:06.523 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.523 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:06.523 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_id": 4, 2024-09-21T09:31:06.523 INFO:tasks.workunit.client.0.smithi022.stdout: "values": [ 2024-09-21T09:31:06.523 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.523 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.523 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.523 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.524 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.524 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.524 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.524 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.524 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.524 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.524 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.524 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.524 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.524 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.524 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.525 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.525 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.525 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.525 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.525 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.525 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.525 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.525 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.525 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.525 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.525 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.525 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.525 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.526 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.526 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.526 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.526 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:31:06.526 INFO:tasks.workunit.client.0.smithi022.stdout: ] 2024-09-21T09:31:06.526 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.526 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:06.526 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_id": 5, 2024-09-21T09:31:06.526 INFO:tasks.workunit.client.0.smithi022.stdout: "values": [ 2024-09-21T09:31:06.526 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.526 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.526 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.526 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.527 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.527 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.527 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.527 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.527 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.527 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.527 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.527 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.527 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.527 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.527 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.527 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.527 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.528 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.528 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.528 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.528 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.528 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.528 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.528 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.528 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.528 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.528 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.528 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.528 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.528 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.528 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.528 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:31:06.529 INFO:tasks.workunit.client.0.smithi022.stdout: ] 2024-09-21T09:31:06.529 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.529 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:06.529 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_id": 6, 2024-09-21T09:31:06.529 INFO:tasks.workunit.client.0.smithi022.stdout: "values": [ 2024-09-21T09:31:06.529 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.529 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.529 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.529 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.529 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.529 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.529 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.529 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.529 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.530 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.530 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.530 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.530 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.530 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.530 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.530 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.530 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.530 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.530 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.530 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.530 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.530 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.531 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.531 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.531 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.531 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.531 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.531 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.531 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.531 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.531 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.531 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:31:06.531 INFO:tasks.workunit.client.0.smithi022.stdout: ] 2024-09-21T09:31:06.531 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.531 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:06.531 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_id": 7, 2024-09-21T09:31:06.532 INFO:tasks.workunit.client.0.smithi022.stdout: "values": [ 2024-09-21T09:31:06.532 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.532 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.532 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.532 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.532 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.532 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.532 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.532 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.532 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.532 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.532 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.532 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.532 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.532 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.533 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.533 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.533 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.533 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.533 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.533 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.533 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.533 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.533 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.533 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.533 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.533 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.533 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.533 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.534 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.534 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.534 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.534 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:31:06.534 INFO:tasks.workunit.client.0.smithi022.stdout: ] 2024-09-21T09:31:06.534 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.534 INFO:tasks.workunit.client.0.smithi022.stdout: ] 2024-09-21T09:31:06.534 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.534 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-21T09:31:06.534 INFO:tasks.workunit.client.0.smithi022.stdout: "axes": [ 2024-09-21T09:31:06.534 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:06.534 INFO:tasks.workunit.client.0.smithi022.stdout: "buckets": 32, 2024-09-21T09:31:06.534 INFO:tasks.workunit.client.0.smithi022.stdout: "min": 0, 2024-09-21T09:31:06.535 INFO:tasks.workunit.client.0.smithi022.stdout: "name": "Latency (usec)", 2024-09-21T09:31:06.535 INFO:tasks.workunit.client.0.smithi022.stdout: "quant_size": 100000, 2024-09-21T09:31:06.535 INFO:tasks.workunit.client.0.smithi022.stdout: "ranges": [ 2024-09-21T09:31:06.535 INFO:tasks.workunit.client.0.smithi022.stdout: "[None, -1]", 2024-09-21T09:31:06.535 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 99999]", 2024-09-21T09:31:06.535 INFO:tasks.workunit.client.0.smithi022.stdout: "[100000, 199999]", 2024-09-21T09:31:06.535 INFO:tasks.workunit.client.0.smithi022.stdout: "[200000, 399999]", 2024-09-21T09:31:06.535 INFO:tasks.workunit.client.0.smithi022.stdout: "[400000, 799999]", 2024-09-21T09:31:06.535 INFO:tasks.workunit.client.0.smithi022.stdout: "[800000, 1599999]", 2024-09-21T09:31:06.535 INFO:tasks.workunit.client.0.smithi022.stdout: "[1600000, 3199999]", 2024-09-21T09:31:06.535 INFO:tasks.workunit.client.0.smithi022.stdout: "[3200000, 6399999]", 2024-09-21T09:31:06.535 INFO:tasks.workunit.client.0.smithi022.stdout: "[6400000, 12799999]", 2024-09-21T09:31:06.535 INFO:tasks.workunit.client.0.smithi022.stdout: "[12800000, 25599999]", 2024-09-21T09:31:06.535 INFO:tasks.workunit.client.0.smithi022.stdout: "[25600000, 51199999]", 2024-09-21T09:31:06.535 INFO:tasks.workunit.client.0.smithi022.stdout: "[51200000, 102399999]", 2024-09-21T09:31:06.536 INFO:tasks.workunit.client.0.smithi022.stdout: "[102400000, 204799999]", 2024-09-21T09:31:06.536 INFO:tasks.workunit.client.0.smithi022.stdout: "[204800000, 409599999]", 2024-09-21T09:31:06.536 INFO:tasks.workunit.client.0.smithi022.stdout: "[409600000, 819199999]", 2024-09-21T09:31:06.536 INFO:tasks.workunit.client.0.smithi022.stdout: "[819200000, 1638399999]", 2024-09-21T09:31:06.536 INFO:tasks.workunit.client.0.smithi022.stdout: "[1638400000, 3276799999]", 2024-09-21T09:31:06.536 INFO:tasks.workunit.client.0.smithi022.stdout: "[3276800000, 6553599999]", 2024-09-21T09:31:06.536 INFO:tasks.workunit.client.0.smithi022.stdout: "[6553600000, 13107199999]", 2024-09-21T09:31:06.536 INFO:tasks.workunit.client.0.smithi022.stdout: "[13107200000, 26214399999]", 2024-09-21T09:31:06.536 INFO:tasks.workunit.client.0.smithi022.stdout: "[26214400000, 52428799999]", 2024-09-21T09:31:06.536 INFO:tasks.workunit.client.0.smithi022.stdout: "[52428800000, 104857599999]", 2024-09-21T09:31:06.536 INFO:tasks.workunit.client.0.smithi022.stdout: "[104857600000, 209715199999]", 2024-09-21T09:31:06.536 INFO:tasks.workunit.client.0.smithi022.stdout: "[209715200000, 419430399999]", 2024-09-21T09:31:06.536 INFO:tasks.workunit.client.0.smithi022.stdout: "[419430400000, 838860799999]", 2024-09-21T09:31:06.537 INFO:tasks.workunit.client.0.smithi022.stdout: "[838860800000, 1677721599999]", 2024-09-21T09:31:06.537 INFO:tasks.workunit.client.0.smithi022.stdout: "[1677721600000, 3355443199999]", 2024-09-21T09:31:06.537 INFO:tasks.workunit.client.0.smithi022.stdout: "[3355443200000, 6710886399999]", 2024-09-21T09:31:06.537 INFO:tasks.workunit.client.0.smithi022.stdout: "[6710886400000, 13421772799999]", 2024-09-21T09:31:06.537 INFO:tasks.workunit.client.0.smithi022.stdout: "[13421772800000, 26843545599999]", 2024-09-21T09:31:06.537 INFO:tasks.workunit.client.0.smithi022.stdout: "[26843545600000, 53687091199999]", 2024-09-21T09:31:06.537 INFO:tasks.workunit.client.0.smithi022.stdout: "[53687091200000, None]" 2024-09-21T09:31:06.537 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:06.537 INFO:tasks.workunit.client.0.smithi022.stdout: "scale_type": "log2" 2024-09-21T09:31:06.537 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.537 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:06.537 INFO:tasks.workunit.client.0.smithi022.stdout: "buckets": 32, 2024-09-21T09:31:06.537 INFO:tasks.workunit.client.0.smithi022.stdout: "min": 0, 2024-09-21T09:31:06.537 INFO:tasks.workunit.client.0.smithi022.stdout: "name": "Request size (bytes)", 2024-09-21T09:31:06.538 INFO:tasks.workunit.client.0.smithi022.stdout: "quant_size": 512, 2024-09-21T09:31:06.538 INFO:tasks.workunit.client.0.smithi022.stdout: "ranges": [ 2024-09-21T09:31:06.538 INFO:tasks.workunit.client.0.smithi022.stdout: "[None, -1]", 2024-09-21T09:31:06.538 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 511]", 2024-09-21T09:31:06.538 INFO:tasks.workunit.client.0.smithi022.stdout: "[512, 1023]", 2024-09-21T09:31:06.538 INFO:tasks.workunit.client.0.smithi022.stdout: "[1024, 2047]", 2024-09-21T09:31:06.538 INFO:tasks.workunit.client.0.smithi022.stdout: "[2048, 4095]", 2024-09-21T09:31:06.538 INFO:tasks.workunit.client.0.smithi022.stdout: "[4096, 8191]", 2024-09-21T09:31:06.538 INFO:tasks.workunit.client.0.smithi022.stdout: "[8192, 16383]", 2024-09-21T09:31:06.538 INFO:tasks.workunit.client.0.smithi022.stdout: "[16384, 32767]", 2024-09-21T09:31:06.538 INFO:tasks.workunit.client.0.smithi022.stdout: "[32768, 65535]", 2024-09-21T09:31:06.538 INFO:tasks.workunit.client.0.smithi022.stdout: "[65536, 131071]", 2024-09-21T09:31:06.539 INFO:tasks.workunit.client.0.smithi022.stdout: "[131072, 262143]", 2024-09-21T09:31:06.539 INFO:tasks.workunit.client.0.smithi022.stdout: "[262144, 524287]", 2024-09-21T09:31:06.539 INFO:tasks.workunit.client.0.smithi022.stdout: "[524288, 1048575]", 2024-09-21T09:31:06.539 INFO:tasks.workunit.client.0.smithi022.stdout: "[1048576, 2097151]", 2024-09-21T09:31:06.539 INFO:tasks.workunit.client.0.smithi022.stdout: "[2097152, 4194303]", 2024-09-21T09:31:06.539 INFO:tasks.workunit.client.0.smithi022.stdout: "[4194304, 8388607]", 2024-09-21T09:31:06.539 INFO:tasks.workunit.client.0.smithi022.stdout: "[8388608, 16777215]", 2024-09-21T09:31:06.539 INFO:tasks.workunit.client.0.smithi022.stdout: "[16777216, 33554431]", 2024-09-21T09:31:06.539 INFO:tasks.workunit.client.0.smithi022.stdout: "[33554432, 67108863]", 2024-09-21T09:31:06.539 INFO:tasks.workunit.client.0.smithi022.stdout: "[67108864, 134217727]", 2024-09-21T09:31:06.539 INFO:tasks.workunit.client.0.smithi022.stdout: "[134217728, 268435455]", 2024-09-21T09:31:06.539 INFO:tasks.workunit.client.0.smithi022.stdout: "[268435456, 536870911]", 2024-09-21T09:31:06.539 INFO:tasks.workunit.client.0.smithi022.stdout: "[536870912, 1073741823]", 2024-09-21T09:31:06.539 INFO:tasks.workunit.client.0.smithi022.stdout: "[1073741824, 2147483647]", 2024-09-21T09:31:06.539 INFO:tasks.workunit.client.0.smithi022.stdout: "[2147483648, 4294967295]", 2024-09-21T09:31:06.540 INFO:tasks.workunit.client.0.smithi022.stdout: "[4294967296, 8589934591]", 2024-09-21T09:31:06.540 INFO:tasks.workunit.client.0.smithi022.stdout: "[8589934592, 17179869183]", 2024-09-21T09:31:06.540 INFO:tasks.workunit.client.0.smithi022.stdout: "[17179869184, 34359738367]", 2024-09-21T09:31:06.540 INFO:tasks.workunit.client.0.smithi022.stdout: "[34359738368, 68719476735]", 2024-09-21T09:31:06.540 INFO:tasks.workunit.client.0.smithi022.stdout: "[68719476736, 137438953471]", 2024-09-21T09:31:06.540 INFO:tasks.workunit.client.0.smithi022.stdout: "[137438953472, 274877906943]", 2024-09-21T09:31:06.540 INFO:tasks.workunit.client.0.smithi022.stdout: "[274877906944, None]" 2024-09-21T09:31:06.540 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:06.540 INFO:tasks.workunit.client.0.smithi022.stdout: "scale_type": "log2" 2024-09-21T09:31:06.540 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.540 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:06.540 INFO:tasks.workunit.client.0.smithi022.stdout: "osds": [ 2024-09-21T09:31:06.540 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:06.540 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_id": 0, 2024-09-21T09:31:06.540 INFO:tasks.workunit.client.0.smithi022.stdout: "values": [ 2024-09-21T09:31:06.540 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.541 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.541 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.541 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.541 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.541 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.541 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.541 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.541 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.541 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.541 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.541 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.541 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.542 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.542 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.542 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.542 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.542 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.542 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.542 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.542 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.542 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.542 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.542 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.542 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.542 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.543 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.543 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.543 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.543 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.543 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.543 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:31:06.543 INFO:tasks.workunit.client.0.smithi022.stdout: ] 2024-09-21T09:31:06.543 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.543 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:06.543 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_id": 1, 2024-09-21T09:31:06.543 INFO:tasks.workunit.client.0.smithi022.stdout: "values": [ 2024-09-21T09:31:06.543 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.543 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.543 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.544 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.544 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.544 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.544 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.544 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.544 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.544 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.544 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.544 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.544 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.544 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.544 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.544 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.544 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.545 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.545 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.545 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.545 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.545 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.545 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.545 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.545 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.545 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.545 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.545 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.545 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.545 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.546 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.546 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:31:06.546 INFO:tasks.workunit.client.0.smithi022.stdout: ] 2024-09-21T09:31:06.546 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.546 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:06.546 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_id": 2, 2024-09-21T09:31:06.546 INFO:tasks.workunit.client.0.smithi022.stdout: "values": [ 2024-09-21T09:31:06.546 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.546 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.546 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.546 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.546 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.546 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.546 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.546 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.547 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.547 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.547 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.547 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.547 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.547 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.547 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.547 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.547 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.547 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.547 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.547 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.547 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.548 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.548 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.548 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.548 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.548 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.548 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.548 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.548 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.548 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.548 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.548 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:31:06.548 INFO:tasks.workunit.client.0.smithi022.stdout: ] 2024-09-21T09:31:06.548 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.548 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:06.549 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_id": 3, 2024-09-21T09:31:06.549 INFO:tasks.workunit.client.0.smithi022.stdout: "values": [ 2024-09-21T09:31:06.549 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.549 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.549 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.549 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.549 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.549 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.549 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.549 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.549 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.549 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.549 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.550 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.550 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.550 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.550 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.550 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.550 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.550 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.550 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.550 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.550 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.550 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.550 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.550 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.550 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.551 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.551 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.551 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.551 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.551 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.551 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.551 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:31:06.551 INFO:tasks.workunit.client.0.smithi022.stdout: ] 2024-09-21T09:31:06.551 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.551 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:06.551 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_id": 4, 2024-09-21T09:31:06.551 INFO:tasks.workunit.client.0.smithi022.stdout: "values": [ 2024-09-21T09:31:06.551 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.551 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.552 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.552 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.552 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.552 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.552 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.552 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.552 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.552 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.552 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.552 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.552 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.552 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.552 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.552 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.552 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.553 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.553 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.553 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.553 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.553 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.553 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.553 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.553 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.553 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.553 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.553 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.553 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.553 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.553 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.553 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:31:06.554 INFO:tasks.workunit.client.0.smithi022.stdout: ] 2024-09-21T09:31:06.554 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.554 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:06.554 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_id": 5, 2024-09-21T09:31:06.554 INFO:tasks.workunit.client.0.smithi022.stdout: "values": [ 2024-09-21T09:31:06.554 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.554 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.554 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.554 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.554 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.554 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.555 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.555 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.555 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.555 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.555 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.555 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.555 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.555 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.555 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.555 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.555 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.555 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.555 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.555 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.555 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.556 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.556 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.556 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.556 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.556 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.556 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.556 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.556 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.556 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.556 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.556 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:31:06.556 INFO:tasks.workunit.client.0.smithi022.stdout: ] 2024-09-21T09:31:06.556 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.556 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:06.557 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_id": 6, 2024-09-21T09:31:06.557 INFO:tasks.workunit.client.0.smithi022.stdout: "values": [ 2024-09-21T09:31:06.557 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.557 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.557 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.557 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.557 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.557 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.557 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.557 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.557 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.557 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.557 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.557 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.558 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.558 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.558 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.558 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.558 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.558 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.558 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.558 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.558 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.558 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.558 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.558 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.558 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.559 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.559 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.559 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.559 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.559 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.559 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.559 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:31:06.559 INFO:tasks.workunit.client.0.smithi022.stdout: ] 2024-09-21T09:31:06.559 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.559 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:06.559 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_id": 7, 2024-09-21T09:31:06.559 INFO:tasks.workunit.client.0.smithi022.stdout: "values": [ 2024-09-21T09:31:06.559 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.560 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.560 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.560 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.560 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.560 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.560 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.560 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.560 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.560 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.560 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.560 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.560 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.560 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.560 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.561 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.561 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.561 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.561 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.561 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.561 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.561 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.561 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.561 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.561 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.561 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.561 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.561 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.561 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.562 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.562 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.562 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:31:06.562 INFO:tasks.workunit.client.0.smithi022.stdout: ] 2024-09-21T09:31:06.562 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.562 INFO:tasks.workunit.client.0.smithi022.stdout: ] 2024-09-21T09:31:06.562 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.562 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-21T09:31:06.562 INFO:tasks.workunit.client.0.smithi022.stdout: "axes": [ 2024-09-21T09:31:06.562 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:06.562 INFO:tasks.workunit.client.0.smithi022.stdout: "buckets": 32, 2024-09-21T09:31:06.562 INFO:tasks.workunit.client.0.smithi022.stdout: "min": 0, 2024-09-21T09:31:06.562 INFO:tasks.workunit.client.0.smithi022.stdout: "name": "Latency (usec)", 2024-09-21T09:31:06.563 INFO:tasks.workunit.client.0.smithi022.stdout: "quant_size": 100000, 2024-09-21T09:31:06.563 INFO:tasks.workunit.client.0.smithi022.stdout: "ranges": [ 2024-09-21T09:31:06.563 INFO:tasks.workunit.client.0.smithi022.stdout: "[None, -1]", 2024-09-21T09:31:06.563 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 99999]", 2024-09-21T09:31:06.563 INFO:tasks.workunit.client.0.smithi022.stdout: "[100000, 199999]", 2024-09-21T09:31:06.563 INFO:tasks.workunit.client.0.smithi022.stdout: "[200000, 399999]", 2024-09-21T09:31:06.563 INFO:tasks.workunit.client.0.smithi022.stdout: "[400000, 799999]", 2024-09-21T09:31:06.563 INFO:tasks.workunit.client.0.smithi022.stdout: "[800000, 1599999]", 2024-09-21T09:31:06.563 INFO:tasks.workunit.client.0.smithi022.stdout: "[1600000, 3199999]", 2024-09-21T09:31:06.563 INFO:tasks.workunit.client.0.smithi022.stdout: "[3200000, 6399999]", 2024-09-21T09:31:06.563 INFO:tasks.workunit.client.0.smithi022.stdout: "[6400000, 12799999]", 2024-09-21T09:31:06.563 INFO:tasks.workunit.client.0.smithi022.stdout: "[12800000, 25599999]", 2024-09-21T09:31:06.563 INFO:tasks.workunit.client.0.smithi022.stdout: "[25600000, 51199999]", 2024-09-21T09:31:06.563 INFO:tasks.workunit.client.0.smithi022.stdout: "[51200000, 102399999]", 2024-09-21T09:31:06.564 INFO:tasks.workunit.client.0.smithi022.stdout: "[102400000, 204799999]", 2024-09-21T09:31:06.564 INFO:tasks.workunit.client.0.smithi022.stdout: "[204800000, 409599999]", 2024-09-21T09:31:06.564 INFO:tasks.workunit.client.0.smithi022.stdout: "[409600000, 819199999]", 2024-09-21T09:31:06.564 INFO:tasks.workunit.client.0.smithi022.stdout: "[819200000, 1638399999]", 2024-09-21T09:31:06.564 INFO:tasks.workunit.client.0.smithi022.stdout: "[1638400000, 3276799999]", 2024-09-21T09:31:06.564 INFO:tasks.workunit.client.0.smithi022.stdout: "[3276800000, 6553599999]", 2024-09-21T09:31:06.564 INFO:tasks.workunit.client.0.smithi022.stdout: "[6553600000, 13107199999]", 2024-09-21T09:31:06.564 INFO:tasks.workunit.client.0.smithi022.stdout: "[13107200000, 26214399999]", 2024-09-21T09:31:06.564 INFO:tasks.workunit.client.0.smithi022.stdout: "[26214400000, 52428799999]", 2024-09-21T09:31:06.564 INFO:tasks.workunit.client.0.smithi022.stdout: "[52428800000, 104857599999]", 2024-09-21T09:31:06.564 INFO:tasks.workunit.client.0.smithi022.stdout: "[104857600000, 209715199999]", 2024-09-21T09:31:06.564 INFO:tasks.workunit.client.0.smithi022.stdout: "[209715200000, 419430399999]", 2024-09-21T09:31:06.564 INFO:tasks.workunit.client.0.smithi022.stdout: "[419430400000, 838860799999]", 2024-09-21T09:31:06.564 INFO:tasks.workunit.client.0.smithi022.stdout: "[838860800000, 1677721599999]", 2024-09-21T09:31:06.564 INFO:tasks.workunit.client.0.smithi022.stdout: "[1677721600000, 3355443199999]", 2024-09-21T09:31:06.565 INFO:tasks.workunit.client.0.smithi022.stdout: "[3355443200000, 6710886399999]", 2024-09-21T09:31:06.565 INFO:tasks.workunit.client.0.smithi022.stdout: "[6710886400000, 13421772799999]", 2024-09-21T09:31:06.565 INFO:tasks.workunit.client.0.smithi022.stdout: "[13421772800000, 26843545599999]", 2024-09-21T09:31:06.565 INFO:tasks.workunit.client.0.smithi022.stdout: "[26843545600000, 53687091199999]", 2024-09-21T09:31:06.565 INFO:tasks.workunit.client.0.smithi022.stdout: "[53687091200000, None]" 2024-09-21T09:31:06.565 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:06.565 INFO:tasks.workunit.client.0.smithi022.stdout: "scale_type": "log2" 2024-09-21T09:31:06.565 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.565 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:06.565 INFO:tasks.workunit.client.0.smithi022.stdout: "buckets": 32, 2024-09-21T09:31:06.565 INFO:tasks.workunit.client.0.smithi022.stdout: "min": 0, 2024-09-21T09:31:06.565 INFO:tasks.workunit.client.0.smithi022.stdout: "name": "Request size (bytes)", 2024-09-21T09:31:06.565 INFO:tasks.workunit.client.0.smithi022.stdout: "quant_size": 512, 2024-09-21T09:31:06.565 INFO:tasks.workunit.client.0.smithi022.stdout: "ranges": [ 2024-09-21T09:31:06.566 INFO:tasks.workunit.client.0.smithi022.stdout: "[None, -1]", 2024-09-21T09:31:06.566 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 511]", 2024-09-21T09:31:06.566 INFO:tasks.workunit.client.0.smithi022.stdout: "[512, 1023]", 2024-09-21T09:31:06.566 INFO:tasks.workunit.client.0.smithi022.stdout: "[1024, 2047]", 2024-09-21T09:31:06.566 INFO:tasks.workunit.client.0.smithi022.stdout: "[2048, 4095]", 2024-09-21T09:31:06.566 INFO:tasks.workunit.client.0.smithi022.stdout: "[4096, 8191]", 2024-09-21T09:31:06.566 INFO:tasks.workunit.client.0.smithi022.stdout: "[8192, 16383]", 2024-09-21T09:31:06.566 INFO:tasks.workunit.client.0.smithi022.stdout: "[16384, 32767]", 2024-09-21T09:31:06.566 INFO:tasks.workunit.client.0.smithi022.stdout: "[32768, 65535]", 2024-09-21T09:31:06.566 INFO:tasks.workunit.client.0.smithi022.stdout: "[65536, 131071]", 2024-09-21T09:31:06.566 INFO:tasks.workunit.client.0.smithi022.stdout: "[131072, 262143]", 2024-09-21T09:31:06.566 INFO:tasks.workunit.client.0.smithi022.stdout: "[262144, 524287]", 2024-09-21T09:31:06.566 INFO:tasks.workunit.client.0.smithi022.stdout: "[524288, 1048575]", 2024-09-21T09:31:06.566 INFO:tasks.workunit.client.0.smithi022.stdout: "[1048576, 2097151]", 2024-09-21T09:31:06.566 INFO:tasks.workunit.client.0.smithi022.stdout: "[2097152, 4194303]", 2024-09-21T09:31:06.567 INFO:tasks.workunit.client.0.smithi022.stdout: "[4194304, 8388607]", 2024-09-21T09:31:06.567 INFO:tasks.workunit.client.0.smithi022.stdout: "[8388608, 16777215]", 2024-09-21T09:31:06.567 INFO:tasks.workunit.client.0.smithi022.stdout: "[16777216, 33554431]", 2024-09-21T09:31:06.567 INFO:tasks.workunit.client.0.smithi022.stdout: "[33554432, 67108863]", 2024-09-21T09:31:06.567 INFO:tasks.workunit.client.0.smithi022.stdout: "[67108864, 134217727]", 2024-09-21T09:31:06.567 INFO:tasks.workunit.client.0.smithi022.stdout: "[134217728, 268435455]", 2024-09-21T09:31:06.567 INFO:tasks.workunit.client.0.smithi022.stdout: "[268435456, 536870911]", 2024-09-21T09:31:06.567 INFO:tasks.workunit.client.0.smithi022.stdout: "[536870912, 1073741823]", 2024-09-21T09:31:06.567 INFO:tasks.workunit.client.0.smithi022.stdout: "[1073741824, 2147483647]", 2024-09-21T09:31:06.567 INFO:tasks.workunit.client.0.smithi022.stdout: "[2147483648, 4294967295]", 2024-09-21T09:31:06.567 INFO:tasks.workunit.client.0.smithi022.stdout: "[4294967296, 8589934591]", 2024-09-21T09:31:06.567 INFO:tasks.workunit.client.0.smithi022.stdout: "[8589934592, 17179869183]", 2024-09-21T09:31:06.567 INFO:tasks.workunit.client.0.smithi022.stdout: "[17179869184, 34359738367]", 2024-09-21T09:31:06.568 INFO:tasks.workunit.client.0.smithi022.stdout: "[34359738368, 68719476735]", 2024-09-21T09:31:06.568 INFO:tasks.workunit.client.0.smithi022.stdout: "[68719476736, 137438953471]", 2024-09-21T09:31:06.568 INFO:tasks.workunit.client.0.smithi022.stdout: "[137438953472, 274877906943]", 2024-09-21T09:31:06.568 INFO:tasks.workunit.client.0.smithi022.stdout: "[274877906944, None]" 2024-09-21T09:31:06.568 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:06.568 INFO:tasks.workunit.client.0.smithi022.stdout: "scale_type": "log2" 2024-09-21T09:31:06.568 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.568 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:06.568 INFO:tasks.workunit.client.0.smithi022.stdout: "osds": [ 2024-09-21T09:31:06.568 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:06.568 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_id": 0, 2024-09-21T09:31:06.568 INFO:tasks.workunit.client.0.smithi022.stdout: "values": [ 2024-09-21T09:31:06.568 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.568 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.569 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.569 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.569 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.569 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.569 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.569 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.569 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.569 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.569 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.569 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.569 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.569 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.569 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.569 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.570 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.570 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.570 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.570 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.570 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.570 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.570 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.570 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.570 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.570 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.570 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.570 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.570 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.570 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.571 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.571 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:31:06.571 INFO:tasks.workunit.client.0.smithi022.stdout: ] 2024-09-21T09:31:06.571 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.571 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:06.571 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_id": 1, 2024-09-21T09:31:06.571 INFO:tasks.workunit.client.0.smithi022.stdout: "values": [ 2024-09-21T09:31:06.571 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.571 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.571 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.571 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.571 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.571 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 6, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.572 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 7, 3, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.572 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 16, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.572 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 3, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.572 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.572 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.572 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.572 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.572 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.572 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.572 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.572 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.572 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.572 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.572 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.573 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.573 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.573 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.573 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.573 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.573 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.573 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.573 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.573 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.573 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.573 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.574 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:31:06.574 INFO:tasks.workunit.client.0.smithi022.stdout: ] 2024-09-21T09:31:06.574 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.574 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:06.574 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_id": 2, 2024-09-21T09:31:06.574 INFO:tasks.workunit.client.0.smithi022.stdout: "values": [ 2024-09-21T09:31:06.574 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.574 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.574 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.574 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.574 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.574 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.574 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.575 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.575 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.575 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.575 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.575 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.575 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.575 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.575 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.575 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.575 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.575 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.575 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.575 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.576 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.576 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.576 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.576 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.576 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.576 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.576 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.576 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.576 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.576 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.576 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.576 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:31:06.576 INFO:tasks.workunit.client.0.smithi022.stdout: ] 2024-09-21T09:31:06.577 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.577 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:06.577 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_id": 3, 2024-09-21T09:31:06.577 INFO:tasks.workunit.client.0.smithi022.stdout: "values": [ 2024-09-21T09:31:06.577 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.577 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.577 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.577 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.577 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.577 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.577 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.577 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.577 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.578 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.578 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.578 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.578 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.578 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.578 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.578 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.578 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.578 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.578 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.578 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.578 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.579 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.579 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.579 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.579 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.579 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.579 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.579 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.579 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.579 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.579 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.579 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:31:06.579 INFO:tasks.workunit.client.0.smithi022.stdout: ] 2024-09-21T09:31:06.580 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.580 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:06.580 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_id": 4, 2024-09-21T09:31:06.580 INFO:tasks.workunit.client.0.smithi022.stdout: "values": [ 2024-09-21T09:31:06.580 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.580 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.580 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.580 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.580 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.580 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.580 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.580 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.580 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.580 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.581 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.581 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.581 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.581 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.581 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.581 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.581 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.581 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.581 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.581 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.581 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.581 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.581 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.582 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.582 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.582 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.582 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.582 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.582 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.582 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.582 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.582 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:31:06.582 INFO:tasks.workunit.client.0.smithi022.stdout: ] 2024-09-21T09:31:06.582 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.583 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:06.583 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_id": 5, 2024-09-21T09:31:06.583 INFO:tasks.workunit.client.0.smithi022.stdout: "values": [ 2024-09-21T09:31:06.583 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.583 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.583 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.583 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.583 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.583 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.583 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.583 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.583 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.583 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.583 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.584 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.584 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.584 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.584 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.584 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.584 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.584 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.584 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.584 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.584 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.584 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.584 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.584 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.585 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.585 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.585 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.585 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.585 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.585 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.585 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.585 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:31:06.585 INFO:tasks.workunit.client.0.smithi022.stdout: ] 2024-09-21T09:31:06.585 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.585 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:06.585 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_id": 6, 2024-09-21T09:31:06.585 INFO:tasks.workunit.client.0.smithi022.stdout: "values": [ 2024-09-21T09:31:06.586 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.586 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.586 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.586 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.586 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.586 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.586 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.586 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.586 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.586 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.586 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.586 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.586 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.586 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.587 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.587 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.587 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.587 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.587 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.587 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.587 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.587 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.587 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.587 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.587 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.587 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.587 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.587 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.588 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.588 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.588 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.588 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:31:06.588 INFO:tasks.workunit.client.0.smithi022.stdout: ] 2024-09-21T09:31:06.588 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.588 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:06.588 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_id": 7, 2024-09-21T09:31:06.588 INFO:tasks.workunit.client.0.smithi022.stdout: "values": [ 2024-09-21T09:31:06.588 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.588 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.588 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.588 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.588 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.588 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.589 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.589 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.589 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.589 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.589 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.589 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.589 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.589 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.589 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.589 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.589 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.589 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.589 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.589 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.590 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.590 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.590 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.590 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.590 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.590 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.590 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.590 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.590 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.590 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.590 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:06.590 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:31:06.590 INFO:tasks.workunit.client.0.smithi022.stdout: ] 2024-09-21T09:31:06.591 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.591 INFO:tasks.workunit.client.0.smithi022.stdout: ] 2024-09-21T09:31:06.591 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.591 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.591 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:06.591 INFO:tasks.workunit.client.0.smithi022.stdout: "perf_counters": { 2024-09-21T09:31:06.591 INFO:tasks.workunit.client.0.smithi022.stdout: "mds.aa5e831b31d98e3163af5edfeb921f9af32463c5": { 2024-09-21T09:31:06.591 INFO:tasks.workunit.client.0.smithi022.stdout: "mds": { 2024-09-21T09:31:06.591 INFO:tasks.workunit.client.0.smithi022.stdout: "caps": { 2024-09-21T09:31:06.591 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.591 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.591 INFO:tasks.workunit.client.0.smithi022.stdout: "ceph_cap_op_flush_ack": { 2024-09-21T09:31:06.591 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.592 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.592 INFO:tasks.workunit.client.0.smithi022.stdout: "ceph_cap_op_flushsnap_ack": { 2024-09-21T09:31:06.592 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.592 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.592 INFO:tasks.workunit.client.0.smithi022.stdout: "ceph_cap_op_grant": { 2024-09-21T09:31:06.592 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.592 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.592 INFO:tasks.workunit.client.0.smithi022.stdout: "ceph_cap_op_revoke": { 2024-09-21T09:31:06.592 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.592 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.592 INFO:tasks.workunit.client.0.smithi022.stdout: "ceph_cap_op_trunc": { 2024-09-21T09:31:06.592 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.593 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.593 INFO:tasks.workunit.client.0.smithi022.stdout: "dir_commit": { 2024-09-21T09:31:06.593 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 12 2024-09-21T09:31:06.593 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.593 INFO:tasks.workunit.client.0.smithi022.stdout: "dir_fetch": { 2024-09-21T09:31:06.593 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.593 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.593 INFO:tasks.workunit.client.0.smithi022.stdout: "dir_merge": { 2024-09-21T09:31:06.593 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.593 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.593 INFO:tasks.workunit.client.0.smithi022.stdout: "dir_split": { 2024-09-21T09:31:06.593 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.593 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.594 INFO:tasks.workunit.client.0.smithi022.stdout: "exported_inodes": { 2024-09-21T09:31:06.594 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.594 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.594 INFO:tasks.workunit.client.0.smithi022.stdout: "forward": { 2024-09-21T09:31:06.594 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.594 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.594 INFO:tasks.workunit.client.0.smithi022.stdout: "handle_client_cap_release": { 2024-09-21T09:31:06.594 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.594 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.594 INFO:tasks.workunit.client.0.smithi022.stdout: "handle_client_caps": { 2024-09-21T09:31:06.594 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.594 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.595 INFO:tasks.workunit.client.0.smithi022.stdout: "handle_client_caps_dirty": { 2024-09-21T09:31:06.595 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.595 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.595 INFO:tasks.workunit.client.0.smithi022.stdout: "handle_inode_file_caps": { 2024-09-21T09:31:06.595 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.595 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.595 INFO:tasks.workunit.client.0.smithi022.stdout: "imported_inodes": { 2024-09-21T09:31:06.595 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.595 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.595 INFO:tasks.workunit.client.0.smithi022.stdout: "inodes": { 2024-09-21T09:31:06.595 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 10 2024-09-21T09:31:06.595 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.596 INFO:tasks.workunit.client.0.smithi022.stdout: "inodes_expired": { 2024-09-21T09:31:06.596 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.596 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.596 INFO:tasks.workunit.client.0.smithi022.stdout: "inodes_pinned": { 2024-09-21T09:31:06.596 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 10 2024-09-21T09:31:06.596 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.596 INFO:tasks.workunit.client.0.smithi022.stdout: "inodes_with_caps": { 2024-09-21T09:31:06.596 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.596 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.596 INFO:tasks.workunit.client.0.smithi022.stdout: "load_cent": { 2024-09-21T09:31:06.596 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.596 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.597 INFO:tasks.workunit.client.0.smithi022.stdout: "openino_dir_fetch": { 2024-09-21T09:31:06.597 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.597 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.597 INFO:tasks.workunit.client.0.smithi022.stdout: "process_request_cap_release": { 2024-09-21T09:31:06.597 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.597 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.597 INFO:tasks.workunit.client.0.smithi022.stdout: "reply_latency": { 2024-09-21T09:31:06.597 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 2, 2024-09-21T09:31:06.597 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 129410 2024-09-21T09:31:06.597 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.597 INFO:tasks.workunit.client.0.smithi022.stdout: "request": { 2024-09-21T09:31:06.597 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2 2024-09-21T09:31:06.598 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.598 INFO:tasks.workunit.client.0.smithi022.stdout: "root_rbytes": { 2024-09-21T09:31:06.598 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.598 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.598 INFO:tasks.workunit.client.0.smithi022.stdout: "root_rfiles": { 2024-09-21T09:31:06.598 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.598 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.598 INFO:tasks.workunit.client.0.smithi022.stdout: "root_rsnaps": { 2024-09-21T09:31:06.598 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.598 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.598 INFO:tasks.workunit.client.0.smithi022.stdout: "slow_reply": { 2024-09-21T09:31:06.598 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.599 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.599 INFO:tasks.workunit.client.0.smithi022.stdout: "subtrees": { 2024-09-21T09:31:06.599 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2 2024-09-21T09:31:06.599 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.599 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.599 INFO:tasks.workunit.client.0.smithi022.stdout: "mds_cache": { 2024-09-21T09:31:06.599 INFO:tasks.workunit.client.0.smithi022.stdout: "ireq_enqueue_scrub": { 2024-09-21T09:31:06.599 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.599 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.599 INFO:tasks.workunit.client.0.smithi022.stdout: "ireq_exportdir": { 2024-09-21T09:31:06.599 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.599 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.600 INFO:tasks.workunit.client.0.smithi022.stdout: "ireq_flush": { 2024-09-21T09:31:06.600 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.600 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.600 INFO:tasks.workunit.client.0.smithi022.stdout: "ireq_fragmentdir": { 2024-09-21T09:31:06.600 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.600 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.600 INFO:tasks.workunit.client.0.smithi022.stdout: "ireq_fragstats": { 2024-09-21T09:31:06.600 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.600 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.600 INFO:tasks.workunit.client.0.smithi022.stdout: "ireq_inodestats": { 2024-09-21T09:31:06.600 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.600 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.600 INFO:tasks.workunit.client.0.smithi022.stdout: "num_recovering_enqueued": { 2024-09-21T09:31:06.600 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.600 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.601 INFO:tasks.workunit.client.0.smithi022.stdout: "num_recovering_prioritized": { 2024-09-21T09:31:06.601 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.601 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.601 INFO:tasks.workunit.client.0.smithi022.stdout: "num_recovering_processing": { 2024-09-21T09:31:06.601 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.601 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.601 INFO:tasks.workunit.client.0.smithi022.stdout: "num_strays": { 2024-09-21T09:31:06.601 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.601 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.601 INFO:tasks.workunit.client.0.smithi022.stdout: "num_strays_delayed": { 2024-09-21T09:31:06.601 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.601 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.601 INFO:tasks.workunit.client.0.smithi022.stdout: "num_strays_enqueuing": { 2024-09-21T09:31:06.601 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.602 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.602 INFO:tasks.workunit.client.0.smithi022.stdout: "recovery_completed": { 2024-09-21T09:31:06.602 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.602 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.602 INFO:tasks.workunit.client.0.smithi022.stdout: "recovery_started": { 2024-09-21T09:31:06.602 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.602 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.602 INFO:tasks.workunit.client.0.smithi022.stdout: "strays_created": { 2024-09-21T09:31:06.602 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.602 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.602 INFO:tasks.workunit.client.0.smithi022.stdout: "strays_enqueued": { 2024-09-21T09:31:06.602 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.602 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.602 INFO:tasks.workunit.client.0.smithi022.stdout: "strays_migrated": { 2024-09-21T09:31:06.603 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.603 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.603 INFO:tasks.workunit.client.0.smithi022.stdout: "strays_reintegrated": { 2024-09-21T09:31:06.603 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.603 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.603 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.603 INFO:tasks.workunit.client.0.smithi022.stdout: "mds_log": { 2024-09-21T09:31:06.603 INFO:tasks.workunit.client.0.smithi022.stdout: "ev": { 2024-09-21T09:31:06.603 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 5 2024-09-21T09:31:06.603 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.603 INFO:tasks.workunit.client.0.smithi022.stdout: "evadd": { 2024-09-21T09:31:06.603 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 5 2024-09-21T09:31:06.603 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.603 INFO:tasks.workunit.client.0.smithi022.stdout: "evex": { 2024-09-21T09:31:06.604 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.604 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.604 INFO:tasks.workunit.client.0.smithi022.stdout: "evexd": { 2024-09-21T09:31:06.604 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.604 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.604 INFO:tasks.workunit.client.0.smithi022.stdout: "evexg": { 2024-09-21T09:31:06.604 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.604 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.604 INFO:tasks.workunit.client.0.smithi022.stdout: "evtrm": { 2024-09-21T09:31:06.604 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.604 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.604 INFO:tasks.workunit.client.0.smithi022.stdout: "jlat": { 2024-09-21T09:31:06.604 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 5, 2024-09-21T09:31:06.604 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 8856762 2024-09-21T09:31:06.604 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.605 INFO:tasks.workunit.client.0.smithi022.stdout: "replayed": { 2024-09-21T09:31:06.605 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.605 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.605 INFO:tasks.workunit.client.0.smithi022.stdout: "seg": { 2024-09-21T09:31:06.605 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1 2024-09-21T09:31:06.605 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.605 INFO:tasks.workunit.client.0.smithi022.stdout: "segadd": { 2024-09-21T09:31:06.605 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1 2024-09-21T09:31:06.605 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.605 INFO:tasks.workunit.client.0.smithi022.stdout: "segex": { 2024-09-21T09:31:06.605 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.605 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.605 INFO:tasks.workunit.client.0.smithi022.stdout: "segexd": { 2024-09-21T09:31:06.605 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.606 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.606 INFO:tasks.workunit.client.0.smithi022.stdout: "segexg": { 2024-09-21T09:31:06.606 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.606 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.606 INFO:tasks.workunit.client.0.smithi022.stdout: "segtrm": { 2024-09-21T09:31:06.606 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.606 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.606 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.606 INFO:tasks.workunit.client.0.smithi022.stdout: "mds_mem": { 2024-09-21T09:31:06.606 INFO:tasks.workunit.client.0.smithi022.stdout: "cap": { 2024-09-21T09:31:06.606 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.606 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.606 INFO:tasks.workunit.client.0.smithi022.stdout: "cap+": { 2024-09-21T09:31:06.606 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2 2024-09-21T09:31:06.607 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.607 INFO:tasks.workunit.client.0.smithi022.stdout: "cap-": { 2024-09-21T09:31:06.607 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2 2024-09-21T09:31:06.607 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.607 INFO:tasks.workunit.client.0.smithi022.stdout: "dir": { 2024-09-21T09:31:06.607 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 12 2024-09-21T09:31:06.607 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.607 INFO:tasks.workunit.client.0.smithi022.stdout: "dir+": { 2024-09-21T09:31:06.607 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 12 2024-09-21T09:31:06.607 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.607 INFO:tasks.workunit.client.0.smithi022.stdout: "dir-": { 2024-09-21T09:31:06.607 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.607 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.608 INFO:tasks.workunit.client.0.smithi022.stdout: "dn": { 2024-09-21T09:31:06.608 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 10 2024-09-21T09:31:06.608 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.608 INFO:tasks.workunit.client.0.smithi022.stdout: "dn+": { 2024-09-21T09:31:06.608 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 10 2024-09-21T09:31:06.608 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.608 INFO:tasks.workunit.client.0.smithi022.stdout: "dn-": { 2024-09-21T09:31:06.608 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.608 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.608 INFO:tasks.workunit.client.0.smithi022.stdout: "heap": { 2024-09-21T09:31:06.608 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 207068 2024-09-21T09:31:06.608 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.608 INFO:tasks.workunit.client.0.smithi022.stdout: "ino": { 2024-09-21T09:31:06.609 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 13 2024-09-21T09:31:06.609 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.609 INFO:tasks.workunit.client.0.smithi022.stdout: "ino+": { 2024-09-21T09:31:06.609 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 13 2024-09-21T09:31:06.609 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.609 INFO:tasks.workunit.client.0.smithi022.stdout: "ino-": { 2024-09-21T09:31:06.609 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.609 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.609 INFO:tasks.workunit.client.0.smithi022.stdout: "rss": { 2024-09-21T09:31:06.609 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 39020 2024-09-21T09:31:06.609 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.609 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.609 INFO:tasks.workunit.client.0.smithi022.stdout: "mds_server": { 2024-09-21T09:31:06.610 INFO:tasks.workunit.client.0.smithi022.stdout: "cap_acquisition_throttle": { 2024-09-21T09:31:06.610 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.610 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.610 INFO:tasks.workunit.client.0.smithi022.stdout: "cap_revoke_eviction": { 2024-09-21T09:31:06.610 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.610 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.610 INFO:tasks.workunit.client.0.smithi022.stdout: "handle_client_request": { 2024-09-21T09:31:06.610 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2 2024-09-21T09:31:06.610 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.610 INFO:tasks.workunit.client.0.smithi022.stdout: "handle_client_session": { 2024-09-21T09:31:06.610 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 16 2024-09-21T09:31:06.610 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.611 INFO:tasks.workunit.client.0.smithi022.stdout: "handle_peer_request": { 2024-09-21T09:31:06.611 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.611 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.611 INFO:tasks.workunit.client.0.smithi022.stdout: "req_create_latency": { 2024-09-21T09:31:06.611 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.611 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.611 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.611 INFO:tasks.workunit.client.0.smithi022.stdout: "req_getattr_latency": { 2024-09-21T09:31:06.611 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 2, 2024-09-21T09:31:06.611 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 129410 2024-09-21T09:31:06.611 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.611 INFO:tasks.workunit.client.0.smithi022.stdout: "req_getfilelock_latency": { 2024-09-21T09:31:06.611 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.611 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.612 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.612 INFO:tasks.workunit.client.0.smithi022.stdout: "req_getvxattr_latency": { 2024-09-21T09:31:06.612 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.612 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.612 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.612 INFO:tasks.workunit.client.0.smithi022.stdout: "req_link_latency": { 2024-09-21T09:31:06.612 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.612 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.612 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.612 INFO:tasks.workunit.client.0.smithi022.stdout: "req_lookup_latency": { 2024-09-21T09:31:06.612 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.612 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.612 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.613 INFO:tasks.workunit.client.0.smithi022.stdout: "req_lookuphash_latency": { 2024-09-21T09:31:06.613 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.613 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.613 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.613 INFO:tasks.workunit.client.0.smithi022.stdout: "req_lookupino_latency": { 2024-09-21T09:31:06.613 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.613 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.613 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.613 INFO:tasks.workunit.client.0.smithi022.stdout: "req_lookupname_latency": { 2024-09-21T09:31:06.613 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.613 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.613 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.613 INFO:tasks.workunit.client.0.smithi022.stdout: "req_lookupparent_latency": { 2024-09-21T09:31:06.613 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.614 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.614 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.614 INFO:tasks.workunit.client.0.smithi022.stdout: "req_lookupsnap_latency": { 2024-09-21T09:31:06.614 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.614 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.614 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.614 INFO:tasks.workunit.client.0.smithi022.stdout: "req_lssnap_latency": { 2024-09-21T09:31:06.614 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.614 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.614 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.614 INFO:tasks.workunit.client.0.smithi022.stdout: "req_mkdir_latency": { 2024-09-21T09:31:06.614 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.614 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.615 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.615 INFO:tasks.workunit.client.0.smithi022.stdout: "req_mknod_latency": { 2024-09-21T09:31:06.615 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.615 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.615 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.615 INFO:tasks.workunit.client.0.smithi022.stdout: "req_mksnap_latency": { 2024-09-21T09:31:06.615 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.615 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.615 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.616 INFO:tasks.workunit.client.0.smithi022.stdout: "req_open_latency": { 2024-09-21T09:31:06.616 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.616 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.616 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.616 INFO:tasks.workunit.client.0.smithi022.stdout: "req_readdir_latency": { 2024-09-21T09:31:06.616 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.616 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.616 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.616 INFO:tasks.workunit.client.0.smithi022.stdout: "req_rename_latency": { 2024-09-21T09:31:06.616 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.616 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.616 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.617 INFO:tasks.workunit.client.0.smithi022.stdout: "req_renamesnap_latency": { 2024-09-21T09:31:06.617 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.617 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.617 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.617 INFO:tasks.workunit.client.0.smithi022.stdout: "req_rmdir_latency": { 2024-09-21T09:31:06.617 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.617 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.617 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.617 INFO:tasks.workunit.client.0.smithi022.stdout: "req_rmsnap_latency": { 2024-09-21T09:31:06.617 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.617 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.617 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.617 INFO:tasks.workunit.client.0.smithi022.stdout: "req_rmxattr_latency": { 2024-09-21T09:31:06.618 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.618 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.618 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.618 INFO:tasks.workunit.client.0.smithi022.stdout: "req_setattr_latency": { 2024-09-21T09:31:06.618 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.618 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.618 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.618 INFO:tasks.workunit.client.0.smithi022.stdout: "req_setdirlayout_latency": { 2024-09-21T09:31:06.618 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.618 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.618 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.618 INFO:tasks.workunit.client.0.smithi022.stdout: "req_setfilelock_latency": { 2024-09-21T09:31:06.618 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.618 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.619 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.619 INFO:tasks.workunit.client.0.smithi022.stdout: "req_setlayout_latency": { 2024-09-21T09:31:06.619 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.619 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.619 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.619 INFO:tasks.workunit.client.0.smithi022.stdout: "req_setxattr_latency": { 2024-09-21T09:31:06.619 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.619 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.619 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.619 INFO:tasks.workunit.client.0.smithi022.stdout: "req_symlink_latency": { 2024-09-21T09:31:06.619 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.619 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.619 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.619 INFO:tasks.workunit.client.0.smithi022.stdout: "req_unlink_latency": { 2024-09-21T09:31:06.620 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.620 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.620 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.620 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.620 INFO:tasks.workunit.client.0.smithi022.stdout: "mds_sessions": { 2024-09-21T09:31:06.620 INFO:tasks.workunit.client.0.smithi022.stdout: "average_load": { 2024-09-21T09:31:06.620 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1 2024-09-21T09:31:06.620 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.620 INFO:tasks.workunit.client.0.smithi022.stdout: "avg_session_uptime": { 2024-09-21T09:31:06.620 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 89 2024-09-21T09:31:06.620 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.620 INFO:tasks.workunit.client.0.smithi022.stdout: "mdthresh_evicted": { 2024-09-21T09:31:06.620 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.620 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.621 INFO:tasks.workunit.client.0.smithi022.stdout: "session_add": { 2024-09-21T09:31:06.621 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2 2024-09-21T09:31:06.621 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.621 INFO:tasks.workunit.client.0.smithi022.stdout: "session_count": { 2024-09-21T09:31:06.621 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.621 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.621 INFO:tasks.workunit.client.0.smithi022.stdout: "session_remove": { 2024-09-21T09:31:06.621 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2 2024-09-21T09:31:06.621 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.621 INFO:tasks.workunit.client.0.smithi022.stdout: "sessions_open": { 2024-09-21T09:31:06.621 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.621 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.621 INFO:tasks.workunit.client.0.smithi022.stdout: "sessions_stale": { 2024-09-21T09:31:06.621 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.621 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.622 INFO:tasks.workunit.client.0.smithi022.stdout: "total_load": { 2024-09-21T09:31:06.622 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2 2024-09-21T09:31:06.622 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.622 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.622 INFO:tasks.workunit.client.0.smithi022.stdout: "objecter": { 2024-09-21T09:31:06.622 INFO:tasks.workunit.client.0.smithi022.stdout: "op_active": { 2024-09-21T09:31:06.622 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.622 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.622 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r": { 2024-09-21T09:31:06.622 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.622 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.622 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rmw": { 2024-09-21T09:31:06.622 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.623 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.623 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w": { 2024-09-21T09:31:06.623 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 45 2024-09-21T09:31:06.623 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.623 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.623 INFO:tasks.workunit.client.0.smithi022.stdout: "purge_queue": { 2024-09-21T09:31:06.623 INFO:tasks.workunit.client.0.smithi022.stdout: "pq_executed": { 2024-09-21T09:31:06.623 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.623 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.623 INFO:tasks.workunit.client.0.smithi022.stdout: "pq_executing": { 2024-09-21T09:31:06.623 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.623 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.623 INFO:tasks.workunit.client.0.smithi022.stdout: "pq_executing_high_water": { 2024-09-21T09:31:06.623 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.624 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.624 INFO:tasks.workunit.client.0.smithi022.stdout: "pq_executing_ops": { 2024-09-21T09:31:06.624 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.624 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.624 INFO:tasks.workunit.client.0.smithi022.stdout: "pq_executing_ops_high_water": { 2024-09-21T09:31:06.624 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.624 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.624 INFO:tasks.workunit.client.0.smithi022.stdout: "pq_item_in_journal": { 2024-09-21T09:31:06.624 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.624 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.624 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.624 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.624 INFO:tasks.workunit.client.0.smithi022.stdout: "mon.5349066dea191aa6e7969482c5114baabfb76f71": { 2024-09-21T09:31:06.625 INFO:tasks.workunit.client.0.smithi022.stdout: "mon": { 2024-09-21T09:31:06.625 INFO:tasks.workunit.client.0.smithi022.stdout: "election_call": { 2024-09-21T09:31:06.625 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1 2024-09-21T09:31:06.625 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.625 INFO:tasks.workunit.client.0.smithi022.stdout: "election_lose": { 2024-09-21T09:31:06.625 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1 2024-09-21T09:31:06.625 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.625 INFO:tasks.workunit.client.0.smithi022.stdout: "election_win": { 2024-09-21T09:31:06.625 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.625 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.625 INFO:tasks.workunit.client.0.smithi022.stdout: "num_elections": { 2024-09-21T09:31:06.625 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 3 2024-09-21T09:31:06.625 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.625 INFO:tasks.workunit.client.0.smithi022.stdout: "num_sessions": { 2024-09-21T09:31:06.625 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 11 2024-09-21T09:31:06.626 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.626 INFO:tasks.workunit.client.0.smithi022.stdout: "session_add": { 2024-09-21T09:31:06.626 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 5 2024-09-21T09:31:06.626 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.626 INFO:tasks.workunit.client.0.smithi022.stdout: "session_rm": { 2024-09-21T09:31:06.626 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 77 2024-09-21T09:31:06.626 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.626 INFO:tasks.workunit.client.0.smithi022.stdout: "session_trim": { 2024-09-21T09:31:06.626 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.626 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.626 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.626 INFO:tasks.workunit.client.0.smithi022.stdout: "paxos": { 2024-09-21T09:31:06.626 INFO:tasks.workunit.client.0.smithi022.stdout: "accept_timeout": { 2024-09-21T09:31:06.627 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.627 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.627 INFO:tasks.workunit.client.0.smithi022.stdout: "begin": { 2024-09-21T09:31:06.627 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 813 2024-09-21T09:31:06.627 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.627 INFO:tasks.workunit.client.0.smithi022.stdout: "begin_bytes": { 2024-09-21T09:31:06.627 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 813, 2024-09-21T09:31:06.627 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 3886043 2024-09-21T09:31:06.627 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.627 INFO:tasks.workunit.client.0.smithi022.stdout: "begin_keys": { 2024-09-21T09:31:06.627 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.627 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.628 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.628 INFO:tasks.workunit.client.0.smithi022.stdout: "begin_latency": { 2024-09-21T09:31:06.628 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 813, 2024-09-21T09:31:06.628 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 92998844 2024-09-21T09:31:06.628 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.628 INFO:tasks.workunit.client.0.smithi022.stdout: "collect": { 2024-09-21T09:31:06.628 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1 2024-09-21T09:31:06.628 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.628 INFO:tasks.workunit.client.0.smithi022.stdout: "collect_bytes": { 2024-09-21T09:31:06.628 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 1, 2024-09-21T09:31:06.628 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 73 2024-09-21T09:31:06.628 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.628 INFO:tasks.workunit.client.0.smithi022.stdout: "collect_keys": { 2024-09-21T09:31:06.628 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 1, 2024-09-21T09:31:06.629 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1 2024-09-21T09:31:06.629 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.629 INFO:tasks.workunit.client.0.smithi022.stdout: "collect_latency": { 2024-09-21T09:31:06.629 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 1, 2024-09-21T09:31:06.629 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.629 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.629 INFO:tasks.workunit.client.0.smithi022.stdout: "collect_timeout": { 2024-09-21T09:31:06.629 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.629 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.629 INFO:tasks.workunit.client.0.smithi022.stdout: "collect_uncommitted": { 2024-09-21T09:31:06.629 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.629 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.629 INFO:tasks.workunit.client.0.smithi022.stdout: "commit": { 2024-09-21T09:31:06.629 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 815 2024-09-21T09:31:06.630 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.630 INFO:tasks.workunit.client.0.smithi022.stdout: "commit_bytes": { 2024-09-21T09:31:06.630 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.630 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.630 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.630 INFO:tasks.workunit.client.0.smithi022.stdout: "commit_keys": { 2024-09-21T09:31:06.630 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.630 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.630 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.630 INFO:tasks.workunit.client.0.smithi022.stdout: "commit_latency": { 2024-09-21T09:31:06.630 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.630 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.631 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.631 INFO:tasks.workunit.client.0.smithi022.stdout: "lease_ack_timeout": { 2024-09-21T09:31:06.631 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.631 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.631 INFO:tasks.workunit.client.0.smithi022.stdout: "lease_timeout": { 2024-09-21T09:31:06.631 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.631 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.631 INFO:tasks.workunit.client.0.smithi022.stdout: "new_pn": { 2024-09-21T09:31:06.631 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.631 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.631 INFO:tasks.workunit.client.0.smithi022.stdout: "new_pn_latency": { 2024-09-21T09:31:06.631 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.631 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.632 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.632 INFO:tasks.workunit.client.0.smithi022.stdout: "refresh": { 2024-09-21T09:31:06.632 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 815 2024-09-21T09:31:06.632 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.632 INFO:tasks.workunit.client.0.smithi022.stdout: "refresh_latency": { 2024-09-21T09:31:06.632 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 815, 2024-09-21T09:31:06.632 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1849976914 2024-09-21T09:31:06.632 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.632 INFO:tasks.workunit.client.0.smithi022.stdout: "restart": { 2024-09-21T09:31:06.632 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 8 2024-09-21T09:31:06.632 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.632 INFO:tasks.workunit.client.0.smithi022.stdout: "share_state": { 2024-09-21T09:31:06.632 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.633 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.633 INFO:tasks.workunit.client.0.smithi022.stdout: "share_state_bytes": { 2024-09-21T09:31:06.633 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.633 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.633 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.633 INFO:tasks.workunit.client.0.smithi022.stdout: "share_state_keys": { 2024-09-21T09:31:06.633 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.633 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.633 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.633 INFO:tasks.workunit.client.0.smithi022.stdout: "start_leader": { 2024-09-21T09:31:06.633 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.633 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.633 INFO:tasks.workunit.client.0.smithi022.stdout: "start_peon": { 2024-09-21T09:31:06.633 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1 2024-09-21T09:31:06.634 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.634 INFO:tasks.workunit.client.0.smithi022.stdout: "store_state": { 2024-09-21T09:31:06.634 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 814 2024-09-21T09:31:06.634 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.634 INFO:tasks.workunit.client.0.smithi022.stdout: "store_state_bytes": { 2024-09-21T09:31:06.634 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 814, 2024-09-21T09:31:06.634 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 7619626 2024-09-21T09:31:06.634 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.634 INFO:tasks.workunit.client.0.smithi022.stdout: "store_state_keys": { 2024-09-21T09:31:06.634 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 814, 2024-09-21T09:31:06.634 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 5476 2024-09-21T09:31:06.634 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.634 INFO:tasks.workunit.client.0.smithi022.stdout: "store_state_latency": { 2024-09-21T09:31:06.634 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 814, 2024-09-21T09:31:06.635 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 99998748 2024-09-21T09:31:06.635 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.635 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.635 INFO:tasks.workunit.client.0.smithi022.stdout: "prioritycache": { 2024-09-21T09:31:06.635 INFO:tasks.workunit.client.0.smithi022.stdout: "cache_bytes": { 2024-09-21T09:31:06.635 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1020054731 2024-09-21T09:31:06.635 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.635 INFO:tasks.workunit.client.0.smithi022.stdout: "heap_bytes": { 2024-09-21T09:31:06.635 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 49078272 2024-09-21T09:31:06.635 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.635 INFO:tasks.workunit.client.0.smithi022.stdout: "mapped_bytes": { 2024-09-21T09:31:06.635 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 33382400 2024-09-21T09:31:06.635 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.635 INFO:tasks.workunit.client.0.smithi022.stdout: "target_bytes": { 2024-09-21T09:31:06.635 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2147483648 2024-09-21T09:31:06.636 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.636 INFO:tasks.workunit.client.0.smithi022.stdout: "unmapped_bytes": { 2024-09-21T09:31:06.636 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 15695872 2024-09-21T09:31:06.636 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.636 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.636 INFO:tasks.workunit.client.0.smithi022.stdout: "prioritycache:full": { 2024-09-21T09:31:06.636 INFO:tasks.workunit.client.0.smithi022.stdout: "committed_bytes": { 2024-09-21T09:31:06.636 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 348127232 2024-09-21T09:31:06.636 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.636 INFO:tasks.workunit.client.0.smithi022.stdout: "pri0_bytes": { 2024-09-21T09:31:06.636 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.636 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.636 INFO:tasks.workunit.client.0.smithi022.stdout: "pri10_bytes": { 2024-09-21T09:31:06.636 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.637 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.637 INFO:tasks.workunit.client.0.smithi022.stdout: "pri11_bytes": { 2024-09-21T09:31:06.637 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 277021337 2024-09-21T09:31:06.637 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.637 INFO:tasks.workunit.client.0.smithi022.stdout: "pri1_bytes": { 2024-09-21T09:31:06.637 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 271552 2024-09-21T09:31:06.637 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.637 INFO:tasks.workunit.client.0.smithi022.stdout: "pri2_bytes": { 2024-09-21T09:31:06.637 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.637 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.637 INFO:tasks.workunit.client.0.smithi022.stdout: "pri3_bytes": { 2024-09-21T09:31:06.637 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.637 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.637 INFO:tasks.workunit.client.0.smithi022.stdout: "pri4_bytes": { 2024-09-21T09:31:06.637 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.638 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.638 INFO:tasks.workunit.client.0.smithi022.stdout: "pri5_bytes": { 2024-09-21T09:31:06.638 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.638 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.638 INFO:tasks.workunit.client.0.smithi022.stdout: "pri6_bytes": { 2024-09-21T09:31:06.638 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.638 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.638 INFO:tasks.workunit.client.0.smithi022.stdout: "pri7_bytes": { 2024-09-21T09:31:06.638 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.638 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.638 INFO:tasks.workunit.client.0.smithi022.stdout: "pri8_bytes": { 2024-09-21T09:31:06.638 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.638 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.638 INFO:tasks.workunit.client.0.smithi022.stdout: "pri9_bytes": { 2024-09-21T09:31:06.639 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.639 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.639 INFO:tasks.workunit.client.0.smithi022.stdout: "reserved_bytes": { 2024-09-21T09:31:06.639 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 70834343 2024-09-21T09:31:06.639 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.639 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.639 INFO:tasks.workunit.client.0.smithi022.stdout: "prioritycache:inc": { 2024-09-21T09:31:06.639 INFO:tasks.workunit.client.0.smithi022.stdout: "committed_bytes": { 2024-09-21T09:31:06.639 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 348127232 2024-09-21T09:31:06.639 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.639 INFO:tasks.workunit.client.0.smithi022.stdout: "pri0_bytes": { 2024-09-21T09:31:06.639 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.639 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.639 INFO:tasks.workunit.client.0.smithi022.stdout: "pri10_bytes": { 2024-09-21T09:31:06.640 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.640 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.640 INFO:tasks.workunit.client.0.smithi022.stdout: "pri11_bytes": { 2024-09-21T09:31:06.640 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 277021337 2024-09-21T09:31:06.640 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.640 INFO:tasks.workunit.client.0.smithi022.stdout: "pri1_bytes": { 2024-09-21T09:31:06.640 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 41245 2024-09-21T09:31:06.640 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.640 INFO:tasks.workunit.client.0.smithi022.stdout: "pri2_bytes": { 2024-09-21T09:31:06.640 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.640 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.640 INFO:tasks.workunit.client.0.smithi022.stdout: "pri3_bytes": { 2024-09-21T09:31:06.640 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.640 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.641 INFO:tasks.workunit.client.0.smithi022.stdout: "pri4_bytes": { 2024-09-21T09:31:06.641 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.641 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.641 INFO:tasks.workunit.client.0.smithi022.stdout: "pri5_bytes": { 2024-09-21T09:31:06.641 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.641 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.641 INFO:tasks.workunit.client.0.smithi022.stdout: "pri6_bytes": { 2024-09-21T09:31:06.641 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.641 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.641 INFO:tasks.workunit.client.0.smithi022.stdout: "pri7_bytes": { 2024-09-21T09:31:06.641 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.641 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.641 INFO:tasks.workunit.client.0.smithi022.stdout: "pri8_bytes": { 2024-09-21T09:31:06.641 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.642 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.642 INFO:tasks.workunit.client.0.smithi022.stdout: "pri9_bytes": { 2024-09-21T09:31:06.642 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.642 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.642 INFO:tasks.workunit.client.0.smithi022.stdout: "reserved_bytes": { 2024-09-21T09:31:06.642 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 71064650 2024-09-21T09:31:06.642 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.642 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.642 INFO:tasks.workunit.client.0.smithi022.stdout: "prioritycache:kv": { 2024-09-21T09:31:06.642 INFO:tasks.workunit.client.0.smithi022.stdout: "committed_bytes": { 2024-09-21T09:31:06.642 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 322961408 2024-09-21T09:31:06.642 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.642 INFO:tasks.workunit.client.0.smithi022.stdout: "pri0_bytes": { 2024-09-21T09:31:06.643 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 67108864 2024-09-21T09:31:06.643 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.643 INFO:tasks.workunit.client.0.smithi022.stdout: "pri10_bytes": { 2024-09-21T09:31:06.643 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.643 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.643 INFO:tasks.workunit.client.0.smithi022.stdout: "pri11_bytes": { 2024-09-21T09:31:06.643 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 184680891 2024-09-21T09:31:06.643 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.643 INFO:tasks.workunit.client.0.smithi022.stdout: "pri1_bytes": { 2024-09-21T09:31:06.643 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.643 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.643 INFO:tasks.workunit.client.0.smithi022.stdout: "pri2_bytes": { 2024-09-21T09:31:06.643 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.643 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.644 INFO:tasks.workunit.client.0.smithi022.stdout: "pri3_bytes": { 2024-09-21T09:31:06.644 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.644 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.644 INFO:tasks.workunit.client.0.smithi022.stdout: "pri4_bytes": { 2024-09-21T09:31:06.644 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.644 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.644 INFO:tasks.workunit.client.0.smithi022.stdout: "pri5_bytes": { 2024-09-21T09:31:06.644 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.644 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.644 INFO:tasks.workunit.client.0.smithi022.stdout: "pri6_bytes": { 2024-09-21T09:31:06.644 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.644 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.645 INFO:tasks.workunit.client.0.smithi022.stdout: "pri7_bytes": { 2024-09-21T09:31:06.645 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.645 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.645 INFO:tasks.workunit.client.0.smithi022.stdout: "pri8_bytes": { 2024-09-21T09:31:06.645 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.645 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.645 INFO:tasks.workunit.client.0.smithi022.stdout: "pri9_bytes": { 2024-09-21T09:31:06.645 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.645 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.645 INFO:tasks.workunit.client.0.smithi022.stdout: "reserved_bytes": { 2024-09-21T09:31:06.645 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 71171653 2024-09-21T09:31:06.645 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.645 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.646 INFO:tasks.workunit.client.0.smithi022.stdout: "rocksdb": { 2024-09-21T09:31:06.646 INFO:tasks.workunit.client.0.smithi022.stdout: "compact": { 2024-09-21T09:31:06.646 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.646 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.646 INFO:tasks.workunit.client.0.smithi022.stdout: "compact_completed": { 2024-09-21T09:31:06.646 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1 2024-09-21T09:31:06.646 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.646 INFO:tasks.workunit.client.0.smithi022.stdout: "compact_lasted": { 2024-09-21T09:31:06.646 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 23852185 2024-09-21T09:31:06.646 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.646 INFO:tasks.workunit.client.0.smithi022.stdout: "compact_queue_len": { 2024-09-21T09:31:06.646 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.646 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.647 INFO:tasks.workunit.client.0.smithi022.stdout: "compact_queue_merge": { 2024-09-21T09:31:06.647 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.647 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.647 INFO:tasks.workunit.client.0.smithi022.stdout: "compact_running": { 2024-09-21T09:31:06.647 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.647 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.647 INFO:tasks.workunit.client.0.smithi022.stdout: "get_latency": { 2024-09-21T09:31:06.647 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 35045, 2024-09-21T09:31:06.647 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 85593023 2024-09-21T09:31:06.647 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.647 INFO:tasks.workunit.client.0.smithi022.stdout: "rocksdb_write_delay_time": { 2024-09-21T09:31:06.647 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.647 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.647 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.648 INFO:tasks.workunit.client.0.smithi022.stdout: "rocksdb_write_memtable_time": { 2024-09-21T09:31:06.648 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.648 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.648 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.648 INFO:tasks.workunit.client.0.smithi022.stdout: "rocksdb_write_pre_and_post_time": { 2024-09-21T09:31:06.648 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.648 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.648 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.648 INFO:tasks.workunit.client.0.smithi022.stdout: "rocksdb_write_wal_time": { 2024-09-21T09:31:06.648 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.648 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.648 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.648 INFO:tasks.workunit.client.0.smithi022.stdout: "submit_latency": { 2024-09-21T09:31:06.648 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.649 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.649 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.649 INFO:tasks.workunit.client.0.smithi022.stdout: "submit_sync_latency": { 2024-09-21T09:31:06.649 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 1747, 2024-09-21T09:31:06.649 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 209900578 2024-09-21T09:31:06.649 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.649 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.649 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.649 INFO:tasks.workunit.client.0.smithi022.stdout: "mon.aa5e831b31d98e3163af5edfeb921f9af32463c5": { 2024-09-21T09:31:06.649 INFO:tasks.workunit.client.0.smithi022.stdout: "mon": { 2024-09-21T09:31:06.649 INFO:tasks.workunit.client.0.smithi022.stdout: "election_call": { 2024-09-21T09:31:06.649 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2 2024-09-21T09:31:06.649 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.649 INFO:tasks.workunit.client.0.smithi022.stdout: "election_lose": { 2024-09-21T09:31:06.650 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2 2024-09-21T09:31:06.650 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.650 INFO:tasks.workunit.client.0.smithi022.stdout: "election_win": { 2024-09-21T09:31:06.650 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.650 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.650 INFO:tasks.workunit.client.0.smithi022.stdout: "num_elections": { 2024-09-21T09:31:06.650 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 7 2024-09-21T09:31:06.650 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.650 INFO:tasks.workunit.client.0.smithi022.stdout: "num_sessions": { 2024-09-21T09:31:06.650 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 10 2024-09-21T09:31:06.650 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.650 INFO:tasks.workunit.client.0.smithi022.stdout: "session_add": { 2024-09-21T09:31:06.650 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 4 2024-09-21T09:31:06.651 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.651 INFO:tasks.workunit.client.0.smithi022.stdout: "session_rm": { 2024-09-21T09:31:06.651 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 51 2024-09-21T09:31:06.651 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.651 INFO:tasks.workunit.client.0.smithi022.stdout: "session_trim": { 2024-09-21T09:31:06.651 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.651 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.651 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.651 INFO:tasks.workunit.client.0.smithi022.stdout: "paxos": { 2024-09-21T09:31:06.651 INFO:tasks.workunit.client.0.smithi022.stdout: "accept_timeout": { 2024-09-21T09:31:06.651 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.651 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.651 INFO:tasks.workunit.client.0.smithi022.stdout: "begin": { 2024-09-21T09:31:06.652 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 823 2024-09-21T09:31:06.652 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.652 INFO:tasks.workunit.client.0.smithi022.stdout: "begin_bytes": { 2024-09-21T09:31:06.652 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 823, 2024-09-21T09:31:06.652 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 3913382 2024-09-21T09:31:06.652 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.652 INFO:tasks.workunit.client.0.smithi022.stdout: "begin_keys": { 2024-09-21T09:31:06.652 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.652 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.652 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.653 INFO:tasks.workunit.client.0.smithi022.stdout: "begin_latency": { 2024-09-21T09:31:06.653 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 823, 2024-09-21T09:31:06.653 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 126997490 2024-09-21T09:31:06.653 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.653 INFO:tasks.workunit.client.0.smithi022.stdout: "collect": { 2024-09-21T09:31:06.653 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2 2024-09-21T09:31:06.653 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.653 INFO:tasks.workunit.client.0.smithi022.stdout: "collect_bytes": { 2024-09-21T09:31:06.653 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 2, 2024-09-21T09:31:06.653 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 146 2024-09-21T09:31:06.653 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.654 INFO:tasks.workunit.client.0.smithi022.stdout: "collect_keys": { 2024-09-21T09:31:06.654 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 2, 2024-09-21T09:31:06.654 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2 2024-09-21T09:31:06.654 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.654 INFO:tasks.workunit.client.0.smithi022.stdout: "collect_latency": { 2024-09-21T09:31:06.654 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 2, 2024-09-21T09:31:06.654 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.654 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.654 INFO:tasks.workunit.client.0.smithi022.stdout: "collect_timeout": { 2024-09-21T09:31:06.654 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.654 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.654 INFO:tasks.workunit.client.0.smithi022.stdout: "collect_uncommitted": { 2024-09-21T09:31:06.654 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.655 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.655 INFO:tasks.workunit.client.0.smithi022.stdout: "commit": { 2024-09-21T09:31:06.655 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 824 2024-09-21T09:31:06.655 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.655 INFO:tasks.workunit.client.0.smithi022.stdout: "commit_bytes": { 2024-09-21T09:31:06.655 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.655 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.655 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.655 INFO:tasks.workunit.client.0.smithi022.stdout: "commit_keys": { 2024-09-21T09:31:06.655 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.655 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.655 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.656 INFO:tasks.workunit.client.0.smithi022.stdout: "commit_latency": { 2024-09-21T09:31:06.656 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.656 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.656 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.656 INFO:tasks.workunit.client.0.smithi022.stdout: "lease_ack_timeout": { 2024-09-21T09:31:06.656 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.656 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.656 INFO:tasks.workunit.client.0.smithi022.stdout: "lease_timeout": { 2024-09-21T09:31:06.656 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.656 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.656 INFO:tasks.workunit.client.0.smithi022.stdout: "new_pn": { 2024-09-21T09:31:06.656 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.656 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.657 INFO:tasks.workunit.client.0.smithi022.stdout: "new_pn_latency": { 2024-09-21T09:31:06.657 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.657 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.657 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.657 INFO:tasks.workunit.client.0.smithi022.stdout: "refresh": { 2024-09-21T09:31:06.657 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 824 2024-09-21T09:31:06.657 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.657 INFO:tasks.workunit.client.0.smithi022.stdout: "refresh_latency": { 2024-09-21T09:31:06.657 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 824, 2024-09-21T09:31:06.657 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1861963206 2024-09-21T09:31:06.657 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.657 INFO:tasks.workunit.client.0.smithi022.stdout: "restart": { 2024-09-21T09:31:06.657 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 13 2024-09-21T09:31:06.657 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.658 INFO:tasks.workunit.client.0.smithi022.stdout: "share_state": { 2024-09-21T09:31:06.658 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.658 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.658 INFO:tasks.workunit.client.0.smithi022.stdout: "share_state_bytes": { 2024-09-21T09:31:06.658 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.658 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.658 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.658 INFO:tasks.workunit.client.0.smithi022.stdout: "share_state_keys": { 2024-09-21T09:31:06.658 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.658 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.658 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.658 INFO:tasks.workunit.client.0.smithi022.stdout: "start_leader": { 2024-09-21T09:31:06.659 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.659 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.659 INFO:tasks.workunit.client.0.smithi022.stdout: "start_peon": { 2024-09-21T09:31:06.659 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2 2024-09-21T09:31:06.659 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.659 INFO:tasks.workunit.client.0.smithi022.stdout: "store_state": { 2024-09-21T09:31:06.659 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 824 2024-09-21T09:31:06.659 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.659 INFO:tasks.workunit.client.0.smithi022.stdout: "store_state_bytes": { 2024-09-21T09:31:06.659 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 824, 2024-09-21T09:31:06.659 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 7672206 2024-09-21T09:31:06.659 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.659 INFO:tasks.workunit.client.0.smithi022.stdout: "store_state_keys": { 2024-09-21T09:31:06.659 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 824, 2024-09-21T09:31:06.660 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 5552 2024-09-21T09:31:06.660 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.660 INFO:tasks.workunit.client.0.smithi022.stdout: "store_state_latency": { 2024-09-21T09:31:06.660 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 824, 2024-09-21T09:31:06.660 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 343993214 2024-09-21T09:31:06.660 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.660 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.660 INFO:tasks.workunit.client.0.smithi022.stdout: "prioritycache": { 2024-09-21T09:31:06.660 INFO:tasks.workunit.client.0.smithi022.stdout: "cache_bytes": { 2024-09-21T09:31:06.660 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1020054731 2024-09-21T09:31:06.660 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.660 INFO:tasks.workunit.client.0.smithi022.stdout: "heap_bytes": { 2024-09-21T09:31:06.660 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 49086464 2024-09-21T09:31:06.660 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.661 INFO:tasks.workunit.client.0.smithi022.stdout: "mapped_bytes": { 2024-09-21T09:31:06.661 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 32604160 2024-09-21T09:31:06.661 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.661 INFO:tasks.workunit.client.0.smithi022.stdout: "target_bytes": { 2024-09-21T09:31:06.661 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2147483648 2024-09-21T09:31:06.661 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.661 INFO:tasks.workunit.client.0.smithi022.stdout: "unmapped_bytes": { 2024-09-21T09:31:06.661 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 16482304 2024-09-21T09:31:06.661 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.661 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.661 INFO:tasks.workunit.client.0.smithi022.stdout: "prioritycache:full": { 2024-09-21T09:31:06.661 INFO:tasks.workunit.client.0.smithi022.stdout: "committed_bytes": { 2024-09-21T09:31:06.661 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 348127232 2024-09-21T09:31:06.661 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.662 INFO:tasks.workunit.client.0.smithi022.stdout: "pri0_bytes": { 2024-09-21T09:31:06.662 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.662 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.662 INFO:tasks.workunit.client.0.smithi022.stdout: "pri10_bytes": { 2024-09-21T09:31:06.662 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.662 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.662 INFO:tasks.workunit.client.0.smithi022.stdout: "pri11_bytes": { 2024-09-21T09:31:06.662 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 277021337 2024-09-21T09:31:06.662 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.662 INFO:tasks.workunit.client.0.smithi022.stdout: "pri1_bytes": { 2024-09-21T09:31:06.662 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 271552 2024-09-21T09:31:06.662 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.662 INFO:tasks.workunit.client.0.smithi022.stdout: "pri2_bytes": { 2024-09-21T09:31:06.663 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.663 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.663 INFO:tasks.workunit.client.0.smithi022.stdout: "pri3_bytes": { 2024-09-21T09:31:06.663 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.663 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.663 INFO:tasks.workunit.client.0.smithi022.stdout: "pri4_bytes": { 2024-09-21T09:31:06.664 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.664 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.664 INFO:tasks.workunit.client.0.smithi022.stdout: "pri5_bytes": { 2024-09-21T09:31:06.664 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.664 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.664 INFO:tasks.workunit.client.0.smithi022.stdout: "pri6_bytes": { 2024-09-21T09:31:06.664 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.664 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.664 INFO:tasks.workunit.client.0.smithi022.stdout: "pri7_bytes": { 2024-09-21T09:31:06.664 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.664 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.664 INFO:tasks.workunit.client.0.smithi022.stdout: "pri8_bytes": { 2024-09-21T09:31:06.665 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.665 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.665 INFO:tasks.workunit.client.0.smithi022.stdout: "pri9_bytes": { 2024-09-21T09:31:06.665 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.665 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.665 INFO:tasks.workunit.client.0.smithi022.stdout: "reserved_bytes": { 2024-09-21T09:31:06.665 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 70834343 2024-09-21T09:31:06.665 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.665 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.665 INFO:tasks.workunit.client.0.smithi022.stdout: "prioritycache:inc": { 2024-09-21T09:31:06.666 INFO:tasks.workunit.client.0.smithi022.stdout: "committed_bytes": { 2024-09-21T09:31:06.666 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 348127232 2024-09-21T09:31:06.666 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.666 INFO:tasks.workunit.client.0.smithi022.stdout: "pri0_bytes": { 2024-09-21T09:31:06.666 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.666 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.666 INFO:tasks.workunit.client.0.smithi022.stdout: "pri10_bytes": { 2024-09-21T09:31:06.666 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.666 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.666 INFO:tasks.workunit.client.0.smithi022.stdout: "pri11_bytes": { 2024-09-21T09:31:06.666 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 277021337 2024-09-21T09:31:06.666 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.666 INFO:tasks.workunit.client.0.smithi022.stdout: "pri1_bytes": { 2024-09-21T09:31:06.667 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 41245 2024-09-21T09:31:06.667 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.667 INFO:tasks.workunit.client.0.smithi022.stdout: "pri2_bytes": { 2024-09-21T09:31:06.667 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.667 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.667 INFO:tasks.workunit.client.0.smithi022.stdout: "pri3_bytes": { 2024-09-21T09:31:06.667 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.667 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.667 INFO:tasks.workunit.client.0.smithi022.stdout: "pri4_bytes": { 2024-09-21T09:31:06.667 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.668 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.668 INFO:tasks.workunit.client.0.smithi022.stdout: "pri5_bytes": { 2024-09-21T09:31:06.668 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.668 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.668 INFO:tasks.workunit.client.0.smithi022.stdout: "pri6_bytes": { 2024-09-21T09:31:06.668 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.668 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.668 INFO:tasks.workunit.client.0.smithi022.stdout: "pri7_bytes": { 2024-09-21T09:31:06.668 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.668 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.668 INFO:tasks.workunit.client.0.smithi022.stdout: "pri8_bytes": { 2024-09-21T09:31:06.668 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.668 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.669 INFO:tasks.workunit.client.0.smithi022.stdout: "pri9_bytes": { 2024-09-21T09:31:06.669 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.669 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.669 INFO:tasks.workunit.client.0.smithi022.stdout: "reserved_bytes": { 2024-09-21T09:31:06.669 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 71064650 2024-09-21T09:31:06.669 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.669 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.669 INFO:tasks.workunit.client.0.smithi022.stdout: "prioritycache:kv": { 2024-09-21T09:31:06.669 INFO:tasks.workunit.client.0.smithi022.stdout: "committed_bytes": { 2024-09-21T09:31:06.669 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 322961408 2024-09-21T09:31:06.669 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.670 INFO:tasks.workunit.client.0.smithi022.stdout: "pri0_bytes": { 2024-09-21T09:31:06.670 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 67108864 2024-09-21T09:31:06.670 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.670 INFO:tasks.workunit.client.0.smithi022.stdout: "pri10_bytes": { 2024-09-21T09:31:06.670 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.670 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.670 INFO:tasks.workunit.client.0.smithi022.stdout: "pri11_bytes": { 2024-09-21T09:31:06.670 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 184680891 2024-09-21T09:31:06.670 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.670 INFO:tasks.workunit.client.0.smithi022.stdout: "pri1_bytes": { 2024-09-21T09:31:06.670 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.671 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.671 INFO:tasks.workunit.client.0.smithi022.stdout: "pri2_bytes": { 2024-09-21T09:31:06.671 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.671 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.671 INFO:tasks.workunit.client.0.smithi022.stdout: "pri3_bytes": { 2024-09-21T09:31:06.671 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.671 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.671 INFO:tasks.workunit.client.0.smithi022.stdout: "pri4_bytes": { 2024-09-21T09:31:06.671 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.671 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.671 INFO:tasks.workunit.client.0.smithi022.stdout: "pri5_bytes": { 2024-09-21T09:31:06.671 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.671 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.671 INFO:tasks.workunit.client.0.smithi022.stdout: "pri6_bytes": { 2024-09-21T09:31:06.672 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.672 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.672 INFO:tasks.workunit.client.0.smithi022.stdout: "pri7_bytes": { 2024-09-21T09:31:06.672 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.672 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.672 INFO:tasks.workunit.client.0.smithi022.stdout: "pri8_bytes": { 2024-09-21T09:31:06.672 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.672 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.672 INFO:tasks.workunit.client.0.smithi022.stdout: "pri9_bytes": { 2024-09-21T09:31:06.672 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.672 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.672 INFO:tasks.workunit.client.0.smithi022.stdout: "reserved_bytes": { 2024-09-21T09:31:06.672 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 71171653 2024-09-21T09:31:06.673 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.673 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.673 INFO:tasks.workunit.client.0.smithi022.stdout: "rocksdb": { 2024-09-21T09:31:06.673 INFO:tasks.workunit.client.0.smithi022.stdout: "compact": { 2024-09-21T09:31:06.673 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.673 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.673 INFO:tasks.workunit.client.0.smithi022.stdout: "compact_completed": { 2024-09-21T09:31:06.673 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1 2024-09-21T09:31:06.673 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.673 INFO:tasks.workunit.client.0.smithi022.stdout: "compact_lasted": { 2024-09-21T09:31:06.673 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 29092821 2024-09-21T09:31:06.673 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.673 INFO:tasks.workunit.client.0.smithi022.stdout: "compact_queue_len": { 2024-09-21T09:31:06.674 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.674 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.674 INFO:tasks.workunit.client.0.smithi022.stdout: "compact_queue_merge": { 2024-09-21T09:31:06.674 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.674 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.674 INFO:tasks.workunit.client.0.smithi022.stdout: "compact_running": { 2024-09-21T09:31:06.674 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.674 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.674 INFO:tasks.workunit.client.0.smithi022.stdout: "get_latency": { 2024-09-21T09:31:06.674 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 35234, 2024-09-21T09:31:06.674 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 86559894 2024-09-21T09:31:06.674 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.674 INFO:tasks.workunit.client.0.smithi022.stdout: "rocksdb_write_delay_time": { 2024-09-21T09:31:06.675 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.675 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.675 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.675 INFO:tasks.workunit.client.0.smithi022.stdout: "rocksdb_write_memtable_time": { 2024-09-21T09:31:06.675 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.675 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.675 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.675 INFO:tasks.workunit.client.0.smithi022.stdout: "rocksdb_write_pre_and_post_time": { 2024-09-21T09:31:06.675 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.675 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.675 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.675 INFO:tasks.workunit.client.0.smithi022.stdout: "rocksdb_write_wal_time": { 2024-09-21T09:31:06.675 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.676 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.676 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.676 INFO:tasks.workunit.client.0.smithi022.stdout: "submit_latency": { 2024-09-21T09:31:06.676 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.676 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.676 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.676 INFO:tasks.workunit.client.0.smithi022.stdout: "submit_sync_latency": { 2024-09-21T09:31:06.676 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 1794, 2024-09-21T09:31:06.676 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 577083552 2024-09-21T09:31:06.676 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.676 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.676 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.676 INFO:tasks.workunit.client.0.smithi022.stdout: "mon.f85c107c3119ea450f9f41ee1907af1822a71ba3": { 2024-09-21T09:31:06.677 INFO:tasks.workunit.client.0.smithi022.stdout: "mon": { 2024-09-21T09:31:06.677 INFO:tasks.workunit.client.0.smithi022.stdout: "election_call": { 2024-09-21T09:31:06.677 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2 2024-09-21T09:31:06.677 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.677 INFO:tasks.workunit.client.0.smithi022.stdout: "election_lose": { 2024-09-21T09:31:06.677 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.677 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.677 INFO:tasks.workunit.client.0.smithi022.stdout: "election_win": { 2024-09-21T09:31:06.677 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 3 2024-09-21T09:31:06.677 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.677 INFO:tasks.workunit.client.0.smithi022.stdout: "num_elections": { 2024-09-21T09:31:06.677 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 9 2024-09-21T09:31:06.677 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.677 INFO:tasks.workunit.client.0.smithi022.stdout: "num_sessions": { 2024-09-21T09:31:06.678 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 8 2024-09-21T09:31:06.678 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.678 INFO:tasks.workunit.client.0.smithi022.stdout: "session_add": { 2024-09-21T09:31:06.678 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 3 2024-09-21T09:31:06.678 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.678 INFO:tasks.workunit.client.0.smithi022.stdout: "session_rm": { 2024-09-21T09:31:06.678 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 177 2024-09-21T09:31:06.678 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.678 INFO:tasks.workunit.client.0.smithi022.stdout: "session_trim": { 2024-09-21T09:31:06.678 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.678 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.679 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.679 INFO:tasks.workunit.client.0.smithi022.stdout: "paxos": { 2024-09-21T09:31:06.679 INFO:tasks.workunit.client.0.smithi022.stdout: "accept_timeout": { 2024-09-21T09:31:06.679 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.679 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.679 INFO:tasks.workunit.client.0.smithi022.stdout: "begin": { 2024-09-21T09:31:06.679 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 951 2024-09-21T09:31:06.679 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.679 INFO:tasks.workunit.client.0.smithi022.stdout: "begin_bytes": { 2024-09-21T09:31:06.679 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 951, 2024-09-21T09:31:06.679 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 4963289 2024-09-21T09:31:06.679 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.679 INFO:tasks.workunit.client.0.smithi022.stdout: "begin_keys": { 2024-09-21T09:31:06.680 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 951, 2024-09-21T09:31:06.680 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2853 2024-09-21T09:31:06.680 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.680 INFO:tasks.workunit.client.0.smithi022.stdout: "begin_latency": { 2024-09-21T09:31:06.680 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 951, 2024-09-21T09:31:06.680 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 138998270 2024-09-21T09:31:06.680 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.680 INFO:tasks.workunit.client.0.smithi022.stdout: "collect": { 2024-09-21T09:31:06.680 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.680 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.680 INFO:tasks.workunit.client.0.smithi022.stdout: "collect_bytes": { 2024-09-21T09:31:06.680 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.680 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.681 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.681 INFO:tasks.workunit.client.0.smithi022.stdout: "collect_keys": { 2024-09-21T09:31:06.681 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.681 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.681 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.681 INFO:tasks.workunit.client.0.smithi022.stdout: "collect_latency": { 2024-09-21T09:31:06.681 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.681 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.681 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.681 INFO:tasks.workunit.client.0.smithi022.stdout: "collect_timeout": { 2024-09-21T09:31:06.681 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.681 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.681 INFO:tasks.workunit.client.0.smithi022.stdout: "collect_uncommitted": { 2024-09-21T09:31:06.681 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.682 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.682 INFO:tasks.workunit.client.0.smithi022.stdout: "commit": { 2024-09-21T09:31:06.682 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 951 2024-09-21T09:31:06.682 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.682 INFO:tasks.workunit.client.0.smithi022.stdout: "commit_bytes": { 2024-09-21T09:31:06.682 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 951, 2024-09-21T09:31:06.682 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 4894875 2024-09-21T09:31:06.682 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.682 INFO:tasks.workunit.client.0.smithi022.stdout: "commit_keys": { 2024-09-21T09:31:06.682 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 951, 2024-09-21T09:31:06.682 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 5416 2024-09-21T09:31:06.682 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.682 INFO:tasks.workunit.client.0.smithi022.stdout: "commit_latency": { 2024-09-21T09:31:06.682 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 951, 2024-09-21T09:31:06.683 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 133493004 2024-09-21T09:31:06.683 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.683 INFO:tasks.workunit.client.0.smithi022.stdout: "lease_ack_timeout": { 2024-09-21T09:31:06.683 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.683 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.683 INFO:tasks.workunit.client.0.smithi022.stdout: "lease_timeout": { 2024-09-21T09:31:06.683 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.683 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.683 INFO:tasks.workunit.client.0.smithi022.stdout: "new_pn": { 2024-09-21T09:31:06.683 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2 2024-09-21T09:31:06.683 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.683 INFO:tasks.workunit.client.0.smithi022.stdout: "new_pn_latency": { 2024-09-21T09:31:06.683 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 2, 2024-09-21T09:31:06.683 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.684 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.684 INFO:tasks.workunit.client.0.smithi022.stdout: "refresh": { 2024-09-21T09:31:06.684 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 953 2024-09-21T09:31:06.684 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.684 INFO:tasks.workunit.client.0.smithi022.stdout: "refresh_latency": { 2024-09-21T09:31:06.684 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 953, 2024-09-21T09:31:06.684 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2276971617 2024-09-21T09:31:06.684 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.684 INFO:tasks.workunit.client.0.smithi022.stdout: "restart": { 2024-09-21T09:31:06.684 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 12 2024-09-21T09:31:06.684 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.684 INFO:tasks.workunit.client.0.smithi022.stdout: "share_state": { 2024-09-21T09:31:06.684 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 3 2024-09-21T09:31:06.684 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.685 INFO:tasks.workunit.client.0.smithi022.stdout: "share_state_bytes": { 2024-09-21T09:31:06.685 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 3, 2024-09-21T09:31:06.685 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1627 2024-09-21T09:31:06.685 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.685 INFO:tasks.workunit.client.0.smithi022.stdout: "share_state_keys": { 2024-09-21T09:31:06.685 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 3, 2024-09-21T09:31:06.685 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 3 2024-09-21T09:31:06.685 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.685 INFO:tasks.workunit.client.0.smithi022.stdout: "start_leader": { 2024-09-21T09:31:06.685 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 3 2024-09-21T09:31:06.685 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.685 INFO:tasks.workunit.client.0.smithi022.stdout: "start_peon": { 2024-09-21T09:31:06.685 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.685 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.685 INFO:tasks.workunit.client.0.smithi022.stdout: "store_state": { 2024-09-21T09:31:06.686 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.686 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.686 INFO:tasks.workunit.client.0.smithi022.stdout: "store_state_bytes": { 2024-09-21T09:31:06.686 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.686 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.686 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.686 INFO:tasks.workunit.client.0.smithi022.stdout: "store_state_keys": { 2024-09-21T09:31:06.686 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.686 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.686 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.686 INFO:tasks.workunit.client.0.smithi022.stdout: "store_state_latency": { 2024-09-21T09:31:06.686 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.686 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.686 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.687 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.687 INFO:tasks.workunit.client.0.smithi022.stdout: "prioritycache": { 2024-09-21T09:31:06.687 INFO:tasks.workunit.client.0.smithi022.stdout: "cache_bytes": { 2024-09-21T09:31:06.687 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1020054731 2024-09-21T09:31:06.687 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.687 INFO:tasks.workunit.client.0.smithi022.stdout: "heap_bytes": { 2024-09-21T09:31:06.687 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 48979968 2024-09-21T09:31:06.687 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.687 INFO:tasks.workunit.client.0.smithi022.stdout: "mapped_bytes": { 2024-09-21T09:31:06.687 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 40542208 2024-09-21T09:31:06.687 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.687 INFO:tasks.workunit.client.0.smithi022.stdout: "target_bytes": { 2024-09-21T09:31:06.687 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2147483648 2024-09-21T09:31:06.687 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.688 INFO:tasks.workunit.client.0.smithi022.stdout: "unmapped_bytes": { 2024-09-21T09:31:06.688 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 8437760 2024-09-21T09:31:06.688 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.688 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.688 INFO:tasks.workunit.client.0.smithi022.stdout: "prioritycache:full": { 2024-09-21T09:31:06.688 INFO:tasks.workunit.client.0.smithi022.stdout: "committed_bytes": { 2024-09-21T09:31:06.688 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 348127232 2024-09-21T09:31:06.688 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.688 INFO:tasks.workunit.client.0.smithi022.stdout: "pri0_bytes": { 2024-09-21T09:31:06.688 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.688 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.688 INFO:tasks.workunit.client.0.smithi022.stdout: "pri10_bytes": { 2024-09-21T09:31:06.688 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.688 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.688 INFO:tasks.workunit.client.0.smithi022.stdout: "pri11_bytes": { 2024-09-21T09:31:06.689 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 277022399 2024-09-21T09:31:06.689 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.689 INFO:tasks.workunit.client.0.smithi022.stdout: "pri1_bytes": { 2024-09-21T09:31:06.689 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 271122 2024-09-21T09:31:06.689 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.689 INFO:tasks.workunit.client.0.smithi022.stdout: "pri2_bytes": { 2024-09-21T09:31:06.689 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.689 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.689 INFO:tasks.workunit.client.0.smithi022.stdout: "pri3_bytes": { 2024-09-21T09:31:06.689 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.689 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.689 INFO:tasks.workunit.client.0.smithi022.stdout: "pri4_bytes": { 2024-09-21T09:31:06.689 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.689 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.690 INFO:tasks.workunit.client.0.smithi022.stdout: "pri5_bytes": { 2024-09-21T09:31:06.690 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.690 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.690 INFO:tasks.workunit.client.0.smithi022.stdout: "pri6_bytes": { 2024-09-21T09:31:06.690 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.690 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.690 INFO:tasks.workunit.client.0.smithi022.stdout: "pri7_bytes": { 2024-09-21T09:31:06.690 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.690 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.690 INFO:tasks.workunit.client.0.smithi022.stdout: "pri8_bytes": { 2024-09-21T09:31:06.690 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.690 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.690 INFO:tasks.workunit.client.0.smithi022.stdout: "pri9_bytes": { 2024-09-21T09:31:06.690 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.691 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.691 INFO:tasks.workunit.client.0.smithi022.stdout: "reserved_bytes": { 2024-09-21T09:31:06.691 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 70833711 2024-09-21T09:31:06.691 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.691 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.691 INFO:tasks.workunit.client.0.smithi022.stdout: "prioritycache:inc": { 2024-09-21T09:31:06.691 INFO:tasks.workunit.client.0.smithi022.stdout: "committed_bytes": { 2024-09-21T09:31:06.691 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 348127232 2024-09-21T09:31:06.691 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.691 INFO:tasks.workunit.client.0.smithi022.stdout: "pri0_bytes": { 2024-09-21T09:31:06.691 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.691 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.691 INFO:tasks.workunit.client.0.smithi022.stdout: "pri10_bytes": { 2024-09-21T09:31:06.691 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.691 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.692 INFO:tasks.workunit.client.0.smithi022.stdout: "pri11_bytes": { 2024-09-21T09:31:06.692 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 277022399 2024-09-21T09:31:06.692 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.692 INFO:tasks.workunit.client.0.smithi022.stdout: "pri1_bytes": { 2024-09-21T09:31:06.692 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 38842 2024-09-21T09:31:06.692 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.692 INFO:tasks.workunit.client.0.smithi022.stdout: "pri2_bytes": { 2024-09-21T09:31:06.692 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.692 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.692 INFO:tasks.workunit.client.0.smithi022.stdout: "pri3_bytes": { 2024-09-21T09:31:06.692 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.692 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.692 INFO:tasks.workunit.client.0.smithi022.stdout: "pri4_bytes": { 2024-09-21T09:31:06.692 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.693 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.693 INFO:tasks.workunit.client.0.smithi022.stdout: "pri5_bytes": { 2024-09-21T09:31:06.693 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.693 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.693 INFO:tasks.workunit.client.0.smithi022.stdout: "pri6_bytes": { 2024-09-21T09:31:06.693 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.693 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.693 INFO:tasks.workunit.client.0.smithi022.stdout: "pri7_bytes": { 2024-09-21T09:31:06.693 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.693 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.693 INFO:tasks.workunit.client.0.smithi022.stdout: "pri8_bytes": { 2024-09-21T09:31:06.693 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.693 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.693 INFO:tasks.workunit.client.0.smithi022.stdout: "pri9_bytes": { 2024-09-21T09:31:06.694 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.694 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.694 INFO:tasks.workunit.client.0.smithi022.stdout: "reserved_bytes": { 2024-09-21T09:31:06.694 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 71065991 2024-09-21T09:31:06.694 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.694 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.694 INFO:tasks.workunit.client.0.smithi022.stdout: "prioritycache:kv": { 2024-09-21T09:31:06.694 INFO:tasks.workunit.client.0.smithi022.stdout: "committed_bytes": { 2024-09-21T09:31:06.694 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 322961408 2024-09-21T09:31:06.694 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.694 INFO:tasks.workunit.client.0.smithi022.stdout: "pri0_bytes": { 2024-09-21T09:31:06.694 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 67108864 2024-09-21T09:31:06.694 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.694 INFO:tasks.workunit.client.0.smithi022.stdout: "pri10_bytes": { 2024-09-21T09:31:06.694 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.695 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.695 INFO:tasks.workunit.client.0.smithi022.stdout: "pri11_bytes": { 2024-09-21T09:31:06.695 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 184681599 2024-09-21T09:31:06.695 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.695 INFO:tasks.workunit.client.0.smithi022.stdout: "pri1_bytes": { 2024-09-21T09:31:06.695 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.695 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.695 INFO:tasks.workunit.client.0.smithi022.stdout: "pri2_bytes": { 2024-09-21T09:31:06.695 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.695 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.695 INFO:tasks.workunit.client.0.smithi022.stdout: "pri3_bytes": { 2024-09-21T09:31:06.695 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.695 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.695 INFO:tasks.workunit.client.0.smithi022.stdout: "pri4_bytes": { 2024-09-21T09:31:06.696 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.696 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.696 INFO:tasks.workunit.client.0.smithi022.stdout: "pri5_bytes": { 2024-09-21T09:31:06.696 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.696 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.696 INFO:tasks.workunit.client.0.smithi022.stdout: "pri6_bytes": { 2024-09-21T09:31:06.696 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.696 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.696 INFO:tasks.workunit.client.0.smithi022.stdout: "pri7_bytes": { 2024-09-21T09:31:06.696 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.696 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.696 INFO:tasks.workunit.client.0.smithi022.stdout: "pri8_bytes": { 2024-09-21T09:31:06.696 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.697 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.697 INFO:tasks.workunit.client.0.smithi022.stdout: "pri9_bytes": { 2024-09-21T09:31:06.697 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.697 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.697 INFO:tasks.workunit.client.0.smithi022.stdout: "reserved_bytes": { 2024-09-21T09:31:06.697 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 71170945 2024-09-21T09:31:06.697 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.697 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.697 INFO:tasks.workunit.client.0.smithi022.stdout: "rocksdb": { 2024-09-21T09:31:06.697 INFO:tasks.workunit.client.0.smithi022.stdout: "compact": { 2024-09-21T09:31:06.697 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.697 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.697 INFO:tasks.workunit.client.0.smithi022.stdout: "compact_completed": { 2024-09-21T09:31:06.697 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1 2024-09-21T09:31:06.697 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.698 INFO:tasks.workunit.client.0.smithi022.stdout: "compact_lasted": { 2024-09-21T09:31:06.698 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 24725695 2024-09-21T09:31:06.698 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.698 INFO:tasks.workunit.client.0.smithi022.stdout: "compact_queue_len": { 2024-09-21T09:31:06.698 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.698 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.698 INFO:tasks.workunit.client.0.smithi022.stdout: "compact_queue_merge": { 2024-09-21T09:31:06.698 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.698 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.698 INFO:tasks.workunit.client.0.smithi022.stdout: "compact_running": { 2024-09-21T09:31:06.698 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.698 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.698 INFO:tasks.workunit.client.0.smithi022.stdout: "get_latency": { 2024-09-21T09:31:06.698 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 42345, 2024-09-21T09:31:06.699 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 170203915 2024-09-21T09:31:06.699 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.699 INFO:tasks.workunit.client.0.smithi022.stdout: "rocksdb_write_delay_time": { 2024-09-21T09:31:06.699 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.699 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.699 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.699 INFO:tasks.workunit.client.0.smithi022.stdout: "rocksdb_write_memtable_time": { 2024-09-21T09:31:06.699 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.699 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.699 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.699 INFO:tasks.workunit.client.0.smithi022.stdout: "rocksdb_write_pre_and_post_time": { 2024-09-21T09:31:06.699 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.699 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.699 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.700 INFO:tasks.workunit.client.0.smithi022.stdout: "rocksdb_write_wal_time": { 2024-09-21T09:31:06.700 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.700 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.700 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.700 INFO:tasks.workunit.client.0.smithi022.stdout: "submit_latency": { 2024-09-21T09:31:06.700 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.700 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.700 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.700 INFO:tasks.workunit.client.0.smithi022.stdout: "submit_sync_latency": { 2024-09-21T09:31:06.700 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 2069, 2024-09-21T09:31:06.700 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 279929127 2024-09-21T09:31:06.700 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.700 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.700 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.700 INFO:tasks.workunit.client.0.smithi022.stdout: "osd.0": { 2024-09-21T09:31:06.701 INFO:tasks.workunit.client.0.smithi022.stdout: "bluefs": { 2024-09-21T09:31:06.701 INFO:tasks.workunit.client.0.smithi022.stdout: "alloc_slow_fallback": { 2024-09-21T09:31:06.701 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.701 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.701 INFO:tasks.workunit.client.0.smithi022.stdout: "alloc_slow_size_fallback": { 2024-09-21T09:31:06.701 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.701 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.701 INFO:tasks.workunit.client.0.smithi022.stdout: "alloc_unit_db": { 2024-09-21T09:31:06.701 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 65536 2024-09-21T09:31:06.701 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.701 INFO:tasks.workunit.client.0.smithi022.stdout: "alloc_unit_main": { 2024-09-21T09:31:06.701 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.701 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.701 INFO:tasks.workunit.client.0.smithi022.stdout: "alloc_unit_wal": { 2024-09-21T09:31:06.702 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.702 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.702 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes_written_slow": { 2024-09-21T09:31:06.702 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.702 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.702 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes_written_sst": { 2024-09-21T09:31:06.702 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 12288 2024-09-21T09:31:06.702 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.702 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes_written_wal": { 2024-09-21T09:31:06.702 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2215936 2024-09-21T09:31:06.702 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.702 INFO:tasks.workunit.client.0.smithi022.stdout: "compact_lat": { 2024-09-21T09:31:06.702 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.702 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.703 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.703 INFO:tasks.workunit.client.0.smithi022.stdout: "compact_lock_lat": { 2024-09-21T09:31:06.703 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.703 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.703 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.703 INFO:tasks.workunit.client.0.smithi022.stdout: "db_total_bytes": { 2024-09-21T09:31:06.703 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 95995035648 2024-09-21T09:31:06.703 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.703 INFO:tasks.workunit.client.0.smithi022.stdout: "db_used_bytes": { 2024-09-21T09:31:06.703 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 304480256 2024-09-21T09:31:06.703 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.703 INFO:tasks.workunit.client.0.smithi022.stdout: "log_bytes": { 2024-09-21T09:31:06.703 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1470464 2024-09-21T09:31:06.703 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.703 INFO:tasks.workunit.client.0.smithi022.stdout: "logged_bytes": { 2024-09-21T09:31:06.704 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1331200 2024-09-21T09:31:06.704 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.704 INFO:tasks.workunit.client.0.smithi022.stdout: "max_bytes_db": { 2024-09-21T09:31:06.704 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 312868864 2024-09-21T09:31:06.704 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.704 INFO:tasks.workunit.client.0.smithi022.stdout: "max_bytes_slow": { 2024-09-21T09:31:06.704 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.704 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.704 INFO:tasks.workunit.client.0.smithi022.stdout: "max_bytes_wal": { 2024-09-21T09:31:06.704 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.704 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.704 INFO:tasks.workunit.client.0.smithi022.stdout: "num_files": { 2024-09-21T09:31:06.704 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 14 2024-09-21T09:31:06.704 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.705 INFO:tasks.workunit.client.0.smithi022.stdout: "read_bytes": { 2024-09-21T09:31:06.705 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 195697 2024-09-21T09:31:06.705 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.705 INFO:tasks.workunit.client.0.smithi022.stdout: "read_count": { 2024-09-21T09:31:06.705 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 23 2024-09-21T09:31:06.705 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.705 INFO:tasks.workunit.client.0.smithi022.stdout: "read_disk_bytes": { 2024-09-21T09:31:06.705 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1269760 2024-09-21T09:31:06.705 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.705 INFO:tasks.workunit.client.0.smithi022.stdout: "read_disk_bytes_db": { 2024-09-21T09:31:06.705 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1273856 2024-09-21T09:31:06.705 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.705 INFO:tasks.workunit.client.0.smithi022.stdout: "read_disk_bytes_slow": { 2024-09-21T09:31:06.705 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.706 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.706 INFO:tasks.workunit.client.0.smithi022.stdout: "read_disk_bytes_wal": { 2024-09-21T09:31:06.706 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.706 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.706 INFO:tasks.workunit.client.0.smithi022.stdout: "read_disk_count": { 2024-09-21T09:31:06.706 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 15 2024-09-21T09:31:06.706 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.706 INFO:tasks.workunit.client.0.smithi022.stdout: "read_prefetch_bytes": { 2024-09-21T09:31:06.706 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 4789 2024-09-21T09:31:06.706 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.706 INFO:tasks.workunit.client.0.smithi022.stdout: "read_prefetch_count": { 2024-09-21T09:31:06.706 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 4 2024-09-21T09:31:06.706 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.706 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_buffer_bytes": { 2024-09-21T09:31:06.706 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 5589 2024-09-21T09:31:06.707 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.707 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_buffer_count": { 2024-09-21T09:31:06.707 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 24 2024-09-21T09:31:06.707 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.707 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_bytes": { 2024-09-21T09:31:06.707 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 5589 2024-09-21T09:31:06.707 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.707 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_count": { 2024-09-21T09:31:06.707 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 24 2024-09-21T09:31:06.707 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.707 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_disk_bytes": { 2024-09-21T09:31:06.707 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.707 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.707 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_disk_bytes_db": { 2024-09-21T09:31:06.708 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.708 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.708 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_disk_bytes_slow": { 2024-09-21T09:31:06.708 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.708 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.708 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_disk_bytes_wal": { 2024-09-21T09:31:06.708 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.708 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.708 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_disk_count": { 2024-09-21T09:31:06.708 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.708 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.708 INFO:tasks.workunit.client.0.smithi022.stdout: "slow_total_bytes": { 2024-09-21T09:31:06.708 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.709 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.709 INFO:tasks.workunit.client.0.smithi022.stdout: "slow_used_bytes": { 2024-09-21T09:31:06.709 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.709 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.709 INFO:tasks.workunit.client.0.smithi022.stdout: "wal_total_bytes": { 2024-09-21T09:31:06.709 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.709 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.709 INFO:tasks.workunit.client.0.smithi022.stdout: "wal_used_bytes": { 2024-09-21T09:31:06.709 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.709 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.709 INFO:tasks.workunit.client.0.smithi022.stdout: "write_bytes": { 2024-09-21T09:31:06.709 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 3620864 2024-09-21T09:31:06.709 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.709 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.709 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore": { 2024-09-21T09:31:06.710 INFO:tasks.workunit.client.0.smithi022.stdout: "alloc_unit": { 2024-09-21T09:31:06.710 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 4096 2024-09-21T09:31:06.710 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.710 INFO:tasks.workunit.client.0.smithi022.stdout: "allocated": { 2024-09-21T09:31:06.710 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1294336 2024-09-21T09:31:06.710 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.710 INFO:tasks.workunit.client.0.smithi022.stdout: "clist_lat": { 2024-09-21T09:31:06.710 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 6, 2024-09-21T09:31:06.710 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 268543 2024-09-21T09:31:06.710 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.710 INFO:tasks.workunit.client.0.smithi022.stdout: "compress_lat": { 2024-09-21T09:31:06.710 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.710 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.711 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.711 INFO:tasks.workunit.client.0.smithi022.stdout: "compressed": { 2024-09-21T09:31:06.711 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.711 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.711 INFO:tasks.workunit.client.0.smithi022.stdout: "compressed_allocated": { 2024-09-21T09:31:06.711 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.711 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.711 INFO:tasks.workunit.client.0.smithi022.stdout: "compressed_original": { 2024-09-21T09:31:06.711 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.711 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.711 INFO:tasks.workunit.client.0.smithi022.stdout: "csum_lat": { 2024-09-21T09:31:06.711 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 24, 2024-09-21T09:31:06.711 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 78025 2024-09-21T09:31:06.712 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.712 INFO:tasks.workunit.client.0.smithi022.stdout: "decompress_lat": { 2024-09-21T09:31:06.712 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.712 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.712 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.712 INFO:tasks.workunit.client.0.smithi022.stdout: "kv_commit_lat": { 2024-09-21T09:31:06.712 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 308, 2024-09-21T09:31:06.712 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 52145765 2024-09-21T09:31:06.712 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.712 INFO:tasks.workunit.client.0.smithi022.stdout: "kv_final_lat": { 2024-09-21T09:31:06.712 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 301, 2024-09-21T09:31:06.712 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 12656371 2024-09-21T09:31:06.712 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.712 INFO:tasks.workunit.client.0.smithi022.stdout: "kv_flush_lat": { 2024-09-21T09:31:06.712 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 308, 2024-09-21T09:31:06.713 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 394762 2024-09-21T09:31:06.713 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.713 INFO:tasks.workunit.client.0.smithi022.stdout: "kv_sync_lat": { 2024-09-21T09:31:06.713 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 308, 2024-09-21T09:31:06.713 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 52540527 2024-09-21T09:31:06.713 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.713 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_get_keys_lat": { 2024-09-21T09:31:06.713 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.713 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.713 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.713 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_get_values_lat": { 2024-09-21T09:31:06.713 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 1, 2024-09-21T09:31:06.713 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 49657 2024-09-21T09:31:06.713 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.714 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_lower_bound_lat": { 2024-09-21T09:31:06.714 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 5, 2024-09-21T09:31:06.714 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 13525 2024-09-21T09:31:06.714 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.714 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_next_lat": { 2024-09-21T09:31:06.714 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.714 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.714 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.714 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_seek_to_first_lat": { 2024-09-21T09:31:06.714 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.714 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.714 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.714 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_upper_bound_lat": { 2024-09-21T09:31:06.714 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 4, 2024-09-21T09:31:06.714 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 12081 2024-09-21T09:31:06.715 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.715 INFO:tasks.workunit.client.0.smithi022.stdout: "onode_hits": { 2024-09-21T09:31:06.715 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 672 2024-09-21T09:31:06.715 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.715 INFO:tasks.workunit.client.0.smithi022.stdout: "onode_misses": { 2024-09-21T09:31:06.715 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 220 2024-09-21T09:31:06.715 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.715 INFO:tasks.workunit.client.0.smithi022.stdout: "read_lat": { 2024-09-21T09:31:06.715 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 105, 2024-09-21T09:31:06.715 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1662758 2024-09-21T09:31:06.715 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.715 INFO:tasks.workunit.client.0.smithi022.stdout: "read_onode_meta_lat": { 2024-09-21T09:31:06.715 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 253, 2024-09-21T09:31:06.716 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 242000 2024-09-21T09:31:06.716 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.716 INFO:tasks.workunit.client.0.smithi022.stdout: "read_wait_aio_lat": { 2024-09-21T09:31:06.716 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 148, 2024-09-21T09:31:06.716 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1037316 2024-09-21T09:31:06.716 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.716 INFO:tasks.workunit.client.0.smithi022.stdout: "reads_with_retries": { 2024-09-21T09:31:06.716 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.716 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.716 INFO:tasks.workunit.client.0.smithi022.stdout: "remove_lat": { 2024-09-21T09:31:06.716 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 3, 2024-09-21T09:31:06.716 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 63010 2024-09-21T09:31:06.716 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.716 INFO:tasks.workunit.client.0.smithi022.stdout: "state_aio_wait_lat": { 2024-09-21T09:31:06.717 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 417, 2024-09-21T09:31:06.717 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 5786609 2024-09-21T09:31:06.717 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.717 INFO:tasks.workunit.client.0.smithi022.stdout: "state_deferred_aio_wait_lat": { 2024-09-21T09:31:06.717 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 98, 2024-09-21T09:31:06.717 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 7343840 2024-09-21T09:31:06.717 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.717 INFO:tasks.workunit.client.0.smithi022.stdout: "state_deferred_cleanup_lat": { 2024-09-21T09:31:06.717 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 97, 2024-09-21T09:31:06.717 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 416582490984 2024-09-21T09:31:06.717 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.717 INFO:tasks.workunit.client.0.smithi022.stdout: "state_deferred_queued_lat": { 2024-09-21T09:31:06.717 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 98, 2024-09-21T09:31:06.717 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 157672872424 2024-09-21T09:31:06.718 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.718 INFO:tasks.workunit.client.0.smithi022.stdout: "state_done_lat": { 2024-09-21T09:31:06.718 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 416, 2024-09-21T09:31:06.718 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 77598915152 2024-09-21T09:31:06.718 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.718 INFO:tasks.workunit.client.0.smithi022.stdout: "state_finishing_lat": { 2024-09-21T09:31:06.718 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 416, 2024-09-21T09:31:06.718 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 43715 2024-09-21T09:31:06.718 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.718 INFO:tasks.workunit.client.0.smithi022.stdout: "state_io_done_lat": { 2024-09-21T09:31:06.718 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 417, 2024-09-21T09:31:06.718 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 422297 2024-09-21T09:31:06.718 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.718 INFO:tasks.workunit.client.0.smithi022.stdout: "state_kv_commiting_lat": { 2024-09-21T09:31:06.719 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 417, 2024-09-21T09:31:06.719 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 131736946 2024-09-21T09:31:06.719 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.719 INFO:tasks.workunit.client.0.smithi022.stdout: "state_kv_done_lat": { 2024-09-21T09:31:06.719 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 417, 2024-09-21T09:31:06.719 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 144824 2024-09-21T09:31:06.719 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.719 INFO:tasks.workunit.client.0.smithi022.stdout: "state_kv_queued_lat": { 2024-09-21T09:31:06.719 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 417, 2024-09-21T09:31:06.719 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 102226352 2024-09-21T09:31:06.719 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.719 INFO:tasks.workunit.client.0.smithi022.stdout: "state_prepare_lat": { 2024-09-21T09:31:06.719 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 417, 2024-09-21T09:31:06.719 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 20572631 2024-09-21T09:31:06.719 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.720 INFO:tasks.workunit.client.0.smithi022.stdout: "stored": { 2024-09-21T09:31:06.720 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 900748 2024-09-21T09:31:06.720 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.720 INFO:tasks.workunit.client.0.smithi022.stdout: "truncate_lat": { 2024-09-21T09:31:06.720 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 58, 2024-09-21T09:31:06.720 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 305527 2024-09-21T09:31:06.720 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.720 INFO:tasks.workunit.client.0.smithi022.stdout: "txc_commit_lat": { 2024-09-21T09:31:06.720 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 417, 2024-09-21T09:31:06.720 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 260803531 2024-09-21T09:31:06.720 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.720 INFO:tasks.workunit.client.0.smithi022.stdout: "txc_submit_lat": { 2024-09-21T09:31:06.720 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 417, 2024-09-21T09:31:06.720 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 24307691 2024-09-21T09:31:06.720 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.721 INFO:tasks.workunit.client.0.smithi022.stdout: "txc_throttle_lat": { 2024-09-21T09:31:06.721 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 417, 2024-09-21T09:31:06.721 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 3885809 2024-09-21T09:31:06.721 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.721 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.721 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore-pricache": { 2024-09-21T09:31:06.721 INFO:tasks.workunit.client.0.smithi022.stdout: "cache_bytes": { 2024-09-21T09:31:06.721 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1862400347 2024-09-21T09:31:06.721 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.721 INFO:tasks.workunit.client.0.smithi022.stdout: "heap_bytes": { 2024-09-21T09:31:06.721 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 270909440 2024-09-21T09:31:06.721 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.722 INFO:tasks.workunit.client.0.smithi022.stdout: "mapped_bytes": { 2024-09-21T09:31:06.722 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 270516224 2024-09-21T09:31:06.722 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.722 INFO:tasks.workunit.client.0.smithi022.stdout: "target_bytes": { 2024-09-21T09:31:06.722 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 3138478489 2024-09-21T09:31:06.722 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.722 INFO:tasks.workunit.client.0.smithi022.stdout: "unmapped_bytes": { 2024-09-21T09:31:06.722 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 393216 2024-09-21T09:31:06.722 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.722 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.722 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore-pricache:data": { 2024-09-21T09:31:06.722 INFO:tasks.workunit.client.0.smithi022.stdout: "committed_bytes": { 2024-09-21T09:31:06.722 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 159383552 2024-09-21T09:31:06.723 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.723 INFO:tasks.workunit.client.0.smithi022.stdout: "pri0_bytes": { 2024-09-21T09:31:06.723 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.723 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.723 INFO:tasks.workunit.client.0.smithi022.stdout: "pri10_bytes": { 2024-09-21T09:31:06.723 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.723 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.723 INFO:tasks.workunit.client.0.smithi022.stdout: "pri11_bytes": { 2024-09-21T09:31:06.723 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 85554152 2024-09-21T09:31:06.723 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.723 INFO:tasks.workunit.client.0.smithi022.stdout: "pri1_bytes": { 2024-09-21T09:31:06.723 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.723 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.723 INFO:tasks.workunit.client.0.smithi022.stdout: "pri2_bytes": { 2024-09-21T09:31:06.724 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.724 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.724 INFO:tasks.workunit.client.0.smithi022.stdout: "pri3_bytes": { 2024-09-21T09:31:06.724 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.724 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.724 INFO:tasks.workunit.client.0.smithi022.stdout: "pri4_bytes": { 2024-09-21T09:31:06.724 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.724 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.724 INFO:tasks.workunit.client.0.smithi022.stdout: "pri5_bytes": { 2024-09-21T09:31:06.724 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.724 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.724 INFO:tasks.workunit.client.0.smithi022.stdout: "pri6_bytes": { 2024-09-21T09:31:06.724 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 8192 2024-09-21T09:31:06.724 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.725 INFO:tasks.workunit.client.0.smithi022.stdout: "pri7_bytes": { 2024-09-21T09:31:06.725 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.725 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.725 INFO:tasks.workunit.client.0.smithi022.stdout: "pri8_bytes": { 2024-09-21T09:31:06.725 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.725 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.725 INFO:tasks.workunit.client.0.smithi022.stdout: "pri9_bytes": { 2024-09-21T09:31:06.725 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.725 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.725 INFO:tasks.workunit.client.0.smithi022.stdout: "reserved_bytes": { 2024-09-21T09:31:06.725 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 73821208 2024-09-21T09:31:06.725 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.725 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.725 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore-pricache:kv": { 2024-09-21T09:31:06.726 INFO:tasks.workunit.client.0.smithi022.stdout: "committed_bytes": { 2024-09-21T09:31:06.726 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 780140544 2024-09-21T09:31:06.726 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.726 INFO:tasks.workunit.client.0.smithi022.stdout: "pri0_bytes": { 2024-09-21T09:31:06.726 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 67108864 2024-09-21T09:31:06.726 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.726 INFO:tasks.workunit.client.0.smithi022.stdout: "pri10_bytes": { 2024-09-21T09:31:06.726 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.726 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.726 INFO:tasks.workunit.client.0.smithi022.stdout: "pri11_bytes": { 2024-09-21T09:31:06.726 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 641656144 2024-09-21T09:31:06.726 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.726 INFO:tasks.workunit.client.0.smithi022.stdout: "pri1_bytes": { 2024-09-21T09:31:06.726 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.727 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.727 INFO:tasks.workunit.client.0.smithi022.stdout: "pri2_bytes": { 2024-09-21T09:31:06.727 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.727 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.727 INFO:tasks.workunit.client.0.smithi022.stdout: "pri3_bytes": { 2024-09-21T09:31:06.727 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.727 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.727 INFO:tasks.workunit.client.0.smithi022.stdout: "pri4_bytes": { 2024-09-21T09:31:06.727 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.727 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.727 INFO:tasks.workunit.client.0.smithi022.stdout: "pri5_bytes": { 2024-09-21T09:31:06.727 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.727 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.727 INFO:tasks.workunit.client.0.smithi022.stdout: "pri6_bytes": { 2024-09-21T09:31:06.728 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2128 2024-09-21T09:31:06.728 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.728 INFO:tasks.workunit.client.0.smithi022.stdout: "pri7_bytes": { 2024-09-21T09:31:06.728 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.728 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.728 INFO:tasks.workunit.client.0.smithi022.stdout: "pri8_bytes": { 2024-09-21T09:31:06.728 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.728 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.728 INFO:tasks.workunit.client.0.smithi022.stdout: "pri9_bytes": { 2024-09-21T09:31:06.728 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.728 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.728 INFO:tasks.workunit.client.0.smithi022.stdout: "reserved_bytes": { 2024-09-21T09:31:06.728 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 71373408 2024-09-21T09:31:06.729 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.729 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.729 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore-pricache:kv_onode": { 2024-09-21T09:31:06.729 INFO:tasks.workunit.client.0.smithi022.stdout: "committed_bytes": { 2024-09-21T09:31:06.729 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 192937984 2024-09-21T09:31:06.729 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.729 INFO:tasks.workunit.client.0.smithi022.stdout: "pri0_bytes": { 2024-09-21T09:31:06.729 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 67108864 2024-09-21T09:31:06.729 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.729 INFO:tasks.workunit.client.0.smithi022.stdout: "pri10_bytes": { 2024-09-21T09:31:06.729 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 464 2024-09-21T09:31:06.729 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.729 INFO:tasks.workunit.client.0.smithi022.stdout: "pri11_bytes": { 2024-09-21T09:31:06.729 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 57036101 2024-09-21T09:31:06.730 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.730 INFO:tasks.workunit.client.0.smithi022.stdout: "pri1_bytes": { 2024-09-21T09:31:06.730 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.730 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.730 INFO:tasks.workunit.client.0.smithi022.stdout: "pri2_bytes": { 2024-09-21T09:31:06.730 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.730 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.730 INFO:tasks.workunit.client.0.smithi022.stdout: "pri3_bytes": { 2024-09-21T09:31:06.730 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.730 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.730 INFO:tasks.workunit.client.0.smithi022.stdout: "pri4_bytes": { 2024-09-21T09:31:06.730 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.730 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.730 INFO:tasks.workunit.client.0.smithi022.stdout: "pri5_bytes": { 2024-09-21T09:31:06.731 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.731 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.731 INFO:tasks.workunit.client.0.smithi022.stdout: "pri6_bytes": { 2024-09-21T09:31:06.731 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.731 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.731 INFO:tasks.workunit.client.0.smithi022.stdout: "pri7_bytes": { 2024-09-21T09:31:06.731 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.731 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.731 INFO:tasks.workunit.client.0.smithi022.stdout: "pri8_bytes": { 2024-09-21T09:31:06.731 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.731 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.731 INFO:tasks.workunit.client.0.smithi022.stdout: "pri9_bytes": { 2024-09-21T09:31:06.731 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.731 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.732 INFO:tasks.workunit.client.0.smithi022.stdout: "reserved_bytes": { 2024-09-21T09:31:06.732 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 68792555 2024-09-21T09:31:06.732 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.732 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.732 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore-pricache:meta": { 2024-09-21T09:31:06.732 INFO:tasks.workunit.client.0.smithi022.stdout: "committed_bytes": { 2024-09-21T09:31:06.732 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 713031680 2024-09-21T09:31:06.732 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.732 INFO:tasks.workunit.client.0.smithi022.stdout: "pri0_bytes": { 2024-09-21T09:31:06.732 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.732 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.732 INFO:tasks.workunit.client.0.smithi022.stdout: "pri10_bytes": { 2024-09-21T09:31:06.732 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.732 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.733 INFO:tasks.workunit.client.0.smithi022.stdout: "pri11_bytes": { 2024-09-21T09:31:06.733 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 641656144 2024-09-21T09:31:06.733 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.733 INFO:tasks.workunit.client.0.smithi022.stdout: "pri1_bytes": { 2024-09-21T09:31:06.733 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.733 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.733 INFO:tasks.workunit.client.0.smithi022.stdout: "pri2_bytes": { 2024-09-21T09:31:06.733 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.733 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.733 INFO:tasks.workunit.client.0.smithi022.stdout: "pri3_bytes": { 2024-09-21T09:31:06.733 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.733 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.733 INFO:tasks.workunit.client.0.smithi022.stdout: "pri4_bytes": { 2024-09-21T09:31:06.733 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.733 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.734 INFO:tasks.workunit.client.0.smithi022.stdout: "pri5_bytes": { 2024-09-21T09:31:06.734 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.734 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.734 INFO:tasks.workunit.client.0.smithi022.stdout: "pri6_bytes": { 2024-09-21T09:31:06.734 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 276551 2024-09-21T09:31:06.734 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.734 INFO:tasks.workunit.client.0.smithi022.stdout: "pri7_bytes": { 2024-09-21T09:31:06.734 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.734 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.734 INFO:tasks.workunit.client.0.smithi022.stdout: "pri8_bytes": { 2024-09-21T09:31:06.734 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.734 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.734 INFO:tasks.workunit.client.0.smithi022.stdout: "pri9_bytes": { 2024-09-21T09:31:06.734 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.735 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.735 INFO:tasks.workunit.client.0.smithi022.stdout: "reserved_bytes": { 2024-09-21T09:31:06.735 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 71098985 2024-09-21T09:31:06.735 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.735 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.735 INFO:tasks.workunit.client.0.smithi022.stdout: "objecter": { 2024-09-21T09:31:06.735 INFO:tasks.workunit.client.0.smithi022.stdout: "op_active": { 2024-09-21T09:31:06.735 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.735 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.735 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r": { 2024-09-21T09:31:06.735 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.735 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.735 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rmw": { 2024-09-21T09:31:06.735 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.736 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.736 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w": { 2024-09-21T09:31:06.736 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.736 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.736 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.736 INFO:tasks.workunit.client.0.smithi022.stdout: "osd": { 2024-09-21T09:31:06.736 INFO:tasks.workunit.client.0.smithi022.stdout: "numpg": { 2024-09-21T09:31:06.736 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 44 2024-09-21T09:31:06.736 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.736 INFO:tasks.workunit.client.0.smithi022.stdout: "numpg_removing": { 2024-09-21T09:31:06.736 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.736 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.736 INFO:tasks.workunit.client.0.smithi022.stdout: "op": { 2024-09-21T09:31:06.737 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 6 2024-09-21T09:31:06.737 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.737 INFO:tasks.workunit.client.0.smithi022.stdout: "op_delayed_degraded": { 2024-09-21T09:31:06.737 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.737 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.737 INFO:tasks.workunit.client.0.smithi022.stdout: "op_delayed_unreadable": { 2024-09-21T09:31:06.737 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.737 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.737 INFO:tasks.workunit.client.0.smithi022.stdout: "op_in_bytes": { 2024-09-21T09:31:06.737 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 604 2024-09-21T09:31:06.737 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.737 INFO:tasks.workunit.client.0.smithi022.stdout: "op_latency": { 2024-09-21T09:31:06.737 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 6, 2024-09-21T09:31:06.738 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 43758824 2024-09-21T09:31:06.738 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.738 INFO:tasks.workunit.client.0.smithi022.stdout: "op_out_bytes": { 2024-09-21T09:31:06.738 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.738 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.738 INFO:tasks.workunit.client.0.smithi022.stdout: "op_prepare_latency": { 2024-09-21T09:31:06.738 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 7, 2024-09-21T09:31:06.738 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 5335765 2024-09-21T09:31:06.738 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.738 INFO:tasks.workunit.client.0.smithi022.stdout: "op_process_latency": { 2024-09-21T09:31:06.738 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 6, 2024-09-21T09:31:06.738 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 13562427 2024-09-21T09:31:06.738 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.739 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r": { 2024-09-21T09:31:06.739 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.739 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.739 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r_latency": { 2024-09-21T09:31:06.739 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.739 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.739 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.739 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-21T09:31:06.739 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.739 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.739 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r_out_bytes": { 2024-09-21T09:31:06.739 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.739 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.739 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r_prepare_latency": { 2024-09-21T09:31:06.740 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.740 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.740 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.740 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r_process_latency": { 2024-09-21T09:31:06.740 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.740 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.740 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.740 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw": { 2024-09-21T09:31:06.740 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.740 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.740 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_in_bytes": { 2024-09-21T09:31:06.740 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.741 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.741 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_latency": { 2024-09-21T09:31:06.741 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.741 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.741 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.741 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-21T09:31:06.741 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.741 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.741 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-21T09:31:06.741 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.741 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.741 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_out_bytes": { 2024-09-21T09:31:06.741 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.741 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.742 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_prepare_latency": { 2024-09-21T09:31:06.742 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.742 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.742 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.742 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_process_latency": { 2024-09-21T09:31:06.742 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.742 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.742 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.742 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w": { 2024-09-21T09:31:06.742 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 6 2024-09-21T09:31:06.742 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.742 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w_in_bytes": { 2024-09-21T09:31:06.742 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 604 2024-09-21T09:31:06.742 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.743 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w_latency": { 2024-09-21T09:31:06.743 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 6, 2024-09-21T09:31:06.743 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 43758824 2024-09-21T09:31:06.743 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.743 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-21T09:31:06.743 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.743 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.743 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w_prepare_latency": { 2024-09-21T09:31:06.743 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 7, 2024-09-21T09:31:06.743 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 5335765 2024-09-21T09:31:06.743 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.743 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w_process_latency": { 2024-09-21T09:31:06.743 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 6, 2024-09-21T09:31:06.743 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 13562427 2024-09-21T09:31:06.744 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.744 INFO:tasks.workunit.client.0.smithi022.stdout: "op_wip": { 2024-09-21T09:31:06.744 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.744 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.744 INFO:tasks.workunit.client.0.smithi022.stdout: "recovery_bytes": { 2024-09-21T09:31:06.744 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.744 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.744 INFO:tasks.workunit.client.0.smithi022.stdout: "recovery_ops": { 2024-09-21T09:31:06.744 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.744 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.744 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_bytes": { 2024-09-21T09:31:06.744 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 95995035648 2024-09-21T09:31:06.744 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.744 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_bytes_used": { 2024-09-21T09:31:06.745 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 305786880 2024-09-21T09:31:06.745 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.745 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.745 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.745 INFO:tasks.workunit.client.0.smithi022.stdout: "osd.1": { 2024-09-21T09:31:06.745 INFO:tasks.workunit.client.0.smithi022.stdout: "bluefs": { 2024-09-21T09:31:06.745 INFO:tasks.workunit.client.0.smithi022.stdout: "alloc_slow_fallback": { 2024-09-21T09:31:06.745 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.745 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.745 INFO:tasks.workunit.client.0.smithi022.stdout: "alloc_slow_size_fallback": { 2024-09-21T09:31:06.745 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.745 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.745 INFO:tasks.workunit.client.0.smithi022.stdout: "alloc_unit_db": { 2024-09-21T09:31:06.745 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 65536 2024-09-21T09:31:06.745 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.746 INFO:tasks.workunit.client.0.smithi022.stdout: "alloc_unit_main": { 2024-09-21T09:31:06.746 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.746 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.746 INFO:tasks.workunit.client.0.smithi022.stdout: "alloc_unit_wal": { 2024-09-21T09:31:06.746 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.746 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.746 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes_written_slow": { 2024-09-21T09:31:06.746 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.746 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.746 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes_written_sst": { 2024-09-21T09:31:06.746 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 12288 2024-09-21T09:31:06.746 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.746 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes_written_wal": { 2024-09-21T09:31:06.747 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2113536 2024-09-21T09:31:06.747 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.747 INFO:tasks.workunit.client.0.smithi022.stdout: "compact_lat": { 2024-09-21T09:31:06.747 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.747 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.747 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.747 INFO:tasks.workunit.client.0.smithi022.stdout: "compact_lock_lat": { 2024-09-21T09:31:06.747 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.747 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.747 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.747 INFO:tasks.workunit.client.0.smithi022.stdout: "db_total_bytes": { 2024-09-21T09:31:06.747 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 95995035648 2024-09-21T09:31:06.747 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.747 INFO:tasks.workunit.client.0.smithi022.stdout: "db_used_bytes": { 2024-09-21T09:31:06.748 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 304480256 2024-09-21T09:31:06.748 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.748 INFO:tasks.workunit.client.0.smithi022.stdout: "log_bytes": { 2024-09-21T09:31:06.748 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1458176 2024-09-21T09:31:06.748 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.748 INFO:tasks.workunit.client.0.smithi022.stdout: "logged_bytes": { 2024-09-21T09:31:06.748 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1318912 2024-09-21T09:31:06.748 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.748 INFO:tasks.workunit.client.0.smithi022.stdout: "max_bytes_db": { 2024-09-21T09:31:06.748 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 312868864 2024-09-21T09:31:06.748 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.748 INFO:tasks.workunit.client.0.smithi022.stdout: "max_bytes_slow": { 2024-09-21T09:31:06.748 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.748 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.749 INFO:tasks.workunit.client.0.smithi022.stdout: "max_bytes_wal": { 2024-09-21T09:31:06.749 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.749 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.749 INFO:tasks.workunit.client.0.smithi022.stdout: "num_files": { 2024-09-21T09:31:06.749 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 14 2024-09-21T09:31:06.749 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.749 INFO:tasks.workunit.client.0.smithi022.stdout: "read_bytes": { 2024-09-21T09:31:06.749 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 195697 2024-09-21T09:31:06.749 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.749 INFO:tasks.workunit.client.0.smithi022.stdout: "read_count": { 2024-09-21T09:31:06.749 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 23 2024-09-21T09:31:06.749 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.749 INFO:tasks.workunit.client.0.smithi022.stdout: "read_disk_bytes": { 2024-09-21T09:31:06.750 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1269760 2024-09-21T09:31:06.750 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.750 INFO:tasks.workunit.client.0.smithi022.stdout: "read_disk_bytes_db": { 2024-09-21T09:31:06.750 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1273856 2024-09-21T09:31:06.750 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.750 INFO:tasks.workunit.client.0.smithi022.stdout: "read_disk_bytes_slow": { 2024-09-21T09:31:06.750 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.750 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.750 INFO:tasks.workunit.client.0.smithi022.stdout: "read_disk_bytes_wal": { 2024-09-21T09:31:06.750 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.750 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.750 INFO:tasks.workunit.client.0.smithi022.stdout: "read_disk_count": { 2024-09-21T09:31:06.750 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 15 2024-09-21T09:31:06.750 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.751 INFO:tasks.workunit.client.0.smithi022.stdout: "read_prefetch_bytes": { 2024-09-21T09:31:06.751 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 4789 2024-09-21T09:31:06.751 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.751 INFO:tasks.workunit.client.0.smithi022.stdout: "read_prefetch_count": { 2024-09-21T09:31:06.751 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 4 2024-09-21T09:31:06.751 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.751 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_buffer_bytes": { 2024-09-21T09:31:06.751 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 5589 2024-09-21T09:31:06.751 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.751 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_buffer_count": { 2024-09-21T09:31:06.751 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 24 2024-09-21T09:31:06.751 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.751 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_bytes": { 2024-09-21T09:31:06.752 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 5589 2024-09-21T09:31:06.752 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.752 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_count": { 2024-09-21T09:31:06.752 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 24 2024-09-21T09:31:06.752 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.752 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_disk_bytes": { 2024-09-21T09:31:06.752 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.752 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.752 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_disk_bytes_db": { 2024-09-21T09:31:06.752 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.752 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.752 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_disk_bytes_slow": { 2024-09-21T09:31:06.752 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.753 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.753 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_disk_bytes_wal": { 2024-09-21T09:31:06.753 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.753 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.753 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_disk_count": { 2024-09-21T09:31:06.753 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.753 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.753 INFO:tasks.workunit.client.0.smithi022.stdout: "slow_total_bytes": { 2024-09-21T09:31:06.753 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.753 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.753 INFO:tasks.workunit.client.0.smithi022.stdout: "slow_used_bytes": { 2024-09-21T09:31:06.753 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.753 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.754 INFO:tasks.workunit.client.0.smithi022.stdout: "wal_total_bytes": { 2024-09-21T09:31:06.754 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.754 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.754 INFO:tasks.workunit.client.0.smithi022.stdout: "wal_used_bytes": { 2024-09-21T09:31:06.754 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.754 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.754 INFO:tasks.workunit.client.0.smithi022.stdout: "write_bytes": { 2024-09-21T09:31:06.754 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 3506176 2024-09-21T09:31:06.754 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.754 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.754 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore": { 2024-09-21T09:31:06.754 INFO:tasks.workunit.client.0.smithi022.stdout: "alloc_unit": { 2024-09-21T09:31:06.755 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 4096 2024-09-21T09:31:06.755 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.755 INFO:tasks.workunit.client.0.smithi022.stdout: "allocated": { 2024-09-21T09:31:06.755 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 700416 2024-09-21T09:31:06.755 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.755 INFO:tasks.workunit.client.0.smithi022.stdout: "clist_lat": { 2024-09-21T09:31:06.755 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 5, 2024-09-21T09:31:06.755 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 212127 2024-09-21T09:31:06.755 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.755 INFO:tasks.workunit.client.0.smithi022.stdout: "compress_lat": { 2024-09-21T09:31:06.755 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.755 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.755 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.755 INFO:tasks.workunit.client.0.smithi022.stdout: "compressed": { 2024-09-21T09:31:06.756 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.756 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.756 INFO:tasks.workunit.client.0.smithi022.stdout: "compressed_allocated": { 2024-09-21T09:31:06.756 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.756 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.756 INFO:tasks.workunit.client.0.smithi022.stdout: "compressed_original": { 2024-09-21T09:31:06.756 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.756 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.756 INFO:tasks.workunit.client.0.smithi022.stdout: "csum_lat": { 2024-09-21T09:31:06.756 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 23, 2024-09-21T09:31:06.757 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 73883 2024-09-21T09:31:06.757 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.757 INFO:tasks.workunit.client.0.smithi022.stdout: "decompress_lat": { 2024-09-21T09:31:06.757 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.757 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.757 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.757 INFO:tasks.workunit.client.0.smithi022.stdout: "kv_commit_lat": { 2024-09-21T09:31:06.757 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 305, 2024-09-21T09:31:06.757 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 59745722 2024-09-21T09:31:06.757 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.757 INFO:tasks.workunit.client.0.smithi022.stdout: "kv_final_lat": { 2024-09-21T09:31:06.757 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 302, 2024-09-21T09:31:06.757 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 10574229 2024-09-21T09:31:06.758 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.758 INFO:tasks.workunit.client.0.smithi022.stdout: "kv_flush_lat": { 2024-09-21T09:31:06.758 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 305, 2024-09-21T09:31:06.758 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 434879 2024-09-21T09:31:06.758 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.758 INFO:tasks.workunit.client.0.smithi022.stdout: "kv_sync_lat": { 2024-09-21T09:31:06.758 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 305, 2024-09-21T09:31:06.758 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 60180601 2024-09-21T09:31:06.758 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.758 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_get_keys_lat": { 2024-09-21T09:31:06.758 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.758 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.758 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.758 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_get_values_lat": { 2024-09-21T09:31:06.759 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 3, 2024-09-21T09:31:06.759 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 38458 2024-09-21T09:31:06.759 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.759 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_lower_bound_lat": { 2024-09-21T09:31:06.759 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 5, 2024-09-21T09:31:06.759 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 10824 2024-09-21T09:31:06.759 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.759 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_next_lat": { 2024-09-21T09:31:06.759 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.759 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.759 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.759 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_seek_to_first_lat": { 2024-09-21T09:31:06.759 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.759 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.760 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.760 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_upper_bound_lat": { 2024-09-21T09:31:06.760 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 5, 2024-09-21T09:31:06.760 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 14996 2024-09-21T09:31:06.760 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.760 INFO:tasks.workunit.client.0.smithi022.stdout: "onode_hits": { 2024-09-21T09:31:06.760 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 685 2024-09-21T09:31:06.760 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.760 INFO:tasks.workunit.client.0.smithi022.stdout: "onode_misses": { 2024-09-21T09:31:06.760 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 224 2024-09-21T09:31:06.760 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.760 INFO:tasks.workunit.client.0.smithi022.stdout: "read_lat": { 2024-09-21T09:31:06.760 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 125, 2024-09-21T09:31:06.760 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1976819 2024-09-21T09:31:06.760 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.761 INFO:tasks.workunit.client.0.smithi022.stdout: "read_onode_meta_lat": { 2024-09-21T09:31:06.761 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 291, 2024-09-21T09:31:06.761 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 280425 2024-09-21T09:31:06.761 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.761 INFO:tasks.workunit.client.0.smithi022.stdout: "read_wait_aio_lat": { 2024-09-21T09:31:06.761 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 166, 2024-09-21T09:31:06.761 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1109474 2024-09-21T09:31:06.761 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.761 INFO:tasks.workunit.client.0.smithi022.stdout: "reads_with_retries": { 2024-09-21T09:31:06.761 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.761 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.761 INFO:tasks.workunit.client.0.smithi022.stdout: "remove_lat": { 2024-09-21T09:31:06.761 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 3, 2024-09-21T09:31:06.761 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 105215 2024-09-21T09:31:06.762 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.762 INFO:tasks.workunit.client.0.smithi022.stdout: "state_aio_wait_lat": { 2024-09-21T09:31:06.762 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 360, 2024-09-21T09:31:06.762 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 5779123 2024-09-21T09:31:06.762 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.762 INFO:tasks.workunit.client.0.smithi022.stdout: "state_deferred_aio_wait_lat": { 2024-09-21T09:31:06.762 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 96, 2024-09-21T09:31:06.762 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 6923216 2024-09-21T09:31:06.762 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.762 INFO:tasks.workunit.client.0.smithi022.stdout: "state_deferred_cleanup_lat": { 2024-09-21T09:31:06.762 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 95, 2024-09-21T09:31:06.762 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 405290222809 2024-09-21T09:31:06.762 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.763 INFO:tasks.workunit.client.0.smithi022.stdout: "state_deferred_queued_lat": { 2024-09-21T09:31:06.763 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 96, 2024-09-21T09:31:06.763 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 156744314046 2024-09-21T09:31:06.763 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.763 INFO:tasks.workunit.client.0.smithi022.stdout: "state_done_lat": { 2024-09-21T09:31:06.763 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 359, 2024-09-21T09:31:06.763 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 77402288748 2024-09-21T09:31:06.763 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.763 INFO:tasks.workunit.client.0.smithi022.stdout: "state_finishing_lat": { 2024-09-21T09:31:06.763 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 359, 2024-09-21T09:31:06.763 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 39606 2024-09-21T09:31:06.763 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.763 INFO:tasks.workunit.client.0.smithi022.stdout: "state_io_done_lat": { 2024-09-21T09:31:06.763 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 360, 2024-09-21T09:31:06.763 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 389361 2024-09-21T09:31:06.764 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.764 INFO:tasks.workunit.client.0.smithi022.stdout: "state_kv_commiting_lat": { 2024-09-21T09:31:06.764 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 360, 2024-09-21T09:31:06.764 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 95465222 2024-09-21T09:31:06.764 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.764 INFO:tasks.workunit.client.0.smithi022.stdout: "state_kv_done_lat": { 2024-09-21T09:31:06.764 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 360, 2024-09-21T09:31:06.764 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 149571 2024-09-21T09:31:06.764 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.764 INFO:tasks.workunit.client.0.smithi022.stdout: "state_kv_queued_lat": { 2024-09-21T09:31:06.764 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 360, 2024-09-21T09:31:06.764 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 46182165 2024-09-21T09:31:06.764 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.764 INFO:tasks.workunit.client.0.smithi022.stdout: "state_prepare_lat": { 2024-09-21T09:31:06.765 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 360, 2024-09-21T09:31:06.765 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 21284805 2024-09-21T09:31:06.765 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.765 INFO:tasks.workunit.client.0.smithi022.stdout: "stored": { 2024-09-21T09:31:06.765 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 310158 2024-09-21T09:31:06.765 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.765 INFO:tasks.workunit.client.0.smithi022.stdout: "truncate_lat": { 2024-09-21T09:31:06.765 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 55, 2024-09-21T09:31:06.765 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 318589 2024-09-21T09:31:06.765 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.765 INFO:tasks.workunit.client.0.smithi022.stdout: "txc_commit_lat": { 2024-09-21T09:31:06.765 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 360, 2024-09-21T09:31:06.765 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 169171709 2024-09-21T09:31:06.766 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.766 INFO:tasks.workunit.client.0.smithi022.stdout: "txc_submit_lat": { 2024-09-21T09:31:06.766 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 360, 2024-09-21T09:31:06.766 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 25744325 2024-09-21T09:31:06.766 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.766 INFO:tasks.workunit.client.0.smithi022.stdout: "txc_throttle_lat": { 2024-09-21T09:31:06.766 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 360, 2024-09-21T09:31:06.766 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 4315358 2024-09-21T09:31:06.766 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.766 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.766 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore-pricache": { 2024-09-21T09:31:06.766 INFO:tasks.workunit.client.0.smithi022.stdout: "cache_bytes": { 2024-09-21T09:31:06.766 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1862400347 2024-09-21T09:31:06.766 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.767 INFO:tasks.workunit.client.0.smithi022.stdout: "heap_bytes": { 2024-09-21T09:31:06.767 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 270974976 2024-09-21T09:31:06.767 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.767 INFO:tasks.workunit.client.0.smithi022.stdout: "mapped_bytes": { 2024-09-21T09:31:06.767 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 269991936 2024-09-21T09:31:06.767 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.767 INFO:tasks.workunit.client.0.smithi022.stdout: "target_bytes": { 2024-09-21T09:31:06.767 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 3138478489 2024-09-21T09:31:06.767 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.767 INFO:tasks.workunit.client.0.smithi022.stdout: "unmapped_bytes": { 2024-09-21T09:31:06.767 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 983040 2024-09-21T09:31:06.767 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.767 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.768 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore-pricache:data": { 2024-09-21T09:31:06.768 INFO:tasks.workunit.client.0.smithi022.stdout: "committed_bytes": { 2024-09-21T09:31:06.768 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 159383552 2024-09-21T09:31:06.768 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.768 INFO:tasks.workunit.client.0.smithi022.stdout: "pri0_bytes": { 2024-09-21T09:31:06.768 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.768 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.768 INFO:tasks.workunit.client.0.smithi022.stdout: "pri10_bytes": { 2024-09-21T09:31:06.768 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.768 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.768 INFO:tasks.workunit.client.0.smithi022.stdout: "pri11_bytes": { 2024-09-21T09:31:06.768 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 85555426 2024-09-21T09:31:06.769 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.769 INFO:tasks.workunit.client.0.smithi022.stdout: "pri1_bytes": { 2024-09-21T09:31:06.769 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.769 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.769 INFO:tasks.workunit.client.0.smithi022.stdout: "pri2_bytes": { 2024-09-21T09:31:06.769 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.769 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.769 INFO:tasks.workunit.client.0.smithi022.stdout: "pri3_bytes": { 2024-09-21T09:31:06.769 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.769 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.769 INFO:tasks.workunit.client.0.smithi022.stdout: "pri4_bytes": { 2024-09-21T09:31:06.769 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.769 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.770 INFO:tasks.workunit.client.0.smithi022.stdout: "pri5_bytes": { 2024-09-21T09:31:06.770 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.770 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.770 INFO:tasks.workunit.client.0.smithi022.stdout: "pri6_bytes": { 2024-09-21T09:31:06.770 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 8192 2024-09-21T09:31:06.770 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.770 INFO:tasks.workunit.client.0.smithi022.stdout: "pri7_bytes": { 2024-09-21T09:31:06.770 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.770 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.770 INFO:tasks.workunit.client.0.smithi022.stdout: "pri8_bytes": { 2024-09-21T09:31:06.770 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.770 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.770 INFO:tasks.workunit.client.0.smithi022.stdout: "pri9_bytes": { 2024-09-21T09:31:06.771 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.771 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.771 INFO:tasks.workunit.client.0.smithi022.stdout: "reserved_bytes": { 2024-09-21T09:31:06.771 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 73819934 2024-09-21T09:31:06.771 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.771 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.771 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore-pricache:kv": { 2024-09-21T09:31:06.771 INFO:tasks.workunit.client.0.smithi022.stdout: "committed_bytes": { 2024-09-21T09:31:06.771 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 780140544 2024-09-21T09:31:06.771 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.771 INFO:tasks.workunit.client.0.smithi022.stdout: "pri0_bytes": { 2024-09-21T09:31:06.771 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 67108864 2024-09-21T09:31:06.771 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.772 INFO:tasks.workunit.client.0.smithi022.stdout: "pri10_bytes": { 2024-09-21T09:31:06.772 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.772 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.772 INFO:tasks.workunit.client.0.smithi022.stdout: "pri11_bytes": { 2024-09-21T09:31:06.772 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 641665696 2024-09-21T09:31:06.772 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.772 INFO:tasks.workunit.client.0.smithi022.stdout: "pri1_bytes": { 2024-09-21T09:31:06.772 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.772 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.772 INFO:tasks.workunit.client.0.smithi022.stdout: "pri2_bytes": { 2024-09-21T09:31:06.772 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.772 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.772 INFO:tasks.workunit.client.0.smithi022.stdout: "pri3_bytes": { 2024-09-21T09:31:06.773 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.773 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.773 INFO:tasks.workunit.client.0.smithi022.stdout: "pri4_bytes": { 2024-09-21T09:31:06.773 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.773 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.773 INFO:tasks.workunit.client.0.smithi022.stdout: "pri5_bytes": { 2024-09-21T09:31:06.773 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.773 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.773 INFO:tasks.workunit.client.0.smithi022.stdout: "pri6_bytes": { 2024-09-21T09:31:06.773 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2128 2024-09-21T09:31:06.773 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.773 INFO:tasks.workunit.client.0.smithi022.stdout: "pri7_bytes": { 2024-09-21T09:31:06.774 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.774 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.774 INFO:tasks.workunit.client.0.smithi022.stdout: "pri8_bytes": { 2024-09-21T09:31:06.774 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.774 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.774 INFO:tasks.workunit.client.0.smithi022.stdout: "pri9_bytes": { 2024-09-21T09:31:06.774 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.774 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.774 INFO:tasks.workunit.client.0.smithi022.stdout: "reserved_bytes": { 2024-09-21T09:31:06.774 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 71363856 2024-09-21T09:31:06.774 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.775 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.775 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore-pricache:kv_onode": { 2024-09-21T09:31:06.775 INFO:tasks.workunit.client.0.smithi022.stdout: "committed_bytes": { 2024-09-21T09:31:06.775 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 192937984 2024-09-21T09:31:06.775 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.775 INFO:tasks.workunit.client.0.smithi022.stdout: "pri0_bytes": { 2024-09-21T09:31:06.775 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 67108864 2024-09-21T09:31:06.775 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.775 INFO:tasks.workunit.client.0.smithi022.stdout: "pri10_bytes": { 2024-09-21T09:31:06.775 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 464 2024-09-21T09:31:06.775 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.775 INFO:tasks.workunit.client.0.smithi022.stdout: "pri11_bytes": { 2024-09-21T09:31:06.776 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 57036950 2024-09-21T09:31:06.776 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.776 INFO:tasks.workunit.client.0.smithi022.stdout: "pri1_bytes": { 2024-09-21T09:31:06.776 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.776 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.776 INFO:tasks.workunit.client.0.smithi022.stdout: "pri2_bytes": { 2024-09-21T09:31:06.776 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.776 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.776 INFO:tasks.workunit.client.0.smithi022.stdout: "pri3_bytes": { 2024-09-21T09:31:06.776 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.776 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.776 INFO:tasks.workunit.client.0.smithi022.stdout: "pri4_bytes": { 2024-09-21T09:31:06.776 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.776 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.776 INFO:tasks.workunit.client.0.smithi022.stdout: "pri5_bytes": { 2024-09-21T09:31:06.777 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.777 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.777 INFO:tasks.workunit.client.0.smithi022.stdout: "pri6_bytes": { 2024-09-21T09:31:06.777 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.777 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.777 INFO:tasks.workunit.client.0.smithi022.stdout: "pri7_bytes": { 2024-09-21T09:31:06.777 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.777 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.777 INFO:tasks.workunit.client.0.smithi022.stdout: "pri8_bytes": { 2024-09-21T09:31:06.777 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.778 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.778 INFO:tasks.workunit.client.0.smithi022.stdout: "pri9_bytes": { 2024-09-21T09:31:06.778 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.778 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.778 INFO:tasks.workunit.client.0.smithi022.stdout: "reserved_bytes": { 2024-09-21T09:31:06.778 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 68791706 2024-09-21T09:31:06.778 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.778 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.778 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore-pricache:meta": { 2024-09-21T09:31:06.778 INFO:tasks.workunit.client.0.smithi022.stdout: "committed_bytes": { 2024-09-21T09:31:06.778 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 713031680 2024-09-21T09:31:06.778 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.778 INFO:tasks.workunit.client.0.smithi022.stdout: "pri0_bytes": { 2024-09-21T09:31:06.778 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.779 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.779 INFO:tasks.workunit.client.0.smithi022.stdout: "pri10_bytes": { 2024-09-21T09:31:06.779 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.779 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.779 INFO:tasks.workunit.client.0.smithi022.stdout: "pri11_bytes": { 2024-09-21T09:31:06.779 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 641665696 2024-09-21T09:31:06.779 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.779 INFO:tasks.workunit.client.0.smithi022.stdout: "pri1_bytes": { 2024-09-21T09:31:06.779 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.779 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.780 INFO:tasks.workunit.client.0.smithi022.stdout: "pri2_bytes": { 2024-09-21T09:31:06.780 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.780 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.780 INFO:tasks.workunit.client.0.smithi022.stdout: "pri3_bytes": { 2024-09-21T09:31:06.780 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.780 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.780 INFO:tasks.workunit.client.0.smithi022.stdout: "pri4_bytes": { 2024-09-21T09:31:06.780 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.780 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.780 INFO:tasks.workunit.client.0.smithi022.stdout: "pri5_bytes": { 2024-09-21T09:31:06.780 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.780 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.781 INFO:tasks.workunit.client.0.smithi022.stdout: "pri6_bytes": { 2024-09-21T09:31:06.781 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 255384 2024-09-21T09:31:06.781 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.781 INFO:tasks.workunit.client.0.smithi022.stdout: "pri7_bytes": { 2024-09-21T09:31:06.781 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.781 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.781 INFO:tasks.workunit.client.0.smithi022.stdout: "pri8_bytes": { 2024-09-21T09:31:06.781 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.781 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.782 INFO:tasks.workunit.client.0.smithi022.stdout: "pri9_bytes": { 2024-09-21T09:31:06.782 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.782 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.782 INFO:tasks.workunit.client.0.smithi022.stdout: "reserved_bytes": { 2024-09-21T09:31:06.782 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 71110600 2024-09-21T09:31:06.782 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.782 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.782 INFO:tasks.workunit.client.0.smithi022.stdout: "objecter": { 2024-09-21T09:31:06.782 INFO:tasks.workunit.client.0.smithi022.stdout: "op_active": { 2024-09-21T09:31:06.782 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.782 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.782 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r": { 2024-09-21T09:31:06.782 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.783 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.783 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rmw": { 2024-09-21T09:31:06.783 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.783 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.783 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w": { 2024-09-21T09:31:06.783 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.783 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.783 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.783 INFO:tasks.workunit.client.0.smithi022.stdout: "osd": { 2024-09-21T09:31:06.783 INFO:tasks.workunit.client.0.smithi022.stdout: "numpg": { 2024-09-21T09:31:06.783 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 38 2024-09-21T09:31:06.783 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.783 INFO:tasks.workunit.client.0.smithi022.stdout: "numpg_removing": { 2024-09-21T09:31:06.784 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.784 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.784 INFO:tasks.workunit.client.0.smithi022.stdout: "op": { 2024-09-21T09:31:06.784 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 97 2024-09-21T09:31:06.784 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.784 INFO:tasks.workunit.client.0.smithi022.stdout: "op_delayed_degraded": { 2024-09-21T09:31:06.784 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.784 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.784 INFO:tasks.workunit.client.0.smithi022.stdout: "op_delayed_unreadable": { 2024-09-21T09:31:06.784 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.784 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.784 INFO:tasks.workunit.client.0.smithi022.stdout: "op_in_bytes": { 2024-09-21T09:31:06.784 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1640827 2024-09-21T09:31:06.785 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.785 INFO:tasks.workunit.client.0.smithi022.stdout: "op_latency": { 2024-09-21T09:31:06.785 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 97, 2024-09-21T09:31:06.785 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 368569962 2024-09-21T09:31:06.785 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.785 INFO:tasks.workunit.client.0.smithi022.stdout: "op_out_bytes": { 2024-09-21T09:31:06.785 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 602 2024-09-21T09:31:06.785 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.785 INFO:tasks.workunit.client.0.smithi022.stdout: "op_prepare_latency": { 2024-09-21T09:31:06.785 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 97, 2024-09-21T09:31:06.785 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 33419604 2024-09-21T09:31:06.785 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.785 INFO:tasks.workunit.client.0.smithi022.stdout: "op_process_latency": { 2024-09-21T09:31:06.785 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 97, 2024-09-21T09:31:06.786 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 106861826 2024-09-21T09:31:06.786 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.786 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r": { 2024-09-21T09:31:06.786 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 18 2024-09-21T09:31:06.786 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.786 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r_latency": { 2024-09-21T09:31:06.786 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 18, 2024-09-21T09:31:06.786 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2701008 2024-09-21T09:31:06.786 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.786 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-21T09:31:06.786 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.786 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.786 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r_out_bytes": { 2024-09-21T09:31:06.787 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 602 2024-09-21T09:31:06.787 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.787 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r_prepare_latency": { 2024-09-21T09:31:06.787 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 18, 2024-09-21T09:31:06.787 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2305491 2024-09-21T09:31:06.787 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.787 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r_process_latency": { 2024-09-21T09:31:06.787 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 18, 2024-09-21T09:31:06.787 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1860842 2024-09-21T09:31:06.787 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.787 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw": { 2024-09-21T09:31:06.787 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 12 2024-09-21T09:31:06.787 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.787 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_in_bytes": { 2024-09-21T09:31:06.788 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.788 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.788 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_latency": { 2024-09-21T09:31:06.788 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 12, 2024-09-21T09:31:06.788 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 10826824 2024-09-21T09:31:06.788 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.788 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-21T09:31:06.788 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.788 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.788 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-21T09:31:06.788 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.788 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.788 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_out_bytes": { 2024-09-21T09:31:06.788 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.788 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.789 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_prepare_latency": { 2024-09-21T09:31:06.789 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 12, 2024-09-21T09:31:06.789 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 4695971 2024-09-21T09:31:06.789 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.789 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_process_latency": { 2024-09-21T09:31:06.789 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 12, 2024-09-21T09:31:06.789 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 10130626 2024-09-21T09:31:06.789 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.789 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w": { 2024-09-21T09:31:06.789 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 67 2024-09-21T09:31:06.789 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.789 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w_in_bytes": { 2024-09-21T09:31:06.789 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1640827 2024-09-21T09:31:06.790 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.790 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w_latency": { 2024-09-21T09:31:06.790 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 67, 2024-09-21T09:31:06.790 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 355042130 2024-09-21T09:31:06.790 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.790 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-21T09:31:06.790 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.790 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.790 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w_prepare_latency": { 2024-09-21T09:31:06.790 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 67, 2024-09-21T09:31:06.790 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 26418142 2024-09-21T09:31:06.790 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.790 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w_process_latency": { 2024-09-21T09:31:06.790 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 67, 2024-09-21T09:31:06.791 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 94870358 2024-09-21T09:31:06.791 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.791 INFO:tasks.workunit.client.0.smithi022.stdout: "op_wip": { 2024-09-21T09:31:06.791 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.791 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.791 INFO:tasks.workunit.client.0.smithi022.stdout: "recovery_bytes": { 2024-09-21T09:31:06.791 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 590368 2024-09-21T09:31:06.791 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.791 INFO:tasks.workunit.client.0.smithi022.stdout: "recovery_ops": { 2024-09-21T09:31:06.791 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2 2024-09-21T09:31:06.791 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.791 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_bytes": { 2024-09-21T09:31:06.792 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 95995035648 2024-09-21T09:31:06.792 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.792 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_bytes_used": { 2024-09-21T09:31:06.792 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 305192960 2024-09-21T09:31:06.792 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.792 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.792 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.792 INFO:tasks.workunit.client.0.smithi022.stdout: "osd.2": { 2024-09-21T09:31:06.792 INFO:tasks.workunit.client.0.smithi022.stdout: "bluefs": { 2024-09-21T09:31:06.792 INFO:tasks.workunit.client.0.smithi022.stdout: "alloc_slow_fallback": { 2024-09-21T09:31:06.792 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.792 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.792 INFO:tasks.workunit.client.0.smithi022.stdout: "alloc_slow_size_fallback": { 2024-09-21T09:31:06.793 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.793 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.793 INFO:tasks.workunit.client.0.smithi022.stdout: "alloc_unit_db": { 2024-09-21T09:31:06.793 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 65536 2024-09-21T09:31:06.793 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.793 INFO:tasks.workunit.client.0.smithi022.stdout: "alloc_unit_main": { 2024-09-21T09:31:06.793 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.793 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.793 INFO:tasks.workunit.client.0.smithi022.stdout: "alloc_unit_wal": { 2024-09-21T09:31:06.793 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.793 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.793 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes_written_slow": { 2024-09-21T09:31:06.794 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.794 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.794 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes_written_sst": { 2024-09-21T09:31:06.794 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 12288 2024-09-21T09:31:06.794 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.794 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes_written_wal": { 2024-09-21T09:31:06.794 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1703936 2024-09-21T09:31:06.794 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.794 INFO:tasks.workunit.client.0.smithi022.stdout: "compact_lat": { 2024-09-21T09:31:06.794 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.794 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.794 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.795 INFO:tasks.workunit.client.0.smithi022.stdout: "compact_lock_lat": { 2024-09-21T09:31:06.795 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.795 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.795 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.795 INFO:tasks.workunit.client.0.smithi022.stdout: "db_total_bytes": { 2024-09-21T09:31:06.795 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 95995035648 2024-09-21T09:31:06.795 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.795 INFO:tasks.workunit.client.0.smithi022.stdout: "db_used_bytes": { 2024-09-21T09:31:06.795 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 304480256 2024-09-21T09:31:06.795 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.795 INFO:tasks.workunit.client.0.smithi022.stdout: "log_bytes": { 2024-09-21T09:31:06.795 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1146880 2024-09-21T09:31:06.795 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.795 INFO:tasks.workunit.client.0.smithi022.stdout: "logged_bytes": { 2024-09-21T09:31:06.796 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1007616 2024-09-21T09:31:06.796 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.796 INFO:tasks.workunit.client.0.smithi022.stdout: "max_bytes_db": { 2024-09-21T09:31:06.796 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 312868864 2024-09-21T09:31:06.796 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.796 INFO:tasks.workunit.client.0.smithi022.stdout: "max_bytes_slow": { 2024-09-21T09:31:06.796 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.796 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.796 INFO:tasks.workunit.client.0.smithi022.stdout: "max_bytes_wal": { 2024-09-21T09:31:06.796 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.796 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.796 INFO:tasks.workunit.client.0.smithi022.stdout: "num_files": { 2024-09-21T09:31:06.796 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 14 2024-09-21T09:31:06.797 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.797 INFO:tasks.workunit.client.0.smithi022.stdout: "read_bytes": { 2024-09-21T09:31:06.797 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 195697 2024-09-21T09:31:06.797 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.797 INFO:tasks.workunit.client.0.smithi022.stdout: "read_count": { 2024-09-21T09:31:06.797 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 23 2024-09-21T09:31:06.797 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.797 INFO:tasks.workunit.client.0.smithi022.stdout: "read_disk_bytes": { 2024-09-21T09:31:06.797 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1269760 2024-09-21T09:31:06.797 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.797 INFO:tasks.workunit.client.0.smithi022.stdout: "read_disk_bytes_db": { 2024-09-21T09:31:06.797 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1273856 2024-09-21T09:31:06.797 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.797 INFO:tasks.workunit.client.0.smithi022.stdout: "read_disk_bytes_slow": { 2024-09-21T09:31:06.798 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.798 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.798 INFO:tasks.workunit.client.0.smithi022.stdout: "read_disk_bytes_wal": { 2024-09-21T09:31:06.798 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.798 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.798 INFO:tasks.workunit.client.0.smithi022.stdout: "read_disk_count": { 2024-09-21T09:31:06.798 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 15 2024-09-21T09:31:06.798 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.798 INFO:tasks.workunit.client.0.smithi022.stdout: "read_prefetch_bytes": { 2024-09-21T09:31:06.798 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 4789 2024-09-21T09:31:06.798 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.798 INFO:tasks.workunit.client.0.smithi022.stdout: "read_prefetch_count": { 2024-09-21T09:31:06.798 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 4 2024-09-21T09:31:06.798 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.799 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_buffer_bytes": { 2024-09-21T09:31:06.799 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 5589 2024-09-21T09:31:06.799 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.799 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_buffer_count": { 2024-09-21T09:31:06.799 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 24 2024-09-21T09:31:06.799 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.799 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_bytes": { 2024-09-21T09:31:06.799 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 5589 2024-09-21T09:31:06.799 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.799 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_count": { 2024-09-21T09:31:06.799 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 24 2024-09-21T09:31:06.799 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.799 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_disk_bytes": { 2024-09-21T09:31:06.800 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.800 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.800 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_disk_bytes_db": { 2024-09-21T09:31:06.800 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.800 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.800 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_disk_bytes_slow": { 2024-09-21T09:31:06.800 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.800 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.800 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_disk_bytes_wal": { 2024-09-21T09:31:06.800 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.800 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.800 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_disk_count": { 2024-09-21T09:31:06.800 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.801 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.801 INFO:tasks.workunit.client.0.smithi022.stdout: "slow_total_bytes": { 2024-09-21T09:31:06.801 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.801 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.801 INFO:tasks.workunit.client.0.smithi022.stdout: "slow_used_bytes": { 2024-09-21T09:31:06.801 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.801 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.801 INFO:tasks.workunit.client.0.smithi022.stdout: "wal_total_bytes": { 2024-09-21T09:31:06.801 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.801 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.801 INFO:tasks.workunit.client.0.smithi022.stdout: "wal_used_bytes": { 2024-09-21T09:31:06.801 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.801 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.801 INFO:tasks.workunit.client.0.smithi022.stdout: "write_bytes": { 2024-09-21T09:31:06.801 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2785280 2024-09-21T09:31:06.802 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.802 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.802 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore": { 2024-09-21T09:31:06.802 INFO:tasks.workunit.client.0.smithi022.stdout: "alloc_unit": { 2024-09-21T09:31:06.802 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 4096 2024-09-21T09:31:06.802 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.802 INFO:tasks.workunit.client.0.smithi022.stdout: "allocated": { 2024-09-21T09:31:06.802 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 688128 2024-09-21T09:31:06.802 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.802 INFO:tasks.workunit.client.0.smithi022.stdout: "clist_lat": { 2024-09-21T09:31:06.802 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 4, 2024-09-21T09:31:06.802 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 197871 2024-09-21T09:31:06.802 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.803 INFO:tasks.workunit.client.0.smithi022.stdout: "compress_lat": { 2024-09-21T09:31:06.803 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.803 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.803 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.803 INFO:tasks.workunit.client.0.smithi022.stdout: "compressed": { 2024-09-21T09:31:06.803 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.803 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.803 INFO:tasks.workunit.client.0.smithi022.stdout: "compressed_allocated": { 2024-09-21T09:31:06.803 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.803 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.803 INFO:tasks.workunit.client.0.smithi022.stdout: "compressed_original": { 2024-09-21T09:31:06.803 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.803 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.803 INFO:tasks.workunit.client.0.smithi022.stdout: "csum_lat": { 2024-09-21T09:31:06.804 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 10, 2024-09-21T09:31:06.804 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 11569 2024-09-21T09:31:06.804 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.804 INFO:tasks.workunit.client.0.smithi022.stdout: "decompress_lat": { 2024-09-21T09:31:06.804 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.804 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.804 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.804 INFO:tasks.workunit.client.0.smithi022.stdout: "kv_commit_lat": { 2024-09-21T09:31:06.804 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 229, 2024-09-21T09:31:06.804 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 40494224 2024-09-21T09:31:06.804 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.804 INFO:tasks.workunit.client.0.smithi022.stdout: "kv_final_lat": { 2024-09-21T09:31:06.804 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 225, 2024-09-21T09:31:06.804 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 8386751 2024-09-21T09:31:06.805 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.805 INFO:tasks.workunit.client.0.smithi022.stdout: "kv_flush_lat": { 2024-09-21T09:31:06.805 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 229, 2024-09-21T09:31:06.805 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 340445 2024-09-21T09:31:06.805 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.805 INFO:tasks.workunit.client.0.smithi022.stdout: "kv_sync_lat": { 2024-09-21T09:31:06.805 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 229, 2024-09-21T09:31:06.805 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 40834669 2024-09-21T09:31:06.805 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.805 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_get_keys_lat": { 2024-09-21T09:31:06.805 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.805 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.805 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.805 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_get_values_lat": { 2024-09-21T09:31:06.805 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 3, 2024-09-21T09:31:06.806 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 40544 2024-09-21T09:31:06.806 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.806 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_lower_bound_lat": { 2024-09-21T09:31:06.806 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 2, 2024-09-21T09:31:06.806 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1415 2024-09-21T09:31:06.806 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.806 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_next_lat": { 2024-09-21T09:31:06.806 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.806 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.806 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.806 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_seek_to_first_lat": { 2024-09-21T09:31:06.806 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.806 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.807 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.807 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_upper_bound_lat": { 2024-09-21T09:31:06.807 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 2, 2024-09-21T09:31:06.807 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 7390 2024-09-21T09:31:06.807 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.807 INFO:tasks.workunit.client.0.smithi022.stdout: "onode_hits": { 2024-09-21T09:31:06.807 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 509 2024-09-21T09:31:06.807 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.807 INFO:tasks.workunit.client.0.smithi022.stdout: "onode_misses": { 2024-09-21T09:31:06.807 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 205 2024-09-21T09:31:06.807 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.807 INFO:tasks.workunit.client.0.smithi022.stdout: "read_lat": { 2024-09-21T09:31:06.807 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 96, 2024-09-21T09:31:06.807 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 901168 2024-09-21T09:31:06.808 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.808 INFO:tasks.workunit.client.0.smithi022.stdout: "read_onode_meta_lat": { 2024-09-21T09:31:06.808 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 227, 2024-09-21T09:31:06.808 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 237836 2024-09-21T09:31:06.808 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.808 INFO:tasks.workunit.client.0.smithi022.stdout: "read_wait_aio_lat": { 2024-09-21T09:31:06.808 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 131, 2024-09-21T09:31:06.808 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 472466 2024-09-21T09:31:06.808 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.808 INFO:tasks.workunit.client.0.smithi022.stdout: "reads_with_retries": { 2024-09-21T09:31:06.808 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.808 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.808 INFO:tasks.workunit.client.0.smithi022.stdout: "remove_lat": { 2024-09-21T09:31:06.808 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 3, 2024-09-21T09:31:06.808 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 105977 2024-09-21T09:31:06.809 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.809 INFO:tasks.workunit.client.0.smithi022.stdout: "state_aio_wait_lat": { 2024-09-21T09:31:06.809 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 287, 2024-09-21T09:31:06.809 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 5234531 2024-09-21T09:31:06.809 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.809 INFO:tasks.workunit.client.0.smithi022.stdout: "state_deferred_aio_wait_lat": { 2024-09-21T09:31:06.809 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 89, 2024-09-21T09:31:06.809 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 6506780 2024-09-21T09:31:06.809 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.809 INFO:tasks.workunit.client.0.smithi022.stdout: "state_deferred_cleanup_lat": { 2024-09-21T09:31:06.809 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 88, 2024-09-21T09:31:06.809 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 330243082590 2024-09-21T09:31:06.809 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.809 INFO:tasks.workunit.client.0.smithi022.stdout: "state_deferred_queued_lat": { 2024-09-21T09:31:06.810 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 89, 2024-09-21T09:31:06.810 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 119707374385 2024-09-21T09:31:06.810 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.810 INFO:tasks.workunit.client.0.smithi022.stdout: "state_done_lat": { 2024-09-21T09:31:06.810 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 286, 2024-09-21T09:31:06.810 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 77969329092 2024-09-21T09:31:06.810 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.810 INFO:tasks.workunit.client.0.smithi022.stdout: "state_finishing_lat": { 2024-09-21T09:31:06.810 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 286, 2024-09-21T09:31:06.810 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 29304 2024-09-21T09:31:06.810 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.810 INFO:tasks.workunit.client.0.smithi022.stdout: "state_io_done_lat": { 2024-09-21T09:31:06.810 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 287, 2024-09-21T09:31:06.810 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 352169 2024-09-21T09:31:06.811 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.811 INFO:tasks.workunit.client.0.smithi022.stdout: "state_kv_commiting_lat": { 2024-09-21T09:31:06.811 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 287, 2024-09-21T09:31:06.811 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 95585029 2024-09-21T09:31:06.811 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.811 INFO:tasks.workunit.client.0.smithi022.stdout: "state_kv_done_lat": { 2024-09-21T09:31:06.811 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 287, 2024-09-21T09:31:06.811 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 105994 2024-09-21T09:31:06.811 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.811 INFO:tasks.workunit.client.0.smithi022.stdout: "state_kv_queued_lat": { 2024-09-21T09:31:06.811 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 287, 2024-09-21T09:31:06.811 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 79771479 2024-09-21T09:31:06.811 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.811 INFO:tasks.workunit.client.0.smithi022.stdout: "state_prepare_lat": { 2024-09-21T09:31:06.812 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 287, 2024-09-21T09:31:06.812 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 16479816 2024-09-21T09:31:06.812 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.812 INFO:tasks.workunit.client.0.smithi022.stdout: "stored": { 2024-09-21T09:31:06.812 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 309979 2024-09-21T09:31:06.812 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.812 INFO:tasks.workunit.client.0.smithi022.stdout: "truncate_lat": { 2024-09-21T09:31:06.812 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 49, 2024-09-21T09:31:06.812 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 232193 2024-09-21T09:31:06.812 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.812 INFO:tasks.workunit.client.0.smithi022.stdout: "txc_commit_lat": { 2024-09-21T09:31:06.812 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 287, 2024-09-21T09:31:06.812 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 197473775 2024-09-21T09:31:06.812 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.812 INFO:tasks.workunit.client.0.smithi022.stdout: "txc_submit_lat": { 2024-09-21T09:31:06.812 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 287, 2024-09-21T09:31:06.813 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 19985041 2024-09-21T09:31:06.813 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.813 INFO:tasks.workunit.client.0.smithi022.stdout: "txc_throttle_lat": { 2024-09-21T09:31:06.813 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 287, 2024-09-21T09:31:06.813 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1943254 2024-09-21T09:31:06.813 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.813 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.813 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore-pricache": { 2024-09-21T09:31:06.813 INFO:tasks.workunit.client.0.smithi022.stdout: "cache_bytes": { 2024-09-21T09:31:06.813 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1862400347 2024-09-21T09:31:06.813 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.813 INFO:tasks.workunit.client.0.smithi022.stdout: "heap_bytes": { 2024-09-21T09:31:06.813 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 270139392 2024-09-21T09:31:06.813 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.813 INFO:tasks.workunit.client.0.smithi022.stdout: "mapped_bytes": { 2024-09-21T09:31:06.814 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 269344768 2024-09-21T09:31:06.814 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.814 INFO:tasks.workunit.client.0.smithi022.stdout: "target_bytes": { 2024-09-21T09:31:06.814 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 3138478489 2024-09-21T09:31:06.814 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.814 INFO:tasks.workunit.client.0.smithi022.stdout: "unmapped_bytes": { 2024-09-21T09:31:06.814 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 794624 2024-09-21T09:31:06.814 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.814 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.814 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore-pricache:data": { 2024-09-21T09:31:06.814 INFO:tasks.workunit.client.0.smithi022.stdout: "committed_bytes": { 2024-09-21T09:31:06.814 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 159383552 2024-09-21T09:31:06.814 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.814 INFO:tasks.workunit.client.0.smithi022.stdout: "pri0_bytes": { 2024-09-21T09:31:06.815 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.815 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.815 INFO:tasks.workunit.client.0.smithi022.stdout: "pri10_bytes": { 2024-09-21T09:31:06.815 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.815 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.815 INFO:tasks.workunit.client.0.smithi022.stdout: "pri11_bytes": { 2024-09-21T09:31:06.815 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 85556261 2024-09-21T09:31:06.815 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.815 INFO:tasks.workunit.client.0.smithi022.stdout: "pri1_bytes": { 2024-09-21T09:31:06.815 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.815 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.815 INFO:tasks.workunit.client.0.smithi022.stdout: "pri2_bytes": { 2024-09-21T09:31:06.815 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.815 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.815 INFO:tasks.workunit.client.0.smithi022.stdout: "pri3_bytes": { 2024-09-21T09:31:06.816 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.816 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.816 INFO:tasks.workunit.client.0.smithi022.stdout: "pri4_bytes": { 2024-09-21T09:31:06.816 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.816 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.816 INFO:tasks.workunit.client.0.smithi022.stdout: "pri5_bytes": { 2024-09-21T09:31:06.816 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.816 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.816 INFO:tasks.workunit.client.0.smithi022.stdout: "pri6_bytes": { 2024-09-21T09:31:06.816 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 8192 2024-09-21T09:31:06.816 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.816 INFO:tasks.workunit.client.0.smithi022.stdout: "pri7_bytes": { 2024-09-21T09:31:06.816 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.816 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.816 INFO:tasks.workunit.client.0.smithi022.stdout: "pri8_bytes": { 2024-09-21T09:31:06.817 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.817 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.817 INFO:tasks.workunit.client.0.smithi022.stdout: "pri9_bytes": { 2024-09-21T09:31:06.817 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.817 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.817 INFO:tasks.workunit.client.0.smithi022.stdout: "reserved_bytes": { 2024-09-21T09:31:06.817 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 73819099 2024-09-21T09:31:06.817 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.817 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.817 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore-pricache:kv": { 2024-09-21T09:31:06.817 INFO:tasks.workunit.client.0.smithi022.stdout: "committed_bytes": { 2024-09-21T09:31:06.817 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 780140544 2024-09-21T09:31:06.817 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.817 INFO:tasks.workunit.client.0.smithi022.stdout: "pri0_bytes": { 2024-09-21T09:31:06.817 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 67108864 2024-09-21T09:31:06.818 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.818 INFO:tasks.workunit.client.0.smithi022.stdout: "pri10_bytes": { 2024-09-21T09:31:06.818 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.818 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.818 INFO:tasks.workunit.client.0.smithi022.stdout: "pri11_bytes": { 2024-09-21T09:31:06.818 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 641671963 2024-09-21T09:31:06.818 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.818 INFO:tasks.workunit.client.0.smithi022.stdout: "pri1_bytes": { 2024-09-21T09:31:06.818 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.818 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.819 INFO:tasks.workunit.client.0.smithi022.stdout: "pri2_bytes": { 2024-09-21T09:31:06.819 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.819 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.819 INFO:tasks.workunit.client.0.smithi022.stdout: "pri3_bytes": { 2024-09-21T09:31:06.819 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.819 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.819 INFO:tasks.workunit.client.0.smithi022.stdout: "pri4_bytes": { 2024-09-21T09:31:06.819 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.819 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.819 INFO:tasks.workunit.client.0.smithi022.stdout: "pri5_bytes": { 2024-09-21T09:31:06.819 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.819 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.819 INFO:tasks.workunit.client.0.smithi022.stdout: "pri6_bytes": { 2024-09-21T09:31:06.819 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2128 2024-09-21T09:31:06.820 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.820 INFO:tasks.workunit.client.0.smithi022.stdout: "pri7_bytes": { 2024-09-21T09:31:06.820 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.820 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.820 INFO:tasks.workunit.client.0.smithi022.stdout: "pri8_bytes": { 2024-09-21T09:31:06.820 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.820 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.820 INFO:tasks.workunit.client.0.smithi022.stdout: "pri9_bytes": { 2024-09-21T09:31:06.820 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.820 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.820 INFO:tasks.workunit.client.0.smithi022.stdout: "reserved_bytes": { 2024-09-21T09:31:06.820 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 71357589 2024-09-21T09:31:06.820 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.821 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.821 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore-pricache:kv_onode": { 2024-09-21T09:31:06.821 INFO:tasks.workunit.client.0.smithi022.stdout: "committed_bytes": { 2024-09-21T09:31:06.821 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 192937984 2024-09-21T09:31:06.821 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.821 INFO:tasks.workunit.client.0.smithi022.stdout: "pri0_bytes": { 2024-09-21T09:31:06.821 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 67108864 2024-09-21T09:31:06.821 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.821 INFO:tasks.workunit.client.0.smithi022.stdout: "pri10_bytes": { 2024-09-21T09:31:06.821 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 464 2024-09-21T09:31:06.821 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.821 INFO:tasks.workunit.client.0.smithi022.stdout: "pri11_bytes": { 2024-09-21T09:31:06.821 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 57037507 2024-09-21T09:31:06.821 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.822 INFO:tasks.workunit.client.0.smithi022.stdout: "pri1_bytes": { 2024-09-21T09:31:06.822 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.822 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.822 INFO:tasks.workunit.client.0.smithi022.stdout: "pri2_bytes": { 2024-09-21T09:31:06.822 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.822 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.822 INFO:tasks.workunit.client.0.smithi022.stdout: "pri3_bytes": { 2024-09-21T09:31:06.822 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.822 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.822 INFO:tasks.workunit.client.0.smithi022.stdout: "pri4_bytes": { 2024-09-21T09:31:06.822 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.822 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.822 INFO:tasks.workunit.client.0.smithi022.stdout: "pri5_bytes": { 2024-09-21T09:31:06.823 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.823 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.823 INFO:tasks.workunit.client.0.smithi022.stdout: "pri6_bytes": { 2024-09-21T09:31:06.823 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.823 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.823 INFO:tasks.workunit.client.0.smithi022.stdout: "pri7_bytes": { 2024-09-21T09:31:06.823 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.823 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.823 INFO:tasks.workunit.client.0.smithi022.stdout: "pri8_bytes": { 2024-09-21T09:31:06.823 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.823 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.824 INFO:tasks.workunit.client.0.smithi022.stdout: "pri9_bytes": { 2024-09-21T09:31:06.824 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.824 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.824 INFO:tasks.workunit.client.0.smithi022.stdout: "reserved_bytes": { 2024-09-21T09:31:06.824 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 68791149 2024-09-21T09:31:06.824 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.824 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.824 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore-pricache:meta": { 2024-09-21T09:31:06.824 INFO:tasks.workunit.client.0.smithi022.stdout: "committed_bytes": { 2024-09-21T09:31:06.824 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 713031680 2024-09-21T09:31:06.824 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.824 INFO:tasks.workunit.client.0.smithi022.stdout: "pri0_bytes": { 2024-09-21T09:31:06.825 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.825 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.825 INFO:tasks.workunit.client.0.smithi022.stdout: "pri10_bytes": { 2024-09-21T09:31:06.825 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.825 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.825 INFO:tasks.workunit.client.0.smithi022.stdout: "pri11_bytes": { 2024-09-21T09:31:06.825 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 641671963 2024-09-21T09:31:06.825 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.825 INFO:tasks.workunit.client.0.smithi022.stdout: "pri1_bytes": { 2024-09-21T09:31:06.825 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.825 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.825 INFO:tasks.workunit.client.0.smithi022.stdout: "pri2_bytes": { 2024-09-21T09:31:06.825 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.825 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.826 INFO:tasks.workunit.client.0.smithi022.stdout: "pri3_bytes": { 2024-09-21T09:31:06.826 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.826 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.826 INFO:tasks.workunit.client.0.smithi022.stdout: "pri4_bytes": { 2024-09-21T09:31:06.826 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.826 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.826 INFO:tasks.workunit.client.0.smithi022.stdout: "pri5_bytes": { 2024-09-21T09:31:06.826 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.826 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.826 INFO:tasks.workunit.client.0.smithi022.stdout: "pri6_bytes": { 2024-09-21T09:31:06.826 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 241458 2024-09-21T09:31:06.826 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.826 INFO:tasks.workunit.client.0.smithi022.stdout: "pri7_bytes": { 2024-09-21T09:31:06.827 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.827 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.827 INFO:tasks.workunit.client.0.smithi022.stdout: "pri8_bytes": { 2024-09-21T09:31:06.827 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.827 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.827 INFO:tasks.workunit.client.0.smithi022.stdout: "pri9_bytes": { 2024-09-21T09:31:06.827 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.827 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.827 INFO:tasks.workunit.client.0.smithi022.stdout: "reserved_bytes": { 2024-09-21T09:31:06.827 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 71118259 2024-09-21T09:31:06.827 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.827 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.827 INFO:tasks.workunit.client.0.smithi022.stdout: "objecter": { 2024-09-21T09:31:06.827 INFO:tasks.workunit.client.0.smithi022.stdout: "op_active": { 2024-09-21T09:31:06.827 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.828 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.828 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r": { 2024-09-21T09:31:06.828 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.828 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.828 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rmw": { 2024-09-21T09:31:06.828 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.828 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.828 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w": { 2024-09-21T09:31:06.828 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.828 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.828 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.828 INFO:tasks.workunit.client.0.smithi022.stdout: "osd": { 2024-09-21T09:31:06.828 INFO:tasks.workunit.client.0.smithi022.stdout: "numpg": { 2024-09-21T09:31:06.828 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 30 2024-09-21T09:31:06.829 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.829 INFO:tasks.workunit.client.0.smithi022.stdout: "numpg_removing": { 2024-09-21T09:31:06.829 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.829 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.829 INFO:tasks.workunit.client.0.smithi022.stdout: "op": { 2024-09-21T09:31:06.829 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 10 2024-09-21T09:31:06.829 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.829 INFO:tasks.workunit.client.0.smithi022.stdout: "op_delayed_degraded": { 2024-09-21T09:31:06.829 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.829 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.829 INFO:tasks.workunit.client.0.smithi022.stdout: "op_delayed_unreadable": { 2024-09-21T09:31:06.829 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.829 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.829 INFO:tasks.workunit.client.0.smithi022.stdout: "op_in_bytes": { 2024-09-21T09:31:06.830 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.830 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.830 INFO:tasks.workunit.client.0.smithi022.stdout: "op_latency": { 2024-09-21T09:31:06.830 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 10, 2024-09-21T09:31:06.830 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 54020340 2024-09-21T09:31:06.830 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.830 INFO:tasks.workunit.client.0.smithi022.stdout: "op_out_bytes": { 2024-09-21T09:31:06.830 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.830 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.830 INFO:tasks.workunit.client.0.smithi022.stdout: "op_prepare_latency": { 2024-09-21T09:31:06.830 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 10, 2024-09-21T09:31:06.830 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 3890283 2024-09-21T09:31:06.830 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.830 INFO:tasks.workunit.client.0.smithi022.stdout: "op_process_latency": { 2024-09-21T09:31:06.830 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 10, 2024-09-21T09:31:06.831 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 42756099 2024-09-21T09:31:06.831 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.831 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r": { 2024-09-21T09:31:06.831 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.831 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.831 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r_latency": { 2024-09-21T09:31:06.831 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.831 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.831 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.831 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-21T09:31:06.831 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.831 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.831 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r_out_bytes": { 2024-09-21T09:31:06.831 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.832 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.832 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r_prepare_latency": { 2024-09-21T09:31:06.832 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.832 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.832 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.832 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r_process_latency": { 2024-09-21T09:31:06.832 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.832 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.832 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.832 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw": { 2024-09-21T09:31:06.832 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.832 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.832 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_in_bytes": { 2024-09-21T09:31:06.832 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.832 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.833 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_latency": { 2024-09-21T09:31:06.833 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.833 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.833 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.833 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-21T09:31:06.833 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.833 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.833 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-21T09:31:06.833 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.833 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.833 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_out_bytes": { 2024-09-21T09:31:06.833 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.833 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.834 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_prepare_latency": { 2024-09-21T09:31:06.834 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.834 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.834 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.834 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_process_latency": { 2024-09-21T09:31:06.834 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.834 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.834 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.834 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w": { 2024-09-21T09:31:06.834 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 10 2024-09-21T09:31:06.834 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.834 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w_in_bytes": { 2024-09-21T09:31:06.834 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.834 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.835 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w_latency": { 2024-09-21T09:31:06.835 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 10, 2024-09-21T09:31:06.835 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 54020340 2024-09-21T09:31:06.835 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.835 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-21T09:31:06.835 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.835 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.835 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w_prepare_latency": { 2024-09-21T09:31:06.835 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 10, 2024-09-21T09:31:06.835 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 3890283 2024-09-21T09:31:06.835 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.835 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w_process_latency": { 2024-09-21T09:31:06.835 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 10, 2024-09-21T09:31:06.836 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 42756099 2024-09-21T09:31:06.836 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.836 INFO:tasks.workunit.client.0.smithi022.stdout: "op_wip": { 2024-09-21T09:31:06.836 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.836 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.836 INFO:tasks.workunit.client.0.smithi022.stdout: "recovery_bytes": { 2024-09-21T09:31:06.836 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.836 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.836 INFO:tasks.workunit.client.0.smithi022.stdout: "recovery_ops": { 2024-09-21T09:31:06.836 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.836 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.836 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_bytes": { 2024-09-21T09:31:06.836 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 95995035648 2024-09-21T09:31:06.837 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.837 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_bytes_used": { 2024-09-21T09:31:06.837 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 305180672 2024-09-21T09:31:06.837 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.837 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.837 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.837 INFO:tasks.workunit.client.0.smithi022.stdout: "osd.3": { 2024-09-21T09:31:06.837 INFO:tasks.workunit.client.0.smithi022.stdout: "bluefs": { 2024-09-21T09:31:06.837 INFO:tasks.workunit.client.0.smithi022.stdout: "alloc_slow_fallback": { 2024-09-21T09:31:06.837 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.837 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.837 INFO:tasks.workunit.client.0.smithi022.stdout: "alloc_slow_size_fallback": { 2024-09-21T09:31:06.838 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.838 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.838 INFO:tasks.workunit.client.0.smithi022.stdout: "alloc_unit_db": { 2024-09-21T09:31:06.838 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 65536 2024-09-21T09:31:06.838 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.838 INFO:tasks.workunit.client.0.smithi022.stdout: "alloc_unit_main": { 2024-09-21T09:31:06.838 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.838 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.838 INFO:tasks.workunit.client.0.smithi022.stdout: "alloc_unit_wal": { 2024-09-21T09:31:06.838 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.838 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.838 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes_written_slow": { 2024-09-21T09:31:06.838 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.839 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.839 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes_written_sst": { 2024-09-21T09:31:06.839 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 12288 2024-09-21T09:31:06.839 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.839 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes_written_wal": { 2024-09-21T09:31:06.839 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1228800 2024-09-21T09:31:06.839 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.839 INFO:tasks.workunit.client.0.smithi022.stdout: "compact_lat": { 2024-09-21T09:31:06.839 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.839 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.839 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.839 INFO:tasks.workunit.client.0.smithi022.stdout: "compact_lock_lat": { 2024-09-21T09:31:06.840 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.840 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.840 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.840 INFO:tasks.workunit.client.0.smithi022.stdout: "db_total_bytes": { 2024-09-21T09:31:06.840 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 95995035648 2024-09-21T09:31:06.840 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.840 INFO:tasks.workunit.client.0.smithi022.stdout: "db_used_bytes": { 2024-09-21T09:31:06.840 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 304480256 2024-09-21T09:31:06.840 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.840 INFO:tasks.workunit.client.0.smithi022.stdout: "log_bytes": { 2024-09-21T09:31:06.840 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 937984 2024-09-21T09:31:06.840 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.840 INFO:tasks.workunit.client.0.smithi022.stdout: "logged_bytes": { 2024-09-21T09:31:06.840 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 798720 2024-09-21T09:31:06.840 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.840 INFO:tasks.workunit.client.0.smithi022.stdout: "max_bytes_db": { 2024-09-21T09:31:06.841 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 312868864 2024-09-21T09:31:06.841 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.841 INFO:tasks.workunit.client.0.smithi022.stdout: "max_bytes_slow": { 2024-09-21T09:31:06.841 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.841 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.841 INFO:tasks.workunit.client.0.smithi022.stdout: "max_bytes_wal": { 2024-09-21T09:31:06.841 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.841 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.841 INFO:tasks.workunit.client.0.smithi022.stdout: "num_files": { 2024-09-21T09:31:06.841 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 14 2024-09-21T09:31:06.841 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.841 INFO:tasks.workunit.client.0.smithi022.stdout: "read_bytes": { 2024-09-21T09:31:06.841 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 195697 2024-09-21T09:31:06.842 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.842 INFO:tasks.workunit.client.0.smithi022.stdout: "read_count": { 2024-09-21T09:31:06.842 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 23 2024-09-21T09:31:06.842 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.842 INFO:tasks.workunit.client.0.smithi022.stdout: "read_disk_bytes": { 2024-09-21T09:31:06.842 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1269760 2024-09-21T09:31:06.842 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.842 INFO:tasks.workunit.client.0.smithi022.stdout: "read_disk_bytes_db": { 2024-09-21T09:31:06.842 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1273856 2024-09-21T09:31:06.842 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.842 INFO:tasks.workunit.client.0.smithi022.stdout: "read_disk_bytes_slow": { 2024-09-21T09:31:06.842 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.842 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.843 INFO:tasks.workunit.client.0.smithi022.stdout: "read_disk_bytes_wal": { 2024-09-21T09:31:06.843 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.843 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.843 INFO:tasks.workunit.client.0.smithi022.stdout: "read_disk_count": { 2024-09-21T09:31:06.843 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 15 2024-09-21T09:31:06.843 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.843 INFO:tasks.workunit.client.0.smithi022.stdout: "read_prefetch_bytes": { 2024-09-21T09:31:06.843 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 4789 2024-09-21T09:31:06.843 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.843 INFO:tasks.workunit.client.0.smithi022.stdout: "read_prefetch_count": { 2024-09-21T09:31:06.843 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 4 2024-09-21T09:31:06.843 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.843 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_buffer_bytes": { 2024-09-21T09:31:06.843 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 5589 2024-09-21T09:31:06.844 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.844 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_buffer_count": { 2024-09-21T09:31:06.844 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 24 2024-09-21T09:31:06.844 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.844 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_bytes": { 2024-09-21T09:31:06.844 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 5589 2024-09-21T09:31:06.844 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.844 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_count": { 2024-09-21T09:31:06.844 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 24 2024-09-21T09:31:06.844 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.844 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_disk_bytes": { 2024-09-21T09:31:06.844 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.844 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.845 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_disk_bytes_db": { 2024-09-21T09:31:06.845 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.845 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.845 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_disk_bytes_slow": { 2024-09-21T09:31:06.845 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.845 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.845 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_disk_bytes_wal": { 2024-09-21T09:31:06.845 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.845 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.845 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_disk_count": { 2024-09-21T09:31:06.845 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.845 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.845 INFO:tasks.workunit.client.0.smithi022.stdout: "slow_total_bytes": { 2024-09-21T09:31:06.846 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.846 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.846 INFO:tasks.workunit.client.0.smithi022.stdout: "slow_used_bytes": { 2024-09-21T09:31:06.846 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.846 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.846 INFO:tasks.workunit.client.0.smithi022.stdout: "wal_total_bytes": { 2024-09-21T09:31:06.846 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.846 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.846 INFO:tasks.workunit.client.0.smithi022.stdout: "wal_used_bytes": { 2024-09-21T09:31:06.846 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.846 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.846 INFO:tasks.workunit.client.0.smithi022.stdout: "write_bytes": { 2024-09-21T09:31:06.846 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2101248 2024-09-21T09:31:06.846 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.847 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.847 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore": { 2024-09-21T09:31:06.847 INFO:tasks.workunit.client.0.smithi022.stdout: "alloc_unit": { 2024-09-21T09:31:06.847 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 4096 2024-09-21T09:31:06.847 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.847 INFO:tasks.workunit.client.0.smithi022.stdout: "allocated": { 2024-09-21T09:31:06.847 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 708608 2024-09-21T09:31:06.847 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.847 INFO:tasks.workunit.client.0.smithi022.stdout: "clist_lat": { 2024-09-21T09:31:06.847 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.847 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.847 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.847 INFO:tasks.workunit.client.0.smithi022.stdout: "compress_lat": { 2024-09-21T09:31:06.847 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.848 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.848 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.848 INFO:tasks.workunit.client.0.smithi022.stdout: "compressed": { 2024-09-21T09:31:06.848 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.848 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.848 INFO:tasks.workunit.client.0.smithi022.stdout: "compressed_allocated": { 2024-09-21T09:31:06.848 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.848 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.848 INFO:tasks.workunit.client.0.smithi022.stdout: "compressed_original": { 2024-09-21T09:31:06.848 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.848 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.848 INFO:tasks.workunit.client.0.smithi022.stdout: "csum_lat": { 2024-09-21T09:31:06.848 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 4, 2024-09-21T09:31:06.849 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 7157 2024-09-21T09:31:06.849 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.849 INFO:tasks.workunit.client.0.smithi022.stdout: "decompress_lat": { 2024-09-21T09:31:06.849 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.849 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.849 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.849 INFO:tasks.workunit.client.0.smithi022.stdout: "kv_commit_lat": { 2024-09-21T09:31:06.849 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 178, 2024-09-21T09:31:06.849 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 35605293 2024-09-21T09:31:06.849 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.849 INFO:tasks.workunit.client.0.smithi022.stdout: "kv_final_lat": { 2024-09-21T09:31:06.849 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 172, 2024-09-21T09:31:06.850 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 9751410 2024-09-21T09:31:06.850 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.850 INFO:tasks.workunit.client.0.smithi022.stdout: "kv_flush_lat": { 2024-09-21T09:31:06.850 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 178, 2024-09-21T09:31:06.850 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 244331 2024-09-21T09:31:06.850 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.850 INFO:tasks.workunit.client.0.smithi022.stdout: "kv_sync_lat": { 2024-09-21T09:31:06.850 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 178, 2024-09-21T09:31:06.850 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 35849624 2024-09-21T09:31:06.850 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.850 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_get_keys_lat": { 2024-09-21T09:31:06.850 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.850 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.851 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.851 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_get_values_lat": { 2024-09-21T09:31:06.851 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 1, 2024-09-21T09:31:06.851 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 28513 2024-09-21T09:31:06.851 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.851 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_lower_bound_lat": { 2024-09-21T09:31:06.851 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 2, 2024-09-21T09:31:06.851 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 916 2024-09-21T09:31:06.851 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.851 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_next_lat": { 2024-09-21T09:31:06.851 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.851 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.851 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.852 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_seek_to_first_lat": { 2024-09-21T09:31:06.852 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.852 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.852 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.852 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_upper_bound_lat": { 2024-09-21T09:31:06.852 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 3, 2024-09-21T09:31:06.852 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 7611 2024-09-21T09:31:06.852 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.852 INFO:tasks.workunit.client.0.smithi022.stdout: "onode_hits": { 2024-09-21T09:31:06.852 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 352 2024-09-21T09:31:06.852 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.852 INFO:tasks.workunit.client.0.smithi022.stdout: "onode_misses": { 2024-09-21T09:31:06.852 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 214 2024-09-21T09:31:06.852 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.853 INFO:tasks.workunit.client.0.smithi022.stdout: "read_lat": { 2024-09-21T09:31:06.853 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 85, 2024-09-21T09:31:06.853 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 738958 2024-09-21T09:31:06.853 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.853 INFO:tasks.workunit.client.0.smithi022.stdout: "read_onode_meta_lat": { 2024-09-21T09:31:06.853 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 149, 2024-09-21T09:31:06.853 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 245550 2024-09-21T09:31:06.853 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.853 INFO:tasks.workunit.client.0.smithi022.stdout: "read_wait_aio_lat": { 2024-09-21T09:31:06.853 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 64, 2024-09-21T09:31:06.853 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 222496 2024-09-21T09:31:06.853 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.853 INFO:tasks.workunit.client.0.smithi022.stdout: "reads_with_retries": { 2024-09-21T09:31:06.853 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.854 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.854 INFO:tasks.workunit.client.0.smithi022.stdout: "remove_lat": { 2024-09-21T09:31:06.854 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.854 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.854 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.854 INFO:tasks.workunit.client.0.smithi022.stdout: "state_aio_wait_lat": { 2024-09-21T09:31:06.854 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 251, 2024-09-21T09:31:06.854 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 3272554 2024-09-21T09:31:06.854 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.854 INFO:tasks.workunit.client.0.smithi022.stdout: "state_deferred_aio_wait_lat": { 2024-09-21T09:31:06.854 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 51, 2024-09-21T09:31:06.854 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 3656446 2024-09-21T09:31:06.855 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.855 INFO:tasks.workunit.client.0.smithi022.stdout: "state_deferred_cleanup_lat": { 2024-09-21T09:31:06.855 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 48, 2024-09-21T09:31:06.855 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 348209618281 2024-09-21T09:31:06.855 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.855 INFO:tasks.workunit.client.0.smithi022.stdout: "state_deferred_queued_lat": { 2024-09-21T09:31:06.855 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 51, 2024-09-21T09:31:06.855 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 60751115424 2024-09-21T09:31:06.855 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.855 INFO:tasks.workunit.client.0.smithi022.stdout: "state_done_lat": { 2024-09-21T09:31:06.855 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 248, 2024-09-21T09:31:06.855 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 509483 2024-09-21T09:31:06.855 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.855 INFO:tasks.workunit.client.0.smithi022.stdout: "state_finishing_lat": { 2024-09-21T09:31:06.856 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 248, 2024-09-21T09:31:06.856 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 25639 2024-09-21T09:31:06.856 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.856 INFO:tasks.workunit.client.0.smithi022.stdout: "state_io_done_lat": { 2024-09-21T09:31:06.856 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 251, 2024-09-21T09:31:06.856 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 233596 2024-09-21T09:31:06.856 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.856 INFO:tasks.workunit.client.0.smithi022.stdout: "state_kv_commiting_lat": { 2024-09-21T09:31:06.856 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 251, 2024-09-21T09:31:06.856 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 85587384 2024-09-21T09:31:06.856 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.856 INFO:tasks.workunit.client.0.smithi022.stdout: "state_kv_done_lat": { 2024-09-21T09:31:06.856 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 251, 2024-09-21T09:31:06.856 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 78983 2024-09-21T09:31:06.856 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.857 INFO:tasks.workunit.client.0.smithi022.stdout: "state_kv_queued_lat": { 2024-09-21T09:31:06.857 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 251, 2024-09-21T09:31:06.857 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 38798576 2024-09-21T09:31:06.857 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.857 INFO:tasks.workunit.client.0.smithi022.stdout: "state_prepare_lat": { 2024-09-21T09:31:06.857 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 251, 2024-09-21T09:31:06.857 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 11159142 2024-09-21T09:31:06.857 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.857 INFO:tasks.workunit.client.0.smithi022.stdout: "stored": { 2024-09-21T09:31:06.857 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 312707 2024-09-21T09:31:06.857 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.857 INFO:tasks.workunit.client.0.smithi022.stdout: "truncate_lat": { 2024-09-21T09:31:06.857 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 42, 2024-09-21T09:31:06.857 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 217112 2024-09-21T09:31:06.858 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.858 INFO:tasks.workunit.client.0.smithi022.stdout: "txc_commit_lat": { 2024-09-21T09:31:06.858 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 251, 2024-09-21T09:31:06.858 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 139086793 2024-09-21T09:31:06.858 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.858 INFO:tasks.workunit.client.0.smithi022.stdout: "txc_submit_lat": { 2024-09-21T09:31:06.858 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 251, 2024-09-21T09:31:06.858 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 13699222 2024-09-21T09:31:06.858 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.858 INFO:tasks.workunit.client.0.smithi022.stdout: "txc_throttle_lat": { 2024-09-21T09:31:06.858 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 251, 2024-09-21T09:31:06.858 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1770625 2024-09-21T09:31:06.858 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.858 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.858 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore-pricache": { 2024-09-21T09:31:06.859 INFO:tasks.workunit.client.0.smithi022.stdout: "cache_bytes": { 2024-09-21T09:31:06.859 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1862400347 2024-09-21T09:31:06.859 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.859 INFO:tasks.workunit.client.0.smithi022.stdout: "heap_bytes": { 2024-09-21T09:31:06.859 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 268877824 2024-09-21T09:31:06.859 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.859 INFO:tasks.workunit.client.0.smithi022.stdout: "mapped_bytes": { 2024-09-21T09:31:06.859 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 268582912 2024-09-21T09:31:06.859 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.859 INFO:tasks.workunit.client.0.smithi022.stdout: "target_bytes": { 2024-09-21T09:31:06.859 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 3138478489 2024-09-21T09:31:06.859 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.860 INFO:tasks.workunit.client.0.smithi022.stdout: "unmapped_bytes": { 2024-09-21T09:31:06.860 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 294912 2024-09-21T09:31:06.860 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.860 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.860 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore-pricache:data": { 2024-09-21T09:31:06.860 INFO:tasks.workunit.client.0.smithi022.stdout: "committed_bytes": { 2024-09-21T09:31:06.860 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 159383552 2024-09-21T09:31:06.860 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.860 INFO:tasks.workunit.client.0.smithi022.stdout: "pri0_bytes": { 2024-09-21T09:31:06.860 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.860 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.860 INFO:tasks.workunit.client.0.smithi022.stdout: "pri10_bytes": { 2024-09-21T09:31:06.860 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.861 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.861 INFO:tasks.workunit.client.0.smithi022.stdout: "pri11_bytes": { 2024-09-21T09:31:06.861 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 85555878 2024-09-21T09:31:06.861 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.861 INFO:tasks.workunit.client.0.smithi022.stdout: "pri1_bytes": { 2024-09-21T09:31:06.861 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.861 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.861 INFO:tasks.workunit.client.0.smithi022.stdout: "pri2_bytes": { 2024-09-21T09:31:06.861 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.861 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.861 INFO:tasks.workunit.client.0.smithi022.stdout: "pri3_bytes": { 2024-09-21T09:31:06.861 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.861 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.861 INFO:tasks.workunit.client.0.smithi022.stdout: "pri4_bytes": { 2024-09-21T09:31:06.862 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.862 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.862 INFO:tasks.workunit.client.0.smithi022.stdout: "pri5_bytes": { 2024-09-21T09:31:06.862 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.862 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.862 INFO:tasks.workunit.client.0.smithi022.stdout: "pri6_bytes": { 2024-09-21T09:31:06.862 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 4096 2024-09-21T09:31:06.862 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.862 INFO:tasks.workunit.client.0.smithi022.stdout: "pri7_bytes": { 2024-09-21T09:31:06.862 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.862 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.862 INFO:tasks.workunit.client.0.smithi022.stdout: "pri8_bytes": { 2024-09-21T09:31:06.862 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.862 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.862 INFO:tasks.workunit.client.0.smithi022.stdout: "pri9_bytes": { 2024-09-21T09:31:06.863 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.863 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.863 INFO:tasks.workunit.client.0.smithi022.stdout: "reserved_bytes": { 2024-09-21T09:31:06.863 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 73823578 2024-09-21T09:31:06.863 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.863 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.863 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore-pricache:kv": { 2024-09-21T09:31:06.863 INFO:tasks.workunit.client.0.smithi022.stdout: "committed_bytes": { 2024-09-21T09:31:06.863 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 780140544 2024-09-21T09:31:06.863 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.863 INFO:tasks.workunit.client.0.smithi022.stdout: "pri0_bytes": { 2024-09-21T09:31:06.863 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 67108864 2024-09-21T09:31:06.863 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.864 INFO:tasks.workunit.client.0.smithi022.stdout: "pri10_bytes": { 2024-09-21T09:31:06.864 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.864 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.864 INFO:tasks.workunit.client.0.smithi022.stdout: "pri11_bytes": { 2024-09-21T09:31:06.864 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 641669085 2024-09-21T09:31:06.864 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.864 INFO:tasks.workunit.client.0.smithi022.stdout: "pri1_bytes": { 2024-09-21T09:31:06.864 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.864 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.864 INFO:tasks.workunit.client.0.smithi022.stdout: "pri2_bytes": { 2024-09-21T09:31:06.864 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.864 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.864 INFO:tasks.workunit.client.0.smithi022.stdout: "pri3_bytes": { 2024-09-21T09:31:06.864 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.864 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.864 INFO:tasks.workunit.client.0.smithi022.stdout: "pri4_bytes": { 2024-09-21T09:31:06.865 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.865 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.865 INFO:tasks.workunit.client.0.smithi022.stdout: "pri5_bytes": { 2024-09-21T09:31:06.865 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.865 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.865 INFO:tasks.workunit.client.0.smithi022.stdout: "pri6_bytes": { 2024-09-21T09:31:06.865 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2128 2024-09-21T09:31:06.865 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.865 INFO:tasks.workunit.client.0.smithi022.stdout: "pri7_bytes": { 2024-09-21T09:31:06.865 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.865 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.865 INFO:tasks.workunit.client.0.smithi022.stdout: "pri8_bytes": { 2024-09-21T09:31:06.865 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.865 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.865 INFO:tasks.workunit.client.0.smithi022.stdout: "pri9_bytes": { 2024-09-21T09:31:06.865 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.866 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.866 INFO:tasks.workunit.client.0.smithi022.stdout: "reserved_bytes": { 2024-09-21T09:31:06.866 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 71360467 2024-09-21T09:31:06.866 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.866 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.866 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore-pricache:kv_onode": { 2024-09-21T09:31:06.866 INFO:tasks.workunit.client.0.smithi022.stdout: "committed_bytes": { 2024-09-21T09:31:06.866 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 192937984 2024-09-21T09:31:06.866 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.866 INFO:tasks.workunit.client.0.smithi022.stdout: "pri0_bytes": { 2024-09-21T09:31:06.866 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 67108864 2024-09-21T09:31:06.866 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.866 INFO:tasks.workunit.client.0.smithi022.stdout: "pri10_bytes": { 2024-09-21T09:31:06.866 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 464 2024-09-21T09:31:06.867 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.867 INFO:tasks.workunit.client.0.smithi022.stdout: "pri11_bytes": { 2024-09-21T09:31:06.867 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 57037252 2024-09-21T09:31:06.867 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.867 INFO:tasks.workunit.client.0.smithi022.stdout: "pri1_bytes": { 2024-09-21T09:31:06.867 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.867 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.867 INFO:tasks.workunit.client.0.smithi022.stdout: "pri2_bytes": { 2024-09-21T09:31:06.867 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.867 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.867 INFO:tasks.workunit.client.0.smithi022.stdout: "pri3_bytes": { 2024-09-21T09:31:06.867 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.867 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.867 INFO:tasks.workunit.client.0.smithi022.stdout: "pri4_bytes": { 2024-09-21T09:31:06.867 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.868 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.868 INFO:tasks.workunit.client.0.smithi022.stdout: "pri5_bytes": { 2024-09-21T09:31:06.868 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.868 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.868 INFO:tasks.workunit.client.0.smithi022.stdout: "pri6_bytes": { 2024-09-21T09:31:06.868 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.868 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.868 INFO:tasks.workunit.client.0.smithi022.stdout: "pri7_bytes": { 2024-09-21T09:31:06.868 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.868 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.868 INFO:tasks.workunit.client.0.smithi022.stdout: "pri8_bytes": { 2024-09-21T09:31:06.868 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.868 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.868 INFO:tasks.workunit.client.0.smithi022.stdout: "pri9_bytes": { 2024-09-21T09:31:06.869 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.869 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.869 INFO:tasks.workunit.client.0.smithi022.stdout: "reserved_bytes": { 2024-09-21T09:31:06.869 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 68791404 2024-09-21T09:31:06.869 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.869 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.869 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore-pricache:meta": { 2024-09-21T09:31:06.869 INFO:tasks.workunit.client.0.smithi022.stdout: "committed_bytes": { 2024-09-21T09:31:06.869 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 713031680 2024-09-21T09:31:06.869 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.869 INFO:tasks.workunit.client.0.smithi022.stdout: "pri0_bytes": { 2024-09-21T09:31:06.869 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.869 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.869 INFO:tasks.workunit.client.0.smithi022.stdout: "pri10_bytes": { 2024-09-21T09:31:06.869 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.870 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.870 INFO:tasks.workunit.client.0.smithi022.stdout: "pri11_bytes": { 2024-09-21T09:31:06.870 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 641669085 2024-09-21T09:31:06.870 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.870 INFO:tasks.workunit.client.0.smithi022.stdout: "pri1_bytes": { 2024-09-21T09:31:06.870 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.870 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.870 INFO:tasks.workunit.client.0.smithi022.stdout: "pri2_bytes": { 2024-09-21T09:31:06.870 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.870 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.870 INFO:tasks.workunit.client.0.smithi022.stdout: "pri3_bytes": { 2024-09-21T09:31:06.870 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.870 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.870 INFO:tasks.workunit.client.0.smithi022.stdout: "pri4_bytes": { 2024-09-21T09:31:06.871 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.871 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.871 INFO:tasks.workunit.client.0.smithi022.stdout: "pri5_bytes": { 2024-09-21T09:31:06.871 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 6960 2024-09-21T09:31:06.871 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.871 INFO:tasks.workunit.client.0.smithi022.stdout: "pri6_bytes": { 2024-09-21T09:31:06.871 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 247783 2024-09-21T09:31:06.871 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.871 INFO:tasks.workunit.client.0.smithi022.stdout: "pri7_bytes": { 2024-09-21T09:31:06.871 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.871 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.871 INFO:tasks.workunit.client.0.smithi022.stdout: "pri8_bytes": { 2024-09-21T09:31:06.871 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.871 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.871 INFO:tasks.workunit.client.0.smithi022.stdout: "pri9_bytes": { 2024-09-21T09:31:06.872 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.872 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.872 INFO:tasks.workunit.client.0.smithi022.stdout: "reserved_bytes": { 2024-09-21T09:31:06.872 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 71107852 2024-09-21T09:31:06.872 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.872 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.872 INFO:tasks.workunit.client.0.smithi022.stdout: "objecter": { 2024-09-21T09:31:06.872 INFO:tasks.workunit.client.0.smithi022.stdout: "op_active": { 2024-09-21T09:31:06.872 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.872 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.872 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r": { 2024-09-21T09:31:06.872 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.872 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.872 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rmw": { 2024-09-21T09:31:06.872 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.873 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.873 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w": { 2024-09-21T09:31:06.873 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.873 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.873 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.873 INFO:tasks.workunit.client.0.smithi022.stdout: "osd": { 2024-09-21T09:31:06.873 INFO:tasks.workunit.client.0.smithi022.stdout: "numpg": { 2024-09-21T09:31:06.873 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 37 2024-09-21T09:31:06.873 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.873 INFO:tasks.workunit.client.0.smithi022.stdout: "numpg_removing": { 2024-09-21T09:31:06.873 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.873 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.873 INFO:tasks.workunit.client.0.smithi022.stdout: "op": { 2024-09-21T09:31:06.873 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.873 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.874 INFO:tasks.workunit.client.0.smithi022.stdout: "op_delayed_degraded": { 2024-09-21T09:31:06.874 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.874 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.874 INFO:tasks.workunit.client.0.smithi022.stdout: "op_delayed_unreadable": { 2024-09-21T09:31:06.874 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.874 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.874 INFO:tasks.workunit.client.0.smithi022.stdout: "op_in_bytes": { 2024-09-21T09:31:06.874 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.874 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.874 INFO:tasks.workunit.client.0.smithi022.stdout: "op_latency": { 2024-09-21T09:31:06.874 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.874 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.874 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.874 INFO:tasks.workunit.client.0.smithi022.stdout: "op_out_bytes": { 2024-09-21T09:31:06.874 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.875 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.875 INFO:tasks.workunit.client.0.smithi022.stdout: "op_prepare_latency": { 2024-09-21T09:31:06.875 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 2, 2024-09-21T09:31:06.875 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 731032 2024-09-21T09:31:06.875 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.875 INFO:tasks.workunit.client.0.smithi022.stdout: "op_process_latency": { 2024-09-21T09:31:06.875 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.875 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.875 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.875 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r": { 2024-09-21T09:31:06.875 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.875 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.875 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r_latency": { 2024-09-21T09:31:06.875 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.875 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.876 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.876 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-21T09:31:06.876 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.876 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.876 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r_out_bytes": { 2024-09-21T09:31:06.876 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.876 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.876 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r_prepare_latency": { 2024-09-21T09:31:06.876 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.876 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.876 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.876 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r_process_latency": { 2024-09-21T09:31:06.876 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.876 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.876 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.876 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw": { 2024-09-21T09:31:06.877 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.877 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.877 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_in_bytes": { 2024-09-21T09:31:06.877 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.877 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.877 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_latency": { 2024-09-21T09:31:06.877 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.877 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.877 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.877 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-21T09:31:06.877 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.877 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.877 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-21T09:31:06.877 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.877 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.877 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_out_bytes": { 2024-09-21T09:31:06.877 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.878 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.878 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_prepare_latency": { 2024-09-21T09:31:06.878 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.878 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.878 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.878 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_process_latency": { 2024-09-21T09:31:06.878 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.878 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.878 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.878 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w": { 2024-09-21T09:31:06.878 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.878 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.878 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w_in_bytes": { 2024-09-21T09:31:06.878 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.879 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.879 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w_latency": { 2024-09-21T09:31:06.879 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.879 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.879 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.879 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-21T09:31:06.879 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.879 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.879 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w_prepare_latency": { 2024-09-21T09:31:06.879 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 2, 2024-09-21T09:31:06.879 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 731032 2024-09-21T09:31:06.880 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.880 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w_process_latency": { 2024-09-21T09:31:06.880 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.880 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.880 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.880 INFO:tasks.workunit.client.0.smithi022.stdout: "op_wip": { 2024-09-21T09:31:06.880 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.880 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.880 INFO:tasks.workunit.client.0.smithi022.stdout: "recovery_bytes": { 2024-09-21T09:31:06.880 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.880 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.880 INFO:tasks.workunit.client.0.smithi022.stdout: "recovery_ops": { 2024-09-21T09:31:06.880 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.880 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.881 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_bytes": { 2024-09-21T09:31:06.881 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 95995035648 2024-09-21T09:31:06.881 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.881 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_bytes_used": { 2024-09-21T09:31:06.881 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 305197056 2024-09-21T09:31:06.881 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.881 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.881 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.881 INFO:tasks.workunit.client.0.smithi022.stdout: "osd.4": { 2024-09-21T09:31:06.881 INFO:tasks.workunit.client.0.smithi022.stdout: "bluefs": { 2024-09-21T09:31:06.881 INFO:tasks.workunit.client.0.smithi022.stdout: "alloc_slow_fallback": { 2024-09-21T09:31:06.881 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.881 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.881 INFO:tasks.workunit.client.0.smithi022.stdout: "alloc_slow_size_fallback": { 2024-09-21T09:31:06.882 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.882 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.882 INFO:tasks.workunit.client.0.smithi022.stdout: "alloc_unit_db": { 2024-09-21T09:31:06.882 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 65536 2024-09-21T09:31:06.882 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.882 INFO:tasks.workunit.client.0.smithi022.stdout: "alloc_unit_main": { 2024-09-21T09:31:06.882 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.882 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.882 INFO:tasks.workunit.client.0.smithi022.stdout: "alloc_unit_wal": { 2024-09-21T09:31:06.882 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.882 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.882 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes_written_slow": { 2024-09-21T09:31:06.882 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.883 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.883 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes_written_sst": { 2024-09-21T09:31:06.883 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 12288 2024-09-21T09:31:06.883 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.883 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes_written_wal": { 2024-09-21T09:31:06.883 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1212416 2024-09-21T09:31:06.883 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.883 INFO:tasks.workunit.client.0.smithi022.stdout: "compact_lat": { 2024-09-21T09:31:06.883 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.883 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.883 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.883 INFO:tasks.workunit.client.0.smithi022.stdout: "compact_lock_lat": { 2024-09-21T09:31:06.883 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.884 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.884 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.884 INFO:tasks.workunit.client.0.smithi022.stdout: "db_total_bytes": { 2024-09-21T09:31:06.884 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 95995035648 2024-09-21T09:31:06.884 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.884 INFO:tasks.workunit.client.0.smithi022.stdout: "db_used_bytes": { 2024-09-21T09:31:06.884 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 304480256 2024-09-21T09:31:06.884 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.884 INFO:tasks.workunit.client.0.smithi022.stdout: "log_bytes": { 2024-09-21T09:31:06.884 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 917504 2024-09-21T09:31:06.884 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.884 INFO:tasks.workunit.client.0.smithi022.stdout: "logged_bytes": { 2024-09-21T09:31:06.884 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 778240 2024-09-21T09:31:06.885 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.885 INFO:tasks.workunit.client.0.smithi022.stdout: "max_bytes_db": { 2024-09-21T09:31:06.885 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 312868864 2024-09-21T09:31:06.885 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.885 INFO:tasks.workunit.client.0.smithi022.stdout: "max_bytes_slow": { 2024-09-21T09:31:06.885 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.885 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.885 INFO:tasks.workunit.client.0.smithi022.stdout: "max_bytes_wal": { 2024-09-21T09:31:06.885 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.885 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.885 INFO:tasks.workunit.client.0.smithi022.stdout: "num_files": { 2024-09-21T09:31:06.885 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 14 2024-09-21T09:31:06.885 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.885 INFO:tasks.workunit.client.0.smithi022.stdout: "read_bytes": { 2024-09-21T09:31:06.885 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 195697 2024-09-21T09:31:06.886 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.886 INFO:tasks.workunit.client.0.smithi022.stdout: "read_count": { 2024-09-21T09:31:06.886 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 23 2024-09-21T09:31:06.886 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.886 INFO:tasks.workunit.client.0.smithi022.stdout: "read_disk_bytes": { 2024-09-21T09:31:06.886 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1269760 2024-09-21T09:31:06.886 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.886 INFO:tasks.workunit.client.0.smithi022.stdout: "read_disk_bytes_db": { 2024-09-21T09:31:06.886 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1273856 2024-09-21T09:31:06.886 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.886 INFO:tasks.workunit.client.0.smithi022.stdout: "read_disk_bytes_slow": { 2024-09-21T09:31:06.886 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.886 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.886 INFO:tasks.workunit.client.0.smithi022.stdout: "read_disk_bytes_wal": { 2024-09-21T09:31:06.887 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.887 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.887 INFO:tasks.workunit.client.0.smithi022.stdout: "read_disk_count": { 2024-09-21T09:31:06.887 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 15 2024-09-21T09:31:06.887 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.887 INFO:tasks.workunit.client.0.smithi022.stdout: "read_prefetch_bytes": { 2024-09-21T09:31:06.887 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 4789 2024-09-21T09:31:06.887 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.887 INFO:tasks.workunit.client.0.smithi022.stdout: "read_prefetch_count": { 2024-09-21T09:31:06.887 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 4 2024-09-21T09:31:06.887 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.887 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_buffer_bytes": { 2024-09-21T09:31:06.887 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 5589 2024-09-21T09:31:06.887 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.887 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_buffer_count": { 2024-09-21T09:31:06.888 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 24 2024-09-21T09:31:06.888 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.888 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_bytes": { 2024-09-21T09:31:06.888 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 5589 2024-09-21T09:31:06.888 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.888 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_count": { 2024-09-21T09:31:06.888 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 24 2024-09-21T09:31:06.888 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.888 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_disk_bytes": { 2024-09-21T09:31:06.888 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.888 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.888 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_disk_bytes_db": { 2024-09-21T09:31:06.888 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.888 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.889 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_disk_bytes_slow": { 2024-09-21T09:31:06.889 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.889 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.889 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_disk_bytes_wal": { 2024-09-21T09:31:06.889 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.889 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.889 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_disk_count": { 2024-09-21T09:31:06.889 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.889 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.889 INFO:tasks.workunit.client.0.smithi022.stdout: "slow_total_bytes": { 2024-09-21T09:31:06.889 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.889 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.889 INFO:tasks.workunit.client.0.smithi022.stdout: "slow_used_bytes": { 2024-09-21T09:31:06.889 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.890 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.890 INFO:tasks.workunit.client.0.smithi022.stdout: "wal_total_bytes": { 2024-09-21T09:31:06.890 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.890 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.890 INFO:tasks.workunit.client.0.smithi022.stdout: "wal_used_bytes": { 2024-09-21T09:31:06.890 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.890 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.890 INFO:tasks.workunit.client.0.smithi022.stdout: "write_bytes": { 2024-09-21T09:31:06.890 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2064384 2024-09-21T09:31:06.890 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.890 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.890 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore": { 2024-09-21T09:31:06.890 INFO:tasks.workunit.client.0.smithi022.stdout: "alloc_unit": { 2024-09-21T09:31:06.890 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 4096 2024-09-21T09:31:06.890 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.891 INFO:tasks.workunit.client.0.smithi022.stdout: "allocated": { 2024-09-21T09:31:06.891 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 708608 2024-09-21T09:31:06.891 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.891 INFO:tasks.workunit.client.0.smithi022.stdout: "clist_lat": { 2024-09-21T09:31:06.891 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 8, 2024-09-21T09:31:06.891 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 332492 2024-09-21T09:31:06.891 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.891 INFO:tasks.workunit.client.0.smithi022.stdout: "compress_lat": { 2024-09-21T09:31:06.891 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.891 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.891 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.891 INFO:tasks.workunit.client.0.smithi022.stdout: "compressed": { 2024-09-21T09:31:06.891 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.891 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.891 INFO:tasks.workunit.client.0.smithi022.stdout: "compressed_allocated": { 2024-09-21T09:31:06.892 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.892 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.892 INFO:tasks.workunit.client.0.smithi022.stdout: "compressed_original": { 2024-09-21T09:31:06.892 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.892 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.892 INFO:tasks.workunit.client.0.smithi022.stdout: "csum_lat": { 2024-09-21T09:31:06.892 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 8, 2024-09-21T09:31:06.892 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 19002 2024-09-21T09:31:06.892 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.892 INFO:tasks.workunit.client.0.smithi022.stdout: "decompress_lat": { 2024-09-21T09:31:06.892 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.892 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.892 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.892 INFO:tasks.workunit.client.0.smithi022.stdout: "kv_commit_lat": { 2024-09-21T09:31:06.892 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 173, 2024-09-21T09:31:06.892 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 48290848 2024-09-21T09:31:06.893 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.893 INFO:tasks.workunit.client.0.smithi022.stdout: "kv_final_lat": { 2024-09-21T09:31:06.893 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 163, 2024-09-21T09:31:06.893 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 10875385 2024-09-21T09:31:06.893 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.893 INFO:tasks.workunit.client.0.smithi022.stdout: "kv_flush_lat": { 2024-09-21T09:31:06.893 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 173, 2024-09-21T09:31:06.893 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 273861 2024-09-21T09:31:06.893 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.893 INFO:tasks.workunit.client.0.smithi022.stdout: "kv_sync_lat": { 2024-09-21T09:31:06.893 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 173, 2024-09-21T09:31:06.893 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 48564709 2024-09-21T09:31:06.893 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.893 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_get_keys_lat": { 2024-09-21T09:31:06.893 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.893 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.894 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.894 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_get_values_lat": { 2024-09-21T09:31:06.894 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 3, 2024-09-21T09:31:06.894 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 45531 2024-09-21T09:31:06.894 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.894 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_lower_bound_lat": { 2024-09-21T09:31:06.894 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 3, 2024-09-21T09:31:06.894 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 4564 2024-09-21T09:31:06.894 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.894 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_next_lat": { 2024-09-21T09:31:06.894 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.894 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.894 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.894 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_seek_to_first_lat": { 2024-09-21T09:31:06.895 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.895 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.895 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.895 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_upper_bound_lat": { 2024-09-21T09:31:06.895 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 3, 2024-09-21T09:31:06.895 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 8263 2024-09-21T09:31:06.895 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.895 INFO:tasks.workunit.client.0.smithi022.stdout: "onode_hits": { 2024-09-21T09:31:06.895 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 364 2024-09-21T09:31:06.895 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.895 INFO:tasks.workunit.client.0.smithi022.stdout: "onode_misses": { 2024-09-21T09:31:06.895 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 233 2024-09-21T09:31:06.895 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.895 INFO:tasks.workunit.client.0.smithi022.stdout: "read_lat": { 2024-09-21T09:31:06.895 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 93, 2024-09-21T09:31:06.895 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1467917 2024-09-21T09:31:06.896 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.896 INFO:tasks.workunit.client.0.smithi022.stdout: "read_onode_meta_lat": { 2024-09-21T09:31:06.896 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 156, 2024-09-21T09:31:06.896 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 259382 2024-09-21T09:31:06.896 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.896 INFO:tasks.workunit.client.0.smithi022.stdout: "read_wait_aio_lat": { 2024-09-21T09:31:06.896 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 63, 2024-09-21T09:31:06.896 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 720054 2024-09-21T09:31:06.896 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.896 INFO:tasks.workunit.client.0.smithi022.stdout: "reads_with_retries": { 2024-09-21T09:31:06.896 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.896 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.896 INFO:tasks.workunit.client.0.smithi022.stdout: "remove_lat": { 2024-09-21T09:31:06.896 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 5, 2024-09-21T09:31:06.897 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 120139 2024-09-21T09:31:06.897 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.897 INFO:tasks.workunit.client.0.smithi022.stdout: "state_aio_wait_lat": { 2024-09-21T09:31:06.897 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 261, 2024-09-21T09:31:06.897 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 3155241 2024-09-21T09:31:06.897 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.897 INFO:tasks.workunit.client.0.smithi022.stdout: "state_deferred_aio_wait_lat": { 2024-09-21T09:31:06.897 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 41, 2024-09-21T09:31:06.897 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2664064 2024-09-21T09:31:06.897 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.897 INFO:tasks.workunit.client.0.smithi022.stdout: "state_deferred_cleanup_lat": { 2024-09-21T09:31:06.897 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 40, 2024-09-21T09:31:06.897 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 234961781205 2024-09-21T09:31:06.897 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.898 INFO:tasks.workunit.client.0.smithi022.stdout: "state_deferred_queued_lat": { 2024-09-21T09:31:06.898 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 41, 2024-09-21T09:31:06.898 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 59734430380 2024-09-21T09:31:06.898 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.898 INFO:tasks.workunit.client.0.smithi022.stdout: "state_done_lat": { 2024-09-21T09:31:06.898 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 260, 2024-09-21T09:31:06.898 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 539644 2024-09-21T09:31:06.898 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.898 INFO:tasks.workunit.client.0.smithi022.stdout: "state_finishing_lat": { 2024-09-21T09:31:06.898 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 260, 2024-09-21T09:31:06.898 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 30665 2024-09-21T09:31:06.898 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.898 INFO:tasks.workunit.client.0.smithi022.stdout: "state_io_done_lat": { 2024-09-21T09:31:06.899 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 261, 2024-09-21T09:31:06.899 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 262379 2024-09-21T09:31:06.899 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.899 INFO:tasks.workunit.client.0.smithi022.stdout: "state_kv_commiting_lat": { 2024-09-21T09:31:06.899 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 261, 2024-09-21T09:31:06.899 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 145402039 2024-09-21T09:31:06.899 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.899 INFO:tasks.workunit.client.0.smithi022.stdout: "state_kv_done_lat": { 2024-09-21T09:31:06.899 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 261, 2024-09-21T09:31:06.899 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 85011 2024-09-21T09:31:06.899 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.899 INFO:tasks.workunit.client.0.smithi022.stdout: "state_kv_queued_lat": { 2024-09-21T09:31:06.899 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 261, 2024-09-21T09:31:06.900 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 83345383 2024-09-21T09:31:06.900 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.900 INFO:tasks.workunit.client.0.smithi022.stdout: "state_prepare_lat": { 2024-09-21T09:31:06.900 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 261, 2024-09-21T09:31:06.900 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 13784998 2024-09-21T09:31:06.900 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.900 INFO:tasks.workunit.client.0.smithi022.stdout: "stored": { 2024-09-21T09:31:06.900 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 310694 2024-09-21T09:31:06.900 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.900 INFO:tasks.workunit.client.0.smithi022.stdout: "truncate_lat": { 2024-09-21T09:31:06.900 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 39, 2024-09-21T09:31:06.900 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 176771 2024-09-21T09:31:06.900 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.900 INFO:tasks.workunit.client.0.smithi022.stdout: "txc_commit_lat": { 2024-09-21T09:31:06.901 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 261, 2024-09-21T09:31:06.901 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 245987319 2024-09-21T09:31:06.901 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.901 INFO:tasks.workunit.client.0.smithi022.stdout: "txc_submit_lat": { 2024-09-21T09:31:06.901 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 261, 2024-09-21T09:31:06.901 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 16421168 2024-09-21T09:31:06.901 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.901 INFO:tasks.workunit.client.0.smithi022.stdout: "txc_throttle_lat": { 2024-09-21T09:31:06.901 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 261, 2024-09-21T09:31:06.901 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1920806 2024-09-21T09:31:06.901 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.901 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.901 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore-pricache": { 2024-09-21T09:31:06.901 INFO:tasks.workunit.client.0.smithi022.stdout: "cache_bytes": { 2024-09-21T09:31:06.901 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1861463884 2024-09-21T09:31:06.902 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.902 INFO:tasks.workunit.client.0.smithi022.stdout: "heap_bytes": { 2024-09-21T09:31:06.902 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 269860864 2024-09-21T09:31:06.902 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.902 INFO:tasks.workunit.client.0.smithi022.stdout: "mapped_bytes": { 2024-09-21T09:31:06.902 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 269139968 2024-09-21T09:31:06.902 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.902 INFO:tasks.workunit.client.0.smithi022.stdout: "target_bytes": { 2024-09-21T09:31:06.902 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 3137376768 2024-09-21T09:31:06.902 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.902 INFO:tasks.workunit.client.0.smithi022.stdout: "unmapped_bytes": { 2024-09-21T09:31:06.902 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 720896 2024-09-21T09:31:06.902 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.902 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.903 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore-pricache:data": { 2024-09-21T09:31:06.903 INFO:tasks.workunit.client.0.smithi022.stdout: "committed_bytes": { 2024-09-21T09:31:06.903 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 159383552 2024-09-21T09:31:06.903 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.903 INFO:tasks.workunit.client.0.smithi022.stdout: "pri0_bytes": { 2024-09-21T09:31:06.903 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.903 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.903 INFO:tasks.workunit.client.0.smithi022.stdout: "pri10_bytes": { 2024-09-21T09:31:06.903 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.903 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.903 INFO:tasks.workunit.client.0.smithi022.stdout: "pri11_bytes": { 2024-09-21T09:31:06.903 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 85499084 2024-09-21T09:31:06.903 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.903 INFO:tasks.workunit.client.0.smithi022.stdout: "pri1_bytes": { 2024-09-21T09:31:06.904 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.904 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.904 INFO:tasks.workunit.client.0.smithi022.stdout: "pri2_bytes": { 2024-09-21T09:31:06.904 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.904 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.904 INFO:tasks.workunit.client.0.smithi022.stdout: "pri3_bytes": { 2024-09-21T09:31:06.904 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.904 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.904 INFO:tasks.workunit.client.0.smithi022.stdout: "pri4_bytes": { 2024-09-21T09:31:06.904 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.904 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.904 INFO:tasks.workunit.client.0.smithi022.stdout: "pri5_bytes": { 2024-09-21T09:31:06.904 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.904 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.905 INFO:tasks.workunit.client.0.smithi022.stdout: "pri6_bytes": { 2024-09-21T09:31:06.905 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 8192 2024-09-21T09:31:06.905 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.905 INFO:tasks.workunit.client.0.smithi022.stdout: "pri7_bytes": { 2024-09-21T09:31:06.905 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.905 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.905 INFO:tasks.workunit.client.0.smithi022.stdout: "pri8_bytes": { 2024-09-21T09:31:06.905 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.905 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.905 INFO:tasks.workunit.client.0.smithi022.stdout: "pri9_bytes": { 2024-09-21T09:31:06.905 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.905 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.905 INFO:tasks.workunit.client.0.smithi022.stdout: "reserved_bytes": { 2024-09-21T09:31:06.905 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 73876276 2024-09-21T09:31:06.906 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.906 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.906 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore-pricache:kv": { 2024-09-21T09:31:06.906 INFO:tasks.workunit.client.0.smithi022.stdout: "committed_bytes": { 2024-09-21T09:31:06.906 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 780140544 2024-09-21T09:31:06.906 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.906 INFO:tasks.workunit.client.0.smithi022.stdout: "pri0_bytes": { 2024-09-21T09:31:06.906 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 67108864 2024-09-21T09:31:06.906 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.906 INFO:tasks.workunit.client.0.smithi022.stdout: "pri10_bytes": { 2024-09-21T09:31:06.906 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.906 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.906 INFO:tasks.workunit.client.0.smithi022.stdout: "pri11_bytes": { 2024-09-21T09:31:06.906 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 641243131 2024-09-21T09:31:06.907 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.907 INFO:tasks.workunit.client.0.smithi022.stdout: "pri1_bytes": { 2024-09-21T09:31:06.907 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.907 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.907 INFO:tasks.workunit.client.0.smithi022.stdout: "pri2_bytes": { 2024-09-21T09:31:06.907 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.907 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.907 INFO:tasks.workunit.client.0.smithi022.stdout: "pri3_bytes": { 2024-09-21T09:31:06.907 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.907 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.907 INFO:tasks.workunit.client.0.smithi022.stdout: "pri4_bytes": { 2024-09-21T09:31:06.907 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.907 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.907 INFO:tasks.workunit.client.0.smithi022.stdout: "pri5_bytes": { 2024-09-21T09:31:06.908 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.908 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.908 INFO:tasks.workunit.client.0.smithi022.stdout: "pri6_bytes": { 2024-09-21T09:31:06.908 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2128 2024-09-21T09:31:06.908 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.908 INFO:tasks.workunit.client.0.smithi022.stdout: "pri7_bytes": { 2024-09-21T09:31:06.908 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.908 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.908 INFO:tasks.workunit.client.0.smithi022.stdout: "pri8_bytes": { 2024-09-21T09:31:06.908 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.908 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.908 INFO:tasks.workunit.client.0.smithi022.stdout: "pri9_bytes": { 2024-09-21T09:31:06.908 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.908 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.908 INFO:tasks.workunit.client.0.smithi022.stdout: "reserved_bytes": { 2024-09-21T09:31:06.909 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 71786421 2024-09-21T09:31:06.909 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.909 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.909 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore-pricache:kv_onode": { 2024-09-21T09:31:06.909 INFO:tasks.workunit.client.0.smithi022.stdout: "committed_bytes": { 2024-09-21T09:31:06.909 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 192937984 2024-09-21T09:31:06.909 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.909 INFO:tasks.workunit.client.0.smithi022.stdout: "pri0_bytes": { 2024-09-21T09:31:06.909 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 67108864 2024-09-21T09:31:06.909 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.909 INFO:tasks.workunit.client.0.smithi022.stdout: "pri10_bytes": { 2024-09-21T09:31:06.909 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 464 2024-09-21T09:31:06.909 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.909 INFO:tasks.workunit.client.0.smithi022.stdout: "pri11_bytes": { 2024-09-21T09:31:06.910 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 56999389 2024-09-21T09:31:06.910 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.910 INFO:tasks.workunit.client.0.smithi022.stdout: "pri1_bytes": { 2024-09-21T09:31:06.910 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.910 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.910 INFO:tasks.workunit.client.0.smithi022.stdout: "pri2_bytes": { 2024-09-21T09:31:06.910 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.910 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.910 INFO:tasks.workunit.client.0.smithi022.stdout: "pri3_bytes": { 2024-09-21T09:31:06.910 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.910 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.910 INFO:tasks.workunit.client.0.smithi022.stdout: "pri4_bytes": { 2024-09-21T09:31:06.910 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.910 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.911 INFO:tasks.workunit.client.0.smithi022.stdout: "pri5_bytes": { 2024-09-21T09:31:06.911 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.911 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.911 INFO:tasks.workunit.client.0.smithi022.stdout: "pri6_bytes": { 2024-09-21T09:31:06.911 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.911 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.911 INFO:tasks.workunit.client.0.smithi022.stdout: "pri7_bytes": { 2024-09-21T09:31:06.911 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.911 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.911 INFO:tasks.workunit.client.0.smithi022.stdout: "pri8_bytes": { 2024-09-21T09:31:06.911 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.911 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.911 INFO:tasks.workunit.client.0.smithi022.stdout: "pri9_bytes": { 2024-09-21T09:31:06.911 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.912 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.912 INFO:tasks.workunit.client.0.smithi022.stdout: "reserved_bytes": { 2024-09-21T09:31:06.912 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 68829267 2024-09-21T09:31:06.912 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.912 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.912 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore-pricache:meta": { 2024-09-21T09:31:06.912 INFO:tasks.workunit.client.0.smithi022.stdout: "committed_bytes": { 2024-09-21T09:31:06.912 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 713031680 2024-09-21T09:31:06.912 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.912 INFO:tasks.workunit.client.0.smithi022.stdout: "pri0_bytes": { 2024-09-21T09:31:06.912 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.912 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.912 INFO:tasks.workunit.client.0.smithi022.stdout: "pri10_bytes": { 2024-09-21T09:31:06.912 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.912 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.913 INFO:tasks.workunit.client.0.smithi022.stdout: "pri11_bytes": { 2024-09-21T09:31:06.913 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 641243131 2024-09-21T09:31:06.913 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.913 INFO:tasks.workunit.client.0.smithi022.stdout: "pri1_bytes": { 2024-09-21T09:31:06.913 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.913 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.913 INFO:tasks.workunit.client.0.smithi022.stdout: "pri2_bytes": { 2024-09-21T09:31:06.913 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.913 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.913 INFO:tasks.workunit.client.0.smithi022.stdout: "pri3_bytes": { 2024-09-21T09:31:06.913 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.913 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.913 INFO:tasks.workunit.client.0.smithi022.stdout: "pri4_bytes": { 2024-09-21T09:31:06.913 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.914 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.914 INFO:tasks.workunit.client.0.smithi022.stdout: "pri5_bytes": { 2024-09-21T09:31:06.914 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.914 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.914 INFO:tasks.workunit.client.0.smithi022.stdout: "pri6_bytes": { 2024-09-21T09:31:06.914 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 257959 2024-09-21T09:31:06.914 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.914 INFO:tasks.workunit.client.0.smithi022.stdout: "pri7_bytes": { 2024-09-21T09:31:06.914 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.914 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.914 INFO:tasks.workunit.client.0.smithi022.stdout: "pri8_bytes": { 2024-09-21T09:31:06.914 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.914 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.914 INFO:tasks.workunit.client.0.smithi022.stdout: "pri9_bytes": { 2024-09-21T09:31:06.914 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.915 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.915 INFO:tasks.workunit.client.0.smithi022.stdout: "reserved_bytes": { 2024-09-21T09:31:06.915 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 71530590 2024-09-21T09:31:06.915 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.915 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.915 INFO:tasks.workunit.client.0.smithi022.stdout: "objecter": { 2024-09-21T09:31:06.915 INFO:tasks.workunit.client.0.smithi022.stdout: "op_active": { 2024-09-21T09:31:06.915 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.915 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.915 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r": { 2024-09-21T09:31:06.915 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.915 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.915 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rmw": { 2024-09-21T09:31:06.915 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.916 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.916 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w": { 2024-09-21T09:31:06.916 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.916 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.916 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.916 INFO:tasks.workunit.client.0.smithi022.stdout: "osd": { 2024-09-21T09:31:06.916 INFO:tasks.workunit.client.0.smithi022.stdout: "numpg": { 2024-09-21T09:31:06.916 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 40 2024-09-21T09:31:06.916 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.916 INFO:tasks.workunit.client.0.smithi022.stdout: "numpg_removing": { 2024-09-21T09:31:06.916 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.916 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.916 INFO:tasks.workunit.client.0.smithi022.stdout: "op": { 2024-09-21T09:31:06.916 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1 2024-09-21T09:31:06.916 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.917 INFO:tasks.workunit.client.0.smithi022.stdout: "op_delayed_degraded": { 2024-09-21T09:31:06.917 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.917 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.917 INFO:tasks.workunit.client.0.smithi022.stdout: "op_delayed_unreadable": { 2024-09-21T09:31:06.917 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.917 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.917 INFO:tasks.workunit.client.0.smithi022.stdout: "op_in_bytes": { 2024-09-21T09:31:06.917 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 22 2024-09-21T09:31:06.917 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.917 INFO:tasks.workunit.client.0.smithi022.stdout: "op_latency": { 2024-09-21T09:31:06.917 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 1, 2024-09-21T09:31:06.917 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 7870434 2024-09-21T09:31:06.917 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.917 INFO:tasks.workunit.client.0.smithi022.stdout: "op_out_bytes": { 2024-09-21T09:31:06.918 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.918 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.918 INFO:tasks.workunit.client.0.smithi022.stdout: "op_prepare_latency": { 2024-09-21T09:31:06.918 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 1, 2024-09-21T09:31:06.918 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 786116 2024-09-21T09:31:06.918 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.918 INFO:tasks.workunit.client.0.smithi022.stdout: "op_process_latency": { 2024-09-21T09:31:06.918 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 1, 2024-09-21T09:31:06.918 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 7748154 2024-09-21T09:31:06.918 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.919 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r": { 2024-09-21T09:31:06.919 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.919 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.919 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r_latency": { 2024-09-21T09:31:06.919 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.919 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.919 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.919 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-21T09:31:06.919 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.919 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.919 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r_out_bytes": { 2024-09-21T09:31:06.919 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.919 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.919 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r_prepare_latency": { 2024-09-21T09:31:06.920 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.920 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.920 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.920 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r_process_latency": { 2024-09-21T09:31:06.920 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.920 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.920 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.920 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw": { 2024-09-21T09:31:06.920 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.920 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.920 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_in_bytes": { 2024-09-21T09:31:06.920 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.920 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.920 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_latency": { 2024-09-21T09:31:06.921 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.921 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.921 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.921 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-21T09:31:06.921 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.921 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.921 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-21T09:31:06.921 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.921 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.921 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_out_bytes": { 2024-09-21T09:31:06.921 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.921 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.921 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_prepare_latency": { 2024-09-21T09:31:06.921 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.922 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.922 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.922 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_process_latency": { 2024-09-21T09:31:06.922 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.922 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.922 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.922 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w": { 2024-09-21T09:31:06.922 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1 2024-09-21T09:31:06.922 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.922 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w_in_bytes": { 2024-09-21T09:31:06.922 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 22 2024-09-21T09:31:06.922 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.922 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w_latency": { 2024-09-21T09:31:06.923 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 1, 2024-09-21T09:31:06.923 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 7870434 2024-09-21T09:31:06.923 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.923 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-21T09:31:06.923 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.923 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.923 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w_prepare_latency": { 2024-09-21T09:31:06.923 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 1, 2024-09-21T09:31:06.923 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 786116 2024-09-21T09:31:06.923 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.923 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w_process_latency": { 2024-09-21T09:31:06.923 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 1, 2024-09-21T09:31:06.923 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 7748154 2024-09-21T09:31:06.923 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.924 INFO:tasks.workunit.client.0.smithi022.stdout: "op_wip": { 2024-09-21T09:31:06.924 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.924 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.924 INFO:tasks.workunit.client.0.smithi022.stdout: "recovery_bytes": { 2024-09-21T09:31:06.924 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.924 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.924 INFO:tasks.workunit.client.0.smithi022.stdout: "recovery_ops": { 2024-09-21T09:31:06.924 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.924 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.924 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_bytes": { 2024-09-21T09:31:06.924 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 95995035648 2024-09-21T09:31:06.924 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.924 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_bytes_used": { 2024-09-21T09:31:06.924 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 305201152 2024-09-21T09:31:06.925 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.925 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.925 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.925 INFO:tasks.workunit.client.0.smithi022.stdout: "osd.5": { 2024-09-21T09:31:06.925 INFO:tasks.workunit.client.0.smithi022.stdout: "bluefs": { 2024-09-21T09:31:06.925 INFO:tasks.workunit.client.0.smithi022.stdout: "alloc_slow_fallback": { 2024-09-21T09:31:06.925 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.925 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.925 INFO:tasks.workunit.client.0.smithi022.stdout: "alloc_slow_size_fallback": { 2024-09-21T09:31:06.925 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.925 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.925 INFO:tasks.workunit.client.0.smithi022.stdout: "alloc_unit_db": { 2024-09-21T09:31:06.926 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 65536 2024-09-21T09:31:06.926 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.926 INFO:tasks.workunit.client.0.smithi022.stdout: "alloc_unit_main": { 2024-09-21T09:31:06.926 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.926 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.926 INFO:tasks.workunit.client.0.smithi022.stdout: "alloc_unit_wal": { 2024-09-21T09:31:06.926 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.926 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.926 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes_written_slow": { 2024-09-21T09:31:06.926 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.926 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.926 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes_written_sst": { 2024-09-21T09:31:06.926 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 12288 2024-09-21T09:31:06.926 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.927 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes_written_wal": { 2024-09-21T09:31:06.927 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1298432 2024-09-21T09:31:06.927 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.927 INFO:tasks.workunit.client.0.smithi022.stdout: "compact_lat": { 2024-09-21T09:31:06.927 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.927 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.927 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.927 INFO:tasks.workunit.client.0.smithi022.stdout: "compact_lock_lat": { 2024-09-21T09:31:06.927 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.927 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.927 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.927 INFO:tasks.workunit.client.0.smithi022.stdout: "db_total_bytes": { 2024-09-21T09:31:06.927 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 95995035648 2024-09-21T09:31:06.927 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.928 INFO:tasks.workunit.client.0.smithi022.stdout: "db_used_bytes": { 2024-09-21T09:31:06.928 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 304480256 2024-09-21T09:31:06.928 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.928 INFO:tasks.workunit.client.0.smithi022.stdout: "log_bytes": { 2024-09-21T09:31:06.928 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 991232 2024-09-21T09:31:06.928 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.928 INFO:tasks.workunit.client.0.smithi022.stdout: "logged_bytes": { 2024-09-21T09:31:06.928 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 851968 2024-09-21T09:31:06.928 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.928 INFO:tasks.workunit.client.0.smithi022.stdout: "max_bytes_db": { 2024-09-21T09:31:06.928 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 312868864 2024-09-21T09:31:06.928 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.929 INFO:tasks.workunit.client.0.smithi022.stdout: "max_bytes_slow": { 2024-09-21T09:31:06.929 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.929 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.929 INFO:tasks.workunit.client.0.smithi022.stdout: "max_bytes_wal": { 2024-09-21T09:31:06.929 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.929 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.929 INFO:tasks.workunit.client.0.smithi022.stdout: "num_files": { 2024-09-21T09:31:06.929 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 14 2024-09-21T09:31:06.929 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.929 INFO:tasks.workunit.client.0.smithi022.stdout: "read_bytes": { 2024-09-21T09:31:06.929 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 195697 2024-09-21T09:31:06.929 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.929 INFO:tasks.workunit.client.0.smithi022.stdout: "read_count": { 2024-09-21T09:31:06.930 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 23 2024-09-21T09:31:06.930 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.930 INFO:tasks.workunit.client.0.smithi022.stdout: "read_disk_bytes": { 2024-09-21T09:31:06.930 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1269760 2024-09-21T09:31:06.930 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.930 INFO:tasks.workunit.client.0.smithi022.stdout: "read_disk_bytes_db": { 2024-09-21T09:31:06.930 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1273856 2024-09-21T09:31:06.930 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.930 INFO:tasks.workunit.client.0.smithi022.stdout: "read_disk_bytes_slow": { 2024-09-21T09:31:06.930 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.930 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.930 INFO:tasks.workunit.client.0.smithi022.stdout: "read_disk_bytes_wal": { 2024-09-21T09:31:06.931 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.931 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.931 INFO:tasks.workunit.client.0.smithi022.stdout: "read_disk_count": { 2024-09-21T09:31:06.931 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 15 2024-09-21T09:31:06.931 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.931 INFO:tasks.workunit.client.0.smithi022.stdout: "read_prefetch_bytes": { 2024-09-21T09:31:06.931 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 4789 2024-09-21T09:31:06.931 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.931 INFO:tasks.workunit.client.0.smithi022.stdout: "read_prefetch_count": { 2024-09-21T09:31:06.931 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 4 2024-09-21T09:31:06.931 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.931 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_buffer_bytes": { 2024-09-21T09:31:06.932 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 5589 2024-09-21T09:31:06.932 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.932 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_buffer_count": { 2024-09-21T09:31:06.932 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 24 2024-09-21T09:31:06.932 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.932 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_bytes": { 2024-09-21T09:31:06.932 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 5589 2024-09-21T09:31:06.932 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.932 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_count": { 2024-09-21T09:31:06.932 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 24 2024-09-21T09:31:06.932 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.932 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_disk_bytes": { 2024-09-21T09:31:06.932 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.932 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.933 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_disk_bytes_db": { 2024-09-21T09:31:06.933 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.933 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.933 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_disk_bytes_slow": { 2024-09-21T09:31:06.933 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.933 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.933 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_disk_bytes_wal": { 2024-09-21T09:31:06.933 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.933 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.933 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_disk_count": { 2024-09-21T09:31:06.933 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.934 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.934 INFO:tasks.workunit.client.0.smithi022.stdout: "slow_total_bytes": { 2024-09-21T09:31:06.934 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.934 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.934 INFO:tasks.workunit.client.0.smithi022.stdout: "slow_used_bytes": { 2024-09-21T09:31:06.934 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.934 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.934 INFO:tasks.workunit.client.0.smithi022.stdout: "wal_total_bytes": { 2024-09-21T09:31:06.934 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.934 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.934 INFO:tasks.workunit.client.0.smithi022.stdout: "wal_used_bytes": { 2024-09-21T09:31:06.934 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.934 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.934 INFO:tasks.workunit.client.0.smithi022.stdout: "write_bytes": { 2024-09-21T09:31:06.935 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2224128 2024-09-21T09:31:06.935 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.935 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.935 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore": { 2024-09-21T09:31:06.935 INFO:tasks.workunit.client.0.smithi022.stdout: "alloc_unit": { 2024-09-21T09:31:06.935 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 4096 2024-09-21T09:31:06.935 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.935 INFO:tasks.workunit.client.0.smithi022.stdout: "allocated": { 2024-09-21T09:31:06.935 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 700416 2024-09-21T09:31:06.935 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.935 INFO:tasks.workunit.client.0.smithi022.stdout: "clist_lat": { 2024-09-21T09:31:06.935 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 6, 2024-09-21T09:31:06.935 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 223263 2024-09-21T09:31:06.936 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.936 INFO:tasks.workunit.client.0.smithi022.stdout: "compress_lat": { 2024-09-21T09:31:06.936 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.936 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.936 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.936 INFO:tasks.workunit.client.0.smithi022.stdout: "compressed": { 2024-09-21T09:31:06.936 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.936 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.936 INFO:tasks.workunit.client.0.smithi022.stdout: "compressed_allocated": { 2024-09-21T09:31:06.936 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.936 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.936 INFO:tasks.workunit.client.0.smithi022.stdout: "compressed_original": { 2024-09-21T09:31:06.936 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.936 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.936 INFO:tasks.workunit.client.0.smithi022.stdout: "csum_lat": { 2024-09-21T09:31:06.937 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 8, 2024-09-21T09:31:06.937 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 21257 2024-09-21T09:31:06.937 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.937 INFO:tasks.workunit.client.0.smithi022.stdout: "decompress_lat": { 2024-09-21T09:31:06.937 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.937 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.937 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.937 INFO:tasks.workunit.client.0.smithi022.stdout: "kv_commit_lat": { 2024-09-21T09:31:06.937 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 191, 2024-09-21T09:31:06.937 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 57204104 2024-09-21T09:31:06.937 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.937 INFO:tasks.workunit.client.0.smithi022.stdout: "kv_final_lat": { 2024-09-21T09:31:06.937 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 185, 2024-09-21T09:31:06.938 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 7085387 2024-09-21T09:31:06.938 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.938 INFO:tasks.workunit.client.0.smithi022.stdout: "kv_flush_lat": { 2024-09-21T09:31:06.938 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 191, 2024-09-21T09:31:06.938 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 202500 2024-09-21T09:31:06.938 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.938 INFO:tasks.workunit.client.0.smithi022.stdout: "kv_sync_lat": { 2024-09-21T09:31:06.938 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 191, 2024-09-21T09:31:06.938 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 57406604 2024-09-21T09:31:06.938 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.938 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_get_keys_lat": { 2024-09-21T09:31:06.938 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.938 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.939 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.939 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_get_values_lat": { 2024-09-21T09:31:06.939 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 3, 2024-09-21T09:31:06.939 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 45534 2024-09-21T09:31:06.939 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.939 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_lower_bound_lat": { 2024-09-21T09:31:06.939 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 3, 2024-09-21T09:31:06.939 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 4226 2024-09-21T09:31:06.939 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.939 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_next_lat": { 2024-09-21T09:31:06.939 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.939 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.940 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.940 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_seek_to_first_lat": { 2024-09-21T09:31:06.940 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.940 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.940 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.940 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_upper_bound_lat": { 2024-09-21T09:31:06.940 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 2, 2024-09-21T09:31:06.940 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 5556 2024-09-21T09:31:06.940 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.940 INFO:tasks.workunit.client.0.smithi022.stdout: "onode_hits": { 2024-09-21T09:31:06.940 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 355 2024-09-21T09:31:06.940 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.940 INFO:tasks.workunit.client.0.smithi022.stdout: "onode_misses": { 2024-09-21T09:31:06.940 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 237 2024-09-21T09:31:06.941 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.941 INFO:tasks.workunit.client.0.smithi022.stdout: "read_lat": { 2024-09-21T09:31:06.941 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 87, 2024-09-21T09:31:06.941 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1264385 2024-09-21T09:31:06.941 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.941 INFO:tasks.workunit.client.0.smithi022.stdout: "read_onode_meta_lat": { 2024-09-21T09:31:06.941 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 140, 2024-09-21T09:31:06.941 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 265863 2024-09-21T09:31:06.941 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.941 INFO:tasks.workunit.client.0.smithi022.stdout: "read_wait_aio_lat": { 2024-09-21T09:31:06.941 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 53, 2024-09-21T09:31:06.941 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 902729 2024-09-21T09:31:06.941 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.942 INFO:tasks.workunit.client.0.smithi022.stdout: "reads_with_retries": { 2024-09-21T09:31:06.942 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.942 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.942 INFO:tasks.workunit.client.0.smithi022.stdout: "remove_lat": { 2024-09-21T09:31:06.942 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 4, 2024-09-21T09:31:06.942 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 84353 2024-09-21T09:31:06.942 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.942 INFO:tasks.workunit.client.0.smithi022.stdout: "state_aio_wait_lat": { 2024-09-21T09:31:06.942 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 284, 2024-09-21T09:31:06.942 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2665786 2024-09-21T09:31:06.942 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.942 INFO:tasks.workunit.client.0.smithi022.stdout: "state_deferred_aio_wait_lat": { 2024-09-21T09:31:06.943 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 41, 2024-09-21T09:31:06.943 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2599677 2024-09-21T09:31:06.943 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.943 INFO:tasks.workunit.client.0.smithi022.stdout: "state_deferred_cleanup_lat": { 2024-09-21T09:31:06.943 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 38, 2024-09-21T09:31:06.943 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 294532670170 2024-09-21T09:31:06.943 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.943 INFO:tasks.workunit.client.0.smithi022.stdout: "state_deferred_queued_lat": { 2024-09-21T09:31:06.943 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 41, 2024-09-21T09:31:06.943 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 66574400736 2024-09-21T09:31:06.943 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.943 INFO:tasks.workunit.client.0.smithi022.stdout: "state_done_lat": { 2024-09-21T09:31:06.943 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 281, 2024-09-21T09:31:06.943 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 7312017343 2024-09-21T09:31:06.943 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.944 INFO:tasks.workunit.client.0.smithi022.stdout: "state_finishing_lat": { 2024-09-21T09:31:06.944 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 281, 2024-09-21T09:31:06.944 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 33542 2024-09-21T09:31:06.944 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.944 INFO:tasks.workunit.client.0.smithi022.stdout: "state_io_done_lat": { 2024-09-21T09:31:06.944 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 284, 2024-09-21T09:31:06.944 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 245343 2024-09-21T09:31:06.944 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.944 INFO:tasks.workunit.client.0.smithi022.stdout: "state_kv_commiting_lat": { 2024-09-21T09:31:06.944 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 284, 2024-09-21T09:31:06.944 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 151544224 2024-09-21T09:31:06.944 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.945 INFO:tasks.workunit.client.0.smithi022.stdout: "state_kv_done_lat": { 2024-09-21T09:31:06.945 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 284, 2024-09-21T09:31:06.945 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 87321 2024-09-21T09:31:06.945 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.945 INFO:tasks.workunit.client.0.smithi022.stdout: "state_kv_queued_lat": { 2024-09-21T09:31:06.945 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 284, 2024-09-21T09:31:06.945 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 115059209 2024-09-21T09:31:06.945 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.945 INFO:tasks.workunit.client.0.smithi022.stdout: "state_prepare_lat": { 2024-09-21T09:31:06.945 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 284, 2024-09-21T09:31:06.945 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 14646137 2024-09-21T09:31:06.945 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.945 INFO:tasks.workunit.client.0.smithi022.stdout: "stored": { 2024-09-21T09:31:06.945 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 312082 2024-09-21T09:31:06.946 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.946 INFO:tasks.workunit.client.0.smithi022.stdout: "truncate_lat": { 2024-09-21T09:31:06.946 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 33, 2024-09-21T09:31:06.946 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 153003 2024-09-21T09:31:06.946 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.946 INFO:tasks.workunit.client.0.smithi022.stdout: "txc_commit_lat": { 2024-09-21T09:31:06.946 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 284, 2024-09-21T09:31:06.946 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 284197396 2024-09-21T09:31:06.946 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.946 INFO:tasks.workunit.client.0.smithi022.stdout: "txc_submit_lat": { 2024-09-21T09:31:06.946 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 284, 2024-09-21T09:31:06.946 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 17576248 2024-09-21T09:31:06.946 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.947 INFO:tasks.workunit.client.0.smithi022.stdout: "txc_throttle_lat": { 2024-09-21T09:31:06.947 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 284, 2024-09-21T09:31:06.947 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2153215 2024-09-21T09:31:06.947 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.947 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.947 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore-pricache": { 2024-09-21T09:31:06.947 INFO:tasks.workunit.client.0.smithi022.stdout: "cache_bytes": { 2024-09-21T09:31:06.947 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1861463884 2024-09-21T09:31:06.947 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.947 INFO:tasks.workunit.client.0.smithi022.stdout: "heap_bytes": { 2024-09-21T09:31:06.947 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 270974976 2024-09-21T09:31:06.947 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.947 INFO:tasks.workunit.client.0.smithi022.stdout: "mapped_bytes": { 2024-09-21T09:31:06.947 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 270041088 2024-09-21T09:31:06.947 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.948 INFO:tasks.workunit.client.0.smithi022.stdout: "target_bytes": { 2024-09-21T09:31:06.948 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 3137376768 2024-09-21T09:31:06.948 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.948 INFO:tasks.workunit.client.0.smithi022.stdout: "unmapped_bytes": { 2024-09-21T09:31:06.948 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 933888 2024-09-21T09:31:06.948 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.948 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.948 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore-pricache:data": { 2024-09-21T09:31:06.948 INFO:tasks.workunit.client.0.smithi022.stdout: "committed_bytes": { 2024-09-21T09:31:06.948 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 159383552 2024-09-21T09:31:06.948 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.948 INFO:tasks.workunit.client.0.smithi022.stdout: "pri0_bytes": { 2024-09-21T09:31:06.949 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.949 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.949 INFO:tasks.workunit.client.0.smithi022.stdout: "pri10_bytes": { 2024-09-21T09:31:06.949 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.949 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.949 INFO:tasks.workunit.client.0.smithi022.stdout: "pri11_bytes": { 2024-09-21T09:31:06.949 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 85499594 2024-09-21T09:31:06.949 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.949 INFO:tasks.workunit.client.0.smithi022.stdout: "pri1_bytes": { 2024-09-21T09:31:06.949 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.949 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.949 INFO:tasks.workunit.client.0.smithi022.stdout: "pri2_bytes": { 2024-09-21T09:31:06.950 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.950 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.950 INFO:tasks.workunit.client.0.smithi022.stdout: "pri3_bytes": { 2024-09-21T09:31:06.950 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.950 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.950 INFO:tasks.workunit.client.0.smithi022.stdout: "pri4_bytes": { 2024-09-21T09:31:06.950 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.950 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.950 INFO:tasks.workunit.client.0.smithi022.stdout: "pri5_bytes": { 2024-09-21T09:31:06.950 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.950 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.950 INFO:tasks.workunit.client.0.smithi022.stdout: "pri6_bytes": { 2024-09-21T09:31:06.951 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.951 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.951 INFO:tasks.workunit.client.0.smithi022.stdout: "pri7_bytes": { 2024-09-21T09:31:06.951 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.951 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.951 INFO:tasks.workunit.client.0.smithi022.stdout: "pri8_bytes": { 2024-09-21T09:31:06.951 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.951 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.951 INFO:tasks.workunit.client.0.smithi022.stdout: "pri9_bytes": { 2024-09-21T09:31:06.951 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.951 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.951 INFO:tasks.workunit.client.0.smithi022.stdout: "reserved_bytes": { 2024-09-21T09:31:06.952 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 73883958 2024-09-21T09:31:06.952 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.952 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.952 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore-pricache:kv": { 2024-09-21T09:31:06.952 INFO:tasks.workunit.client.0.smithi022.stdout: "committed_bytes": { 2024-09-21T09:31:06.952 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 780140544 2024-09-21T09:31:06.952 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.952 INFO:tasks.workunit.client.0.smithi022.stdout: "pri0_bytes": { 2024-09-21T09:31:06.952 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 67108864 2024-09-21T09:31:06.952 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.952 INFO:tasks.workunit.client.0.smithi022.stdout: "pri10_bytes": { 2024-09-21T09:31:06.953 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.953 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.953 INFO:tasks.workunit.client.0.smithi022.stdout: "pri11_bytes": { 2024-09-21T09:31:06.953 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 641246959 2024-09-21T09:31:06.953 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.953 INFO:tasks.workunit.client.0.smithi022.stdout: "pri1_bytes": { 2024-09-21T09:31:06.953 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.953 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.953 INFO:tasks.workunit.client.0.smithi022.stdout: "pri2_bytes": { 2024-09-21T09:31:06.953 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.953 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.953 INFO:tasks.workunit.client.0.smithi022.stdout: "pri3_bytes": { 2024-09-21T09:31:06.953 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.954 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.954 INFO:tasks.workunit.client.0.smithi022.stdout: "pri4_bytes": { 2024-09-21T09:31:06.954 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.954 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.954 INFO:tasks.workunit.client.0.smithi022.stdout: "pri5_bytes": { 2024-09-21T09:31:06.954 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.954 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.954 INFO:tasks.workunit.client.0.smithi022.stdout: "pri6_bytes": { 2024-09-21T09:31:06.954 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2128 2024-09-21T09:31:06.954 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.954 INFO:tasks.workunit.client.0.smithi022.stdout: "pri7_bytes": { 2024-09-21T09:31:06.954 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.954 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.955 INFO:tasks.workunit.client.0.smithi022.stdout: "pri8_bytes": { 2024-09-21T09:31:06.955 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.955 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.955 INFO:tasks.workunit.client.0.smithi022.stdout: "pri9_bytes": { 2024-09-21T09:31:06.955 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.955 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.955 INFO:tasks.workunit.client.0.smithi022.stdout: "reserved_bytes": { 2024-09-21T09:31:06.955 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 71782593 2024-09-21T09:31:06.955 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.955 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.955 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore-pricache:kv_onode": { 2024-09-21T09:31:06.955 INFO:tasks.workunit.client.0.smithi022.stdout: "committed_bytes": { 2024-09-21T09:31:06.956 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 192937984 2024-09-21T09:31:06.956 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.956 INFO:tasks.workunit.client.0.smithi022.stdout: "pri0_bytes": { 2024-09-21T09:31:06.956 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 67108864 2024-09-21T09:31:06.956 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.956 INFO:tasks.workunit.client.0.smithi022.stdout: "pri10_bytes": { 2024-09-21T09:31:06.956 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 464 2024-09-21T09:31:06.956 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.956 INFO:tasks.workunit.client.0.smithi022.stdout: "pri11_bytes": { 2024-09-21T09:31:06.956 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 56999729 2024-09-21T09:31:06.956 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.956 INFO:tasks.workunit.client.0.smithi022.stdout: "pri1_bytes": { 2024-09-21T09:31:06.956 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.956 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.957 INFO:tasks.workunit.client.0.smithi022.stdout: "pri2_bytes": { 2024-09-21T09:31:06.957 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.957 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.957 INFO:tasks.workunit.client.0.smithi022.stdout: "pri3_bytes": { 2024-09-21T09:31:06.957 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.957 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.957 INFO:tasks.workunit.client.0.smithi022.stdout: "pri4_bytes": { 2024-09-21T09:31:06.957 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.957 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.957 INFO:tasks.workunit.client.0.smithi022.stdout: "pri5_bytes": { 2024-09-21T09:31:06.957 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.957 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.957 INFO:tasks.workunit.client.0.smithi022.stdout: "pri6_bytes": { 2024-09-21T09:31:06.957 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.957 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.958 INFO:tasks.workunit.client.0.smithi022.stdout: "pri7_bytes": { 2024-09-21T09:31:06.958 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.958 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.958 INFO:tasks.workunit.client.0.smithi022.stdout: "pri8_bytes": { 2024-09-21T09:31:06.958 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.958 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.958 INFO:tasks.workunit.client.0.smithi022.stdout: "pri9_bytes": { 2024-09-21T09:31:06.958 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.958 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.958 INFO:tasks.workunit.client.0.smithi022.stdout: "reserved_bytes": { 2024-09-21T09:31:06.958 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 68828927 2024-09-21T09:31:06.958 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.958 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.958 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore-pricache:meta": { 2024-09-21T09:31:06.958 INFO:tasks.workunit.client.0.smithi022.stdout: "committed_bytes": { 2024-09-21T09:31:06.959 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 713031680 2024-09-21T09:31:06.959 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.959 INFO:tasks.workunit.client.0.smithi022.stdout: "pri0_bytes": { 2024-09-21T09:31:06.959 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.959 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.959 INFO:tasks.workunit.client.0.smithi022.stdout: "pri10_bytes": { 2024-09-21T09:31:06.959 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.959 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.959 INFO:tasks.workunit.client.0.smithi022.stdout: "pri11_bytes": { 2024-09-21T09:31:06.959 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 641246959 2024-09-21T09:31:06.959 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.959 INFO:tasks.workunit.client.0.smithi022.stdout: "pri1_bytes": { 2024-09-21T09:31:06.959 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.959 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.960 INFO:tasks.workunit.client.0.smithi022.stdout: "pri2_bytes": { 2024-09-21T09:31:06.960 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.960 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.960 INFO:tasks.workunit.client.0.smithi022.stdout: "pri3_bytes": { 2024-09-21T09:31:06.960 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.960 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.960 INFO:tasks.workunit.client.0.smithi022.stdout: "pri4_bytes": { 2024-09-21T09:31:06.960 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.960 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.960 INFO:tasks.workunit.client.0.smithi022.stdout: "pri5_bytes": { 2024-09-21T09:31:06.960 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 6963 2024-09-21T09:31:06.960 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.960 INFO:tasks.workunit.client.0.smithi022.stdout: "pri6_bytes": { 2024-09-21T09:31:06.960 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 253470 2024-09-21T09:31:06.961 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.961 INFO:tasks.workunit.client.0.smithi022.stdout: "pri7_bytes": { 2024-09-21T09:31:06.961 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.961 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.961 INFO:tasks.workunit.client.0.smithi022.stdout: "pri8_bytes": { 2024-09-21T09:31:06.961 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.961 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.961 INFO:tasks.workunit.client.0.smithi022.stdout: "pri9_bytes": { 2024-09-21T09:31:06.961 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.961 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.961 INFO:tasks.workunit.client.0.smithi022.stdout: "reserved_bytes": { 2024-09-21T09:31:06.961 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 71524288 2024-09-21T09:31:06.961 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.961 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.961 INFO:tasks.workunit.client.0.smithi022.stdout: "objecter": { 2024-09-21T09:31:06.962 INFO:tasks.workunit.client.0.smithi022.stdout: "op_active": { 2024-09-21T09:31:06.962 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.962 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.962 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r": { 2024-09-21T09:31:06.962 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.962 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.962 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rmw": { 2024-09-21T09:31:06.962 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.962 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.962 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w": { 2024-09-21T09:31:06.962 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.962 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.962 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.962 INFO:tasks.workunit.client.0.smithi022.stdout: "osd": { 2024-09-21T09:31:06.962 INFO:tasks.workunit.client.0.smithi022.stdout: "numpg": { 2024-09-21T09:31:06.963 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 41 2024-09-21T09:31:06.963 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.963 INFO:tasks.workunit.client.0.smithi022.stdout: "numpg_removing": { 2024-09-21T09:31:06.963 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.963 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.963 INFO:tasks.workunit.client.0.smithi022.stdout: "op": { 2024-09-21T09:31:06.963 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.963 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.963 INFO:tasks.workunit.client.0.smithi022.stdout: "op_delayed_degraded": { 2024-09-21T09:31:06.963 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.963 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.963 INFO:tasks.workunit.client.0.smithi022.stdout: "op_delayed_unreadable": { 2024-09-21T09:31:06.963 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.963 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.963 INFO:tasks.workunit.client.0.smithi022.stdout: "op_in_bytes": { 2024-09-21T09:31:06.963 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.964 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.964 INFO:tasks.workunit.client.0.smithi022.stdout: "op_latency": { 2024-09-21T09:31:06.964 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.964 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.964 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.964 INFO:tasks.workunit.client.0.smithi022.stdout: "op_out_bytes": { 2024-09-21T09:31:06.964 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.964 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.964 INFO:tasks.workunit.client.0.smithi022.stdout: "op_prepare_latency": { 2024-09-21T09:31:06.964 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.964 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.964 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.964 INFO:tasks.workunit.client.0.smithi022.stdout: "op_process_latency": { 2024-09-21T09:31:06.964 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.964 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.965 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.965 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r": { 2024-09-21T09:31:06.965 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.965 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.965 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r_latency": { 2024-09-21T09:31:06.965 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.965 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.965 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.965 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-21T09:31:06.965 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.965 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.965 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r_out_bytes": { 2024-09-21T09:31:06.965 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.965 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.965 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r_prepare_latency": { 2024-09-21T09:31:06.966 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.966 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.966 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.966 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r_process_latency": { 2024-09-21T09:31:06.966 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.966 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.966 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.966 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw": { 2024-09-21T09:31:06.966 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.966 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.966 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_in_bytes": { 2024-09-21T09:31:06.966 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.966 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.966 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_latency": { 2024-09-21T09:31:06.966 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.967 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.967 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.967 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-21T09:31:06.967 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.967 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.967 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-21T09:31:06.967 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.967 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.967 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_out_bytes": { 2024-09-21T09:31:06.967 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.967 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.967 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_prepare_latency": { 2024-09-21T09:31:06.967 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.967 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.968 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.968 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_process_latency": { 2024-09-21T09:31:06.968 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.968 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.968 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.968 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w": { 2024-09-21T09:31:06.968 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.968 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.968 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w_in_bytes": { 2024-09-21T09:31:06.968 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.968 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.968 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w_latency": { 2024-09-21T09:31:06.968 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.968 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.968 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.969 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-21T09:31:06.969 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.969 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.969 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w_prepare_latency": { 2024-09-21T09:31:06.969 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.969 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.969 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.969 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w_process_latency": { 2024-09-21T09:31:06.969 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.969 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.969 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.969 INFO:tasks.workunit.client.0.smithi022.stdout: "op_wip": { 2024-09-21T09:31:06.969 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.969 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.970 INFO:tasks.workunit.client.0.smithi022.stdout: "recovery_bytes": { 2024-09-21T09:31:06.970 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 590368 2024-09-21T09:31:06.970 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.970 INFO:tasks.workunit.client.0.smithi022.stdout: "recovery_ops": { 2024-09-21T09:31:06.970 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2 2024-09-21T09:31:06.970 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.970 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_bytes": { 2024-09-21T09:31:06.970 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 95995035648 2024-09-21T09:31:06.970 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.970 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_bytes_used": { 2024-09-21T09:31:06.970 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 305188864 2024-09-21T09:31:06.970 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.970 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.970 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.971 INFO:tasks.workunit.client.0.smithi022.stdout: "osd.6": { 2024-09-21T09:31:06.971 INFO:tasks.workunit.client.0.smithi022.stdout: "bluefs": { 2024-09-21T09:31:06.971 INFO:tasks.workunit.client.0.smithi022.stdout: "alloc_slow_fallback": { 2024-09-21T09:31:06.971 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.971 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.971 INFO:tasks.workunit.client.0.smithi022.stdout: "alloc_slow_size_fallback": { 2024-09-21T09:31:06.971 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.971 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.971 INFO:tasks.workunit.client.0.smithi022.stdout: "alloc_unit_db": { 2024-09-21T09:31:06.971 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 65536 2024-09-21T09:31:06.971 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.971 INFO:tasks.workunit.client.0.smithi022.stdout: "alloc_unit_main": { 2024-09-21T09:31:06.971 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.971 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.971 INFO:tasks.workunit.client.0.smithi022.stdout: "alloc_unit_wal": { 2024-09-21T09:31:06.972 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.972 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.972 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes_written_slow": { 2024-09-21T09:31:06.972 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.972 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.972 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes_written_sst": { 2024-09-21T09:31:06.972 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 12288 2024-09-21T09:31:06.972 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.972 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes_written_wal": { 2024-09-21T09:31:06.972 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1167360 2024-09-21T09:31:06.972 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.972 INFO:tasks.workunit.client.0.smithi022.stdout: "compact_lat": { 2024-09-21T09:31:06.972 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.972 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.973 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.973 INFO:tasks.workunit.client.0.smithi022.stdout: "compact_lock_lat": { 2024-09-21T09:31:06.973 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.973 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.973 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.973 INFO:tasks.workunit.client.0.smithi022.stdout: "db_total_bytes": { 2024-09-21T09:31:06.973 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 95995035648 2024-09-21T09:31:06.973 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.973 INFO:tasks.workunit.client.0.smithi022.stdout: "db_used_bytes": { 2024-09-21T09:31:06.973 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 304480256 2024-09-21T09:31:06.973 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.973 INFO:tasks.workunit.client.0.smithi022.stdout: "log_bytes": { 2024-09-21T09:31:06.973 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 917504 2024-09-21T09:31:06.973 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.973 INFO:tasks.workunit.client.0.smithi022.stdout: "logged_bytes": { 2024-09-21T09:31:06.974 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 778240 2024-09-21T09:31:06.974 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.974 INFO:tasks.workunit.client.0.smithi022.stdout: "max_bytes_db": { 2024-09-21T09:31:06.974 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 312868864 2024-09-21T09:31:06.974 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.974 INFO:tasks.workunit.client.0.smithi022.stdout: "max_bytes_slow": { 2024-09-21T09:31:06.974 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.974 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.974 INFO:tasks.workunit.client.0.smithi022.stdout: "max_bytes_wal": { 2024-09-21T09:31:06.974 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.974 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.974 INFO:tasks.workunit.client.0.smithi022.stdout: "num_files": { 2024-09-21T09:31:06.974 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 14 2024-09-21T09:31:06.974 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.975 INFO:tasks.workunit.client.0.smithi022.stdout: "read_bytes": { 2024-09-21T09:31:06.975 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 195697 2024-09-21T09:31:06.975 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.975 INFO:tasks.workunit.client.0.smithi022.stdout: "read_count": { 2024-09-21T09:31:06.975 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 23 2024-09-21T09:31:06.975 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.975 INFO:tasks.workunit.client.0.smithi022.stdout: "read_disk_bytes": { 2024-09-21T09:31:06.975 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1269760 2024-09-21T09:31:06.975 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.975 INFO:tasks.workunit.client.0.smithi022.stdout: "read_disk_bytes_db": { 2024-09-21T09:31:06.975 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1273856 2024-09-21T09:31:06.975 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.975 INFO:tasks.workunit.client.0.smithi022.stdout: "read_disk_bytes_slow": { 2024-09-21T09:31:06.975 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.976 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.976 INFO:tasks.workunit.client.0.smithi022.stdout: "read_disk_bytes_wal": { 2024-09-21T09:31:06.976 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.976 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.976 INFO:tasks.workunit.client.0.smithi022.stdout: "read_disk_count": { 2024-09-21T09:31:06.976 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 15 2024-09-21T09:31:06.976 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.976 INFO:tasks.workunit.client.0.smithi022.stdout: "read_prefetch_bytes": { 2024-09-21T09:31:06.976 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 4789 2024-09-21T09:31:06.976 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.976 INFO:tasks.workunit.client.0.smithi022.stdout: "read_prefetch_count": { 2024-09-21T09:31:06.976 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 4 2024-09-21T09:31:06.976 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.977 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_buffer_bytes": { 2024-09-21T09:31:06.977 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 5589 2024-09-21T09:31:06.977 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.977 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_buffer_count": { 2024-09-21T09:31:06.977 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 24 2024-09-21T09:31:06.977 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.977 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_bytes": { 2024-09-21T09:31:06.977 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 5589 2024-09-21T09:31:06.977 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.977 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_count": { 2024-09-21T09:31:06.977 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 24 2024-09-21T09:31:06.977 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.977 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_disk_bytes": { 2024-09-21T09:31:06.977 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.977 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.977 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_disk_bytes_db": { 2024-09-21T09:31:06.978 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.978 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.978 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_disk_bytes_slow": { 2024-09-21T09:31:06.978 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.978 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.978 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_disk_bytes_wal": { 2024-09-21T09:31:06.978 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.978 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.978 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_disk_count": { 2024-09-21T09:31:06.978 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.978 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.978 INFO:tasks.workunit.client.0.smithi022.stdout: "slow_total_bytes": { 2024-09-21T09:31:06.978 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.978 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.979 INFO:tasks.workunit.client.0.smithi022.stdout: "slow_used_bytes": { 2024-09-21T09:31:06.979 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.979 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.979 INFO:tasks.workunit.client.0.smithi022.stdout: "wal_total_bytes": { 2024-09-21T09:31:06.979 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.979 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.979 INFO:tasks.workunit.client.0.smithi022.stdout: "wal_used_bytes": { 2024-09-21T09:31:06.979 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.979 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.979 INFO:tasks.workunit.client.0.smithi022.stdout: "write_bytes": { 2024-09-21T09:31:06.979 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2019328 2024-09-21T09:31:06.979 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.979 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.981 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore": { 2024-09-21T09:31:06.981 INFO:tasks.workunit.client.0.smithi022.stdout: "alloc_unit": { 2024-09-21T09:31:06.981 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 4096 2024-09-21T09:31:06.982 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.982 INFO:tasks.workunit.client.0.smithi022.stdout: "allocated": { 2024-09-21T09:31:06.982 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1290240 2024-09-21T09:31:06.982 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.982 INFO:tasks.workunit.client.0.smithi022.stdout: "clist_lat": { 2024-09-21T09:31:06.982 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.982 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.982 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.982 INFO:tasks.workunit.client.0.smithi022.stdout: "compress_lat": { 2024-09-21T09:31:06.982 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.982 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.982 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.982 INFO:tasks.workunit.client.0.smithi022.stdout: "compressed": { 2024-09-21T09:31:06.982 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.983 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.983 INFO:tasks.workunit.client.0.smithi022.stdout: "compressed_allocated": { 2024-09-21T09:31:06.983 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.983 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.983 INFO:tasks.workunit.client.0.smithi022.stdout: "compressed_original": { 2024-09-21T09:31:06.983 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.983 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.983 INFO:tasks.workunit.client.0.smithi022.stdout: "csum_lat": { 2024-09-21T09:31:06.983 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 5, 2024-09-21T09:31:06.983 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 8640 2024-09-21T09:31:06.983 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.983 INFO:tasks.workunit.client.0.smithi022.stdout: "decompress_lat": { 2024-09-21T09:31:06.983 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.983 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.983 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.983 INFO:tasks.workunit.client.0.smithi022.stdout: "kv_commit_lat": { 2024-09-21T09:31:06.984 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 173, 2024-09-21T09:31:06.984 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 49130440 2024-09-21T09:31:06.984 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.984 INFO:tasks.workunit.client.0.smithi022.stdout: "kv_final_lat": { 2024-09-21T09:31:06.984 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 167, 2024-09-21T09:31:06.984 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 4765920 2024-09-21T09:31:06.984 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.984 INFO:tasks.workunit.client.0.smithi022.stdout: "kv_flush_lat": { 2024-09-21T09:31:06.984 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 173, 2024-09-21T09:31:06.984 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 172237 2024-09-21T09:31:06.984 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.984 INFO:tasks.workunit.client.0.smithi022.stdout: "kv_sync_lat": { 2024-09-21T09:31:06.984 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 173, 2024-09-21T09:31:06.984 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 49302677 2024-09-21T09:31:06.985 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.985 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_get_keys_lat": { 2024-09-21T09:31:06.985 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.985 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.985 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.985 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_get_values_lat": { 2024-09-21T09:31:06.985 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 1, 2024-09-21T09:31:06.985 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 39943 2024-09-21T09:31:06.985 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.986 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_lower_bound_lat": { 2024-09-21T09:31:06.986 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 2, 2024-09-21T09:31:06.986 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 12870 2024-09-21T09:31:06.986 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.986 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_next_lat": { 2024-09-21T09:31:06.986 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.986 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.986 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.986 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_seek_to_first_lat": { 2024-09-21T09:31:06.986 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.987 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.987 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.987 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_upper_bound_lat": { 2024-09-21T09:31:06.987 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 3, 2024-09-21T09:31:06.987 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 8352 2024-09-21T09:31:06.987 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.987 INFO:tasks.workunit.client.0.smithi022.stdout: "onode_hits": { 2024-09-21T09:31:06.987 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 308 2024-09-21T09:31:06.987 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.987 INFO:tasks.workunit.client.0.smithi022.stdout: "onode_misses": { 2024-09-21T09:31:06.987 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 241 2024-09-21T09:31:06.987 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.987 INFO:tasks.workunit.client.0.smithi022.stdout: "read_lat": { 2024-09-21T09:31:06.987 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 86, 2024-09-21T09:31:06.987 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 900908 2024-09-21T09:31:06.988 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.988 INFO:tasks.workunit.client.0.smithi022.stdout: "read_onode_meta_lat": { 2024-09-21T09:31:06.988 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 134, 2024-09-21T09:31:06.988 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 251268 2024-09-21T09:31:06.988 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.988 INFO:tasks.workunit.client.0.smithi022.stdout: "read_wait_aio_lat": { 2024-09-21T09:31:06.988 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 48, 2024-09-21T09:31:06.988 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 612055 2024-09-21T09:31:06.988 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.988 INFO:tasks.workunit.client.0.smithi022.stdout: "reads_with_retries": { 2024-09-21T09:31:06.988 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.988 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.988 INFO:tasks.workunit.client.0.smithi022.stdout: "remove_lat": { 2024-09-21T09:31:06.988 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:06.989 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.989 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.989 INFO:tasks.workunit.client.0.smithi022.stdout: "state_aio_wait_lat": { 2024-09-21T09:31:06.989 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 245, 2024-09-21T09:31:06.989 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2747998 2024-09-21T09:31:06.989 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.989 INFO:tasks.workunit.client.0.smithi022.stdout: "state_deferred_aio_wait_lat": { 2024-09-21T09:31:06.989 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 36, 2024-09-21T09:31:06.989 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2510672 2024-09-21T09:31:06.989 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.989 INFO:tasks.workunit.client.0.smithi022.stdout: "state_deferred_cleanup_lat": { 2024-09-21T09:31:06.989 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 33, 2024-09-21T09:31:06.989 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 251855406390 2024-09-21T09:31:06.989 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.990 INFO:tasks.workunit.client.0.smithi022.stdout: "state_deferred_queued_lat": { 2024-09-21T09:31:06.990 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 36, 2024-09-21T09:31:06.990 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 54118203799 2024-09-21T09:31:06.990 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.990 INFO:tasks.workunit.client.0.smithi022.stdout: "state_done_lat": { 2024-09-21T09:31:06.990 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 242, 2024-09-21T09:31:06.990 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 460560 2024-09-21T09:31:06.990 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.990 INFO:tasks.workunit.client.0.smithi022.stdout: "state_finishing_lat": { 2024-09-21T09:31:06.990 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 242, 2024-09-21T09:31:06.990 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 27951 2024-09-21T09:31:06.990 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.990 INFO:tasks.workunit.client.0.smithi022.stdout: "state_io_done_lat": { 2024-09-21T09:31:06.990 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 245, 2024-09-21T09:31:06.990 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 200260 2024-09-21T09:31:06.991 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.991 INFO:tasks.workunit.client.0.smithi022.stdout: "state_kv_commiting_lat": { 2024-09-21T09:31:06.991 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 245, 2024-09-21T09:31:06.991 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 99524523 2024-09-21T09:31:06.991 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.991 INFO:tasks.workunit.client.0.smithi022.stdout: "state_kv_done_lat": { 2024-09-21T09:31:06.991 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 245, 2024-09-21T09:31:06.991 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 79625 2024-09-21T09:31:06.991 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.991 INFO:tasks.workunit.client.0.smithi022.stdout: "state_kv_queued_lat": { 2024-09-21T09:31:06.991 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 245, 2024-09-21T09:31:06.991 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 67615661 2024-09-21T09:31:06.991 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.991 INFO:tasks.workunit.client.0.smithi022.stdout: "state_prepare_lat": { 2024-09-21T09:31:06.991 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 245, 2024-09-21T09:31:06.992 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 17205604 2024-09-21T09:31:06.992 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.992 INFO:tasks.workunit.client.0.smithi022.stdout: "stored": { 2024-09-21T09:31:06.992 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 902656 2024-09-21T09:31:06.992 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.992 INFO:tasks.workunit.client.0.smithi022.stdout: "truncate_lat": { 2024-09-21T09:31:06.992 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 29, 2024-09-21T09:31:06.992 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 128923 2024-09-21T09:31:06.992 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.992 INFO:tasks.workunit.client.0.smithi022.stdout: "txc_commit_lat": { 2024-09-21T09:31:06.992 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 245, 2024-09-21T09:31:06.992 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 187328793 2024-09-21T09:31:06.992 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.992 INFO:tasks.workunit.client.0.smithi022.stdout: "txc_submit_lat": { 2024-09-21T09:31:06.992 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 245, 2024-09-21T09:31:06.992 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 19285473 2024-09-21T09:31:06.993 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.993 INFO:tasks.workunit.client.0.smithi022.stdout: "txc_throttle_lat": { 2024-09-21T09:31:06.993 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 245, 2024-09-21T09:31:06.993 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 5521377 2024-09-21T09:31:06.993 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.993 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.993 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore-pricache": { 2024-09-21T09:31:06.993 INFO:tasks.workunit.client.0.smithi022.stdout: "cache_bytes": { 2024-09-21T09:31:06.993 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1861463884 2024-09-21T09:31:06.993 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.993 INFO:tasks.workunit.client.0.smithi022.stdout: "heap_bytes": { 2024-09-21T09:31:06.993 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 269090816 2024-09-21T09:31:06.993 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.993 INFO:tasks.workunit.client.0.smithi022.stdout: "mapped_bytes": { 2024-09-21T09:31:06.993 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 268804096 2024-09-21T09:31:06.993 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.994 INFO:tasks.workunit.client.0.smithi022.stdout: "target_bytes": { 2024-09-21T09:31:06.994 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 3137376768 2024-09-21T09:31:06.994 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.994 INFO:tasks.workunit.client.0.smithi022.stdout: "unmapped_bytes": { 2024-09-21T09:31:06.994 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 286720 2024-09-21T09:31:06.994 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.994 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.994 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore-pricache:data": { 2024-09-21T09:31:06.994 INFO:tasks.workunit.client.0.smithi022.stdout: "committed_bytes": { 2024-09-21T09:31:06.994 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 159383552 2024-09-21T09:31:06.994 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.994 INFO:tasks.workunit.client.0.smithi022.stdout: "pri0_bytes": { 2024-09-21T09:31:06.994 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.994 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.994 INFO:tasks.workunit.client.0.smithi022.stdout: "pri10_bytes": { 2024-09-21T09:31:06.995 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.995 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.995 INFO:tasks.workunit.client.0.smithi022.stdout: "pri11_bytes": { 2024-09-21T09:31:06.995 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 85498940 2024-09-21T09:31:06.995 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.995 INFO:tasks.workunit.client.0.smithi022.stdout: "pri1_bytes": { 2024-09-21T09:31:06.995 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.995 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.995 INFO:tasks.workunit.client.0.smithi022.stdout: "pri2_bytes": { 2024-09-21T09:31:06.995 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.995 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.995 INFO:tasks.workunit.client.0.smithi022.stdout: "pri3_bytes": { 2024-09-21T09:31:06.995 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.995 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.996 INFO:tasks.workunit.client.0.smithi022.stdout: "pri4_bytes": { 2024-09-21T09:31:06.996 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.996 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.996 INFO:tasks.workunit.client.0.smithi022.stdout: "pri5_bytes": { 2024-09-21T09:31:06.996 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.996 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.996 INFO:tasks.workunit.client.0.smithi022.stdout: "pri6_bytes": { 2024-09-21T09:31:06.996 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 8192 2024-09-21T09:31:06.996 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.996 INFO:tasks.workunit.client.0.smithi022.stdout: "pri7_bytes": { 2024-09-21T09:31:06.996 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.996 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.996 INFO:tasks.workunit.client.0.smithi022.stdout: "pri8_bytes": { 2024-09-21T09:31:06.996 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.996 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.996 INFO:tasks.workunit.client.0.smithi022.stdout: "pri9_bytes": { 2024-09-21T09:31:06.997 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.997 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.997 INFO:tasks.workunit.client.0.smithi022.stdout: "reserved_bytes": { 2024-09-21T09:31:06.997 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 73876420 2024-09-21T09:31:06.997 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:06.997 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.997 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore-pricache:kv": { 2024-09-21T09:31:06.997 INFO:tasks.workunit.client.0.smithi022.stdout: "committed_bytes": { 2024-09-21T09:31:06.997 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 780140544 2024-09-21T09:31:06.997 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.997 INFO:tasks.workunit.client.0.smithi022.stdout: "pri0_bytes": { 2024-09-21T09:31:06.997 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 67108864 2024-09-21T09:31:06.997 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.997 INFO:tasks.workunit.client.0.smithi022.stdout: "pri10_bytes": { 2024-09-21T09:31:06.997 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.997 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.998 INFO:tasks.workunit.client.0.smithi022.stdout: "pri11_bytes": { 2024-09-21T09:31:06.998 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 641242056 2024-09-21T09:31:06.998 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.998 INFO:tasks.workunit.client.0.smithi022.stdout: "pri1_bytes": { 2024-09-21T09:31:06.998 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.998 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.998 INFO:tasks.workunit.client.0.smithi022.stdout: "pri2_bytes": { 2024-09-21T09:31:06.998 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.998 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.998 INFO:tasks.workunit.client.0.smithi022.stdout: "pri3_bytes": { 2024-09-21T09:31:06.998 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.998 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.998 INFO:tasks.workunit.client.0.smithi022.stdout: "pri4_bytes": { 2024-09-21T09:31:06.999 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.999 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.999 INFO:tasks.workunit.client.0.smithi022.stdout: "pri5_bytes": { 2024-09-21T09:31:06.999 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.999 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.999 INFO:tasks.workunit.client.0.smithi022.stdout: "pri6_bytes": { 2024-09-21T09:31:06.999 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2128 2024-09-21T09:31:06.999 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.999 INFO:tasks.workunit.client.0.smithi022.stdout: "pri7_bytes": { 2024-09-21T09:31:06.999 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.999 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.999 INFO:tasks.workunit.client.0.smithi022.stdout: "pri8_bytes": { 2024-09-21T09:31:06.999 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:06.999 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:06.999 INFO:tasks.workunit.client.0.smithi022.stdout: "pri9_bytes": { 2024-09-21T09:31:07.000 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.000 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.000 INFO:tasks.workunit.client.0.smithi022.stdout: "reserved_bytes": { 2024-09-21T09:31:07.000 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 71787496 2024-09-21T09:31:07.000 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:07.000 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.000 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore-pricache:kv_onode": { 2024-09-21T09:31:07.000 INFO:tasks.workunit.client.0.smithi022.stdout: "committed_bytes": { 2024-09-21T09:31:07.000 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 192937984 2024-09-21T09:31:07.000 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.000 INFO:tasks.workunit.client.0.smithi022.stdout: "pri0_bytes": { 2024-09-21T09:31:07.000 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 67108864 2024-09-21T09:31:07.000 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.000 INFO:tasks.workunit.client.0.smithi022.stdout: "pri10_bytes": { 2024-09-21T09:31:07.001 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 464 2024-09-21T09:31:07.001 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.001 INFO:tasks.workunit.client.0.smithi022.stdout: "pri11_bytes": { 2024-09-21T09:31:07.001 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 56999293 2024-09-21T09:31:07.001 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.001 INFO:tasks.workunit.client.0.smithi022.stdout: "pri1_bytes": { 2024-09-21T09:31:07.001 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.001 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.001 INFO:tasks.workunit.client.0.smithi022.stdout: "pri2_bytes": { 2024-09-21T09:31:07.001 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.001 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.001 INFO:tasks.workunit.client.0.smithi022.stdout: "pri3_bytes": { 2024-09-21T09:31:07.001 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.001 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.001 INFO:tasks.workunit.client.0.smithi022.stdout: "pri4_bytes": { 2024-09-21T09:31:07.001 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.002 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.002 INFO:tasks.workunit.client.0.smithi022.stdout: "pri5_bytes": { 2024-09-21T09:31:07.002 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.002 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.002 INFO:tasks.workunit.client.0.smithi022.stdout: "pri6_bytes": { 2024-09-21T09:31:07.002 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.002 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.002 INFO:tasks.workunit.client.0.smithi022.stdout: "pri7_bytes": { 2024-09-21T09:31:07.002 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.002 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.002 INFO:tasks.workunit.client.0.smithi022.stdout: "pri8_bytes": { 2024-09-21T09:31:07.002 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.002 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.002 INFO:tasks.workunit.client.0.smithi022.stdout: "pri9_bytes": { 2024-09-21T09:31:07.002 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.002 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.003 INFO:tasks.workunit.client.0.smithi022.stdout: "reserved_bytes": { 2024-09-21T09:31:07.003 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 68829363 2024-09-21T09:31:07.003 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:07.003 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.003 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore-pricache:meta": { 2024-09-21T09:31:07.003 INFO:tasks.workunit.client.0.smithi022.stdout: "committed_bytes": { 2024-09-21T09:31:07.003 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 713031680 2024-09-21T09:31:07.003 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.003 INFO:tasks.workunit.client.0.smithi022.stdout: "pri0_bytes": { 2024-09-21T09:31:07.003 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.003 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.003 INFO:tasks.workunit.client.0.smithi022.stdout: "pri10_bytes": { 2024-09-21T09:31:07.003 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.003 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.003 INFO:tasks.workunit.client.0.smithi022.stdout: "pri11_bytes": { 2024-09-21T09:31:07.003 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 641242056 2024-09-21T09:31:07.004 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.004 INFO:tasks.workunit.client.0.smithi022.stdout: "pri1_bytes": { 2024-09-21T09:31:07.004 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.004 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.004 INFO:tasks.workunit.client.0.smithi022.stdout: "pri2_bytes": { 2024-09-21T09:31:07.004 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.004 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.004 INFO:tasks.workunit.client.0.smithi022.stdout: "pri3_bytes": { 2024-09-21T09:31:07.004 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.004 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.004 INFO:tasks.workunit.client.0.smithi022.stdout: "pri4_bytes": { 2024-09-21T09:31:07.004 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.004 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.004 INFO:tasks.workunit.client.0.smithi022.stdout: "pri5_bytes": { 2024-09-21T09:31:07.004 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 7111 2024-09-21T09:31:07.004 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.005 INFO:tasks.workunit.client.0.smithi022.stdout: "pri6_bytes": { 2024-09-21T09:31:07.005 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 256025 2024-09-21T09:31:07.005 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.005 INFO:tasks.workunit.client.0.smithi022.stdout: "pri7_bytes": { 2024-09-21T09:31:07.005 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.005 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.005 INFO:tasks.workunit.client.0.smithi022.stdout: "pri8_bytes": { 2024-09-21T09:31:07.005 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.005 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.005 INFO:tasks.workunit.client.0.smithi022.stdout: "pri9_bytes": { 2024-09-21T09:31:07.005 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.005 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.005 INFO:tasks.workunit.client.0.smithi022.stdout: "reserved_bytes": { 2024-09-21T09:31:07.006 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 71526488 2024-09-21T09:31:07.006 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:07.006 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.006 INFO:tasks.workunit.client.0.smithi022.stdout: "objecter": { 2024-09-21T09:31:07.006 INFO:tasks.workunit.client.0.smithi022.stdout: "op_active": { 2024-09-21T09:31:07.006 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.006 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.006 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r": { 2024-09-21T09:31:07.006 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.006 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.006 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rmw": { 2024-09-21T09:31:07.006 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.006 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.007 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w": { 2024-09-21T09:31:07.007 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.007 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:07.007 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.007 INFO:tasks.workunit.client.0.smithi022.stdout: "osd": { 2024-09-21T09:31:07.007 INFO:tasks.workunit.client.0.smithi022.stdout: "numpg": { 2024-09-21T09:31:07.007 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 39 2024-09-21T09:31:07.007 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.007 INFO:tasks.workunit.client.0.smithi022.stdout: "numpg_removing": { 2024-09-21T09:31:07.007 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.007 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.007 INFO:tasks.workunit.client.0.smithi022.stdout: "op": { 2024-09-21T09:31:07.007 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 13 2024-09-21T09:31:07.007 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.008 INFO:tasks.workunit.client.0.smithi022.stdout: "op_delayed_degraded": { 2024-09-21T09:31:07.008 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.008 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.008 INFO:tasks.workunit.client.0.smithi022.stdout: "op_delayed_unreadable": { 2024-09-21T09:31:07.008 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.008 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.008 INFO:tasks.workunit.client.0.smithi022.stdout: "op_in_bytes": { 2024-09-21T09:31:07.008 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2804 2024-09-21T09:31:07.008 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.008 INFO:tasks.workunit.client.0.smithi022.stdout: "op_latency": { 2024-09-21T09:31:07.008 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 13, 2024-09-21T09:31:07.008 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 68095634 2024-09-21T09:31:07.008 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.008 INFO:tasks.workunit.client.0.smithi022.stdout: "op_out_bytes": { 2024-09-21T09:31:07.009 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.009 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.009 INFO:tasks.workunit.client.0.smithi022.stdout: "op_prepare_latency": { 2024-09-21T09:31:07.009 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 15, 2024-09-21T09:31:07.009 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 9750663 2024-09-21T09:31:07.009 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.009 INFO:tasks.workunit.client.0.smithi022.stdout: "op_process_latency": { 2024-09-21T09:31:07.009 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 13, 2024-09-21T09:31:07.009 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 27964068 2024-09-21T09:31:07.009 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.009 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r": { 2024-09-21T09:31:07.009 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.009 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.009 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r_latency": { 2024-09-21T09:31:07.010 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:07.010 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.010 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.010 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-21T09:31:07.010 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.010 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.010 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r_out_bytes": { 2024-09-21T09:31:07.010 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.010 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.010 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r_prepare_latency": { 2024-09-21T09:31:07.010 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:07.010 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.010 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.011 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r_process_latency": { 2024-09-21T09:31:07.011 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:07.011 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.011 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.011 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw": { 2024-09-21T09:31:07.011 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.011 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.011 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_in_bytes": { 2024-09-21T09:31:07.011 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.011 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.011 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_latency": { 2024-09-21T09:31:07.011 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:07.011 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.012 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.012 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-21T09:31:07.012 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.012 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.012 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-21T09:31:07.012 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.012 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.012 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_out_bytes": { 2024-09-21T09:31:07.012 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.012 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.012 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_prepare_latency": { 2024-09-21T09:31:07.012 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:07.012 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.013 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.013 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_process_latency": { 2024-09-21T09:31:07.013 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:07.013 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.013 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.013 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w": { 2024-09-21T09:31:07.013 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 13 2024-09-21T09:31:07.013 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.013 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w_in_bytes": { 2024-09-21T09:31:07.013 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2804 2024-09-21T09:31:07.013 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.013 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w_latency": { 2024-09-21T09:31:07.014 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 13, 2024-09-21T09:31:07.014 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 68095634 2024-09-21T09:31:07.014 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.014 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-21T09:31:07.014 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.014 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.014 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w_prepare_latency": { 2024-09-21T09:31:07.014 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 15, 2024-09-21T09:31:07.014 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 9750663 2024-09-21T09:31:07.014 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.014 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w_process_latency": { 2024-09-21T09:31:07.015 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 13, 2024-09-21T09:31:07.015 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 27964068 2024-09-21T09:31:07.015 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.015 INFO:tasks.workunit.client.0.smithi022.stdout: "op_wip": { 2024-09-21T09:31:07.015 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.015 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.015 INFO:tasks.workunit.client.0.smithi022.stdout: "recovery_bytes": { 2024-09-21T09:31:07.015 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 590368 2024-09-21T09:31:07.015 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.015 INFO:tasks.workunit.client.0.smithi022.stdout: "recovery_ops": { 2024-09-21T09:31:07.015 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2 2024-09-21T09:31:07.015 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.015 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_bytes": { 2024-09-21T09:31:07.015 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 95995035648 2024-09-21T09:31:07.016 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.016 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_bytes_used": { 2024-09-21T09:31:07.016 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 305778688 2024-09-21T09:31:07.016 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:07.016 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:07.016 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.016 INFO:tasks.workunit.client.0.smithi022.stdout: "osd.7": { 2024-09-21T09:31:07.016 INFO:tasks.workunit.client.0.smithi022.stdout: "bluefs": { 2024-09-21T09:31:07.016 INFO:tasks.workunit.client.0.smithi022.stdout: "alloc_slow_fallback": { 2024-09-21T09:31:07.016 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.016 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.016 INFO:tasks.workunit.client.0.smithi022.stdout: "alloc_slow_size_fallback": { 2024-09-21T09:31:07.016 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.016 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.016 INFO:tasks.workunit.client.0.smithi022.stdout: "alloc_unit_db": { 2024-09-21T09:31:07.017 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 65536 2024-09-21T09:31:07.017 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.017 INFO:tasks.workunit.client.0.smithi022.stdout: "alloc_unit_main": { 2024-09-21T09:31:07.017 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.017 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.017 INFO:tasks.workunit.client.0.smithi022.stdout: "alloc_unit_wal": { 2024-09-21T09:31:07.017 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.017 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.017 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes_written_slow": { 2024-09-21T09:31:07.017 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.017 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.017 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes_written_sst": { 2024-09-21T09:31:07.017 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 12288 2024-09-21T09:31:07.017 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.018 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes_written_wal": { 2024-09-21T09:31:07.018 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1101824 2024-09-21T09:31:07.018 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.018 INFO:tasks.workunit.client.0.smithi022.stdout: "compact_lat": { 2024-09-21T09:31:07.018 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:07.018 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.018 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.018 INFO:tasks.workunit.client.0.smithi022.stdout: "compact_lock_lat": { 2024-09-21T09:31:07.018 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:07.018 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.018 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.018 INFO:tasks.workunit.client.0.smithi022.stdout: "db_total_bytes": { 2024-09-21T09:31:07.019 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 95995035648 2024-09-21T09:31:07.019 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.019 INFO:tasks.workunit.client.0.smithi022.stdout: "db_used_bytes": { 2024-09-21T09:31:07.019 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 304480256 2024-09-21T09:31:07.019 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.019 INFO:tasks.workunit.client.0.smithi022.stdout: "log_bytes": { 2024-09-21T09:31:07.019 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 872448 2024-09-21T09:31:07.019 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.019 INFO:tasks.workunit.client.0.smithi022.stdout: "logged_bytes": { 2024-09-21T09:31:07.019 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 733184 2024-09-21T09:31:07.019 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.019 INFO:tasks.workunit.client.0.smithi022.stdout: "max_bytes_db": { 2024-09-21T09:31:07.019 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 312868864 2024-09-21T09:31:07.019 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.020 INFO:tasks.workunit.client.0.smithi022.stdout: "max_bytes_slow": { 2024-09-21T09:31:07.020 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.020 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.020 INFO:tasks.workunit.client.0.smithi022.stdout: "max_bytes_wal": { 2024-09-21T09:31:07.020 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.020 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.020 INFO:tasks.workunit.client.0.smithi022.stdout: "num_files": { 2024-09-21T09:31:07.020 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 14 2024-09-21T09:31:07.020 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.020 INFO:tasks.workunit.client.0.smithi022.stdout: "read_bytes": { 2024-09-21T09:31:07.020 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 195697 2024-09-21T09:31:07.020 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.020 INFO:tasks.workunit.client.0.smithi022.stdout: "read_count": { 2024-09-21T09:31:07.020 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 23 2024-09-21T09:31:07.021 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.021 INFO:tasks.workunit.client.0.smithi022.stdout: "read_disk_bytes": { 2024-09-21T09:31:07.021 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1269760 2024-09-21T09:31:07.021 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.021 INFO:tasks.workunit.client.0.smithi022.stdout: "read_disk_bytes_db": { 2024-09-21T09:31:07.021 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1273856 2024-09-21T09:31:07.021 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.021 INFO:tasks.workunit.client.0.smithi022.stdout: "read_disk_bytes_slow": { 2024-09-21T09:31:07.021 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.021 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.021 INFO:tasks.workunit.client.0.smithi022.stdout: "read_disk_bytes_wal": { 2024-09-21T09:31:07.021 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.021 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.021 INFO:tasks.workunit.client.0.smithi022.stdout: "read_disk_count": { 2024-09-21T09:31:07.022 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 15 2024-09-21T09:31:07.022 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.022 INFO:tasks.workunit.client.0.smithi022.stdout: "read_prefetch_bytes": { 2024-09-21T09:31:07.022 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 4789 2024-09-21T09:31:07.022 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.022 INFO:tasks.workunit.client.0.smithi022.stdout: "read_prefetch_count": { 2024-09-21T09:31:07.022 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 4 2024-09-21T09:31:07.022 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.022 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_buffer_bytes": { 2024-09-21T09:31:07.022 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 5589 2024-09-21T09:31:07.022 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.022 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_buffer_count": { 2024-09-21T09:31:07.022 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 24 2024-09-21T09:31:07.022 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.023 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_bytes": { 2024-09-21T09:31:07.023 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 5589 2024-09-21T09:31:07.023 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.023 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_count": { 2024-09-21T09:31:07.023 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 24 2024-09-21T09:31:07.023 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.023 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_disk_bytes": { 2024-09-21T09:31:07.023 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.023 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.023 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_disk_bytes_db": { 2024-09-21T09:31:07.023 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.023 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.023 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_disk_bytes_slow": { 2024-09-21T09:31:07.024 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.024 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.024 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_disk_bytes_wal": { 2024-09-21T09:31:07.024 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.024 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.024 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_disk_count": { 2024-09-21T09:31:07.024 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.024 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.024 INFO:tasks.workunit.client.0.smithi022.stdout: "slow_total_bytes": { 2024-09-21T09:31:07.024 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.024 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.024 INFO:tasks.workunit.client.0.smithi022.stdout: "slow_used_bytes": { 2024-09-21T09:31:07.024 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.024 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.025 INFO:tasks.workunit.client.0.smithi022.stdout: "wal_total_bytes": { 2024-09-21T09:31:07.025 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.025 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.025 INFO:tasks.workunit.client.0.smithi022.stdout: "wal_used_bytes": { 2024-09-21T09:31:07.025 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.025 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.025 INFO:tasks.workunit.client.0.smithi022.stdout: "write_bytes": { 2024-09-21T09:31:07.025 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1908736 2024-09-21T09:31:07.025 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:07.025 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.025 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore": { 2024-09-21T09:31:07.025 INFO:tasks.workunit.client.0.smithi022.stdout: "alloc_unit": { 2024-09-21T09:31:07.025 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 4096 2024-09-21T09:31:07.025 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.026 INFO:tasks.workunit.client.0.smithi022.stdout: "allocated": { 2024-09-21T09:31:07.026 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1290240 2024-09-21T09:31:07.026 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.026 INFO:tasks.workunit.client.0.smithi022.stdout: "clist_lat": { 2024-09-21T09:31:07.026 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 3, 2024-09-21T09:31:07.026 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 137214 2024-09-21T09:31:07.026 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.026 INFO:tasks.workunit.client.0.smithi022.stdout: "compress_lat": { 2024-09-21T09:31:07.026 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:07.026 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.026 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.026 INFO:tasks.workunit.client.0.smithi022.stdout: "compressed": { 2024-09-21T09:31:07.026 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.026 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.027 INFO:tasks.workunit.client.0.smithi022.stdout: "compressed_allocated": { 2024-09-21T09:31:07.027 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.027 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.027 INFO:tasks.workunit.client.0.smithi022.stdout: "compressed_original": { 2024-09-21T09:31:07.027 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.027 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.027 INFO:tasks.workunit.client.0.smithi022.stdout: "csum_lat": { 2024-09-21T09:31:07.027 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 6, 2024-09-21T09:31:07.027 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 18916 2024-09-21T09:31:07.027 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.027 INFO:tasks.workunit.client.0.smithi022.stdout: "decompress_lat": { 2024-09-21T09:31:07.027 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:07.027 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.027 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.027 INFO:tasks.workunit.client.0.smithi022.stdout: "kv_commit_lat": { 2024-09-21T09:31:07.028 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 162, 2024-09-21T09:31:07.028 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 44052102 2024-09-21T09:31:07.028 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.028 INFO:tasks.workunit.client.0.smithi022.stdout: "kv_final_lat": { 2024-09-21T09:31:07.028 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 148, 2024-09-21T09:31:07.028 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 9993526 2024-09-21T09:31:07.028 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.028 INFO:tasks.workunit.client.0.smithi022.stdout: "kv_flush_lat": { 2024-09-21T09:31:07.028 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 162, 2024-09-21T09:31:07.028 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 138154 2024-09-21T09:31:07.028 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.028 INFO:tasks.workunit.client.0.smithi022.stdout: "kv_sync_lat": { 2024-09-21T09:31:07.028 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 162, 2024-09-21T09:31:07.028 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 44190256 2024-09-21T09:31:07.029 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.029 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_get_keys_lat": { 2024-09-21T09:31:07.029 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:07.029 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.029 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.029 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_get_values_lat": { 2024-09-21T09:31:07.029 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 1, 2024-09-21T09:31:07.029 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 35500 2024-09-21T09:31:07.029 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.029 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_lower_bound_lat": { 2024-09-21T09:31:07.029 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 2, 2024-09-21T09:31:07.029 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1026 2024-09-21T09:31:07.029 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.030 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_next_lat": { 2024-09-21T09:31:07.030 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:07.030 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.030 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.030 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_seek_to_first_lat": { 2024-09-21T09:31:07.030 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:07.030 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.030 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.030 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_upper_bound_lat": { 2024-09-21T09:31:07.030 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 2, 2024-09-21T09:31:07.030 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 4201 2024-09-21T09:31:07.030 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.030 INFO:tasks.workunit.client.0.smithi022.stdout: "onode_hits": { 2024-09-21T09:31:07.030 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 360 2024-09-21T09:31:07.030 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.031 INFO:tasks.workunit.client.0.smithi022.stdout: "onode_misses": { 2024-09-21T09:31:07.031 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 257 2024-09-21T09:31:07.031 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.031 INFO:tasks.workunit.client.0.smithi022.stdout: "read_lat": { 2024-09-21T09:31:07.031 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 90, 2024-09-21T09:31:07.031 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1583211 2024-09-21T09:31:07.031 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.031 INFO:tasks.workunit.client.0.smithi022.stdout: "read_onode_meta_lat": { 2024-09-21T09:31:07.031 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 134, 2024-09-21T09:31:07.031 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 239391 2024-09-21T09:31:07.031 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.031 INFO:tasks.workunit.client.0.smithi022.stdout: "read_wait_aio_lat": { 2024-09-21T09:31:07.031 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 44, 2024-09-21T09:31:07.031 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1027252 2024-09-21T09:31:07.031 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.032 INFO:tasks.workunit.client.0.smithi022.stdout: "reads_with_retries": { 2024-09-21T09:31:07.032 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.032 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.032 INFO:tasks.workunit.client.0.smithi022.stdout: "remove_lat": { 2024-09-21T09:31:07.032 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 1, 2024-09-21T09:31:07.032 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 15458 2024-09-21T09:31:07.032 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.032 INFO:tasks.workunit.client.0.smithi022.stdout: "state_aio_wait_lat": { 2024-09-21T09:31:07.032 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 273, 2024-09-21T09:31:07.032 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2128426 2024-09-21T09:31:07.032 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.032 INFO:tasks.workunit.client.0.smithi022.stdout: "state_deferred_aio_wait_lat": { 2024-09-21T09:31:07.032 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 24, 2024-09-21T09:31:07.032 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1767705 2024-09-21T09:31:07.032 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.032 INFO:tasks.workunit.client.0.smithi022.stdout: "state_deferred_cleanup_lat": { 2024-09-21T09:31:07.033 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 23, 2024-09-21T09:31:07.033 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 153476156123 2024-09-21T09:31:07.033 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.033 INFO:tasks.workunit.client.0.smithi022.stdout: "state_deferred_queued_lat": { 2024-09-21T09:31:07.033 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 24, 2024-09-21T09:31:07.033 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 43631242718 2024-09-21T09:31:07.033 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.034 INFO:tasks.workunit.client.0.smithi022.stdout: "state_done_lat": { 2024-09-21T09:31:07.034 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 272, 2024-09-21T09:31:07.034 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 424514 2024-09-21T09:31:07.034 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.034 INFO:tasks.workunit.client.0.smithi022.stdout: "state_finishing_lat": { 2024-09-21T09:31:07.034 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 272, 2024-09-21T09:31:07.034 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 32163 2024-09-21T09:31:07.034 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.034 INFO:tasks.workunit.client.0.smithi022.stdout: "state_io_done_lat": { 2024-09-21T09:31:07.034 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 273, 2024-09-21T09:31:07.034 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 200068 2024-09-21T09:31:07.034 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.034 INFO:tasks.workunit.client.0.smithi022.stdout: "state_kv_commiting_lat": { 2024-09-21T09:31:07.034 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 273, 2024-09-21T09:31:07.035 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 172372448 2024-09-21T09:31:07.035 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.035 INFO:tasks.workunit.client.0.smithi022.stdout: "state_kv_done_lat": { 2024-09-21T09:31:07.035 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 273, 2024-09-21T09:31:07.035 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 81417 2024-09-21T09:31:07.035 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.035 INFO:tasks.workunit.client.0.smithi022.stdout: "state_kv_queued_lat": { 2024-09-21T09:31:07.035 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 273, 2024-09-21T09:31:07.035 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 105292381 2024-09-21T09:31:07.035 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.035 INFO:tasks.workunit.client.0.smithi022.stdout: "state_prepare_lat": { 2024-09-21T09:31:07.035 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 273, 2024-09-21T09:31:07.035 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 12978219 2024-09-21T09:31:07.035 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.035 INFO:tasks.workunit.client.0.smithi022.stdout: "stored": { 2024-09-21T09:31:07.036 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 900939 2024-09-21T09:31:07.036 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.036 INFO:tasks.workunit.client.0.smithi022.stdout: "truncate_lat": { 2024-09-21T09:31:07.036 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 23, 2024-09-21T09:31:07.036 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 110508 2024-09-21T09:31:07.036 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.036 INFO:tasks.workunit.client.0.smithi022.stdout: "txc_commit_lat": { 2024-09-21T09:31:07.036 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 273, 2024-09-21T09:31:07.036 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 293007041 2024-09-21T09:31:07.036 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.036 INFO:tasks.workunit.client.0.smithi022.stdout: "txc_submit_lat": { 2024-09-21T09:31:07.036 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 273, 2024-09-21T09:31:07.036 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 15401487 2024-09-21T09:31:07.037 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.037 INFO:tasks.workunit.client.0.smithi022.stdout: "txc_throttle_lat": { 2024-09-21T09:31:07.037 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 273, 2024-09-21T09:31:07.037 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2362761 2024-09-21T09:31:07.037 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:07.037 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.037 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore-pricache": { 2024-09-21T09:31:07.037 INFO:tasks.workunit.client.0.smithi022.stdout: "cache_bytes": { 2024-09-21T09:31:07.037 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1861463884 2024-09-21T09:31:07.037 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.037 INFO:tasks.workunit.client.0.smithi022.stdout: "heap_bytes": { 2024-09-21T09:31:07.037 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 270974976 2024-09-21T09:31:07.037 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.037 INFO:tasks.workunit.client.0.smithi022.stdout: "mapped_bytes": { 2024-09-21T09:31:07.037 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 270049280 2024-09-21T09:31:07.038 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.038 INFO:tasks.workunit.client.0.smithi022.stdout: "target_bytes": { 2024-09-21T09:31:07.038 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 3137376768 2024-09-21T09:31:07.038 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.038 INFO:tasks.workunit.client.0.smithi022.stdout: "unmapped_bytes": { 2024-09-21T09:31:07.038 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 925696 2024-09-21T09:31:07.038 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:07.038 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.038 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore-pricache:data": { 2024-09-21T09:31:07.038 INFO:tasks.workunit.client.0.smithi022.stdout: "committed_bytes": { 2024-09-21T09:31:07.038 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 159383552 2024-09-21T09:31:07.038 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.038 INFO:tasks.workunit.client.0.smithi022.stdout: "pri0_bytes": { 2024-09-21T09:31:07.039 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.039 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.039 INFO:tasks.workunit.client.0.smithi022.stdout: "pri10_bytes": { 2024-09-21T09:31:07.039 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.039 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.039 INFO:tasks.workunit.client.0.smithi022.stdout: "pri11_bytes": { 2024-09-21T09:31:07.039 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 85490200 2024-09-21T09:31:07.039 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.039 INFO:tasks.workunit.client.0.smithi022.stdout: "pri1_bytes": { 2024-09-21T09:31:07.039 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.039 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.039 INFO:tasks.workunit.client.0.smithi022.stdout: "pri2_bytes": { 2024-09-21T09:31:07.039 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.039 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.040 INFO:tasks.workunit.client.0.smithi022.stdout: "pri3_bytes": { 2024-09-21T09:31:07.040 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.040 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.040 INFO:tasks.workunit.client.0.smithi022.stdout: "pri4_bytes": { 2024-09-21T09:31:07.040 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.040 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.040 INFO:tasks.workunit.client.0.smithi022.stdout: "pri5_bytes": { 2024-09-21T09:31:07.040 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.040 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.040 INFO:tasks.workunit.client.0.smithi022.stdout: "pri6_bytes": { 2024-09-21T09:31:07.040 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 139264 2024-09-21T09:31:07.040 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.041 INFO:tasks.workunit.client.0.smithi022.stdout: "pri7_bytes": { 2024-09-21T09:31:07.041 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.041 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.041 INFO:tasks.workunit.client.0.smithi022.stdout: "pri8_bytes": { 2024-09-21T09:31:07.041 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.041 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.041 INFO:tasks.workunit.client.0.smithi022.stdout: "pri9_bytes": { 2024-09-21T09:31:07.041 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.041 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.041 INFO:tasks.workunit.client.0.smithi022.stdout: "reserved_bytes": { 2024-09-21T09:31:07.041 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 73754088 2024-09-21T09:31:07.041 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:07.041 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.041 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore-pricache:kv": { 2024-09-21T09:31:07.042 INFO:tasks.workunit.client.0.smithi022.stdout: "committed_bytes": { 2024-09-21T09:31:07.042 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 780140544 2024-09-21T09:31:07.042 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.042 INFO:tasks.workunit.client.0.smithi022.stdout: "pri0_bytes": { 2024-09-21T09:31:07.042 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 67108864 2024-09-21T09:31:07.042 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.042 INFO:tasks.workunit.client.0.smithi022.stdout: "pri10_bytes": { 2024-09-21T09:31:07.042 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.042 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.042 INFO:tasks.workunit.client.0.smithi022.stdout: "pri11_bytes": { 2024-09-21T09:31:07.042 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 641176501 2024-09-21T09:31:07.042 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.042 INFO:tasks.workunit.client.0.smithi022.stdout: "pri1_bytes": { 2024-09-21T09:31:07.043 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.043 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.043 INFO:tasks.workunit.client.0.smithi022.stdout: "pri2_bytes": { 2024-09-21T09:31:07.043 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.043 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.043 INFO:tasks.workunit.client.0.smithi022.stdout: "pri3_bytes": { 2024-09-21T09:31:07.043 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.043 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.043 INFO:tasks.workunit.client.0.smithi022.stdout: "pri4_bytes": { 2024-09-21T09:31:07.043 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.043 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.043 INFO:tasks.workunit.client.0.smithi022.stdout: "pri5_bytes": { 2024-09-21T09:31:07.043 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.044 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.044 INFO:tasks.workunit.client.0.smithi022.stdout: "pri6_bytes": { 2024-09-21T09:31:07.044 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2128 2024-09-21T09:31:07.044 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.044 INFO:tasks.workunit.client.0.smithi022.stdout: "pri7_bytes": { 2024-09-21T09:31:07.044 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.044 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.044 INFO:tasks.workunit.client.0.smithi022.stdout: "pri8_bytes": { 2024-09-21T09:31:07.044 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.044 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.044 INFO:tasks.workunit.client.0.smithi022.stdout: "pri9_bytes": { 2024-09-21T09:31:07.044 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.044 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.045 INFO:tasks.workunit.client.0.smithi022.stdout: "reserved_bytes": { 2024-09-21T09:31:07.045 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 71853051 2024-09-21T09:31:07.045 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:07.045 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.045 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore-pricache:kv_onode": { 2024-09-21T09:31:07.045 INFO:tasks.workunit.client.0.smithi022.stdout: "committed_bytes": { 2024-09-21T09:31:07.045 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 192937984 2024-09-21T09:31:07.045 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.045 INFO:tasks.workunit.client.0.smithi022.stdout: "pri0_bytes": { 2024-09-21T09:31:07.045 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 67108864 2024-09-21T09:31:07.045 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.045 INFO:tasks.workunit.client.0.smithi022.stdout: "pri10_bytes": { 2024-09-21T09:31:07.045 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 464 2024-09-21T09:31:07.046 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.046 INFO:tasks.workunit.client.0.smithi022.stdout: "pri11_bytes": { 2024-09-21T09:31:07.046 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 56993466 2024-09-21T09:31:07.046 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.046 INFO:tasks.workunit.client.0.smithi022.stdout: "pri1_bytes": { 2024-09-21T09:31:07.046 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.046 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.046 INFO:tasks.workunit.client.0.smithi022.stdout: "pri2_bytes": { 2024-09-21T09:31:07.046 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.046 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.046 INFO:tasks.workunit.client.0.smithi022.stdout: "pri3_bytes": { 2024-09-21T09:31:07.046 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.046 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.046 INFO:tasks.workunit.client.0.smithi022.stdout: "pri4_bytes": { 2024-09-21T09:31:07.047 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.047 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.047 INFO:tasks.workunit.client.0.smithi022.stdout: "pri5_bytes": { 2024-09-21T09:31:07.047 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.047 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.047 INFO:tasks.workunit.client.0.smithi022.stdout: "pri6_bytes": { 2024-09-21T09:31:07.047 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.047 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.047 INFO:tasks.workunit.client.0.smithi022.stdout: "pri7_bytes": { 2024-09-21T09:31:07.047 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.047 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.047 INFO:tasks.workunit.client.0.smithi022.stdout: "pri8_bytes": { 2024-09-21T09:31:07.047 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.047 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.048 INFO:tasks.workunit.client.0.smithi022.stdout: "pri9_bytes": { 2024-09-21T09:31:07.048 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.048 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.048 INFO:tasks.workunit.client.0.smithi022.stdout: "reserved_bytes": { 2024-09-21T09:31:07.048 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 68835190 2024-09-21T09:31:07.048 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:07.048 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.048 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore-pricache:meta": { 2024-09-21T09:31:07.048 INFO:tasks.workunit.client.0.smithi022.stdout: "committed_bytes": { 2024-09-21T09:31:07.048 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 713031680 2024-09-21T09:31:07.048 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.048 INFO:tasks.workunit.client.0.smithi022.stdout: "pri0_bytes": { 2024-09-21T09:31:07.048 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.049 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.049 INFO:tasks.workunit.client.0.smithi022.stdout: "pri10_bytes": { 2024-09-21T09:31:07.049 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.049 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.049 INFO:tasks.workunit.client.0.smithi022.stdout: "pri11_bytes": { 2024-09-21T09:31:07.049 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 641176501 2024-09-21T09:31:07.049 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.049 INFO:tasks.workunit.client.0.smithi022.stdout: "pri1_bytes": { 2024-09-21T09:31:07.049 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.049 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.049 INFO:tasks.workunit.client.0.smithi022.stdout: "pri2_bytes": { 2024-09-21T09:31:07.049 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.049 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.049 INFO:tasks.workunit.client.0.smithi022.stdout: "pri3_bytes": { 2024-09-21T09:31:07.049 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.050 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.050 INFO:tasks.workunit.client.0.smithi022.stdout: "pri4_bytes": { 2024-09-21T09:31:07.050 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.050 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.050 INFO:tasks.workunit.client.0.smithi022.stdout: "pri5_bytes": { 2024-09-21T09:31:07.050 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.050 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.050 INFO:tasks.workunit.client.0.smithi022.stdout: "pri6_bytes": { 2024-09-21T09:31:07.050 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 272044 2024-09-21T09:31:07.050 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.050 INFO:tasks.workunit.client.0.smithi022.stdout: "pri7_bytes": { 2024-09-21T09:31:07.050 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.050 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.050 INFO:tasks.workunit.client.0.smithi022.stdout: "pri8_bytes": { 2024-09-21T09:31:07.050 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.051 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.051 INFO:tasks.workunit.client.0.smithi022.stdout: "pri9_bytes": { 2024-09-21T09:31:07.051 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.051 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.051 INFO:tasks.workunit.client.0.smithi022.stdout: "reserved_bytes": { 2024-09-21T09:31:07.051 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 71583135 2024-09-21T09:31:07.051 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:07.051 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.051 INFO:tasks.workunit.client.0.smithi022.stdout: "objecter": { 2024-09-21T09:31:07.051 INFO:tasks.workunit.client.0.smithi022.stdout: "op_active": { 2024-09-21T09:31:07.051 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.051 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.051 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r": { 2024-09-21T09:31:07.051 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.051 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.052 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rmw": { 2024-09-21T09:31:07.052 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.052 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.052 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w": { 2024-09-21T09:31:07.052 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.052 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:07.052 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.052 INFO:tasks.workunit.client.0.smithi022.stdout: "osd": { 2024-09-21T09:31:07.052 INFO:tasks.workunit.client.0.smithi022.stdout: "numpg": { 2024-09-21T09:31:07.052 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 46 2024-09-21T09:31:07.052 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.052 INFO:tasks.workunit.client.0.smithi022.stdout: "numpg_removing": { 2024-09-21T09:31:07.052 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.052 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.052 INFO:tasks.workunit.client.0.smithi022.stdout: "op": { 2024-09-21T09:31:07.052 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 29 2024-09-21T09:31:07.053 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.053 INFO:tasks.workunit.client.0.smithi022.stdout: "op_delayed_degraded": { 2024-09-21T09:31:07.053 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.053 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.053 INFO:tasks.workunit.client.0.smithi022.stdout: "op_delayed_unreadable": { 2024-09-21T09:31:07.053 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.053 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.053 INFO:tasks.workunit.client.0.smithi022.stdout: "op_in_bytes": { 2024-09-21T09:31:07.053 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.053 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.053 INFO:tasks.workunit.client.0.smithi022.stdout: "op_latency": { 2024-09-21T09:31:07.053 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 29, 2024-09-21T09:31:07.053 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 47302505 2024-09-21T09:31:07.053 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.053 INFO:tasks.workunit.client.0.smithi022.stdout: "op_out_bytes": { 2024-09-21T09:31:07.054 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 131762 2024-09-21T09:31:07.054 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.054 INFO:tasks.workunit.client.0.smithi022.stdout: "op_prepare_latency": { 2024-09-21T09:31:07.054 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 31, 2024-09-21T09:31:07.054 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 8253325 2024-09-21T09:31:07.054 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.054 INFO:tasks.workunit.client.0.smithi022.stdout: "op_process_latency": { 2024-09-21T09:31:07.054 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 29, 2024-09-21T09:31:07.054 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 29473970 2024-09-21T09:31:07.054 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.054 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r": { 2024-09-21T09:31:07.054 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 18 2024-09-21T09:31:07.054 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.055 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r_latency": { 2024-09-21T09:31:07.055 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 18, 2024-09-21T09:31:07.055 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 3096894 2024-09-21T09:31:07.055 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.055 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-21T09:31:07.055 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.055 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.055 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r_out_bytes": { 2024-09-21T09:31:07.055 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 131762 2024-09-21T09:31:07.055 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.055 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r_prepare_latency": { 2024-09-21T09:31:07.055 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 18, 2024-09-21T09:31:07.055 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2895370 2024-09-21T09:31:07.055 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.056 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r_process_latency": { 2024-09-21T09:31:07.056 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 18, 2024-09-21T09:31:07.056 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2472706 2024-09-21T09:31:07.056 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.056 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw": { 2024-09-21T09:31:07.056 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 6 2024-09-21T09:31:07.056 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.056 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_in_bytes": { 2024-09-21T09:31:07.056 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.056 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.056 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_latency": { 2024-09-21T09:31:07.056 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 6, 2024-09-21T09:31:07.056 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 5010170 2024-09-21T09:31:07.056 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.056 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-21T09:31:07.057 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.057 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.057 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-21T09:31:07.057 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.057 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.057 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_out_bytes": { 2024-09-21T09:31:07.057 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.057 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.057 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_prepare_latency": { 2024-09-21T09:31:07.057 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 6, 2024-09-21T09:31:07.057 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2293509 2024-09-21T09:31:07.057 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.057 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_process_latency": { 2024-09-21T09:31:07.057 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 6, 2024-09-21T09:31:07.058 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 4778960 2024-09-21T09:31:07.058 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.058 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w": { 2024-09-21T09:31:07.058 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 5 2024-09-21T09:31:07.058 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.058 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w_in_bytes": { 2024-09-21T09:31:07.058 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.058 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.058 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w_latency": { 2024-09-21T09:31:07.058 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 5, 2024-09-21T09:31:07.058 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 39195441 2024-09-21T09:31:07.059 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.059 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-21T09:31:07.059 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.059 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.059 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w_prepare_latency": { 2024-09-21T09:31:07.059 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 7, 2024-09-21T09:31:07.059 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 3064446 2024-09-21T09:31:07.059 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.059 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w_process_latency": { 2024-09-21T09:31:07.059 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 5, 2024-09-21T09:31:07.059 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 22222304 2024-09-21T09:31:07.059 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.059 INFO:tasks.workunit.client.0.smithi022.stdout: "op_wip": { 2024-09-21T09:31:07.059 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.060 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.060 INFO:tasks.workunit.client.0.smithi022.stdout: "recovery_bytes": { 2024-09-21T09:31:07.060 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 590368 2024-09-21T09:31:07.060 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.060 INFO:tasks.workunit.client.0.smithi022.stdout: "recovery_ops": { 2024-09-21T09:31:07.060 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2 2024-09-21T09:31:07.060 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.060 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_bytes": { 2024-09-21T09:31:07.060 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 95995035648 2024-09-21T09:31:07.060 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.060 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_bytes_used": { 2024-09-21T09:31:07.060 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 305782784 2024-09-21T09:31:07.060 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:07.060 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:07.061 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:07.061 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.061 INFO:tasks.workunit.client.0.smithi022.stdout: "pools": [ 2024-09-21T09:31:07.061 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.061 INFO:tasks.workunit.client.0.smithi022.stdout: "application": [ 2024-09-21T09:31:07.061 INFO:tasks.workunit.client.0.smithi022.stdout: "mgr" 2024-09-21T09:31:07.061 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.061 INFO:tasks.workunit.client.0.smithi022.stdout: "cache_mode": "none", 2024-09-21T09:31:07.061 INFO:tasks.workunit.client.0.smithi022.stdout: "erasure_code_profile": {}, 2024-09-21T09:31:07.061 INFO:tasks.workunit.client.0.smithi022.stdout: "min_size": 2, 2024-09-21T09:31:07.061 INFO:tasks.workunit.client.0.smithi022.stdout: "options": {}, 2024-09-21T09:31:07.061 INFO:tasks.workunit.client.0.smithi022.stdout: "pg_autoscale_mode": "off", 2024-09-21T09:31:07.061 INFO:tasks.workunit.client.0.smithi022.stdout: "pg_num": 1, 2024-09-21T09:31:07.061 INFO:tasks.workunit.client.0.smithi022.stdout: "pgp_num": 1, 2024-09-21T09:31:07.062 INFO:tasks.workunit.client.0.smithi022.stdout: "pool": 1, 2024-09-21T09:31:07.062 INFO:tasks.workunit.client.0.smithi022.stdout: "size": 3, 2024-09-21T09:31:07.062 INFO:tasks.workunit.client.0.smithi022.stdout: "stats": { 2024-09-21T09:31:07.062 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_raw": 727115966730, 2024-09-21T09:31:07.062 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes_used": 1781760, 2024-09-21T09:31:07.062 INFO:tasks.workunit.client.0.smithi022.stdout: "compress_bytes_used": 0, 2024-09-21T09:31:07.062 INFO:tasks.workunit.client.0.smithi022.stdout: "compress_under_bytes": 0, 2024-09-21T09:31:07.062 INFO:tasks.workunit.client.0.smithi022.stdout: "data_bytes_used": 1781760, 2024-09-21T09:31:07.062 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty": 0, 2024-09-21T09:31:07.062 INFO:tasks.workunit.client.0.smithi022.stdout: "max_avail": 242371985408, 2024-09-21T09:31:07.062 INFO:tasks.workunit.client.0.smithi022.stdout: "objects": 2, 2024-09-21T09:31:07.062 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_bytes_used": 0, 2024-09-21T09:31:07.062 INFO:tasks.workunit.client.0.smithi022.stdout: "percent_used": 2.4504422526661074e-06, 2024-09-21T09:31:07.062 INFO:tasks.workunit.client.0.smithi022.stdout: "quota_bytes": 0, 2024-09-21T09:31:07.063 INFO:tasks.workunit.client.0.smithi022.stdout: "quota_objects": 0, 2024-09-21T09:31:07.063 INFO:tasks.workunit.client.0.smithi022.stdout: "rd": 126, 2024-09-21T09:31:07.063 INFO:tasks.workunit.client.0.smithi022.stdout: "rd_bytes": 236544, 2024-09-21T09:31:07.063 INFO:tasks.workunit.client.0.smithi022.stdout: "stored": 590368, 2024-09-21T09:31:07.063 INFO:tasks.workunit.client.0.smithi022.stdout: "stored_data": 590368, 2024-09-21T09:31:07.063 INFO:tasks.workunit.client.0.smithi022.stdout: "stored_omap": 0, 2024-09-21T09:31:07.063 INFO:tasks.workunit.client.0.smithi022.stdout: "stored_raw": 1771104, 2024-09-21T09:31:07.063 INFO:tasks.workunit.client.0.smithi022.stdout: "wr": 113, 2024-09-21T09:31:07.063 INFO:tasks.workunit.client.0.smithi022.stdout: "wr_bytes": 1667072 2024-09-21T09:31:07.063 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.063 INFO:tasks.workunit.client.0.smithi022.stdout: "target_max_bytes": 0, 2024-09-21T09:31:07.063 INFO:tasks.workunit.client.0.smithi022.stdout: "target_max_objects": 0, 2024-09-21T09:31:07.063 INFO:tasks.workunit.client.0.smithi022.stdout: "type": "replicated" 2024-09-21T09:31:07.063 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.063 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.064 INFO:tasks.workunit.client.0.smithi022.stdout: "application": [ 2024-09-21T09:31:07.064 INFO:tasks.workunit.client.0.smithi022.stdout: "cephfs" 2024-09-21T09:31:07.064 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.064 INFO:tasks.workunit.client.0.smithi022.stdout: "cache_mode": "none", 2024-09-21T09:31:07.064 INFO:tasks.workunit.client.0.smithi022.stdout: "erasure_code_profile": {}, 2024-09-21T09:31:07.064 INFO:tasks.workunit.client.0.smithi022.stdout: "min_size": 2, 2024-09-21T09:31:07.064 INFO:tasks.workunit.client.0.smithi022.stdout: "options": {}, 2024-09-21T09:31:07.064 INFO:tasks.workunit.client.0.smithi022.stdout: "pg_autoscale_mode": "off", 2024-09-21T09:31:07.064 INFO:tasks.workunit.client.0.smithi022.stdout: "pg_num": 32, 2024-09-21T09:31:07.064 INFO:tasks.workunit.client.0.smithi022.stdout: "pgp_num": 32, 2024-09-21T09:31:07.064 INFO:tasks.workunit.client.0.smithi022.stdout: "pool": 2, 2024-09-21T09:31:07.064 INFO:tasks.workunit.client.0.smithi022.stdout: "size": 3, 2024-09-21T09:31:07.064 INFO:tasks.workunit.client.0.smithi022.stdout: "stats": { 2024-09-21T09:31:07.064 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_raw": 727115966730, 2024-09-21T09:31:07.064 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes_used": 98304, 2024-09-21T09:31:07.065 INFO:tasks.workunit.client.0.smithi022.stdout: "compress_bytes_used": 0, 2024-09-21T09:31:07.065 INFO:tasks.workunit.client.0.smithi022.stdout: "compress_under_bytes": 0, 2024-09-21T09:31:07.065 INFO:tasks.workunit.client.0.smithi022.stdout: "data_bytes_used": 98304, 2024-09-21T09:31:07.065 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty": 0, 2024-09-21T09:31:07.065 INFO:tasks.workunit.client.0.smithi022.stdout: "max_avail": 242371985408, 2024-09-21T09:31:07.065 INFO:tasks.workunit.client.0.smithi022.stdout: "objects": 22, 2024-09-21T09:31:07.065 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_bytes_used": 0, 2024-09-21T09:31:07.065 INFO:tasks.workunit.client.0.smithi022.stdout: "percent_used": 1.3519712638299097e-07, 2024-09-21T09:31:07.065 INFO:tasks.workunit.client.0.smithi022.stdout: "quota_bytes": 0, 2024-09-21T09:31:07.065 INFO:tasks.workunit.client.0.smithi022.stdout: "quota_objects": 0, 2024-09-21T09:31:07.065 INFO:tasks.workunit.client.0.smithi022.stdout: "rd": 0, 2024-09-21T09:31:07.065 INFO:tasks.workunit.client.0.smithi022.stdout: "rd_bytes": 0, 2024-09-21T09:31:07.065 INFO:tasks.workunit.client.0.smithi022.stdout: "stored": 3410, 2024-09-21T09:31:07.065 INFO:tasks.workunit.client.0.smithi022.stdout: "stored_data": 3410, 2024-09-21T09:31:07.066 INFO:tasks.workunit.client.0.smithi022.stdout: "stored_omap": 0, 2024-09-21T09:31:07.066 INFO:tasks.workunit.client.0.smithi022.stdout: "stored_raw": 10230, 2024-09-21T09:31:07.066 INFO:tasks.workunit.client.0.smithi022.stdout: "wr": 51, 2024-09-21T09:31:07.066 INFO:tasks.workunit.client.0.smithi022.stdout: "wr_bytes": 19456 2024-09-21T09:31:07.066 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.066 INFO:tasks.workunit.client.0.smithi022.stdout: "target_max_bytes": 0, 2024-09-21T09:31:07.066 INFO:tasks.workunit.client.0.smithi022.stdout: "target_max_objects": 0, 2024-09-21T09:31:07.066 INFO:tasks.workunit.client.0.smithi022.stdout: "type": "replicated" 2024-09-21T09:31:07.066 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.066 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.066 INFO:tasks.workunit.client.0.smithi022.stdout: "application": [ 2024-09-21T09:31:07.066 INFO:tasks.workunit.client.0.smithi022.stdout: "cephfs" 2024-09-21T09:31:07.066 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.066 INFO:tasks.workunit.client.0.smithi022.stdout: "cache_mode": "none", 2024-09-21T09:31:07.066 INFO:tasks.workunit.client.0.smithi022.stdout: "erasure_code_profile": {}, 2024-09-21T09:31:07.067 INFO:tasks.workunit.client.0.smithi022.stdout: "min_size": 2, 2024-09-21T09:31:07.067 INFO:tasks.workunit.client.0.smithi022.stdout: "options": {}, 2024-09-21T09:31:07.067 INFO:tasks.workunit.client.0.smithi022.stdout: "pg_autoscale_mode": "off", 2024-09-21T09:31:07.067 INFO:tasks.workunit.client.0.smithi022.stdout: "pg_num": 64, 2024-09-21T09:31:07.067 INFO:tasks.workunit.client.0.smithi022.stdout: "pgp_num": 64, 2024-09-21T09:31:07.067 INFO:tasks.workunit.client.0.smithi022.stdout: "pool": 3, 2024-09-21T09:31:07.067 INFO:tasks.workunit.client.0.smithi022.stdout: "size": 3, 2024-09-21T09:31:07.067 INFO:tasks.workunit.client.0.smithi022.stdout: "stats": { 2024-09-21T09:31:07.067 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_raw": 727115966730, 2024-09-21T09:31:07.067 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes_used": 0, 2024-09-21T09:31:07.067 INFO:tasks.workunit.client.0.smithi022.stdout: "compress_bytes_used": 0, 2024-09-21T09:31:07.067 INFO:tasks.workunit.client.0.smithi022.stdout: "compress_under_bytes": 0, 2024-09-21T09:31:07.067 INFO:tasks.workunit.client.0.smithi022.stdout: "data_bytes_used": 0, 2024-09-21T09:31:07.067 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty": 0, 2024-09-21T09:31:07.068 INFO:tasks.workunit.client.0.smithi022.stdout: "max_avail": 242371985408, 2024-09-21T09:31:07.068 INFO:tasks.workunit.client.0.smithi022.stdout: "objects": 0, 2024-09-21T09:31:07.068 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_bytes_used": 0, 2024-09-21T09:31:07.068 INFO:tasks.workunit.client.0.smithi022.stdout: "percent_used": 0.0, 2024-09-21T09:31:07.068 INFO:tasks.workunit.client.0.smithi022.stdout: "quota_bytes": 0, 2024-09-21T09:31:07.068 INFO:tasks.workunit.client.0.smithi022.stdout: "quota_objects": 0, 2024-09-21T09:31:07.068 INFO:tasks.workunit.client.0.smithi022.stdout: "rd": 0, 2024-09-21T09:31:07.068 INFO:tasks.workunit.client.0.smithi022.stdout: "rd_bytes": 0, 2024-09-21T09:31:07.068 INFO:tasks.workunit.client.0.smithi022.stdout: "stored": 0, 2024-09-21T09:31:07.068 INFO:tasks.workunit.client.0.smithi022.stdout: "stored_data": 0, 2024-09-21T09:31:07.068 INFO:tasks.workunit.client.0.smithi022.stdout: "stored_omap": 0, 2024-09-21T09:31:07.068 INFO:tasks.workunit.client.0.smithi022.stdout: "stored_raw": 0, 2024-09-21T09:31:07.068 INFO:tasks.workunit.client.0.smithi022.stdout: "wr": 0, 2024-09-21T09:31:07.068 INFO:tasks.workunit.client.0.smithi022.stdout: "wr_bytes": 0 2024-09-21T09:31:07.068 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.069 INFO:tasks.workunit.client.0.smithi022.stdout: "target_max_bytes": 0, 2024-09-21T09:31:07.069 INFO:tasks.workunit.client.0.smithi022.stdout: "target_max_objects": 0, 2024-09-21T09:31:07.069 INFO:tasks.workunit.client.0.smithi022.stdout: "type": "replicated" 2024-09-21T09:31:07.069 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.069 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.069 INFO:tasks.workunit.client.0.smithi022.stdout: "application": [ 2024-09-21T09:31:07.069 INFO:tasks.workunit.client.0.smithi022.stdout: "rbd" 2024-09-21T09:31:07.069 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.069 INFO:tasks.workunit.client.0.smithi022.stdout: "cache_mode": "none", 2024-09-21T09:31:07.069 INFO:tasks.workunit.client.0.smithi022.stdout: "erasure_code_profile": {}, 2024-09-21T09:31:07.069 INFO:tasks.workunit.client.0.smithi022.stdout: "min_size": 2, 2024-09-21T09:31:07.069 INFO:tasks.workunit.client.0.smithi022.stdout: "options": {}, 2024-09-21T09:31:07.069 INFO:tasks.workunit.client.0.smithi022.stdout: "pg_autoscale_mode": "off", 2024-09-21T09:31:07.069 INFO:tasks.workunit.client.0.smithi022.stdout: "pg_num": 8, 2024-09-21T09:31:07.069 INFO:tasks.workunit.client.0.smithi022.stdout: "pgp_num": 8, 2024-09-21T09:31:07.070 INFO:tasks.workunit.client.0.smithi022.stdout: "pool": 4, 2024-09-21T09:31:07.070 INFO:tasks.workunit.client.0.smithi022.stdout: "size": 3, 2024-09-21T09:31:07.070 INFO:tasks.workunit.client.0.smithi022.stdout: "stats": { 2024-09-21T09:31:07.070 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_raw": 727115966730, 2024-09-21T09:31:07.070 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes_used": 12288, 2024-09-21T09:31:07.070 INFO:tasks.workunit.client.0.smithi022.stdout: "compress_bytes_used": 0, 2024-09-21T09:31:07.070 INFO:tasks.workunit.client.0.smithi022.stdout: "compress_under_bytes": 0, 2024-09-21T09:31:07.070 INFO:tasks.workunit.client.0.smithi022.stdout: "data_bytes_used": 12288, 2024-09-21T09:31:07.070 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty": 0, 2024-09-21T09:31:07.070 INFO:tasks.workunit.client.0.smithi022.stdout: "max_avail": 242371985408, 2024-09-21T09:31:07.070 INFO:tasks.workunit.client.0.smithi022.stdout: "objects": 2, 2024-09-21T09:31:07.070 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_bytes_used": 0, 2024-09-21T09:31:07.070 INFO:tasks.workunit.client.0.smithi022.stdout: "percent_used": 1.689964257423071e-08, 2024-09-21T09:31:07.070 INFO:tasks.workunit.client.0.smithi022.stdout: "quota_bytes": 0, 2024-09-21T09:31:07.071 INFO:tasks.workunit.client.0.smithi022.stdout: "quota_objects": 0, 2024-09-21T09:31:07.071 INFO:tasks.workunit.client.0.smithi022.stdout: "rd": 0, 2024-09-21T09:31:07.071 INFO:tasks.workunit.client.0.smithi022.stdout: "rd_bytes": 0, 2024-09-21T09:31:07.071 INFO:tasks.workunit.client.0.smithi022.stdout: "stored": 19, 2024-09-21T09:31:07.071 INFO:tasks.workunit.client.0.smithi022.stdout: "stored_data": 19, 2024-09-21T09:31:07.071 INFO:tasks.workunit.client.0.smithi022.stdout: "stored_omap": 0, 2024-09-21T09:31:07.071 INFO:tasks.workunit.client.0.smithi022.stdout: "stored_raw": 57, 2024-09-21T09:31:07.071 INFO:tasks.workunit.client.0.smithi022.stdout: "wr": 2, 2024-09-21T09:31:07.071 INFO:tasks.workunit.client.0.smithi022.stdout: "wr_bytes": 2048 2024-09-21T09:31:07.071 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.071 INFO:tasks.workunit.client.0.smithi022.stdout: "target_max_bytes": 0, 2024-09-21T09:31:07.071 INFO:tasks.workunit.client.0.smithi022.stdout: "target_max_objects": 0, 2024-09-21T09:31:07.071 INFO:tasks.workunit.client.0.smithi022.stdout: "type": "replicated" 2024-09-21T09:31:07.071 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:07.071 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.072 INFO:tasks.workunit.client.0.smithi022.stdout: "rbd": { 2024-09-21T09:31:07.072 INFO:tasks.workunit.client.0.smithi022.stdout: "mirroring_by_pool": [ 2024-09-21T09:31:07.072 INFO:tasks.workunit.client.0.smithi022.stdout: false 2024-09-21T09:31:07.072 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.072 INFO:tasks.workunit.client.0.smithi022.stdout: "num_images_by_pool": [ 2024-09-21T09:31:07.072 INFO:tasks.workunit.client.0.smithi022.stdout: 0 2024-09-21T09:31:07.072 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.072 INFO:tasks.workunit.client.0.smithi022.stdout: "num_pools": 1 2024-09-21T09:31:07.072 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.072 INFO:tasks.workunit.client.0.smithi022.stdout: "report_id": "77addb0d-e829-4757-884e-46deb5965591", 2024-09-21T09:31:07.072 INFO:tasks.workunit.client.0.smithi022.stdout: "report_timestamp": "2024-09-21T09:31:06.195062", 2024-09-21T09:31:07.072 INFO:tasks.workunit.client.0.smithi022.stdout: "report_version": 1, 2024-09-21T09:31:07.072 INFO:tasks.workunit.client.0.smithi022.stdout: "rocksdb_stats": { 2024-09-21T09:31:07.072 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "6.15.5" 2024-09-21T09:31:07.072 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.073 INFO:tasks.workunit.client.0.smithi022.stdout: "rook": { 2024-09-21T09:31:07.073 INFO:tasks.workunit.client.0.smithi022.stdout: "cluster": { 2024-09-21T09:31:07.073 INFO:tasks.workunit.client.0.smithi022.stdout: "external-mode": null, 2024-09-21T09:31:07.073 INFO:tasks.workunit.client.0.smithi022.stdout: "mon": { 2024-09-21T09:31:07.073 INFO:tasks.workunit.client.0.smithi022.stdout: "allow-multiple-per-node": null, 2024-09-21T09:31:07.073 INFO:tasks.workunit.client.0.smithi022.stdout: "count": null, 2024-09-21T09:31:07.073 INFO:tasks.workunit.client.0.smithi022.stdout: "max-id": null, 2024-09-21T09:31:07.073 INFO:tasks.workunit.client.0.smithi022.stdout: "pvc": { 2024-09-21T09:31:07.073 INFO:tasks.workunit.client.0.smithi022.stdout: "enabled": null 2024-09-21T09:31:07.073 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.073 INFO:tasks.workunit.client.0.smithi022.stdout: "stretch": { 2024-09-21T09:31:07.073 INFO:tasks.workunit.client.0.smithi022.stdout: "enabled": null 2024-09-21T09:31:07.073 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:07.073 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.073 INFO:tasks.workunit.client.0.smithi022.stdout: "network": { 2024-09-21T09:31:07.074 INFO:tasks.workunit.client.0.smithi022.stdout: "provider": null 2024-09-21T09:31:07.074 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.074 INFO:tasks.workunit.client.0.smithi022.stdout: "storage": { 2024-09-21T09:31:07.074 INFO:tasks.workunit.client.0.smithi022.stdout: "device-set": { 2024-09-21T09:31:07.074 INFO:tasks.workunit.client.0.smithi022.stdout: "count": { 2024-09-21T09:31:07.074 INFO:tasks.workunit.client.0.smithi022.stdout: "non-portable": null, 2024-09-21T09:31:07.074 INFO:tasks.workunit.client.0.smithi022.stdout: "portable": null, 2024-09-21T09:31:07.074 INFO:tasks.workunit.client.0.smithi022.stdout: "total": null 2024-09-21T09:31:07.074 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:07.074 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:07.074 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:07.074 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.074 INFO:tasks.workunit.client.0.smithi022.stdout: "csi": { 2024-09-21T09:31:07.074 INFO:tasks.workunit.client.0.smithi022.stdout: "version": null 2024-09-21T09:31:07.075 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.075 INFO:tasks.workunit.client.0.smithi022.stdout: "kubernetes": { 2024-09-21T09:31:07.075 INFO:tasks.workunit.client.0.smithi022.stdout: "version": null 2024-09-21T09:31:07.075 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.075 INFO:tasks.workunit.client.0.smithi022.stdout: "node": { 2024-09-21T09:31:07.075 INFO:tasks.workunit.client.0.smithi022.stdout: "count": { 2024-09-21T09:31:07.075 INFO:tasks.workunit.client.0.smithi022.stdout: "kubernetes-total": null, 2024-09-21T09:31:07.075 INFO:tasks.workunit.client.0.smithi022.stdout: "with-ceph-daemons": null, 2024-09-21T09:31:07.075 INFO:tasks.workunit.client.0.smithi022.stdout: "with-csi-cephfs-plugin": null, 2024-09-21T09:31:07.075 INFO:tasks.workunit.client.0.smithi022.stdout: "with-csi-nfs-plugin": null, 2024-09-21T09:31:07.075 INFO:tasks.workunit.client.0.smithi022.stdout: "with-csi-rbd-plugin": null 2024-09-21T09:31:07.075 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:07.075 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.075 INFO:tasks.workunit.client.0.smithi022.stdout: "usage": { 2024-09-21T09:31:07.075 INFO:tasks.workunit.client.0.smithi022.stdout: "storage-class": { 2024-09-21T09:31:07.076 INFO:tasks.workunit.client.0.smithi022.stdout: "count": { 2024-09-21T09:31:07.076 INFO:tasks.workunit.client.0.smithi022.stdout: "bucket": null, 2024-09-21T09:31:07.076 INFO:tasks.workunit.client.0.smithi022.stdout: "cephfs": null, 2024-09-21T09:31:07.076 INFO:tasks.workunit.client.0.smithi022.stdout: "nfs": null, 2024-09-21T09:31:07.076 INFO:tasks.workunit.client.0.smithi022.stdout: "rbd": null, 2024-09-21T09:31:07.076 INFO:tasks.workunit.client.0.smithi022.stdout: "total": null 2024-09-21T09:31:07.076 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:07.076 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:07.076 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.076 INFO:tasks.workunit.client.0.smithi022.stdout: "version": null 2024-09-21T09:31:07.076 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.076 INFO:tasks.workunit.client.0.smithi022.stdout: "services": {}, 2024-09-21T09:31:07.076 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_per_pg": [ 2024-09-21T09:31:07.076 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.077 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:07.077 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:07.077 INFO:tasks.workunit.client.0.smithi022.stdout: 6, 2024-09-21T09:31:07.077 INFO:tasks.workunit.client.0.smithi022.stdout: 0 2024-09-21T09:31:07.077 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.077 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 7, 2024-09-21T09:31:07.077 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:07.077 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:07.077 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:07.077 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:07.077 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:07.077 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:07.077 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.359261+0000", 2024-09-21T09:31:07.077 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.635255+0000", 2024-09-21T09:31:07.077 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.635255+0000", 2024-09-21T09:31:07.078 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.635695+0000", 2024-09-21T09:31:07.078 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.359261+0000", 2024-09-21T09:31:07.078 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.078 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:07.078 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.078 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:07.078 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.359261+0000", 2024-09-21T09:31:07.078 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.359261+0000", 2024-09-21T09:31:07.078 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.359261+0000", 2024-09-21T09:31:07.078 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:07.078 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:07.078 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.079 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.359261+0000", 2024-09-21T09:31:07.079 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.359261+0000", 2024-09-21T09:31:07.079 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:07.079 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:07.079 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:07.079 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:07.079 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:07.079 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:07.079 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:07.079 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:07.079 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:07.079 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:07.079 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:07.080 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:07.080 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:07.080 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.3f", 2024-09-21T09:31:07.080 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:07.080 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:07.080 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:07.080 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:07.080 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:07.080 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T16:15:50.156743+0000", 2024-09-21T09:31:07.080 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:07.080 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:07.080 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.080 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:07.080 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.081 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.081 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.081 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.081 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.081 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.081 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.081 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.081 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.081 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.081 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.081 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.081 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.081 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.081 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.082 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:07.082 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:07.082 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.082 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:07.082 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.082 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.082 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.082 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.082 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.082 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:07.082 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.082 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.082 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.083 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.083 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.083 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.083 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.083 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.083 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.083 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.083 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.083 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.083 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:07.083 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:07.083 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.083 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:07.083 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:07.084 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:07.084 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:07.084 INFO:tasks.workunit.client.0.smithi022.stdout: 6, 2024-09-21T09:31:07.084 INFO:tasks.workunit.client.0.smithi022.stdout: 0 2024-09-21T09:31:07.084 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.084 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 7, 2024-09-21T09:31:07.084 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:07.084 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.084 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.084 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:07.084 INFO:tasks.workunit.client.0.smithi022.stdout: 1, 2024-09-21T09:31:07.084 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:07.084 INFO:tasks.workunit.client.0.smithi022.stdout: 6 2024-09-21T09:31:07.084 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.084 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 1, 2024-09-21T09:31:07.085 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:07.085 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:07.085 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:07.085 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:07.085 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:07.085 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:07.085 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.062366+0000", 2024-09-21T09:31:07.085 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.635778+0000", 2024-09-21T09:31:07.085 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.635778+0000", 2024-09-21T09:31:07.085 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.636115+0000", 2024-09-21T09:31:07.085 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.062366+0000", 2024-09-21T09:31:07.085 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.086 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:07.086 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.086 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:07.086 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.062366+0000", 2024-09-21T09:31:07.086 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.062366+0000", 2024-09-21T09:31:07.086 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.062366+0000", 2024-09-21T09:31:07.086 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:07.086 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:07.086 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.086 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.062366+0000", 2024-09-21T09:31:07.086 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.062366+0000", 2024-09-21T09:31:07.087 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:07.087 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:07.087 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:07.087 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:07.087 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:07.087 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:07.087 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:07.087 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:07.087 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:07.087 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:07.087 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:07.087 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:07.087 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:07.088 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.3e", 2024-09-21T09:31:07.088 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:07.088 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:07.088 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:07.088 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:07.088 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:07.088 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T11:36:35.607057+0000", 2024-09-21T09:31:07.088 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:07.088 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:07.088 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.088 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:07.088 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.088 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.089 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.089 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.089 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.089 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.089 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.089 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.089 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.089 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.089 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.089 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.089 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.089 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.089 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.089 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:07.090 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:07.090 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.090 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:07.090 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.090 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.090 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.090 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.090 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.090 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:07.090 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.090 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.090 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.090 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.090 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.090 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.091 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.091 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.091 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.091 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.091 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.091 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.091 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:07.091 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:07.091 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.091 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:07.091 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:07.091 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:07.091 INFO:tasks.workunit.client.0.smithi022.stdout: 1, 2024-09-21T09:31:07.091 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:07.092 INFO:tasks.workunit.client.0.smithi022.stdout: 6 2024-09-21T09:31:07.092 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.092 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 1, 2024-09-21T09:31:07.092 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:07.092 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.092 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.092 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:07.092 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:07.092 INFO:tasks.workunit.client.0.smithi022.stdout: 2, 2024-09-21T09:31:07.092 INFO:tasks.workunit.client.0.smithi022.stdout: 1 2024-09-21T09:31:07.092 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.092 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 0, 2024-09-21T09:31:07.092 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:07.092 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:07.093 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:07.093 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:07.093 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:07.093 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:07.093 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.141182+0000", 2024-09-21T09:31:07.093 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.640415+0000", 2024-09-21T09:31:07.093 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.640415+0000", 2024-09-21T09:31:07.093 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.640689+0000", 2024-09-21T09:31:07.093 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.141182+0000", 2024-09-21T09:31:07.093 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.093 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:07.093 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.093 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:07.093 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.141182+0000", 2024-09-21T09:31:07.093 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.141182+0000", 2024-09-21T09:31:07.094 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.141182+0000", 2024-09-21T09:31:07.094 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:07.094 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:07.094 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.094 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.141182+0000", 2024-09-21T09:31:07.094 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.141182+0000", 2024-09-21T09:31:07.094 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:07.094 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:07.094 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:07.094 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:07.094 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:07.094 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:07.095 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:07.095 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:07.095 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:07.095 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:07.095 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:07.095 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:07.095 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:07.095 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.3d", 2024-09-21T09:31:07.095 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:07.095 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:07.095 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:07.095 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:07.095 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:07.095 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T21:12:22.314199+0000", 2024-09-21T09:31:07.096 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:07.096 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:07.096 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.096 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:07.096 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.096 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.096 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.096 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.096 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.096 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.096 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.096 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.096 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.096 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.096 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.097 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.097 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.097 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.097 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.097 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:07.097 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:07.097 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.097 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:07.097 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.097 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.097 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.097 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.097 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.097 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:07.097 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.098 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.098 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.098 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.098 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.098 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.098 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.098 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.098 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.098 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.098 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.098 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.098 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:07.098 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:07.099 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.099 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:07.099 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:07.099 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:07.099 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:07.099 INFO:tasks.workunit.client.0.smithi022.stdout: 2, 2024-09-21T09:31:07.099 INFO:tasks.workunit.client.0.smithi022.stdout: 1 2024-09-21T09:31:07.099 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.099 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 0, 2024-09-21T09:31:07.099 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:07.099 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.099 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.099 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:07.099 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:07.100 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:07.100 INFO:tasks.workunit.client.0.smithi022.stdout: 7 2024-09-21T09:31:07.100 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.100 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 5, 2024-09-21T09:31:07.100 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:07.100 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:07.100 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:07.100 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:07.100 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:07.100 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:07.100 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.066315+0000", 2024-09-21T09:31:07.100 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.644246+0000", 2024-09-21T09:31:07.100 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.644246+0000", 2024-09-21T09:31:07.100 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.644536+0000", 2024-09-21T09:31:07.101 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.066315+0000", 2024-09-21T09:31:07.101 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.101 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:07.101 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.101 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:07.101 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.066315+0000", 2024-09-21T09:31:07.101 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.066315+0000", 2024-09-21T09:31:07.101 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.066315+0000", 2024-09-21T09:31:07.101 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:07.101 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:07.101 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.101 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.066315+0000", 2024-09-21T09:31:07.101 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.066315+0000", 2024-09-21T09:31:07.101 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:07.101 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:07.102 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:07.102 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:07.102 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:07.102 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:07.102 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:07.102 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:07.102 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:07.102 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:07.102 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:07.102 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:07.102 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:07.102 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.3c", 2024-09-21T09:31:07.102 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:07.102 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:07.103 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:07.103 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:07.103 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:07.103 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T17:31:37.886172+0000", 2024-09-21T09:31:07.103 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:07.103 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:07.103 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.103 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:07.103 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.103 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.103 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.103 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.103 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.103 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.104 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.104 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.104 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.104 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.104 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.104 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.104 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.104 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.104 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.104 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:07.104 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:07.104 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.104 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:07.104 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.105 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.105 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.105 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.105 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.105 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:07.105 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.105 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.105 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.105 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.105 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.105 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.105 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.105 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.105 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.105 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.106 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.106 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.106 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:07.106 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:07.106 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.106 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:07.106 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:07.106 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:07.106 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:07.106 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:07.106 INFO:tasks.workunit.client.0.smithi022.stdout: 7 2024-09-21T09:31:07.106 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.106 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 5, 2024-09-21T09:31:07.106 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:07.107 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.107 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.107 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:07.107 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:07.107 INFO:tasks.workunit.client.0.smithi022.stdout: 2, 2024-09-21T09:31:07.107 INFO:tasks.workunit.client.0.smithi022.stdout: 4 2024-09-21T09:31:07.107 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.107 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 7, 2024-09-21T09:31:07.107 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:07.107 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:07.107 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:07.107 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:07.107 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:07.107 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:07.108 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.358990+0000", 2024-09-21T09:31:07.108 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.642487+0000", 2024-09-21T09:31:07.108 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.642487+0000", 2024-09-21T09:31:07.108 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.642779+0000", 2024-09-21T09:31:07.108 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.358990+0000", 2024-09-21T09:31:07.108 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.108 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:07.108 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.108 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:07.108 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.358990+0000", 2024-09-21T09:31:07.108 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.358990+0000", 2024-09-21T09:31:07.108 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.358990+0000", 2024-09-21T09:31:07.108 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:07.109 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:07.109 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.109 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.358990+0000", 2024-09-21T09:31:07.109 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.358990+0000", 2024-09-21T09:31:07.109 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:07.109 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:07.109 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:07.109 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:07.109 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:07.109 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:07.109 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:07.109 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:07.109 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:07.110 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:07.110 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:07.110 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:07.110 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:07.110 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.3b", 2024-09-21T09:31:07.110 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:07.110 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:07.110 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:07.110 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:07.110 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:07.110 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T11:17:29.631740+0000", 2024-09-21T09:31:07.110 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:07.111 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:07.111 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.111 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:07.111 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.111 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.111 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.111 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.111 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.111 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.111 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.111 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.111 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.111 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.111 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.111 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.112 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.112 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.112 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.112 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:07.112 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:07.112 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.112 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:07.112 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.112 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.112 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.112 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.112 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.112 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:07.112 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.113 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.113 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.113 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.113 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.113 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.113 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.113 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.113 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.113 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.113 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.113 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.113 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:07.113 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:07.113 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.113 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:07.113 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:07.114 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:07.114 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:07.114 INFO:tasks.workunit.client.0.smithi022.stdout: 2, 2024-09-21T09:31:07.114 INFO:tasks.workunit.client.0.smithi022.stdout: 4 2024-09-21T09:31:07.114 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.114 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 7, 2024-09-21T09:31:07.114 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:07.114 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.114 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.114 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:07.114 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:07.114 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:07.114 INFO:tasks.workunit.client.0.smithi022.stdout: 1 2024-09-21T09:31:07.115 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.115 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 5, 2024-09-21T09:31:07.115 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:07.115 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:07.115 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:07.115 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:07.115 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:07.115 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:07.115 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.066399+0000", 2024-09-21T09:31:07.115 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.650742+0000", 2024-09-21T09:31:07.115 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.650742+0000", 2024-09-21T09:31:07.115 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.651016+0000", 2024-09-21T09:31:07.115 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.066399+0000", 2024-09-21T09:31:07.115 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.115 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:07.116 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.116 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:07.116 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.066399+0000", 2024-09-21T09:31:07.116 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.066399+0000", 2024-09-21T09:31:07.116 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.066399+0000", 2024-09-21T09:31:07.116 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:07.116 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:07.116 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.116 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.066399+0000", 2024-09-21T09:31:07.116 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.066399+0000", 2024-09-21T09:31:07.116 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:07.116 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:07.116 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:07.116 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:07.116 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:07.117 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:07.117 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:07.117 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:07.117 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:07.117 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:07.117 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:07.117 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:07.117 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:07.117 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.3a", 2024-09-21T09:31:07.117 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:07.117 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:07.117 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:07.117 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:07.117 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:07.117 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T17:11:11.493990+0000", 2024-09-21T09:31:07.117 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:07.118 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:07.118 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.118 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:07.118 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.118 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.118 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.118 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.118 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.118 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.118 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.118 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.118 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.118 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.119 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.119 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.119 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.119 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.119 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.119 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:07.119 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:07.119 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.119 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:07.119 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.119 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.119 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.119 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.119 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.120 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:07.120 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.120 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.120 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.120 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.120 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.120 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.120 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.120 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.120 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.120 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.120 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.120 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.121 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:07.121 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:07.121 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.121 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:07.121 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:07.121 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:07.121 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:07.121 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:07.121 INFO:tasks.workunit.client.0.smithi022.stdout: 1 2024-09-21T09:31:07.121 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.121 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 5, 2024-09-21T09:31:07.121 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:07.121 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.122 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.122 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:07.122 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:07.122 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:07.122 INFO:tasks.workunit.client.0.smithi022.stdout: 4 2024-09-21T09:31:07.122 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.122 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 7, 2024-09-21T09:31:07.122 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:07.122 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:07.122 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:07.122 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:07.122 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:07.123 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:07.123 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.358835+0000", 2024-09-21T09:31:07.123 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.646890+0000", 2024-09-21T09:31:07.123 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.646890+0000", 2024-09-21T09:31:07.123 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.647235+0000", 2024-09-21T09:31:07.123 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.358835+0000", 2024-09-21T09:31:07.123 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.123 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:07.123 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.123 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:07.123 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.358835+0000", 2024-09-21T09:31:07.123 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.358835+0000", 2024-09-21T09:31:07.124 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.358835+0000", 2024-09-21T09:31:07.124 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:07.124 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:07.124 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.124 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.358835+0000", 2024-09-21T09:31:07.124 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.358835+0000", 2024-09-21T09:31:07.124 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:07.124 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:07.124 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:07.124 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:07.124 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:07.124 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:07.125 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:07.125 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:07.125 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:07.125 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:07.125 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:07.125 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:07.125 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:07.125 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.39", 2024-09-21T09:31:07.125 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:07.125 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:07.125 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:07.126 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:07.126 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:07.126 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T11:21:45.995387+0000", 2024-09-21T09:31:07.126 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:07.126 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:07.126 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.126 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:07.126 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.126 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.126 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.126 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.127 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.127 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.127 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.127 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.127 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.127 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.127 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.127 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.127 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.127 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.127 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.127 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:07.127 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:07.128 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.128 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:07.128 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.128 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.128 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.128 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.128 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.128 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:07.128 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.128 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.128 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.128 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.128 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.129 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.129 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.129 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.129 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.129 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.129 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.129 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.129 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:07.129 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:07.129 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.129 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:07.129 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:07.129 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:07.130 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:07.130 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:07.130 INFO:tasks.workunit.client.0.smithi022.stdout: 4 2024-09-21T09:31:07.130 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.130 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 7, 2024-09-21T09:31:07.130 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:07.130 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.130 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.130 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:07.130 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:07.130 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:07.131 INFO:tasks.workunit.client.0.smithi022.stdout: 6 2024-09-21T09:31:07.131 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.131 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 3, 2024-09-21T09:31:07.131 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:07.131 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:07.131 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:07.131 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:07.131 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:07.131 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:07.131 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.124976+0000", 2024-09-21T09:31:07.131 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.637751+0000", 2024-09-21T09:31:07.131 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.637751+0000", 2024-09-21T09:31:07.131 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.638161+0000", 2024-09-21T09:31:07.131 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.124976+0000", 2024-09-21T09:31:07.131 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.132 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:07.132 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.132 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:07.132 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.124976+0000", 2024-09-21T09:31:07.132 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.124976+0000", 2024-09-21T09:31:07.132 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.124976+0000", 2024-09-21T09:31:07.132 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:07.132 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:07.132 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.132 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.124976+0000", 2024-09-21T09:31:07.132 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.124976+0000", 2024-09-21T09:31:07.132 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:07.132 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:07.132 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:07.133 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:07.133 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:07.133 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:07.133 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:07.133 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:07.133 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:07.133 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:07.133 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:07.133 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:07.133 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:07.133 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.38", 2024-09-21T09:31:07.133 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:07.133 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:07.133 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:07.133 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:07.134 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:07.134 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T15:07:40.751242+0000", 2024-09-21T09:31:07.134 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:07.134 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:07.134 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.134 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:07.134 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.134 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.134 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.134 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.134 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.134 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.134 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.134 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.135 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.135 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.135 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.135 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.135 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.135 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.135 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.135 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:07.135 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:07.135 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.135 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:07.135 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.135 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.136 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.136 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.136 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.136 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:07.136 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.136 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.136 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.136 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.136 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.136 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.136 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.136 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.136 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.137 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.137 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.137 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.137 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:07.137 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:07.137 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.137 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:07.137 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:07.137 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:07.137 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:07.137 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:07.137 INFO:tasks.workunit.client.0.smithi022.stdout: 6 2024-09-21T09:31:07.137 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.137 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 3, 2024-09-21T09:31:07.138 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:07.138 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.138 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.138 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:07.138 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:07.138 INFO:tasks.workunit.client.0.smithi022.stdout: 1, 2024-09-21T09:31:07.138 INFO:tasks.workunit.client.0.smithi022.stdout: 0 2024-09-21T09:31:07.138 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.138 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 7, 2024-09-21T09:31:07.138 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:07.138 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:07.138 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:07.138 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:07.139 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:07.139 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:07.139 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.359502+0000", 2024-09-21T09:31:07.139 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:52.079352+0000", 2024-09-21T09:31:07.139 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:52.079352+0000", 2024-09-21T09:31:07.139 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:52.079760+0000", 2024-09-21T09:31:07.139 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.359502+0000", 2024-09-21T09:31:07.139 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.139 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:07.139 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.139 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 59, 2024-09-21T09:31:07.139 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.359502+0000", 2024-09-21T09:31:07.140 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.359502+0000", 2024-09-21T09:31:07.140 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.359502+0000", 2024-09-21T09:31:07.140 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:07.140 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:07.140 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.140 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.359502+0000", 2024-09-21T09:31:07.140 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.359502+0000", 2024-09-21T09:31:07.140 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:07.140 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:07.140 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:07.140 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:07.140 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 58, 2024-09-21T09:31:07.140 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:07.140 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:07.140 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:07.141 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:07.141 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:07.141 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:07.141 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:07.141 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:07.141 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.37", 2024-09-21T09:31:07.141 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:07.141 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:07.141 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:07.141 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 31, 2024-09-21T09:31:07.141 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:07.141 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T11:57:57.194876+0000", 2024-09-21T09:31:07.141 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:07.141 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:07.141 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.141 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:07.142 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.142 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.142 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.142 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.142 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.142 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.142 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.142 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.142 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.142 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.142 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.142 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.142 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.142 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.142 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.143 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:07.143 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:07.143 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.143 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:07.143 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.143 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.143 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.143 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.143 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.143 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:07.143 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.143 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.143 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.144 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.144 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.144 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.144 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.144 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.144 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.144 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.144 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.144 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.144 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:07.144 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:07.144 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.144 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:07.144 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:07.144 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:07.145 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:07.145 INFO:tasks.workunit.client.0.smithi022.stdout: 1, 2024-09-21T09:31:07.145 INFO:tasks.workunit.client.0.smithi022.stdout: 0 2024-09-21T09:31:07.145 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.145 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 7, 2024-09-21T09:31:07.145 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:07.145 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.145 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.145 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:07.145 INFO:tasks.workunit.client.0.smithi022.stdout: 6, 2024-09-21T09:31:07.145 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:07.145 INFO:tasks.workunit.client.0.smithi022.stdout: 1 2024-09-21T09:31:07.145 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.145 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 6, 2024-09-21T09:31:07.146 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:07.146 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:07.146 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:07.146 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:07.146 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:07.146 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:07.146 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.484301+0000", 2024-09-21T09:31:07.146 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.635770+0000", 2024-09-21T09:31:07.146 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.635770+0000", 2024-09-21T09:31:07.146 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.636094+0000", 2024-09-21T09:31:07.146 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.484301+0000", 2024-09-21T09:31:07.147 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.147 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:07.147 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.147 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:07.147 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.484301+0000", 2024-09-21T09:31:07.147 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.484301+0000", 2024-09-21T09:31:07.147 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.484301+0000", 2024-09-21T09:31:07.147 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:07.147 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:07.147 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.147 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.484301+0000", 2024-09-21T09:31:07.147 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.484301+0000", 2024-09-21T09:31:07.148 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:07.148 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:07.148 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:07.148 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:07.148 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:07.148 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:07.148 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:07.148 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:07.148 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:07.148 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:07.148 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:07.148 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:07.148 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:07.148 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.36", 2024-09-21T09:31:07.149 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:07.149 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:07.149 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:07.149 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:07.149 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:07.149 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T12:41:52.909549+0000", 2024-09-21T09:31:07.149 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:07.149 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:07.149 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.149 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:07.149 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.149 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.149 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.150 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.150 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.150 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.150 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.150 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.150 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.150 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.150 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.150 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.150 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.150 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.150 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.150 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:07.150 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:07.151 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.151 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:07.151 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.151 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.151 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.151 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.151 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.151 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:07.151 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.151 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.151 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.151 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.151 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.151 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.152 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.152 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.152 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.152 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.152 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.152 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.152 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:07.152 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:07.152 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.152 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:07.152 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:07.152 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:07.152 INFO:tasks.workunit.client.0.smithi022.stdout: 6, 2024-09-21T09:31:07.152 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:07.152 INFO:tasks.workunit.client.0.smithi022.stdout: 1 2024-09-21T09:31:07.153 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.153 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 6, 2024-09-21T09:31:07.153 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:07.153 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.153 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.153 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:07.153 INFO:tasks.workunit.client.0.smithi022.stdout: 6, 2024-09-21T09:31:07.153 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:07.153 INFO:tasks.workunit.client.0.smithi022.stdout: 0 2024-09-21T09:31:07.153 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.153 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 6, 2024-09-21T09:31:07.153 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:07.153 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:07.153 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:07.154 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:07.154 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:07.154 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:07.154 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.484371+0000", 2024-09-21T09:31:07.154 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.640253+0000", 2024-09-21T09:31:07.154 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.640253+0000", 2024-09-21T09:31:07.154 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.640571+0000", 2024-09-21T09:31:07.154 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.484371+0000", 2024-09-21T09:31:07.154 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.154 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:07.154 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.154 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:07.154 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.484371+0000", 2024-09-21T09:31:07.155 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.484371+0000", 2024-09-21T09:31:07.155 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.484371+0000", 2024-09-21T09:31:07.155 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:07.155 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:07.155 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.155 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.484371+0000", 2024-09-21T09:31:07.155 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.484371+0000", 2024-09-21T09:31:07.155 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:07.155 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:07.155 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:07.155 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:07.155 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:07.155 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:07.155 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:07.156 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:07.156 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:07.156 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:07.156 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:07.156 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:07.156 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:07.156 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.35", 2024-09-21T09:31:07.156 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:07.156 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:07.156 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:07.156 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:07.156 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:07.156 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T12:40:44.721446+0000", 2024-09-21T09:31:07.157 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:07.157 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:07.157 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.157 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:07.157 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.157 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.157 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.157 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.157 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.157 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.157 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.157 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.157 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.157 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.158 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.158 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.158 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.158 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.158 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.158 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:07.158 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:07.158 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.158 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:07.158 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.158 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.158 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.158 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.159 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.159 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:07.159 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.159 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.159 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.159 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.159 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.159 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.159 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.159 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.159 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.159 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.160 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.160 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.160 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:07.160 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:07.160 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.160 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:07.160 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:07.160 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:07.160 INFO:tasks.workunit.client.0.smithi022.stdout: 6, 2024-09-21T09:31:07.160 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:07.160 INFO:tasks.workunit.client.0.smithi022.stdout: 0 2024-09-21T09:31:07.160 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.160 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 6, 2024-09-21T09:31:07.161 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:07.161 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.161 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.161 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:07.161 INFO:tasks.workunit.client.0.smithi022.stdout: 4, 2024-09-21T09:31:07.161 INFO:tasks.workunit.client.0.smithi022.stdout: 6, 2024-09-21T09:31:07.161 INFO:tasks.workunit.client.0.smithi022.stdout: 0 2024-09-21T09:31:07.161 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.161 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 4, 2024-09-21T09:31:07.161 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:07.161 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:07.161 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:07.161 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:07.161 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:07.162 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:07.162 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.062533+0000", 2024-09-21T09:31:07.162 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.639056+0000", 2024-09-21T09:31:07.162 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.639056+0000", 2024-09-21T09:31:07.162 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.639471+0000", 2024-09-21T09:31:07.162 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.062533+0000", 2024-09-21T09:31:07.162 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.162 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:07.162 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.162 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:07.162 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.062533+0000", 2024-09-21T09:31:07.162 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.062533+0000", 2024-09-21T09:31:07.162 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.062533+0000", 2024-09-21T09:31:07.163 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:07.163 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:07.163 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.163 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.062533+0000", 2024-09-21T09:31:07.163 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.062533+0000", 2024-09-21T09:31:07.163 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:07.163 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:07.163 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:07.163 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:07.163 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:07.163 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:07.163 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:07.163 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:07.163 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:07.164 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:07.164 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:07.164 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:07.164 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:07.164 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.34", 2024-09-21T09:31:07.164 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:07.164 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:07.164 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:07.164 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:07.164 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:07.164 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T21:16:15.724781+0000", 2024-09-21T09:31:07.164 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:07.165 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:07.165 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.165 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:07.165 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.165 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.165 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.165 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.165 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.165 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.165 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.165 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.165 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.165 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.165 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.166 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.166 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.166 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.166 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.166 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:07.166 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:07.166 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.166 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:07.166 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.166 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.166 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.166 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.166 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.167 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:07.167 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.167 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.167 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.167 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.167 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.167 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.167 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.167 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.167 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.167 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.167 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.167 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.168 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:07.168 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:07.168 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.168 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:07.168 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:07.168 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:07.168 INFO:tasks.workunit.client.0.smithi022.stdout: 4, 2024-09-21T09:31:07.168 INFO:tasks.workunit.client.0.smithi022.stdout: 6, 2024-09-21T09:31:07.168 INFO:tasks.workunit.client.0.smithi022.stdout: 0 2024-09-21T09:31:07.168 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.168 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 4, 2024-09-21T09:31:07.168 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:07.168 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.169 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.169 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:07.169 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:07.169 INFO:tasks.workunit.client.0.smithi022.stdout: 6, 2024-09-21T09:31:07.169 INFO:tasks.workunit.client.0.smithi022.stdout: 1 2024-09-21T09:31:07.169 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.169 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 3, 2024-09-21T09:31:07.169 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:07.169 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:07.169 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:07.169 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:07.169 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:07.169 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:07.170 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.125281+0000", 2024-09-21T09:31:07.170 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.637068+0000", 2024-09-21T09:31:07.170 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.637068+0000", 2024-09-21T09:31:07.170 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.637331+0000", 2024-09-21T09:31:07.170 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.125281+0000", 2024-09-21T09:31:07.170 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.170 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:07.170 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.170 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:07.170 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.125281+0000", 2024-09-21T09:31:07.170 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.125281+0000", 2024-09-21T09:31:07.170 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.125281+0000", 2024-09-21T09:31:07.170 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:07.171 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:07.171 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.171 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.125281+0000", 2024-09-21T09:31:07.171 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.125281+0000", 2024-09-21T09:31:07.171 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:07.171 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:07.171 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:07.171 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:07.171 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:07.171 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:07.171 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:07.171 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:07.171 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:07.172 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:07.172 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:07.172 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:07.172 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:07.172 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.33", 2024-09-21T09:31:07.172 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:07.172 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:07.172 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:07.172 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:07.172 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:07.172 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T15:24:52.176104+0000", 2024-09-21T09:31:07.172 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:07.172 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:07.173 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.173 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:07.173 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.173 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.173 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.173 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.173 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.173 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.173 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.173 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.173 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.173 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.173 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.173 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.174 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.174 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.174 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.174 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:07.174 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:07.174 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.174 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:07.174 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.174 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.174 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.174 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.174 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.174 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:07.174 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.175 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.175 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.175 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.175 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.175 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.175 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.175 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.175 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.175 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.175 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.175 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.175 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:07.175 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:07.175 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.175 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:07.176 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:07.176 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:07.176 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:07.176 INFO:tasks.workunit.client.0.smithi022.stdout: 6, 2024-09-21T09:31:07.176 INFO:tasks.workunit.client.0.smithi022.stdout: 1 2024-09-21T09:31:07.176 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.176 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 3, 2024-09-21T09:31:07.176 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:07.176 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.176 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.176 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:07.176 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:07.176 INFO:tasks.workunit.client.0.smithi022.stdout: 1, 2024-09-21T09:31:07.176 INFO:tasks.workunit.client.0.smithi022.stdout: 5 2024-09-21T09:31:07.176 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.177 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 0, 2024-09-21T09:31:07.177 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:07.177 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:07.177 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:07.177 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:07.177 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:07.177 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:07.177 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.140751+0000", 2024-09-21T09:31:07.177 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.644114+0000", 2024-09-21T09:31:07.177 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.644114+0000", 2024-09-21T09:31:07.177 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.645565+0000", 2024-09-21T09:31:07.177 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.140751+0000", 2024-09-21T09:31:07.177 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.177 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:07.177 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.178 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:07.178 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.140751+0000", 2024-09-21T09:31:07.178 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.140751+0000", 2024-09-21T09:31:07.178 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.140751+0000", 2024-09-21T09:31:07.178 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:07.178 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:07.178 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.178 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.140751+0000", 2024-09-21T09:31:07.178 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.140751+0000", 2024-09-21T09:31:07.178 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:07.178 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:07.178 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:07.178 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:07.178 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:07.179 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:07.179 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:07.179 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:07.179 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:07.179 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:07.179 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:07.179 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:07.179 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:07.179 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.32", 2024-09-21T09:31:07.179 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:07.179 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:07.179 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:07.179 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:07.179 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:07.180 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T10:27:18.008605+0000", 2024-09-21T09:31:07.180 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:07.180 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:07.180 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.180 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:07.180 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.180 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.180 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.180 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.180 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.180 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.180 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.180 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.180 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.180 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.181 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.181 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.181 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.181 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.181 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.181 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:07.181 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:07.181 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.181 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:07.181 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.181 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.181 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.181 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.181 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.182 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:07.182 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.182 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.182 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.182 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.182 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.182 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.182 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.182 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.182 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.182 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.182 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.183 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.183 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:07.183 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:07.183 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.183 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:07.183 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:07.183 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:07.183 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:07.183 INFO:tasks.workunit.client.0.smithi022.stdout: 1, 2024-09-21T09:31:07.183 INFO:tasks.workunit.client.0.smithi022.stdout: 5 2024-09-21T09:31:07.183 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.183 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 0, 2024-09-21T09:31:07.183 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:07.184 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.184 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.184 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:07.184 INFO:tasks.workunit.client.0.smithi022.stdout: 1, 2024-09-21T09:31:07.184 INFO:tasks.workunit.client.0.smithi022.stdout: 4, 2024-09-21T09:31:07.185 INFO:tasks.workunit.client.0.smithi022.stdout: 5 2024-09-21T09:31:07.185 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.185 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 1, 2024-09-21T09:31:07.185 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:07.185 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:07.186 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:07.186 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:07.186 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:07.186 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:07.186 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.062608+0000", 2024-09-21T09:31:07.186 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.634089+0000", 2024-09-21T09:31:07.186 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.634089+0000", 2024-09-21T09:31:07.186 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.634448+0000", 2024-09-21T09:31:07.186 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.062608+0000", 2024-09-21T09:31:07.186 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.186 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:07.186 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.186 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:07.186 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.062608+0000", 2024-09-21T09:31:07.186 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.062608+0000", 2024-09-21T09:31:07.187 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.062608+0000", 2024-09-21T09:31:07.187 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:07.187 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:07.187 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.187 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.062608+0000", 2024-09-21T09:31:07.187 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.062608+0000", 2024-09-21T09:31:07.187 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:07.187 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:07.187 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:07.187 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:07.187 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:07.187 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:07.187 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:07.187 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:07.188 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:07.188 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:07.188 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:07.188 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:07.188 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:07.188 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.31", 2024-09-21T09:31:07.188 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:07.188 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:07.188 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:07.188 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:07.188 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:07.188 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T10:14:14.726492+0000", 2024-09-21T09:31:07.188 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:07.188 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:07.189 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.189 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:07.189 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.189 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.189 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.189 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.189 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.189 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.189 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.189 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.189 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.189 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.189 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.190 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.190 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.190 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.190 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.190 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:07.190 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:07.190 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.190 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:07.190 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.190 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.190 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.190 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.190 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.190 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:07.191 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.191 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.191 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.191 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.191 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.191 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.191 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.191 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.191 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.191 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.191 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.191 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.191 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:07.191 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:07.192 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.192 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:07.192 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:07.192 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:07.192 INFO:tasks.workunit.client.0.smithi022.stdout: 1, 2024-09-21T09:31:07.192 INFO:tasks.workunit.client.0.smithi022.stdout: 4, 2024-09-21T09:31:07.192 INFO:tasks.workunit.client.0.smithi022.stdout: 5 2024-09-21T09:31:07.192 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.192 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 1, 2024-09-21T09:31:07.192 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:07.192 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.192 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.192 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:07.192 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:07.192 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:07.193 INFO:tasks.workunit.client.0.smithi022.stdout: 7 2024-09-21T09:31:07.193 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.193 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 5, 2024-09-21T09:31:07.193 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:07.193 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:07.193 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:07.193 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:07.193 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:07.193 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:07.193 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.066102+0000", 2024-09-21T09:31:07.193 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.651698+0000", 2024-09-21T09:31:07.193 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.651698+0000", 2024-09-21T09:31:07.193 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.651997+0000", 2024-09-21T09:31:07.193 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.066102+0000", 2024-09-21T09:31:07.193 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.193 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:07.194 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.194 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:07.194 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.066102+0000", 2024-09-21T09:31:07.194 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.066102+0000", 2024-09-21T09:31:07.194 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.066102+0000", 2024-09-21T09:31:07.194 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:07.194 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:07.194 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.194 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.066102+0000", 2024-09-21T09:31:07.194 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.066102+0000", 2024-09-21T09:31:07.194 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:07.194 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:07.194 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:07.194 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:07.195 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:07.195 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:07.195 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:07.195 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:07.195 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:07.195 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:07.195 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:07.195 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:07.195 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:07.195 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.30", 2024-09-21T09:31:07.195 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:07.195 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:07.195 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:07.195 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:07.195 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:07.195 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T09:43:39.574788+0000", 2024-09-21T09:31:07.196 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:07.196 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:07.196 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.196 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:07.196 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.196 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.196 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.196 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.196 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.196 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.196 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.196 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.196 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.196 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.197 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.197 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.197 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.197 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.197 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.197 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:07.197 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:07.197 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.197 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:07.197 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.197 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.197 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.197 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.197 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.197 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:07.198 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.198 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.198 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.198 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.198 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.198 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.198 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.198 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.198 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.198 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.198 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.198 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.198 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:07.198 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:07.198 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.199 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:07.199 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:07.199 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:07.199 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:07.199 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:07.199 INFO:tasks.workunit.client.0.smithi022.stdout: 7 2024-09-21T09:31:07.199 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.199 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 5, 2024-09-21T09:31:07.199 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:07.199 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.199 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.199 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:07.199 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:07.199 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:07.200 INFO:tasks.workunit.client.0.smithi022.stdout: 4 2024-09-21T09:31:07.200 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.200 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 5, 2024-09-21T09:31:07.200 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:07.200 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:07.200 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:07.200 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:07.200 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:07.200 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:07.200 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.066495+0000", 2024-09-21T09:31:07.200 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.646084+0000", 2024-09-21T09:31:07.200 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.646084+0000", 2024-09-21T09:31:07.200 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.646363+0000", 2024-09-21T09:31:07.200 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.066495+0000", 2024-09-21T09:31:07.200 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.200 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:07.201 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.201 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:07.201 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.066495+0000", 2024-09-21T09:31:07.201 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.066495+0000", 2024-09-21T09:31:07.201 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.066495+0000", 2024-09-21T09:31:07.201 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:07.201 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:07.201 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.201 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.066495+0000", 2024-09-21T09:31:07.201 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.066495+0000", 2024-09-21T09:31:07.201 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:07.201 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:07.201 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:07.201 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:07.201 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:07.202 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:07.202 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:07.202 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:07.202 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:07.202 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:07.202 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:07.202 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:07.202 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:07.202 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.2f", 2024-09-21T09:31:07.202 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:07.202 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:07.202 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:07.202 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:07.202 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:07.203 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T15:35:38.711926+0000", 2024-09-21T09:31:07.203 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:07.203 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:07.203 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.203 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:07.203 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.203 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.203 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.203 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.203 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.203 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.203 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.203 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.204 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.204 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.204 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.204 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.204 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.204 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.204 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.204 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:07.204 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:07.204 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.204 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:07.204 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.204 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.204 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.204 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.205 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.205 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:07.205 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.205 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.205 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.205 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.205 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.205 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.205 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.205 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.205 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.205 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.205 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.205 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.205 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:07.205 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:07.206 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.206 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:07.206 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:07.206 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:07.206 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:07.206 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:07.206 INFO:tasks.workunit.client.0.smithi022.stdout: 4 2024-09-21T09:31:07.206 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.206 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 5, 2024-09-21T09:31:07.206 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:07.206 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.206 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.206 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:07.206 INFO:tasks.workunit.client.0.smithi022.stdout: 2, 2024-09-21T09:31:07.207 INFO:tasks.workunit.client.0.smithi022.stdout: 6, 2024-09-21T09:31:07.207 INFO:tasks.workunit.client.0.smithi022.stdout: 7 2024-09-21T09:31:07.207 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.207 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 2, 2024-09-21T09:31:07.207 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:07.207 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:07.207 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:07.207 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:07.207 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:07.207 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:07.207 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.061977+0000", 2024-09-21T09:31:07.207 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.636438+0000", 2024-09-21T09:31:07.207 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.636438+0000", 2024-09-21T09:31:07.207 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.636697+0000", 2024-09-21T09:31:07.208 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.061977+0000", 2024-09-21T09:31:07.208 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.208 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:07.208 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.208 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:07.208 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.061977+0000", 2024-09-21T09:31:07.208 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.061977+0000", 2024-09-21T09:31:07.208 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.061977+0000", 2024-09-21T09:31:07.208 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:07.208 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:07.208 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.208 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.061977+0000", 2024-09-21T09:31:07.208 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.061977+0000", 2024-09-21T09:31:07.209 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:07.209 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:07.209 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:07.209 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:07.209 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:07.209 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:07.209 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:07.209 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:07.209 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:07.209 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:07.209 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:07.209 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:07.209 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:07.209 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.2e", 2024-09-21T09:31:07.210 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:07.210 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:07.210 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:07.210 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:07.210 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:07.210 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T15:55:44.287544+0000", 2024-09-21T09:31:07.210 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:07.210 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:07.210 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.210 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:07.210 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.210 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.210 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.210 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.210 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.211 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.211 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.211 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.211 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.211 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.211 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.211 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.211 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.211 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.211 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.211 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:07.211 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:07.211 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.212 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:07.212 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.212 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.212 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.212 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.212 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.212 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:07.212 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.212 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.212 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.212 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.212 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.212 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.212 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.212 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.213 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.213 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.213 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.213 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.213 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:07.213 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:07.213 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.213 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:07.213 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:07.213 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:07.213 INFO:tasks.workunit.client.0.smithi022.stdout: 2, 2024-09-21T09:31:07.213 INFO:tasks.workunit.client.0.smithi022.stdout: 6, 2024-09-21T09:31:07.213 INFO:tasks.workunit.client.0.smithi022.stdout: 7 2024-09-21T09:31:07.213 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.214 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 2, 2024-09-21T09:31:07.214 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:07.214 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.214 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.214 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:07.214 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:07.214 INFO:tasks.workunit.client.0.smithi022.stdout: 2, 2024-09-21T09:31:07.214 INFO:tasks.workunit.client.0.smithi022.stdout: 6 2024-09-21T09:31:07.214 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.214 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 7, 2024-09-21T09:31:07.214 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:07.214 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:07.214 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:07.214 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:07.215 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:07.215 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:07.215 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.359395+0000", 2024-09-21T09:31:07.215 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.635807+0000", 2024-09-21T09:31:07.215 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.635807+0000", 2024-09-21T09:31:07.215 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.636110+0000", 2024-09-21T09:31:07.215 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.359395+0000", 2024-09-21T09:31:07.215 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.215 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:07.215 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.215 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:07.215 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.359395+0000", 2024-09-21T09:31:07.215 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.359395+0000", 2024-09-21T09:31:07.215 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.359395+0000", 2024-09-21T09:31:07.215 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:07.216 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:07.216 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.216 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.359395+0000", 2024-09-21T09:31:07.216 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.359395+0000", 2024-09-21T09:31:07.216 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:07.216 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:07.216 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:07.216 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:07.216 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:07.216 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:07.216 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:07.216 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:07.216 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:07.216 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:07.217 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:07.217 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:07.217 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:07.217 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.2d", 2024-09-21T09:31:07.217 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:07.217 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:07.217 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:07.217 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:07.217 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:07.217 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T16:22:43.942811+0000", 2024-09-21T09:31:07.217 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:07.217 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:07.217 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.218 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:07.218 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.218 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.218 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.218 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.218 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.218 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.218 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.218 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.218 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.218 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.218 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.218 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.218 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.219 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.219 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.219 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:07.219 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:07.219 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.219 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:07.219 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.219 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.219 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.219 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.219 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.219 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:07.219 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.220 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.220 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.220 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.220 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.220 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.220 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.220 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.220 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.220 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.220 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.220 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.220 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:07.220 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:07.221 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.221 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:07.221 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:07.221 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:07.221 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:07.221 INFO:tasks.workunit.client.0.smithi022.stdout: 2, 2024-09-21T09:31:07.221 INFO:tasks.workunit.client.0.smithi022.stdout: 6 2024-09-21T09:31:07.221 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.221 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 7, 2024-09-21T09:31:07.221 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:07.221 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.221 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.221 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:07.221 INFO:tasks.workunit.client.0.smithi022.stdout: 6, 2024-09-21T09:31:07.221 INFO:tasks.workunit.client.0.smithi022.stdout: 4, 2024-09-21T09:31:07.222 INFO:tasks.workunit.client.0.smithi022.stdout: 0 2024-09-21T09:31:07.222 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.222 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 6, 2024-09-21T09:31:07.222 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:07.222 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:07.222 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:07.222 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:07.222 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:07.222 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:07.222 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.484182+0000", 2024-09-21T09:31:07.222 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.649905+0000", 2024-09-21T09:31:07.222 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.649905+0000", 2024-09-21T09:31:07.222 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.650351+0000", 2024-09-21T09:31:07.222 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.484182+0000", 2024-09-21T09:31:07.223 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.223 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:07.223 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.223 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:07.223 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.484182+0000", 2024-09-21T09:31:07.223 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.484182+0000", 2024-09-21T09:31:07.223 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.484182+0000", 2024-09-21T09:31:07.223 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:07.223 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:07.223 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.223 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.484182+0000", 2024-09-21T09:31:07.223 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.484182+0000", 2024-09-21T09:31:07.223 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:07.223 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:07.224 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:07.224 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:07.224 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:07.224 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:07.224 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:07.224 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:07.224 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:07.224 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:07.224 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:07.224 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:07.224 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:07.224 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.2c", 2024-09-21T09:31:07.224 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:07.224 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:07.225 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:07.225 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:07.225 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:07.225 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T13:12:31.390666+0000", 2024-09-21T09:31:07.225 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:07.225 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:07.225 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.225 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:07.225 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.225 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.225 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.225 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.225 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.225 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.226 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.226 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.226 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.226 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.226 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.226 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.226 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.226 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.226 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.226 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:07.226 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:07.226 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.226 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:07.226 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.227 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.227 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.227 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.227 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.227 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:07.227 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.227 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.227 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.227 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.227 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.227 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.227 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.227 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.227 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.228 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.228 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.228 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.228 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:07.228 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:07.228 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.228 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:07.228 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:07.228 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:07.228 INFO:tasks.workunit.client.0.smithi022.stdout: 6, 2024-09-21T09:31:07.228 INFO:tasks.workunit.client.0.smithi022.stdout: 4, 2024-09-21T09:31:07.228 INFO:tasks.workunit.client.0.smithi022.stdout: 0 2024-09-21T09:31:07.228 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.228 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 6, 2024-09-21T09:31:07.229 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:07.229 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.229 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.229 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:07.229 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:07.229 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:07.229 INFO:tasks.workunit.client.0.smithi022.stdout: 1 2024-09-21T09:31:07.229 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.229 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 0, 2024-09-21T09:31:07.229 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:07.229 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:07.229 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:07.229 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:07.229 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:07.229 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:07.230 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.141020+0000", 2024-09-21T09:31:07.230 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.642408+0000", 2024-09-21T09:31:07.230 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.642408+0000", 2024-09-21T09:31:07.230 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.643345+0000", 2024-09-21T09:31:07.230 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.141020+0000", 2024-09-21T09:31:07.230 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.230 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:07.230 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.230 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:07.230 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.141020+0000", 2024-09-21T09:31:07.230 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.141020+0000", 2024-09-21T09:31:07.230 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.141020+0000", 2024-09-21T09:31:07.230 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:07.230 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:07.231 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.231 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.141020+0000", 2024-09-21T09:31:07.231 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.141020+0000", 2024-09-21T09:31:07.231 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:07.231 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:07.231 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:07.231 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:07.231 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:07.231 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:07.231 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:07.231 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:07.231 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:07.231 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:07.231 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:07.231 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:07.232 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:07.232 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.2b", 2024-09-21T09:31:07.232 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:07.232 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:07.232 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:07.232 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:07.232 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:07.232 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T17:34:59.698334+0000", 2024-09-21T09:31:07.232 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:07.232 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:07.232 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.232 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:07.232 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.233 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.233 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.233 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.233 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.233 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.233 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.233 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.233 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.233 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.233 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.233 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.233 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.234 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.234 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.234 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:07.234 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:07.234 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.234 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:07.234 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.234 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.234 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.234 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.234 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.234 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:07.234 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.235 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.235 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.235 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.235 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.235 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.235 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.235 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.235 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.235 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.235 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.235 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.235 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:07.235 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:07.235 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.235 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:07.236 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:07.236 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:07.236 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:07.236 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:07.236 INFO:tasks.workunit.client.0.smithi022.stdout: 1 2024-09-21T09:31:07.236 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.236 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 0, 2024-09-21T09:31:07.236 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:07.236 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.236 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.236 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:07.236 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:07.236 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:07.236 INFO:tasks.workunit.client.0.smithi022.stdout: 6 2024-09-21T09:31:07.237 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.237 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 5, 2024-09-21T09:31:07.237 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:07.237 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:07.237 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:07.237 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:07.237 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:07.237 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:07.237 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.066015+0000", 2024-09-21T09:31:07.237 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.644322+0000", 2024-09-21T09:31:07.237 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.644322+0000", 2024-09-21T09:31:07.237 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.645032+0000", 2024-09-21T09:31:07.237 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.066015+0000", 2024-09-21T09:31:07.237 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.237 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:07.238 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.238 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:07.238 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.066015+0000", 2024-09-21T09:31:07.238 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.066015+0000", 2024-09-21T09:31:07.238 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.066015+0000", 2024-09-21T09:31:07.238 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:07.238 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:07.238 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.238 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.066015+0000", 2024-09-21T09:31:07.238 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.066015+0000", 2024-09-21T09:31:07.238 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:07.238 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:07.238 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:07.239 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:07.239 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:07.239 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:07.239 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:07.239 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:07.239 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:07.239 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:07.239 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:07.239 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:07.239 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:07.239 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.2a", 2024-09-21T09:31:07.239 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:07.239 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:07.239 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:07.239 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:07.240 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:07.240 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T16:10:14.511772+0000", 2024-09-21T09:31:07.240 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:07.240 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:07.240 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.240 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:07.240 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.240 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.240 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.240 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.240 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.240 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.240 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.240 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.240 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.241 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.241 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.241 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.241 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.241 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.241 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.241 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:07.241 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:07.241 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.241 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:07.241 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.241 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.241 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.241 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.241 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.241 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:07.242 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.242 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.242 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.242 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.242 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.242 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.242 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.242 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.242 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.242 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.242 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.242 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.242 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:07.242 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:07.242 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.243 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:07.243 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:07.243 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:07.243 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:07.243 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:07.243 INFO:tasks.workunit.client.0.smithi022.stdout: 6 2024-09-21T09:31:07.243 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.243 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 5, 2024-09-21T09:31:07.243 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:07.243 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.243 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.243 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:07.243 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:07.243 INFO:tasks.workunit.client.0.smithi022.stdout: 4, 2024-09-21T09:31:07.244 INFO:tasks.workunit.client.0.smithi022.stdout: 1 2024-09-21T09:31:07.244 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.244 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 3, 2024-09-21T09:31:07.244 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:07.244 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:07.244 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:07.244 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:07.244 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:07.244 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:07.244 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.125084+0000", 2024-09-21T09:31:07.244 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.638036+0000", 2024-09-21T09:31:07.244 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.638036+0000", 2024-09-21T09:31:07.244 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.638305+0000", 2024-09-21T09:31:07.244 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.125084+0000", 2024-09-21T09:31:07.245 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.245 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:07.245 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.245 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:07.245 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.125084+0000", 2024-09-21T09:31:07.245 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.125084+0000", 2024-09-21T09:31:07.245 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.125084+0000", 2024-09-21T09:31:07.245 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:07.245 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:07.245 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.245 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.125084+0000", 2024-09-21T09:31:07.245 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.125084+0000", 2024-09-21T09:31:07.245 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:07.245 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:07.246 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:07.246 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:07.246 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:07.246 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:07.246 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:07.246 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:07.246 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:07.246 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:07.246 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:07.246 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:07.246 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:07.246 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.29", 2024-09-21T09:31:07.246 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:07.247 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:07.247 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:07.247 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:07.247 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:07.247 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T10:38:54.627840+0000", 2024-09-21T09:31:07.247 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:07.247 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:07.247 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.247 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:07.247 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.247 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.247 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.247 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.248 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.248 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.248 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.248 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.248 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.248 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.248 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.248 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.248 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.248 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.248 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.248 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:07.248 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:07.248 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.249 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:07.249 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.249 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.249 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.249 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.249 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.249 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:07.249 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.249 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.249 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.249 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.249 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.249 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.249 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.250 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.250 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.250 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.250 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.250 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.250 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:07.250 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:07.250 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.250 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:07.250 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:07.250 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:07.250 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:07.250 INFO:tasks.workunit.client.0.smithi022.stdout: 4, 2024-09-21T09:31:07.251 INFO:tasks.workunit.client.0.smithi022.stdout: 1 2024-09-21T09:31:07.251 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.251 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 3, 2024-09-21T09:31:07.251 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:07.251 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.251 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.251 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:07.251 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:07.251 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:07.251 INFO:tasks.workunit.client.0.smithi022.stdout: 2 2024-09-21T09:31:07.251 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.251 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 5, 2024-09-21T09:31:07.251 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:07.251 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:07.252 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:07.252 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:07.252 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:07.252 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:07.252 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.066181+0000", 2024-09-21T09:31:07.252 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.636541+0000", 2024-09-21T09:31:07.252 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.636541+0000", 2024-09-21T09:31:07.252 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.637277+0000", 2024-09-21T09:31:07.252 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.066181+0000", 2024-09-21T09:31:07.252 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.252 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:07.252 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.252 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:07.252 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.066181+0000", 2024-09-21T09:31:07.253 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.066181+0000", 2024-09-21T09:31:07.253 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.066181+0000", 2024-09-21T09:31:07.253 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:07.253 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:07.253 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.253 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.066181+0000", 2024-09-21T09:31:07.253 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.066181+0000", 2024-09-21T09:31:07.253 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:07.253 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:07.253 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:07.253 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:07.253 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:07.253 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:07.253 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:07.253 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:07.254 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:07.254 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:07.254 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:07.254 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:07.254 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:07.254 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.28", 2024-09-21T09:31:07.254 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:07.254 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:07.254 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:07.254 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:07.254 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:07.254 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T09:45:47.090531+0000", 2024-09-21T09:31:07.254 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:07.254 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:07.255 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.255 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:07.255 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.255 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.255 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.255 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.255 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.255 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.255 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.255 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.255 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.255 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.255 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.255 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.256 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.256 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.256 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.256 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:07.256 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:07.256 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.256 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:07.256 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.256 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.256 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.256 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.256 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.256 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:07.256 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.256 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.257 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.257 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.257 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.257 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.257 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.257 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.257 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.257 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.257 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.257 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.257 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:07.257 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:07.257 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.258 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:07.258 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:07.258 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:07.258 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:07.258 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:07.258 INFO:tasks.workunit.client.0.smithi022.stdout: 2 2024-09-21T09:31:07.258 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.258 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 5, 2024-09-21T09:31:07.258 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:07.258 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.258 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.258 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:07.258 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:07.258 INFO:tasks.workunit.client.0.smithi022.stdout: 4, 2024-09-21T09:31:07.259 INFO:tasks.workunit.client.0.smithi022.stdout: 2 2024-09-21T09:31:07.259 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.259 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 3, 2024-09-21T09:31:07.259 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:07.259 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:07.259 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:07.259 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:07.259 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:07.259 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:07.259 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.125608+0000", 2024-09-21T09:31:07.259 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.638663+0000", 2024-09-21T09:31:07.259 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.638663+0000", 2024-09-21T09:31:07.259 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.638946+0000", 2024-09-21T09:31:07.260 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.125608+0000", 2024-09-21T09:31:07.260 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.260 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:07.260 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.260 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:07.260 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.125608+0000", 2024-09-21T09:31:07.260 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.125608+0000", 2024-09-21T09:31:07.260 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.125608+0000", 2024-09-21T09:31:07.260 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:07.260 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:07.260 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.260 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.125608+0000", 2024-09-21T09:31:07.260 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.125608+0000", 2024-09-21T09:31:07.260 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:07.260 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:07.261 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:07.261 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:07.261 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:07.261 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:07.261 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:07.261 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:07.261 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:07.261 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:07.261 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:07.261 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:07.261 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:07.261 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.27", 2024-09-21T09:31:07.261 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:07.261 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:07.261 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:07.262 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:07.262 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:07.262 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T12:49:57.591531+0000", 2024-09-21T09:31:07.262 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:07.262 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:07.262 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.262 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:07.262 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.262 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.262 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.262 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.262 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.262 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.263 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.263 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.263 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.263 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.263 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.263 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.263 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.263 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.263 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.263 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:07.263 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:07.263 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.263 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:07.264 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.264 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.264 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.264 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.264 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.264 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:07.264 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.264 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.264 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.264 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.264 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.264 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.264 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.264 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.265 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.265 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.265 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.265 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.265 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:07.265 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:07.265 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.265 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:07.265 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:07.265 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:07.265 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:07.265 INFO:tasks.workunit.client.0.smithi022.stdout: 4, 2024-09-21T09:31:07.265 INFO:tasks.workunit.client.0.smithi022.stdout: 2 2024-09-21T09:31:07.265 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.266 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 3, 2024-09-21T09:31:07.266 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:07.266 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.266 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.266 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:07.266 INFO:tasks.workunit.client.0.smithi022.stdout: 6, 2024-09-21T09:31:07.266 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:07.266 INFO:tasks.workunit.client.0.smithi022.stdout: 4 2024-09-21T09:31:07.266 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.266 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 6, 2024-09-21T09:31:07.266 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:07.266 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:07.266 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:07.266 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:07.267 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:07.267 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:07.267 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.484321+0000", 2024-09-21T09:31:07.267 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.638383+0000", 2024-09-21T09:31:07.267 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.638383+0000", 2024-09-21T09:31:07.267 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.638679+0000", 2024-09-21T09:31:07.267 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.484321+0000", 2024-09-21T09:31:07.267 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.267 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:07.267 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.267 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:07.267 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.484321+0000", 2024-09-21T09:31:07.267 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.484321+0000", 2024-09-21T09:31:07.267 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.484321+0000", 2024-09-21T09:31:07.268 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:07.268 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:07.268 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.268 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.484321+0000", 2024-09-21T09:31:07.268 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.484321+0000", 2024-09-21T09:31:07.268 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:07.268 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:07.268 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:07.268 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:07.268 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:07.268 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:07.268 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:07.268 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:07.268 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:07.268 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:07.268 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:07.269 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:07.269 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:07.269 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.26", 2024-09-21T09:31:07.269 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:07.269 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:07.269 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:07.269 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:07.269 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:07.269 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T18:16:11.663503+0000", 2024-09-21T09:31:07.269 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:07.269 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:07.269 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.269 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:07.269 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.270 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.270 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.270 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.270 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.270 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.270 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.270 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.270 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.270 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.270 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.270 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.270 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.270 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.270 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.271 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:07.271 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:07.271 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.271 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:07.271 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.271 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.271 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.271 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.271 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.271 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:07.271 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.271 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.272 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.272 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.272 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.272 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.272 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.272 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.272 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.272 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.272 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.272 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.272 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:07.272 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:07.272 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.272 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:07.273 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:07.273 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:07.273 INFO:tasks.workunit.client.0.smithi022.stdout: 6, 2024-09-21T09:31:07.273 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:07.273 INFO:tasks.workunit.client.0.smithi022.stdout: 4 2024-09-21T09:31:07.273 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.273 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 6, 2024-09-21T09:31:07.273 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:07.273 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.273 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.273 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:07.273 INFO:tasks.workunit.client.0.smithi022.stdout: 1, 2024-09-21T09:31:07.273 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:07.273 INFO:tasks.workunit.client.0.smithi022.stdout: 2 2024-09-21T09:31:07.273 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.274 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 1, 2024-09-21T09:31:07.274 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:07.274 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:07.274 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:07.274 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:07.274 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:07.274 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:07.274 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.061992+0000", 2024-09-21T09:31:07.274 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.632569+0000", 2024-09-21T09:31:07.274 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.632569+0000", 2024-09-21T09:31:07.274 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.632894+0000", 2024-09-21T09:31:07.274 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.061992+0000", 2024-09-21T09:31:07.274 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.275 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:07.275 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.275 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:07.275 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.061992+0000", 2024-09-21T09:31:07.275 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.061992+0000", 2024-09-21T09:31:07.275 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.061992+0000", 2024-09-21T09:31:07.275 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:07.275 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:07.275 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.275 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.061992+0000", 2024-09-21T09:31:07.275 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.061992+0000", 2024-09-21T09:31:07.275 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:07.275 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:07.276 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:07.276 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:07.276 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:07.276 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:07.276 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:07.276 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:07.276 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:07.276 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:07.276 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:07.276 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:07.276 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:07.276 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.25", 2024-09-21T09:31:07.276 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:07.277 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:07.277 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:07.277 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:07.277 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:07.277 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T13:25:22.833535+0000", 2024-09-21T09:31:07.277 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:07.277 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:07.277 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.277 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:07.277 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.277 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.277 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.277 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.278 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.278 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.278 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.278 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.278 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.278 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.278 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.278 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.278 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.278 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.278 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.279 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:07.279 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:07.279 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.279 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:07.279 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.279 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.279 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.279 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.279 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.279 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:07.279 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.279 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.279 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.279 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.280 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.280 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.280 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.280 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.280 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.280 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.280 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.280 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.280 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:07.280 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:07.280 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.280 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:07.280 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:07.280 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:07.281 INFO:tasks.workunit.client.0.smithi022.stdout: 1, 2024-09-21T09:31:07.281 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:07.281 INFO:tasks.workunit.client.0.smithi022.stdout: 2 2024-09-21T09:31:07.281 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.281 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 1, 2024-09-21T09:31:07.281 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:07.281 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.281 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.281 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:07.281 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:07.281 INFO:tasks.workunit.client.0.smithi022.stdout: 2, 2024-09-21T09:31:07.281 INFO:tasks.workunit.client.0.smithi022.stdout: 0 2024-09-21T09:31:07.281 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.281 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 3, 2024-09-21T09:31:07.282 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:07.282 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:07.282 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:07.282 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:07.282 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:07.282 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:07.282 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.125399+0000", 2024-09-21T09:31:07.282 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:52.079395+0000", 2024-09-21T09:31:07.282 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:52.079395+0000", 2024-09-21T09:31:07.282 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:52.079786+0000", 2024-09-21T09:31:07.282 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.125399+0000", 2024-09-21T09:31:07.282 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.282 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:07.282 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.283 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 59, 2024-09-21T09:31:07.283 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.125399+0000", 2024-09-21T09:31:07.283 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.125399+0000", 2024-09-21T09:31:07.283 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.125399+0000", 2024-09-21T09:31:07.283 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:07.283 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:07.283 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.283 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.125399+0000", 2024-09-21T09:31:07.283 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.125399+0000", 2024-09-21T09:31:07.283 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:07.283 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:07.283 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:07.283 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:07.283 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 58, 2024-09-21T09:31:07.284 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:07.284 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:07.284 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:07.284 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:07.284 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:07.284 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:07.284 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:07.284 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:07.284 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.24", 2024-09-21T09:31:07.284 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:07.284 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:07.284 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:07.285 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 31, 2024-09-21T09:31:07.285 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:07.285 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T12:17:07.388900+0000", 2024-09-21T09:31:07.285 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:07.285 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:07.285 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.285 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:07.285 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.285 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.285 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.285 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.285 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.285 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.285 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.286 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.286 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.286 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.286 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.286 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.286 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.286 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.286 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.286 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:07.286 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:07.286 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.286 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:07.286 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.286 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.287 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.287 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.287 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.287 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:07.287 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.287 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.287 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.287 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.287 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.287 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.287 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.287 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.287 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.287 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.287 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.288 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.288 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:07.288 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:07.288 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.288 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:07.288 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:07.288 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:07.288 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:07.288 INFO:tasks.workunit.client.0.smithi022.stdout: 2, 2024-09-21T09:31:07.288 INFO:tasks.workunit.client.0.smithi022.stdout: 0 2024-09-21T09:31:07.288 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.288 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 3, 2024-09-21T09:31:07.288 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:07.289 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.289 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.289 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:07.289 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:07.289 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:07.289 INFO:tasks.workunit.client.0.smithi022.stdout: 4 2024-09-21T09:31:07.289 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.289 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 0, 2024-09-21T09:31:07.289 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:07.289 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:07.289 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:07.289 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:07.289 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:07.289 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:07.289 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.141121+0000", 2024-09-21T09:31:07.290 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.634065+0000", 2024-09-21T09:31:07.290 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.634065+0000", 2024-09-21T09:31:07.290 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.634785+0000", 2024-09-21T09:31:07.290 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.141121+0000", 2024-09-21T09:31:07.290 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.290 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:07.290 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.290 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:07.290 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.141121+0000", 2024-09-21T09:31:07.290 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.141121+0000", 2024-09-21T09:31:07.290 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.141121+0000", 2024-09-21T09:31:07.290 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:07.290 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:07.290 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.291 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.141121+0000", 2024-09-21T09:31:07.291 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.141121+0000", 2024-09-21T09:31:07.291 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:07.291 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:07.291 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:07.291 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:07.291 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:07.291 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:07.291 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:07.291 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:07.291 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:07.291 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:07.291 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:07.292 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:07.292 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:07.292 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.23", 2024-09-21T09:31:07.292 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:07.292 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:07.292 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:07.292 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:07.292 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:07.292 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T19:54:21.482132+0000", 2024-09-21T09:31:07.292 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:07.293 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:07.293 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.293 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:07.293 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.293 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.293 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.293 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.293 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.293 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.293 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.293 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.293 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.293 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.294 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.294 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.294 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.294 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.294 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.294 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:07.294 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:07.294 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.294 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:07.294 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.294 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.294 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.295 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.295 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.295 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:07.295 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.295 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.295 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.295 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.295 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.295 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.295 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.295 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.296 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.296 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.296 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.296 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.296 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:07.296 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:07.296 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.296 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:07.296 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:07.296 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:07.296 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:07.297 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:07.297 INFO:tasks.workunit.client.0.smithi022.stdout: 4 2024-09-21T09:31:07.297 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.297 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 0, 2024-09-21T09:31:07.297 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:07.297 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.297 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.297 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:07.297 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:07.297 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:07.297 INFO:tasks.workunit.client.0.smithi022.stdout: 1 2024-09-21T09:31:07.297 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.297 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 5, 2024-09-21T09:31:07.297 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:07.297 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:07.298 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:07.298 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:07.298 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:07.298 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:07.298 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.066111+0000", 2024-09-21T09:31:07.298 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:52.048885+0000", 2024-09-21T09:31:07.298 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:52.048885+0000", 2024-09-21T09:31:07.298 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:52.049243+0000", 2024-09-21T09:31:07.298 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.066111+0000", 2024-09-21T09:31:07.298 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.298 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:07.298 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.299 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 59, 2024-09-21T09:31:07.299 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.066111+0000", 2024-09-21T09:31:07.299 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.066111+0000", 2024-09-21T09:31:07.299 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.066111+0000", 2024-09-21T09:31:07.299 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:07.299 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:07.299 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.299 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.066111+0000", 2024-09-21T09:31:07.299 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.066111+0000", 2024-09-21T09:31:07.299 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:07.299 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:07.299 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:07.300 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:07.300 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 58, 2024-09-21T09:31:07.300 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:07.300 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:07.300 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:07.300 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:07.300 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:07.300 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:07.300 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:07.300 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:07.300 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.22", 2024-09-21T09:31:07.300 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:07.301 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:07.301 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:07.301 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 31, 2024-09-21T09:31:07.301 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:07.301 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T10:44:22.262414+0000", 2024-09-21T09:31:07.301 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:07.301 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:07.301 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.301 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:07.301 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.301 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.301 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.302 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.302 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.302 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.302 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.302 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.302 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.302 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.302 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.302 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.302 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.302 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.302 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.303 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:07.303 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:07.303 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.303 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:07.303 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.303 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.303 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.303 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.303 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.303 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:07.303 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.304 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.304 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.304 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.304 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.304 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.304 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.304 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.304 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.304 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.304 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.304 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.304 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:07.305 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:07.305 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.305 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:07.305 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:07.305 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:07.305 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:07.305 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:07.305 INFO:tasks.workunit.client.0.smithi022.stdout: 1 2024-09-21T09:31:07.305 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.305 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 5, 2024-09-21T09:31:07.306 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:07.306 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.306 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.306 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:07.306 INFO:tasks.workunit.client.0.smithi022.stdout: 2, 2024-09-21T09:31:07.306 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:07.306 INFO:tasks.workunit.client.0.smithi022.stdout: 6 2024-09-21T09:31:07.306 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.306 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 2, 2024-09-21T09:31:07.306 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:07.306 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:07.307 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:07.307 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:07.307 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:07.307 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:07.307 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.061749+0000", 2024-09-21T09:31:07.307 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.635935+0000", 2024-09-21T09:31:07.307 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.635935+0000", 2024-09-21T09:31:07.307 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.636222+0000", 2024-09-21T09:31:07.307 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.061749+0000", 2024-09-21T09:31:07.307 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.307 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:07.308 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.308 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:07.308 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.061749+0000", 2024-09-21T09:31:07.308 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.061749+0000", 2024-09-21T09:31:07.308 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.061749+0000", 2024-09-21T09:31:07.308 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:07.308 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:07.308 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.308 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.061749+0000", 2024-09-21T09:31:07.308 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.061749+0000", 2024-09-21T09:31:07.308 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:07.309 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:07.309 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:07.309 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:07.309 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:07.309 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:07.309 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:07.309 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:07.309 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:07.309 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:07.309 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:07.309 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:07.310 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:07.310 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.21", 2024-09-21T09:31:07.310 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:07.310 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:07.310 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:07.310 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:07.310 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:07.310 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T14:05:50.402224+0000", 2024-09-21T09:31:07.310 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:07.310 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:07.310 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.310 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:07.310 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.310 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.310 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.311 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.311 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.311 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.311 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.311 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.311 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.311 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.311 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.311 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.311 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.311 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.311 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.311 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:07.312 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:07.312 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.312 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:07.312 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.312 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.312 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.312 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.312 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.312 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:07.312 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.312 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.312 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.312 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.312 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.312 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.313 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.313 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.313 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.313 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.313 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.313 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.313 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:07.313 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:07.313 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.313 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:07.313 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:07.313 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:07.313 INFO:tasks.workunit.client.0.smithi022.stdout: 2, 2024-09-21T09:31:07.313 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:07.313 INFO:tasks.workunit.client.0.smithi022.stdout: 6 2024-09-21T09:31:07.313 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.314 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 2, 2024-09-21T09:31:07.314 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:07.314 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.314 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.314 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:07.314 INFO:tasks.workunit.client.0.smithi022.stdout: 6, 2024-09-21T09:31:07.314 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:07.314 INFO:tasks.workunit.client.0.smithi022.stdout: 5 2024-09-21T09:31:07.314 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.314 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 6, 2024-09-21T09:31:07.314 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:07.314 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:07.314 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:07.314 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:07.315 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:07.315 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:07.315 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.483839+0000", 2024-09-21T09:31:07.315 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.641409+0000", 2024-09-21T09:31:07.315 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.641409+0000", 2024-09-21T09:31:07.315 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.641740+0000", 2024-09-21T09:31:07.315 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.483839+0000", 2024-09-21T09:31:07.315 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.315 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:07.315 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.315 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:07.315 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.483839+0000", 2024-09-21T09:31:07.315 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.483839+0000", 2024-09-21T09:31:07.315 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.483839+0000", 2024-09-21T09:31:07.315 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:07.316 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:07.316 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.316 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.483839+0000", 2024-09-21T09:31:07.316 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.483839+0000", 2024-09-21T09:31:07.316 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:07.316 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:07.316 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:07.316 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:07.316 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:07.316 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:07.316 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:07.316 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:07.316 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:07.316 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:07.316 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:07.317 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:07.317 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:07.317 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.20", 2024-09-21T09:31:07.317 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:07.317 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:07.317 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:07.317 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:07.317 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:07.317 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T18:59:30.487730+0000", 2024-09-21T09:31:07.317 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:07.317 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:07.317 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.317 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:07.317 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.318 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.318 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.318 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.318 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.318 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.318 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.318 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.318 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.318 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.318 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.318 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.318 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.318 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.318 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.319 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:07.319 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:07.319 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.319 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:07.319 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.319 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.319 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.319 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.319 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.319 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:07.319 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.319 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.320 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.320 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.320 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.320 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.320 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.320 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.320 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.320 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.320 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.320 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.320 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:07.320 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:07.320 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.320 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:07.321 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:07.321 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:07.321 INFO:tasks.workunit.client.0.smithi022.stdout: 6, 2024-09-21T09:31:07.321 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:07.321 INFO:tasks.workunit.client.0.smithi022.stdout: 5 2024-09-21T09:31:07.321 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.321 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 6, 2024-09-21T09:31:07.321 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:07.321 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.321 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.321 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:07.321 INFO:tasks.workunit.client.0.smithi022.stdout: 4, 2024-09-21T09:31:07.321 INFO:tasks.workunit.client.0.smithi022.stdout: 1, 2024-09-21T09:31:07.321 INFO:tasks.workunit.client.0.smithi022.stdout: 2 2024-09-21T09:31:07.321 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.322 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 4, 2024-09-21T09:31:07.322 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:07.322 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:07.322 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:07.322 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:07.322 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:07.322 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:07.322 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.062273+0000", 2024-09-21T09:31:07.322 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.629167+0000", 2024-09-21T09:31:07.322 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.629167+0000", 2024-09-21T09:31:07.322 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.629525+0000", 2024-09-21T09:31:07.322 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.062273+0000", 2024-09-21T09:31:07.322 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.322 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:07.323 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.323 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:07.323 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.062273+0000", 2024-09-21T09:31:07.323 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.062273+0000", 2024-09-21T09:31:07.323 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.062273+0000", 2024-09-21T09:31:07.323 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:07.323 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:07.323 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.323 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.062273+0000", 2024-09-21T09:31:07.323 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.062273+0000", 2024-09-21T09:31:07.323 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:07.323 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:07.323 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:07.324 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:07.324 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:07.324 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:07.324 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:07.324 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:07.324 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:07.324 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:07.324 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:07.324 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:07.324 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:07.324 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.1a", 2024-09-21T09:31:07.324 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:07.324 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:07.324 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:07.324 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:07.325 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:07.325 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T21:06:24.323825+0000", 2024-09-21T09:31:07.325 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:07.325 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:07.325 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.325 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:07.325 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.325 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.325 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.325 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.325 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.325 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.325 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.325 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.325 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.326 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.326 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.326 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.326 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.326 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.326 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.326 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:07.326 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:07.326 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.326 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:07.326 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.326 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.326 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.326 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.327 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.327 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:07.327 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.327 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.327 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.327 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.327 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.327 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.327 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.327 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.327 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.327 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.327 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.327 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.327 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:07.328 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:07.328 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.328 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:07.328 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:07.328 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:07.328 INFO:tasks.workunit.client.0.smithi022.stdout: 4, 2024-09-21T09:31:07.328 INFO:tasks.workunit.client.0.smithi022.stdout: 1, 2024-09-21T09:31:07.328 INFO:tasks.workunit.client.0.smithi022.stdout: 2 2024-09-21T09:31:07.328 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.328 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 4, 2024-09-21T09:31:07.328 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:07.328 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.328 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.328 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:07.329 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:07.329 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:07.329 INFO:tasks.workunit.client.0.smithi022.stdout: 6 2024-09-21T09:31:07.329 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.329 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 3, 2024-09-21T09:31:07.329 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:07.329 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:07.329 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 51, 2024-09-21T09:31:07.329 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:07.329 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:07.329 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:07.329 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.125391+0000", 2024-09-21T09:31:07.329 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:32.264963+0000", 2024-09-21T09:31:07.329 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:32.264963+0000", 2024-09-21T09:31:07.330 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:32.265235+0000", 2024-09-21T09:31:07.330 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.125391+0000", 2024-09-21T09:31:07.330 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:07.330 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:07.330 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:07.330 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 52, 2024-09-21T09:31:07.330 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.125391+0000", 2024-09-21T09:31:07.330 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.125391+0000", 2024-09-21T09:31:07.330 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.125391+0000", 2024-09-21T09:31:07.330 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:07.330 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:07.330 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:07.330 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.125391+0000", 2024-09-21T09:31:07.330 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.125391+0000", 2024-09-21T09:31:07.330 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:07.331 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 1, 2024-09-21T09:31:07.331 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:07.331 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:07.331 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 51, 2024-09-21T09:31:07.331 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:07.331 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:07.331 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:07.331 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:07.331 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 1, 2024-09-21T09:31:07.331 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:07.331 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:07.331 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:07.331 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "2.1b", 2024-09-21T09:31:07.332 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:07.332 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:07.332 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:07.332 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 25, 2024-09-21T09:31:07.332 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:07.332 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T18:09:03.927341+0000", 2024-09-21T09:31:07.332 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:07.332 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:07.332 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.332 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:07.332 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.332 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.332 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.332 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.333 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.333 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.333 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.333 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.333 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.333 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.333 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.333 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.333 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.333 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.333 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.333 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:07.333 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:07.333 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.333 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:07.333 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.334 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.334 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.334 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.334 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.334 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:07.334 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.334 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.334 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.334 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.334 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.334 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.334 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.334 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.334 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.335 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.335 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.335 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.335 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:07.335 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:07.335 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.335 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:07.335 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:07.335 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:07.335 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:07.335 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:07.335 INFO:tasks.workunit.client.0.smithi022.stdout: 6 2024-09-21T09:31:07.335 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.336 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 3, 2024-09-21T09:31:07.336 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "57'1" 2024-09-21T09:31:07.336 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.336 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.336 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:07.336 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:07.336 INFO:tasks.workunit.client.0.smithi022.stdout: 4, 2024-09-21T09:31:07.336 INFO:tasks.workunit.client.0.smithi022.stdout: 7 2024-09-21T09:31:07.336 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.336 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 0, 2024-09-21T09:31:07.336 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:07.336 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:07.336 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:07.336 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:07.336 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:07.336 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:07.337 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.141127+0000", 2024-09-21T09:31:07.337 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.641801+0000", 2024-09-21T09:31:07.337 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.641801+0000", 2024-09-21T09:31:07.337 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.642154+0000", 2024-09-21T09:31:07.337 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.141127+0000", 2024-09-21T09:31:07.337 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.337 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:07.337 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.337 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:07.337 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.141127+0000", 2024-09-21T09:31:07.337 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.141127+0000", 2024-09-21T09:31:07.337 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.141127+0000", 2024-09-21T09:31:07.337 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:07.337 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:07.337 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.338 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.141127+0000", 2024-09-21T09:31:07.338 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.141127+0000", 2024-09-21T09:31:07.338 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:07.338 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:07.338 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:07.338 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:07.338 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:07.338 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:07.338 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:07.338 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:07.338 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:07.338 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:07.338 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:07.338 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:07.339 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:07.339 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.1b", 2024-09-21T09:31:07.339 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:07.339 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:07.339 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:07.339 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:07.339 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:07.339 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T10:50:45.075506+0000", 2024-09-21T09:31:07.339 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:07.339 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:07.339 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.339 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:07.339 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.339 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.339 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.340 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.340 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.340 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.340 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.340 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.340 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.340 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.340 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.340 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.340 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.340 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.340 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.340 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:07.340 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:07.340 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.340 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:07.341 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.341 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.341 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.341 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.341 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.341 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:07.341 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.341 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.341 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.341 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.341 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.341 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.341 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.341 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.342 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.342 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.342 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.342 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.342 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:07.342 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:07.342 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.342 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:07.342 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:07.342 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:07.342 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:07.342 INFO:tasks.workunit.client.0.smithi022.stdout: 4, 2024-09-21T09:31:07.342 INFO:tasks.workunit.client.0.smithi022.stdout: 7 2024-09-21T09:31:07.342 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.343 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 0, 2024-09-21T09:31:07.343 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:07.343 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.343 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.343 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:07.343 INFO:tasks.workunit.client.0.smithi022.stdout: 6, 2024-09-21T09:31:07.343 INFO:tasks.workunit.client.0.smithi022.stdout: 4, 2024-09-21T09:31:07.343 INFO:tasks.workunit.client.0.smithi022.stdout: 7 2024-09-21T09:31:07.343 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.343 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 6, 2024-09-21T09:31:07.343 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:07.343 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:07.343 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 51, 2024-09-21T09:31:07.343 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:07.343 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:07.344 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:07.344 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.484044+0000", 2024-09-21T09:31:07.344 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:32.269026+0000", 2024-09-21T09:31:07.344 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:32.269026+0000", 2024-09-21T09:31:07.344 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:32.269358+0000", 2024-09-21T09:31:07.344 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.484044+0000", 2024-09-21T09:31:07.344 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:07.344 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:07.344 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:07.344 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 52, 2024-09-21T09:31:07.344 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.484044+0000", 2024-09-21T09:31:07.344 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.484044+0000", 2024-09-21T09:31:07.344 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.484044+0000", 2024-09-21T09:31:07.344 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:07.344 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:07.345 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:07.345 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.484044+0000", 2024-09-21T09:31:07.345 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.484044+0000", 2024-09-21T09:31:07.345 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:07.345 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 1, 2024-09-21T09:31:07.345 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:07.345 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:07.345 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 51, 2024-09-21T09:31:07.345 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:07.345 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:07.345 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:07.345 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:07.345 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 1, 2024-09-21T09:31:07.345 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:07.345 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:07.346 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:07.346 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "2.1a", 2024-09-21T09:31:07.346 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:07.346 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:07.346 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:07.346 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 25, 2024-09-21T09:31:07.346 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:07.346 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T09:59:17.916370+0000", 2024-09-21T09:31:07.346 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:07.346 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:07.346 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.346 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:07.346 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.346 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.347 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.347 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.347 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.347 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.347 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.347 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.347 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.347 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.347 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.347 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.347 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.347 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.347 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.347 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:07.348 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:07.348 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.348 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:07.348 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.348 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.348 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.348 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.348 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.348 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:07.348 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.348 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.348 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.348 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.349 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.349 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.349 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.349 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.349 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.349 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.349 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.349 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.349 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:07.349 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:07.349 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.349 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:07.349 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:07.349 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:07.350 INFO:tasks.workunit.client.0.smithi022.stdout: 6, 2024-09-21T09:31:07.350 INFO:tasks.workunit.client.0.smithi022.stdout: 4, 2024-09-21T09:31:07.350 INFO:tasks.workunit.client.0.smithi022.stdout: 7 2024-09-21T09:31:07.350 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.350 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 6, 2024-09-21T09:31:07.350 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "57'1" 2024-09-21T09:31:07.350 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.350 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.350 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:07.350 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:07.350 INFO:tasks.workunit.client.0.smithi022.stdout: 6, 2024-09-21T09:31:07.350 INFO:tasks.workunit.client.0.smithi022.stdout: 0 2024-09-21T09:31:07.350 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.351 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 3, 2024-09-21T09:31:07.351 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:07.351 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:07.351 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 51, 2024-09-21T09:31:07.351 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:07.351 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:07.351 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:07.351 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.125195+0000", 2024-09-21T09:31:07.351 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:32.264381+0000", 2024-09-21T09:31:07.351 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:32.264381+0000", 2024-09-21T09:31:07.351 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:32.264601+0000", 2024-09-21T09:31:07.351 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.125195+0000", 2024-09-21T09:31:07.351 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:07.351 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:07.352 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:07.352 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 52, 2024-09-21T09:31:07.352 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.125195+0000", 2024-09-21T09:31:07.352 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.125195+0000", 2024-09-21T09:31:07.352 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.125195+0000", 2024-09-21T09:31:07.352 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:07.352 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:07.352 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:07.352 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.125195+0000", 2024-09-21T09:31:07.352 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.125195+0000", 2024-09-21T09:31:07.352 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:07.352 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:07.352 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:07.352 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:07.353 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 51, 2024-09-21T09:31:07.353 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:07.353 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:07.353 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:07.353 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:07.353 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:07.353 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:07.353 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:07.353 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:07.353 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "2.19", 2024-09-21T09:31:07.353 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:07.353 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:07.353 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:07.353 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 24, 2024-09-21T09:31:07.354 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:07.354 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T19:35:57.463156+0000", 2024-09-21T09:31:07.354 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:07.354 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:07.354 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.354 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:07.354 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.354 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.354 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.354 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.354 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.354 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.354 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.354 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.355 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.355 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.355 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.355 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.355 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.355 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.355 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.355 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:07.355 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:07.355 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.355 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:07.355 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.355 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.355 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.355 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.356 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.356 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:07.356 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.356 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.356 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.356 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.356 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.356 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.356 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.356 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.356 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.356 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.356 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.356 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.356 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:07.357 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:07.357 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.357 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:07.357 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:07.357 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:07.357 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:07.357 INFO:tasks.workunit.client.0.smithi022.stdout: 6, 2024-09-21T09:31:07.357 INFO:tasks.workunit.client.0.smithi022.stdout: 0 2024-09-21T09:31:07.357 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.357 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 3, 2024-09-21T09:31:07.357 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:07.357 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.357 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.357 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:07.357 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:07.357 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:07.358 INFO:tasks.workunit.client.0.smithi022.stdout: 1 2024-09-21T09:31:07.358 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.358 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 3, 2024-09-21T09:31:07.358 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:07.358 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:07.358 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:07.358 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:07.358 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:07.358 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:07.358 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.125067+0000", 2024-09-21T09:31:07.358 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.634224+0000", 2024-09-21T09:31:07.358 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.634224+0000", 2024-09-21T09:31:07.358 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.634475+0000", 2024-09-21T09:31:07.358 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.125067+0000", 2024-09-21T09:31:07.359 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.359 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:07.359 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.359 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:07.359 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.125067+0000", 2024-09-21T09:31:07.359 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.125067+0000", 2024-09-21T09:31:07.359 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.125067+0000", 2024-09-21T09:31:07.359 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:07.359 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:07.359 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.359 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.125067+0000", 2024-09-21T09:31:07.359 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.125067+0000", 2024-09-21T09:31:07.359 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:07.360 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:07.360 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:07.360 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:07.360 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:07.360 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:07.360 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:07.360 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:07.360 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:07.360 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:07.360 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:07.360 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:07.360 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:07.360 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.18", 2024-09-21T09:31:07.360 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:07.361 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:07.361 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:07.361 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:07.361 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:07.361 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T16:38:21.241590+0000", 2024-09-21T09:31:07.361 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:07.361 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:07.361 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.361 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:07.361 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.361 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.361 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.361 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.361 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.361 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.362 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.362 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.362 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.362 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.362 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.362 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.362 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.362 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.362 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.362 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:07.362 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:07.362 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.362 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:07.363 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.363 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.363 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.363 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.363 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.363 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:07.363 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.363 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.363 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.363 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.363 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.363 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.363 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.363 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.364 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.364 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.364 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.364 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.364 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:07.364 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:07.364 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.364 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:07.364 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:07.364 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:07.364 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:07.364 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:07.364 INFO:tasks.workunit.client.0.smithi022.stdout: 1 2024-09-21T09:31:07.364 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.364 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 3, 2024-09-21T09:31:07.365 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:07.365 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.365 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.365 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:07.365 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:07.365 INFO:tasks.workunit.client.0.smithi022.stdout: 4, 2024-09-21T09:31:07.365 INFO:tasks.workunit.client.0.smithi022.stdout: 7 2024-09-21T09:31:07.365 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.365 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 5, 2024-09-21T09:31:07.365 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:07.365 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:07.365 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 51, 2024-09-21T09:31:07.365 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:07.365 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:07.366 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:07.366 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.066263+0000", 2024-09-21T09:31:07.366 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:32.267484+0000", 2024-09-21T09:31:07.366 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:32.267484+0000", 2024-09-21T09:31:07.366 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:32.267805+0000", 2024-09-21T09:31:07.366 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.066263+0000", 2024-09-21T09:31:07.366 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:07.366 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:07.366 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:07.366 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 52, 2024-09-21T09:31:07.366 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.066263+0000", 2024-09-21T09:31:07.366 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.066263+0000", 2024-09-21T09:31:07.366 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.066263+0000", 2024-09-21T09:31:07.366 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:07.367 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:07.367 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:07.367 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.066263+0000", 2024-09-21T09:31:07.367 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.066263+0000", 2024-09-21T09:31:07.367 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:07.367 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:07.367 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:07.367 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:07.367 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 51, 2024-09-21T09:31:07.367 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:07.367 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:07.367 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:07.367 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:07.367 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:07.367 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:07.368 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:07.368 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:07.368 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "2.18", 2024-09-21T09:31:07.368 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:07.368 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:07.368 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:07.368 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 24, 2024-09-21T09:31:07.368 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:07.368 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T19:00:43.105362+0000", 2024-09-21T09:31:07.368 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:07.368 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:07.368 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.368 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:07.368 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.369 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.369 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.369 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.369 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.369 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.369 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.369 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.369 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.369 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.369 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.369 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.369 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.369 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.369 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.370 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:07.370 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:07.370 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.370 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:07.370 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.370 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.370 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.370 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.370 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.370 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:07.370 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.370 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.370 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.370 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.371 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.371 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.371 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.371 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.371 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.371 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.371 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.371 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.371 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:07.371 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:07.371 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.371 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:07.372 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:07.372 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:07.372 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:07.372 INFO:tasks.workunit.client.0.smithi022.stdout: 4, 2024-09-21T09:31:07.372 INFO:tasks.workunit.client.0.smithi022.stdout: 7 2024-09-21T09:31:07.372 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.372 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 5, 2024-09-21T09:31:07.372 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:07.372 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.372 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.372 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:07.372 INFO:tasks.workunit.client.0.smithi022.stdout: 1, 2024-09-21T09:31:07.372 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:07.373 INFO:tasks.workunit.client.0.smithi022.stdout: 4 2024-09-21T09:31:07.373 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.373 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 1, 2024-09-21T09:31:07.373 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:07.373 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:07.373 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:07.373 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:07.373 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:07.373 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:07.373 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.062486+0000", 2024-09-21T09:31:07.373 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.630723+0000", 2024-09-21T09:31:07.373 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.630723+0000", 2024-09-21T09:31:07.373 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.631350+0000", 2024-09-21T09:31:07.373 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.062486+0000", 2024-09-21T09:31:07.373 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.373 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:07.374 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.374 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:07.374 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.062486+0000", 2024-09-21T09:31:07.374 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.062486+0000", 2024-09-21T09:31:07.374 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.062486+0000", 2024-09-21T09:31:07.374 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:07.374 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:07.374 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.374 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.062486+0000", 2024-09-21T09:31:07.374 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.062486+0000", 2024-09-21T09:31:07.374 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:07.374 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:07.375 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:07.375 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:07.375 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:07.375 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:07.375 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:07.375 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:07.375 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:07.375 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:07.375 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:07.375 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:07.375 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:07.375 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.19", 2024-09-21T09:31:07.375 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:07.375 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:07.375 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:07.376 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:07.376 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:07.376 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T15:46:32.135037+0000", 2024-09-21T09:31:07.376 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:07.376 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:07.376 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.376 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:07.376 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.376 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.376 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.376 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.376 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.376 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.376 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.376 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.376 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.377 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.377 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.377 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.377 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.377 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.377 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.377 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:07.377 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:07.377 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.377 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:07.377 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.377 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.377 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.378 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.378 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.378 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:07.378 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.378 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.378 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.378 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.378 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.378 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.378 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.378 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.378 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.378 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.378 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.379 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.379 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:07.379 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:07.379 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.379 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:07.379 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:07.379 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:07.379 INFO:tasks.workunit.client.0.smithi022.stdout: 1, 2024-09-21T09:31:07.379 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:07.379 INFO:tasks.workunit.client.0.smithi022.stdout: 4 2024-09-21T09:31:07.379 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.379 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 1, 2024-09-21T09:31:07.379 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:07.379 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.380 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.380 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:07.380 INFO:tasks.workunit.client.0.smithi022.stdout: 6, 2024-09-21T09:31:07.380 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:07.380 INFO:tasks.workunit.client.0.smithi022.stdout: 2 2024-09-21T09:31:07.380 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.380 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 6, 2024-09-21T09:31:07.380 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:07.380 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:07.380 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 51, 2024-09-21T09:31:07.380 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:07.380 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:07.380 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:07.380 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.484390+0000", 2024-09-21T09:31:07.381 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:32.265995+0000", 2024-09-21T09:31:07.381 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:32.265995+0000", 2024-09-21T09:31:07.381 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:32.266307+0000", 2024-09-21T09:31:07.381 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.484390+0000", 2024-09-21T09:31:07.381 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:07.381 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:07.381 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:07.381 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 52, 2024-09-21T09:31:07.381 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.484390+0000", 2024-09-21T09:31:07.381 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.484390+0000", 2024-09-21T09:31:07.381 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.484390+0000", 2024-09-21T09:31:07.381 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:07.381 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:07.381 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:07.381 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.484390+0000", 2024-09-21T09:31:07.381 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.484390+0000", 2024-09-21T09:31:07.382 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:07.382 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:07.382 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:07.382 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:07.382 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 51, 2024-09-21T09:31:07.382 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:07.382 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:07.382 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:07.382 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:07.382 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:07.382 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:07.382 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:07.382 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:07.382 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "2.17", 2024-09-21T09:31:07.382 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:07.383 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:07.383 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:07.383 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 24, 2024-09-21T09:31:07.383 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:07.383 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T16:24:28.036534+0000", 2024-09-21T09:31:07.383 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:07.383 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:07.383 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.383 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:07.383 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.383 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.383 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.383 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.383 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.384 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.384 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.384 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.384 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.384 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.384 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.384 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.384 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.384 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.384 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.384 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:07.384 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:07.384 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.384 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:07.384 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.385 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.385 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.385 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.385 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.385 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:07.385 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.385 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.385 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.385 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.385 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.385 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.385 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.385 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.385 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.385 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.386 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.386 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.386 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:07.386 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:07.386 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.386 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:07.386 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:07.386 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:07.386 INFO:tasks.workunit.client.0.smithi022.stdout: 6, 2024-09-21T09:31:07.386 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:07.386 INFO:tasks.workunit.client.0.smithi022.stdout: 2 2024-09-21T09:31:07.386 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.386 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 6, 2024-09-21T09:31:07.386 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:07.387 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.387 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.387 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:07.387 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:07.387 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:07.387 INFO:tasks.workunit.client.0.smithi022.stdout: 1 2024-09-21T09:31:07.387 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.387 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 5, 2024-09-21T09:31:07.387 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:07.387 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:07.387 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:07.387 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:07.387 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:07.387 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:07.387 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.066174+0000", 2024-09-21T09:31:07.388 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.647649+0000", 2024-09-21T09:31:07.388 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.647649+0000", 2024-09-21T09:31:07.389 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.647952+0000", 2024-09-21T09:31:07.389 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.066174+0000", 2024-09-21T09:31:07.389 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.390 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:07.390 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.390 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:07.390 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.066174+0000", 2024-09-21T09:31:07.390 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.066174+0000", 2024-09-21T09:31:07.390 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.066174+0000", 2024-09-21T09:31:07.390 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:07.390 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:07.390 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.390 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.066174+0000", 2024-09-21T09:31:07.390 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.066174+0000", 2024-09-21T09:31:07.390 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:07.390 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:07.391 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:07.391 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:07.391 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:07.391 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:07.391 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:07.391 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:07.391 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:07.391 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:07.391 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:07.391 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:07.391 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:07.391 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.16", 2024-09-21T09:31:07.392 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:07.392 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:07.392 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:07.392 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:07.392 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:07.392 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T12:12:29.850835+0000", 2024-09-21T09:31:07.392 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:07.392 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:07.392 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.392 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:07.392 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.392 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.392 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.393 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.393 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.393 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.393 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.393 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.393 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.393 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.393 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.393 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.393 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.393 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.393 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.393 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:07.393 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:07.393 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.394 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:07.394 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.394 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.394 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.394 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.394 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.394 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:07.394 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.394 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.394 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.394 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.394 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.394 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.395 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.395 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.395 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.395 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.395 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.395 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.395 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:07.395 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:07.395 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.395 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:07.395 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:07.395 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:07.395 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:07.396 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:07.396 INFO:tasks.workunit.client.0.smithi022.stdout: 1 2024-09-21T09:31:07.396 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.396 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 5, 2024-09-21T09:31:07.396 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:07.396 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.396 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.396 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:07.396 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:07.396 INFO:tasks.workunit.client.0.smithi022.stdout: 6, 2024-09-21T09:31:07.396 INFO:tasks.workunit.client.0.smithi022.stdout: 2 2024-09-21T09:31:07.396 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.396 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 5, 2024-09-21T09:31:07.396 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:07.396 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:07.397 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 51, 2024-09-21T09:31:07.397 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:07.397 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:07.397 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:07.397 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.066325+0000", 2024-09-21T09:31:07.397 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:32.268832+0000", 2024-09-21T09:31:07.397 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:32.268832+0000", 2024-09-21T09:31:07.397 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:32.269137+0000", 2024-09-21T09:31:07.397 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.066325+0000", 2024-09-21T09:31:07.397 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:07.397 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:07.397 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:07.397 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 52, 2024-09-21T09:31:07.397 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.066325+0000", 2024-09-21T09:31:07.398 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.066325+0000", 2024-09-21T09:31:07.398 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.066325+0000", 2024-09-21T09:31:07.398 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:07.398 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:07.398 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:07.398 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.066325+0000", 2024-09-21T09:31:07.398 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.066325+0000", 2024-09-21T09:31:07.398 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:07.398 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:07.398 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:07.398 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:07.398 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 51, 2024-09-21T09:31:07.398 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:07.398 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:07.399 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:07.399 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:07.399 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:07.399 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:07.399 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:07.399 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:07.399 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "2.16", 2024-09-21T09:31:07.399 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:07.399 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:07.399 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:07.399 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 24, 2024-09-21T09:31:07.399 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:07.399 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T15:51:58.668390+0000", 2024-09-21T09:31:07.399 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:07.399 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:07.400 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.400 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:07.400 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.400 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.400 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.400 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.400 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.400 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.400 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.400 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.400 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.400 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.400 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.400 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.401 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.401 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.401 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.401 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:07.401 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:07.401 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.401 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:07.401 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.401 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.401 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.401 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.401 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.401 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:07.402 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.402 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.402 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.402 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.402 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.402 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.402 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.402 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.402 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.402 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.402 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.402 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.402 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:07.402 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:07.403 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.403 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:07.403 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:07.403 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:07.403 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:07.403 INFO:tasks.workunit.client.0.smithi022.stdout: 6, 2024-09-21T09:31:07.403 INFO:tasks.workunit.client.0.smithi022.stdout: 2 2024-09-21T09:31:07.403 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.403 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 5, 2024-09-21T09:31:07.403 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:07.403 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.403 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.403 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:07.404 INFO:tasks.workunit.client.0.smithi022.stdout: 2, 2024-09-21T09:31:07.404 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:07.404 INFO:tasks.workunit.client.0.smithi022.stdout: 3 2024-09-21T09:31:07.404 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.404 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 2, 2024-09-21T09:31:07.404 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:07.404 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:07.404 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:07.404 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:07.404 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:07.404 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:07.404 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.062462+0000", 2024-09-21T09:31:07.404 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:52.049549+0000", 2024-09-21T09:31:07.404 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:52.049549+0000", 2024-09-21T09:31:07.404 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:52.050121+0000", 2024-09-21T09:31:07.405 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.062462+0000", 2024-09-21T09:31:07.405 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.405 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:07.405 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.405 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 59, 2024-09-21T09:31:07.405 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.062462+0000", 2024-09-21T09:31:07.405 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.062462+0000", 2024-09-21T09:31:07.405 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.062462+0000", 2024-09-21T09:31:07.405 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:07.405 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:07.405 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.405 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.062462+0000", 2024-09-21T09:31:07.405 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.062462+0000", 2024-09-21T09:31:07.406 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:07.406 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:07.406 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:07.406 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:07.406 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 58, 2024-09-21T09:31:07.406 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:07.406 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:07.406 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:07.406 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:07.406 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:07.406 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:07.406 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:07.406 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:07.406 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.17", 2024-09-21T09:31:07.407 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:07.407 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:07.407 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:07.407 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 18, 2024-09-21T09:31:07.407 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:07.407 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T11:25:27.303500+0000", 2024-09-21T09:31:07.407 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:07.407 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:07.407 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.407 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:07.407 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.407 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.407 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.407 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.408 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.408 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.408 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.408 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.408 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.408 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.408 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.408 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.408 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.408 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.408 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.408 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:07.408 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:07.408 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.409 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:07.409 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.409 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.409 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.409 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.409 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.409 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:07.409 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.409 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.409 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.409 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.409 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.409 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.409 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.410 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.410 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.410 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.410 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.410 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.410 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:07.410 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:07.410 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.410 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:07.410 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:07.410 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:07.410 INFO:tasks.workunit.client.0.smithi022.stdout: 2, 2024-09-21T09:31:07.410 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:07.410 INFO:tasks.workunit.client.0.smithi022.stdout: 3 2024-09-21T09:31:07.411 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.411 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 2, 2024-09-21T09:31:07.411 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:07.411 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.411 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.411 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:07.411 INFO:tasks.workunit.client.0.smithi022.stdout: 1, 2024-09-21T09:31:07.411 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:07.411 INFO:tasks.workunit.client.0.smithi022.stdout: 0 2024-09-21T09:31:07.411 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.411 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 1, 2024-09-21T09:31:07.411 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:07.411 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:07.411 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 51, 2024-09-21T09:31:07.411 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:07.412 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:07.412 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:07.412 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.062137+0000", 2024-09-21T09:31:07.412 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:32.269808+0000", 2024-09-21T09:31:07.412 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:32.269808+0000", 2024-09-21T09:31:07.412 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:32.270145+0000", 2024-09-21T09:31:07.412 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.062137+0000", 2024-09-21T09:31:07.412 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:07.412 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:07.412 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:07.412 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 52, 2024-09-21T09:31:07.412 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.062137+0000", 2024-09-21T09:31:07.412 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.062137+0000", 2024-09-21T09:31:07.412 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.062137+0000", 2024-09-21T09:31:07.413 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:07.413 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:07.413 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:07.413 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.062137+0000", 2024-09-21T09:31:07.413 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.062137+0000", 2024-09-21T09:31:07.413 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:07.413 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 3, 2024-09-21T09:31:07.413 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:07.413 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:07.413 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 51, 2024-09-21T09:31:07.413 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:07.413 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:07.413 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:07.413 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:07.414 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 3, 2024-09-21T09:31:07.414 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:07.414 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:07.414 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:07.414 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "2.15", 2024-09-21T09:31:07.414 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:07.414 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:07.414 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:07.414 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 27, 2024-09-21T09:31:07.414 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:07.414 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T15:35:53.397343+0000", 2024-09-21T09:31:07.414 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:07.414 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:07.414 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.415 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 90, 2024-09-21T09:31:07.415 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.415 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.415 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.415 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.415 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.415 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.415 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.415 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.415 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.415 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.415 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.415 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.415 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.416 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.416 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.416 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 6, 2024-09-21T09:31:07.416 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 2, 2024-09-21T09:31:07.416 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.416 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 2, 2024-09-21T09:31:07.416 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.416 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.416 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.416 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.416 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.416 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 1, 2024-09-21T09:31:07.416 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.416 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.416 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.417 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.417 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.417 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.417 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.417 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.417 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.417 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.417 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.417 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.417 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 4, 2024-09-21T09:31:07.417 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 1 2024-09-21T09:31:07.417 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.417 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:07.417 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:07.417 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:07.418 INFO:tasks.workunit.client.0.smithi022.stdout: 1, 2024-09-21T09:31:07.418 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:07.418 INFO:tasks.workunit.client.0.smithi022.stdout: 0 2024-09-21T09:31:07.418 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.418 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 1, 2024-09-21T09:31:07.418 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "57'3" 2024-09-21T09:31:07.418 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.418 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.418 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:07.418 INFO:tasks.workunit.client.0.smithi022.stdout: 4, 2024-09-21T09:31:07.418 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:07.418 INFO:tasks.workunit.client.0.smithi022.stdout: 6 2024-09-21T09:31:07.418 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.418 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 4, 2024-09-21T09:31:07.419 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:07.419 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:07.419 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:07.419 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:07.419 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:07.419 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:07.419 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.062595+0000", 2024-09-21T09:31:07.419 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.639976+0000", 2024-09-21T09:31:07.419 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.639976+0000", 2024-09-21T09:31:07.419 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.640314+0000", 2024-09-21T09:31:07.419 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.062595+0000", 2024-09-21T09:31:07.419 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.419 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:07.419 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.420 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:07.420 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.062595+0000", 2024-09-21T09:31:07.420 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.062595+0000", 2024-09-21T09:31:07.420 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.062595+0000", 2024-09-21T09:31:07.420 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:07.420 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:07.420 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.420 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.062595+0000", 2024-09-21T09:31:07.420 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.062595+0000", 2024-09-21T09:31:07.420 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:07.420 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:07.420 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:07.420 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:07.420 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:07.421 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:07.421 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:07.421 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:07.421 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:07.421 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:07.421 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:07.421 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:07.421 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:07.421 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.14", 2024-09-21T09:31:07.421 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:07.421 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:07.421 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:07.421 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:07.421 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:07.421 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T15:55:21.229874+0000", 2024-09-21T09:31:07.422 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:07.422 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:07.422 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.422 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:07.422 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.422 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.422 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.422 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.422 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.422 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.422 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.422 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.422 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.422 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.423 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.423 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.423 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.423 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.423 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.423 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:07.423 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:07.423 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.423 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:07.423 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.423 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.423 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.423 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.423 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.424 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:07.424 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.424 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.424 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.424 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.424 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.424 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.424 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.424 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.424 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.424 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.424 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.424 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.424 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:07.425 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:07.425 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.425 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:07.425 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:07.425 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:07.425 INFO:tasks.workunit.client.0.smithi022.stdout: 4, 2024-09-21T09:31:07.425 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:07.425 INFO:tasks.workunit.client.0.smithi022.stdout: 6 2024-09-21T09:31:07.425 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.425 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 4, 2024-09-21T09:31:07.425 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:07.425 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.425 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.425 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:07.426 INFO:tasks.workunit.client.0.smithi022.stdout: 6, 2024-09-21T09:31:07.426 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:07.426 INFO:tasks.workunit.client.0.smithi022.stdout: 5 2024-09-21T09:31:07.426 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.426 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 6, 2024-09-21T09:31:07.426 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:07.426 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:07.426 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 51, 2024-09-21T09:31:07.426 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:07.426 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:07.426 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:07.426 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:29:56.877634+0000", 2024-09-21T09:31:07.426 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:32.265712+0000", 2024-09-21T09:31:07.427 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:32.265712+0000", 2024-09-21T09:31:07.427 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:32.266499+0000", 2024-09-21T09:31:07.427 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:29:56.877634+0000", 2024-09-21T09:31:07.427 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:07.427 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:07.427 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:07.427 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 52, 2024-09-21T09:31:07.427 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:29:56.877634+0000", 2024-09-21T09:31:07.427 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:29:56.877634+0000", 2024-09-21T09:31:07.427 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:29:56.877634+0000", 2024-09-21T09:31:07.427 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:07.427 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:07.427 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:07.427 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:29:56.877634+0000", 2024-09-21T09:31:07.428 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:29:56.877634+0000", 2024-09-21T09:31:07.428 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:07.428 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 12, 2024-09-21T09:31:07.428 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:07.428 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:07.428 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 51, 2024-09-21T09:31:07.428 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:07.428 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:07.428 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:07.428 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:07.428 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 12, 2024-09-21T09:31:07.428 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:07.428 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:07.428 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:07.429 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "2.14", 2024-09-21T09:31:07.429 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:07.429 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:07.429 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 67, 2024-09-21T09:31:07.429 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 36, 2024-09-21T09:31:07.429 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:07.429 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T14:19:17.014990+0000", 2024-09-21T09:31:07.429 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:07.429 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:07.429 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.429 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 2624, 2024-09-21T09:31:07.429 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.429 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.429 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.429 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.430 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.430 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.430 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.430 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.430 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.430 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.430 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.430 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.430 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.430 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.430 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.430 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 12, 2024-09-21T09:31:07.430 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 4, 2024-09-21T09:31:07.430 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.431 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 4, 2024-09-21T09:31:07.431 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.431 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.431 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.431 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.431 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.431 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 1, 2024-09-21T09:31:07.431 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.431 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.431 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.431 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.431 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.431 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.432 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.432 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.432 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.432 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.432 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.432 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.432 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 12, 2024-09-21T09:31:07.432 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 9 2024-09-21T09:31:07.432 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.432 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:07.432 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:07.432 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:07.432 INFO:tasks.workunit.client.0.smithi022.stdout: 6, 2024-09-21T09:31:07.432 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:07.432 INFO:tasks.workunit.client.0.smithi022.stdout: 5 2024-09-21T09:31:07.433 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.433 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 6, 2024-09-21T09:31:07.433 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "67'12" 2024-09-21T09:31:07.433 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.433 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.433 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:07.433 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:07.433 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:07.433 INFO:tasks.workunit.client.0.smithi022.stdout: 4 2024-09-21T09:31:07.433 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.433 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 7, 2024-09-21T09:31:07.433 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:07.433 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:07.433 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:07.434 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:07.434 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:07.434 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:07.434 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.359220+0000", 2024-09-21T09:31:07.434 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.641272+0000", 2024-09-21T09:31:07.434 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.641272+0000", 2024-09-21T09:31:07.434 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.641626+0000", 2024-09-21T09:31:07.434 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.359220+0000", 2024-09-21T09:31:07.434 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.434 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:07.434 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.434 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:07.434 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.359220+0000", 2024-09-21T09:31:07.434 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.359220+0000", 2024-09-21T09:31:07.435 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.359220+0000", 2024-09-21T09:31:07.435 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:07.435 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:07.435 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.435 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.359220+0000", 2024-09-21T09:31:07.435 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.359220+0000", 2024-09-21T09:31:07.435 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:07.435 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:07.435 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:07.435 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:07.435 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:07.435 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:07.436 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:07.436 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:07.436 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:07.436 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:07.436 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:07.436 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:07.436 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:07.436 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.15", 2024-09-21T09:31:07.436 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:07.436 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:07.436 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:07.436 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:07.436 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:07.436 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T17:15:05.108072+0000", 2024-09-21T09:31:07.436 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:07.437 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:07.437 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.437 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:07.437 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.437 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.437 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.437 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.437 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.437 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.437 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.437 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.437 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.437 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.437 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.437 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.438 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.438 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.438 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.438 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:07.438 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:07.438 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.438 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:07.438 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.438 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.438 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.438 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.438 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.438 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:07.438 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.439 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.439 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.439 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.439 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.439 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.439 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.439 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.439 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.439 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.439 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.439 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.439 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:07.439 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:07.439 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.439 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:07.440 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:07.440 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:07.440 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:07.440 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:07.440 INFO:tasks.workunit.client.0.smithi022.stdout: 4 2024-09-21T09:31:07.440 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.440 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 7, 2024-09-21T09:31:07.440 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:07.440 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.440 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.440 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:07.440 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:07.440 INFO:tasks.workunit.client.0.smithi022.stdout: 4, 2024-09-21T09:31:07.440 INFO:tasks.workunit.client.0.smithi022.stdout: 3 2024-09-21T09:31:07.440 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.441 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 0, 2024-09-21T09:31:07.441 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:07.441 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:07.441 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 51, 2024-09-21T09:31:07.441 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:07.441 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:07.441 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:07.441 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.141220+0000", 2024-09-21T09:31:07.441 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:32.267108+0000", 2024-09-21T09:31:07.441 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:32.267108+0000", 2024-09-21T09:31:07.441 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:32.267412+0000", 2024-09-21T09:31:07.441 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.141220+0000", 2024-09-21T09:31:07.441 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:07.441 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:07.441 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:07.442 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 52, 2024-09-21T09:31:07.442 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.141220+0000", 2024-09-21T09:31:07.442 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.141220+0000", 2024-09-21T09:31:07.442 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.141220+0000", 2024-09-21T09:31:07.442 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:07.442 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:07.442 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:07.442 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.141220+0000", 2024-09-21T09:31:07.442 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.141220+0000", 2024-09-21T09:31:07.442 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:07.442 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 2, 2024-09-21T09:31:07.442 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:07.442 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:07.442 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 51, 2024-09-21T09:31:07.443 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:07.443 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:07.443 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:07.443 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:07.443 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 2, 2024-09-21T09:31:07.443 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:07.443 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:07.443 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:07.443 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "2.13", 2024-09-21T09:31:07.443 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:07.443 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:07.443 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:07.443 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 26, 2024-09-21T09:31:07.443 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:07.444 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T17:54:11.228702+0000", 2024-09-21T09:31:07.444 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:07.444 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:07.444 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.444 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 34, 2024-09-21T09:31:07.444 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.444 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.444 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.444 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.444 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.444 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.444 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.444 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.445 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.445 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.445 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.445 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.445 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.445 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.445 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.445 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 6, 2024-09-21T09:31:07.445 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 2, 2024-09-21T09:31:07.445 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.445 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 2, 2024-09-21T09:31:07.445 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.445 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.445 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.446 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.446 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.446 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 1, 2024-09-21T09:31:07.446 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.446 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.446 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.446 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.446 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.446 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.446 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.446 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.446 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.446 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.447 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.447 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.447 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 3, 2024-09-21T09:31:07.447 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 6 2024-09-21T09:31:07.447 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.447 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:07.447 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:07.447 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:07.447 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:07.447 INFO:tasks.workunit.client.0.smithi022.stdout: 4, 2024-09-21T09:31:07.447 INFO:tasks.workunit.client.0.smithi022.stdout: 3 2024-09-21T09:31:07.447 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.447 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 0, 2024-09-21T09:31:07.448 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "57'2" 2024-09-21T09:31:07.448 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.448 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.448 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:07.448 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:07.448 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:07.448 INFO:tasks.workunit.client.0.smithi022.stdout: 3 2024-09-21T09:31:07.448 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.448 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 0, 2024-09-21T09:31:07.448 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:07.448 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:07.448 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:07.448 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:07.448 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:07.449 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:07.449 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.140829+0000", 2024-09-21T09:31:07.449 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.641185+0000", 2024-09-21T09:31:07.449 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.641185+0000", 2024-09-21T09:31:07.449 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.641572+0000", 2024-09-21T09:31:07.449 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.140829+0000", 2024-09-21T09:31:07.449 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.449 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:07.449 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.449 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:07.449 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.140829+0000", 2024-09-21T09:31:07.449 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.140829+0000", 2024-09-21T09:31:07.449 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.140829+0000", 2024-09-21T09:31:07.449 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:07.450 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:07.450 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.450 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.140829+0000", 2024-09-21T09:31:07.450 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.140829+0000", 2024-09-21T09:31:07.450 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:07.450 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:07.450 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:07.450 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:07.450 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:07.450 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:07.450 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:07.450 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:07.450 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:07.450 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:07.451 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:07.451 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:07.451 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:07.451 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.12", 2024-09-21T09:31:07.451 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:07.451 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:07.451 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:07.451 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:07.451 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:07.451 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T09:52:30.993491+0000", 2024-09-21T09:31:07.451 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:07.451 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:07.451 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.451 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:07.452 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.452 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.452 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.452 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.452 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.452 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.452 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.452 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.452 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.452 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.452 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.452 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.452 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.453 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.453 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.453 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:07.453 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:07.453 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.453 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:07.453 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.453 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.453 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.453 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.453 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.453 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:07.453 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.454 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.454 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.454 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.454 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.454 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.454 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.454 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.454 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.454 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.454 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.454 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.454 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:07.454 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:07.454 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.455 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:07.455 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:07.455 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:07.455 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:07.455 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:07.455 INFO:tasks.workunit.client.0.smithi022.stdout: 3 2024-09-21T09:31:07.455 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.455 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 0, 2024-09-21T09:31:07.455 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:07.455 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.455 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.455 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:07.455 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:07.456 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:07.456 INFO:tasks.workunit.client.0.smithi022.stdout: 7 2024-09-21T09:31:07.456 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.456 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 5, 2024-09-21T09:31:07.456 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:07.456 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:07.456 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 51, 2024-09-21T09:31:07.456 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:07.456 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:07.456 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:07.456 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.066197+0000", 2024-09-21T09:31:07.456 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:32.267232+0000", 2024-09-21T09:31:07.457 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:32.267232+0000", 2024-09-21T09:31:07.457 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:32.267644+0000", 2024-09-21T09:31:07.457 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.066197+0000", 2024-09-21T09:31:07.457 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:07.457 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:07.457 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:07.457 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 52, 2024-09-21T09:31:07.457 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.066197+0000", 2024-09-21T09:31:07.457 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.066197+0000", 2024-09-21T09:31:07.457 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.066197+0000", 2024-09-21T09:31:07.457 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:07.457 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:07.457 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:07.457 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.066197+0000", 2024-09-21T09:31:07.457 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.066197+0000", 2024-09-21T09:31:07.458 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:07.458 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:07.458 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:07.458 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:07.458 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 51, 2024-09-21T09:31:07.458 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:07.458 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:07.458 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:07.458 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:07.458 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:07.458 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:07.458 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:07.459 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:07.459 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "2.12", 2024-09-21T09:31:07.459 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:07.459 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:07.459 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:07.459 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 24, 2024-09-21T09:31:07.459 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:07.459 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T19:14:19.578564+0000", 2024-09-21T09:31:07.459 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:07.459 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:07.459 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.459 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:07.459 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.459 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.460 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.460 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.460 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.460 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.460 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.460 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.460 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.460 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.460 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.460 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.460 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.460 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.460 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.460 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:07.461 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:07.461 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.461 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:07.461 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.461 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.461 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.461 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.461 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.461 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:07.461 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.461 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.461 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.461 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.461 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.462 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.462 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.462 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.462 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.462 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.462 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.462 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.462 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:07.462 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:07.462 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.462 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:07.462 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:07.463 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:07.463 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:07.463 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:07.463 INFO:tasks.workunit.client.0.smithi022.stdout: 7 2024-09-21T09:31:07.463 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.463 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 5, 2024-09-21T09:31:07.463 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:07.463 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.463 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.463 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:07.463 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:07.463 INFO:tasks.workunit.client.0.smithi022.stdout: 4, 2024-09-21T09:31:07.463 INFO:tasks.workunit.client.0.smithi022.stdout: 2 2024-09-21T09:31:07.463 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.464 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 7, 2024-09-21T09:31:07.464 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:07.464 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:07.464 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:07.464 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:07.464 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:07.464 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:07.464 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.358995+0000", 2024-09-21T09:31:07.464 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.643681+0000", 2024-09-21T09:31:07.464 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.643681+0000", 2024-09-21T09:31:07.464 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.644217+0000", 2024-09-21T09:31:07.464 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.358995+0000", 2024-09-21T09:31:07.464 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.465 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:07.465 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.465 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:07.465 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.358995+0000", 2024-09-21T09:31:07.465 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.358995+0000", 2024-09-21T09:31:07.465 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.358995+0000", 2024-09-21T09:31:07.465 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:07.465 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:07.465 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.465 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.358995+0000", 2024-09-21T09:31:07.465 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.358995+0000", 2024-09-21T09:31:07.465 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:07.465 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:07.465 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:07.466 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:07.466 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:07.466 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:07.466 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:07.466 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:07.466 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:07.466 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:07.466 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:07.466 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:07.466 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:07.466 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.13", 2024-09-21T09:31:07.466 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:07.467 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:07.467 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:07.467 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:07.467 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:07.467 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T14:07:58.418951+0000", 2024-09-21T09:31:07.467 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:07.467 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:07.467 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.467 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:07.467 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.467 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.467 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.467 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.467 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.467 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.468 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.468 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.468 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.468 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.468 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.468 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.468 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.468 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.468 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.468 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:07.468 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:07.468 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.468 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:07.469 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.469 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.469 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.469 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.469 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.469 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:07.469 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.469 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.469 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.469 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.469 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.469 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.469 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.469 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.470 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.470 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.470 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.470 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.470 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:07.470 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:07.470 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.470 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:07.470 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:07.470 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:07.470 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:07.470 INFO:tasks.workunit.client.0.smithi022.stdout: 4, 2024-09-21T09:31:07.470 INFO:tasks.workunit.client.0.smithi022.stdout: 2 2024-09-21T09:31:07.470 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.471 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 7, 2024-09-21T09:31:07.471 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:07.471 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.471 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.471 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:07.471 INFO:tasks.workunit.client.0.smithi022.stdout: 6, 2024-09-21T09:31:07.471 INFO:tasks.workunit.client.0.smithi022.stdout: 4, 2024-09-21T09:31:07.471 INFO:tasks.workunit.client.0.smithi022.stdout: 1 2024-09-21T09:31:07.471 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.471 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 6, 2024-09-21T09:31:07.471 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:07.471 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:07.471 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 51, 2024-09-21T09:31:07.471 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:07.472 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:07.472 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:07.472 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.483959+0000", 2024-09-21T09:31:07.472 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:32.269808+0000", 2024-09-21T09:31:07.472 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:32.269808+0000", 2024-09-21T09:31:07.472 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:32.270501+0000", 2024-09-21T09:31:07.472 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.483959+0000", 2024-09-21T09:31:07.472 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:07.472 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:07.472 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:07.472 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 52, 2024-09-21T09:31:07.472 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.483959+0000", 2024-09-21T09:31:07.472 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.483959+0000", 2024-09-21T09:31:07.472 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.483959+0000", 2024-09-21T09:31:07.473 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:07.473 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:07.473 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:07.473 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.483959+0000", 2024-09-21T09:31:07.473 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.483959+0000", 2024-09-21T09:31:07.473 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:07.473 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:07.473 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:07.473 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:07.473 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 51, 2024-09-21T09:31:07.473 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:07.473 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:07.473 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:07.473 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:07.473 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:07.474 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:07.474 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:07.474 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:07.474 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "2.11", 2024-09-21T09:31:07.474 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:07.474 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:07.474 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:07.474 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 24, 2024-09-21T09:31:07.474 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:07.474 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T11:28:03.705631+0000", 2024-09-21T09:31:07.474 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:07.474 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:07.474 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.474 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:07.475 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.475 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.475 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.475 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.475 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.475 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.475 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.475 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.475 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.475 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.475 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.475 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.475 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.475 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.476 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.476 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:07.476 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:07.476 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.476 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:07.476 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.476 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.476 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.476 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.476 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.476 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:07.476 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.476 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.476 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.477 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.477 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.477 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.477 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.477 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.477 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.477 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.477 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.477 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.477 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:07.477 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:07.477 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.477 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:07.477 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:07.478 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:07.478 INFO:tasks.workunit.client.0.smithi022.stdout: 6, 2024-09-21T09:31:07.478 INFO:tasks.workunit.client.0.smithi022.stdout: 4, 2024-09-21T09:31:07.478 INFO:tasks.workunit.client.0.smithi022.stdout: 1 2024-09-21T09:31:07.478 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.478 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 6, 2024-09-21T09:31:07.478 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:07.478 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.478 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.478 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:07.478 INFO:tasks.workunit.client.0.smithi022.stdout: 6, 2024-09-21T09:31:07.478 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:07.478 INFO:tasks.workunit.client.0.smithi022.stdout: 5 2024-09-21T09:31:07.478 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.479 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 6, 2024-09-21T09:31:07.479 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:07.479 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:07.479 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:07.479 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:07.479 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:07.479 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:07.479 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.483855+0000", 2024-09-21T09:31:07.479 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.640996+0000", 2024-09-21T09:31:07.479 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.640996+0000", 2024-09-21T09:31:07.479 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.641508+0000", 2024-09-21T09:31:07.479 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.483855+0000", 2024-09-21T09:31:07.479 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.479 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:07.479 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.480 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:07.480 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.483855+0000", 2024-09-21T09:31:07.480 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.483855+0000", 2024-09-21T09:31:07.480 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.483855+0000", 2024-09-21T09:31:07.480 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:07.480 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:07.480 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.480 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.483855+0000", 2024-09-21T09:31:07.480 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.483855+0000", 2024-09-21T09:31:07.480 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:07.480 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:07.480 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:07.480 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:07.481 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:07.481 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:07.481 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:07.481 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:07.481 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:07.481 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:07.481 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:07.481 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:07.481 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:07.481 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.10", 2024-09-21T09:31:07.481 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:07.481 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:07.481 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:07.481 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:07.481 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:07.482 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T12:02:24.674295+0000", 2024-09-21T09:31:07.482 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:07.482 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:07.482 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.482 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:07.482 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.482 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.482 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.482 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.482 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.482 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.482 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.482 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.482 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.483 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.483 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.483 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.483 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.483 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.483 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.483 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:07.483 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:07.483 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.483 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:07.483 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.483 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.483 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.483 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.484 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.484 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:07.484 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.484 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.484 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.484 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.484 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.484 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.484 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.484 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.484 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.484 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.484 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.484 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.484 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:07.485 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:07.485 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.485 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:07.485 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:07.485 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:07.485 INFO:tasks.workunit.client.0.smithi022.stdout: 6, 2024-09-21T09:31:07.485 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:07.485 INFO:tasks.workunit.client.0.smithi022.stdout: 5 2024-09-21T09:31:07.485 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.485 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 6, 2024-09-21T09:31:07.485 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:07.485 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.485 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.486 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:07.486 INFO:tasks.workunit.client.0.smithi022.stdout: 2, 2024-09-21T09:31:07.486 INFO:tasks.workunit.client.0.smithi022.stdout: 1, 2024-09-21T09:31:07.486 INFO:tasks.workunit.client.0.smithi022.stdout: 0 2024-09-21T09:31:07.486 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.486 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 2, 2024-09-21T09:31:07.486 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:07.486 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:07.486 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 51, 2024-09-21T09:31:07.486 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:07.486 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:07.486 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:07.486 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.061786+0000", 2024-09-21T09:31:07.487 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:32.261324+0000", 2024-09-21T09:31:07.487 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:32.261324+0000", 2024-09-21T09:31:07.487 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:32.264620+0000", 2024-09-21T09:31:07.487 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.061786+0000", 2024-09-21T09:31:07.487 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:07.487 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:07.487 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:07.487 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 52, 2024-09-21T09:31:07.487 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.061786+0000", 2024-09-21T09:31:07.487 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.061786+0000", 2024-09-21T09:31:07.487 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.061786+0000", 2024-09-21T09:31:07.487 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:07.487 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:07.487 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:07.488 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.061786+0000", 2024-09-21T09:31:07.488 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.061786+0000", 2024-09-21T09:31:07.488 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:07.488 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 8, 2024-09-21T09:31:07.488 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:07.488 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:07.488 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 51, 2024-09-21T09:31:07.488 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:07.488 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:07.488 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:07.488 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:07.488 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 8, 2024-09-21T09:31:07.488 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:07.488 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:07.488 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:07.489 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "2.10", 2024-09-21T09:31:07.489 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:07.489 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:07.489 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:07.489 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 32, 2024-09-21T09:31:07.489 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:07.489 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T14:11:04.933873+0000", 2024-09-21T09:31:07.489 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:07.489 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:07.489 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.489 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:07.489 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.489 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.489 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.490 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.490 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.490 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.490 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.490 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.490 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.490 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.490 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.490 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.490 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.490 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.490 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.490 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 12, 2024-09-21T09:31:07.490 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 4, 2024-09-21T09:31:07.491 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.491 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 4, 2024-09-21T09:31:07.491 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.491 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.491 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.491 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.491 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.491 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 4, 2024-09-21T09:31:07.491 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.491 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.491 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.491 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.491 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.491 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.491 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.492 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.492 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.492 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.492 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.492 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.492 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 12, 2024-09-21T09:31:07.492 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:07.492 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.492 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:07.492 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:07.492 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:07.492 INFO:tasks.workunit.client.0.smithi022.stdout: 2, 2024-09-21T09:31:07.492 INFO:tasks.workunit.client.0.smithi022.stdout: 1, 2024-09-21T09:31:07.493 INFO:tasks.workunit.client.0.smithi022.stdout: 0 2024-09-21T09:31:07.493 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.493 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 2, 2024-09-21T09:31:07.493 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "57'8" 2024-09-21T09:31:07.493 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.493 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.493 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:07.493 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:07.493 INFO:tasks.workunit.client.0.smithi022.stdout: 4, 2024-09-21T09:31:07.493 INFO:tasks.workunit.client.0.smithi022.stdout: 6 2024-09-21T09:31:07.493 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.493 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 7, 2024-09-21T09:31:07.493 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:07.493 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:07.494 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:07.494 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:07.494 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:07.494 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:07.494 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.358860+0000", 2024-09-21T09:31:07.494 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.641261+0000", 2024-09-21T09:31:07.494 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.641261+0000", 2024-09-21T09:31:07.494 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.641576+0000", 2024-09-21T09:31:07.494 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.358860+0000", 2024-09-21T09:31:07.494 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.494 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:07.494 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.494 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:07.495 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.358860+0000", 2024-09-21T09:31:07.495 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.358860+0000", 2024-09-21T09:31:07.495 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.358860+0000", 2024-09-21T09:31:07.495 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:07.495 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:07.495 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.495 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.358860+0000", 2024-09-21T09:31:07.495 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.358860+0000", 2024-09-21T09:31:07.495 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:07.495 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:07.495 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:07.495 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:07.495 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:07.495 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:07.496 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:07.496 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:07.496 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:07.496 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:07.496 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:07.496 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:07.496 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:07.496 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.11", 2024-09-21T09:31:07.496 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:07.496 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:07.496 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:07.496 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:07.497 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:07.497 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T14:59:48.868788+0000", 2024-09-21T09:31:07.497 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:07.497 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:07.497 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.497 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:07.497 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.497 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.497 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.497 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.497 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.497 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.497 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.497 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.498 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.498 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.498 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.498 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.498 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.498 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.498 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.498 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:07.498 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:07.498 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.498 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:07.498 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.499 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.499 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.499 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.499 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.499 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:07.499 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.499 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.499 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.499 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.499 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.499 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.499 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.499 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.500 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.500 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.500 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.500 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.500 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:07.500 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:07.500 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.500 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:07.500 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:07.500 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:07.500 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:07.500 INFO:tasks.workunit.client.0.smithi022.stdout: 4, 2024-09-21T09:31:07.500 INFO:tasks.workunit.client.0.smithi022.stdout: 6 2024-09-21T09:31:07.500 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.501 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 7, 2024-09-21T09:31:07.501 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:07.501 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.501 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.501 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:07.501 INFO:tasks.workunit.client.0.smithi022.stdout: 4, 2024-09-21T09:31:07.501 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:07.501 INFO:tasks.workunit.client.0.smithi022.stdout: 7 2024-09-21T09:31:07.501 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.501 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 4, 2024-09-21T09:31:07.501 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:07.501 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:07.501 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 51, 2024-09-21T09:31:07.501 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:07.501 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:07.502 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:07.502 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.062869+0000", 2024-09-21T09:31:07.502 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:32.269748+0000", 2024-09-21T09:31:07.502 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:32.269748+0000", 2024-09-21T09:31:07.502 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:32.270055+0000", 2024-09-21T09:31:07.502 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.062869+0000", 2024-09-21T09:31:07.502 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:07.502 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:07.502 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:07.502 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 52, 2024-09-21T09:31:07.502 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.062869+0000", 2024-09-21T09:31:07.502 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.062869+0000", 2024-09-21T09:31:07.502 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.062869+0000", 2024-09-21T09:31:07.502 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:07.503 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:07.503 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:07.503 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.062869+0000", 2024-09-21T09:31:07.503 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.062869+0000", 2024-09-21T09:31:07.503 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:07.503 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 1, 2024-09-21T09:31:07.503 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:07.503 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:07.503 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 51, 2024-09-21T09:31:07.503 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:07.503 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:07.503 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:07.503 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:07.503 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 1, 2024-09-21T09:31:07.504 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:07.504 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:07.504 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:07.504 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "2.f", 2024-09-21T09:31:07.504 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:07.504 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:07.504 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:07.504 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 25, 2024-09-21T09:31:07.504 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:07.504 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T14:50:54.853319+0000", 2024-09-21T09:31:07.504 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:07.504 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:07.504 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.504 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 22, 2024-09-21T09:31:07.505 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.505 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.505 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.505 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.505 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.505 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.505 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.505 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.505 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.505 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.505 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.505 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.505 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.505 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.505 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.506 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 3, 2024-09-21T09:31:07.506 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 1, 2024-09-21T09:31:07.506 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.506 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 1, 2024-09-21T09:31:07.506 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.506 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.506 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.506 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.506 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.506 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:07.506 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.506 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.506 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.506 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.507 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.507 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.507 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.507 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.507 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.507 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.507 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.507 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.507 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 1, 2024-09-21T09:31:07.507 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 1 2024-09-21T09:31:07.507 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.507 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:07.507 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:07.507 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:07.508 INFO:tasks.workunit.client.0.smithi022.stdout: 4, 2024-09-21T09:31:07.508 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:07.508 INFO:tasks.workunit.client.0.smithi022.stdout: 7 2024-09-21T09:31:07.508 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.508 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 4, 2024-09-21T09:31:07.508 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "57'1" 2024-09-21T09:31:07.508 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.508 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.508 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:07.508 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:07.508 INFO:tasks.workunit.client.0.smithi022.stdout: 4, 2024-09-21T09:31:07.508 INFO:tasks.workunit.client.0.smithi022.stdout: 1 2024-09-21T09:31:07.508 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.508 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 7, 2024-09-21T09:31:07.509 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:07.509 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:07.509 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:07.509 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:07.509 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:07.509 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:07.509 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.359401+0000", 2024-09-21T09:31:07.509 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.641348+0000", 2024-09-21T09:31:07.509 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.641348+0000", 2024-09-21T09:31:07.509 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.641790+0000", 2024-09-21T09:31:07.509 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.359401+0000", 2024-09-21T09:31:07.509 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.509 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:07.509 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.510 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:07.510 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.359401+0000", 2024-09-21T09:31:07.510 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.359401+0000", 2024-09-21T09:31:07.510 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.359401+0000", 2024-09-21T09:31:07.510 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:07.510 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:07.510 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.510 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.359401+0000", 2024-09-21T09:31:07.510 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.359401+0000", 2024-09-21T09:31:07.510 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:07.510 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:07.510 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:07.510 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:07.511 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:07.511 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:07.511 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:07.511 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:07.511 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:07.511 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:07.511 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:07.511 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:07.511 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:07.511 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.e", 2024-09-21T09:31:07.511 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:07.511 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:07.511 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:07.511 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:07.512 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:07.512 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T18:25:10.635353+0000", 2024-09-21T09:31:07.512 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:07.512 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:07.512 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.512 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:07.512 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.512 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.512 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.512 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.512 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.512 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.512 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.512 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.512 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.513 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.513 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.513 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.513 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.513 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.513 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.513 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:07.513 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:07.513 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.513 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:07.513 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.513 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.513 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.513 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.514 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.514 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:07.514 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.514 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.514 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.514 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.514 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.514 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.514 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.514 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.514 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.514 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.515 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.515 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.515 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:07.515 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:07.515 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.515 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:07.515 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:07.515 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:07.515 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:07.515 INFO:tasks.workunit.client.0.smithi022.stdout: 4, 2024-09-21T09:31:07.515 INFO:tasks.workunit.client.0.smithi022.stdout: 1 2024-09-21T09:31:07.515 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.515 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 7, 2024-09-21T09:31:07.516 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:07.516 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.516 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.516 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:07.516 INFO:tasks.workunit.client.0.smithi022.stdout: 2, 2024-09-21T09:31:07.516 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:07.516 INFO:tasks.workunit.client.0.smithi022.stdout: 7 2024-09-21T09:31:07.516 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.516 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 2, 2024-09-21T09:31:07.516 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:07.516 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:07.516 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 51, 2024-09-21T09:31:07.516 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:07.516 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:07.517 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:07.517 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.062124+0000", 2024-09-21T09:31:07.517 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:32.263984+0000", 2024-09-21T09:31:07.517 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:32.263984+0000", 2024-09-21T09:31:07.517 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:32.264276+0000", 2024-09-21T09:31:07.517 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.062124+0000", 2024-09-21T09:31:07.517 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:07.517 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:07.517 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:07.517 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 52, 2024-09-21T09:31:07.517 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.062124+0000", 2024-09-21T09:31:07.517 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.062124+0000", 2024-09-21T09:31:07.517 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.062124+0000", 2024-09-21T09:31:07.517 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:07.518 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:07.518 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:07.518 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.062124+0000", 2024-09-21T09:31:07.518 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.062124+0000", 2024-09-21T09:31:07.518 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:07.518 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 2, 2024-09-21T09:31:07.518 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:07.518 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:07.518 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 51, 2024-09-21T09:31:07.518 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:07.518 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:07.518 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:07.518 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:07.519 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 2, 2024-09-21T09:31:07.519 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:07.519 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:07.519 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:07.519 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "2.e", 2024-09-21T09:31:07.519 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:07.519 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:07.519 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:07.519 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 26, 2024-09-21T09:31:07.519 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:07.519 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T17:18:50.045457+0000", 2024-09-21T09:31:07.519 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:07.519 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:07.519 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.519 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:07.520 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.520 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.520 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.520 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.520 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.520 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.520 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.520 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.520 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.520 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.520 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.520 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.520 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.521 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.521 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.521 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 3, 2024-09-21T09:31:07.521 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 1, 2024-09-21T09:31:07.521 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.521 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 1, 2024-09-21T09:31:07.521 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.521 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.521 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.521 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.521 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.521 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 1, 2024-09-21T09:31:07.521 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.521 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.521 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.522 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.522 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.522 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.522 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.522 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.522 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.522 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.522 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.522 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.522 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 3, 2024-09-21T09:31:07.522 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:07.522 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.522 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:07.522 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:07.523 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:07.523 INFO:tasks.workunit.client.0.smithi022.stdout: 2, 2024-09-21T09:31:07.523 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:07.523 INFO:tasks.workunit.client.0.smithi022.stdout: 7 2024-09-21T09:31:07.523 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.523 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 2, 2024-09-21T09:31:07.523 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "57'2" 2024-09-21T09:31:07.523 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.523 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.523 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:07.523 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:07.523 INFO:tasks.workunit.client.0.smithi022.stdout: 4, 2024-09-21T09:31:07.523 INFO:tasks.workunit.client.0.smithi022.stdout: 0 2024-09-21T09:31:07.523 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.524 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 7, 2024-09-21T09:31:07.524 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:07.524 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:07.524 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:07.524 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:07.524 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:07.524 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:07.524 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.359338+0000", 2024-09-21T09:31:07.524 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.649033+0000", 2024-09-21T09:31:07.524 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.649033+0000", 2024-09-21T09:31:07.524 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.649353+0000", 2024-09-21T09:31:07.524 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.359338+0000", 2024-09-21T09:31:07.524 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.524 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:07.525 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.525 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:07.525 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.359338+0000", 2024-09-21T09:31:07.525 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.359338+0000", 2024-09-21T09:31:07.525 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.359338+0000", 2024-09-21T09:31:07.525 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:07.525 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:07.525 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.525 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.359338+0000", 2024-09-21T09:31:07.525 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.359338+0000", 2024-09-21T09:31:07.525 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:07.525 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:07.526 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:07.526 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:07.526 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:07.526 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:07.526 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:07.526 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:07.526 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:07.526 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:07.526 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:07.526 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:07.526 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:07.526 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.f", 2024-09-21T09:31:07.526 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:07.527 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:07.527 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:07.527 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:07.527 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:07.527 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T15:13:12.611019+0000", 2024-09-21T09:31:07.527 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:07.527 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:07.527 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.527 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:07.527 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.527 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.527 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.527 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.527 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.528 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.528 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.528 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.528 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.528 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.528 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.528 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.528 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.528 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.528 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.528 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:07.529 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:07.529 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.529 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:07.529 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.529 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.529 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.529 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.529 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.529 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:07.529 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.529 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.529 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.530 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.530 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.530 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.530 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.530 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.530 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.530 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.530 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.530 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.530 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:07.530 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:07.530 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.530 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:07.530 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:07.531 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:07.531 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:07.531 INFO:tasks.workunit.client.0.smithi022.stdout: 4, 2024-09-21T09:31:07.531 INFO:tasks.workunit.client.0.smithi022.stdout: 0 2024-09-21T09:31:07.531 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.531 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 7, 2024-09-21T09:31:07.531 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:07.531 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.531 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.531 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:07.531 INFO:tasks.workunit.client.0.smithi022.stdout: 1, 2024-09-21T09:31:07.531 INFO:tasks.workunit.client.0.smithi022.stdout: 4, 2024-09-21T09:31:07.531 INFO:tasks.workunit.client.0.smithi022.stdout: 3 2024-09-21T09:31:07.532 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.532 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 1, 2024-09-21T09:31:07.532 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:07.532 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:07.532 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 51, 2024-09-21T09:31:07.532 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:07.532 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:07.532 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:07.532 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.062217+0000", 2024-09-21T09:31:07.532 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:32.259209+0000", 2024-09-21T09:31:07.532 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:32.259209+0000", 2024-09-21T09:31:07.532 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:32.260033+0000", 2024-09-21T09:31:07.532 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.062217+0000", 2024-09-21T09:31:07.532 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:07.533 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:07.533 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:07.533 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 52, 2024-09-21T09:31:07.533 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.062217+0000", 2024-09-21T09:31:07.533 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.062217+0000", 2024-09-21T09:31:07.533 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.062217+0000", 2024-09-21T09:31:07.533 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:07.533 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:07.533 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:07.533 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.062217+0000", 2024-09-21T09:31:07.533 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.062217+0000", 2024-09-21T09:31:07.533 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:07.533 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 1, 2024-09-21T09:31:07.534 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:07.534 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:07.534 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 51, 2024-09-21T09:31:07.534 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:07.534 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:07.534 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:07.534 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:07.534 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 1, 2024-09-21T09:31:07.534 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:07.534 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:07.534 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:07.534 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "2.d", 2024-09-21T09:31:07.534 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:07.534 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:07.535 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:07.535 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 25, 2024-09-21T09:31:07.535 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:07.535 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T21:16:09.835879+0000", 2024-09-21T09:31:07.535 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:07.535 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:07.535 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.535 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 70, 2024-09-21T09:31:07.535 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.535 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.535 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.535 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.535 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.535 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.536 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.536 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.536 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.536 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.536 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.536 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.536 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.536 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.536 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.536 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 3, 2024-09-21T09:31:07.536 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 1, 2024-09-21T09:31:07.536 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.536 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 1, 2024-09-21T09:31:07.536 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.537 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.537 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.537 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.537 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.537 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:07.537 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.537 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.537 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.537 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.537 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.537 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.537 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.537 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.537 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.538 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.538 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.538 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.538 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 1, 2024-09-21T09:31:07.538 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 1 2024-09-21T09:31:07.538 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.538 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:07.538 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:07.538 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:07.538 INFO:tasks.workunit.client.0.smithi022.stdout: 1, 2024-09-21T09:31:07.538 INFO:tasks.workunit.client.0.smithi022.stdout: 4, 2024-09-21T09:31:07.538 INFO:tasks.workunit.client.0.smithi022.stdout: 3 2024-09-21T09:31:07.539 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.539 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 1, 2024-09-21T09:31:07.539 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "57'1" 2024-09-21T09:31:07.539 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.539 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.539 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:07.539 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:07.539 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:07.539 INFO:tasks.workunit.client.0.smithi022.stdout: 6 2024-09-21T09:31:07.539 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.539 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 5, 2024-09-21T09:31:07.539 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:07.539 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:07.539 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:07.540 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:07.540 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:07.540 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:07.540 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.066397+0000", 2024-09-21T09:31:07.540 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.638802+0000", 2024-09-21T09:31:07.540 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.638802+0000", 2024-09-21T09:31:07.540 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.639076+0000", 2024-09-21T09:31:07.540 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.066397+0000", 2024-09-21T09:31:07.540 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.540 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:07.540 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.540 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:07.540 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.066397+0000", 2024-09-21T09:31:07.540 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.066397+0000", 2024-09-21T09:31:07.541 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.066397+0000", 2024-09-21T09:31:07.541 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:07.541 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:07.541 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.541 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.066397+0000", 2024-09-21T09:31:07.541 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.066397+0000", 2024-09-21T09:31:07.541 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:07.541 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:07.541 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:07.541 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:07.541 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:07.541 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:07.541 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:07.542 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:07.542 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:07.542 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:07.542 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:07.542 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:07.542 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:07.542 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.c", 2024-09-21T09:31:07.542 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:07.542 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:07.542 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:07.542 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:07.542 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:07.542 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T17:36:18.524320+0000", 2024-09-21T09:31:07.542 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:07.543 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:07.543 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.543 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:07.543 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.543 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.543 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.543 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.543 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.543 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.543 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.543 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.543 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.543 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.544 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.544 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.544 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.544 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.544 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.544 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:07.544 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:07.544 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.544 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:07.544 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.544 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.544 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.544 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.545 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.545 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:07.545 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.545 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.545 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.545 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.545 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.545 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.545 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.545 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.545 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.545 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.545 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.546 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.546 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:07.546 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:07.546 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.546 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:07.546 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:07.546 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:07.546 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:07.546 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:07.546 INFO:tasks.workunit.client.0.smithi022.stdout: 6 2024-09-21T09:31:07.546 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.546 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 5, 2024-09-21T09:31:07.546 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:07.547 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.547 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.547 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:07.547 INFO:tasks.workunit.client.0.smithi022.stdout: 2, 2024-09-21T09:31:07.547 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:07.547 INFO:tasks.workunit.client.0.smithi022.stdout: 0 2024-09-21T09:31:07.547 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.547 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 2, 2024-09-21T09:31:07.547 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:07.547 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:07.547 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 51, 2024-09-21T09:31:07.547 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:07.547 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:07.547 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:07.548 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.062156+0000", 2024-09-21T09:31:07.548 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:32.269717+0000", 2024-09-21T09:31:07.548 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:32.269717+0000", 2024-09-21T09:31:07.548 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:32.270060+0000", 2024-09-21T09:31:07.548 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.062156+0000", 2024-09-21T09:31:07.548 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:07.548 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:07.548 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:07.548 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 52, 2024-09-21T09:31:07.548 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.062156+0000", 2024-09-21T09:31:07.548 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.062156+0000", 2024-09-21T09:31:07.549 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.062156+0000", 2024-09-21T09:31:07.549 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:07.549 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:07.549 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:07.549 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.062156+0000", 2024-09-21T09:31:07.549 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.062156+0000", 2024-09-21T09:31:07.549 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:07.549 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:07.549 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:07.549 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:07.549 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 51, 2024-09-21T09:31:07.549 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:07.549 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:07.550 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:07.550 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:07.550 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:07.550 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:07.550 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:07.550 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:07.550 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "2.c", 2024-09-21T09:31:07.550 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:07.550 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:07.550 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:07.550 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 24, 2024-09-21T09:31:07.550 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:07.550 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T19:59:27.725031+0000", 2024-09-21T09:31:07.551 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:07.551 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:07.551 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.551 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:07.551 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.551 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.551 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.551 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.551 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.551 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.551 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.551 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.551 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.551 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.551 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.552 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.552 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.552 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.552 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.552 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:07.552 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:07.552 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.552 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:07.552 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.552 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.552 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.552 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.552 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.552 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:07.552 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.553 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.553 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.553 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.553 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.553 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.553 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.553 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.553 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.553 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.553 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.553 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.553 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:07.553 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:07.553 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.554 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:07.554 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:07.554 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:07.554 INFO:tasks.workunit.client.0.smithi022.stdout: 2, 2024-09-21T09:31:07.554 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:07.554 INFO:tasks.workunit.client.0.smithi022.stdout: 0 2024-09-21T09:31:07.554 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.554 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 2, 2024-09-21T09:31:07.554 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:07.554 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.554 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.554 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:07.554 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:07.555 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:07.555 INFO:tasks.workunit.client.0.smithi022.stdout: 6 2024-09-21T09:31:07.555 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.555 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 7, 2024-09-21T09:31:07.555 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:07.555 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:07.555 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:07.555 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:07.555 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:07.555 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:07.555 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.359314+0000", 2024-09-21T09:31:07.555 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.639746+0000", 2024-09-21T09:31:07.555 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.639746+0000", 2024-09-21T09:31:07.555 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.640059+0000", 2024-09-21T09:31:07.556 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.359314+0000", 2024-09-21T09:31:07.556 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.556 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:07.556 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.556 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:07.556 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.359314+0000", 2024-09-21T09:31:07.556 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.359314+0000", 2024-09-21T09:31:07.556 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.359314+0000", 2024-09-21T09:31:07.556 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:07.556 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:07.556 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.556 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.359314+0000", 2024-09-21T09:31:07.556 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.359314+0000", 2024-09-21T09:31:07.556 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:07.557 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:07.557 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:07.557 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:07.557 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:07.557 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:07.557 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:07.557 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:07.557 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:07.557 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:07.557 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:07.557 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:07.557 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:07.557 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.d", 2024-09-21T09:31:07.557 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:07.558 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:07.558 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:07.558 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:07.558 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:07.558 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T11:33:15.402410+0000", 2024-09-21T09:31:07.558 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:07.558 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:07.558 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.558 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:07.558 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.558 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.558 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.558 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.558 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.559 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.559 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.559 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.559 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.559 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.559 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.559 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.559 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.559 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.559 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.559 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:07.559 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:07.559 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.559 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:07.560 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.560 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.560 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.560 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.560 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.560 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:07.560 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.560 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.560 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.560 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.560 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.560 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.560 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.560 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.561 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.561 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.561 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.561 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.561 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:07.561 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:07.561 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.561 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:07.561 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:07.561 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:07.561 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:07.561 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:07.561 INFO:tasks.workunit.client.0.smithi022.stdout: 6 2024-09-21T09:31:07.561 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.562 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 7, 2024-09-21T09:31:07.562 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:07.562 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.562 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.562 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:07.562 INFO:tasks.workunit.client.0.smithi022.stdout: 6, 2024-09-21T09:31:07.562 INFO:tasks.workunit.client.0.smithi022.stdout: 4, 2024-09-21T09:31:07.562 INFO:tasks.workunit.client.0.smithi022.stdout: 7 2024-09-21T09:31:07.562 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.562 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 6, 2024-09-21T09:31:07.562 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:07.562 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:07.562 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 61, 2024-09-21T09:31:07.563 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:07.563 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:07.563 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:07.563 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:41.371073+0000", 2024-09-21T09:31:07.563 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:28:28.844168+0000", 2024-09-21T09:31:07.563 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:28:28.844168+0000", 2024-09-21T09:31:07.563 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:28:33.484488+0000", 2024-09-21T09:31:07.563 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:41.371073+0000", 2024-09-21T09:31:07.563 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:28:27.830318+0000", 2024-09-21T09:31:07.563 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:07.563 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:28:27.830318+0000", 2024-09-21T09:31:07.564 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 62, 2024-09-21T09:31:07.564 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:41.371073+0000", 2024-09-21T09:31:07.564 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:41.371073+0000", 2024-09-21T09:31:07.564 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:41.371073+0000", 2024-09-21T09:31:07.564 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:07.564 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:07.564 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:28:27.830318+0000", 2024-09-21T09:31:07.564 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:41.371073+0000", 2024-09-21T09:31:07.564 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:41.371073+0000", 2024-09-21T09:31:07.564 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:07.564 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:07.564 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:07.564 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:07.564 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 61, 2024-09-21T09:31:07.564 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:07.565 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:07.565 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:07.565 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:07.565 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:07.565 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:07.565 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:07.565 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:07.565 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "4.5", 2024-09-21T09:31:07.565 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:07.565 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:07.565 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 67, 2024-09-21T09:31:07.565 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 19, 2024-09-21T09:31:07.565 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:07.565 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T14:07:12.698450+0000", 2024-09-21T09:31:07.565 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.000304273, 2024-09-21T09:31:07.566 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:07.566 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.566 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:07.566 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.566 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.566 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.566 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.566 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.566 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.566 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.566 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.566 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.566 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.566 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.567 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.567 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.567 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.567 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.567 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:07.567 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:07.567 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.567 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:07.567 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.567 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.567 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.567 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.567 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.567 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:07.568 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.568 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.568 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.568 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.568 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.568 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.568 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.568 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.568 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.568 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.568 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.568 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.568 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:07.568 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:07.568 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.569 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:07.569 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:07.569 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:07.569 INFO:tasks.workunit.client.0.smithi022.stdout: 6, 2024-09-21T09:31:07.569 INFO:tasks.workunit.client.0.smithi022.stdout: 4, 2024-09-21T09:31:07.569 INFO:tasks.workunit.client.0.smithi022.stdout: 7 2024-09-21T09:31:07.569 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.569 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 6, 2024-09-21T09:31:07.569 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:07.569 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.569 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.569 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:07.569 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:07.569 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:07.570 INFO:tasks.workunit.client.0.smithi022.stdout: 6 2024-09-21T09:31:07.570 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.570 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 7, 2024-09-21T09:31:07.570 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:07.570 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:07.570 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 19, 2024-09-21T09:31:07.570 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:07.570 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:07.570 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:07.570 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.359151+0000", 2024-09-21T09:31:07.570 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:08.454770+0000", 2024-09-21T09:31:07.570 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:08.454770+0000", 2024-09-21T09:31:07.571 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:08.460023+0000", 2024-09-21T09:31:07.571 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.359151+0000", 2024-09-21T09:31:07.571 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:25:07.646659+0000", 2024-09-21T09:31:07.571 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:07.571 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:25:07.646659+0000", 2024-09-21T09:31:07.571 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 50, 2024-09-21T09:31:07.571 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.359151+0000", 2024-09-21T09:31:07.571 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.359151+0000", 2024-09-21T09:31:07.571 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.359151+0000", 2024-09-21T09:31:07.571 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:07.571 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:07.571 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:25:07.646659+0000", 2024-09-21T09:31:07.571 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.359151+0000", 2024-09-21T09:31:07.571 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.359151+0000", 2024-09-21T09:31:07.572 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:07.572 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 79, 2024-09-21T09:31:07.572 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:07.572 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:07.572 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 49, 2024-09-21T09:31:07.572 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:07.572 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:07.572 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:07.572 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:07.572 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 79, 2024-09-21T09:31:07.572 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:07.572 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:07.572 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:07.572 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "1.0", 2024-09-21T09:31:07.573 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:07.573 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:07.573 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:07.573 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 58, 2024-09-21T09:31:07.573 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:07.573 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T19:08:45.658466+0000", 2024-09-21T09:31:07.573 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:07.573 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:07.573 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.573 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 590368, 2024-09-21T09:31:07.573 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.573 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 590368, 2024-09-21T09:31:07.573 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.573 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.573 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.574 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.574 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.574 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.574 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.574 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.574 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.574 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.574 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.574 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.574 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.574 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 6, 2024-09-21T09:31:07.574 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 2, 2024-09-21T09:31:07.574 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.574 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 2, 2024-09-21T09:31:07.575 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.575 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.575 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.575 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.575 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.575 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:07.575 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.575 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 2, 2024-09-21T09:31:07.575 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.575 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.575 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.575 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.575 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.575 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 126, 2024-09-21T09:31:07.576 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 231, 2024-09-21T09:31:07.576 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.576 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.576 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.576 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 113, 2024-09-21T09:31:07.576 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 1628 2024-09-21T09:31:07.576 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.576 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:07.576 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:07.576 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:07.576 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:07.576 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:07.576 INFO:tasks.workunit.client.0.smithi022.stdout: 6 2024-09-21T09:31:07.576 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.576 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 7, 2024-09-21T09:31:07.577 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "62'79" 2024-09-21T09:31:07.577 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.577 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.577 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:07.577 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:07.577 INFO:tasks.workunit.client.0.smithi022.stdout: 2, 2024-09-21T09:31:07.577 INFO:tasks.workunit.client.0.smithi022.stdout: 4 2024-09-21T09:31:07.577 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.577 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 5, 2024-09-21T09:31:07.577 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:07.577 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:07.577 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 51, 2024-09-21T09:31:07.577 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:07.577 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:07.578 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:07.578 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.066105+0000", 2024-09-21T09:31:07.578 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:52.080680+0000", 2024-09-21T09:31:07.578 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:52.080680+0000", 2024-09-21T09:31:07.578 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:52.081088+0000", 2024-09-21T09:31:07.578 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.066105+0000", 2024-09-21T09:31:07.578 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:07.578 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:07.578 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:07.578 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 59, 2024-09-21T09:31:07.578 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.066105+0000", 2024-09-21T09:31:07.578 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.066105+0000", 2024-09-21T09:31:07.578 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.066105+0000", 2024-09-21T09:31:07.578 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:07.579 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:07.579 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:07.579 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.066105+0000", 2024-09-21T09:31:07.579 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.066105+0000", 2024-09-21T09:31:07.579 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:07.579 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:07.579 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:07.579 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:07.579 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 58, 2024-09-21T09:31:07.579 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:07.579 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:07.579 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:07.579 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:07.579 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:07.580 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:07.580 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:07.580 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:07.580 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "2.3", 2024-09-21T09:31:07.580 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:07.580 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:07.580 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:07.580 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 34, 2024-09-21T09:31:07.580 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:07.580 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T14:01:04.317396+0000", 2024-09-21T09:31:07.580 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:07.580 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:07.580 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.580 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:07.581 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.581 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.581 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.581 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.581 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.581 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.581 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.581 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.581 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.581 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.581 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.581 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.581 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.581 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.582 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.582 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:07.582 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:07.582 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.582 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:07.582 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.582 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.582 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.582 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.582 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.582 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:07.582 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.582 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.582 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.583 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.583 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.583 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.583 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.583 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.583 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.583 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.583 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.583 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.583 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:07.583 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:07.583 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.583 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:07.583 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:07.584 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:07.584 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:07.584 INFO:tasks.workunit.client.0.smithi022.stdout: 2, 2024-09-21T09:31:07.584 INFO:tasks.workunit.client.0.smithi022.stdout: 4 2024-09-21T09:31:07.584 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.584 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 5, 2024-09-21T09:31:07.584 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:07.584 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.584 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.584 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:07.584 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:07.584 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:07.584 INFO:tasks.workunit.client.0.smithi022.stdout: 6 2024-09-21T09:31:07.585 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.585 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 3, 2024-09-21T09:31:07.585 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:07.585 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:07.585 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:07.585 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:07.585 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:07.585 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:07.585 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.125141+0000", 2024-09-21T09:31:07.585 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.635939+0000", 2024-09-21T09:31:07.585 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.635939+0000", 2024-09-21T09:31:07.585 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.636200+0000", 2024-09-21T09:31:07.585 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.125141+0000", 2024-09-21T09:31:07.585 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.586 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:07.586 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.586 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:07.586 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.125141+0000", 2024-09-21T09:31:07.586 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.125141+0000", 2024-09-21T09:31:07.586 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.125141+0000", 2024-09-21T09:31:07.586 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:07.586 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:07.586 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.586 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.125141+0000", 2024-09-21T09:31:07.586 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.125141+0000", 2024-09-21T09:31:07.586 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:07.587 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:07.587 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:07.587 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:07.587 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:07.587 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:07.587 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:07.587 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:07.587 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:07.587 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:07.587 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:07.587 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:07.587 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:07.587 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.2", 2024-09-21T09:31:07.587 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:07.588 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:07.588 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:07.588 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:07.588 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:07.588 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T12:02:06.551561+0000", 2024-09-21T09:31:07.588 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:07.588 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:07.588 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.588 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:07.588 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.588 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.588 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.588 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.588 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.589 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.589 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.589 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.589 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.589 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.589 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.589 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.589 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.589 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.589 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.589 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:07.589 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:07.589 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.590 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:07.590 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.590 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.590 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.590 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.590 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.590 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:07.590 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.590 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.590 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.590 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.590 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.590 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.590 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.591 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.591 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.591 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.591 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.591 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.591 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:07.591 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:07.591 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.591 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:07.591 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:07.591 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:07.591 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:07.591 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:07.592 INFO:tasks.workunit.client.0.smithi022.stdout: 6 2024-09-21T09:31:07.592 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.592 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 3, 2024-09-21T09:31:07.592 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:07.592 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.592 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.592 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:07.592 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:07.592 INFO:tasks.workunit.client.0.smithi022.stdout: 1, 2024-09-21T09:31:07.592 INFO:tasks.workunit.client.0.smithi022.stdout: 2 2024-09-21T09:31:07.592 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.592 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 0, 2024-09-21T09:31:07.592 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:07.593 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:07.593 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 61, 2024-09-21T09:31:07.593 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:07.593 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:07.593 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:07.593 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:40.923121+0000", 2024-09-21T09:31:07.593 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:28:28.846894+0000", 2024-09-21T09:31:07.593 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:28:28.846894+0000", 2024-09-21T09:31:07.593 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:28:33.141556+0000", 2024-09-21T09:31:07.593 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:40.923121+0000", 2024-09-21T09:31:07.593 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:28:27.830318+0000", 2024-09-21T09:31:07.593 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:07.593 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:28:27.830318+0000", 2024-09-21T09:31:07.593 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 62, 2024-09-21T09:31:07.593 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:40.923121+0000", 2024-09-21T09:31:07.593 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:40.923121+0000", 2024-09-21T09:31:07.594 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:40.923121+0000", 2024-09-21T09:31:07.594 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:07.594 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:07.594 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:28:27.830318+0000", 2024-09-21T09:31:07.594 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:40.923121+0000", 2024-09-21T09:31:07.594 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:40.923121+0000", 2024-09-21T09:31:07.594 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:07.594 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:07.594 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:07.594 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:07.594 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 61, 2024-09-21T09:31:07.594 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:07.594 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:07.594 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:07.595 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:07.595 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:07.595 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:07.595 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:07.595 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:07.595 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "4.6", 2024-09-21T09:31:07.595 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:07.595 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:07.595 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 67, 2024-09-21T09:31:07.595 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 19, 2024-09-21T09:31:07.595 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:07.595 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T10:54:49.861215+0000", 2024-09-21T09:31:07.595 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.000276941, 2024-09-21T09:31:07.595 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:07.595 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.596 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:07.596 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.596 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.596 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.596 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.596 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.596 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.596 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.596 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.596 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.596 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.596 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.596 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.597 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.597 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.597 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.597 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:07.597 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:07.597 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.597 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:07.597 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.597 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.597 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.597 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.597 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.597 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:07.597 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.598 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.598 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.598 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.598 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.598 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.598 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.598 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.598 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.598 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.598 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.598 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.598 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:07.598 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:07.598 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.599 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:07.599 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:07.599 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:07.599 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:07.599 INFO:tasks.workunit.client.0.smithi022.stdout: 1, 2024-09-21T09:31:07.599 INFO:tasks.workunit.client.0.smithi022.stdout: 2 2024-09-21T09:31:07.599 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.599 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 0, 2024-09-21T09:31:07.599 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:07.599 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.599 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.599 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:07.599 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:07.599 INFO:tasks.workunit.client.0.smithi022.stdout: 1, 2024-09-21T09:31:07.600 INFO:tasks.workunit.client.0.smithi022.stdout: 0 2024-09-21T09:31:07.600 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.600 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 7, 2024-09-21T09:31:07.600 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:07.600 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:07.600 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 51, 2024-09-21T09:31:07.600 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:07.600 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:07.600 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:07.600 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.359086+0000", 2024-09-21T09:31:07.600 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:32.268026+0000", 2024-09-21T09:31:07.600 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:32.268026+0000", 2024-09-21T09:31:07.600 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:32.268500+0000", 2024-09-21T09:31:07.600 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.359086+0000", 2024-09-21T09:31:07.600 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:07.601 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:07.601 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:07.601 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 52, 2024-09-21T09:31:07.601 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.359086+0000", 2024-09-21T09:31:07.601 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.359086+0000", 2024-09-21T09:31:07.601 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.359086+0000", 2024-09-21T09:31:07.601 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:07.601 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:07.601 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:07.601 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.359086+0000", 2024-09-21T09:31:07.601 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.359086+0000", 2024-09-21T09:31:07.601 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:07.601 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:07.601 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:07.601 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:07.602 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 51, 2024-09-21T09:31:07.602 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:07.602 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:07.602 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:07.602 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:07.602 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:07.602 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:07.602 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:07.602 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:07.602 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "2.0", 2024-09-21T09:31:07.602 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:07.602 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:07.602 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:07.602 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 24, 2024-09-21T09:31:07.603 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:07.603 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T13:44:43.463552+0000", 2024-09-21T09:31:07.603 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:07.603 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:07.603 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.603 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:07.603 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.603 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.603 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.603 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.603 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.603 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.603 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.603 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.603 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.604 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.604 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.604 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.604 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.604 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.604 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.604 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:07.604 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:07.604 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.604 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:07.604 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.604 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.604 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.604 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.605 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.605 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:07.605 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.605 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.605 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.605 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.605 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.605 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.605 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.605 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.605 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.605 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.605 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.605 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.605 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:07.606 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:07.606 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.606 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:07.606 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:07.606 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:07.606 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:07.606 INFO:tasks.workunit.client.0.smithi022.stdout: 1, 2024-09-21T09:31:07.606 INFO:tasks.workunit.client.0.smithi022.stdout: 0 2024-09-21T09:31:07.606 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.606 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 7, 2024-09-21T09:31:07.606 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:07.606 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.606 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.606 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:07.607 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:07.607 INFO:tasks.workunit.client.0.smithi022.stdout: 4, 2024-09-21T09:31:07.607 INFO:tasks.workunit.client.0.smithi022.stdout: 3 2024-09-21T09:31:07.607 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.607 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 0, 2024-09-21T09:31:07.607 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:07.607 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:07.607 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:07.607 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:07.607 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:07.607 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:07.607 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.140711+0000", 2024-09-21T09:31:07.607 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.633746+0000", 2024-09-21T09:31:07.608 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.633746+0000", 2024-09-21T09:31:07.608 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.634025+0000", 2024-09-21T09:31:07.608 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.140711+0000", 2024-09-21T09:31:07.608 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.608 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:07.608 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.608 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:07.608 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.140711+0000", 2024-09-21T09:31:07.608 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.140711+0000", 2024-09-21T09:31:07.608 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.140711+0000", 2024-09-21T09:31:07.608 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:07.608 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:07.608 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.608 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.140711+0000", 2024-09-21T09:31:07.609 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.140711+0000", 2024-09-21T09:31:07.609 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:07.609 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:07.609 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:07.609 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:07.609 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:07.609 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:07.609 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:07.609 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:07.609 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:07.609 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:07.609 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:07.609 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:07.610 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:07.610 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.1", 2024-09-21T09:31:07.610 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:07.610 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:07.610 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:07.610 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:07.610 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:07.610 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T15:18:40.511292+0000", 2024-09-21T09:31:07.610 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:07.610 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:07.610 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.610 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:07.610 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.610 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.611 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.611 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.611 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.611 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.611 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.611 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.611 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.611 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.611 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.611 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.611 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.611 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.611 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.611 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:07.612 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:07.612 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.612 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:07.612 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.612 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.612 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.612 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.612 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.612 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:07.612 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.612 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.612 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.612 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.612 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.613 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.613 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.613 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.613 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.613 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.613 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.613 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.613 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:07.613 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:07.613 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.613 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:07.613 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:07.613 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:07.613 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:07.613 INFO:tasks.workunit.client.0.smithi022.stdout: 4, 2024-09-21T09:31:07.614 INFO:tasks.workunit.client.0.smithi022.stdout: 3 2024-09-21T09:31:07.614 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.614 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 0, 2024-09-21T09:31:07.614 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:07.614 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.614 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.614 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:07.614 INFO:tasks.workunit.client.0.smithi022.stdout: 1, 2024-09-21T09:31:07.614 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:07.614 INFO:tasks.workunit.client.0.smithi022.stdout: 0 2024-09-21T09:31:07.614 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.614 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 1, 2024-09-21T09:31:07.614 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:07.614 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:07.615 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 61, 2024-09-21T09:31:07.615 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:07.615 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:07.615 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:07.615 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:40.375561+0000", 2024-09-21T09:31:07.615 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:28:28.924106+0000", 2024-09-21T09:31:07.615 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:28:28.924106+0000", 2024-09-21T09:31:07.615 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:28:33.062917+0000", 2024-09-21T09:31:07.615 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:40.375561+0000", 2024-09-21T09:31:07.615 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:28:27.830318+0000", 2024-09-21T09:31:07.615 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:07.615 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:28:27.830318+0000", 2024-09-21T09:31:07.615 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 62, 2024-09-21T09:31:07.616 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:40.375561+0000", 2024-09-21T09:31:07.616 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:40.375561+0000", 2024-09-21T09:31:07.616 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:40.375561+0000", 2024-09-21T09:31:07.616 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:07.616 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:07.616 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:28:27.830318+0000", 2024-09-21T09:31:07.616 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:40.375561+0000", 2024-09-21T09:31:07.616 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:40.375561+0000", 2024-09-21T09:31:07.616 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:07.616 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:07.616 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:07.616 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:07.616 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 61, 2024-09-21T09:31:07.616 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:07.616 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:07.617 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:07.617 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:07.617 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:07.617 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:07.617 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:07.617 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:07.617 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "4.7", 2024-09-21T09:31:07.617 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:07.617 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:07.617 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 67, 2024-09-21T09:31:07.617 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 19, 2024-09-21T09:31:07.617 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:07.617 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T12:40:32.351197+0000", 2024-09-21T09:31:07.617 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.000580449, 2024-09-21T09:31:07.618 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:07.618 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.618 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:07.618 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.618 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.618 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.618 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.618 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.618 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.618 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.618 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.618 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.618 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.618 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.619 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.619 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.619 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.619 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.619 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:07.619 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:07.619 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.619 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:07.619 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.619 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.619 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.619 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.619 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.619 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:07.619 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.620 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.620 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.620 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.620 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.620 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.620 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.620 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.620 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.620 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.620 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.620 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.620 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:07.620 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:07.620 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.621 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:07.621 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:07.621 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:07.621 INFO:tasks.workunit.client.0.smithi022.stdout: 1, 2024-09-21T09:31:07.621 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:07.621 INFO:tasks.workunit.client.0.smithi022.stdout: 0 2024-09-21T09:31:07.621 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.621 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 1, 2024-09-21T09:31:07.621 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:07.621 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.621 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.621 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:07.621 INFO:tasks.workunit.client.0.smithi022.stdout: 2, 2024-09-21T09:31:07.621 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:07.622 INFO:tasks.workunit.client.0.smithi022.stdout: 0 2024-09-21T09:31:07.622 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.622 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 2, 2024-09-21T09:31:07.622 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:07.622 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:07.622 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 51, 2024-09-21T09:31:07.622 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:07.622 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:07.622 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:07.622 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.061847+0000", 2024-09-21T09:31:07.622 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:32.262373+0000", 2024-09-21T09:31:07.622 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:32.262373+0000", 2024-09-21T09:31:07.622 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:32.262645+0000", 2024-09-21T09:31:07.623 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.061847+0000", 2024-09-21T09:31:07.623 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:07.623 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:07.623 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:07.623 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 52, 2024-09-21T09:31:07.623 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.061847+0000", 2024-09-21T09:31:07.623 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.061847+0000", 2024-09-21T09:31:07.623 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.061847+0000", 2024-09-21T09:31:07.623 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:07.623 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:07.623 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:07.623 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.061847+0000", 2024-09-21T09:31:07.623 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.061847+0000", 2024-09-21T09:31:07.623 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:07.623 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:07.624 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:07.624 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:07.624 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 51, 2024-09-21T09:31:07.624 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:07.624 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:07.624 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:07.624 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:07.624 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:07.624 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:07.624 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:07.624 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:07.624 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "2.1", 2024-09-21T09:31:07.624 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:07.624 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:07.624 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:07.625 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 24, 2024-09-21T09:31:07.625 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:07.625 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T10:37:23.684561+0000", 2024-09-21T09:31:07.625 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:07.625 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:07.625 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.625 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:07.625 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.625 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.625 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.625 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.625 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.625 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.625 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.626 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.626 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.626 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.626 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.626 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.626 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.626 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.626 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.626 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:07.626 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:07.626 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.626 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:07.626 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.627 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.627 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.627 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.627 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.627 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:07.627 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.627 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.627 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.627 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.627 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.627 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.627 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.627 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.627 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.628 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.628 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.628 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.628 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:07.628 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:07.628 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.628 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:07.628 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:07.628 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:07.628 INFO:tasks.workunit.client.0.smithi022.stdout: 2, 2024-09-21T09:31:07.628 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:07.628 INFO:tasks.workunit.client.0.smithi022.stdout: 0 2024-09-21T09:31:07.629 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.629 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 2, 2024-09-21T09:31:07.629 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:07.629 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.629 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.629 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:07.629 INFO:tasks.workunit.client.0.smithi022.stdout: 1, 2024-09-21T09:31:07.629 INFO:tasks.workunit.client.0.smithi022.stdout: 2, 2024-09-21T09:31:07.629 INFO:tasks.workunit.client.0.smithi022.stdout: 6 2024-09-21T09:31:07.629 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.629 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 1, 2024-09-21T09:31:07.629 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:07.629 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:07.629 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:07.630 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:07.630 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:07.630 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:07.630 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.061597+0000", 2024-09-21T09:31:07.630 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.635805+0000", 2024-09-21T09:31:07.630 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.635805+0000", 2024-09-21T09:31:07.630 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.636105+0000", 2024-09-21T09:31:07.630 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.061597+0000", 2024-09-21T09:31:07.630 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.630 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:07.630 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.630 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:07.630 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.061597+0000", 2024-09-21T09:31:07.631 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.061597+0000", 2024-09-21T09:31:07.631 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.061597+0000", 2024-09-21T09:31:07.631 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:07.631 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:07.631 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.631 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.061597+0000", 2024-09-21T09:31:07.631 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.061597+0000", 2024-09-21T09:31:07.631 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:07.631 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:07.631 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:07.631 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:07.631 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:07.631 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:07.632 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:07.632 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:07.632 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:07.632 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:07.632 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:07.632 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:07.632 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:07.632 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.0", 2024-09-21T09:31:07.632 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:07.632 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:07.632 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:07.632 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:07.632 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:07.633 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T13:22:53.711130+0000", 2024-09-21T09:31:07.633 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:07.633 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:07.633 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.633 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:07.633 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.633 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.633 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.633 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.633 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.633 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.633 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.633 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.633 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.634 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.634 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.634 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.634 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.634 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.634 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.634 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:07.634 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:07.634 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.634 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:07.634 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.634 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.634 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.634 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.635 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.635 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:07.635 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.635 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.635 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.635 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.635 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.635 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.635 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.635 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.635 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.635 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.635 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.635 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.636 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:07.636 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:07.636 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.636 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:07.636 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:07.636 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:07.636 INFO:tasks.workunit.client.0.smithi022.stdout: 1, 2024-09-21T09:31:07.636 INFO:tasks.workunit.client.0.smithi022.stdout: 2, 2024-09-21T09:31:07.636 INFO:tasks.workunit.client.0.smithi022.stdout: 6 2024-09-21T09:31:07.636 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.636 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 1, 2024-09-21T09:31:07.636 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:07.636 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.636 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.637 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:07.637 INFO:tasks.workunit.client.0.smithi022.stdout: 1, 2024-09-21T09:31:07.637 INFO:tasks.workunit.client.0.smithi022.stdout: 2, 2024-09-21T09:31:07.637 INFO:tasks.workunit.client.0.smithi022.stdout: 3 2024-09-21T09:31:07.637 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.637 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 1, 2024-09-21T09:31:07.637 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:07.637 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:07.637 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 61, 2024-09-21T09:31:07.637 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:07.637 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:07.637 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:07.637 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:40.376066+0000", 2024-09-21T09:31:07.637 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:28:28.844698+0000", 2024-09-21T09:31:07.638 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:28:28.844698+0000", 2024-09-21T09:31:07.638 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:28:33.062726+0000", 2024-09-21T09:31:07.638 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:40.376066+0000", 2024-09-21T09:31:07.638 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:28:27.830318+0000", 2024-09-21T09:31:07.638 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:07.638 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:28:27.830318+0000", 2024-09-21T09:31:07.638 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 62, 2024-09-21T09:31:07.638 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:40.376066+0000", 2024-09-21T09:31:07.638 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:40.376066+0000", 2024-09-21T09:31:07.638 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:40.376066+0000", 2024-09-21T09:31:07.638 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:07.638 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:07.638 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:28:27.830318+0000", 2024-09-21T09:31:07.638 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:40.376066+0000", 2024-09-21T09:31:07.639 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:40.376066+0000", 2024-09-21T09:31:07.639 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:07.639 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:07.639 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:07.639 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:07.639 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 61, 2024-09-21T09:31:07.639 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:07.639 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:07.639 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:07.639 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:07.639 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:07.639 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:07.639 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:07.639 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:07.640 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "4.4", 2024-09-21T09:31:07.640 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:07.640 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:07.640 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 67, 2024-09-21T09:31:07.640 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 19, 2024-09-21T09:31:07.640 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:07.640 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T14:24:39.284345+0000", 2024-09-21T09:31:07.640 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.000725062, 2024-09-21T09:31:07.640 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:07.640 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.640 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:07.640 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.641 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.641 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.641 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.641 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.641 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.641 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.641 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.641 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.641 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.641 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.641 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.642 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.642 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.642 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.642 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:07.642 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:07.642 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.642 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:07.642 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.642 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.642 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.642 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.643 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.643 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:07.643 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.643 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.643 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.643 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.643 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.643 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.643 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.643 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.643 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.643 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.643 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.643 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.644 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:07.644 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:07.644 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.644 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:07.644 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:07.644 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:07.644 INFO:tasks.workunit.client.0.smithi022.stdout: 1, 2024-09-21T09:31:07.644 INFO:tasks.workunit.client.0.smithi022.stdout: 2, 2024-09-21T09:31:07.644 INFO:tasks.workunit.client.0.smithi022.stdout: 3 2024-09-21T09:31:07.644 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.644 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 1, 2024-09-21T09:31:07.644 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:07.644 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.644 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.644 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:07.645 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:07.645 INFO:tasks.workunit.client.0.smithi022.stdout: 1, 2024-09-21T09:31:07.645 INFO:tasks.workunit.client.0.smithi022.stdout: 6 2024-09-21T09:31:07.645 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.645 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 5, 2024-09-21T09:31:07.645 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:07.645 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:07.645 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 51, 2024-09-21T09:31:07.645 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:07.645 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:07.645 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:07.645 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.066308+0000", 2024-09-21T09:31:07.645 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:32.264740+0000", 2024-09-21T09:31:07.646 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:32.264740+0000", 2024-09-21T09:31:07.646 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:32.265210+0000", 2024-09-21T09:31:07.646 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.066308+0000", 2024-09-21T09:31:07.646 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:07.646 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:07.646 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:07.646 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 52, 2024-09-21T09:31:07.646 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.066308+0000", 2024-09-21T09:31:07.646 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.066308+0000", 2024-09-21T09:31:07.646 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.066308+0000", 2024-09-21T09:31:07.646 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:07.646 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:07.646 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:07.646 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.066308+0000", 2024-09-21T09:31:07.647 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.066308+0000", 2024-09-21T09:31:07.647 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:07.647 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:07.647 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:07.647 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:07.647 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 51, 2024-09-21T09:31:07.647 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:07.647 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:07.647 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:07.647 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:07.647 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:07.647 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:07.647 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:07.647 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:07.648 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "2.2", 2024-09-21T09:31:07.648 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:07.648 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:07.648 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:07.648 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 24, 2024-09-21T09:31:07.648 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:07.648 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T19:14:01.491316+0000", 2024-09-21T09:31:07.648 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:07.648 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:07.648 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.648 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:07.648 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.648 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.648 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.649 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.649 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.649 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.649 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.649 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.649 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.649 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.649 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.649 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.649 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.649 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.649 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.649 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:07.649 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:07.650 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.650 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:07.650 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.650 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.650 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.650 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.650 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.650 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:07.650 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.650 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.650 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.650 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.651 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.651 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.651 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.651 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.651 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.651 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.651 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.651 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.651 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:07.651 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:07.651 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.651 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:07.651 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:07.652 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:07.652 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:07.652 INFO:tasks.workunit.client.0.smithi022.stdout: 1, 2024-09-21T09:31:07.652 INFO:tasks.workunit.client.0.smithi022.stdout: 6 2024-09-21T09:31:07.652 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.652 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 5, 2024-09-21T09:31:07.652 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:07.652 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.652 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.652 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:07.652 INFO:tasks.workunit.client.0.smithi022.stdout: 4, 2024-09-21T09:31:07.652 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:07.652 INFO:tasks.workunit.client.0.smithi022.stdout: 6 2024-09-21T09:31:07.652 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.653 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 4, 2024-09-21T09:31:07.653 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:07.653 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:07.653 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:07.653 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:07.653 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:07.653 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:07.653 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.062810+0000", 2024-09-21T09:31:07.653 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.638269+0000", 2024-09-21T09:31:07.653 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.638269+0000", 2024-09-21T09:31:07.653 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.638582+0000", 2024-09-21T09:31:07.653 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.062810+0000", 2024-09-21T09:31:07.653 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.654 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:07.654 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.654 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:07.654 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.062810+0000", 2024-09-21T09:31:07.654 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.062810+0000", 2024-09-21T09:31:07.654 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.062810+0000", 2024-09-21T09:31:07.654 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:07.654 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:07.654 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.654 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.062810+0000", 2024-09-21T09:31:07.654 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.062810+0000", 2024-09-21T09:31:07.654 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:07.655 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:07.655 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:07.655 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:07.655 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:07.655 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:07.655 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:07.655 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:07.655 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:07.655 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:07.655 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:07.655 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:07.655 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:07.655 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.3", 2024-09-21T09:31:07.656 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:07.656 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:07.656 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:07.656 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:07.656 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:07.656 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T20:52:20.849743+0000", 2024-09-21T09:31:07.656 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:07.656 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:07.656 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.656 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:07.656 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.656 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.656 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.656 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.656 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.657 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.657 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.657 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.657 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.657 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.657 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.657 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.657 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.657 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.657 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.657 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:07.657 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:07.657 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.658 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:07.658 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.658 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.658 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.658 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.658 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.658 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:07.658 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.658 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.658 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.658 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.658 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.658 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.658 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.659 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.659 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.659 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.659 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.659 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.659 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:07.659 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:07.659 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.659 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:07.659 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:07.659 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:07.659 INFO:tasks.workunit.client.0.smithi022.stdout: 4, 2024-09-21T09:31:07.659 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:07.660 INFO:tasks.workunit.client.0.smithi022.stdout: 6 2024-09-21T09:31:07.660 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.660 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 4, 2024-09-21T09:31:07.660 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:07.660 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.660 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.660 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:07.660 INFO:tasks.workunit.client.0.smithi022.stdout: 1, 2024-09-21T09:31:07.660 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:07.660 INFO:tasks.workunit.client.0.smithi022.stdout: 4 2024-09-21T09:31:07.660 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.660 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 1, 2024-09-21T09:31:07.661 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:07.661 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:07.661 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 61, 2024-09-21T09:31:07.661 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:07.661 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:07.661 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:07.661 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:40.375501+0000", 2024-09-21T09:31:07.661 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:28:28.924195+0000", 2024-09-21T09:31:07.661 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:28:28.924195+0000", 2024-09-21T09:31:07.661 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:28:33.063343+0000", 2024-09-21T09:31:07.661 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:40.375501+0000", 2024-09-21T09:31:07.661 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:28:27.830318+0000", 2024-09-21T09:31:07.661 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:07.662 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:28:27.830318+0000", 2024-09-21T09:31:07.662 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 62, 2024-09-21T09:31:07.662 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:40.375501+0000", 2024-09-21T09:31:07.662 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:40.375501+0000", 2024-09-21T09:31:07.662 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:40.375501+0000", 2024-09-21T09:31:07.662 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:07.662 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:07.662 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:28:27.830318+0000", 2024-09-21T09:31:07.662 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:40.375501+0000", 2024-09-21T09:31:07.662 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:40.375501+0000", 2024-09-21T09:31:07.662 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:07.662 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 2, 2024-09-21T09:31:07.662 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:07.663 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:07.663 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 61, 2024-09-21T09:31:07.663 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:07.663 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:07.663 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:07.663 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:07.663 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 2, 2024-09-21T09:31:07.663 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:07.663 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:07.663 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:07.663 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "4.2", 2024-09-21T09:31:07.663 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:07.663 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:07.664 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 67, 2024-09-21T09:31:07.664 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:07.664 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:07.664 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T10:33:35.119949+0000", 2024-09-21T09:31:07.664 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.000800248, 2024-09-21T09:31:07.664 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:07.664 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.664 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 19, 2024-09-21T09:31:07.664 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.664 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.664 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.664 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.664 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.665 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.665 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.665 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.665 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.665 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.665 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.665 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.665 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.665 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.665 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.665 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 3, 2024-09-21T09:31:07.665 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 1, 2024-09-21T09:31:07.665 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.666 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 1, 2024-09-21T09:31:07.666 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.666 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.666 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.666 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.666 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.666 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:07.666 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.666 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.666 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.666 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.666 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.666 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.666 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.667 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.667 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.667 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.667 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.667 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.667 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 2, 2024-09-21T09:31:07.667 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 2 2024-09-21T09:31:07.667 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.667 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:07.667 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:07.667 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:07.667 INFO:tasks.workunit.client.0.smithi022.stdout: 1, 2024-09-21T09:31:07.667 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:07.667 INFO:tasks.workunit.client.0.smithi022.stdout: 4 2024-09-21T09:31:07.668 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.668 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 1, 2024-09-21T09:31:07.668 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "66'2" 2024-09-21T09:31:07.668 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.668 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.668 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:07.668 INFO:tasks.workunit.client.0.smithi022.stdout: 1, 2024-09-21T09:31:07.668 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:07.668 INFO:tasks.workunit.client.0.smithi022.stdout: 7 2024-09-21T09:31:07.668 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.668 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 1, 2024-09-21T09:31:07.668 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:07.668 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:07.668 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 51, 2024-09-21T09:31:07.669 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:07.669 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:07.669 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:07.669 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.062271+0000", 2024-09-21T09:31:07.669 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:32.264201+0000", 2024-09-21T09:31:07.669 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:32.264201+0000", 2024-09-21T09:31:07.669 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:32.264482+0000", 2024-09-21T09:31:07.669 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.062271+0000", 2024-09-21T09:31:07.669 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:07.669 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:07.669 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:07.669 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 52, 2024-09-21T09:31:07.669 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.062271+0000", 2024-09-21T09:31:07.669 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.062271+0000", 2024-09-21T09:31:07.669 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.062271+0000", 2024-09-21T09:31:07.670 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:07.670 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:07.670 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:07.670 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.062271+0000", 2024-09-21T09:31:07.670 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.062271+0000", 2024-09-21T09:31:07.670 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:07.670 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:07.670 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:07.670 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:07.670 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 51, 2024-09-21T09:31:07.670 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:07.670 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:07.670 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:07.670 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:07.671 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:07.671 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:07.671 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:07.671 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:07.671 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "2.4", 2024-09-21T09:31:07.671 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:07.671 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:07.671 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:07.671 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 24, 2024-09-21T09:31:07.671 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:07.671 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T20:37:15.565839+0000", 2024-09-21T09:31:07.671 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:07.671 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:07.672 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.672 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:07.672 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.672 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.672 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.672 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.672 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.672 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.672 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.672 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.672 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.672 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.672 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.672 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.673 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.673 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.673 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.673 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:07.673 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:07.673 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.673 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:07.673 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.673 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.673 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.673 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.673 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.673 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:07.673 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.674 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.674 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.674 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.674 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.674 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.674 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.674 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.674 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.674 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.674 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.674 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.674 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:07.674 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:07.674 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.675 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:07.675 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:07.675 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:07.675 INFO:tasks.workunit.client.0.smithi022.stdout: 1, 2024-09-21T09:31:07.675 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:07.675 INFO:tasks.workunit.client.0.smithi022.stdout: 7 2024-09-21T09:31:07.675 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.675 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 1, 2024-09-21T09:31:07.675 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:07.675 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.675 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.675 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:07.675 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:07.675 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:07.676 INFO:tasks.workunit.client.0.smithi022.stdout: 2 2024-09-21T09:31:07.676 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.676 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 5, 2024-09-21T09:31:07.676 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:07.676 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:07.676 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:07.676 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:07.676 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:07.676 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:07.676 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.066122+0000", 2024-09-21T09:31:07.676 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.634594+0000", 2024-09-21T09:31:07.676 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.634594+0000", 2024-09-21T09:31:07.676 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.634935+0000", 2024-09-21T09:31:07.677 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.066122+0000", 2024-09-21T09:31:07.677 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.677 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:07.677 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.677 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:07.677 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.066122+0000", 2024-09-21T09:31:07.677 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.066122+0000", 2024-09-21T09:31:07.677 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.066122+0000", 2024-09-21T09:31:07.677 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:07.677 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:07.677 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.677 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.066122+0000", 2024-09-21T09:31:07.678 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.066122+0000", 2024-09-21T09:31:07.678 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:07.678 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:07.678 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:07.678 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:07.678 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:07.678 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:07.678 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:07.678 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:07.678 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:07.678 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:07.678 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:07.678 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:07.679 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:07.679 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.5", 2024-09-21T09:31:07.679 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:07.679 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:07.679 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:07.679 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:07.679 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:07.679 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T11:53:03.606929+0000", 2024-09-21T09:31:07.679 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:07.679 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:07.679 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.679 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:07.679 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.680 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.680 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.680 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.680 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.680 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.680 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.680 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.680 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.680 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.680 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.680 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.680 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.680 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.680 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.680 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:07.681 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:07.681 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.681 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:07.681 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.681 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.681 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.681 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.681 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.681 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:07.681 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.681 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.681 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.681 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.681 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.682 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.682 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.682 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.682 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.682 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.682 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.682 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.682 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:07.682 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:07.682 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.682 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:07.682 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:07.682 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:07.682 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:07.683 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:07.683 INFO:tasks.workunit.client.0.smithi022.stdout: 2 2024-09-21T09:31:07.683 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.683 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 5, 2024-09-21T09:31:07.683 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:07.683 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.683 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.683 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:07.683 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:07.683 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:07.683 INFO:tasks.workunit.client.0.smithi022.stdout: 7 2024-09-21T09:31:07.683 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.683 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 0, 2024-09-21T09:31:07.683 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:07.684 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:07.684 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 61, 2024-09-21T09:31:07.684 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:07.684 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:07.684 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:07.684 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:40.922745+0000", 2024-09-21T09:31:07.684 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:28:28.923619+0000", 2024-09-21T09:31:07.684 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:28:28.923619+0000", 2024-09-21T09:31:07.684 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:28:33.141355+0000", 2024-09-21T09:31:07.684 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:40.922745+0000", 2024-09-21T09:31:07.684 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:28:27.830318+0000", 2024-09-21T09:31:07.684 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:07.684 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:28:27.830318+0000", 2024-09-21T09:31:07.684 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 62, 2024-09-21T09:31:07.685 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:40.922745+0000", 2024-09-21T09:31:07.685 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:40.922745+0000", 2024-09-21T09:31:07.685 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:40.922745+0000", 2024-09-21T09:31:07.685 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:07.685 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:07.685 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:28:27.830318+0000", 2024-09-21T09:31:07.685 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:40.922745+0000", 2024-09-21T09:31:07.685 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:40.922745+0000", 2024-09-21T09:31:07.685 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:07.685 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 1, 2024-09-21T09:31:07.685 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:07.685 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:07.685 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 61, 2024-09-21T09:31:07.685 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:07.686 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:07.686 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:07.686 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:07.686 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 1, 2024-09-21T09:31:07.686 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:07.686 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:07.686 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:07.686 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "4.3", 2024-09-21T09:31:07.686 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:07.686 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:07.686 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 67, 2024-09-21T09:31:07.686 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 20, 2024-09-21T09:31:07.686 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:07.687 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T10:40:50.990669+0000", 2024-09-21T09:31:07.687 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.000417457, 2024-09-21T09:31:07.687 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:07.687 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.687 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:07.687 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.687 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.687 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.687 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.687 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.687 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.687 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.687 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.687 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.688 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.688 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.688 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.688 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.688 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.688 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.688 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 3, 2024-09-21T09:31:07.688 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 1, 2024-09-21T09:31:07.688 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.688 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 1, 2024-09-21T09:31:07.688 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.688 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.688 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.688 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.689 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.689 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:07.689 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.689 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.689 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.689 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.689 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.689 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.689 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.689 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.689 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.690 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.690 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.690 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.690 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:07.690 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:07.690 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.690 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:07.690 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:07.690 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:07.690 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:07.690 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:07.690 INFO:tasks.workunit.client.0.smithi022.stdout: 7 2024-09-21T09:31:07.691 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.691 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 0, 2024-09-21T09:31:07.691 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "64'1" 2024-09-21T09:31:07.691 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.691 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.691 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:07.691 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:07.691 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:07.691 INFO:tasks.workunit.client.0.smithi022.stdout: 4 2024-09-21T09:31:07.691 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.691 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 7, 2024-09-21T09:31:07.691 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:07.692 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:07.692 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 51, 2024-09-21T09:31:07.692 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:07.692 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:07.692 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:07.692 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.359440+0000", 2024-09-21T09:31:07.692 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:32.274058+0000", 2024-09-21T09:31:07.692 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:32.274058+0000", 2024-09-21T09:31:07.692 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:32.274427+0000", 2024-09-21T09:31:07.692 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.359440+0000", 2024-09-21T09:31:07.692 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:07.692 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:07.692 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:07.693 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 52, 2024-09-21T09:31:07.693 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.359440+0000", 2024-09-21T09:31:07.693 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.359440+0000", 2024-09-21T09:31:07.693 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.359440+0000", 2024-09-21T09:31:07.693 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:07.693 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:07.693 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:07.693 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.359440+0000", 2024-09-21T09:31:07.693 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.359440+0000", 2024-09-21T09:31:07.693 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:07.693 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:07.693 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:07.693 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:07.694 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 51, 2024-09-21T09:31:07.694 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:07.694 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:07.694 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:07.694 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:07.694 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:07.694 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:07.694 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:07.694 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:07.694 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "2.5", 2024-09-21T09:31:07.694 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:07.694 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:07.694 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:07.695 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 24, 2024-09-21T09:31:07.695 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:07.695 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T16:09:38.714482+0000", 2024-09-21T09:31:07.695 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:07.695 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:07.695 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.695 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:07.695 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.695 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.695 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.695 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.695 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.695 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.696 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.696 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.696 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.696 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.696 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.696 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.696 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.696 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.696 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.696 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:07.696 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:07.696 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.696 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:07.696 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.697 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.697 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.697 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.697 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.697 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:07.697 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.697 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.697 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.697 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.697 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.697 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.697 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.697 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.697 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.697 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.698 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.698 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.698 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:07.698 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:07.698 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.698 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:07.698 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:07.698 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:07.698 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:07.698 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:07.698 INFO:tasks.workunit.client.0.smithi022.stdout: 4 2024-09-21T09:31:07.698 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.698 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 7, 2024-09-21T09:31:07.699 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:07.699 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.699 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.699 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:07.699 INFO:tasks.workunit.client.0.smithi022.stdout: 1, 2024-09-21T09:31:07.699 INFO:tasks.workunit.client.0.smithi022.stdout: 2, 2024-09-21T09:31:07.699 INFO:tasks.workunit.client.0.smithi022.stdout: 5 2024-09-21T09:31:07.699 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.699 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 1, 2024-09-21T09:31:07.699 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:07.699 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:07.699 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:07.699 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:07.699 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:07.700 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:07.700 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.062188+0000", 2024-09-21T09:31:07.700 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.642792+0000", 2024-09-21T09:31:07.700 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.642792+0000", 2024-09-21T09:31:07.700 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.643081+0000", 2024-09-21T09:31:07.700 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.062188+0000", 2024-09-21T09:31:07.700 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.700 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:07.700 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.700 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:07.700 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.062188+0000", 2024-09-21T09:31:07.700 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.062188+0000", 2024-09-21T09:31:07.700 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.062188+0000", 2024-09-21T09:31:07.700 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:07.701 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:07.701 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.701 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.062188+0000", 2024-09-21T09:31:07.701 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.062188+0000", 2024-09-21T09:31:07.701 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:07.701 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:07.701 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:07.701 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:07.701 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:07.701 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:07.701 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:07.701 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:07.701 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:07.702 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:07.702 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:07.702 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:07.702 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:07.702 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.4", 2024-09-21T09:31:07.702 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:07.702 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:07.702 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:07.702 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:07.702 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:07.702 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T17:05:03.157102+0000", 2024-09-21T09:31:07.702 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:07.702 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:07.703 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.703 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:07.703 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.703 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.703 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.703 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.703 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.703 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.703 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.703 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.703 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.703 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.703 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.703 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.704 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.704 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.704 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.704 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:07.704 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:07.704 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.704 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:07.704 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.704 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.704 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.704 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.704 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.704 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:07.704 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.705 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.705 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.705 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.705 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.705 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.705 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.705 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.705 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.705 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.705 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.705 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.705 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:07.705 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:07.705 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.706 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:07.706 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:07.706 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:07.706 INFO:tasks.workunit.client.0.smithi022.stdout: 1, 2024-09-21T09:31:07.706 INFO:tasks.workunit.client.0.smithi022.stdout: 2, 2024-09-21T09:31:07.706 INFO:tasks.workunit.client.0.smithi022.stdout: 5 2024-09-21T09:31:07.706 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.706 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 1, 2024-09-21T09:31:07.706 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:07.706 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.706 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.706 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:07.706 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:07.706 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:07.707 INFO:tasks.workunit.client.0.smithi022.stdout: 0 2024-09-21T09:31:07.707 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.707 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 3, 2024-09-21T09:31:07.707 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:07.707 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:07.707 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 61, 2024-09-21T09:31:07.707 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:07.707 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:07.707 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:07.707 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:40.484507+0000", 2024-09-21T09:31:07.707 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:28:28.845163+0000", 2024-09-21T09:31:07.707 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:28:28.845163+0000", 2024-09-21T09:31:07.707 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:28:33.125263+0000", 2024-09-21T09:31:07.708 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:40.484507+0000", 2024-09-21T09:31:07.708 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:28:27.830318+0000", 2024-09-21T09:31:07.708 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:07.708 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:28:27.830318+0000", 2024-09-21T09:31:07.708 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 62, 2024-09-21T09:31:07.708 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:40.484507+0000", 2024-09-21T09:31:07.708 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:40.484507+0000", 2024-09-21T09:31:07.708 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:40.484507+0000", 2024-09-21T09:31:07.708 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:07.708 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:07.708 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:28:27.830318+0000", 2024-09-21T09:31:07.708 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:40.484507+0000", 2024-09-21T09:31:07.708 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:40.484507+0000", 2024-09-21T09:31:07.708 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:07.709 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:07.709 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:07.709 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:07.709 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 61, 2024-09-21T09:31:07.709 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:07.709 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:07.709 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:07.709 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:07.709 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:07.709 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:07.709 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:07.709 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:07.709 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "4.0", 2024-09-21T09:31:07.709 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:07.710 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:07.710 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 67, 2024-09-21T09:31:07.710 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 19, 2024-09-21T09:31:07.710 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:07.710 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T13:56:39.868147+0000", 2024-09-21T09:31:07.710 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.000345912, 2024-09-21T09:31:07.710 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:07.710 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.710 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:07.710 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.710 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.710 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.710 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.711 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.711 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.711 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.711 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.711 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.711 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.711 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.711 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.711 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.711 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.711 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.711 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:07.711 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:07.711 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.712 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:07.712 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.712 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.712 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.712 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.712 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.712 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:07.712 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.712 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.712 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.712 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.712 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.712 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.712 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.712 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.713 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.713 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.713 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.713 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.713 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:07.713 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:07.713 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.713 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:07.713 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:07.713 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:07.713 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:07.713 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:07.713 INFO:tasks.workunit.client.0.smithi022.stdout: 0 2024-09-21T09:31:07.713 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.713 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 3, 2024-09-21T09:31:07.713 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:07.714 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.714 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.714 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:07.714 INFO:tasks.workunit.client.0.smithi022.stdout: 1, 2024-09-21T09:31:07.714 INFO:tasks.workunit.client.0.smithi022.stdout: 6, 2024-09-21T09:31:07.714 INFO:tasks.workunit.client.0.smithi022.stdout: 4 2024-09-21T09:31:07.714 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.714 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 1, 2024-09-21T09:31:07.714 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:07.714 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:07.714 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 51, 2024-09-21T09:31:07.714 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:07.714 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:07.714 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:07.715 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.062459+0000", 2024-09-21T09:31:07.715 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:32.265171+0000", 2024-09-21T09:31:07.715 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:32.265171+0000", 2024-09-21T09:31:07.715 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:32.265476+0000", 2024-09-21T09:31:07.715 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.062459+0000", 2024-09-21T09:31:07.715 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:07.715 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:07.715 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:07.715 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 52, 2024-09-21T09:31:07.715 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.062459+0000", 2024-09-21T09:31:07.715 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.062459+0000", 2024-09-21T09:31:07.715 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.062459+0000", 2024-09-21T09:31:07.715 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:07.716 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:07.716 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:07.716 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.062459+0000", 2024-09-21T09:31:07.716 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.062459+0000", 2024-09-21T09:31:07.716 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:07.716 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 1, 2024-09-21T09:31:07.716 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:07.716 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:07.716 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 51, 2024-09-21T09:31:07.716 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:07.716 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:07.716 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:07.716 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:07.716 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 1, 2024-09-21T09:31:07.717 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:07.717 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:07.717 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:07.717 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "2.6", 2024-09-21T09:31:07.717 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:07.717 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:07.717 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:07.717 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 25, 2024-09-21T09:31:07.717 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:07.717 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T12:42:48.279357+0000", 2024-09-21T09:31:07.717 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:07.717 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:07.717 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.717 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:07.717 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.718 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.718 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.718 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.718 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.718 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.718 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.718 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.718 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.718 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.718 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.718 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.718 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.718 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.719 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.719 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 3, 2024-09-21T09:31:07.719 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 1, 2024-09-21T09:31:07.719 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.719 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 1, 2024-09-21T09:31:07.719 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.719 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.719 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.719 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.719 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.719 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 1, 2024-09-21T09:31:07.719 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.719 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.719 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.720 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.720 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.720 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.720 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.720 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.720 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.720 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.720 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.720 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.720 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 1, 2024-09-21T09:31:07.720 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:07.720 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.720 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:07.720 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:07.721 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:07.721 INFO:tasks.workunit.client.0.smithi022.stdout: 1, 2024-09-21T09:31:07.721 INFO:tasks.workunit.client.0.smithi022.stdout: 6, 2024-09-21T09:31:07.721 INFO:tasks.workunit.client.0.smithi022.stdout: 4 2024-09-21T09:31:07.721 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.721 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 1, 2024-09-21T09:31:07.721 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "57'1" 2024-09-21T09:31:07.721 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.721 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.721 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:07.721 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:07.721 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:07.721 INFO:tasks.workunit.client.0.smithi022.stdout: 0 2024-09-21T09:31:07.721 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.722 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 3, 2024-09-21T09:31:07.722 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:07.722 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:07.722 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:07.722 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:07.722 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:07.722 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:07.722 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.125628+0000", 2024-09-21T09:31:07.722 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.629345+0000", 2024-09-21T09:31:07.722 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.629345+0000", 2024-09-21T09:31:07.722 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.629856+0000", 2024-09-21T09:31:07.722 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.125628+0000", 2024-09-21T09:31:07.722 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.722 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:07.722 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.723 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:07.723 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.125628+0000", 2024-09-21T09:31:07.723 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.125628+0000", 2024-09-21T09:31:07.723 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.125628+0000", 2024-09-21T09:31:07.723 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:07.723 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:07.723 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.723 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.125628+0000", 2024-09-21T09:31:07.723 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.125628+0000", 2024-09-21T09:31:07.723 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:07.723 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:07.723 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:07.724 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:07.724 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:07.724 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:07.724 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:07.724 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:07.724 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:07.724 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:07.724 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:07.724 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:07.724 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:07.724 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.7", 2024-09-21T09:31:07.724 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:07.724 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:07.724 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:07.724 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:07.725 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:07.725 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T17:30:14.361278+0000", 2024-09-21T09:31:07.725 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:07.725 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:07.725 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.725 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:07.725 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.725 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.725 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.725 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.725 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.725 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.725 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.725 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.725 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.726 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.726 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.726 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.726 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.726 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.726 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.726 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:07.726 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:07.726 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.726 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:07.726 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.726 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.726 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.726 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.727 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.727 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:07.727 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.727 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.727 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.727 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.727 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.727 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.727 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.727 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.727 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.727 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.727 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.727 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.727 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:07.727 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:07.728 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.728 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:07.728 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:07.728 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:07.728 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:07.728 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:07.728 INFO:tasks.workunit.client.0.smithi022.stdout: 0 2024-09-21T09:31:07.728 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.728 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 3, 2024-09-21T09:31:07.728 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:07.728 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.728 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.728 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:07.728 INFO:tasks.workunit.client.0.smithi022.stdout: 4, 2024-09-21T09:31:07.729 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:07.729 INFO:tasks.workunit.client.0.smithi022.stdout: 6 2024-09-21T09:31:07.729 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.729 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 4, 2024-09-21T09:31:07.729 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:07.729 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:07.729 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 61, 2024-09-21T09:31:07.729 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:07.729 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:07.729 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:07.729 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:40.749241+0000", 2024-09-21T09:31:07.729 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:28:28.924207+0000", 2024-09-21T09:31:07.729 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:28:28.924207+0000", 2024-09-21T09:31:07.729 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:28:33.062764+0000", 2024-09-21T09:31:07.729 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:40.749241+0000", 2024-09-21T09:31:07.729 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:28:27.830318+0000", 2024-09-21T09:31:07.730 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:07.730 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:28:27.830318+0000", 2024-09-21T09:31:07.730 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 62, 2024-09-21T09:31:07.730 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:40.749241+0000", 2024-09-21T09:31:07.730 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:40.749241+0000", 2024-09-21T09:31:07.730 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:40.749241+0000", 2024-09-21T09:31:07.730 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:07.730 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:07.730 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:28:27.830318+0000", 2024-09-21T09:31:07.730 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:40.749241+0000", 2024-09-21T09:31:07.730 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:40.749241+0000", 2024-09-21T09:31:07.730 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:07.730 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:07.730 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:07.731 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:07.731 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 61, 2024-09-21T09:31:07.731 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:07.731 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:07.731 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:07.731 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:07.731 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:07.731 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:07.731 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:07.731 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:07.731 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "4.1", 2024-09-21T09:31:07.731 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:07.731 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:07.731 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 67, 2024-09-21T09:31:07.732 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 19, 2024-09-21T09:31:07.732 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:07.732 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T11:08:48.406053+0000", 2024-09-21T09:31:07.732 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.000555972, 2024-09-21T09:31:07.732 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:07.732 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.732 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:07.732 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.732 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.732 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.732 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.732 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.732 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.732 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.733 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.733 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.733 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.733 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.733 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.733 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.733 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.733 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.733 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:07.733 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:07.733 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.733 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:07.733 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.734 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.734 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.734 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.734 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.734 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:07.734 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.734 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.734 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.734 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.734 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.734 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.734 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.734 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.734 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.735 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.735 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.735 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.735 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:07.735 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:07.735 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.735 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:07.735 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:07.735 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:07.735 INFO:tasks.workunit.client.0.smithi022.stdout: 4, 2024-09-21T09:31:07.735 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:07.735 INFO:tasks.workunit.client.0.smithi022.stdout: 6 2024-09-21T09:31:07.735 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.735 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 4, 2024-09-21T09:31:07.735 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:07.736 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.736 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.736 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:07.736 INFO:tasks.workunit.client.0.smithi022.stdout: 6, 2024-09-21T09:31:07.736 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:07.736 INFO:tasks.workunit.client.0.smithi022.stdout: 2 2024-09-21T09:31:07.736 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.736 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 6, 2024-09-21T09:31:07.736 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:07.736 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:07.736 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 51, 2024-09-21T09:31:07.736 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:07.736 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:07.737 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:07.737 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.484453+0000", 2024-09-21T09:31:07.737 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:32.264444+0000", 2024-09-21T09:31:07.737 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:32.264444+0000", 2024-09-21T09:31:07.737 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:32.264763+0000", 2024-09-21T09:31:07.737 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.484453+0000", 2024-09-21T09:31:07.737 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:07.737 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:07.737 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:07.737 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 52, 2024-09-21T09:31:07.737 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.484453+0000", 2024-09-21T09:31:07.737 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.484453+0000", 2024-09-21T09:31:07.737 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.484453+0000", 2024-09-21T09:31:07.738 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:07.738 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:07.738 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:07.738 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.484453+0000", 2024-09-21T09:31:07.738 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.484453+0000", 2024-09-21T09:31:07.738 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:07.738 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 2, 2024-09-21T09:31:07.738 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:07.738 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:07.738 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 51, 2024-09-21T09:31:07.738 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:07.738 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:07.738 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:07.739 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:07.739 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 2, 2024-09-21T09:31:07.739 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:07.739 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:07.739 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:07.739 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "2.7", 2024-09-21T09:31:07.739 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:07.739 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:07.739 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:07.739 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 26, 2024-09-21T09:31:07.739 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:07.739 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T12:01:52.871773+0000", 2024-09-21T09:31:07.739 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:07.739 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:07.740 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.740 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:07.740 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.740 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.740 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.740 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.740 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.740 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.740 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.740 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.740 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.740 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.740 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.740 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.740 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.741 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.741 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.741 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 3, 2024-09-21T09:31:07.741 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 1, 2024-09-21T09:31:07.741 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.741 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 1, 2024-09-21T09:31:07.741 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.741 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.741 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.741 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.741 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.741 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 1, 2024-09-21T09:31:07.741 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.741 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.742 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.742 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.742 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.742 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.742 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.742 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.742 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.742 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.742 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.742 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.742 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 3, 2024-09-21T09:31:07.742 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:07.742 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.742 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:07.743 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:07.743 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:07.743 INFO:tasks.workunit.client.0.smithi022.stdout: 6, 2024-09-21T09:31:07.743 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:07.743 INFO:tasks.workunit.client.0.smithi022.stdout: 2 2024-09-21T09:31:07.743 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.743 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 6, 2024-09-21T09:31:07.743 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "57'2" 2024-09-21T09:31:07.743 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.743 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.743 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:07.743 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:07.743 INFO:tasks.workunit.client.0.smithi022.stdout: 1, 2024-09-21T09:31:07.743 INFO:tasks.workunit.client.0.smithi022.stdout: 4 2024-09-21T09:31:07.744 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.744 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 0, 2024-09-21T09:31:07.744 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:07.744 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:07.744 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:07.744 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:07.744 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:07.744 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:07.744 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.141267+0000", 2024-09-21T09:31:07.744 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.634680+0000", 2024-09-21T09:31:07.744 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.634680+0000", 2024-09-21T09:31:07.744 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.637964+0000", 2024-09-21T09:31:07.744 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.141267+0000", 2024-09-21T09:31:07.744 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.744 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:07.745 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.745 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:07.745 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.141267+0000", 2024-09-21T09:31:07.745 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.141267+0000", 2024-09-21T09:31:07.745 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.141267+0000", 2024-09-21T09:31:07.745 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:07.745 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:07.745 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.745 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.141267+0000", 2024-09-21T09:31:07.745 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.141267+0000", 2024-09-21T09:31:07.745 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:07.745 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:07.745 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:07.746 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:07.746 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:07.746 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:07.746 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:07.746 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:07.746 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:07.746 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:07.746 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:07.746 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:07.746 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:07.746 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.6", 2024-09-21T09:31:07.746 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:07.746 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:07.746 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:07.746 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:07.747 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:07.747 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T19:50:36.642394+0000", 2024-09-21T09:31:07.747 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:07.747 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:07.747 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.747 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:07.747 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.747 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.747 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.747 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.747 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.747 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.747 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.747 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.748 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.748 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.748 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.748 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.748 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.748 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.748 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.748 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:07.748 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:07.748 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.748 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:07.748 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.748 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.748 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.748 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.749 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.749 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:07.749 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.749 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.749 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.749 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.749 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.749 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.749 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.749 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.749 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.749 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.749 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.749 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.749 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:07.750 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:07.750 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.750 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:07.750 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:07.750 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:07.750 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:07.750 INFO:tasks.workunit.client.0.smithi022.stdout: 1, 2024-09-21T09:31:07.750 INFO:tasks.workunit.client.0.smithi022.stdout: 4 2024-09-21T09:31:07.750 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.750 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 0, 2024-09-21T09:31:07.750 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:07.750 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.750 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.751 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:07.751 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:07.751 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:07.751 INFO:tasks.workunit.client.0.smithi022.stdout: 1 2024-09-21T09:31:07.751 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.751 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 7, 2024-09-21T09:31:07.751 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:07.751 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:07.751 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 51, 2024-09-21T09:31:07.751 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:07.751 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:07.751 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:07.751 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.359011+0000", 2024-09-21T09:31:07.751 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:32.268720+0000", 2024-09-21T09:31:07.752 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:32.268720+0000", 2024-09-21T09:31:07.752 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:32.270810+0000", 2024-09-21T09:31:07.752 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.359011+0000", 2024-09-21T09:31:07.752 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:07.752 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:07.752 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:07.752 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 52, 2024-09-21T09:31:07.752 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.359011+0000", 2024-09-21T09:31:07.752 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.359011+0000", 2024-09-21T09:31:07.752 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.359011+0000", 2024-09-21T09:31:07.752 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:07.752 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:07.752 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:07.752 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.359011+0000", 2024-09-21T09:31:07.753 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.359011+0000", 2024-09-21T09:31:07.753 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:07.753 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:07.753 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:07.753 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:07.753 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 51, 2024-09-21T09:31:07.753 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:07.753 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:07.753 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:07.753 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:07.753 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:07.753 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:07.753 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:07.754 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:07.754 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "2.8", 2024-09-21T09:31:07.754 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:07.754 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:07.754 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:07.754 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 24, 2024-09-21T09:31:07.754 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:07.754 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T13:58:54.820070+0000", 2024-09-21T09:31:07.754 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:07.754 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:07.754 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.754 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:07.754 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.755 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.755 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.755 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.755 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.755 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.755 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.755 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.755 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.755 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.755 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.755 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.755 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.755 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.756 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.756 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:07.756 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:07.756 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.756 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:07.756 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.756 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.756 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.756 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.756 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.756 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:07.756 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.756 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.756 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.757 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.757 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.757 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.757 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.757 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.757 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.757 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.757 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.757 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.757 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:07.757 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:07.757 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.757 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:07.757 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:07.757 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:07.758 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:07.758 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:07.758 INFO:tasks.workunit.client.0.smithi022.stdout: 1 2024-09-21T09:31:07.758 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.758 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 7, 2024-09-21T09:31:07.758 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:07.758 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.758 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.758 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:07.758 INFO:tasks.workunit.client.0.smithi022.stdout: 4, 2024-09-21T09:31:07.758 INFO:tasks.workunit.client.0.smithi022.stdout: 2, 2024-09-21T09:31:07.758 INFO:tasks.workunit.client.0.smithi022.stdout: 7 2024-09-21T09:31:07.758 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.758 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 4, 2024-09-21T09:31:07.758 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:07.759 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:07.759 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:07.759 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:07.759 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:07.759 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:07.759 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.062395+0000", 2024-09-21T09:31:07.759 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.637064+0000", 2024-09-21T09:31:07.759 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.637064+0000", 2024-09-21T09:31:07.759 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.637942+0000", 2024-09-21T09:31:07.759 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.062395+0000", 2024-09-21T09:31:07.759 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.759 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:07.759 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.760 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:07.760 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.062395+0000", 2024-09-21T09:31:07.760 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.062395+0000", 2024-09-21T09:31:07.760 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.062395+0000", 2024-09-21T09:31:07.760 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:07.760 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:07.760 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.760 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.062395+0000", 2024-09-21T09:31:07.760 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.062395+0000", 2024-09-21T09:31:07.760 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:07.760 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:07.760 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:07.760 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:07.760 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:07.761 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:07.761 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:07.761 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:07.761 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:07.761 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:07.761 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:07.761 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:07.761 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:07.761 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.9", 2024-09-21T09:31:07.761 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:07.761 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:07.761 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:07.761 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:07.761 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:07.762 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T10:16:53.911245+0000", 2024-09-21T09:31:07.762 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:07.762 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:07.762 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.762 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:07.762 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.762 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.762 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.762 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.762 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.762 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.762 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.762 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.763 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.763 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.763 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.763 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.763 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.763 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.763 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.763 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:07.763 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:07.763 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.763 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:07.763 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.763 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.763 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.763 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.764 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.764 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:07.764 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.764 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.764 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.764 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.764 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.764 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.764 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.764 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.764 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.764 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.764 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.764 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.765 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:07.765 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:07.765 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.765 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:07.765 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:07.765 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:07.765 INFO:tasks.workunit.client.0.smithi022.stdout: 4, 2024-09-21T09:31:07.765 INFO:tasks.workunit.client.0.smithi022.stdout: 2, 2024-09-21T09:31:07.765 INFO:tasks.workunit.client.0.smithi022.stdout: 7 2024-09-21T09:31:07.765 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.765 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 4, 2024-09-21T09:31:07.765 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:07.765 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.765 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.765 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:07.766 INFO:tasks.workunit.client.0.smithi022.stdout: 1, 2024-09-21T09:31:07.766 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:07.766 INFO:tasks.workunit.client.0.smithi022.stdout: 3 2024-09-21T09:31:07.766 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.766 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 1, 2024-09-21T09:31:07.766 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:07.766 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:07.766 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 51, 2024-09-21T09:31:07.766 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:07.766 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:07.766 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:07.766 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.062682+0000", 2024-09-21T09:31:07.766 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:32.262555+0000", 2024-09-21T09:31:07.767 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:32.262555+0000", 2024-09-21T09:31:07.767 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:32.262848+0000", 2024-09-21T09:31:07.767 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.062682+0000", 2024-09-21T09:31:07.767 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:07.767 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:07.767 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:07.767 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 52, 2024-09-21T09:31:07.767 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.062682+0000", 2024-09-21T09:31:07.767 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.062682+0000", 2024-09-21T09:31:07.767 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.062682+0000", 2024-09-21T09:31:07.767 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:07.767 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:07.767 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:07.767 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.062682+0000", 2024-09-21T09:31:07.768 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.062682+0000", 2024-09-21T09:31:07.768 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:07.768 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:07.768 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:07.768 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:07.768 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 51, 2024-09-21T09:31:07.768 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:07.768 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:07.768 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:07.768 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:07.768 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:07.768 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:07.769 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:07.769 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:07.769 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "2.9", 2024-09-21T09:31:07.769 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:07.769 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:07.769 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:07.769 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 24, 2024-09-21T09:31:07.769 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:07.769 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T20:27:02.670310+0000", 2024-09-21T09:31:07.769 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:07.769 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:07.769 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.769 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:07.769 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.769 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.770 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.770 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.770 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.770 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.770 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.770 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.770 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.770 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.770 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.770 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.770 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.770 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.770 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.771 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:07.771 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:07.771 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.771 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:07.771 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.771 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.771 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.771 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.771 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.771 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:07.771 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.771 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.771 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.771 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.772 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.772 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.772 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.772 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.772 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.772 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.772 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.772 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.772 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:07.772 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:07.772 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.772 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:07.772 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:07.772 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:07.772 INFO:tasks.workunit.client.0.smithi022.stdout: 1, 2024-09-21T09:31:07.773 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:07.773 INFO:tasks.workunit.client.0.smithi022.stdout: 3 2024-09-21T09:31:07.773 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.773 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 1, 2024-09-21T09:31:07.773 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:07.773 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.773 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.773 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:07.773 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:07.773 INFO:tasks.workunit.client.0.smithi022.stdout: 1, 2024-09-21T09:31:07.773 INFO:tasks.workunit.client.0.smithi022.stdout: 7 2024-09-21T09:31:07.773 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.773 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 3, 2024-09-21T09:31:07.773 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:07.774 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:07.774 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:07.774 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:07.774 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:07.774 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:07.774 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.125068+0000", 2024-09-21T09:31:07.774 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.625619+0000", 2024-09-21T09:31:07.774 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.625619+0000", 2024-09-21T09:31:07.774 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.625899+0000", 2024-09-21T09:31:07.774 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.125068+0000", 2024-09-21T09:31:07.774 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.774 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:07.774 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.774 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:07.775 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.125068+0000", 2024-09-21T09:31:07.775 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.125068+0000", 2024-09-21T09:31:07.775 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.125068+0000", 2024-09-21T09:31:07.775 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:07.775 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:07.775 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.775 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.125068+0000", 2024-09-21T09:31:07.775 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.125068+0000", 2024-09-21T09:31:07.775 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:07.775 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:07.775 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:07.775 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:07.775 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:07.775 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:07.776 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:07.776 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:07.776 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:07.776 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:07.776 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:07.776 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:07.776 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:07.776 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.8", 2024-09-21T09:31:07.776 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:07.776 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:07.776 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:07.776 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:07.776 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:07.776 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T14:38:48.577776+0000", 2024-09-21T09:31:07.777 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:07.777 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:07.777 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.777 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:07.777 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.777 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.777 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.777 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.777 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.777 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.777 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.777 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.777 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.777 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.777 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.778 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.778 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.778 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.778 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.778 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:07.778 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:07.778 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.778 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:07.778 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.778 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.778 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.778 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.778 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.778 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:07.779 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.779 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.779 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.779 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.779 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.779 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.779 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.779 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.779 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.779 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.779 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.779 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.779 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:07.779 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:07.779 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.780 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:07.780 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:07.780 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:07.780 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:07.780 INFO:tasks.workunit.client.0.smithi022.stdout: 1, 2024-09-21T09:31:07.780 INFO:tasks.workunit.client.0.smithi022.stdout: 7 2024-09-21T09:31:07.780 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.780 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 3, 2024-09-21T09:31:07.780 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:07.780 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.780 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.780 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:07.780 INFO:tasks.workunit.client.0.smithi022.stdout: 1, 2024-09-21T09:31:07.780 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:07.781 INFO:tasks.workunit.client.0.smithi022.stdout: 7 2024-09-21T09:31:07.781 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.781 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 1, 2024-09-21T09:31:07.781 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:07.781 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:07.781 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 51, 2024-09-21T09:31:07.781 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:07.781 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:07.781 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:07.781 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.063179+0000", 2024-09-21T09:31:07.781 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:32.264353+0000", 2024-09-21T09:31:07.781 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:32.264353+0000", 2024-09-21T09:31:07.781 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:32.264667+0000", 2024-09-21T09:31:07.781 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.063179+0000", 2024-09-21T09:31:07.781 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:07.782 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:07.782 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:07.782 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 52, 2024-09-21T09:31:07.782 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.063179+0000", 2024-09-21T09:31:07.782 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.063179+0000", 2024-09-21T09:31:07.782 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.063179+0000", 2024-09-21T09:31:07.782 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:07.782 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:07.782 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:07.782 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.063179+0000", 2024-09-21T09:31:07.782 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.063179+0000", 2024-09-21T09:31:07.782 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:07.782 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:07.782 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:07.783 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:07.783 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 51, 2024-09-21T09:31:07.783 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:07.783 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:07.783 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:07.783 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:07.783 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:07.783 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:07.783 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:07.783 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:07.783 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "2.a", 2024-09-21T09:31:07.783 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:07.783 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:07.783 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:07.783 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 24, 2024-09-21T09:31:07.784 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:07.784 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T17:21:53.938525+0000", 2024-09-21T09:31:07.784 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:07.784 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:07.784 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.784 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:07.784 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.784 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.784 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.784 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.784 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.784 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.784 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.784 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.785 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.785 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.785 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.785 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.785 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.785 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.785 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.785 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:07.785 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:07.785 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.785 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:07.785 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.785 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.786 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.786 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.786 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.786 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:07.786 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.786 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.786 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.786 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.786 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.786 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.786 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.786 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.786 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.786 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.787 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.787 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.787 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:07.787 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:07.787 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.787 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:07.787 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:07.787 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:07.787 INFO:tasks.workunit.client.0.smithi022.stdout: 1, 2024-09-21T09:31:07.787 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:07.787 INFO:tasks.workunit.client.0.smithi022.stdout: 7 2024-09-21T09:31:07.787 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.787 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 1, 2024-09-21T09:31:07.788 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:07.788 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.788 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.788 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:07.788 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:07.788 INFO:tasks.workunit.client.0.smithi022.stdout: 2, 2024-09-21T09:31:07.788 INFO:tasks.workunit.client.0.smithi022.stdout: 4 2024-09-21T09:31:07.788 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.788 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 3, 2024-09-21T09:31:07.788 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:07.788 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:07.788 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:07.788 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:07.788 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:07.788 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:07.789 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.125251+0000", 2024-09-21T09:31:07.789 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:52.080726+0000", 2024-09-21T09:31:07.789 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:52.080726+0000", 2024-09-21T09:31:07.789 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:52.081164+0000", 2024-09-21T09:31:07.789 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.125251+0000", 2024-09-21T09:31:07.789 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.789 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:07.789 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.789 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 59, 2024-09-21T09:31:07.789 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.125251+0000", 2024-09-21T09:31:07.789 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.125251+0000", 2024-09-21T09:31:07.789 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.125251+0000", 2024-09-21T09:31:07.789 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:07.789 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:07.789 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.790 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.125251+0000", 2024-09-21T09:31:07.790 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.125251+0000", 2024-09-21T09:31:07.790 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:07.790 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:07.790 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:07.790 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:07.790 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 58, 2024-09-21T09:31:07.790 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:07.790 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:07.790 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:07.790 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:07.790 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:07.790 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:07.790 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:07.791 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:07.791 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.b", 2024-09-21T09:31:07.791 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:07.791 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:07.791 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:07.791 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 31, 2024-09-21T09:31:07.791 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:07.791 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T17:37:01.620726+0000", 2024-09-21T09:31:07.791 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:07.791 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:07.791 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.791 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:07.791 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.791 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.792 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.792 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.792 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.792 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.792 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.792 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.792 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.792 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.792 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.792 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.792 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.792 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.792 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.792 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:07.793 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:07.793 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.793 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:07.793 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.793 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.793 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.793 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.793 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.793 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:07.793 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.793 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.793 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.793 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.793 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.794 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.794 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.794 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.794 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.794 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.794 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.794 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.794 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:07.794 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:07.794 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.794 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:07.794 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:07.794 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:07.794 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:07.795 INFO:tasks.workunit.client.0.smithi022.stdout: 2, 2024-09-21T09:31:07.795 INFO:tasks.workunit.client.0.smithi022.stdout: 4 2024-09-21T09:31:07.795 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.795 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 3, 2024-09-21T09:31:07.795 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:07.795 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.795 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.795 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:07.795 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:07.795 INFO:tasks.workunit.client.0.smithi022.stdout: 4, 2024-09-21T09:31:07.795 INFO:tasks.workunit.client.0.smithi022.stdout: 5 2024-09-21T09:31:07.795 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.795 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 7, 2024-09-21T09:31:07.795 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:07.796 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:07.796 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 51, 2024-09-21T09:31:07.796 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:07.796 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:07.796 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:07.796 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.359154+0000", 2024-09-21T09:31:07.796 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:32.274109+0000", 2024-09-21T09:31:07.796 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:32.274109+0000", 2024-09-21T09:31:07.796 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:32.274404+0000", 2024-09-21T09:31:07.796 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.359154+0000", 2024-09-21T09:31:07.796 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:07.796 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:07.796 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:07.796 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 52, 2024-09-21T09:31:07.797 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.359154+0000", 2024-09-21T09:31:07.797 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.359154+0000", 2024-09-21T09:31:07.797 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.359154+0000", 2024-09-21T09:31:07.797 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:07.797 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:07.797 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:07.797 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.359154+0000", 2024-09-21T09:31:07.797 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.359154+0000", 2024-09-21T09:31:07.797 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:07.797 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 2, 2024-09-21T09:31:07.797 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:07.797 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:07.797 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 51, 2024-09-21T09:31:07.797 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:07.798 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:07.798 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:07.798 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:07.798 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 2, 2024-09-21T09:31:07.798 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:07.798 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:07.798 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:07.798 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "2.b", 2024-09-21T09:31:07.798 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:07.798 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:07.798 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:07.798 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 26, 2024-09-21T09:31:07.798 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:07.799 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T14:24:06.607203+0000", 2024-09-21T09:31:07.799 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:07.799 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:07.799 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.799 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:07.799 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.799 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.799 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.799 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.799 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.799 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.799 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.800 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.800 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.800 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.800 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.800 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.800 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.800 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.800 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.800 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 3, 2024-09-21T09:31:07.800 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 1, 2024-09-21T09:31:07.800 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.800 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 1, 2024-09-21T09:31:07.800 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.800 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.800 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.801 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.801 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.801 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 1, 2024-09-21T09:31:07.801 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.801 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.801 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.801 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.801 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.801 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.801 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.801 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.801 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.801 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.801 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.802 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.802 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 1, 2024-09-21T09:31:07.802 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:07.802 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.802 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:07.802 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:07.802 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:07.802 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:07.802 INFO:tasks.workunit.client.0.smithi022.stdout: 4, 2024-09-21T09:31:07.802 INFO:tasks.workunit.client.0.smithi022.stdout: 5 2024-09-21T09:31:07.802 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.802 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 7, 2024-09-21T09:31:07.802 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "57'2" 2024-09-21T09:31:07.802 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.803 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.803 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:07.803 INFO:tasks.workunit.client.0.smithi022.stdout: 6, 2024-09-21T09:31:07.803 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:07.803 INFO:tasks.workunit.client.0.smithi022.stdout: 1 2024-09-21T09:31:07.803 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.803 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 6, 2024-09-21T09:31:07.803 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:07.803 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:07.803 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:07.803 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:07.803 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:07.803 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:07.803 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.483953+0000", 2024-09-21T09:31:07.804 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:52.050052+0000", 2024-09-21T09:31:07.804 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:52.050052+0000", 2024-09-21T09:31:07.804 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:52.050465+0000", 2024-09-21T09:31:07.804 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.483953+0000", 2024-09-21T09:31:07.804 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.804 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:07.804 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.804 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 59, 2024-09-21T09:31:07.804 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.483953+0000", 2024-09-21T09:31:07.804 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.483953+0000", 2024-09-21T09:31:07.804 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.483953+0000", 2024-09-21T09:31:07.804 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:07.804 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:07.804 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.805 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.483953+0000", 2024-09-21T09:31:07.805 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.483953+0000", 2024-09-21T09:31:07.805 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:07.805 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:07.805 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:07.805 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:07.805 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 58, 2024-09-21T09:31:07.805 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:07.805 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:07.805 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:07.805 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:07.805 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:07.805 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:07.805 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:07.806 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:07.806 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.a", 2024-09-21T09:31:07.806 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:07.806 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:07.806 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:07.806 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 31, 2024-09-21T09:31:07.806 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:07.806 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T17:58:35.370217+0000", 2024-09-21T09:31:07.806 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:07.806 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:07.806 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.806 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:07.806 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.806 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.807 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.807 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.807 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.807 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.807 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.807 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.807 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.807 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.807 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.807 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.807 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.807 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.807 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.807 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:07.808 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:07.808 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.808 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:07.808 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.808 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.808 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.808 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.808 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.808 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:07.808 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.808 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.808 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.808 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.809 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.809 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.809 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.809 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.809 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.809 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.809 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.809 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.809 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:07.809 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:07.809 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.809 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:07.809 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:07.810 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:07.810 INFO:tasks.workunit.client.0.smithi022.stdout: 6, 2024-09-21T09:31:07.810 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:07.810 INFO:tasks.workunit.client.0.smithi022.stdout: 1 2024-09-21T09:31:07.810 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.810 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 6, 2024-09-21T09:31:07.810 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:07.810 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.810 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.810 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:07.810 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:07.810 INFO:tasks.workunit.client.0.smithi022.stdout: 4, 2024-09-21T09:31:07.810 INFO:tasks.workunit.client.0.smithi022.stdout: 6 2024-09-21T09:31:07.810 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.811 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 5, 2024-09-21T09:31:07.811 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:07.811 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:07.811 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:07.811 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:07.811 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:07.811 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:07.811 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.066042+0000", 2024-09-21T09:31:07.811 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.637664+0000", 2024-09-21T09:31:07.811 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.637664+0000", 2024-09-21T09:31:07.811 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.637946+0000", 2024-09-21T09:31:07.811 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.066042+0000", 2024-09-21T09:31:07.811 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.811 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:07.812 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.812 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:07.812 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.066042+0000", 2024-09-21T09:31:07.812 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.066042+0000", 2024-09-21T09:31:07.812 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.066042+0000", 2024-09-21T09:31:07.812 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:07.812 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:07.812 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.812 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.066042+0000", 2024-09-21T09:31:07.812 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.066042+0000", 2024-09-21T09:31:07.812 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:07.812 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:07.813 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:07.813 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:07.813 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:07.813 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:07.813 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:07.813 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:07.813 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:07.813 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:07.813 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:07.814 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:07.814 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:07.814 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.1d", 2024-09-21T09:31:07.814 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:07.814 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:07.814 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:07.814 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:07.814 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:07.814 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T18:22:17.260192+0000", 2024-09-21T09:31:07.815 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:07.815 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:07.815 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.815 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:07.815 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.815 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.815 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.815 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.815 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.815 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.815 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.816 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.816 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.816 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.816 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.816 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.816 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.816 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.816 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.816 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:07.816 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:07.816 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.816 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:07.817 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.817 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.817 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.817 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.817 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.817 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:07.817 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.817 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.817 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.817 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.817 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.817 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.817 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.817 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.818 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.818 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.818 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.818 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.818 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:07.818 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:07.818 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.818 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:07.818 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:07.818 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:07.818 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:07.818 INFO:tasks.workunit.client.0.smithi022.stdout: 4, 2024-09-21T09:31:07.818 INFO:tasks.workunit.client.0.smithi022.stdout: 6 2024-09-21T09:31:07.818 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.819 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 5, 2024-09-21T09:31:07.819 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:07.819 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.819 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.819 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:07.819 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:07.819 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:07.819 INFO:tasks.workunit.client.0.smithi022.stdout: 2 2024-09-21T09:31:07.819 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.819 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 7, 2024-09-21T09:31:07.819 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:07.819 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:07.819 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 51, 2024-09-21T09:31:07.819 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:07.820 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:07.820 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:07.820 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.359224+0000", 2024-09-21T09:31:07.820 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:32.268038+0000", 2024-09-21T09:31:07.820 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:32.268038+0000", 2024-09-21T09:31:07.820 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:32.268336+0000", 2024-09-21T09:31:07.820 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.359224+0000", 2024-09-21T09:31:07.820 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:07.820 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:07.820 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:07.820 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 52, 2024-09-21T09:31:07.820 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.359224+0000", 2024-09-21T09:31:07.821 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.359224+0000", 2024-09-21T09:31:07.821 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.359224+0000", 2024-09-21T09:31:07.821 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:07.821 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:07.821 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:07.821 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.359224+0000", 2024-09-21T09:31:07.821 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.359224+0000", 2024-09-21T09:31:07.821 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:07.821 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 2, 2024-09-21T09:31:07.821 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:07.821 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:07.821 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 51, 2024-09-21T09:31:07.821 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:07.822 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:07.822 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:07.822 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:07.822 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 2, 2024-09-21T09:31:07.822 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:07.822 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:07.822 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:07.822 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "2.1c", 2024-09-21T09:31:07.822 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:07.822 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:07.822 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:07.822 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 26, 2024-09-21T09:31:07.822 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:07.823 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T16:57:13.991593+0000", 2024-09-21T09:31:07.823 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:07.823 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:07.823 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.823 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:07.823 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.823 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.823 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.823 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.823 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.823 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.823 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.823 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.823 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.824 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.824 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.824 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.824 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.824 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.824 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.824 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 3, 2024-09-21T09:31:07.824 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 1, 2024-09-21T09:31:07.824 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.824 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 1, 2024-09-21T09:31:07.824 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.824 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.824 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.825 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.825 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.825 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 1, 2024-09-21T09:31:07.825 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.825 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.825 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.825 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.825 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.825 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.825 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.825 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.825 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.825 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.825 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.826 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.826 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 3, 2024-09-21T09:31:07.826 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:07.826 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.826 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:07.826 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:07.826 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:07.826 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:07.826 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:07.826 INFO:tasks.workunit.client.0.smithi022.stdout: 2 2024-09-21T09:31:07.826 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.826 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 7, 2024-09-21T09:31:07.826 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "57'2" 2024-09-21T09:31:07.827 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.827 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.827 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:07.827 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:07.827 INFO:tasks.workunit.client.0.smithi022.stdout: 4, 2024-09-21T09:31:07.827 INFO:tasks.workunit.client.0.smithi022.stdout: 1 2024-09-21T09:31:07.827 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.827 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 5, 2024-09-21T09:31:07.827 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:07.827 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:07.827 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:07.827 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:07.827 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:07.827 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:07.828 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.066636+0000", 2024-09-21T09:31:07.828 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.642487+0000", 2024-09-21T09:31:07.828 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.642487+0000", 2024-09-21T09:31:07.828 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.642780+0000", 2024-09-21T09:31:07.828 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.066636+0000", 2024-09-21T09:31:07.828 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.828 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:07.828 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.828 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:07.828 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.066636+0000", 2024-09-21T09:31:07.828 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.066636+0000", 2024-09-21T09:31:07.828 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.066636+0000", 2024-09-21T09:31:07.828 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:07.828 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:07.829 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.829 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.066636+0000", 2024-09-21T09:31:07.829 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.066636+0000", 2024-09-21T09:31:07.829 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:07.829 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:07.829 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:07.829 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:07.829 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:07.829 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:07.829 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:07.829 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:07.829 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:07.829 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:07.830 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:07.830 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:07.830 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:07.830 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.1c", 2024-09-21T09:31:07.830 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:07.830 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:07.830 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:07.830 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:07.830 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:07.830 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T15:28:54.864609+0000", 2024-09-21T09:31:07.830 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:07.830 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:07.830 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.831 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:07.831 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.831 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.831 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.831 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.831 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.831 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.831 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.831 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.831 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.831 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.831 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.831 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.831 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.832 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.832 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.832 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:07.832 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:07.832 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.832 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:07.832 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.832 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.832 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.832 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.832 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.832 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:07.832 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.832 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.833 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.833 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.833 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.833 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.833 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.833 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.833 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.833 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.833 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.833 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.833 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:07.833 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:07.833 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.833 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:07.833 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:07.834 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:07.834 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:07.834 INFO:tasks.workunit.client.0.smithi022.stdout: 4, 2024-09-21T09:31:07.834 INFO:tasks.workunit.client.0.smithi022.stdout: 1 2024-09-21T09:31:07.834 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.834 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 5, 2024-09-21T09:31:07.834 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:07.834 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.834 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.834 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:07.834 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:07.834 INFO:tasks.workunit.client.0.smithi022.stdout: 6, 2024-09-21T09:31:07.834 INFO:tasks.workunit.client.0.smithi022.stdout: 0 2024-09-21T09:31:07.834 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.835 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 7, 2024-09-21T09:31:07.835 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:07.835 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:07.835 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 51, 2024-09-21T09:31:07.835 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:07.835 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:07.835 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:07.835 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.359353+0000", 2024-09-21T09:31:07.835 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:32.269469+0000", 2024-09-21T09:31:07.835 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:32.269469+0000", 2024-09-21T09:31:07.835 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:32.270398+0000", 2024-09-21T09:31:07.835 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.359353+0000", 2024-09-21T09:31:07.835 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:07.835 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:07.836 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:07.836 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 52, 2024-09-21T09:31:07.836 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.359353+0000", 2024-09-21T09:31:07.836 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.359353+0000", 2024-09-21T09:31:07.836 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.359353+0000", 2024-09-21T09:31:07.836 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:07.836 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:07.836 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:07.836 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.359353+0000", 2024-09-21T09:31:07.836 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.359353+0000", 2024-09-21T09:31:07.836 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:07.836 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 2, 2024-09-21T09:31:07.836 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:07.837 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:07.837 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 51, 2024-09-21T09:31:07.837 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:07.837 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:07.837 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:07.837 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:07.837 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 2, 2024-09-21T09:31:07.837 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:07.837 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:07.837 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:07.837 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "2.1d", 2024-09-21T09:31:07.837 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:07.837 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:07.837 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:07.838 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 26, 2024-09-21T09:31:07.838 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:07.838 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T17:11:17.381607+0000", 2024-09-21T09:31:07.838 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:07.838 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:07.838 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.838 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:07.838 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.838 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.838 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.838 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.838 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.838 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.838 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.839 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.839 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.839 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.839 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.839 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.839 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.839 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.839 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.839 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 3, 2024-09-21T09:31:07.839 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 1, 2024-09-21T09:31:07.839 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.839 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 1, 2024-09-21T09:31:07.839 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.839 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.840 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.840 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.840 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.840 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 1, 2024-09-21T09:31:07.840 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.840 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.840 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.840 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.840 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.840 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.840 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.840 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.840 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.840 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.841 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.841 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.841 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 3, 2024-09-21T09:31:07.841 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:07.841 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.841 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:07.841 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:07.841 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:07.841 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:07.841 INFO:tasks.workunit.client.0.smithi022.stdout: 6, 2024-09-21T09:31:07.841 INFO:tasks.workunit.client.0.smithi022.stdout: 0 2024-09-21T09:31:07.841 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.841 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 7, 2024-09-21T09:31:07.841 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "57'2" 2024-09-21T09:31:07.841 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.842 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.842 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:07.842 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:07.842 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:07.842 INFO:tasks.workunit.client.0.smithi022.stdout: 2 2024-09-21T09:31:07.842 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.842 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 0, 2024-09-21T09:31:07.842 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:07.842 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:07.842 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:07.842 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:07.842 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:07.842 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:07.843 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.141385+0000", 2024-09-21T09:31:07.843 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.646141+0000", 2024-09-21T09:31:07.843 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.646141+0000", 2024-09-21T09:31:07.843 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.646662+0000", 2024-09-21T09:31:07.843 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.141385+0000", 2024-09-21T09:31:07.843 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.843 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:07.843 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.843 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:07.843 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.141385+0000", 2024-09-21T09:31:07.843 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.141385+0000", 2024-09-21T09:31:07.843 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.141385+0000", 2024-09-21T09:31:07.843 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:07.843 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:07.844 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.844 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.141385+0000", 2024-09-21T09:31:07.844 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.141385+0000", 2024-09-21T09:31:07.844 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:07.844 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:07.844 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:07.844 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:07.844 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:07.844 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:07.844 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:07.844 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:07.844 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:07.844 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:07.844 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:07.845 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:07.845 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:07.845 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.1f", 2024-09-21T09:31:07.845 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:07.845 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:07.845 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:07.845 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:07.845 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:07.845 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T19:22:15.834922+0000", 2024-09-21T09:31:07.845 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:07.845 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:07.845 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.845 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:07.846 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.846 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.846 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.846 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.846 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.846 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.846 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.846 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.846 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.846 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.846 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.846 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.846 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.846 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.847 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.847 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:07.847 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:07.847 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.847 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:07.847 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.847 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.847 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.847 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.847 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.847 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:07.847 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.847 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.847 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.847 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.848 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.848 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.848 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.848 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.848 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.848 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.848 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.848 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.848 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:07.848 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:07.848 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.848 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:07.848 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:07.848 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:07.849 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:07.849 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:07.849 INFO:tasks.workunit.client.0.smithi022.stdout: 2 2024-09-21T09:31:07.849 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.849 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 0, 2024-09-21T09:31:07.849 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:07.849 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.849 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.849 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:07.849 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:07.849 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:07.849 INFO:tasks.workunit.client.0.smithi022.stdout: 5 2024-09-21T09:31:07.849 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.849 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 3, 2024-09-21T09:31:07.850 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:07.850 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:07.850 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 51, 2024-09-21T09:31:07.850 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:07.850 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:07.850 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:07.850 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.125593+0000", 2024-09-21T09:31:07.850 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:32.269813+0000", 2024-09-21T09:31:07.850 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:32.269813+0000", 2024-09-21T09:31:07.850 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:32.270481+0000", 2024-09-21T09:31:07.850 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.125593+0000", 2024-09-21T09:31:07.850 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:07.850 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:07.850 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:07.851 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 52, 2024-09-21T09:31:07.851 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.125593+0000", 2024-09-21T09:31:07.851 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.125593+0000", 2024-09-21T09:31:07.851 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.125593+0000", 2024-09-21T09:31:07.851 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:07.851 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:07.851 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:07.851 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.125593+0000", 2024-09-21T09:31:07.851 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.125593+0000", 2024-09-21T09:31:07.851 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:07.851 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 1, 2024-09-21T09:31:07.851 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:07.851 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:07.852 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 51, 2024-09-21T09:31:07.852 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:07.852 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:07.852 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:07.852 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:07.852 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 1, 2024-09-21T09:31:07.852 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:07.852 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:07.852 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:07.852 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "2.1e", 2024-09-21T09:31:07.852 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:07.852 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:07.852 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:07.852 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 25, 2024-09-21T09:31:07.853 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:07.853 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T15:28:41.266917+0000", 2024-09-21T09:31:07.853 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:07.853 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:07.853 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.853 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:07.853 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.853 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.853 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.853 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.853 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.853 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.853 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.853 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.854 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.854 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.854 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.854 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.854 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.854 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.854 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.854 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:07.854 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:07.854 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.854 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:07.854 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.854 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.854 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.855 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.855 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.855 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:07.855 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.855 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.855 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.855 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.855 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.855 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.855 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.855 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.855 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.855 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.855 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.855 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.856 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:07.856 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:07.856 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.856 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:07.856 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:07.856 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:07.856 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:07.856 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:07.856 INFO:tasks.workunit.client.0.smithi022.stdout: 5 2024-09-21T09:31:07.856 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.856 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 3, 2024-09-21T09:31:07.856 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "57'1" 2024-09-21T09:31:07.856 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.856 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.856 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:07.857 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:07.857 INFO:tasks.workunit.client.0.smithi022.stdout: 6, 2024-09-21T09:31:07.857 INFO:tasks.workunit.client.0.smithi022.stdout: 2 2024-09-21T09:31:07.857 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.857 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 3, 2024-09-21T09:31:07.857 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:07.857 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:07.857 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:07.857 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:07.857 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:07.857 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:07.857 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.125527+0000", 2024-09-21T09:31:07.857 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.636625+0000", 2024-09-21T09:31:07.857 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.636625+0000", 2024-09-21T09:31:07.858 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.636905+0000", 2024-09-21T09:31:07.858 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.125527+0000", 2024-09-21T09:31:07.858 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.858 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:07.858 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.858 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:07.858 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.125527+0000", 2024-09-21T09:31:07.858 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.125527+0000", 2024-09-21T09:31:07.858 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.125527+0000", 2024-09-21T09:31:07.858 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:07.858 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:07.858 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:07.858 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.125527+0000", 2024-09-21T09:31:07.859 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.125527+0000", 2024-09-21T09:31:07.859 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:07.859 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:07.859 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:07.859 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:07.859 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:07.859 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:07.859 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:07.859 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:07.859 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:07.859 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:07.859 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:07.859 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:07.860 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:07.860 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.1e", 2024-09-21T09:31:07.860 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:07.860 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:07.860 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:07.860 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:07.860 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:07.860 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T12:19:39.022799+0000", 2024-09-21T09:31:07.860 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:07.860 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:07.860 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.860 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:07.860 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.861 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.861 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.861 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.861 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.861 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.861 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.861 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.861 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.861 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.861 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.861 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.861 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.861 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.862 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.862 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:07.862 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:07.862 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.862 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:07.862 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.862 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.862 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.862 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.862 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.862 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:07.862 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.862 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.862 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.863 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.863 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.863 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.863 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.863 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.863 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.863 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.863 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.863 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.863 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:07.863 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:07.863 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.863 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:07.863 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:07.864 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:07.864 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:07.864 INFO:tasks.workunit.client.0.smithi022.stdout: 6, 2024-09-21T09:31:07.864 INFO:tasks.workunit.client.0.smithi022.stdout: 2 2024-09-21T09:31:07.864 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.864 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 3, 2024-09-21T09:31:07.864 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:07.864 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.864 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.864 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:07.864 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:07.864 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:07.864 INFO:tasks.workunit.client.0.smithi022.stdout: 4 2024-09-21T09:31:07.865 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.865 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 0, 2024-09-21T09:31:07.865 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:07.865 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:07.865 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 51, 2024-09-21T09:31:07.865 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:07.865 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:07.865 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:07.865 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.141529+0000", 2024-09-21T09:31:07.865 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:32.267583+0000", 2024-09-21T09:31:07.865 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:32.267583+0000", 2024-09-21T09:31:07.865 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:32.267938+0000", 2024-09-21T09:31:07.865 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.141529+0000", 2024-09-21T09:31:07.866 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:07.866 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:07.866 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:07.866 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 52, 2024-09-21T09:31:07.866 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.141529+0000", 2024-09-21T09:31:07.866 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.141529+0000", 2024-09-21T09:31:07.866 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.141529+0000", 2024-09-21T09:31:07.866 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:07.866 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:07.866 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:07.866 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.141529+0000", 2024-09-21T09:31:07.866 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.141529+0000", 2024-09-21T09:31:07.866 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:07.866 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 4, 2024-09-21T09:31:07.867 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:07.867 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:07.867 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 51, 2024-09-21T09:31:07.867 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:07.867 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:07.867 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:07.867 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:07.867 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 4, 2024-09-21T09:31:07.867 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:07.867 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:07.867 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:07.867 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "2.1f", 2024-09-21T09:31:07.867 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:07.867 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:07.868 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:07.868 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 28, 2024-09-21T09:31:07.868 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:07.868 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T12:35:00.591469+0000", 2024-09-21T09:31:07.868 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:07.868 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:07.868 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.868 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 570, 2024-09-21T09:31:07.868 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.868 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.868 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.868 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.868 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.869 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.869 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.869 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.869 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.869 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.869 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.869 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.869 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.869 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.869 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.869 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 6, 2024-09-21T09:31:07.869 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 2, 2024-09-21T09:31:07.870 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.870 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 2, 2024-09-21T09:31:07.870 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.870 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.870 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.870 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.870 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.870 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 1, 2024-09-21T09:31:07.870 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.870 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.870 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.870 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.870 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.871 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.871 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.871 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.871 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.871 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.871 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.871 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.871 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 4, 2024-09-21T09:31:07.871 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 1 2024-09-21T09:31:07.871 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.871 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:07.871 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:07.871 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:07.872 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:07.872 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:07.872 INFO:tasks.workunit.client.0.smithi022.stdout: 4 2024-09-21T09:31:07.872 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.872 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 0, 2024-09-21T09:31:07.872 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "57'4" 2024-09-21T09:31:07.872 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:07.872 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.872 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_per_pool": [ 2024-09-21T09:31:07.872 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.872 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": 24, 2024-09-21T09:31:07.872 INFO:tasks.workunit.client.0.smithi022.stdout: "application": [ 2024-09-21T09:31:07.872 INFO:tasks.workunit.client.0.smithi022.stdout: "rbd" 2024-09-21T09:31:07.873 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.873 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 3, 2024-09-21T09:31:07.873 INFO:tasks.workunit.client.0.smithi022.stdout: "num_pg": 8, 2024-09-21T09:31:07.873 INFO:tasks.workunit.client.0.smithi022.stdout: "num_store_stats": 8, 2024-09-21T09:31:07.873 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 3, 2024-09-21T09:31:07.873 INFO:tasks.workunit.client.0.smithi022.stdout: "poolid": 4, 2024-09-21T09:31:07.873 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.873 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 19, 2024-09-21T09:31:07.873 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.873 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.873 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.873 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.873 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.873 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.874 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.874 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.874 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.874 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.874 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.874 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.874 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.874 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.874 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.874 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 6, 2024-09-21T09:31:07.874 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 2, 2024-09-21T09:31:07.874 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.874 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 2, 2024-09-21T09:31:07.875 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.875 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.875 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.875 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.875 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.875 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:07.875 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.875 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.875 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.875 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.875 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.875 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.876 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.876 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.876 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.876 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.876 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.876 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.876 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 2, 2024-09-21T09:31:07.876 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 2 2024-09-21T09:31:07.876 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.876 INFO:tasks.workunit.client.0.smithi022.stdout: "store_stats": { 2024-09-21T09:31:07.876 INFO:tasks.workunit.client.0.smithi022.stdout: "allocated": 12288, 2024-09-21T09:31:07.876 INFO:tasks.workunit.client.0.smithi022.stdout: "available": 0, 2024-09-21T09:31:07.876 INFO:tasks.workunit.client.0.smithi022.stdout: "data_compressed": 0, 2024-09-21T09:31:07.877 INFO:tasks.workunit.client.0.smithi022.stdout: "data_compressed_allocated": 0, 2024-09-21T09:31:07.877 INFO:tasks.workunit.client.0.smithi022.stdout: "data_compressed_original": 0, 2024-09-21T09:31:07.877 INFO:tasks.workunit.client.0.smithi022.stdout: "data_stored": 57, 2024-09-21T09:31:07.877 INFO:tasks.workunit.client.0.smithi022.stdout: "internal_metadata": 0, 2024-09-21T09:31:07.877 INFO:tasks.workunit.client.0.smithi022.stdout: "internally_reserved": 0, 2024-09-21T09:31:07.877 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_allocated": 0, 2024-09-21T09:31:07.877 INFO:tasks.workunit.client.0.smithi022.stdout: "total": 0 2024-09-21T09:31:07.877 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.877 INFO:tasks.workunit.client.0.smithi022.stdout: "up": 24 2024-09-21T09:31:07.877 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.877 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.877 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": 192, 2024-09-21T09:31:07.877 INFO:tasks.workunit.client.0.smithi022.stdout: "application": [ 2024-09-21T09:31:07.878 INFO:tasks.workunit.client.0.smithi022.stdout: "cephfs" 2024-09-21T09:31:07.878 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.878 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:07.878 INFO:tasks.workunit.client.0.smithi022.stdout: "num_pg": 64, 2024-09-21T09:31:07.878 INFO:tasks.workunit.client.0.smithi022.stdout: "num_store_stats": 8, 2024-09-21T09:31:07.878 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:07.878 INFO:tasks.workunit.client.0.smithi022.stdout: "poolid": 3, 2024-09-21T09:31:07.878 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.878 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:07.878 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.878 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.878 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.878 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.879 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.879 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.879 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.879 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.879 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.879 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.879 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.879 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.879 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.879 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.879 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.879 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:07.879 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:07.879 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.880 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:07.880 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.880 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.880 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.880 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.880 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.880 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:07.880 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.880 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.880 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.880 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.880 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.880 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.881 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.881 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.881 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.881 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.881 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.881 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.881 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:07.881 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:07.881 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.881 INFO:tasks.workunit.client.0.smithi022.stdout: "store_stats": { 2024-09-21T09:31:07.881 INFO:tasks.workunit.client.0.smithi022.stdout: "allocated": 0, 2024-09-21T09:31:07.881 INFO:tasks.workunit.client.0.smithi022.stdout: "available": 0, 2024-09-21T09:31:07.881 INFO:tasks.workunit.client.0.smithi022.stdout: "data_compressed": 0, 2024-09-21T09:31:07.881 INFO:tasks.workunit.client.0.smithi022.stdout: "data_compressed_allocated": 0, 2024-09-21T09:31:07.882 INFO:tasks.workunit.client.0.smithi022.stdout: "data_compressed_original": 0, 2024-09-21T09:31:07.882 INFO:tasks.workunit.client.0.smithi022.stdout: "data_stored": 0, 2024-09-21T09:31:07.882 INFO:tasks.workunit.client.0.smithi022.stdout: "internal_metadata": 0, 2024-09-21T09:31:07.882 INFO:tasks.workunit.client.0.smithi022.stdout: "internally_reserved": 0, 2024-09-21T09:31:07.882 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_allocated": 0, 2024-09-21T09:31:07.882 INFO:tasks.workunit.client.0.smithi022.stdout: "total": 0 2024-09-21T09:31:07.882 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.882 INFO:tasks.workunit.client.0.smithi022.stdout: "up": 192 2024-09-21T09:31:07.882 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.882 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.882 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": 96, 2024-09-21T09:31:07.882 INFO:tasks.workunit.client.0.smithi022.stdout: "application": [ 2024-09-21T09:31:07.882 INFO:tasks.workunit.client.0.smithi022.stdout: "cephfs" 2024-09-21T09:31:07.882 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.883 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 45, 2024-09-21T09:31:07.883 INFO:tasks.workunit.client.0.smithi022.stdout: "num_pg": 32, 2024-09-21T09:31:07.883 INFO:tasks.workunit.client.0.smithi022.stdout: "num_store_stats": 8, 2024-09-21T09:31:07.883 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 45, 2024-09-21T09:31:07.883 INFO:tasks.workunit.client.0.smithi022.stdout: "poolid": 2, 2024-09-21T09:31:07.883 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.883 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 3410, 2024-09-21T09:31:07.883 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.883 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:07.883 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.883 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.883 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.883 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.883 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.884 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.884 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.884 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.884 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.884 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.884 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.884 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.884 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.884 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 66, 2024-09-21T09:31:07.884 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 22, 2024-09-21T09:31:07.884 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.884 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 22, 2024-09-21T09:31:07.884 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.884 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.884 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.885 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.885 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.885 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 14, 2024-09-21T09:31:07.885 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.885 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:07.885 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.885 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.885 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.885 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.885 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.885 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:07.885 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:07.885 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.885 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.886 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.886 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 51, 2024-09-21T09:31:07.886 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 19 2024-09-21T09:31:07.886 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.886 INFO:tasks.workunit.client.0.smithi022.stdout: "store_stats": { 2024-09-21T09:31:07.886 INFO:tasks.workunit.client.0.smithi022.stdout: "allocated": 98304, 2024-09-21T09:31:07.886 INFO:tasks.workunit.client.0.smithi022.stdout: "available": 0, 2024-09-21T09:31:07.886 INFO:tasks.workunit.client.0.smithi022.stdout: "data_compressed": 0, 2024-09-21T09:31:07.886 INFO:tasks.workunit.client.0.smithi022.stdout: "data_compressed_allocated": 0, 2024-09-21T09:31:07.886 INFO:tasks.workunit.client.0.smithi022.stdout: "data_compressed_original": 0, 2024-09-21T09:31:07.886 INFO:tasks.workunit.client.0.smithi022.stdout: "data_stored": 10230, 2024-09-21T09:31:07.886 INFO:tasks.workunit.client.0.smithi022.stdout: "internal_metadata": 0, 2024-09-21T09:31:07.886 INFO:tasks.workunit.client.0.smithi022.stdout: "internally_reserved": 0, 2024-09-21T09:31:07.887 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_allocated": 0, 2024-09-21T09:31:07.887 INFO:tasks.workunit.client.0.smithi022.stdout: "total": 0 2024-09-21T09:31:07.887 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.887 INFO:tasks.workunit.client.0.smithi022.stdout: "up": 96 2024-09-21T09:31:07.887 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.887 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.887 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": 3, 2024-09-21T09:31:07.887 INFO:tasks.workunit.client.0.smithi022.stdout: "application": [ 2024-09-21T09:31:07.887 INFO:tasks.workunit.client.0.smithi022.stdout: "mgr" 2024-09-21T09:31:07.887 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.887 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 79, 2024-09-21T09:31:07.887 INFO:tasks.workunit.client.0.smithi022.stdout: "num_pg": 1, 2024-09-21T09:31:07.887 INFO:tasks.workunit.client.0.smithi022.stdout: "num_store_stats": 3, 2024-09-21T09:31:07.888 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 79, 2024-09-21T09:31:07.888 INFO:tasks.workunit.client.0.smithi022.stdout: "poolid": 1, 2024-09-21T09:31:07.888 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:07.888 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 590368, 2024-09-21T09:31:07.888 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:07.888 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 590368, 2024-09-21T09:31:07.888 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:07.888 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:07.888 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:07.888 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:07.888 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:07.888 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:07.888 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:07.888 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:07.889 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:07.889 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:07.889 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:07.889 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:07.889 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:07.889 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 6, 2024-09-21T09:31:07.889 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 2, 2024-09-21T09:31:07.889 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:07.889 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 2, 2024-09-21T09:31:07.889 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:07.889 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:07.889 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:07.889 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:07.889 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:07.890 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:07.890 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:07.890 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 2, 2024-09-21T09:31:07.890 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:07.890 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:07.890 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:07.890 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:07.890 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:07.890 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 126, 2024-09-21T09:31:07.890 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 231, 2024-09-21T09:31:07.890 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:07.890 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:07.890 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:07.891 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 113, 2024-09-21T09:31:07.891 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 1628 2024-09-21T09:31:07.891 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.891 INFO:tasks.workunit.client.0.smithi022.stdout: "store_stats": { 2024-09-21T09:31:07.891 INFO:tasks.workunit.client.0.smithi022.stdout: "allocated": 1781760, 2024-09-21T09:31:07.891 INFO:tasks.workunit.client.0.smithi022.stdout: "available": 0, 2024-09-21T09:31:07.891 INFO:tasks.workunit.client.0.smithi022.stdout: "data_compressed": 0, 2024-09-21T09:31:07.891 INFO:tasks.workunit.client.0.smithi022.stdout: "data_compressed_allocated": 0, 2024-09-21T09:31:07.891 INFO:tasks.workunit.client.0.smithi022.stdout: "data_compressed_original": 0, 2024-09-21T09:31:07.891 INFO:tasks.workunit.client.0.smithi022.stdout: "data_stored": 1771104, 2024-09-21T09:31:07.891 INFO:tasks.workunit.client.0.smithi022.stdout: "internal_metadata": 0, 2024-09-21T09:31:07.891 INFO:tasks.workunit.client.0.smithi022.stdout: "internally_reserved": 0, 2024-09-21T09:31:07.891 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_allocated": 0, 2024-09-21T09:31:07.891 INFO:tasks.workunit.client.0.smithi022.stdout: "total": 0 2024-09-21T09:31:07.892 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.892 INFO:tasks.workunit.client.0.smithi022.stdout: "up": 3 2024-09-21T09:31:07.892 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:07.892 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.892 INFO:tasks.workunit.client.0.smithi022.stdout: "usage": { 2024-09-21T09:31:07.892 INFO:tasks.workunit.client.0.smithi022.stdout: "pg_num": 105, 2024-09-21T09:31:07.892 INFO:tasks.workunit.client.0.smithi022.stdout: "pools": 4, 2024-09-21T09:31:07.892 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_by_class": { 2024-09-21T09:31:07.892 INFO:tasks.workunit.client.0.smithi022.stdout: "ssd": { 2024-09-21T09:31:07.892 INFO:tasks.workunit.client.0.smithi022.stdout: "total_avail_bytes": 765516976128, 2024-09-21T09:31:07.892 INFO:tasks.workunit.client.0.smithi022.stdout: "total_bytes": 767960285184, 2024-09-21T09:31:07.892 INFO:tasks.workunit.client.0.smithi022.stdout: "total_used_bytes": 2443309056, 2024-09-21T09:31:07.892 INFO:tasks.workunit.client.0.smithi022.stdout: "total_used_raw_bytes": 2443309056, 2024-09-21T09:31:07.892 INFO:tasks.workunit.client.0.smithi022.stdout: "total_used_raw_ratio": 0.003181556472554803 2024-09-21T09:31:07.893 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:07.893 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.893 INFO:tasks.workunit.client.0.smithi022.stdout: "total_avail_bytes": 765516976128, 2024-09-21T09:31:07.893 INFO:tasks.workunit.client.0.smithi022.stdout: "total_bytes": 767960285184, 2024-09-21T09:31:07.893 INFO:tasks.workunit.client.0.smithi022.stdout: "total_used_bytes": 2443309056 2024-09-21T09:31:07.893 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:07.893 INFO:tasks.workunit.client.0.smithi022.stdout:} 2024-09-21T09:31:07.893 INFO:tasks.workunit.client.0.smithi022.stdout:Device report is generated separately. To see it run 'ceph telemetry show-device'. 2024-09-21T09:31:07.894 INFO:tasks.workunit.client.0.smithi022.stderr:+ ceph telemetry show-device 2024-09-21T09:31:07.895 INFO:tasks.workunit.client.0.smithi022.stdout:{ 2024-09-21T09:31:07.895 INFO:tasks.workunit.client.0.smithi022.stdout: "384b39b0-77fc-11ef-b8c0-0cc47a595de4": { 2024-09-21T09:31:07.895 INFO:tasks.workunit.client.0.smithi022.stdout: "INTEL_SSDPED1D480GA_384bca4c-77fc-11ef-b8c0-0cc47a595de4": { 2024-09-21T09:31:07.895 INFO:tasks.workunit.client.0.smithi022.stdout: "20240921-092509": { 2024-09-21T09:31:07.895 INFO:tasks.workunit.client.0.smithi022.stdout: "device": { 2024-09-21T09:31:07.895 INFO:tasks.workunit.client.0.smithi022.stdout: "info_name": "/dev/nvme0n1", 2024-09-21T09:31:07.895 INFO:tasks.workunit.client.0.smithi022.stdout: "name": "/dev/nvme0n1", 2024-09-21T09:31:07.895 INFO:tasks.workunit.client.0.smithi022.stdout: "protocol": "NVMe", 2024-09-21T09:31:07.895 INFO:tasks.workunit.client.0.smithi022.stdout: "type": "nvme" 2024-09-21T09:31:07.895 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.895 INFO:tasks.workunit.client.0.smithi022.stdout: "firmware_version": "E2010325", 2024-09-21T09:31:07.896 INFO:tasks.workunit.client.0.smithi022.stdout: "host_id": "384b39b0-77fc-11ef-b8c0-0cc47a595de4", 2024-09-21T09:31:07.896 INFO:tasks.workunit.client.0.smithi022.stdout: "json_format_version": [ 2024-09-21T09:31:07.896 INFO:tasks.workunit.client.0.smithi022.stdout: 1, 2024-09-21T09:31:07.896 INFO:tasks.workunit.client.0.smithi022.stdout: 0 2024-09-21T09:31:07.896 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.896 INFO:tasks.workunit.client.0.smithi022.stdout: "local_time": { 2024-09-21T09:31:07.896 INFO:tasks.workunit.client.0.smithi022.stdout: "asctime": "Sat Sep 21 09:25:09 2024 UTC", 2024-09-21T09:31:07.896 INFO:tasks.workunit.client.0.smithi022.stdout: "time_t": 1726910709 2024-09-21T09:31:07.896 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.896 INFO:tasks.workunit.client.0.smithi022.stdout: "logical_block_size": 512, 2024-09-21T09:31:07.896 INFO:tasks.workunit.client.0.smithi022.stdout: "model_name": "INTEL SSDPED1D480GA", 2024-09-21T09:31:07.896 INFO:tasks.workunit.client.0.smithi022.stdout: "nvme_controller_id": 0, 2024-09-21T09:31:07.896 INFO:tasks.workunit.client.0.smithi022.stdout: "nvme_ieee_oui_identifier": 6083300, 2024-09-21T09:31:07.897 INFO:tasks.workunit.client.0.smithi022.stdout: "nvme_namespaces": [ 2024-09-21T09:31:07.897 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.897 INFO:tasks.workunit.client.0.smithi022.stdout: "capacity": { 2024-09-21T09:31:07.897 INFO:tasks.workunit.client.0.smithi022.stdout: "blocks": 937703088, 2024-09-21T09:31:07.897 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 480103981056 2024-09-21T09:31:07.897 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.897 INFO:tasks.workunit.client.0.smithi022.stdout: "formatted_lba_size": 512, 2024-09-21T09:31:07.897 INFO:tasks.workunit.client.0.smithi022.stdout: "id": 1, 2024-09-21T09:31:07.897 INFO:tasks.workunit.client.0.smithi022.stdout: "size": { 2024-09-21T09:31:07.897 INFO:tasks.workunit.client.0.smithi022.stdout: "blocks": 937703088, 2024-09-21T09:31:07.897 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 480103981056 2024-09-21T09:31:07.897 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.897 INFO:tasks.workunit.client.0.smithi022.stdout: "utilization": { 2024-09-21T09:31:07.897 INFO:tasks.workunit.client.0.smithi022.stdout: "blocks": 937703088, 2024-09-21T09:31:07.898 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 480103981056 2024-09-21T09:31:07.898 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:07.898 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:07.898 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.898 INFO:tasks.workunit.client.0.smithi022.stdout: "nvme_number_of_namespaces": 1, 2024-09-21T09:31:07.898 INFO:tasks.workunit.client.0.smithi022.stdout: "nvme_pci_vendor": { 2024-09-21T09:31:07.898 INFO:tasks.workunit.client.0.smithi022.stdout: "id": 32902, 2024-09-21T09:31:07.898 INFO:tasks.workunit.client.0.smithi022.stdout: "subsystem_id": 32902 2024-09-21T09:31:07.898 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.898 INFO:tasks.workunit.client.0.smithi022.stdout: "nvme_smart_health_information_add_log": { 2024-09-21T09:31:07.898 INFO:tasks.workunit.client.0.smithi022.stdout: "Device stats": { 2024-09-21T09:31:07.898 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_fw_downgrades": { 2024-09-21T09:31:07.898 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 0, 2024-09-21T09:31:07.899 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 0 2024-09-21T09:31:07.899 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.899 INFO:tasks.workunit.client.0.smithi022.stdout: "crc_error_count": { 2024-09-21T09:31:07.899 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 100, 2024-09-21T09:31:07.899 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 0 2024-09-21T09:31:07.899 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.899 INFO:tasks.workunit.client.0.smithi022.stdout: "end_to_end_error_detection_count": { 2024-09-21T09:31:07.899 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 100, 2024-09-21T09:31:07.899 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 0 2024-09-21T09:31:07.899 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.899 INFO:tasks.workunit.client.0.smithi022.stdout: "erase_fail_count": { 2024-09-21T09:31:07.899 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 0, 2024-09-21T09:31:07.899 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 0 2024-09-21T09:31:07.899 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.900 INFO:tasks.workunit.client.0.smithi022.stdout: "host_bytes_written": { 2024-09-21T09:31:07.900 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 100, 2024-09-21T09:31:07.900 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 6829746 2024-09-21T09:31:07.900 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.900 INFO:tasks.workunit.client.0.smithi022.stdout: "host_ctx_wear_used": { 2024-09-21T09:31:07.900 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 0, 2024-09-21T09:31:07.900 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 0 2024-09-21T09:31:07.900 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.900 INFO:tasks.workunit.client.0.smithi022.stdout: "media_bytes_read": { 2024-09-21T09:31:07.900 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 0, 2024-09-21T09:31:07.900 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 0 2024-09-21T09:31:07.900 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.900 INFO:tasks.workunit.client.0.smithi022.stdout: "nand_bytes_written": { 2024-09-21T09:31:07.900 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 0, 2024-09-21T09:31:07.901 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 0 2024-09-21T09:31:07.901 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.901 INFO:tasks.workunit.client.0.smithi022.stdout: "perf_stat_indicator": { 2024-09-21T09:31:07.901 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 0, 2024-09-21T09:31:07.901 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 0 2024-09-21T09:31:07.901 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.901 INFO:tasks.workunit.client.0.smithi022.stdout: "pll_lock_loss_count": { 2024-09-21T09:31:07.901 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 100, 2024-09-21T09:31:07.901 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 0 2024-09-21T09:31:07.901 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.901 INFO:tasks.workunit.client.0.smithi022.stdout: "program_fail_count": { 2024-09-21T09:31:07.901 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 0, 2024-09-21T09:31:07.901 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 0 2024-09-21T09:31:07.902 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.902 INFO:tasks.workunit.client.0.smithi022.stdout: "re_alloc_sectr_cnt": { 2024-09-21T09:31:07.902 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 0, 2024-09-21T09:31:07.902 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 0 2024-09-21T09:31:07.902 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.902 INFO:tasks.workunit.client.0.smithi022.stdout: "retry_buffer_overflow_count": { 2024-09-21T09:31:07.902 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 100, 2024-09-21T09:31:07.902 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 0 2024-09-21T09:31:07.902 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.902 INFO:tasks.workunit.client.0.smithi022.stdout: "soft_ecc_err_rate": { 2024-09-21T09:31:07.902 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 0, 2024-09-21T09:31:07.902 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 0 2024-09-21T09:31:07.902 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.903 INFO:tasks.workunit.client.0.smithi022.stdout: "thermal_throttle_status": { 2024-09-21T09:31:07.903 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 100, 2024-09-21T09:31:07.903 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": { 2024-09-21T09:31:07.903 INFO:tasks.workunit.client.0.smithi022.stdout: "cnt": 0, 2024-09-21T09:31:07.903 INFO:tasks.workunit.client.0.smithi022.stdout: "pct": 0 2024-09-21T09:31:07.903 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:07.903 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.903 INFO:tasks.workunit.client.0.smithi022.stdout: "timed_workload_host_reads": { 2024-09-21T09:31:07.903 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 100, 2024-09-21T09:31:07.903 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 16 2024-09-21T09:31:07.903 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.903 INFO:tasks.workunit.client.0.smithi022.stdout: "timed_workload_media_wear": { 2024-09-21T09:31:07.903 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 100, 2024-09-21T09:31:07.903 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": "0.000000" 2024-09-21T09:31:07.904 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.904 INFO:tasks.workunit.client.0.smithi022.stdout: "timed_workload_timer": { 2024-09-21T09:31:07.904 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 100, 2024-09-21T09:31:07.904 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 1212022 2024-09-21T09:31:07.904 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.904 INFO:tasks.workunit.client.0.smithi022.stdout: "unexp_power_loss": { 2024-09-21T09:31:07.904 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 0, 2024-09-21T09:31:07.904 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 0 2024-09-21T09:31:07.904 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.904 INFO:tasks.workunit.client.0.smithi022.stdout: "wear_leveling": { 2024-09-21T09:31:07.904 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 0, 2024-09-21T09:31:07.904 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": { 2024-09-21T09:31:07.904 INFO:tasks.workunit.client.0.smithi022.stdout: "avg": 0, 2024-09-21T09:31:07.904 INFO:tasks.workunit.client.0.smithi022.stdout: "max": 0, 2024-09-21T09:31:07.904 INFO:tasks.workunit.client.0.smithi022.stdout: "min": 0 2024-09-21T09:31:07.905 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:07.905 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:07.905 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.905 INFO:tasks.workunit.client.0.smithi022.stdout: "Intel Smart log": "nvme0n1" 2024-09-21T09:31:07.905 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.905 INFO:tasks.workunit.client.0.smithi022.stdout: "nvme_smart_health_information_log": { 2024-09-21T09:31:07.905 INFO:tasks.workunit.client.0.smithi022.stdout: "available_spare": 100, 2024-09-21T09:31:07.905 INFO:tasks.workunit.client.0.smithi022.stdout: "available_spare_threshold": 0, 2024-09-21T09:31:07.905 INFO:tasks.workunit.client.0.smithi022.stdout: "controller_busy_time": 2680, 2024-09-21T09:31:07.905 INFO:tasks.workunit.client.0.smithi022.stdout: "critical_warning": 0, 2024-09-21T09:31:07.905 INFO:tasks.workunit.client.0.smithi022.stdout: "data_units_read": 95367672, 2024-09-21T09:31:07.905 INFO:tasks.workunit.client.0.smithi022.stdout: "data_units_written": 447594268, 2024-09-21T09:31:07.905 INFO:tasks.workunit.client.0.smithi022.stdout: "host_reads": 1403952490, 2024-09-21T09:31:07.905 INFO:tasks.workunit.client.0.smithi022.stdout: "host_writes": 10869660916, 2024-09-21T09:31:07.905 INFO:tasks.workunit.client.0.smithi022.stdout: "media_errors": 0, 2024-09-21T09:31:07.906 INFO:tasks.workunit.client.0.smithi022.stdout: "num_err_log_entries": 0, 2024-09-21T09:31:07.906 INFO:tasks.workunit.client.0.smithi022.stdout: "percentage_used": 0, 2024-09-21T09:31:07.906 INFO:tasks.workunit.client.0.smithi022.stdout: "power_cycles": 16189, 2024-09-21T09:31:07.906 INFO:tasks.workunit.client.0.smithi022.stdout: "power_on_hours": 27263, 2024-09-21T09:31:07.906 INFO:tasks.workunit.client.0.smithi022.stdout: "temperature": 31, 2024-09-21T09:31:07.906 INFO:tasks.workunit.client.0.smithi022.stdout: "unsafe_shutdowns": 5631 2024-09-21T09:31:07.906 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.906 INFO:tasks.workunit.client.0.smithi022.stdout: "nvme_vendor": "intel", 2024-09-21T09:31:07.906 INFO:tasks.workunit.client.0.smithi022.stdout: "nvme_version": { 2024-09-21T09:31:07.906 INFO:tasks.workunit.client.0.smithi022.stdout: "string": "<1.2", 2024-09-21T09:31:07.906 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.906 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.906 INFO:tasks.workunit.client.0.smithi022.stdout: "power_cycle_count": 16189, 2024-09-21T09:31:07.907 INFO:tasks.workunit.client.0.smithi022.stdout: "power_on_time": { 2024-09-21T09:31:07.907 INFO:tasks.workunit.client.0.smithi022.stdout: "hours": 27263 2024-09-21T09:31:07.907 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.907 INFO:tasks.workunit.client.0.smithi022.stdout: "serial_number": "deleted", 2024-09-21T09:31:07.907 INFO:tasks.workunit.client.0.smithi022.stdout: "smart_status": { 2024-09-21T09:31:07.907 INFO:tasks.workunit.client.0.smithi022.stdout: "nvme": { 2024-09-21T09:31:07.907 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.907 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.907 INFO:tasks.workunit.client.0.smithi022.stdout: "passed": true 2024-09-21T09:31:07.907 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.907 INFO:tasks.workunit.client.0.smithi022.stdout: "smartctl": { 2024-09-21T09:31:07.907 INFO:tasks.workunit.client.0.smithi022.stdout: "argv": [ 2024-09-21T09:31:07.907 INFO:tasks.workunit.client.0.smithi022.stdout: "smartctl", 2024-09-21T09:31:07.907 INFO:tasks.workunit.client.0.smithi022.stdout: "-x", 2024-09-21T09:31:07.908 INFO:tasks.workunit.client.0.smithi022.stdout: "--json=o", 2024-09-21T09:31:07.908 INFO:tasks.workunit.client.0.smithi022.stdout: "/dev/nvme0n1" 2024-09-21T09:31:07.908 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.908 INFO:tasks.workunit.client.0.smithi022.stdout: "build_info": "(local build)", 2024-09-21T09:31:07.908 INFO:tasks.workunit.client.0.smithi022.stdout: "exit_status": 0, 2024-09-21T09:31:07.908 INFO:tasks.workunit.client.0.smithi022.stdout: "output": [ 2024-09-21T09:31:07.908 INFO:tasks.workunit.client.0.smithi022.stdout: "smartctl 7.2 2020-12-30 r5155 [x86_64-linux-5.14.0-508.el9.x86_64] (local build)", 2024-09-21T09:31:07.908 INFO:tasks.workunit.client.0.smithi022.stdout: "Copyright (C) 2002-20, Bruce Allen, Christian Franke, www.smartmontools.org", 2024-09-21T09:31:07.908 INFO:tasks.workunit.client.0.smithi022.stdout: "", 2024-09-21T09:31:07.908 INFO:tasks.workunit.client.0.smithi022.stdout: "=== START OF INFORMATION SECTION ===", 2024-09-21T09:31:07.908 INFO:tasks.workunit.client.0.smithi022.stdout: "Model Number: INTEL SSDPED1D480GA", 2024-09-21T09:31:07.908 INFO:tasks.workunit.client.0.smithi022.stdout: "Serial Number: deleted", 2024-09-21T09:31:07.908 INFO:tasks.workunit.client.0.smithi022.stdout: "Firmware Version: E2010325", 2024-09-21T09:31:07.908 INFO:tasks.workunit.client.0.smithi022.stdout: "PCI Vendor/Subsystem ID: 0x8086", 2024-09-21T09:31:07.909 INFO:tasks.workunit.client.0.smithi022.stdout: "IEEE OUI Identifier: 0x5cd2e4", 2024-09-21T09:31:07.909 INFO:tasks.workunit.client.0.smithi022.stdout: "Controller ID: 0", 2024-09-21T09:31:07.909 INFO:tasks.workunit.client.0.smithi022.stdout: "NVMe Version: <1.2", 2024-09-21T09:31:07.909 INFO:tasks.workunit.client.0.smithi022.stdout: "Number of Namespaces: 1", 2024-09-21T09:31:07.909 INFO:tasks.workunit.client.0.smithi022.stdout: "Namespace 1 Size/Capacity: 480,103,981,056 [480 GB]", 2024-09-21T09:31:07.909 INFO:tasks.workunit.client.0.smithi022.stdout: "Namespace 1 Formatted LBA Size: 512", 2024-09-21T09:31:07.909 INFO:tasks.workunit.client.0.smithi022.stdout: "Local Time is: Sat Sep 21 09:25:09 2024 UTC", 2024-09-21T09:31:07.909 INFO:tasks.workunit.client.0.smithi022.stdout: "Firmware Updates (0x02): 1 Slot", 2024-09-21T09:31:07.909 INFO:tasks.workunit.client.0.smithi022.stdout: "Optional Admin Commands (0x0007): Security Format Frmw_DL", 2024-09-21T09:31:07.909 INFO:tasks.workunit.client.0.smithi022.stdout: "Optional NVM Commands (0x0006): Wr_Unc DS_Mngmt", 2024-09-21T09:31:07.909 INFO:tasks.workunit.client.0.smithi022.stdout: "Log Page Attributes (0x02): Cmd_Eff_Lg", 2024-09-21T09:31:07.909 INFO:tasks.workunit.client.0.smithi022.stdout: "Maximum Data Transfer Size: 32 Pages", 2024-09-21T09:31:07.909 INFO:tasks.workunit.client.0.smithi022.stdout: "", 2024-09-21T09:31:07.910 INFO:tasks.workunit.client.0.smithi022.stdout: "Supported Power States", 2024-09-21T09:31:07.910 INFO:tasks.workunit.client.0.smithi022.stdout: "St Op Max Active Idle RL RT WL WT Ent_Lat Ex_Lat", 2024-09-21T09:31:07.910 INFO:tasks.workunit.client.0.smithi022.stdout: " 0 + 18.00W - - 0 0 0 0 0 0", 2024-09-21T09:31:07.910 INFO:tasks.workunit.client.0.smithi022.stdout: "", 2024-09-21T09:31:07.910 INFO:tasks.workunit.client.0.smithi022.stdout: "Supported LBA Sizes (NSID 0x1)", 2024-09-21T09:31:07.910 INFO:tasks.workunit.client.0.smithi022.stdout: "Id Fmt Data Metadt Rel_Perf", 2024-09-21T09:31:07.910 INFO:tasks.workunit.client.0.smithi022.stdout: " 0 + 512 0 2", 2024-09-21T09:31:07.910 INFO:tasks.workunit.client.0.smithi022.stdout: "", 2024-09-21T09:31:07.910 INFO:tasks.workunit.client.0.smithi022.stdout: "=== START OF SMART DATA SECTION ===", 2024-09-21T09:31:07.910 INFO:tasks.workunit.client.0.smithi022.stdout: "SMART overall-health self-assessment test result: PASSED", 2024-09-21T09:31:07.910 INFO:tasks.workunit.client.0.smithi022.stdout: "", 2024-09-21T09:31:07.910 INFO:tasks.workunit.client.0.smithi022.stdout: "SMART/Health Information (NVMe Log 0x02)", 2024-09-21T09:31:07.910 INFO:tasks.workunit.client.0.smithi022.stdout: "Critical Warning: 0x00", 2024-09-21T09:31:07.910 INFO:tasks.workunit.client.0.smithi022.stdout: "Temperature: 31 Celsius", 2024-09-21T09:31:07.911 INFO:tasks.workunit.client.0.smithi022.stdout: "Available Spare: 100%", 2024-09-21T09:31:07.911 INFO:tasks.workunit.client.0.smithi022.stdout: "Available Spare Threshold: 0%", 2024-09-21T09:31:07.911 INFO:tasks.workunit.client.0.smithi022.stdout: "Percentage Used: 0%", 2024-09-21T09:31:07.911 INFO:tasks.workunit.client.0.smithi022.stdout: "Data Units Read: 95,367,672 [48.8 TB]", 2024-09-21T09:31:07.911 INFO:tasks.workunit.client.0.smithi022.stdout: "Data Units Written: 447,594,268 [229 TB]", 2024-09-21T09:31:07.911 INFO:tasks.workunit.client.0.smithi022.stdout: "Host Read Commands: 1,403,952,490", 2024-09-21T09:31:07.911 INFO:tasks.workunit.client.0.smithi022.stdout: "Host Write Commands: 10,869,660,916", 2024-09-21T09:31:07.911 INFO:tasks.workunit.client.0.smithi022.stdout: "Controller Busy Time: 2,680", 2024-09-21T09:31:07.911 INFO:tasks.workunit.client.0.smithi022.stdout: "Power Cycles: 16,189", 2024-09-21T09:31:07.911 INFO:tasks.workunit.client.0.smithi022.stdout: "Power On Hours: 27,263", 2024-09-21T09:31:07.911 INFO:tasks.workunit.client.0.smithi022.stdout: "Unsafe Shutdowns: 5,631", 2024-09-21T09:31:07.911 INFO:tasks.workunit.client.0.smithi022.stdout: "Media and Data Integrity Errors: 0", 2024-09-21T09:31:07.911 INFO:tasks.workunit.client.0.smithi022.stdout: "Error Information Log Entries: 0", 2024-09-21T09:31:07.911 INFO:tasks.workunit.client.0.smithi022.stdout: "", 2024-09-21T09:31:07.912 INFO:tasks.workunit.client.0.smithi022.stdout: "Error Information (NVMe Log 0x01, 16 of 64 entries)", 2024-09-21T09:31:07.912 INFO:tasks.workunit.client.0.smithi022.stdout: "No Errors Logged", 2024-09-21T09:31:07.912 INFO:tasks.workunit.client.0.smithi022.stdout: "" 2024-09-21T09:31:07.912 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.912 INFO:tasks.workunit.client.0.smithi022.stdout: "platform_info": "x86_64-linux-5.14.0-508.el9.x86_64", 2024-09-21T09:31:07.912 INFO:tasks.workunit.client.0.smithi022.stdout: "svn_revision": "5155", 2024-09-21T09:31:07.912 INFO:tasks.workunit.client.0.smithi022.stdout: "version": [ 2024-09-21T09:31:07.912 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:07.912 INFO:tasks.workunit.client.0.smithi022.stdout: 2 2024-09-21T09:31:07.912 INFO:tasks.workunit.client.0.smithi022.stdout: ] 2024-09-21T09:31:07.912 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.912 INFO:tasks.workunit.client.0.smithi022.stdout: "temperature": { 2024-09-21T09:31:07.912 INFO:tasks.workunit.client.0.smithi022.stdout: "current": 31 2024-09-21T09:31:07.912 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.913 INFO:tasks.workunit.client.0.smithi022.stdout: "user_capacity": { 2024-09-21T09:31:07.913 INFO:tasks.workunit.client.0.smithi022.stdout: "blocks": 937703088, 2024-09-21T09:31:07.913 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 480103981056 2024-09-21T09:31:07.913 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:07.913 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:07.913 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:07.913 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.913 INFO:tasks.workunit.client.0.smithi022.stdout: "384d1fc8-77fc-11ef-b8c0-0cc47a595de4": { 2024-09-21T09:31:07.913 INFO:tasks.workunit.client.0.smithi022.stdout: "INTEL_SSDPEDMD400G4_384da218-77fc-11ef-b8c0-0cc47a595de4": { 2024-09-21T09:31:07.913 INFO:tasks.workunit.client.0.smithi022.stdout: "20240921-092513": { 2024-09-21T09:31:07.913 INFO:tasks.workunit.client.0.smithi022.stdout: "device": { 2024-09-21T09:31:07.913 INFO:tasks.workunit.client.0.smithi022.stdout: "info_name": "/dev/nvme0n1", 2024-09-21T09:31:07.913 INFO:tasks.workunit.client.0.smithi022.stdout: "name": "/dev/nvme0n1", 2024-09-21T09:31:07.913 INFO:tasks.workunit.client.0.smithi022.stdout: "protocol": "NVMe", 2024-09-21T09:31:07.914 INFO:tasks.workunit.client.0.smithi022.stdout: "type": "nvme" 2024-09-21T09:31:07.914 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.914 INFO:tasks.workunit.client.0.smithi022.stdout: "firmware_version": "8DV101H0", 2024-09-21T09:31:07.914 INFO:tasks.workunit.client.0.smithi022.stdout: "host_id": "384d1fc8-77fc-11ef-b8c0-0cc47a595de4", 2024-09-21T09:31:07.914 INFO:tasks.workunit.client.0.smithi022.stdout: "json_format_version": [ 2024-09-21T09:31:07.914 INFO:tasks.workunit.client.0.smithi022.stdout: 1, 2024-09-21T09:31:07.914 INFO:tasks.workunit.client.0.smithi022.stdout: 0 2024-09-21T09:31:07.914 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.914 INFO:tasks.workunit.client.0.smithi022.stdout: "local_time": { 2024-09-21T09:31:07.914 INFO:tasks.workunit.client.0.smithi022.stdout: "asctime": "Sat Sep 21 09:25:12 2024 UTC", 2024-09-21T09:31:07.914 INFO:tasks.workunit.client.0.smithi022.stdout: "time_t": 1726910712 2024-09-21T09:31:07.914 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.914 INFO:tasks.workunit.client.0.smithi022.stdout: "logical_block_size": 512, 2024-09-21T09:31:07.914 INFO:tasks.workunit.client.0.smithi022.stdout: "model_name": "INTEL SSDPEDMD400G4", 2024-09-21T09:31:07.915 INFO:tasks.workunit.client.0.smithi022.stdout: "nvme_controller_id": 0, 2024-09-21T09:31:07.915 INFO:tasks.workunit.client.0.smithi022.stdout: "nvme_ieee_oui_identifier": 6083300, 2024-09-21T09:31:07.915 INFO:tasks.workunit.client.0.smithi022.stdout: "nvme_namespaces": [ 2024-09-21T09:31:07.915 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.915 INFO:tasks.workunit.client.0.smithi022.stdout: "capacity": { 2024-09-21T09:31:07.915 INFO:tasks.workunit.client.0.smithi022.stdout: "blocks": 781422768, 2024-09-21T09:31:07.915 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 400088457216 2024-09-21T09:31:07.915 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.915 INFO:tasks.workunit.client.0.smithi022.stdout: "formatted_lba_size": 512, 2024-09-21T09:31:07.915 INFO:tasks.workunit.client.0.smithi022.stdout: "id": 1, 2024-09-21T09:31:07.915 INFO:tasks.workunit.client.0.smithi022.stdout: "size": { 2024-09-21T09:31:07.915 INFO:tasks.workunit.client.0.smithi022.stdout: "blocks": 781422768, 2024-09-21T09:31:07.915 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 400088457216 2024-09-21T09:31:07.915 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.916 INFO:tasks.workunit.client.0.smithi022.stdout: "utilization": { 2024-09-21T09:31:07.916 INFO:tasks.workunit.client.0.smithi022.stdout: "blocks": 781422768, 2024-09-21T09:31:07.916 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 400088457216 2024-09-21T09:31:07.916 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:07.916 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:07.916 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.916 INFO:tasks.workunit.client.0.smithi022.stdout: "nvme_number_of_namespaces": 1, 2024-09-21T09:31:07.916 INFO:tasks.workunit.client.0.smithi022.stdout: "nvme_pci_vendor": { 2024-09-21T09:31:07.916 INFO:tasks.workunit.client.0.smithi022.stdout: "id": 32902, 2024-09-21T09:31:07.916 INFO:tasks.workunit.client.0.smithi022.stdout: "subsystem_id": 32902 2024-09-21T09:31:07.916 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.916 INFO:tasks.workunit.client.0.smithi022.stdout: "nvme_smart_health_information_add_log": { 2024-09-21T09:31:07.916 INFO:tasks.workunit.client.0.smithi022.stdout: "Device stats": { 2024-09-21T09:31:07.916 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_fw_downgrades": { 2024-09-21T09:31:07.917 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 0, 2024-09-21T09:31:07.917 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 0 2024-09-21T09:31:07.917 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.917 INFO:tasks.workunit.client.0.smithi022.stdout: "crc_error_count": { 2024-09-21T09:31:07.917 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 100, 2024-09-21T09:31:07.917 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 0 2024-09-21T09:31:07.917 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.917 INFO:tasks.workunit.client.0.smithi022.stdout: "end_to_end_error_detection_count": { 2024-09-21T09:31:07.917 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 100, 2024-09-21T09:31:07.917 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 0 2024-09-21T09:31:07.917 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.917 INFO:tasks.workunit.client.0.smithi022.stdout: "erase_fail_count": { 2024-09-21T09:31:07.917 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 100, 2024-09-21T09:31:07.917 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 0 2024-09-21T09:31:07.918 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.918 INFO:tasks.workunit.client.0.smithi022.stdout: "host_bytes_written": { 2024-09-21T09:31:07.918 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 100, 2024-09-21T09:31:07.918 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 34797061 2024-09-21T09:31:07.918 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.918 INFO:tasks.workunit.client.0.smithi022.stdout: "host_ctx_wear_used": { 2024-09-21T09:31:07.918 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 100, 2024-09-21T09:31:07.918 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 1 2024-09-21T09:31:07.918 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.918 INFO:tasks.workunit.client.0.smithi022.stdout: "media_bytes_read": { 2024-09-21T09:31:07.918 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 0, 2024-09-21T09:31:07.918 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 0 2024-09-21T09:31:07.918 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.918 INFO:tasks.workunit.client.0.smithi022.stdout: "nand_bytes_written": { 2024-09-21T09:31:07.919 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 100, 2024-09-21T09:31:07.919 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 57236814 2024-09-21T09:31:07.919 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.919 INFO:tasks.workunit.client.0.smithi022.stdout: "perf_stat_indicator": { 2024-09-21T09:31:07.919 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 0, 2024-09-21T09:31:07.919 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 0 2024-09-21T09:31:07.919 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.919 INFO:tasks.workunit.client.0.smithi022.stdout: "pll_lock_loss_count": { 2024-09-21T09:31:07.919 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 100, 2024-09-21T09:31:07.919 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 0 2024-09-21T09:31:07.919 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.919 INFO:tasks.workunit.client.0.smithi022.stdout: "program_fail_count": { 2024-09-21T09:31:07.919 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 100, 2024-09-21T09:31:07.919 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 0 2024-09-21T09:31:07.919 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.920 INFO:tasks.workunit.client.0.smithi022.stdout: "re_alloc_sectr_cnt": { 2024-09-21T09:31:07.920 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 0, 2024-09-21T09:31:07.920 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 0 2024-09-21T09:31:07.920 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.920 INFO:tasks.workunit.client.0.smithi022.stdout: "retry_buffer_overflow_count": { 2024-09-21T09:31:07.920 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 100, 2024-09-21T09:31:07.920 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 0 2024-09-21T09:31:07.920 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.920 INFO:tasks.workunit.client.0.smithi022.stdout: "soft_ecc_err_rate": { 2024-09-21T09:31:07.920 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 0, 2024-09-21T09:31:07.920 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 0 2024-09-21T09:31:07.920 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.920 INFO:tasks.workunit.client.0.smithi022.stdout: "thermal_throttle_status": { 2024-09-21T09:31:07.920 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 100, 2024-09-21T09:31:07.921 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": { 2024-09-21T09:31:07.921 INFO:tasks.workunit.client.0.smithi022.stdout: "cnt": 0, 2024-09-21T09:31:07.921 INFO:tasks.workunit.client.0.smithi022.stdout: "pct": 0 2024-09-21T09:31:07.921 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:07.921 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.921 INFO:tasks.workunit.client.0.smithi022.stdout: "timed_workload_host_reads": { 2024-09-21T09:31:07.921 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 100, 2024-09-21T09:31:07.921 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 65535 2024-09-21T09:31:07.921 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.921 INFO:tasks.workunit.client.0.smithi022.stdout: "timed_workload_media_wear": { 2024-09-21T09:31:07.921 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 100, 2024-09-21T09:31:07.921 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": "63.999023" 2024-09-21T09:31:07.921 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.921 INFO:tasks.workunit.client.0.smithi022.stdout: "timed_workload_timer": { 2024-09-21T09:31:07.922 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 100, 2024-09-21T09:31:07.922 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 65535 2024-09-21T09:31:07.922 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.922 INFO:tasks.workunit.client.0.smithi022.stdout: "unexp_power_loss": { 2024-09-21T09:31:07.922 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 0, 2024-09-21T09:31:07.922 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 0 2024-09-21T09:31:07.922 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.922 INFO:tasks.workunit.client.0.smithi022.stdout: "wear_leveling": { 2024-09-21T09:31:07.922 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 88, 2024-09-21T09:31:07.922 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": { 2024-09-21T09:31:07.922 INFO:tasks.workunit.client.0.smithi022.stdout: "avg": 3844, 2024-09-21T09:31:07.922 INFO:tasks.workunit.client.0.smithi022.stdout: "max": 3860, 2024-09-21T09:31:07.922 INFO:tasks.workunit.client.0.smithi022.stdout: "min": 3828 2024-09-21T09:31:07.922 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:07.923 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:07.923 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.923 INFO:tasks.workunit.client.0.smithi022.stdout: "Intel Smart log": "nvme0n1" 2024-09-21T09:31:07.923 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.923 INFO:tasks.workunit.client.0.smithi022.stdout: "nvme_smart_health_information_log": { 2024-09-21T09:31:07.923 INFO:tasks.workunit.client.0.smithi022.stdout: "available_spare": 100, 2024-09-21T09:31:07.923 INFO:tasks.workunit.client.0.smithi022.stdout: "available_spare_threshold": 10, 2024-09-21T09:31:07.923 INFO:tasks.workunit.client.0.smithi022.stdout: "controller_busy_time": 39, 2024-09-21T09:31:07.923 INFO:tasks.workunit.client.0.smithi022.stdout: "critical_warning": 0, 2024-09-21T09:31:07.923 INFO:tasks.workunit.client.0.smithi022.stdout: "data_units_read": 467647523, 2024-09-21T09:31:07.923 INFO:tasks.workunit.client.0.smithi022.stdout: "data_units_written": 2280460231, 2024-09-21T09:31:07.923 INFO:tasks.workunit.client.0.smithi022.stdout: "host_reads": 8064859300, 2024-09-21T09:31:07.923 INFO:tasks.workunit.client.0.smithi022.stdout: "host_writes": 41665631302, 2024-09-21T09:31:07.923 INFO:tasks.workunit.client.0.smithi022.stdout: "media_errors": 0, 2024-09-21T09:31:07.924 INFO:tasks.workunit.client.0.smithi022.stdout: "num_err_log_entries": 0, 2024-09-21T09:31:07.924 INFO:tasks.workunit.client.0.smithi022.stdout: "percentage_used": 12, 2024-09-21T09:31:07.924 INFO:tasks.workunit.client.0.smithi022.stdout: "power_cycles": 55086, 2024-09-21T09:31:07.924 INFO:tasks.workunit.client.0.smithi022.stdout: "power_on_hours": 60974, 2024-09-21T09:31:07.924 INFO:tasks.workunit.client.0.smithi022.stdout: "temperature": 22, 2024-09-21T09:31:07.924 INFO:tasks.workunit.client.0.smithi022.stdout: "unsafe_shutdowns": 2616 2024-09-21T09:31:07.924 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.924 INFO:tasks.workunit.client.0.smithi022.stdout: "nvme_vendor": "intel", 2024-09-21T09:31:07.924 INFO:tasks.workunit.client.0.smithi022.stdout: "nvme_version": { 2024-09-21T09:31:07.924 INFO:tasks.workunit.client.0.smithi022.stdout: "string": "<1.2", 2024-09-21T09:31:07.924 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.924 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.924 INFO:tasks.workunit.client.0.smithi022.stdout: "power_cycle_count": 55086, 2024-09-21T09:31:07.924 INFO:tasks.workunit.client.0.smithi022.stdout: "power_on_time": { 2024-09-21T09:31:07.925 INFO:tasks.workunit.client.0.smithi022.stdout: "hours": 60974 2024-09-21T09:31:07.925 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.925 INFO:tasks.workunit.client.0.smithi022.stdout: "serial_number": "deleted", 2024-09-21T09:31:07.925 INFO:tasks.workunit.client.0.smithi022.stdout: "smart_status": { 2024-09-21T09:31:07.925 INFO:tasks.workunit.client.0.smithi022.stdout: "nvme": { 2024-09-21T09:31:07.925 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.925 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.925 INFO:tasks.workunit.client.0.smithi022.stdout: "passed": true 2024-09-21T09:31:07.925 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.925 INFO:tasks.workunit.client.0.smithi022.stdout: "smartctl": { 2024-09-21T09:31:07.925 INFO:tasks.workunit.client.0.smithi022.stdout: "argv": [ 2024-09-21T09:31:07.925 INFO:tasks.workunit.client.0.smithi022.stdout: "smartctl", 2024-09-21T09:31:07.925 INFO:tasks.workunit.client.0.smithi022.stdout: "-x", 2024-09-21T09:31:07.925 INFO:tasks.workunit.client.0.smithi022.stdout: "--json=o", 2024-09-21T09:31:07.925 INFO:tasks.workunit.client.0.smithi022.stdout: "/dev/nvme0n1" 2024-09-21T09:31:07.926 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.926 INFO:tasks.workunit.client.0.smithi022.stdout: "build_info": "(local build)", 2024-09-21T09:31:07.926 INFO:tasks.workunit.client.0.smithi022.stdout: "exit_status": 0, 2024-09-21T09:31:07.926 INFO:tasks.workunit.client.0.smithi022.stdout: "output": [ 2024-09-21T09:31:07.926 INFO:tasks.workunit.client.0.smithi022.stdout: "smartctl 7.2 2020-12-30 r5155 [x86_64-linux-5.14.0-508.el9.x86_64] (local build)", 2024-09-21T09:31:07.926 INFO:tasks.workunit.client.0.smithi022.stdout: "Copyright (C) 2002-20, Bruce Allen, Christian Franke, www.smartmontools.org", 2024-09-21T09:31:07.926 INFO:tasks.workunit.client.0.smithi022.stdout: "", 2024-09-21T09:31:07.926 INFO:tasks.workunit.client.0.smithi022.stdout: "=== START OF INFORMATION SECTION ===", 2024-09-21T09:31:07.926 INFO:tasks.workunit.client.0.smithi022.stdout: "Model Number: INTEL SSDPEDMD400G4", 2024-09-21T09:31:07.926 INFO:tasks.workunit.client.0.smithi022.stdout: "Serial Number: deleted", 2024-09-21T09:31:07.926 INFO:tasks.workunit.client.0.smithi022.stdout: "Firmware Version: 8DV101H0", 2024-09-21T09:31:07.926 INFO:tasks.workunit.client.0.smithi022.stdout: "PCI Vendor/Subsystem ID: 0x8086", 2024-09-21T09:31:07.926 INFO:tasks.workunit.client.0.smithi022.stdout: "IEEE OUI Identifier: 0x5cd2e4", 2024-09-21T09:31:07.927 INFO:tasks.workunit.client.0.smithi022.stdout: "Controller ID: 0", 2024-09-21T09:31:07.927 INFO:tasks.workunit.client.0.smithi022.stdout: "NVMe Version: <1.2", 2024-09-21T09:31:07.927 INFO:tasks.workunit.client.0.smithi022.stdout: "Number of Namespaces: 1", 2024-09-21T09:31:07.927 INFO:tasks.workunit.client.0.smithi022.stdout: "Namespace 1 Size/Capacity: 400,088,457,216 [400 GB]", 2024-09-21T09:31:07.927 INFO:tasks.workunit.client.0.smithi022.stdout: "Namespace 1 Formatted LBA Size: 512", 2024-09-21T09:31:07.927 INFO:tasks.workunit.client.0.smithi022.stdout: "Local Time is: Sat Sep 21 09:25:12 2024 UTC", 2024-09-21T09:31:07.927 INFO:tasks.workunit.client.0.smithi022.stdout: "Firmware Updates (0x02): 1 Slot", 2024-09-21T09:31:07.927 INFO:tasks.workunit.client.0.smithi022.stdout: "Optional Admin Commands (0x0006): Format Frmw_DL", 2024-09-21T09:31:07.927 INFO:tasks.workunit.client.0.smithi022.stdout: "Optional NVM Commands (0x0006): Wr_Unc DS_Mngmt", 2024-09-21T09:31:07.927 INFO:tasks.workunit.client.0.smithi022.stdout: "Log Page Attributes (0x02): Cmd_Eff_Lg", 2024-09-21T09:31:07.927 INFO:tasks.workunit.client.0.smithi022.stdout: "Maximum Data Transfer Size: 32 Pages", 2024-09-21T09:31:07.927 INFO:tasks.workunit.client.0.smithi022.stdout: "", 2024-09-21T09:31:07.927 INFO:tasks.workunit.client.0.smithi022.stdout: "Supported Power States", 2024-09-21T09:31:07.927 INFO:tasks.workunit.client.0.smithi022.stdout: "St Op Max Active Idle RL RT WL WT Ent_Lat Ex_Lat", 2024-09-21T09:31:07.928 INFO:tasks.workunit.client.0.smithi022.stdout: " 0 + 25.00W - - 0 0 0 0 0 0", 2024-09-21T09:31:07.928 INFO:tasks.workunit.client.0.smithi022.stdout: "", 2024-09-21T09:31:07.928 INFO:tasks.workunit.client.0.smithi022.stdout: "Supported LBA Sizes (NSID 0x1)", 2024-09-21T09:31:07.928 INFO:tasks.workunit.client.0.smithi022.stdout: "Id Fmt Data Metadt Rel_Perf", 2024-09-21T09:31:07.928 INFO:tasks.workunit.client.0.smithi022.stdout: " 0 + 512 0 2", 2024-09-21T09:31:07.928 INFO:tasks.workunit.client.0.smithi022.stdout: " 1 - 512 8 2", 2024-09-21T09:31:07.928 INFO:tasks.workunit.client.0.smithi022.stdout: " 2 - 512 16 2", 2024-09-21T09:31:07.928 INFO:tasks.workunit.client.0.smithi022.stdout: " 3 - 4096 0 0", 2024-09-21T09:31:07.928 INFO:tasks.workunit.client.0.smithi022.stdout: " 4 - 4096 8 0", 2024-09-21T09:31:07.928 INFO:tasks.workunit.client.0.smithi022.stdout: " 5 - 4096 64 0", 2024-09-21T09:31:07.928 INFO:tasks.workunit.client.0.smithi022.stdout: " 6 - 4096 128 0", 2024-09-21T09:31:07.928 INFO:tasks.workunit.client.0.smithi022.stdout: "", 2024-09-21T09:31:07.928 INFO:tasks.workunit.client.0.smithi022.stdout: "=== START OF SMART DATA SECTION ===", 2024-09-21T09:31:07.929 INFO:tasks.workunit.client.0.smithi022.stdout: "SMART overall-health self-assessment test result: PASSED", 2024-09-21T09:31:07.929 INFO:tasks.workunit.client.0.smithi022.stdout: "", 2024-09-21T09:31:07.929 INFO:tasks.workunit.client.0.smithi022.stdout: "SMART/Health Information (NVMe Log 0x02)", 2024-09-21T09:31:07.929 INFO:tasks.workunit.client.0.smithi022.stdout: "Critical Warning: 0x00", 2024-09-21T09:31:07.929 INFO:tasks.workunit.client.0.smithi022.stdout: "Temperature: 22 Celsius", 2024-09-21T09:31:07.929 INFO:tasks.workunit.client.0.smithi022.stdout: "Available Spare: 100%", 2024-09-21T09:31:07.929 INFO:tasks.workunit.client.0.smithi022.stdout: "Available Spare Threshold: 10%", 2024-09-21T09:31:07.929 INFO:tasks.workunit.client.0.smithi022.stdout: "Percentage Used: 12%", 2024-09-21T09:31:07.929 INFO:tasks.workunit.client.0.smithi022.stdout: "Data Units Read: 467,647,523 [239 TB]", 2024-09-21T09:31:07.929 INFO:tasks.workunit.client.0.smithi022.stdout: "Data Units Written: 2,280,460,231 [1.16 PB]", 2024-09-21T09:31:07.929 INFO:tasks.workunit.client.0.smithi022.stdout: "Host Read Commands: 8,064,859,300", 2024-09-21T09:31:07.929 INFO:tasks.workunit.client.0.smithi022.stdout: "Host Write Commands: 41,665,631,302", 2024-09-21T09:31:07.929 INFO:tasks.workunit.client.0.smithi022.stdout: "Controller Busy Time: 39", 2024-09-21T09:31:07.929 INFO:tasks.workunit.client.0.smithi022.stdout: "Power Cycles: 55,086", 2024-09-21T09:31:07.930 INFO:tasks.workunit.client.0.smithi022.stdout: "Power On Hours: 60,974", 2024-09-21T09:31:07.930 INFO:tasks.workunit.client.0.smithi022.stdout: "Unsafe Shutdowns: 2,616", 2024-09-21T09:31:07.930 INFO:tasks.workunit.client.0.smithi022.stdout: "Media and Data Integrity Errors: 0", 2024-09-21T09:31:07.930 INFO:tasks.workunit.client.0.smithi022.stdout: "Error Information Log Entries: 0", 2024-09-21T09:31:07.930 INFO:tasks.workunit.client.0.smithi022.stdout: "", 2024-09-21T09:31:07.930 INFO:tasks.workunit.client.0.smithi022.stdout: "Error Information (NVMe Log 0x01, 16 of 64 entries)", 2024-09-21T09:31:07.930 INFO:tasks.workunit.client.0.smithi022.stdout: "No Errors Logged", 2024-09-21T09:31:07.930 INFO:tasks.workunit.client.0.smithi022.stdout: "" 2024-09-21T09:31:07.930 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.930 INFO:tasks.workunit.client.0.smithi022.stdout: "platform_info": "x86_64-linux-5.14.0-508.el9.x86_64", 2024-09-21T09:31:07.930 INFO:tasks.workunit.client.0.smithi022.stdout: "svn_revision": "5155", 2024-09-21T09:31:07.930 INFO:tasks.workunit.client.0.smithi022.stdout: "version": [ 2024-09-21T09:31:07.930 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:07.930 INFO:tasks.workunit.client.0.smithi022.stdout: 2 2024-09-21T09:31:07.931 INFO:tasks.workunit.client.0.smithi022.stdout: ] 2024-09-21T09:31:07.931 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.931 INFO:tasks.workunit.client.0.smithi022.stdout: "temperature": { 2024-09-21T09:31:07.931 INFO:tasks.workunit.client.0.smithi022.stdout: "current": 22 2024-09-21T09:31:07.931 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.931 INFO:tasks.workunit.client.0.smithi022.stdout: "user_capacity": { 2024-09-21T09:31:07.931 INFO:tasks.workunit.client.0.smithi022.stdout: "blocks": 781422768, 2024-09-21T09:31:07.931 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 400088457216 2024-09-21T09:31:07.931 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:07.931 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:07.931 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:07.931 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:07.931 INFO:tasks.workunit.client.0.smithi022.stdout:} 2024-09-21T09:31:07.932 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:06 smithi064 ceph-mon[31124]: from='client.14841 -' entity='client.admin' cmd=[{"prefix": "telemetry enable channel", "channels": ["perf"], "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:31:07.932 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:06 smithi064 ceph-mon[31124]: from='client.? 172.21.15.22:0/1027081936' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/telemetry/last_opt_revision"}]: dispatch 2024-09-21T09:31:07.932 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:06 smithi064 ceph-mon[31124]: from='client.24758 -' entity='client.admin' cmd=[{"prefix": "telemetry show", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:31:07.932 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:06 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:31:07.932 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:06 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-21T09:31:07.932 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:06 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:31:07.932 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:06 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:07.932 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:06 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:07.932 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:06 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:31:07.932 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:06 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:07.933 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:06 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:07.933 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:06 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:31:07.933 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:06 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:07.933 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:06 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:07.933 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:06 smithi064 ceph-mon[31124]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:31:07.933 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:06 smithi064 ceph-mon[31124]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:31:07.933 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:06 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:31:07.933 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:06 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:07.933 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:06 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:07.933 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:06 smithi064 ceph-mon[31124]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:31:07.933 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:06 smithi064 ceph-mon[31124]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:31:07.933 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:06 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:31:07.933 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:06 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:07.933 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:06 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:07.933 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:06 smithi064 ceph-mon[31124]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:31:07.934 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:06 smithi064 ceph-mon[31124]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:31:07.934 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:06 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:31:07.934 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:06 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:07.934 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:06 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:07.934 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:06 smithi064 ceph-mon[31124]: from='client.24758 -' entity='client.admin' cmd=[{"prefix": "telemetry show", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:31:07.934 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:06 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:31:07.934 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:06 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-21T09:31:07.934 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:06 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:31:07.934 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:06 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:07.934 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:06 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:07.934 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:06 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:31:07.934 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:06 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:07.934 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:06 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:07.934 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:06 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:31:07.935 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:06 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:07.935 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:06 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:07.935 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:06 smithi064 ceph-mon[31124]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:31:07.935 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:06 smithi064 ceph-mon[31124]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:31:07.935 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:06 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:31:07.935 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:06 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:07.935 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:06 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:07.935 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:06 smithi064 ceph-mon[31124]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:31:07.935 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:06 smithi064 ceph-mon[31124]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:31:07.935 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:06 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:31:07.935 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:06 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:07.935 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:06 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:07.935 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:06 smithi064 ceph-mon[31124]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:31:07.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:06 smithi064 ceph-mon[31124]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:31:07.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:06 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:31:07.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:06 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:07.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:06 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:07.936 INFO:tasks.workunit.client.0.smithi022.stderr:+ ceph telemetry show-all 2024-09-21T09:31:07.936 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[39525]: from='client.14841 -' entity='client.admin' cmd=[{"prefix": "telemetry enable channel", "channels": ["perf"], "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:31:07.936 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[39525]: from='client.? 172.21.15.22:0/1027081936' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/telemetry/last_opt_revision"}]: dispatch 2024-09-21T09:31:07.936 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[39525]: from='client.24758 -' entity='client.admin' cmd=[{"prefix": "telemetry show", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:31:07.936 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:31:07.937 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-21T09:31:07.937 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:31:07.937 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:07.937 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:07.937 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:31:07.937 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:07.937 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:07.937 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:31:07.937 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:07.937 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:07.937 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[39525]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:31:07.937 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[39525]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:31:07.937 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:31:07.937 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:07.938 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:07.938 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[39525]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:31:07.938 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[39525]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:31:07.938 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:31:07.938 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:07.938 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:07.938 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[39525]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:31:07.938 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[39525]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:31:07.938 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:31:07.939 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[27763]: from='client.14841 -' entity='client.admin' cmd=[{"prefix": "telemetry enable channel", "channels": ["perf"], "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:31:07.939 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[27763]: from='client.? 172.21.15.22:0/1027081936' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/telemetry/last_opt_revision"}]: dispatch 2024-09-21T09:31:07.939 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[27763]: from='client.24758 -' entity='client.admin' cmd=[{"prefix": "telemetry show", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:31:07.939 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:31:07.939 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-21T09:31:07.939 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:31:07.939 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:07.939 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:07.939 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:31:07.939 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:07.939 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:07.939 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:31:07.940 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:07.940 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:07.940 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[27763]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:31:07.940 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[27763]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:31:07.940 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:31:07.940 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:07.940 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:07.940 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[27763]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:31:07.940 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[27763]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:31:07.940 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:31:07.940 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:07.940 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:07.940 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[27763]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:31:07.940 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[27763]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:31:07.941 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:31:07.941 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:07.941 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:07.941 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[27763]: from='client.24758 -' entity='client.admin' cmd=[{"prefix": "telemetry show", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:31:07.941 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:31:07.941 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-21T09:31:07.941 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:31:07.941 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:07.941 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:07.941 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:31:07.941 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:07.941 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:07.941 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:31:07.942 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:07.942 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:07.942 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[27763]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:31:07.942 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[27763]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:31:07.942 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:31:07.942 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:07.942 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:07.942 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[27763]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:31:07.942 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[27763]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:31:07.942 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:31:07.942 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:07.942 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:07.942 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[27763]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:31:07.943 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[27763]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:31:07.943 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:31:07.943 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:07.943 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:07.943 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:07.943 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:07.943 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[39525]: from='client.24758 -' entity='client.admin' cmd=[{"prefix": "telemetry show", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:31:07.944 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:31:07.944 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-21T09:31:07.944 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:31:07.944 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:07.944 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:07.944 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:31:07.944 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:07.944 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:07.944 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:31:07.944 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:07.944 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:07.944 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[39525]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:31:07.944 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[39525]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:31:07.944 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:31:07.945 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:07.945 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:07.945 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[39525]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:31:07.945 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[39525]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:31:07.945 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:31:07.945 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:07.945 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:07.945 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[39525]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:31:07.945 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[39525]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:31:07.945 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:31:07.945 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:07.945 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:06 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:07.948 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:07 smithi064 ceph-mon[31124]: from='client.14871 -' entity='client.admin' cmd=[{"prefix": "telemetry show-device", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:31:07.948 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:07 smithi064 ceph-mon[31124]: from='client.14871 -' entity='client.admin' cmd=[{"prefix": "telemetry show-device", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:31:07.948 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:07 smithi064 ceph-mon[31124]: pgmap v90: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:31:07.948 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:07 smithi064 ceph-mon[31124]: from='client.24782 -' entity='client.admin' cmd=[{"prefix": "telemetry show-all", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:31:07.948 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:07 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:31:07.948 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:07 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-21T09:31:07.948 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:07 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:31:07.949 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:07 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:07.949 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:07 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:07.949 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:07 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:31:07.949 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:07 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:07.949 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:07 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:07.949 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:07 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:31:07.949 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:07 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:07.949 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:07 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:07.949 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:07 smithi064 ceph-mon[31124]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:31:07.949 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:07 smithi064 ceph-mon[31124]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:31:07.949 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:07 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:31:07.949 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:07 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:07.949 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:07 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:07.949 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:07 smithi064 ceph-mon[31124]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:31:07.950 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:07 smithi064 ceph-mon[31124]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:31:07.950 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:07 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:31:07.950 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:07 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:07.950 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:07 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:07.950 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:07 smithi064 ceph-mon[31124]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:31:07.950 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:07 smithi064 ceph-mon[31124]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:31:07.950 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:07 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:31:07.950 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:07 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:07.950 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:07 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:07.950 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:07 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:31:07.950 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:07 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-21T09:31:07.950 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:07 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:31:07.950 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:07 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:07.950 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:07 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:07.951 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:07 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:31:07.951 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:07 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:07.951 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:07 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:07.951 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:07 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:31:07.951 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:07 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:07.951 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:07 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:07.951 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:07 smithi064 ceph-mon[31124]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:31:07.951 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:07 smithi064 ceph-mon[31124]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:31:07.951 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:07 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:31:07.951 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:07 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:07.951 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:07 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:07.951 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:07 smithi064 ceph-mon[31124]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:31:07.951 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:07 smithi064 ceph-mon[31124]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:31:07.951 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:07 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:31:07.952 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:07 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:07.952 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:07 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:07.952 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:07 smithi064 ceph-mon[31124]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:31:07.952 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:07 smithi064 ceph-mon[31124]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:31:07.952 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:07 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:31:07.952 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:07 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:07.952 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:07 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:07.955 INFO:tasks.workunit.client.0.smithi022.stdout:{ 2024-09-21T09:31:07.955 INFO:tasks.workunit.client.0.smithi022.stdout: "device_report": { 2024-09-21T09:31:07.955 INFO:tasks.workunit.client.0.smithi022.stdout: "384b39b0-77fc-11ef-b8c0-0cc47a595de4": { 2024-09-21T09:31:07.956 INFO:tasks.workunit.client.0.smithi022.stdout: "INTEL_SSDPED1D480GA_384bca4c-77fc-11ef-b8c0-0cc47a595de4": { 2024-09-21T09:31:07.956 INFO:tasks.workunit.client.0.smithi022.stdout: "20240921-092509": { 2024-09-21T09:31:07.956 INFO:tasks.workunit.client.0.smithi022.stdout: "device": { 2024-09-21T09:31:07.956 INFO:tasks.workunit.client.0.smithi022.stdout: "info_name": "/dev/nvme0n1", 2024-09-21T09:31:07.956 INFO:tasks.workunit.client.0.smithi022.stdout: "name": "/dev/nvme0n1", 2024-09-21T09:31:07.956 INFO:tasks.workunit.client.0.smithi022.stdout: "protocol": "NVMe", 2024-09-21T09:31:07.956 INFO:tasks.workunit.client.0.smithi022.stdout: "type": "nvme" 2024-09-21T09:31:07.956 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.956 INFO:tasks.workunit.client.0.smithi022.stdout: "firmware_version": "E2010325", 2024-09-21T09:31:07.956 INFO:tasks.workunit.client.0.smithi022.stdout: "host_id": "384b39b0-77fc-11ef-b8c0-0cc47a595de4", 2024-09-21T09:31:07.956 INFO:tasks.workunit.client.0.smithi022.stdout: "json_format_version": [ 2024-09-21T09:31:07.956 INFO:tasks.workunit.client.0.smithi022.stdout: 1, 2024-09-21T09:31:07.956 INFO:tasks.workunit.client.0.smithi022.stdout: 0 2024-09-21T09:31:07.956 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.957 INFO:tasks.workunit.client.0.smithi022.stdout: "local_time": { 2024-09-21T09:31:07.957 INFO:tasks.workunit.client.0.smithi022.stdout: "asctime": "Sat Sep 21 09:25:09 2024 UTC", 2024-09-21T09:31:07.957 INFO:tasks.workunit.client.0.smithi022.stdout: "time_t": 1726910709 2024-09-21T09:31:07.957 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.957 INFO:tasks.workunit.client.0.smithi022.stdout: "logical_block_size": 512, 2024-09-21T09:31:07.957 INFO:tasks.workunit.client.0.smithi022.stdout: "model_name": "INTEL SSDPED1D480GA", 2024-09-21T09:31:07.957 INFO:tasks.workunit.client.0.smithi022.stdout: "nvme_controller_id": 0, 2024-09-21T09:31:07.957 INFO:tasks.workunit.client.0.smithi022.stdout: "nvme_ieee_oui_identifier": 6083300, 2024-09-21T09:31:07.957 INFO:tasks.workunit.client.0.smithi022.stdout: "nvme_namespaces": [ 2024-09-21T09:31:07.957 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.957 INFO:tasks.workunit.client.0.smithi022.stdout: "capacity": { 2024-09-21T09:31:07.957 INFO:tasks.workunit.client.0.smithi022.stdout: "blocks": 937703088, 2024-09-21T09:31:07.957 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 480103981056 2024-09-21T09:31:07.957 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.957 INFO:tasks.workunit.client.0.smithi022.stdout: "formatted_lba_size": 512, 2024-09-21T09:31:07.958 INFO:tasks.workunit.client.0.smithi022.stdout: "id": 1, 2024-09-21T09:31:07.958 INFO:tasks.workunit.client.0.smithi022.stdout: "size": { 2024-09-21T09:31:07.958 INFO:tasks.workunit.client.0.smithi022.stdout: "blocks": 937703088, 2024-09-21T09:31:07.958 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 480103981056 2024-09-21T09:31:07.958 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.958 INFO:tasks.workunit.client.0.smithi022.stdout: "utilization": { 2024-09-21T09:31:07.958 INFO:tasks.workunit.client.0.smithi022.stdout: "blocks": 937703088, 2024-09-21T09:31:07.958 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 480103981056 2024-09-21T09:31:07.958 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:07.958 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:07.958 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.958 INFO:tasks.workunit.client.0.smithi022.stdout: "nvme_number_of_namespaces": 1, 2024-09-21T09:31:07.958 INFO:tasks.workunit.client.0.smithi022.stdout: "nvme_pci_vendor": { 2024-09-21T09:31:07.958 INFO:tasks.workunit.client.0.smithi022.stdout: "id": 32902, 2024-09-21T09:31:07.959 INFO:tasks.workunit.client.0.smithi022.stdout: "subsystem_id": 32902 2024-09-21T09:31:07.959 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.959 INFO:tasks.workunit.client.0.smithi022.stdout: "nvme_smart_health_information_add_log": { 2024-09-21T09:31:07.959 INFO:tasks.workunit.client.0.smithi022.stdout: "Device stats": { 2024-09-21T09:31:07.959 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_fw_downgrades": { 2024-09-21T09:31:07.959 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 0, 2024-09-21T09:31:07.959 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 0 2024-09-21T09:31:07.959 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.959 INFO:tasks.workunit.client.0.smithi022.stdout: "crc_error_count": { 2024-09-21T09:31:07.959 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 100, 2024-09-21T09:31:07.959 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 0 2024-09-21T09:31:07.959 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.959 INFO:tasks.workunit.client.0.smithi022.stdout: "end_to_end_error_detection_count": { 2024-09-21T09:31:07.959 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 100, 2024-09-21T09:31:07.959 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 0 2024-09-21T09:31:07.960 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.960 INFO:tasks.workunit.client.0.smithi022.stdout: "erase_fail_count": { 2024-09-21T09:31:07.960 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 0, 2024-09-21T09:31:07.960 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 0 2024-09-21T09:31:07.960 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.960 INFO:tasks.workunit.client.0.smithi022.stdout: "host_bytes_written": { 2024-09-21T09:31:07.960 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 100, 2024-09-21T09:31:07.960 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 6829746 2024-09-21T09:31:07.960 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.960 INFO:tasks.workunit.client.0.smithi022.stdout: "host_ctx_wear_used": { 2024-09-21T09:31:07.960 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 0, 2024-09-21T09:31:07.960 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 0 2024-09-21T09:31:07.960 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.961 INFO:tasks.workunit.client.0.smithi022.stdout: "media_bytes_read": { 2024-09-21T09:31:07.961 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 0, 2024-09-21T09:31:07.961 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 0 2024-09-21T09:31:07.961 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.961 INFO:tasks.workunit.client.0.smithi022.stdout: "nand_bytes_written": { 2024-09-21T09:31:07.961 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 0, 2024-09-21T09:31:07.961 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 0 2024-09-21T09:31:07.961 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.961 INFO:tasks.workunit.client.0.smithi022.stdout: "perf_stat_indicator": { 2024-09-21T09:31:07.961 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 0, 2024-09-21T09:31:07.961 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 0 2024-09-21T09:31:07.961 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.961 INFO:tasks.workunit.client.0.smithi022.stdout: "pll_lock_loss_count": { 2024-09-21T09:31:07.961 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 100, 2024-09-21T09:31:07.962 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 0 2024-09-21T09:31:07.962 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.962 INFO:tasks.workunit.client.0.smithi022.stdout: "program_fail_count": { 2024-09-21T09:31:07.962 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 0, 2024-09-21T09:31:07.962 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 0 2024-09-21T09:31:07.962 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.962 INFO:tasks.workunit.client.0.smithi022.stdout: "re_alloc_sectr_cnt": { 2024-09-21T09:31:07.962 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 0, 2024-09-21T09:31:07.962 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 0 2024-09-21T09:31:07.962 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.962 INFO:tasks.workunit.client.0.smithi022.stdout: "retry_buffer_overflow_count": { 2024-09-21T09:31:07.962 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 100, 2024-09-21T09:31:07.962 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 0 2024-09-21T09:31:07.962 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.963 INFO:tasks.workunit.client.0.smithi022.stdout: "soft_ecc_err_rate": { 2024-09-21T09:31:07.963 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 0, 2024-09-21T09:31:07.963 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 0 2024-09-21T09:31:07.963 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.963 INFO:tasks.workunit.client.0.smithi022.stdout: "thermal_throttle_status": { 2024-09-21T09:31:07.963 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 100, 2024-09-21T09:31:07.963 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": { 2024-09-21T09:31:07.963 INFO:tasks.workunit.client.0.smithi022.stdout: "cnt": 0, 2024-09-21T09:31:07.963 INFO:tasks.workunit.client.0.smithi022.stdout: "pct": 0 2024-09-21T09:31:07.963 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:07.963 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.963 INFO:tasks.workunit.client.0.smithi022.stdout: "timed_workload_host_reads": { 2024-09-21T09:31:07.963 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 100, 2024-09-21T09:31:07.964 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 16 2024-09-21T09:31:07.964 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.964 INFO:tasks.workunit.client.0.smithi022.stdout: "timed_workload_media_wear": { 2024-09-21T09:31:07.964 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 100, 2024-09-21T09:31:07.964 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": "0.000000" 2024-09-21T09:31:07.964 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.964 INFO:tasks.workunit.client.0.smithi022.stdout: "timed_workload_timer": { 2024-09-21T09:31:07.964 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 100, 2024-09-21T09:31:07.964 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 1212022 2024-09-21T09:31:07.964 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.964 INFO:tasks.workunit.client.0.smithi022.stdout: "unexp_power_loss": { 2024-09-21T09:31:07.964 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 0, 2024-09-21T09:31:07.964 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 0 2024-09-21T09:31:07.965 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.965 INFO:tasks.workunit.client.0.smithi022.stdout: "wear_leveling": { 2024-09-21T09:31:07.965 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 0, 2024-09-21T09:31:07.965 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": { 2024-09-21T09:31:07.965 INFO:tasks.workunit.client.0.smithi022.stdout: "avg": 0, 2024-09-21T09:31:07.965 INFO:tasks.workunit.client.0.smithi022.stdout: "max": 0, 2024-09-21T09:31:07.965 INFO:tasks.workunit.client.0.smithi022.stdout: "min": 0 2024-09-21T09:31:07.965 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:07.965 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:07.965 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.965 INFO:tasks.workunit.client.0.smithi022.stdout: "Intel Smart log": "nvme0n1" 2024-09-21T09:31:07.965 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.965 INFO:tasks.workunit.client.0.smithi022.stdout: "nvme_smart_health_information_log": { 2024-09-21T09:31:07.965 INFO:tasks.workunit.client.0.smithi022.stdout: "available_spare": 100, 2024-09-21T09:31:07.966 INFO:tasks.workunit.client.0.smithi022.stdout: "available_spare_threshold": 0, 2024-09-21T09:31:07.966 INFO:tasks.workunit.client.0.smithi022.stdout: "controller_busy_time": 2680, 2024-09-21T09:31:07.966 INFO:tasks.workunit.client.0.smithi022.stdout: "critical_warning": 0, 2024-09-21T09:31:07.966 INFO:tasks.workunit.client.0.smithi022.stdout: "data_units_read": 95367672, 2024-09-21T09:31:07.966 INFO:tasks.workunit.client.0.smithi022.stdout: "data_units_written": 447594268, 2024-09-21T09:31:07.966 INFO:tasks.workunit.client.0.smithi022.stdout: "host_reads": 1403952490, 2024-09-21T09:31:07.966 INFO:tasks.workunit.client.0.smithi022.stdout: "host_writes": 10869660916, 2024-09-21T09:31:07.966 INFO:tasks.workunit.client.0.smithi022.stdout: "media_errors": 0, 2024-09-21T09:31:07.966 INFO:tasks.workunit.client.0.smithi022.stdout: "num_err_log_entries": 0, 2024-09-21T09:31:07.966 INFO:tasks.workunit.client.0.smithi022.stdout: "percentage_used": 0, 2024-09-21T09:31:07.966 INFO:tasks.workunit.client.0.smithi022.stdout: "power_cycles": 16189, 2024-09-21T09:31:07.966 INFO:tasks.workunit.client.0.smithi022.stdout: "power_on_hours": 27263, 2024-09-21T09:31:07.966 INFO:tasks.workunit.client.0.smithi022.stdout: "temperature": 31, 2024-09-21T09:31:07.966 INFO:tasks.workunit.client.0.smithi022.stdout: "unsafe_shutdowns": 5631 2024-09-21T09:31:07.967 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.967 INFO:tasks.workunit.client.0.smithi022.stdout: "nvme_vendor": "intel", 2024-09-21T09:31:07.967 INFO:tasks.workunit.client.0.smithi022.stdout: "nvme_version": { 2024-09-21T09:31:07.967 INFO:tasks.workunit.client.0.smithi022.stdout: "string": "<1.2", 2024-09-21T09:31:07.967 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.967 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.967 INFO:tasks.workunit.client.0.smithi022.stdout: "power_cycle_count": 16189, 2024-09-21T09:31:07.967 INFO:tasks.workunit.client.0.smithi022.stdout: "power_on_time": { 2024-09-21T09:31:07.967 INFO:tasks.workunit.client.0.smithi022.stdout: "hours": 27263 2024-09-21T09:31:07.967 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.967 INFO:tasks.workunit.client.0.smithi022.stdout: "serial_number": "deleted", 2024-09-21T09:31:07.967 INFO:tasks.workunit.client.0.smithi022.stdout: "smart_status": { 2024-09-21T09:31:07.967 INFO:tasks.workunit.client.0.smithi022.stdout: "nvme": { 2024-09-21T09:31:07.967 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.968 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.968 INFO:tasks.workunit.client.0.smithi022.stdout: "passed": true 2024-09-21T09:31:07.968 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.968 INFO:tasks.workunit.client.0.smithi022.stdout: "smartctl": { 2024-09-21T09:31:07.968 INFO:tasks.workunit.client.0.smithi022.stdout: "argv": [ 2024-09-21T09:31:07.968 INFO:tasks.workunit.client.0.smithi022.stdout: "smartctl", 2024-09-21T09:31:07.968 INFO:tasks.workunit.client.0.smithi022.stdout: "-x", 2024-09-21T09:31:07.968 INFO:tasks.workunit.client.0.smithi022.stdout: "--json=o", 2024-09-21T09:31:07.968 INFO:tasks.workunit.client.0.smithi022.stdout: "/dev/nvme0n1" 2024-09-21T09:31:07.968 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.968 INFO:tasks.workunit.client.0.smithi022.stdout: "build_info": "(local build)", 2024-09-21T09:31:07.968 INFO:tasks.workunit.client.0.smithi022.stdout: "exit_status": 0, 2024-09-21T09:31:07.968 INFO:tasks.workunit.client.0.smithi022.stdout: "output": [ 2024-09-21T09:31:07.968 INFO:tasks.workunit.client.0.smithi022.stdout: "smartctl 7.2 2020-12-30 r5155 [x86_64-linux-5.14.0-508.el9.x86_64] (local build)", 2024-09-21T09:31:07.968 INFO:tasks.workunit.client.0.smithi022.stdout: "Copyright (C) 2002-20, Bruce Allen, Christian Franke, www.smartmontools.org", 2024-09-21T09:31:07.969 INFO:tasks.workunit.client.0.smithi022.stdout: "", 2024-09-21T09:31:07.969 INFO:tasks.workunit.client.0.smithi022.stdout: "=== START OF INFORMATION SECTION ===", 2024-09-21T09:31:07.969 INFO:tasks.workunit.client.0.smithi022.stdout: "Model Number: INTEL SSDPED1D480GA", 2024-09-21T09:31:07.969 INFO:tasks.workunit.client.0.smithi022.stdout: "Serial Number: deleted", 2024-09-21T09:31:07.969 INFO:tasks.workunit.client.0.smithi022.stdout: "Firmware Version: E2010325", 2024-09-21T09:31:07.969 INFO:tasks.workunit.client.0.smithi022.stdout: "PCI Vendor/Subsystem ID: 0x8086", 2024-09-21T09:31:07.969 INFO:tasks.workunit.client.0.smithi022.stdout: "IEEE OUI Identifier: 0x5cd2e4", 2024-09-21T09:31:07.969 INFO:tasks.workunit.client.0.smithi022.stdout: "Controller ID: 0", 2024-09-21T09:31:07.969 INFO:tasks.workunit.client.0.smithi022.stdout: "NVMe Version: <1.2", 2024-09-21T09:31:07.969 INFO:tasks.workunit.client.0.smithi022.stdout: "Number of Namespaces: 1", 2024-09-21T09:31:07.969 INFO:tasks.workunit.client.0.smithi022.stdout: "Namespace 1 Size/Capacity: 480,103,981,056 [480 GB]", 2024-09-21T09:31:07.969 INFO:tasks.workunit.client.0.smithi022.stdout: "Namespace 1 Formatted LBA Size: 512", 2024-09-21T09:31:07.969 INFO:tasks.workunit.client.0.smithi022.stdout: "Local Time is: Sat Sep 21 09:25:09 2024 UTC", 2024-09-21T09:31:07.969 INFO:tasks.workunit.client.0.smithi022.stdout: "Firmware Updates (0x02): 1 Slot", 2024-09-21T09:31:07.970 INFO:tasks.workunit.client.0.smithi022.stdout: "Optional Admin Commands (0x0007): Security Format Frmw_DL", 2024-09-21T09:31:07.970 INFO:tasks.workunit.client.0.smithi022.stdout: "Optional NVM Commands (0x0006): Wr_Unc DS_Mngmt", 2024-09-21T09:31:07.970 INFO:tasks.workunit.client.0.smithi022.stdout: "Log Page Attributes (0x02): Cmd_Eff_Lg", 2024-09-21T09:31:07.970 INFO:tasks.workunit.client.0.smithi022.stdout: "Maximum Data Transfer Size: 32 Pages", 2024-09-21T09:31:07.970 INFO:tasks.workunit.client.0.smithi022.stdout: "", 2024-09-21T09:31:07.970 INFO:tasks.workunit.client.0.smithi022.stdout: "Supported Power States", 2024-09-21T09:31:07.970 INFO:tasks.workunit.client.0.smithi022.stdout: "St Op Max Active Idle RL RT WL WT Ent_Lat Ex_Lat", 2024-09-21T09:31:07.970 INFO:tasks.workunit.client.0.smithi022.stdout: " 0 + 18.00W - - 0 0 0 0 0 0", 2024-09-21T09:31:07.970 INFO:tasks.workunit.client.0.smithi022.stdout: "", 2024-09-21T09:31:07.970 INFO:tasks.workunit.client.0.smithi022.stdout: "Supported LBA Sizes (NSID 0x1)", 2024-09-21T09:31:07.970 INFO:tasks.workunit.client.0.smithi022.stdout: "Id Fmt Data Metadt Rel_Perf", 2024-09-21T09:31:07.970 INFO:tasks.workunit.client.0.smithi022.stdout: " 0 + 512 0 2", 2024-09-21T09:31:07.970 INFO:tasks.workunit.client.0.smithi022.stdout: "", 2024-09-21T09:31:07.971 INFO:tasks.workunit.client.0.smithi022.stdout: "=== START OF SMART DATA SECTION ===", 2024-09-21T09:31:07.971 INFO:tasks.workunit.client.0.smithi022.stdout: "SMART overall-health self-assessment test result: PASSED", 2024-09-21T09:31:07.971 INFO:tasks.workunit.client.0.smithi022.stdout: "", 2024-09-21T09:31:07.971 INFO:tasks.workunit.client.0.smithi022.stdout: "SMART/Health Information (NVMe Log 0x02)", 2024-09-21T09:31:07.971 INFO:tasks.workunit.client.0.smithi022.stdout: "Critical Warning: 0x00", 2024-09-21T09:31:07.971 INFO:tasks.workunit.client.0.smithi022.stdout: "Temperature: 31 Celsius", 2024-09-21T09:31:07.971 INFO:tasks.workunit.client.0.smithi022.stdout: "Available Spare: 100%", 2024-09-21T09:31:07.971 INFO:tasks.workunit.client.0.smithi022.stdout: "Available Spare Threshold: 0%", 2024-09-21T09:31:07.971 INFO:tasks.workunit.client.0.smithi022.stdout: "Percentage Used: 0%", 2024-09-21T09:31:07.971 INFO:tasks.workunit.client.0.smithi022.stdout: "Data Units Read: 95,367,672 [48.8 TB]", 2024-09-21T09:31:07.971 INFO:tasks.workunit.client.0.smithi022.stdout: "Data Units Written: 447,594,268 [229 TB]", 2024-09-21T09:31:07.971 INFO:tasks.workunit.client.0.smithi022.stdout: "Host Read Commands: 1,403,952,490", 2024-09-21T09:31:07.971 INFO:tasks.workunit.client.0.smithi022.stdout: "Host Write Commands: 10,869,660,916", 2024-09-21T09:31:07.972 INFO:tasks.workunit.client.0.smithi022.stdout: "Controller Busy Time: 2,680", 2024-09-21T09:31:07.972 INFO:tasks.workunit.client.0.smithi022.stdout: "Power Cycles: 16,189", 2024-09-21T09:31:07.972 INFO:tasks.workunit.client.0.smithi022.stdout: "Power On Hours: 27,263", 2024-09-21T09:31:07.972 INFO:tasks.workunit.client.0.smithi022.stdout: "Unsafe Shutdowns: 5,631", 2024-09-21T09:31:07.972 INFO:tasks.workunit.client.0.smithi022.stdout: "Media and Data Integrity Errors: 0", 2024-09-21T09:31:07.972 INFO:tasks.workunit.client.0.smithi022.stdout: "Error Information Log Entries: 0", 2024-09-21T09:31:07.972 INFO:tasks.workunit.client.0.smithi022.stdout: "", 2024-09-21T09:31:07.972 INFO:tasks.workunit.client.0.smithi022.stdout: "Error Information (NVMe Log 0x01, 16 of 64 entries)", 2024-09-21T09:31:07.972 INFO:tasks.workunit.client.0.smithi022.stdout: "No Errors Logged", 2024-09-21T09:31:07.972 INFO:tasks.workunit.client.0.smithi022.stdout: "" 2024-09-21T09:31:07.972 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.972 INFO:tasks.workunit.client.0.smithi022.stdout: "platform_info": "x86_64-linux-5.14.0-508.el9.x86_64", 2024-09-21T09:31:07.972 INFO:tasks.workunit.client.0.smithi022.stdout: "svn_revision": "5155", 2024-09-21T09:31:07.972 INFO:tasks.workunit.client.0.smithi022.stdout: "version": [ 2024-09-21T09:31:07.973 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:07.973 INFO:tasks.workunit.client.0.smithi022.stdout: 2 2024-09-21T09:31:07.973 INFO:tasks.workunit.client.0.smithi022.stdout: ] 2024-09-21T09:31:07.973 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.973 INFO:tasks.workunit.client.0.smithi022.stdout: "temperature": { 2024-09-21T09:31:07.973 INFO:tasks.workunit.client.0.smithi022.stdout: "current": 31 2024-09-21T09:31:07.973 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.973 INFO:tasks.workunit.client.0.smithi022.stdout: "user_capacity": { 2024-09-21T09:31:07.973 INFO:tasks.workunit.client.0.smithi022.stdout: "blocks": 937703088, 2024-09-21T09:31:07.973 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 480103981056 2024-09-21T09:31:07.973 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:07.973 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:07.973 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:07.973 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.974 INFO:tasks.workunit.client.0.smithi022.stdout: "384d1fc8-77fc-11ef-b8c0-0cc47a595de4": { 2024-09-21T09:31:07.974 INFO:tasks.workunit.client.0.smithi022.stdout: "INTEL_SSDPEDMD400G4_384da218-77fc-11ef-b8c0-0cc47a595de4": { 2024-09-21T09:31:07.974 INFO:tasks.workunit.client.0.smithi022.stdout: "20240921-092513": { 2024-09-21T09:31:07.974 INFO:tasks.workunit.client.0.smithi022.stdout: "device": { 2024-09-21T09:31:07.974 INFO:tasks.workunit.client.0.smithi022.stdout: "info_name": "/dev/nvme0n1", 2024-09-21T09:31:07.974 INFO:tasks.workunit.client.0.smithi022.stdout: "name": "/dev/nvme0n1", 2024-09-21T09:31:07.974 INFO:tasks.workunit.client.0.smithi022.stdout: "protocol": "NVMe", 2024-09-21T09:31:07.974 INFO:tasks.workunit.client.0.smithi022.stdout: "type": "nvme" 2024-09-21T09:31:07.974 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.974 INFO:tasks.workunit.client.0.smithi022.stdout: "firmware_version": "8DV101H0", 2024-09-21T09:31:07.974 INFO:tasks.workunit.client.0.smithi022.stdout: "host_id": "384d1fc8-77fc-11ef-b8c0-0cc47a595de4", 2024-09-21T09:31:07.974 INFO:tasks.workunit.client.0.smithi022.stdout: "json_format_version": [ 2024-09-21T09:31:07.974 INFO:tasks.workunit.client.0.smithi022.stdout: 1, 2024-09-21T09:31:07.974 INFO:tasks.workunit.client.0.smithi022.stdout: 0 2024-09-21T09:31:07.975 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.975 INFO:tasks.workunit.client.0.smithi022.stdout: "local_time": { 2024-09-21T09:31:07.975 INFO:tasks.workunit.client.0.smithi022.stdout: "asctime": "Sat Sep 21 09:25:12 2024 UTC", 2024-09-21T09:31:07.975 INFO:tasks.workunit.client.0.smithi022.stdout: "time_t": 1726910712 2024-09-21T09:31:07.975 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.975 INFO:tasks.workunit.client.0.smithi022.stdout: "logical_block_size": 512, 2024-09-21T09:31:07.975 INFO:tasks.workunit.client.0.smithi022.stdout: "model_name": "INTEL SSDPEDMD400G4", 2024-09-21T09:31:07.975 INFO:tasks.workunit.client.0.smithi022.stdout: "nvme_controller_id": 0, 2024-09-21T09:31:07.975 INFO:tasks.workunit.client.0.smithi022.stdout: "nvme_ieee_oui_identifier": 6083300, 2024-09-21T09:31:07.975 INFO:tasks.workunit.client.0.smithi022.stdout: "nvme_namespaces": [ 2024-09-21T09:31:07.975 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:07.975 INFO:tasks.workunit.client.0.smithi022.stdout: "capacity": { 2024-09-21T09:31:07.975 INFO:tasks.workunit.client.0.smithi022.stdout: "blocks": 781422768, 2024-09-21T09:31:07.975 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 400088457216 2024-09-21T09:31:07.976 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.976 INFO:tasks.workunit.client.0.smithi022.stdout: "formatted_lba_size": 512, 2024-09-21T09:31:07.976 INFO:tasks.workunit.client.0.smithi022.stdout: "id": 1, 2024-09-21T09:31:07.976 INFO:tasks.workunit.client.0.smithi022.stdout: "size": { 2024-09-21T09:31:07.976 INFO:tasks.workunit.client.0.smithi022.stdout: "blocks": 781422768, 2024-09-21T09:31:07.976 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 400088457216 2024-09-21T09:31:07.976 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.976 INFO:tasks.workunit.client.0.smithi022.stdout: "utilization": { 2024-09-21T09:31:07.976 INFO:tasks.workunit.client.0.smithi022.stdout: "blocks": 781422768, 2024-09-21T09:31:07.976 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 400088457216 2024-09-21T09:31:07.976 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:07.976 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:07.976 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.976 INFO:tasks.workunit.client.0.smithi022.stdout: "nvme_number_of_namespaces": 1, 2024-09-21T09:31:07.977 INFO:tasks.workunit.client.0.smithi022.stdout: "nvme_pci_vendor": { 2024-09-21T09:31:07.977 INFO:tasks.workunit.client.0.smithi022.stdout: "id": 32902, 2024-09-21T09:31:07.977 INFO:tasks.workunit.client.0.smithi022.stdout: "subsystem_id": 32902 2024-09-21T09:31:07.977 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.977 INFO:tasks.workunit.client.0.smithi022.stdout: "nvme_smart_health_information_add_log": { 2024-09-21T09:31:07.977 INFO:tasks.workunit.client.0.smithi022.stdout: "Device stats": { 2024-09-21T09:31:07.977 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_fw_downgrades": { 2024-09-21T09:31:07.977 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 0, 2024-09-21T09:31:07.977 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 0 2024-09-21T09:31:07.977 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.977 INFO:tasks.workunit.client.0.smithi022.stdout: "crc_error_count": { 2024-09-21T09:31:07.977 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 100, 2024-09-21T09:31:07.977 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 0 2024-09-21T09:31:07.977 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.977 INFO:tasks.workunit.client.0.smithi022.stdout: "end_to_end_error_detection_count": { 2024-09-21T09:31:07.978 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 100, 2024-09-21T09:31:07.978 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 0 2024-09-21T09:31:07.978 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.978 INFO:tasks.workunit.client.0.smithi022.stdout: "erase_fail_count": { 2024-09-21T09:31:07.978 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 100, 2024-09-21T09:31:07.978 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 0 2024-09-21T09:31:07.978 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.978 INFO:tasks.workunit.client.0.smithi022.stdout: "host_bytes_written": { 2024-09-21T09:31:07.978 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 100, 2024-09-21T09:31:07.978 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 34797061 2024-09-21T09:31:07.978 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.978 INFO:tasks.workunit.client.0.smithi022.stdout: "host_ctx_wear_used": { 2024-09-21T09:31:07.978 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 100, 2024-09-21T09:31:07.978 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 1 2024-09-21T09:31:07.979 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.979 INFO:tasks.workunit.client.0.smithi022.stdout: "media_bytes_read": { 2024-09-21T09:31:07.979 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 0, 2024-09-21T09:31:07.979 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 0 2024-09-21T09:31:07.979 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.979 INFO:tasks.workunit.client.0.smithi022.stdout: "nand_bytes_written": { 2024-09-21T09:31:07.979 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 100, 2024-09-21T09:31:07.979 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 57236814 2024-09-21T09:31:07.979 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.979 INFO:tasks.workunit.client.0.smithi022.stdout: "perf_stat_indicator": { 2024-09-21T09:31:07.979 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 0, 2024-09-21T09:31:07.979 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 0 2024-09-21T09:31:07.979 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.979 INFO:tasks.workunit.client.0.smithi022.stdout: "pll_lock_loss_count": { 2024-09-21T09:31:07.979 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 100, 2024-09-21T09:31:07.980 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 0 2024-09-21T09:31:07.980 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.980 INFO:tasks.workunit.client.0.smithi022.stdout: "program_fail_count": { 2024-09-21T09:31:07.980 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 100, 2024-09-21T09:31:07.980 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 0 2024-09-21T09:31:07.980 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.980 INFO:tasks.workunit.client.0.smithi022.stdout: "re_alloc_sectr_cnt": { 2024-09-21T09:31:07.980 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 0, 2024-09-21T09:31:07.980 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 0 2024-09-21T09:31:07.980 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.980 INFO:tasks.workunit.client.0.smithi022.stdout: "retry_buffer_overflow_count": { 2024-09-21T09:31:07.980 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 100, 2024-09-21T09:31:07.980 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 0 2024-09-21T09:31:07.981 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.981 INFO:tasks.workunit.client.0.smithi022.stdout: "soft_ecc_err_rate": { 2024-09-21T09:31:07.981 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 0, 2024-09-21T09:31:07.981 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 0 2024-09-21T09:31:07.981 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.981 INFO:tasks.workunit.client.0.smithi022.stdout: "thermal_throttle_status": { 2024-09-21T09:31:07.981 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 100, 2024-09-21T09:31:07.981 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": { 2024-09-21T09:31:07.981 INFO:tasks.workunit.client.0.smithi022.stdout: "cnt": 0, 2024-09-21T09:31:07.981 INFO:tasks.workunit.client.0.smithi022.stdout: "pct": 0 2024-09-21T09:31:07.981 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:07.981 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.981 INFO:tasks.workunit.client.0.smithi022.stdout: "timed_workload_host_reads": { 2024-09-21T09:31:07.981 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 100, 2024-09-21T09:31:07.982 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 65535 2024-09-21T09:31:07.982 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.982 INFO:tasks.workunit.client.0.smithi022.stdout: "timed_workload_media_wear": { 2024-09-21T09:31:07.982 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 100, 2024-09-21T09:31:07.982 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": "63.999023" 2024-09-21T09:31:07.982 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.982 INFO:tasks.workunit.client.0.smithi022.stdout: "timed_workload_timer": { 2024-09-21T09:31:07.982 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 100, 2024-09-21T09:31:07.982 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 65535 2024-09-21T09:31:07.982 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.982 INFO:tasks.workunit.client.0.smithi022.stdout: "unexp_power_loss": { 2024-09-21T09:31:07.982 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 0, 2024-09-21T09:31:07.982 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": 0 2024-09-21T09:31:07.982 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.983 INFO:tasks.workunit.client.0.smithi022.stdout: "wear_leveling": { 2024-09-21T09:31:07.983 INFO:tasks.workunit.client.0.smithi022.stdout: "normalized": 88, 2024-09-21T09:31:07.983 INFO:tasks.workunit.client.0.smithi022.stdout: "raw": { 2024-09-21T09:31:07.983 INFO:tasks.workunit.client.0.smithi022.stdout: "avg": 3844, 2024-09-21T09:31:07.983 INFO:tasks.workunit.client.0.smithi022.stdout: "max": 3860, 2024-09-21T09:31:07.983 INFO:tasks.workunit.client.0.smithi022.stdout: "min": 3828 2024-09-21T09:31:07.983 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:07.983 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:07.983 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.983 INFO:tasks.workunit.client.0.smithi022.stdout: "Intel Smart log": "nvme0n1" 2024-09-21T09:31:07.983 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.983 INFO:tasks.workunit.client.0.smithi022.stdout: "nvme_smart_health_information_log": { 2024-09-21T09:31:07.983 INFO:tasks.workunit.client.0.smithi022.stdout: "available_spare": 100, 2024-09-21T09:31:07.984 INFO:tasks.workunit.client.0.smithi022.stdout: "available_spare_threshold": 10, 2024-09-21T09:31:07.984 INFO:tasks.workunit.client.0.smithi022.stdout: "controller_busy_time": 39, 2024-09-21T09:31:07.984 INFO:tasks.workunit.client.0.smithi022.stdout: "critical_warning": 0, 2024-09-21T09:31:07.984 INFO:tasks.workunit.client.0.smithi022.stdout: "data_units_read": 467647523, 2024-09-21T09:31:07.984 INFO:tasks.workunit.client.0.smithi022.stdout: "data_units_written": 2280460231, 2024-09-21T09:31:07.984 INFO:tasks.workunit.client.0.smithi022.stdout: "host_reads": 8064859300, 2024-09-21T09:31:07.984 INFO:tasks.workunit.client.0.smithi022.stdout: "host_writes": 41665631302, 2024-09-21T09:31:07.984 INFO:tasks.workunit.client.0.smithi022.stdout: "media_errors": 0, 2024-09-21T09:31:07.984 INFO:tasks.workunit.client.0.smithi022.stdout: "num_err_log_entries": 0, 2024-09-21T09:31:07.984 INFO:tasks.workunit.client.0.smithi022.stdout: "percentage_used": 12, 2024-09-21T09:31:07.984 INFO:tasks.workunit.client.0.smithi022.stdout: "power_cycles": 55086, 2024-09-21T09:31:07.984 INFO:tasks.workunit.client.0.smithi022.stdout: "power_on_hours": 60974, 2024-09-21T09:31:07.984 INFO:tasks.workunit.client.0.smithi022.stdout: "temperature": 22, 2024-09-21T09:31:07.984 INFO:tasks.workunit.client.0.smithi022.stdout: "unsafe_shutdowns": 2616 2024-09-21T09:31:07.985 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.985 INFO:tasks.workunit.client.0.smithi022.stdout: "nvme_vendor": "intel", 2024-09-21T09:31:07.985 INFO:tasks.workunit.client.0.smithi022.stdout: "nvme_version": { 2024-09-21T09:31:07.985 INFO:tasks.workunit.client.0.smithi022.stdout: "string": "<1.2", 2024-09-21T09:31:07.985 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.985 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.985 INFO:tasks.workunit.client.0.smithi022.stdout: "power_cycle_count": 55086, 2024-09-21T09:31:07.985 INFO:tasks.workunit.client.0.smithi022.stdout: "power_on_time": { 2024-09-21T09:31:07.985 INFO:tasks.workunit.client.0.smithi022.stdout: "hours": 60974 2024-09-21T09:31:07.985 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.985 INFO:tasks.workunit.client.0.smithi022.stdout: "serial_number": "deleted", 2024-09-21T09:31:07.985 INFO:tasks.workunit.client.0.smithi022.stdout: "smart_status": { 2024-09-21T09:31:07.985 INFO:tasks.workunit.client.0.smithi022.stdout: "nvme": { 2024-09-21T09:31:07.985 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:07.986 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.986 INFO:tasks.workunit.client.0.smithi022.stdout: "passed": true 2024-09-21T09:31:07.986 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.986 INFO:tasks.workunit.client.0.smithi022.stdout: "smartctl": { 2024-09-21T09:31:07.986 INFO:tasks.workunit.client.0.smithi022.stdout: "argv": [ 2024-09-21T09:31:07.986 INFO:tasks.workunit.client.0.smithi022.stdout: "smartctl", 2024-09-21T09:31:07.986 INFO:tasks.workunit.client.0.smithi022.stdout: "-x", 2024-09-21T09:31:07.986 INFO:tasks.workunit.client.0.smithi022.stdout: "--json=o", 2024-09-21T09:31:07.986 INFO:tasks.workunit.client.0.smithi022.stdout: "/dev/nvme0n1" 2024-09-21T09:31:07.986 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.986 INFO:tasks.workunit.client.0.smithi022.stdout: "build_info": "(local build)", 2024-09-21T09:31:07.986 INFO:tasks.workunit.client.0.smithi022.stdout: "exit_status": 0, 2024-09-21T09:31:07.986 INFO:tasks.workunit.client.0.smithi022.stdout: "output": [ 2024-09-21T09:31:07.986 INFO:tasks.workunit.client.0.smithi022.stdout: "smartctl 7.2 2020-12-30 r5155 [x86_64-linux-5.14.0-508.el9.x86_64] (local build)", 2024-09-21T09:31:07.987 INFO:tasks.workunit.client.0.smithi022.stdout: "Copyright (C) 2002-20, Bruce Allen, Christian Franke, www.smartmontools.org", 2024-09-21T09:31:07.987 INFO:tasks.workunit.client.0.smithi022.stdout: "", 2024-09-21T09:31:07.987 INFO:tasks.workunit.client.0.smithi022.stdout: "=== START OF INFORMATION SECTION ===", 2024-09-21T09:31:07.987 INFO:tasks.workunit.client.0.smithi022.stdout: "Model Number: INTEL SSDPEDMD400G4", 2024-09-21T09:31:07.987 INFO:tasks.workunit.client.0.smithi022.stdout: "Serial Number: deleted", 2024-09-21T09:31:07.987 INFO:tasks.workunit.client.0.smithi022.stdout: "Firmware Version: 8DV101H0", 2024-09-21T09:31:07.987 INFO:tasks.workunit.client.0.smithi022.stdout: "PCI Vendor/Subsystem ID: 0x8086", 2024-09-21T09:31:07.987 INFO:tasks.workunit.client.0.smithi022.stdout: "IEEE OUI Identifier: 0x5cd2e4", 2024-09-21T09:31:07.987 INFO:tasks.workunit.client.0.smithi022.stdout: "Controller ID: 0", 2024-09-21T09:31:07.987 INFO:tasks.workunit.client.0.smithi022.stdout: "NVMe Version: <1.2", 2024-09-21T09:31:07.987 INFO:tasks.workunit.client.0.smithi022.stdout: "Number of Namespaces: 1", 2024-09-21T09:31:07.987 INFO:tasks.workunit.client.0.smithi022.stdout: "Namespace 1 Size/Capacity: 400,088,457,216 [400 GB]", 2024-09-21T09:31:07.987 INFO:tasks.workunit.client.0.smithi022.stdout: "Namespace 1 Formatted LBA Size: 512", 2024-09-21T09:31:07.987 INFO:tasks.workunit.client.0.smithi022.stdout: "Local Time is: Sat Sep 21 09:25:12 2024 UTC", 2024-09-21T09:31:07.987 INFO:tasks.workunit.client.0.smithi022.stdout: "Firmware Updates (0x02): 1 Slot", 2024-09-21T09:31:07.988 INFO:tasks.workunit.client.0.smithi022.stdout: "Optional Admin Commands (0x0006): Format Frmw_DL", 2024-09-21T09:31:07.988 INFO:tasks.workunit.client.0.smithi022.stdout: "Optional NVM Commands (0x0006): Wr_Unc DS_Mngmt", 2024-09-21T09:31:07.988 INFO:tasks.workunit.client.0.smithi022.stdout: "Log Page Attributes (0x02): Cmd_Eff_Lg", 2024-09-21T09:31:07.988 INFO:tasks.workunit.client.0.smithi022.stdout: "Maximum Data Transfer Size: 32 Pages", 2024-09-21T09:31:07.988 INFO:tasks.workunit.client.0.smithi022.stdout: "", 2024-09-21T09:31:07.988 INFO:tasks.workunit.client.0.smithi022.stdout: "Supported Power States", 2024-09-21T09:31:07.988 INFO:tasks.workunit.client.0.smithi022.stdout: "St Op Max Active Idle RL RT WL WT Ent_Lat Ex_Lat", 2024-09-21T09:31:07.988 INFO:tasks.workunit.client.0.smithi022.stdout: " 0 + 25.00W - - 0 0 0 0 0 0", 2024-09-21T09:31:07.988 INFO:tasks.workunit.client.0.smithi022.stdout: "", 2024-09-21T09:31:07.988 INFO:tasks.workunit.client.0.smithi022.stdout: "Supported LBA Sizes (NSID 0x1)", 2024-09-21T09:31:07.988 INFO:tasks.workunit.client.0.smithi022.stdout: "Id Fmt Data Metadt Rel_Perf", 2024-09-21T09:31:07.988 INFO:tasks.workunit.client.0.smithi022.stdout: " 0 + 512 0 2", 2024-09-21T09:31:07.988 INFO:tasks.workunit.client.0.smithi022.stdout: " 1 - 512 8 2", 2024-09-21T09:31:07.988 INFO:tasks.workunit.client.0.smithi022.stdout: " 2 - 512 16 2", 2024-09-21T09:31:07.989 INFO:tasks.workunit.client.0.smithi022.stdout: " 3 - 4096 0 0", 2024-09-21T09:31:07.989 INFO:tasks.workunit.client.0.smithi022.stdout: " 4 - 4096 8 0", 2024-09-21T09:31:07.989 INFO:tasks.workunit.client.0.smithi022.stdout: " 5 - 4096 64 0", 2024-09-21T09:31:07.989 INFO:tasks.workunit.client.0.smithi022.stdout: " 6 - 4096 128 0", 2024-09-21T09:31:07.989 INFO:tasks.workunit.client.0.smithi022.stdout: "", 2024-09-21T09:31:07.989 INFO:tasks.workunit.client.0.smithi022.stdout: "=== START OF SMART DATA SECTION ===", 2024-09-21T09:31:07.989 INFO:tasks.workunit.client.0.smithi022.stdout: "SMART overall-health self-assessment test result: PASSED", 2024-09-21T09:31:07.989 INFO:tasks.workunit.client.0.smithi022.stdout: "", 2024-09-21T09:31:07.989 INFO:tasks.workunit.client.0.smithi022.stdout: "SMART/Health Information (NVMe Log 0x02)", 2024-09-21T09:31:07.989 INFO:tasks.workunit.client.0.smithi022.stdout: "Critical Warning: 0x00", 2024-09-21T09:31:07.989 INFO:tasks.workunit.client.0.smithi022.stdout: "Temperature: 22 Celsius", 2024-09-21T09:31:07.989 INFO:tasks.workunit.client.0.smithi022.stdout: "Available Spare: 100%", 2024-09-21T09:31:07.989 INFO:tasks.workunit.client.0.smithi022.stdout: "Available Spare Threshold: 10%", 2024-09-21T09:31:07.989 INFO:tasks.workunit.client.0.smithi022.stdout: "Percentage Used: 12%", 2024-09-21T09:31:07.989 INFO:tasks.workunit.client.0.smithi022.stdout: "Data Units Read: 467,647,523 [239 TB]", 2024-09-21T09:31:07.990 INFO:tasks.workunit.client.0.smithi022.stdout: "Data Units Written: 2,280,460,231 [1.16 PB]", 2024-09-21T09:31:07.990 INFO:tasks.workunit.client.0.smithi022.stdout: "Host Read Commands: 8,064,859,300", 2024-09-21T09:31:07.990 INFO:tasks.workunit.client.0.smithi022.stdout: "Host Write Commands: 41,665,631,302", 2024-09-21T09:31:07.990 INFO:tasks.workunit.client.0.smithi022.stdout: "Controller Busy Time: 39", 2024-09-21T09:31:07.990 INFO:tasks.workunit.client.0.smithi022.stdout: "Power Cycles: 55,086", 2024-09-21T09:31:07.990 INFO:tasks.workunit.client.0.smithi022.stdout: "Power On Hours: 60,974", 2024-09-21T09:31:07.990 INFO:tasks.workunit.client.0.smithi022.stdout: "Unsafe Shutdowns: 2,616", 2024-09-21T09:31:07.990 INFO:tasks.workunit.client.0.smithi022.stdout: "Media and Data Integrity Errors: 0", 2024-09-21T09:31:07.990 INFO:tasks.workunit.client.0.smithi022.stdout: "Error Information Log Entries: 0", 2024-09-21T09:31:07.990 INFO:tasks.workunit.client.0.smithi022.stdout: "", 2024-09-21T09:31:07.990 INFO:tasks.workunit.client.0.smithi022.stdout: "Error Information (NVMe Log 0x01, 16 of 64 entries)", 2024-09-21T09:31:07.990 INFO:tasks.workunit.client.0.smithi022.stdout: "No Errors Logged", 2024-09-21T09:31:07.990 INFO:tasks.workunit.client.0.smithi022.stdout: "" 2024-09-21T09:31:07.990 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.991 INFO:tasks.workunit.client.0.smithi022.stdout: "platform_info": "x86_64-linux-5.14.0-508.el9.x86_64", 2024-09-21T09:31:07.991 INFO:tasks.workunit.client.0.smithi022.stdout: "svn_revision": "5155", 2024-09-21T09:31:07.991 INFO:tasks.workunit.client.0.smithi022.stdout: "version": [ 2024-09-21T09:31:07.991 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:07.991 INFO:tasks.workunit.client.0.smithi022.stdout: 2 2024-09-21T09:31:07.991 INFO:tasks.workunit.client.0.smithi022.stdout: ] 2024-09-21T09:31:07.991 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.991 INFO:tasks.workunit.client.0.smithi022.stdout: "temperature": { 2024-09-21T09:31:07.991 INFO:tasks.workunit.client.0.smithi022.stdout: "current": 22 2024-09-21T09:31:07.991 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.991 INFO:tasks.workunit.client.0.smithi022.stdout: "user_capacity": { 2024-09-21T09:31:07.991 INFO:tasks.workunit.client.0.smithi022.stdout: "blocks": 781422768, 2024-09-21T09:31:07.991 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 400088457216 2024-09-21T09:31:07.991 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:07.991 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:07.991 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:07.992 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:07.992 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.992 INFO:tasks.workunit.client.0.smithi022.stdout: "report": { 2024-09-21T09:31:07.992 INFO:tasks.workunit.client.0.smithi022.stdout: "balancer": { 2024-09-21T09:31:07.992 INFO:tasks.workunit.client.0.smithi022.stdout: "active": true, 2024-09-21T09:31:07.992 INFO:tasks.workunit.client.0.smithi022.stdout: "mode": "" 2024-09-21T09:31:07.992 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:07.992 INFO:tasks.workunit.client.0.smithi022.stdout: "channels": [ 2024-09-21T09:31:07.992 INFO:tasks.workunit.client.0.smithi022.stdout: "basic", 2024-09-21T09:31:07.992 INFO:tasks.workunit.client.0.smithi022.stdout: "crash", 2024-09-21T09:31:07.992 INFO:tasks.workunit.client.0.smithi022.stdout: "device", 2024-09-21T09:31:07.992 INFO:tasks.workunit.client.0.smithi022.stdout: "ident", 2024-09-21T09:31:07.992 INFO:tasks.workunit.client.0.smithi022.stdout: "perf" 2024-09-21T09:31:07.992 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.993 INFO:tasks.workunit.client.0.smithi022.stdout: "channels_available": [ 2024-09-21T09:31:07.993 INFO:tasks.workunit.client.0.smithi022.stdout: "basic", 2024-09-21T09:31:07.993 INFO:tasks.workunit.client.0.smithi022.stdout: "ident", 2024-09-21T09:31:07.993 INFO:tasks.workunit.client.0.smithi022.stdout: "crash", 2024-09-21T09:31:07.993 INFO:tasks.workunit.client.0.smithi022.stdout: "device", 2024-09-21T09:31:07.993 INFO:tasks.workunit.client.0.smithi022.stdout: "perf" 2024-09-21T09:31:07.993 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.993 INFO:tasks.workunit.client.0.smithi022.stdout: "collections_available": [ 2024-09-21T09:31:07.993 INFO:tasks.workunit.client.0.smithi022.stdout: "basic_base", 2024-09-21T09:31:07.993 INFO:tasks.workunit.client.0.smithi022.stdout: "device_base", 2024-09-21T09:31:07.993 INFO:tasks.workunit.client.0.smithi022.stdout: "crash_base", 2024-09-21T09:31:07.993 INFO:tasks.workunit.client.0.smithi022.stdout: "ident_base", 2024-09-21T09:31:07.993 INFO:tasks.workunit.client.0.smithi022.stdout: "perf_perf", 2024-09-21T09:31:07.993 INFO:tasks.workunit.client.0.smithi022.stdout: "basic_mds_metadata", 2024-09-21T09:31:07.994 INFO:tasks.workunit.client.0.smithi022.stdout: "basic_pool_usage", 2024-09-21T09:31:07.994 INFO:tasks.workunit.client.0.smithi022.stdout: "basic_usage_by_class", 2024-09-21T09:31:07.994 INFO:tasks.workunit.client.0.smithi022.stdout: "basic_rook_v01", 2024-09-21T09:31:07.994 INFO:tasks.workunit.client.0.smithi022.stdout: "perf_memory_metrics", 2024-09-21T09:31:07.994 INFO:tasks.workunit.client.0.smithi022.stdout: "basic_pool_options_bluestore" 2024-09-21T09:31:07.994 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.994 INFO:tasks.workunit.client.0.smithi022.stdout: "collections_opted_in": [ 2024-09-21T09:31:07.994 INFO:tasks.workunit.client.0.smithi022.stdout: "basic_base", 2024-09-21T09:31:07.994 INFO:tasks.workunit.client.0.smithi022.stdout: "device_base", 2024-09-21T09:31:07.994 INFO:tasks.workunit.client.0.smithi022.stdout: "crash_base", 2024-09-21T09:31:07.994 INFO:tasks.workunit.client.0.smithi022.stdout: "ident_base", 2024-09-21T09:31:07.994 INFO:tasks.workunit.client.0.smithi022.stdout: "perf_perf", 2024-09-21T09:31:07.994 INFO:tasks.workunit.client.0.smithi022.stdout: "basic_mds_metadata", 2024-09-21T09:31:07.994 INFO:tasks.workunit.client.0.smithi022.stdout: "basic_pool_usage", 2024-09-21T09:31:07.995 INFO:tasks.workunit.client.0.smithi022.stdout: "basic_usage_by_class", 2024-09-21T09:31:07.995 INFO:tasks.workunit.client.0.smithi022.stdout: "basic_rook_v01", 2024-09-21T09:31:07.995 INFO:tasks.workunit.client.0.smithi022.stdout: "perf_memory_metrics", 2024-09-21T09:31:07.995 INFO:tasks.workunit.client.0.smithi022.stdout: "basic_pool_options_bluestore" 2024-09-21T09:31:07.995 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:07.995 INFO:tasks.workunit.client.0.smithi022.stdout: "config": { 2024-09-21T09:31:07.995 INFO:tasks.workunit.client.0.smithi022.stdout: "active_changed": [ 2024-09-21T09:31:07.995 INFO:tasks.workunit.client.0.smithi022.stdout: "auth_allow_insecure_global_id_reclaim", 2024-09-21T09:31:07.995 INFO:tasks.workunit.client.0.smithi022.stdout: "auth_debug", 2024-09-21T09:31:07.995 INFO:tasks.workunit.client.0.smithi022.stdout: "auth_mon_ticket_ttl", 2024-09-21T09:31:07.995 INFO:tasks.workunit.client.0.smithi022.stdout: "auth_service_ticket_ttl", 2024-09-21T09:31:07.995 INFO:tasks.workunit.client.0.smithi022.stdout: "bdev_debug_aio", 2024-09-21T09:31:07.995 INFO:tasks.workunit.client.0.smithi022.stdout: "client_check_pool_perm", 2024-09-21T09:31:07.996 INFO:tasks.workunit.client.0.smithi022.stdout: "client_mount_gid", 2024-09-21T09:31:07.996 INFO:tasks.workunit.client.0.smithi022.stdout: "client_mount_uid", 2024-09-21T09:31:07.996 INFO:tasks.workunit.client.0.smithi022.stdout: "client_quota", 2024-09-21T09:31:07.996 INFO:tasks.workunit.client.0.smithi022.stdout: "container_image", 2024-09-21T09:31:07.996 INFO:tasks.workunit.client.0.smithi022.stdout: "daemonize", 2024-09-21T09:31:07.996 INFO:tasks.workunit.client.0.smithi022.stdout: "debug_asserts_on_shutdown", 2024-09-21T09:31:07.996 INFO:tasks.workunit.client.0.smithi022.stdout: "debug_mgr", 2024-09-21T09:31:07.996 INFO:tasks.workunit.client.0.smithi022.stdout: "debug_mon", 2024-09-21T09:31:07.996 INFO:tasks.workunit.client.0.smithi022.stdout: "debug_ms", 2024-09-21T09:31:07.996 INFO:tasks.workunit.client.0.smithi022.stdout: "debug_osd", 2024-09-21T09:31:07.996 INFO:tasks.workunit.client.0.smithi022.stdout: "debug_paxos", 2024-09-21T09:31:07.996 INFO:tasks.workunit.client.0.smithi022.stdout: "keyring", 2024-09-21T09:31:07.996 INFO:tasks.workunit.client.0.smithi022.stdout: "leveldb_block_size", 2024-09-21T09:31:07.996 INFO:tasks.workunit.client.0.smithi022.stdout: "leveldb_cache_size", 2024-09-21T09:31:07.996 INFO:tasks.workunit.client.0.smithi022.stdout: "leveldb_compression", 2024-09-21T09:31:07.997 INFO:tasks.workunit.client.0.smithi022.stdout: "leveldb_log", 2024-09-21T09:31:07.997 INFO:tasks.workunit.client.0.smithi022.stdout: "leveldb_write_buffer_size", 2024-09-21T09:31:07.997 INFO:tasks.workunit.client.0.smithi022.stdout: "log_to_file", 2024-09-21T09:31:07.997 INFO:tasks.workunit.client.0.smithi022.stdout: "log_to_journald", 2024-09-21T09:31:07.997 INFO:tasks.workunit.client.0.smithi022.stdout: "log_to_stderr", 2024-09-21T09:31:07.997 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_allow_pool_delete", 2024-09-21T09:31:07.997 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_clock_drift_allowed", 2024-09-21T09:31:07.997 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_cluster_log_to_file", 2024-09-21T09:31:07.997 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_cluster_log_to_journald", 2024-09-21T09:31:07.997 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_cluster_log_to_stderr", 2024-09-21T09:31:07.997 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_data_avail_warn", 2024-09-21T09:31:07.997 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_election_default_strategy", 2024-09-21T09:31:07.997 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_host", 2024-09-21T09:31:07.997 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_max_pg_per_osd", 2024-09-21T09:31:07.998 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_mgr_mkfs_grace", 2024-09-21T09:31:07.998 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_osd_prime_pg_temp", 2024-09-21T09:31:07.998 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_osd_reporter_subtree_level", 2024-09-21T09:31:07.998 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_pg_warn_max_object_skew", 2024-09-21T09:31:07.998 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_reweight_min_bytes_per_osd", 2024-09-21T09:31:07.998 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_reweight_min_pgs_per_osd", 2024-09-21T09:31:07.998 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_warn_on_crush_straw_calc_version_zero", 2024-09-21T09:31:07.998 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_warn_on_insecure_global_id_reclaim", 2024-09-21T09:31:07.998 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_warn_on_insecure_global_id_reclaim_allowed", 2024-09-21T09:31:07.998 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_warn_on_legacy_crush_tunables", 2024-09-21T09:31:07.998 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_warn_on_osd_down_out_interval_zero", 2024-09-21T09:31:07.998 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_warn_on_pool_pg_num_not_power_of_two", 2024-09-21T09:31:07.998 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_warn_on_too_few_osds", 2024-09-21T09:31:07.999 INFO:tasks.workunit.client.0.smithi022.stdout: "ms_die_on_bug", 2024-09-21T09:31:07.999 INFO:tasks.workunit.client.0.smithi022.stdout: "ms_die_on_old_message", 2024-09-21T09:31:07.999 INFO:tasks.workunit.client.0.smithi022.stdout: "no_config_file", 2024-09-21T09:31:07.999 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_class_default_list", 2024-09-21T09:31:07.999 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_class_load_list", 2024-09-21T09:31:07.999 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_debug_misdirected_ops", 2024-09-21T09:31:07.999 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_debug_op_order", 2024-09-21T09:31:07.999 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_debug_pg_log_writeout", 2024-09-21T09:31:07.999 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_debug_shutdown", 2024-09-21T09:31:07.999 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_debug_verify_cached_snaps", 2024-09-21T09:31:07.999 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_debug_verify_missing_on_start", 2024-09-21T09:31:07.999 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_debug_verify_stray_on_activate", 2024-09-21T09:31:07.999 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_deep_scrub_update_digest_min_age", 2024-09-21T09:31:07.999 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_mclock_profile", 2024-09-21T09:31:08.000 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_mclock_scheduler_background_best_effort_lim", 2024-09-21T09:31:08.000 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_mclock_scheduler_background_best_effort_res", 2024-09-21T09:31:08.000 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_mclock_scheduler_background_best_effort_wgt", 2024-09-21T09:31:08.000 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_mclock_scheduler_background_recovery_lim", 2024-09-21T09:31:08.000 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_mclock_scheduler_background_recovery_res", 2024-09-21T09:31:08.000 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_mclock_scheduler_background_recovery_wgt", 2024-09-21T09:31:08.000 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_mclock_scheduler_client_lim", 2024-09-21T09:31:08.000 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_mclock_scheduler_client_res", 2024-09-21T09:31:08.000 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_mclock_scheduler_client_wgt", 2024-09-21T09:31:08.000 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_memory_target", 2024-09-21T09:31:08.000 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_memory_target_autotune", 2024-09-21T09:31:08.000 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_op_queue", 2024-09-21T09:31:08.000 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_op_queue_cut_off", 2024-09-21T09:31:08.000 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_pool_default_erasure_code_profile", 2024-09-21T09:31:08.001 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_pool_default_pg_autoscale_mode", 2024-09-21T09:31:08.001 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_recover_clone_overlap", 2024-09-21T09:31:08.001 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_recovery_max_chunk", 2024-09-21T09:31:08.001 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_scrub_load_threshold", 2024-09-21T09:31:08.001 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_scrub_max_interval", 2024-09-21T09:31:08.001 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_shutdown_pgref_assert", 2024-09-21T09:31:08.001 INFO:tasks.workunit.client.0.smithi022.stdout: "public_addr", 2024-09-21T09:31:08.001 INFO:tasks.workunit.client.0.smithi022.stdout: "public_addrv", 2024-09-21T09:31:08.001 INFO:tasks.workunit.client.0.smithi022.stdout: "public_network", 2024-09-21T09:31:08.001 INFO:tasks.workunit.client.0.smithi022.stdout: "rbd_default_features", 2024-09-21T09:31:08.001 INFO:tasks.workunit.client.0.smithi022.stdout: "rbd_qos_exclude_ops", 2024-09-21T09:31:08.001 INFO:tasks.workunit.client.0.smithi022.stdout: "setgroup", 2024-09-21T09:31:08.001 INFO:tasks.workunit.client.0.smithi022.stdout: "setuser" 2024-09-21T09:31:08.001 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:08.002 INFO:tasks.workunit.client.0.smithi022.stdout: "cluster_changed": [ 2024-09-21T09:31:08.002 INFO:tasks.workunit.client.0.smithi022.stdout: "auth_allow_insecure_global_id_reclaim", 2024-09-21T09:31:08.002 INFO:tasks.workunit.client.0.smithi022.stdout: "auth_debug", 2024-09-21T09:31:08.002 INFO:tasks.workunit.client.0.smithi022.stdout: "auth_mon_ticket_ttl", 2024-09-21T09:31:08.002 INFO:tasks.workunit.client.0.smithi022.stdout: "auth_service_ticket_ttl", 2024-09-21T09:31:08.002 INFO:tasks.workunit.client.0.smithi022.stdout: "bdev_debug_aio", 2024-09-21T09:31:08.002 INFO:tasks.workunit.client.0.smithi022.stdout: "container_image", 2024-09-21T09:31:08.002 INFO:tasks.workunit.client.0.smithi022.stdout: "debug_asserts_on_shutdown", 2024-09-21T09:31:08.002 INFO:tasks.workunit.client.0.smithi022.stdout: "debug_mgr", 2024-09-21T09:31:08.002 INFO:tasks.workunit.client.0.smithi022.stdout: "debug_mon", 2024-09-21T09:31:08.002 INFO:tasks.workunit.client.0.smithi022.stdout: "debug_ms", 2024-09-21T09:31:08.002 INFO:tasks.workunit.client.0.smithi022.stdout: "debug_osd", 2024-09-21T09:31:08.002 INFO:tasks.workunit.client.0.smithi022.stdout: "debug_paxos", 2024-09-21T09:31:08.002 INFO:tasks.workunit.client.0.smithi022.stdout: "log_to_file", 2024-09-21T09:31:08.003 INFO:tasks.workunit.client.0.smithi022.stdout: "log_to_journald", 2024-09-21T09:31:08.003 INFO:tasks.workunit.client.0.smithi022.stdout: "log_to_stderr", 2024-09-21T09:31:08.003 INFO:tasks.workunit.client.0.smithi022.stdout: "mds_join_fs", 2024-09-21T09:31:08.003 INFO:tasks.workunit.client.0.smithi022.stdout: "mgr/cephadm/allow_ptrace", 2024-09-21T09:31:08.003 INFO:tasks.workunit.client.0.smithi022.stdout: "mgr/cephadm/container_init", 2024-09-21T09:31:08.003 INFO:tasks.workunit.client.0.smithi022.stdout: "mgr/cephadm/migration_current", 2024-09-21T09:31:08.003 INFO:tasks.workunit.client.0.smithi022.stdout: "mgr/cephadm/use_repo_digest", 2024-09-21T09:31:08.003 INFO:tasks.workunit.client.0.smithi022.stdout: "mgr/dashboard/ALERTMANAGER_API_HOST", 2024-09-21T09:31:08.003 INFO:tasks.workunit.client.0.smithi022.stdout: "mgr/dashboard/GRAFANA_API_SSL_VERIFY", 2024-09-21T09:31:08.003 INFO:tasks.workunit.client.0.smithi022.stdout: "mgr/dashboard/GRAFANA_API_URL", 2024-09-21T09:31:08.003 INFO:tasks.workunit.client.0.smithi022.stdout: "mgr/dashboard/PROMETHEUS_API_HOST", 2024-09-21T09:31:08.003 INFO:tasks.workunit.client.0.smithi022.stdout: "mgr/dashboard/ssl_server_port", 2024-09-21T09:31:08.003 INFO:tasks.workunit.client.0.smithi022.stdout: "mgr/orchestrator/orchestrator", 2024-09-21T09:31:08.003 INFO:tasks.workunit.client.0.smithi022.stdout: "mgr/telemetry/channel_ident", 2024-09-21T09:31:08.004 INFO:tasks.workunit.client.0.smithi022.stdout: "mgr/telemetry/channel_perf", 2024-09-21T09:31:08.004 INFO:tasks.workunit.client.0.smithi022.stdout: "mgr/telemetry/description", 2024-09-21T09:31:08.004 INFO:tasks.workunit.client.0.smithi022.stdout: "mgr/telemetry/enabled", 2024-09-21T09:31:08.004 INFO:tasks.workunit.client.0.smithi022.stdout: "mgr/telemetry/organization", 2024-09-21T09:31:08.004 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_allow_pool_delete", 2024-09-21T09:31:08.004 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_clock_drift_allowed", 2024-09-21T09:31:08.004 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_cluster_log_to_file", 2024-09-21T09:31:08.004 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_data_avail_warn", 2024-09-21T09:31:08.004 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_election_default_strategy", 2024-09-21T09:31:08.004 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_max_pg_per_osd", 2024-09-21T09:31:08.004 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_mgr_mkfs_grace", 2024-09-21T09:31:08.004 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_osd_prime_pg_temp", 2024-09-21T09:31:08.004 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_osd_reporter_subtree_level", 2024-09-21T09:31:08.004 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_pg_warn_max_object_skew", 2024-09-21T09:31:08.005 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_reweight_min_bytes_per_osd", 2024-09-21T09:31:08.005 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_reweight_min_pgs_per_osd", 2024-09-21T09:31:08.005 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_warn_on_crush_straw_calc_version_zero", 2024-09-21T09:31:08.005 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_warn_on_insecure_global_id_reclaim", 2024-09-21T09:31:08.005 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_warn_on_insecure_global_id_reclaim_allowed", 2024-09-21T09:31:08.005 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_warn_on_legacy_crush_tunables", 2024-09-21T09:31:08.005 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_warn_on_osd_down_out_interval_zero", 2024-09-21T09:31:08.005 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_warn_on_pool_pg_num_not_power_of_two", 2024-09-21T09:31:08.005 INFO:tasks.workunit.client.0.smithi022.stdout: "mon_warn_on_too_few_osds", 2024-09-21T09:31:08.005 INFO:tasks.workunit.client.0.smithi022.stdout: "ms_die_on_bug", 2024-09-21T09:31:08.005 INFO:tasks.workunit.client.0.smithi022.stdout: "ms_die_on_old_message", 2024-09-21T09:31:08.005 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_class_default_list", 2024-09-21T09:31:08.005 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_class_load_list", 2024-09-21T09:31:08.005 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_debug_misdirected_ops", 2024-09-21T09:31:08.005 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_debug_op_order", 2024-09-21T09:31:08.006 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_debug_pg_log_writeout", 2024-09-21T09:31:08.006 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_debug_shutdown", 2024-09-21T09:31:08.006 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_debug_verify_cached_snaps", 2024-09-21T09:31:08.006 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_debug_verify_missing_on_start", 2024-09-21T09:31:08.006 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_debug_verify_stray_on_activate", 2024-09-21T09:31:08.006 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_deep_scrub_update_digest_min_age", 2024-09-21T09:31:08.006 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_mclock_profile", 2024-09-21T09:31:08.006 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_memory_target", 2024-09-21T09:31:08.006 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_memory_target_autotune", 2024-09-21T09:31:08.006 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_op_queue", 2024-09-21T09:31:08.006 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_op_queue_cut_off", 2024-09-21T09:31:08.006 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_pool_default_erasure_code_profile", 2024-09-21T09:31:08.006 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_pool_default_pg_autoscale_mode", 2024-09-21T09:31:08.007 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_recover_clone_overlap", 2024-09-21T09:31:08.007 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_recovery_max_chunk", 2024-09-21T09:31:08.007 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_scrub_load_threshold", 2024-09-21T09:31:08.007 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_scrub_max_interval", 2024-09-21T09:31:08.007 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_shutdown_pgref_assert", 2024-09-21T09:31:08.007 INFO:tasks.workunit.client.0.smithi022.stdout: "public_network", 2024-09-21T09:31:08.007 INFO:tasks.workunit.client.0.smithi022.stdout: "rgw_cache_enabled", 2024-09-21T09:31:08.007 INFO:tasks.workunit.client.0.smithi022.stdout: "rgw_enable_ops_log", 2024-09-21T09:31:08.007 INFO:tasks.workunit.client.0.smithi022.stdout: "rgw_enable_usage_log" 2024-09-21T09:31:08.007 INFO:tasks.workunit.client.0.smithi022.stdout: ] 2024-09-21T09:31:08.007 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.007 INFO:tasks.workunit.client.0.smithi022.stdout: "contact": null, 2024-09-21T09:31:08.007 INFO:tasks.workunit.client.0.smithi022.stdout: "crashes": [], 2024-09-21T09:31:08.008 INFO:tasks.workunit.client.0.smithi022.stdout: "created": "2024-09-21T09:22:01.045656Z", 2024-09-21T09:31:08.008 INFO:tasks.workunit.client.0.smithi022.stdout: "crush": { 2024-09-21T09:31:08.008 INFO:tasks.workunit.client.0.smithi022.stdout: "bucket_algs": { 2024-09-21T09:31:08.008 INFO:tasks.workunit.client.0.smithi022.stdout: "straw2": 3 2024-09-21T09:31:08.008 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.008 INFO:tasks.workunit.client.0.smithi022.stdout: "bucket_sizes": { 2024-09-21T09:31:08.008 INFO:tasks.workunit.client.0.smithi022.stdout: "2": 1, 2024-09-21T09:31:08.008 INFO:tasks.workunit.client.0.smithi022.stdout: "4": 2 2024-09-21T09:31:08.008 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.008 INFO:tasks.workunit.client.0.smithi022.stdout: "bucket_types": { 2024-09-21T09:31:08.008 INFO:tasks.workunit.client.0.smithi022.stdout: "1": 2, 2024-09-21T09:31:08.008 INFO:tasks.workunit.client.0.smithi022.stdout: "11": 1 2024-09-21T09:31:08.008 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.008 INFO:tasks.workunit.client.0.smithi022.stdout: "compat_weight_set": false, 2024-09-21T09:31:08.009 INFO:tasks.workunit.client.0.smithi022.stdout: "device_classes": [ 2024-09-21T09:31:08.009 INFO:tasks.workunit.client.0.smithi022.stdout: 8 2024-09-21T09:31:08.009 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:08.009 INFO:tasks.workunit.client.0.smithi022.stdout: "num_buckets": 6, 2024-09-21T09:31:08.009 INFO:tasks.workunit.client.0.smithi022.stdout: "num_devices": 8, 2024-09-21T09:31:08.009 INFO:tasks.workunit.client.0.smithi022.stdout: "num_rules": 1, 2024-09-21T09:31:08.009 INFO:tasks.workunit.client.0.smithi022.stdout: "num_types": 12, 2024-09-21T09:31:08.009 INFO:tasks.workunit.client.0.smithi022.stdout: "num_weight_sets": 0, 2024-09-21T09:31:08.009 INFO:tasks.workunit.client.0.smithi022.stdout: "tunables": { 2024-09-21T09:31:08.009 INFO:tasks.workunit.client.0.smithi022.stdout: "allowed_bucket_algs": 54, 2024-09-21T09:31:08.009 INFO:tasks.workunit.client.0.smithi022.stdout: "choose_local_fallback_tries": 0, 2024-09-21T09:31:08.009 INFO:tasks.workunit.client.0.smithi022.stdout: "choose_local_tries": 0, 2024-09-21T09:31:08.009 INFO:tasks.workunit.client.0.smithi022.stdout: "choose_total_tries": 50, 2024-09-21T09:31:08.010 INFO:tasks.workunit.client.0.smithi022.stdout: "chooseleaf_descend_once": 1, 2024-09-21T09:31:08.010 INFO:tasks.workunit.client.0.smithi022.stdout: "chooseleaf_stable": 1, 2024-09-21T09:31:08.010 INFO:tasks.workunit.client.0.smithi022.stdout: "chooseleaf_vary_r": 1, 2024-09-21T09:31:08.010 INFO:tasks.workunit.client.0.smithi022.stdout: "has_v2_rules": 0, 2024-09-21T09:31:08.010 INFO:tasks.workunit.client.0.smithi022.stdout: "has_v3_rules": 0, 2024-09-21T09:31:08.010 INFO:tasks.workunit.client.0.smithi022.stdout: "has_v4_buckets": 1, 2024-09-21T09:31:08.010 INFO:tasks.workunit.client.0.smithi022.stdout: "has_v5_rules": 0, 2024-09-21T09:31:08.010 INFO:tasks.workunit.client.0.smithi022.stdout: "legacy_tunables": 0, 2024-09-21T09:31:08.010 INFO:tasks.workunit.client.0.smithi022.stdout: "minimum_required_version": "jewel", 2024-09-21T09:31:08.010 INFO:tasks.workunit.client.0.smithi022.stdout: "optimal_tunables": 1, 2024-09-21T09:31:08.010 INFO:tasks.workunit.client.0.smithi022.stdout: "profile": "jewel", 2024-09-21T09:31:08.010 INFO:tasks.workunit.client.0.smithi022.stdout: "require_feature_tunables": 1, 2024-09-21T09:31:08.010 INFO:tasks.workunit.client.0.smithi022.stdout: "require_feature_tunables2": 1, 2024-09-21T09:31:08.010 INFO:tasks.workunit.client.0.smithi022.stdout: "require_feature_tunables3": 1, 2024-09-21T09:31:08.011 INFO:tasks.workunit.client.0.smithi022.stdout: "require_feature_tunables5": 1, 2024-09-21T09:31:08.011 INFO:tasks.workunit.client.0.smithi022.stdout: "straw_calc_version": 1 2024-09-21T09:31:08.011 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.011 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.011 INFO:tasks.workunit.client.0.smithi022.stdout: "description": "upgrade test cluster", 2024-09-21T09:31:08.011 INFO:tasks.workunit.client.0.smithi022.stdout: "fs": { 2024-09-21T09:31:08.011 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 1, 2024-09-21T09:31:08.011 INFO:tasks.workunit.client.0.smithi022.stdout: "feature_flags": { 2024-09-21T09:31:08.011 INFO:tasks.workunit.client.0.smithi022.stdout: "enable_multiple": true, 2024-09-21T09:31:08.011 INFO:tasks.workunit.client.0.smithi022.stdout: "ever_enabled_multiple": true 2024-09-21T09:31:08.011 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.011 INFO:tasks.workunit.client.0.smithi022.stdout: "filesystems": [ 2024-09-21T09:31:08.011 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:08.011 INFO:tasks.workunit.client.0.smithi022.stdout: "approx_ctime": "2024-09", 2024-09-21T09:31:08.012 INFO:tasks.workunit.client.0.smithi022.stdout: "balancer_enabled": false, 2024-09-21T09:31:08.012 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.012 INFO:tasks.workunit.client.0.smithi022.stdout: "cached_caps": 0, 2024-09-21T09:31:08.012 INFO:tasks.workunit.client.0.smithi022.stdout: "cached_dns": 10, 2024-09-21T09:31:08.012 INFO:tasks.workunit.client.0.smithi022.stdout: "cached_inos": 13, 2024-09-21T09:31:08.012 INFO:tasks.workunit.client.0.smithi022.stdout: "cached_subtrees": 2, 2024-09-21T09:31:08.012 INFO:tasks.workunit.client.0.smithi022.stdout: "ever_allowed_features": 0, 2024-09-21T09:31:08.012 INFO:tasks.workunit.client.0.smithi022.stdout: "explicitly_allowed_features": 0, 2024-09-21T09:31:08.012 INFO:tasks.workunit.client.0.smithi022.stdout: "files": 0, 2024-09-21T09:31:08.012 INFO:tasks.workunit.client.0.smithi022.stdout: "max_mds": 1, 2024-09-21T09:31:08.012 INFO:tasks.workunit.client.0.smithi022.stdout: "num_data_pools": 1, 2024-09-21T09:31:08.012 INFO:tasks.workunit.client.0.smithi022.stdout: "num_in": 1, 2024-09-21T09:31:08.012 INFO:tasks.workunit.client.0.smithi022.stdout: "num_mds": 1, 2024-09-21T09:31:08.013 INFO:tasks.workunit.client.0.smithi022.stdout: "num_sessions": 0, 2024-09-21T09:31:08.013 INFO:tasks.workunit.client.0.smithi022.stdout: "num_standby_replay": 0, 2024-09-21T09:31:08.013 INFO:tasks.workunit.client.0.smithi022.stdout: "num_up": 1, 2024-09-21T09:31:08.013 INFO:tasks.workunit.client.0.smithi022.stdout: "snaps": 0, 2024-09-21T09:31:08.013 INFO:tasks.workunit.client.0.smithi022.stdout: "standby_count_wanted": 0 2024-09-21T09:31:08.013 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.013 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:08.013 INFO:tasks.workunit.client.0.smithi022.stdout: "num_standby_mds": 1, 2024-09-21T09:31:08.013 INFO:tasks.workunit.client.0.smithi022.stdout: "total_num_mds": 2 2024-09-21T09:31:08.013 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.013 INFO:tasks.workunit.client.0.smithi022.stdout: "heap_stats": { 2024-09-21T09:31:08.013 INFO:tasks.workunit.client.0.smithi022.stdout: "mds": { 2024-09-21T09:31:08.014 INFO:tasks.workunit.client.0.smithi022.stdout: "mds.aa5e831b31d98e3163af5edfeb921f9af32463c5": { 2024-09-21T09:31:08.014 INFO:tasks.workunit.client.0.smithi022.stdout: "actual_memory_used": 20594688, 2024-09-21T09:31:08.014 INFO:tasks.workunit.client.0.smithi022.stdout: "central_cache_freelist": 399824, 2024-09-21T09:31:08.014 INFO:tasks.workunit.client.0.smithi022.stdout: "malloc_metadata": 2621440, 2024-09-21T09:31:08.014 INFO:tasks.workunit.client.0.smithi022.stdout: "page_heap_freelist": 0, 2024-09-21T09:31:08.014 INFO:tasks.workunit.client.0.smithi022.stdout: "released_to_os": 589824, 2024-09-21T09:31:08.014 INFO:tasks.workunit.client.0.smithi022.stdout: "spans_in_use": 434, 2024-09-21T09:31:08.014 INFO:tasks.workunit.client.0.smithi022.stdout: "tcmalloc_page_size": 8192, 2024-09-21T09:31:08.014 INFO:tasks.workunit.client.0.smithi022.stdout: "thread_cache_freelists": 1399160, 2024-09-21T09:31:08.014 INFO:tasks.workunit.client.0.smithi022.stdout: "thread_heaps_in_use": 20, 2024-09-21T09:31:08.014 INFO:tasks.workunit.client.0.smithi022.stdout: "transfer_cache_freelist": 1070592, 2024-09-21T09:31:08.014 INFO:tasks.workunit.client.0.smithi022.stdout: "use_by_application": 15103672, 2024-09-21T09:31:08.014 INFO:tasks.workunit.client.0.smithi022.stdout: "virtual_address_space_used": 21184512 2024-09-21T09:31:08.015 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.015 INFO:tasks.workunit.client.0.smithi022.stdout: "mds.f85c107c3119ea450f9f41ee1907af1822a71ba3": { 2024-09-21T09:31:08.015 INFO:tasks.workunit.client.0.smithi022.stdout: "actual_memory_used": 20135936, 2024-09-21T09:31:08.015 INFO:tasks.workunit.client.0.smithi022.stdout: "central_cache_freelist": 361248, 2024-09-21T09:31:08.015 INFO:tasks.workunit.client.0.smithi022.stdout: "malloc_metadata": 2621440, 2024-09-21T09:31:08.015 INFO:tasks.workunit.client.0.smithi022.stdout: "page_heap_freelist": 704512, 2024-09-21T09:31:08.015 INFO:tasks.workunit.client.0.smithi022.stdout: "released_to_os": 0, 2024-09-21T09:31:08.015 INFO:tasks.workunit.client.0.smithi022.stdout: "spans_in_use": 354, 2024-09-21T09:31:08.015 INFO:tasks.workunit.client.0.smithi022.stdout: "tcmalloc_page_size": 8192, 2024-09-21T09:31:08.015 INFO:tasks.workunit.client.0.smithi022.stdout: "thread_cache_freelists": 1200960, 2024-09-21T09:31:08.015 INFO:tasks.workunit.client.0.smithi022.stdout: "thread_heaps_in_use": 12, 2024-09-21T09:31:08.015 INFO:tasks.workunit.client.0.smithi022.stdout: "transfer_cache_freelist": 551936, 2024-09-21T09:31:08.015 INFO:tasks.workunit.client.0.smithi022.stdout: "use_by_application": 14695840, 2024-09-21T09:31:08.016 INFO:tasks.workunit.client.0.smithi022.stdout: "virtual_address_space_used": 20135936 2024-09-21T09:31:08.016 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.016 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.016 INFO:tasks.workunit.client.0.smithi022.stdout: "mon": { 2024-09-21T09:31:08.016 INFO:tasks.workunit.client.0.smithi022.stdout: "mon.5349066dea191aa6e7969482c5114baabfb76f71": { 2024-09-21T09:31:08.016 INFO:tasks.workunit.client.0.smithi022.stdout: "actual_memory_used": 37830656, 2024-09-21T09:31:08.016 INFO:tasks.workunit.client.0.smithi022.stdout: "central_cache_freelist": 1653776, 2024-09-21T09:31:08.016 INFO:tasks.workunit.client.0.smithi022.stdout: "malloc_metadata": 2621440, 2024-09-21T09:31:08.016 INFO:tasks.workunit.client.0.smithi022.stdout: "page_heap_freelist": 860160, 2024-09-21T09:31:08.016 INFO:tasks.workunit.client.0.smithi022.stdout: "released_to_os": 13869056, 2024-09-21T09:31:08.016 INFO:tasks.workunit.client.0.smithi022.stdout: "spans_in_use": 1147, 2024-09-21T09:31:08.016 INFO:tasks.workunit.client.0.smithi022.stdout: "tcmalloc_page_size": 8192, 2024-09-21T09:31:08.016 INFO:tasks.workunit.client.0.smithi022.stdout: "thread_cache_freelists": 6351064, 2024-09-21T09:31:08.016 INFO:tasks.workunit.client.0.smithi022.stdout: "thread_heaps_in_use": 22, 2024-09-21T09:31:08.017 INFO:tasks.workunit.client.0.smithi022.stdout: "transfer_cache_freelist": 2290432, 2024-09-21T09:31:08.017 INFO:tasks.workunit.client.0.smithi022.stdout: "use_by_application": 24053784, 2024-09-21T09:31:08.017 INFO:tasks.workunit.client.0.smithi022.stdout: "virtual_address_space_used": 51699712 2024-09-21T09:31:08.017 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.017 INFO:tasks.workunit.client.0.smithi022.stdout: "mon.aa5e831b31d98e3163af5edfeb921f9af32463c5": { 2024-09-21T09:31:08.017 INFO:tasks.workunit.client.0.smithi022.stdout: "actual_memory_used": 35962880, 2024-09-21T09:31:08.017 INFO:tasks.workunit.client.0.smithi022.stdout: "central_cache_freelist": 1414344, 2024-09-21T09:31:08.017 INFO:tasks.workunit.client.0.smithi022.stdout: "malloc_metadata": 2621440, 2024-09-21T09:31:08.017 INFO:tasks.workunit.client.0.smithi022.stdout: "page_heap_freelist": 0, 2024-09-21T09:31:08.017 INFO:tasks.workunit.client.0.smithi022.stdout: "released_to_os": 15745024, 2024-09-21T09:31:08.017 INFO:tasks.workunit.client.0.smithi022.stdout: "spans_in_use": 1101, 2024-09-21T09:31:08.017 INFO:tasks.workunit.client.0.smithi022.stdout: "tcmalloc_page_size": 8192, 2024-09-21T09:31:08.017 INFO:tasks.workunit.client.0.smithi022.stdout: "thread_cache_freelists": 6704896, 2024-09-21T09:31:08.017 INFO:tasks.workunit.client.0.smithi022.stdout: "thread_heaps_in_use": 21, 2024-09-21T09:31:08.017 INFO:tasks.workunit.client.0.smithi022.stdout: "transfer_cache_freelist": 1635328, 2024-09-21T09:31:08.018 INFO:tasks.workunit.client.0.smithi022.stdout: "use_by_application": 23586872, 2024-09-21T09:31:08.018 INFO:tasks.workunit.client.0.smithi022.stdout: "virtual_address_space_used": 51707904 2024-09-21T09:31:08.018 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.018 INFO:tasks.workunit.client.0.smithi022.stdout: "mon.f85c107c3119ea450f9f41ee1907af1822a71ba3": { 2024-09-21T09:31:08.018 INFO:tasks.workunit.client.0.smithi022.stdout: "actual_memory_used": 45375488, 2024-09-21T09:31:08.018 INFO:tasks.workunit.client.0.smithi022.stdout: "central_cache_freelist": 1905464, 2024-09-21T09:31:08.018 INFO:tasks.workunit.client.0.smithi022.stdout: "malloc_metadata": 2621440, 2024-09-21T09:31:08.018 INFO:tasks.workunit.client.0.smithi022.stdout: "page_heap_freelist": 770048, 2024-09-21T09:31:08.018 INFO:tasks.workunit.client.0.smithi022.stdout: "released_to_os": 6225920, 2024-09-21T09:31:08.018 INFO:tasks.workunit.client.0.smithi022.stdout: "spans_in_use": 1760, 2024-09-21T09:31:08.018 INFO:tasks.workunit.client.0.smithi022.stdout: "tcmalloc_page_size": 8192, 2024-09-21T09:31:08.018 INFO:tasks.workunit.client.0.smithi022.stdout: "thread_cache_freelists": 9567752, 2024-09-21T09:31:08.018 INFO:tasks.workunit.client.0.smithi022.stdout: "thread_heaps_in_use": 22, 2024-09-21T09:31:08.019 INFO:tasks.workunit.client.0.smithi022.stdout: "transfer_cache_freelist": 2816256, 2024-09-21T09:31:08.019 INFO:tasks.workunit.client.0.smithi022.stdout: "use_by_application": 27694528, 2024-09-21T09:31:08.019 INFO:tasks.workunit.client.0.smithi022.stdout: "virtual_address_space_used": 51601408 2024-09-21T09:31:08.019 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.019 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.019 INFO:tasks.workunit.client.0.smithi022.stdout: "osd": { 2024-09-21T09:31:08.019 INFO:tasks.workunit.client.0.smithi022.stdout: "osd.0": { 2024-09-21T09:31:08.019 INFO:tasks.workunit.client.0.smithi022.stdout: "actual_memory_used": 273334272, 2024-09-21T09:31:08.019 INFO:tasks.workunit.client.0.smithi022.stdout: "central_cache_freelist": 781184, 2024-09-21T09:31:08.019 INFO:tasks.workunit.client.0.smithi022.stdout: "malloc_metadata": 2752512, 2024-09-21T09:31:08.019 INFO:tasks.workunit.client.0.smithi022.stdout: "page_heap_freelist": 0, 2024-09-21T09:31:08.019 INFO:tasks.workunit.client.0.smithi022.stdout: "released_to_os": 327680, 2024-09-21T09:31:08.019 INFO:tasks.workunit.client.0.smithi022.stdout: "spans_in_use": 1355, 2024-09-21T09:31:08.019 INFO:tasks.workunit.client.0.smithi022.stdout: "tcmalloc_page_size": 8192, 2024-09-21T09:31:08.020 INFO:tasks.workunit.client.0.smithi022.stdout: "thread_cache_freelists": 5358968, 2024-09-21T09:31:08.020 INFO:tasks.workunit.client.0.smithi022.stdout: "thread_heaps_in_use": 46, 2024-09-21T09:31:08.020 INFO:tasks.workunit.client.0.smithi022.stdout: "transfer_cache_freelist": 1986816, 2024-09-21T09:31:08.020 INFO:tasks.workunit.client.0.smithi022.stdout: "use_by_application": 262454792, 2024-09-21T09:31:08.020 INFO:tasks.workunit.client.0.smithi022.stdout: "virtual_address_space_used": 273661952 2024-09-21T09:31:08.020 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.020 INFO:tasks.workunit.client.0.smithi022.stdout: "osd.1": { 2024-09-21T09:31:08.020 INFO:tasks.workunit.client.0.smithi022.stdout: "actual_memory_used": 272850944, 2024-09-21T09:31:08.020 INFO:tasks.workunit.client.0.smithi022.stdout: "central_cache_freelist": 608320, 2024-09-21T09:31:08.020 INFO:tasks.workunit.client.0.smithi022.stdout: "malloc_metadata": 2752512, 2024-09-21T09:31:08.020 INFO:tasks.workunit.client.0.smithi022.stdout: "page_heap_freelist": 0, 2024-09-21T09:31:08.020 INFO:tasks.workunit.client.0.smithi022.stdout: "released_to_os": 876544, 2024-09-21T09:31:08.020 INFO:tasks.workunit.client.0.smithi022.stdout: "spans_in_use": 1321, 2024-09-21T09:31:08.020 INFO:tasks.workunit.client.0.smithi022.stdout: "tcmalloc_page_size": 8192, 2024-09-21T09:31:08.020 INFO:tasks.workunit.client.0.smithi022.stdout: "thread_cache_freelists": 5472488, 2024-09-21T09:31:08.021 INFO:tasks.workunit.client.0.smithi022.stdout: "thread_heaps_in_use": 46, 2024-09-21T09:31:08.021 INFO:tasks.workunit.client.0.smithi022.stdout: "transfer_cache_freelist": 1896960, 2024-09-21T09:31:08.021 INFO:tasks.workunit.client.0.smithi022.stdout: "use_by_application": 262120664, 2024-09-21T09:31:08.021 INFO:tasks.workunit.client.0.smithi022.stdout: "virtual_address_space_used": 273727488 2024-09-21T09:31:08.021 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.021 INFO:tasks.workunit.client.0.smithi022.stdout: "osd.2": { 2024-09-21T09:31:08.021 INFO:tasks.workunit.client.0.smithi022.stdout: "actual_memory_used": 272048128, 2024-09-21T09:31:08.021 INFO:tasks.workunit.client.0.smithi022.stdout: "central_cache_freelist": 786184, 2024-09-21T09:31:08.021 INFO:tasks.workunit.client.0.smithi022.stdout: "malloc_metadata": 2752512, 2024-09-21T09:31:08.021 INFO:tasks.workunit.client.0.smithi022.stdout: "page_heap_freelist": 0, 2024-09-21T09:31:08.021 INFO:tasks.workunit.client.0.smithi022.stdout: "released_to_os": 843776, 2024-09-21T09:31:08.021 INFO:tasks.workunit.client.0.smithi022.stdout: "spans_in_use": 1251, 2024-09-21T09:31:08.021 INFO:tasks.workunit.client.0.smithi022.stdout: "tcmalloc_page_size": 8192, 2024-09-21T09:31:08.022 INFO:tasks.workunit.client.0.smithi022.stdout: "thread_cache_freelists": 4670304, 2024-09-21T09:31:08.022 INFO:tasks.workunit.client.0.smithi022.stdout: "thread_heaps_in_use": 47, 2024-09-21T09:31:08.022 INFO:tasks.workunit.client.0.smithi022.stdout: "transfer_cache_freelist": 2202624, 2024-09-21T09:31:08.022 INFO:tasks.workunit.client.0.smithi022.stdout: "use_by_application": 261636504, 2024-09-21T09:31:08.022 INFO:tasks.workunit.client.0.smithi022.stdout: "virtual_address_space_used": 272891904 2024-09-21T09:31:08.022 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.022 INFO:tasks.workunit.client.0.smithi022.stdout: "osd.3": { 2024-09-21T09:31:08.022 INFO:tasks.workunit.client.0.smithi022.stdout: "actual_memory_used": 271597568, 2024-09-21T09:31:08.022 INFO:tasks.workunit.client.0.smithi022.stdout: "central_cache_freelist": 601592, 2024-09-21T09:31:08.022 INFO:tasks.workunit.client.0.smithi022.stdout: "malloc_metadata": 2752512, 2024-09-21T09:31:08.022 INFO:tasks.workunit.client.0.smithi022.stdout: "page_heap_freelist": 24576, 2024-09-21T09:31:08.022 INFO:tasks.workunit.client.0.smithi022.stdout: "released_to_os": 32768, 2024-09-21T09:31:08.022 INFO:tasks.workunit.client.0.smithi022.stdout: "spans_in_use": 1246, 2024-09-21T09:31:08.023 INFO:tasks.workunit.client.0.smithi022.stdout: "tcmalloc_page_size": 8192, 2024-09-21T09:31:08.023 INFO:tasks.workunit.client.0.smithi022.stdout: "thread_cache_freelists": 4730552, 2024-09-21T09:31:08.023 INFO:tasks.workunit.client.0.smithi022.stdout: "thread_heaps_in_use": 46, 2024-09-21T09:31:08.023 INFO:tasks.workunit.client.0.smithi022.stdout: "transfer_cache_freelist": 1671936, 2024-09-21T09:31:08.023 INFO:tasks.workunit.client.0.smithi022.stdout: "use_by_application": 261816400, 2024-09-21T09:31:08.023 INFO:tasks.workunit.client.0.smithi022.stdout: "virtual_address_space_used": 271630336 2024-09-21T09:31:08.023 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.023 INFO:tasks.workunit.client.0.smithi022.stdout: "osd.4": { 2024-09-21T09:31:08.023 INFO:tasks.workunit.client.0.smithi022.stdout: "actual_memory_used": 272146432, 2024-09-21T09:31:08.023 INFO:tasks.workunit.client.0.smithi022.stdout: "central_cache_freelist": 705456, 2024-09-21T09:31:08.023 INFO:tasks.workunit.client.0.smithi022.stdout: "malloc_metadata": 2752512, 2024-09-21T09:31:08.023 INFO:tasks.workunit.client.0.smithi022.stdout: "page_heap_freelist": 0, 2024-09-21T09:31:08.023 INFO:tasks.workunit.client.0.smithi022.stdout: "released_to_os": 466944, 2024-09-21T09:31:08.024 INFO:tasks.workunit.client.0.smithi022.stdout: "spans_in_use": 1260, 2024-09-21T09:31:08.024 INFO:tasks.workunit.client.0.smithi022.stdout: "tcmalloc_page_size": 8192, 2024-09-21T09:31:08.024 INFO:tasks.workunit.client.0.smithi022.stdout: "thread_cache_freelists": 4604952, 2024-09-21T09:31:08.024 INFO:tasks.workunit.client.0.smithi022.stdout: "thread_heaps_in_use": 46, 2024-09-21T09:31:08.024 INFO:tasks.workunit.client.0.smithi022.stdout: "transfer_cache_freelist": 2297856, 2024-09-21T09:31:08.024 INFO:tasks.workunit.client.0.smithi022.stdout: "use_by_application": 261785656, 2024-09-21T09:31:08.024 INFO:tasks.workunit.client.0.smithi022.stdout: "virtual_address_space_used": 272613376 2024-09-21T09:31:08.024 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.024 INFO:tasks.workunit.client.0.smithi022.stdout: "osd.5": { 2024-09-21T09:31:08.024 INFO:tasks.workunit.client.0.smithi022.stdout: "actual_memory_used": 272990208, 2024-09-21T09:31:08.024 INFO:tasks.workunit.client.0.smithi022.stdout: "central_cache_freelist": 824872, 2024-09-21T09:31:08.024 INFO:tasks.workunit.client.0.smithi022.stdout: "malloc_metadata": 2752512, 2024-09-21T09:31:08.024 INFO:tasks.workunit.client.0.smithi022.stdout: "page_heap_freelist": 0, 2024-09-21T09:31:08.024 INFO:tasks.workunit.client.0.smithi022.stdout: "released_to_os": 737280, 2024-09-21T09:31:08.024 INFO:tasks.workunit.client.0.smithi022.stdout: "spans_in_use": 1265, 2024-09-21T09:31:08.025 INFO:tasks.workunit.client.0.smithi022.stdout: "tcmalloc_page_size": 8192, 2024-09-21T09:31:08.025 INFO:tasks.workunit.client.0.smithi022.stdout: "thread_cache_freelists": 5257336, 2024-09-21T09:31:08.025 INFO:tasks.workunit.client.0.smithi022.stdout: "thread_heaps_in_use": 46, 2024-09-21T09:31:08.025 INFO:tasks.workunit.client.0.smithi022.stdout: "transfer_cache_freelist": 1883392, 2024-09-21T09:31:08.025 INFO:tasks.workunit.client.0.smithi022.stdout: "use_by_application": 262272096, 2024-09-21T09:31:08.025 INFO:tasks.workunit.client.0.smithi022.stdout: "virtual_address_space_used": 273727488 2024-09-21T09:31:08.025 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.025 INFO:tasks.workunit.client.0.smithi022.stdout: "osd.6": { 2024-09-21T09:31:08.025 INFO:tasks.workunit.client.0.smithi022.stdout: "actual_memory_used": 271679488, 2024-09-21T09:31:08.025 INFO:tasks.workunit.client.0.smithi022.stdout: "central_cache_freelist": 738144, 2024-09-21T09:31:08.025 INFO:tasks.workunit.client.0.smithi022.stdout: "malloc_metadata": 2752512, 2024-09-21T09:31:08.025 INFO:tasks.workunit.client.0.smithi022.stdout: "page_heap_freelist": 0, 2024-09-21T09:31:08.025 INFO:tasks.workunit.client.0.smithi022.stdout: "released_to_os": 163840, 2024-09-21T09:31:08.025 INFO:tasks.workunit.client.0.smithi022.stdout: "spans_in_use": 1241, 2024-09-21T09:31:08.026 INFO:tasks.workunit.client.0.smithi022.stdout: "tcmalloc_page_size": 8192, 2024-09-21T09:31:08.026 INFO:tasks.workunit.client.0.smithi022.stdout: "thread_cache_freelists": 4662000, 2024-09-21T09:31:08.026 INFO:tasks.workunit.client.0.smithi022.stdout: "thread_heaps_in_use": 46, 2024-09-21T09:31:08.026 INFO:tasks.workunit.client.0.smithi022.stdout: "transfer_cache_freelist": 1976832, 2024-09-21T09:31:08.026 INFO:tasks.workunit.client.0.smithi022.stdout: "use_by_application": 261550000, 2024-09-21T09:31:08.026 INFO:tasks.workunit.client.0.smithi022.stdout: "virtual_address_space_used": 271843328 2024-09-21T09:31:08.026 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.026 INFO:tasks.workunit.client.0.smithi022.stdout: "osd.7": { 2024-09-21T09:31:08.026 INFO:tasks.workunit.client.0.smithi022.stdout: "actual_memory_used": 272662528, 2024-09-21T09:31:08.026 INFO:tasks.workunit.client.0.smithi022.stdout: "central_cache_freelist": 988832, 2024-09-21T09:31:08.026 INFO:tasks.workunit.client.0.smithi022.stdout: "malloc_metadata": 2752512, 2024-09-21T09:31:08.026 INFO:tasks.workunit.client.0.smithi022.stdout: "page_heap_freelist": 0, 2024-09-21T09:31:08.027 INFO:tasks.workunit.client.0.smithi022.stdout: "released_to_os": 1064960, 2024-09-21T09:31:08.027 INFO:tasks.workunit.client.0.smithi022.stdout: "spans_in_use": 1296, 2024-09-21T09:31:08.027 INFO:tasks.workunit.client.0.smithi022.stdout: "tcmalloc_page_size": 8192, 2024-09-21T09:31:08.027 INFO:tasks.workunit.client.0.smithi022.stdout: "thread_cache_freelists": 4922312, 2024-09-21T09:31:08.027 INFO:tasks.workunit.client.0.smithi022.stdout: "thread_heaps_in_use": 45, 2024-09-21T09:31:08.027 INFO:tasks.workunit.client.0.smithi022.stdout: "transfer_cache_freelist": 1961984, 2024-09-21T09:31:08.027 INFO:tasks.workunit.client.0.smithi022.stdout: "use_by_application": 262036888, 2024-09-21T09:31:08.027 INFO:tasks.workunit.client.0.smithi022.stdout: "virtual_address_space_used": 273727488 2024-09-21T09:31:08.027 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.027 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.027 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.028 INFO:tasks.workunit.client.0.smithi022.stdout: "hosts": { 2024-09-21T09:31:08.028 INFO:tasks.workunit.client.0.smithi022.stdout: "num": 2, 2024-09-21T09:31:08.028 INFO:tasks.workunit.client.0.smithi022.stdout: "num_with_mds": 2, 2024-09-21T09:31:08.028 INFO:tasks.workunit.client.0.smithi022.stdout: "num_with_mgr": 2, 2024-09-21T09:31:08.028 INFO:tasks.workunit.client.0.smithi022.stdout: "num_with_mon": 2, 2024-09-21T09:31:08.028 INFO:tasks.workunit.client.0.smithi022.stdout: "num_with_osd": 2 2024-09-21T09:31:08.028 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.028 INFO:tasks.workunit.client.0.smithi022.stdout: "io_rate": { 2024-09-21T09:31:08.028 INFO:tasks.workunit.client.0.smithi022.stdout: "pg_stats_delta": { 2024-09-21T09:31:08.028 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": 0, 2024-09-21T09:31:08.028 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:08.028 INFO:tasks.workunit.client.0.smithi022.stdout: "num_store_stats": 0, 2024-09-21T09:31:08.028 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:08.028 INFO:tasks.workunit.client.0.smithi022.stdout: "stamp_delta": "12.002359", 2024-09-21T09:31:08.028 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:08.029 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:08.029 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:08.029 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:08.029 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:08.029 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:08.029 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:08.029 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:08.029 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:08.029 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:08.029 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:08.029 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:08.029 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:08.029 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:08.029 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:08.030 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:08.030 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:08.030 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:08.030 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:08.030 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:08.030 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:08.030 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:08.030 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:08.030 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:08.030 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:08.030 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:08.030 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:08.030 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:08.031 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:08.031 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:08.031 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:08.031 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:08.031 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:08.031 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:08.031 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:08.031 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:08.031 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:08.031 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:08.031 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:08.031 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:08.031 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:08.031 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.032 INFO:tasks.workunit.client.0.smithi022.stdout: "store_stats": { 2024-09-21T09:31:08.032 INFO:tasks.workunit.client.0.smithi022.stdout: "allocated": 0, 2024-09-21T09:31:08.032 INFO:tasks.workunit.client.0.smithi022.stdout: "available": 0, 2024-09-21T09:31:08.032 INFO:tasks.workunit.client.0.smithi022.stdout: "data_compressed": 0, 2024-09-21T09:31:08.032 INFO:tasks.workunit.client.0.smithi022.stdout: "data_compressed_allocated": 0, 2024-09-21T09:31:08.032 INFO:tasks.workunit.client.0.smithi022.stdout: "data_compressed_original": 0, 2024-09-21T09:31:08.032 INFO:tasks.workunit.client.0.smithi022.stdout: "data_stored": 0, 2024-09-21T09:31:08.032 INFO:tasks.workunit.client.0.smithi022.stdout: "internal_metadata": 0, 2024-09-21T09:31:08.032 INFO:tasks.workunit.client.0.smithi022.stdout: "internally_reserved": 0, 2024-09-21T09:31:08.032 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_allocated": 0, 2024-09-21T09:31:08.032 INFO:tasks.workunit.client.0.smithi022.stdout: "total": 0 2024-09-21T09:31:08.032 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.032 INFO:tasks.workunit.client.0.smithi022.stdout: "up": 0 2024-09-21T09:31:08.032 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.032 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.033 INFO:tasks.workunit.client.0.smithi022.stdout: "leaderboard": false, 2024-09-21T09:31:08.033 INFO:tasks.workunit.client.0.smithi022.stdout: "license": "sharing-1-0", 2024-09-21T09:31:08.033 INFO:tasks.workunit.client.0.smithi022.stdout: "mempool": { 2024-09-21T09:31:08.033 INFO:tasks.workunit.client.0.smithi022.stdout: "mds": { 2024-09-21T09:31:08.033 INFO:tasks.workunit.client.0.smithi022.stdout: "mds.aa5e831b31d98e3163af5edfeb921f9af32463c5": { 2024-09-21T09:31:08.033 INFO:tasks.workunit.client.0.smithi022.stdout: "bloom_filter": { 2024-09-21T09:31:08.033 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.033 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.033 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.033 INFO:tasks.workunit.client.0.smithi022.stdout: "bluefs": { 2024-09-21T09:31:08.033 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.033 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.033 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.033 INFO:tasks.workunit.client.0.smithi022.stdout: "bluefs_file_reader": { 2024-09-21T09:31:08.033 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.034 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.034 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.034 INFO:tasks.workunit.client.0.smithi022.stdout: "bluefs_file_writer": { 2024-09-21T09:31:08.034 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.034 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.034 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.034 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_Blob": { 2024-09-21T09:31:08.034 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.034 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.034 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.034 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_Buffer": { 2024-09-21T09:31:08.034 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.034 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.034 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.035 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_Extent": { 2024-09-21T09:31:08.035 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.035 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.035 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.035 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_SharedBlob": { 2024-09-21T09:31:08.035 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.035 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.035 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.035 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_alloc": { 2024-09-21T09:31:08.035 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.035 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.035 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.035 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_cache_data": { 2024-09-21T09:31:08.035 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.035 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.036 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.036 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_cache_meta": { 2024-09-21T09:31:08.036 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.036 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.036 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.036 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_cache_onode": { 2024-09-21T09:31:08.036 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.036 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.036 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.036 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_cache_other": { 2024-09-21T09:31:08.036 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.037 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.037 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.037 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_fsck": { 2024-09-21T09:31:08.037 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.037 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.037 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.037 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_inline_bl": { 2024-09-21T09:31:08.037 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.037 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.037 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.037 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_txc": { 2024-09-21T09:31:08.037 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.037 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.037 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.038 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_writing": { 2024-09-21T09:31:08.038 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.038 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.038 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.038 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_writing_deferred": { 2024-09-21T09:31:08.038 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.038 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.038 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.038 INFO:tasks.workunit.client.0.smithi022.stdout: "buffer_anon": { 2024-09-21T09:31:08.038 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 222471, 2024-09-21T09:31:08.038 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 72 2024-09-21T09:31:08.038 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.038 INFO:tasks.workunit.client.0.smithi022.stdout: "buffer_meta": { 2024-09-21T09:31:08.039 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.039 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.039 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.039 INFO:tasks.workunit.client.0.smithi022.stdout: "mds_co": { 2024-09-21T09:31:08.039 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 50488, 2024-09-21T09:31:08.039 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 208 2024-09-21T09:31:08.039 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.039 INFO:tasks.workunit.client.0.smithi022.stdout: "osd": { 2024-09-21T09:31:08.039 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.039 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.039 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.039 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_mapbl": { 2024-09-21T09:31:08.040 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.040 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.040 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.040 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_pglog": { 2024-09-21T09:31:08.040 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.040 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.040 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.040 INFO:tasks.workunit.client.0.smithi022.stdout: "osdmap": { 2024-09-21T09:31:08.040 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 8136, 2024-09-21T09:31:08.040 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 139 2024-09-21T09:31:08.040 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.040 INFO:tasks.workunit.client.0.smithi022.stdout: "osdmap_mapping": { 2024-09-21T09:31:08.041 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.041 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.041 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.041 INFO:tasks.workunit.client.0.smithi022.stdout: "pgmap": { 2024-09-21T09:31:08.041 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.041 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.041 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.041 INFO:tasks.workunit.client.0.smithi022.stdout: "unittest_1": { 2024-09-21T09:31:08.041 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.041 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.041 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.041 INFO:tasks.workunit.client.0.smithi022.stdout: "unittest_2": { 2024-09-21T09:31:08.041 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.041 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.042 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.042 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.042 INFO:tasks.workunit.client.0.smithi022.stdout: "mds.f85c107c3119ea450f9f41ee1907af1822a71ba3": { 2024-09-21T09:31:08.042 INFO:tasks.workunit.client.0.smithi022.stdout: "bloom_filter": { 2024-09-21T09:31:08.042 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.042 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.042 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.042 INFO:tasks.workunit.client.0.smithi022.stdout: "bluefs": { 2024-09-21T09:31:08.042 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.042 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.042 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.042 INFO:tasks.workunit.client.0.smithi022.stdout: "bluefs_file_reader": { 2024-09-21T09:31:08.042 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.043 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.043 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.043 INFO:tasks.workunit.client.0.smithi022.stdout: "bluefs_file_writer": { 2024-09-21T09:31:08.043 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.043 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.043 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.043 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_Blob": { 2024-09-21T09:31:08.043 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.043 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.043 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.043 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_Buffer": { 2024-09-21T09:31:08.043 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.043 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.043 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.044 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_Extent": { 2024-09-21T09:31:08.044 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.044 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.044 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.044 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_SharedBlob": { 2024-09-21T09:31:08.044 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.044 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.044 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.044 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_alloc": { 2024-09-21T09:31:08.044 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.044 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.044 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.044 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_cache_data": { 2024-09-21T09:31:08.045 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.045 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.045 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.045 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_cache_meta": { 2024-09-21T09:31:08.045 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.045 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.045 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.045 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_cache_onode": { 2024-09-21T09:31:08.045 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.045 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.045 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.045 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_cache_other": { 2024-09-21T09:31:08.045 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.046 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.046 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.046 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_fsck": { 2024-09-21T09:31:08.046 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.046 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.046 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.046 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_inline_bl": { 2024-09-21T09:31:08.046 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.046 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.046 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.046 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_txc": { 2024-09-21T09:31:08.046 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.046 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.046 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.047 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_writing": { 2024-09-21T09:31:08.047 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.047 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.047 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.047 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_writing_deferred": { 2024-09-21T09:31:08.047 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.047 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.047 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.047 INFO:tasks.workunit.client.0.smithi022.stdout: "buffer_anon": { 2024-09-21T09:31:08.047 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 149097, 2024-09-21T09:31:08.047 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 33 2024-09-21T09:31:08.047 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.047 INFO:tasks.workunit.client.0.smithi022.stdout: "buffer_meta": { 2024-09-21T09:31:08.048 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.048 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.048 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.048 INFO:tasks.workunit.client.0.smithi022.stdout: "mds_co": { 2024-09-21T09:31:08.048 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 856, 2024-09-21T09:31:08.048 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 2 2024-09-21T09:31:08.048 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.048 INFO:tasks.workunit.client.0.smithi022.stdout: "osd": { 2024-09-21T09:31:08.048 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.048 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.048 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.048 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_mapbl": { 2024-09-21T09:31:08.048 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.049 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.049 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.049 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_pglog": { 2024-09-21T09:31:08.049 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.049 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.049 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.049 INFO:tasks.workunit.client.0.smithi022.stdout: "osdmap": { 2024-09-21T09:31:08.049 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.049 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.049 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.049 INFO:tasks.workunit.client.0.smithi022.stdout: "osdmap_mapping": { 2024-09-21T09:31:08.049 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.049 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.050 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.050 INFO:tasks.workunit.client.0.smithi022.stdout: "pgmap": { 2024-09-21T09:31:08.050 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.050 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.050 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.050 INFO:tasks.workunit.client.0.smithi022.stdout: "unittest_1": { 2024-09-21T09:31:08.050 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.050 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.050 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.050 INFO:tasks.workunit.client.0.smithi022.stdout: "unittest_2": { 2024-09-21T09:31:08.050 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.050 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.050 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.051 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.051 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.051 INFO:tasks.workunit.client.0.smithi022.stdout: "mon": { 2024-09-21T09:31:08.051 INFO:tasks.workunit.client.0.smithi022.stdout: "mon.5349066dea191aa6e7969482c5114baabfb76f71": { 2024-09-21T09:31:08.051 INFO:tasks.workunit.client.0.smithi022.stdout: "bloom_filter": { 2024-09-21T09:31:08.051 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.051 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.051 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.051 INFO:tasks.workunit.client.0.smithi022.stdout: "bluefs": { 2024-09-21T09:31:08.051 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.051 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.051 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.051 INFO:tasks.workunit.client.0.smithi022.stdout: "bluefs_file_reader": { 2024-09-21T09:31:08.052 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.052 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.052 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.052 INFO:tasks.workunit.client.0.smithi022.stdout: "bluefs_file_writer": { 2024-09-21T09:31:08.052 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.052 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.052 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.052 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_Blob": { 2024-09-21T09:31:08.052 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.052 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.052 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.052 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_Buffer": { 2024-09-21T09:31:08.052 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.052 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.053 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.053 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_Extent": { 2024-09-21T09:31:08.053 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.053 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.053 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.053 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_SharedBlob": { 2024-09-21T09:31:08.053 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.053 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.053 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.053 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_alloc": { 2024-09-21T09:31:08.053 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.053 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.053 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.053 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_cache_data": { 2024-09-21T09:31:08.054 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.054 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.054 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.054 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_cache_meta": { 2024-09-21T09:31:08.054 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.054 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.054 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.054 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_cache_onode": { 2024-09-21T09:31:08.054 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.054 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.054 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.054 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_cache_other": { 2024-09-21T09:31:08.054 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.054 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.055 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.055 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_fsck": { 2024-09-21T09:31:08.055 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.055 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.055 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.055 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_inline_bl": { 2024-09-21T09:31:08.055 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.055 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.055 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.055 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_txc": { 2024-09-21T09:31:08.055 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.055 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.055 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.056 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_writing": { 2024-09-21T09:31:08.056 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.056 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.056 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.056 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_writing_deferred": { 2024-09-21T09:31:08.056 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.056 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.056 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.056 INFO:tasks.workunit.client.0.smithi022.stdout: "buffer_anon": { 2024-09-21T09:31:08.056 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 1302464, 2024-09-21T09:31:08.056 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 422 2024-09-21T09:31:08.056 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.056 INFO:tasks.workunit.client.0.smithi022.stdout: "buffer_meta": { 2024-09-21T09:31:08.056 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.057 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.057 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.057 INFO:tasks.workunit.client.0.smithi022.stdout: "mds_co": { 2024-09-21T09:31:08.057 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.057 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.057 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.057 INFO:tasks.workunit.client.0.smithi022.stdout: "osd": { 2024-09-21T09:31:08.057 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.057 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.057 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.057 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_mapbl": { 2024-09-21T09:31:08.057 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.057 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.057 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.058 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_pglog": { 2024-09-21T09:31:08.058 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.058 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.058 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.058 INFO:tasks.workunit.client.0.smithi022.stdout: "osdmap": { 2024-09-21T09:31:08.058 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 6784, 2024-09-21T09:31:08.058 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 138 2024-09-21T09:31:08.058 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.058 INFO:tasks.workunit.client.0.smithi022.stdout: "osdmap_mapping": { 2024-09-21T09:31:08.058 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 12744, 2024-09-21T09:31:08.058 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 1546 2024-09-21T09:31:08.058 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.059 INFO:tasks.workunit.client.0.smithi022.stdout: "pgmap": { 2024-09-21T09:31:08.059 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 5312, 2024-09-21T09:31:08.059 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 103 2024-09-21T09:31:08.059 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.059 INFO:tasks.workunit.client.0.smithi022.stdout: "unittest_1": { 2024-09-21T09:31:08.059 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.059 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.059 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.059 INFO:tasks.workunit.client.0.smithi022.stdout: "unittest_2": { 2024-09-21T09:31:08.059 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.059 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.059 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.059 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.059 INFO:tasks.workunit.client.0.smithi022.stdout: "mon.aa5e831b31d98e3163af5edfeb921f9af32463c5": { 2024-09-21T09:31:08.060 INFO:tasks.workunit.client.0.smithi022.stdout: "bloom_filter": { 2024-09-21T09:31:08.060 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.060 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.060 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.060 INFO:tasks.workunit.client.0.smithi022.stdout: "bluefs": { 2024-09-21T09:31:08.060 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.060 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.060 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.060 INFO:tasks.workunit.client.0.smithi022.stdout: "bluefs_file_reader": { 2024-09-21T09:31:08.060 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.060 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.060 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.060 INFO:tasks.workunit.client.0.smithi022.stdout: "bluefs_file_writer": { 2024-09-21T09:31:08.061 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.061 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.061 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.061 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_Blob": { 2024-09-21T09:31:08.061 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.061 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.061 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.061 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_Buffer": { 2024-09-21T09:31:08.061 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.061 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.061 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.061 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_Extent": { 2024-09-21T09:31:08.061 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.061 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.062 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.062 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_SharedBlob": { 2024-09-21T09:31:08.062 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.062 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.062 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.062 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_alloc": { 2024-09-21T09:31:08.062 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.062 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.062 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.062 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_cache_data": { 2024-09-21T09:31:08.062 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.062 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.062 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.062 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_cache_meta": { 2024-09-21T09:31:08.063 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.063 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.063 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.063 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_cache_onode": { 2024-09-21T09:31:08.063 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.063 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.063 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.063 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_cache_other": { 2024-09-21T09:31:08.063 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.063 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.063 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.063 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_fsck": { 2024-09-21T09:31:08.063 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.064 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.064 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.064 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_inline_bl": { 2024-09-21T09:31:08.064 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.064 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.064 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.064 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_txc": { 2024-09-21T09:31:08.064 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.064 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.064 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.064 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_writing": { 2024-09-21T09:31:08.064 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.064 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.064 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.065 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_writing_deferred": { 2024-09-21T09:31:08.065 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.065 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.065 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.065 INFO:tasks.workunit.client.0.smithi022.stdout: "buffer_anon": { 2024-09-21T09:31:08.065 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 1091750, 2024-09-21T09:31:08.065 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 346 2024-09-21T09:31:08.065 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.065 INFO:tasks.workunit.client.0.smithi022.stdout: "buffer_meta": { 2024-09-21T09:31:08.065 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.065 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.065 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.065 INFO:tasks.workunit.client.0.smithi022.stdout: "mds_co": { 2024-09-21T09:31:08.065 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.065 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.066 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.066 INFO:tasks.workunit.client.0.smithi022.stdout: "osd": { 2024-09-21T09:31:08.066 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.066 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.066 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.066 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_mapbl": { 2024-09-21T09:31:08.066 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.066 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.066 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.066 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_pglog": { 2024-09-21T09:31:08.066 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.066 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.066 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.066 INFO:tasks.workunit.client.0.smithi022.stdout: "osdmap": { 2024-09-21T09:31:08.067 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 6784, 2024-09-21T09:31:08.067 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 138 2024-09-21T09:31:08.067 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.067 INFO:tasks.workunit.client.0.smithi022.stdout: "osdmap_mapping": { 2024-09-21T09:31:08.067 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 12744, 2024-09-21T09:31:08.067 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 1546 2024-09-21T09:31:08.067 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.067 INFO:tasks.workunit.client.0.smithi022.stdout: "pgmap": { 2024-09-21T09:31:08.067 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 5312, 2024-09-21T09:31:08.067 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 103 2024-09-21T09:31:08.067 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.067 INFO:tasks.workunit.client.0.smithi022.stdout: "unittest_1": { 2024-09-21T09:31:08.067 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.067 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.068 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.068 INFO:tasks.workunit.client.0.smithi022.stdout: "unittest_2": { 2024-09-21T09:31:08.068 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.068 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.068 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.068 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.068 INFO:tasks.workunit.client.0.smithi022.stdout: "mon.f85c107c3119ea450f9f41ee1907af1822a71ba3": { 2024-09-21T09:31:08.068 INFO:tasks.workunit.client.0.smithi022.stdout: "bloom_filter": { 2024-09-21T09:31:08.068 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.068 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.068 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.068 INFO:tasks.workunit.client.0.smithi022.stdout: "bluefs": { 2024-09-21T09:31:08.068 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.068 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.068 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.069 INFO:tasks.workunit.client.0.smithi022.stdout: "bluefs_file_reader": { 2024-09-21T09:31:08.069 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.069 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.069 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.069 INFO:tasks.workunit.client.0.smithi022.stdout: "bluefs_file_writer": { 2024-09-21T09:31:08.069 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.069 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.069 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.069 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_Blob": { 2024-09-21T09:31:08.069 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.069 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.069 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.069 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_Buffer": { 2024-09-21T09:31:08.069 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.069 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.070 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.070 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_Extent": { 2024-09-21T09:31:08.070 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.070 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.070 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.070 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_SharedBlob": { 2024-09-21T09:31:08.070 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.070 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.070 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.070 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_alloc": { 2024-09-21T09:31:08.070 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.070 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.070 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.071 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_cache_data": { 2024-09-21T09:31:08.071 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.071 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.071 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.071 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_cache_meta": { 2024-09-21T09:31:08.071 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.071 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.071 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.071 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_cache_onode": { 2024-09-21T09:31:08.071 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.071 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.071 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.071 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_cache_other": { 2024-09-21T09:31:08.071 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.072 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.072 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.072 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_fsck": { 2024-09-21T09:31:08.072 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.072 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.072 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.072 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_inline_bl": { 2024-09-21T09:31:08.072 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.072 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.072 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.072 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_txc": { 2024-09-21T09:31:08.072 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.072 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.072 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.072 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_writing": { 2024-09-21T09:31:08.073 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.073 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.073 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.073 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_writing_deferred": { 2024-09-21T09:31:08.073 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.073 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.073 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.073 INFO:tasks.workunit.client.0.smithi022.stdout: "buffer_anon": { 2024-09-21T09:31:08.073 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 1135964, 2024-09-21T09:31:08.073 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 354 2024-09-21T09:31:08.073 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.073 INFO:tasks.workunit.client.0.smithi022.stdout: "buffer_meta": { 2024-09-21T09:31:08.073 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 440, 2024-09-21T09:31:08.073 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 5 2024-09-21T09:31:08.074 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.074 INFO:tasks.workunit.client.0.smithi022.stdout: "mds_co": { 2024-09-21T09:31:08.074 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.074 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.074 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.074 INFO:tasks.workunit.client.0.smithi022.stdout: "osd": { 2024-09-21T09:31:08.074 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.074 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.074 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.074 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_mapbl": { 2024-09-21T09:31:08.074 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.074 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.074 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.075 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_pglog": { 2024-09-21T09:31:08.075 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.075 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.075 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.075 INFO:tasks.workunit.client.0.smithi022.stdout: "osdmap": { 2024-09-21T09:31:08.075 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 6784, 2024-09-21T09:31:08.075 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 138 2024-09-21T09:31:08.075 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.075 INFO:tasks.workunit.client.0.smithi022.stdout: "osdmap_mapping": { 2024-09-21T09:31:08.075 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 12744, 2024-09-21T09:31:08.075 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 1546 2024-09-21T09:31:08.075 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.075 INFO:tasks.workunit.client.0.smithi022.stdout: "pgmap": { 2024-09-21T09:31:08.075 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 10624, 2024-09-21T09:31:08.076 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 206 2024-09-21T09:31:08.076 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.076 INFO:tasks.workunit.client.0.smithi022.stdout: "unittest_1": { 2024-09-21T09:31:08.076 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.076 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.076 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.076 INFO:tasks.workunit.client.0.smithi022.stdout: "unittest_2": { 2024-09-21T09:31:08.076 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.076 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.076 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.076 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.076 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.076 INFO:tasks.workunit.client.0.smithi022.stdout: "osd": { 2024-09-21T09:31:08.076 INFO:tasks.workunit.client.0.smithi022.stdout: "osd.0": { 2024-09-21T09:31:08.076 INFO:tasks.workunit.client.0.smithi022.stdout: "bloom_filter": { 2024-09-21T09:31:08.077 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.077 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.077 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.077 INFO:tasks.workunit.client.0.smithi022.stdout: "bluefs": { 2024-09-21T09:31:08.077 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 6304, 2024-09-21T09:31:08.077 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 115 2024-09-21T09:31:08.077 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.077 INFO:tasks.workunit.client.0.smithi022.stdout: "bluefs_file_reader": { 2024-09-21T09:31:08.077 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 16896, 2024-09-21T09:31:08.077 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 8 2024-09-21T09:31:08.077 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.077 INFO:tasks.workunit.client.0.smithi022.stdout: "bluefs_file_writer": { 2024-09-21T09:31:08.077 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 576, 2024-09-21T09:31:08.078 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 3 2024-09-21T09:31:08.078 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.078 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_Blob": { 2024-09-21T09:31:08.078 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 20272, 2024-09-21T09:31:08.078 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 181 2024-09-21T09:31:08.078 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.078 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_Buffer": { 2024-09-21T09:31:08.078 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 224, 2024-09-21T09:31:08.078 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 2 2024-09-21T09:31:08.078 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.078 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_Extent": { 2024-09-21T09:31:08.078 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 8688, 2024-09-21T09:31:08.078 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 181 2024-09-21T09:31:08.079 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.079 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_SharedBlob": { 2024-09-21T09:31:08.079 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 20272, 2024-09-21T09:31:08.079 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 181 2024-09-21T09:31:08.079 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.079 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_alloc": { 2024-09-21T09:31:08.079 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 5440, 2024-09-21T09:31:08.079 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 68 2024-09-21T09:31:08.079 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.079 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_cache_data": { 2024-09-21T09:31:08.079 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 8192, 2024-09-21T09:31:08.079 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 2 2024-09-21T09:31:08.079 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.079 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_cache_meta": { 2024-09-21T09:31:08.079 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 53743, 2024-09-21T09:31:08.080 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 9742 2024-09-21T09:31:08.080 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.080 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_cache_onode": { 2024-09-21T09:31:08.080 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 133280, 2024-09-21T09:31:08.080 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 196 2024-09-21T09:31:08.080 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.080 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_cache_other": { 2024-09-21T09:31:08.080 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 10952, 2024-09-21T09:31:08.080 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 706 2024-09-21T09:31:08.080 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.080 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_fsck": { 2024-09-21T09:31:08.080 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.080 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.080 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.081 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_inline_bl": { 2024-09-21T09:31:08.081 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 31156, 2024-09-21T09:31:08.081 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 196 2024-09-21T09:31:08.081 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.081 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_txc": { 2024-09-21T09:31:08.081 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.081 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.081 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.081 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_writing": { 2024-09-21T09:31:08.081 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 380503, 2024-09-21T09:31:08.081 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 82 2024-09-21T09:31:08.081 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.081 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_writing_deferred": { 2024-09-21T09:31:08.081 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.081 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.082 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.082 INFO:tasks.workunit.client.0.smithi022.stdout: "buffer_anon": { 2024-09-21T09:31:08.082 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 2442382, 2024-09-21T09:31:08.082 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 196 2024-09-21T09:31:08.082 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.082 INFO:tasks.workunit.client.0.smithi022.stdout: "buffer_meta": { 2024-09-21T09:31:08.082 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 5544, 2024-09-21T09:31:08.082 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 63 2024-09-21T09:31:08.082 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.082 INFO:tasks.workunit.client.0.smithi022.stdout: "mds_co": { 2024-09-21T09:31:08.082 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.082 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.082 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.082 INFO:tasks.workunit.client.0.smithi022.stdout: "osd": { 2024-09-21T09:31:08.083 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 509696, 2024-09-21T09:31:08.083 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 44 2024-09-21T09:31:08.083 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.083 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_mapbl": { 2024-09-21T09:31:08.083 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.083 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.083 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.083 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_pglog": { 2024-09-21T09:31:08.083 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 69784, 2024-09-21T09:31:08.083 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 139 2024-09-21T09:31:08.083 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.083 INFO:tasks.workunit.client.0.smithi022.stdout: "osdmap": { 2024-09-21T09:31:08.083 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 236944, 2024-09-21T09:31:08.083 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 3552 2024-09-21T09:31:08.083 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.084 INFO:tasks.workunit.client.0.smithi022.stdout: "osdmap_mapping": { 2024-09-21T09:31:08.084 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.084 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.084 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.084 INFO:tasks.workunit.client.0.smithi022.stdout: "pgmap": { 2024-09-21T09:31:08.084 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.084 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.084 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.084 INFO:tasks.workunit.client.0.smithi022.stdout: "unittest_1": { 2024-09-21T09:31:08.084 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.084 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.084 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.084 INFO:tasks.workunit.client.0.smithi022.stdout: "unittest_2": { 2024-09-21T09:31:08.085 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.085 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.085 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.085 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.085 INFO:tasks.workunit.client.0.smithi022.stdout: "osd.1": { 2024-09-21T09:31:08.085 INFO:tasks.workunit.client.0.smithi022.stdout: "bloom_filter": { 2024-09-21T09:31:08.085 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.085 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.085 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.085 INFO:tasks.workunit.client.0.smithi022.stdout: "bluefs": { 2024-09-21T09:31:08.085 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 6304, 2024-09-21T09:31:08.085 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 115 2024-09-21T09:31:08.085 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.086 INFO:tasks.workunit.client.0.smithi022.stdout: "bluefs_file_reader": { 2024-09-21T09:31:08.086 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 16896, 2024-09-21T09:31:08.086 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 8 2024-09-21T09:31:08.086 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.086 INFO:tasks.workunit.client.0.smithi022.stdout: "bluefs_file_writer": { 2024-09-21T09:31:08.086 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 576, 2024-09-21T09:31:08.086 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 3 2024-09-21T09:31:08.086 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.086 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_Blob": { 2024-09-21T09:31:08.086 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 19152, 2024-09-21T09:31:08.086 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 171 2024-09-21T09:31:08.086 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.086 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_Buffer": { 2024-09-21T09:31:08.086 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 784, 2024-09-21T09:31:08.087 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 7 2024-09-21T09:31:08.087 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.087 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_Extent": { 2024-09-21T09:31:08.087 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 8208, 2024-09-21T09:31:08.087 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 171 2024-09-21T09:31:08.087 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.087 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_SharedBlob": { 2024-09-21T09:31:08.087 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 19152, 2024-09-21T09:31:08.087 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 171 2024-09-21T09:31:08.087 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.087 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_alloc": { 2024-09-21T09:31:08.087 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 4880, 2024-09-21T09:31:08.087 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 61 2024-09-21T09:31:08.087 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.087 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_cache_data": { 2024-09-21T09:31:08.088 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 8192, 2024-09-21T09:31:08.088 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 2 2024-09-21T09:31:08.088 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.088 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_cache_meta": { 2024-09-21T09:31:08.088 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 48508, 2024-09-21T09:31:08.088 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 8876 2024-09-21T09:31:08.088 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.088 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_cache_onode": { 2024-09-21T09:31:08.088 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 125800, 2024-09-21T09:31:08.088 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 185 2024-09-21T09:31:08.088 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.088 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_cache_other": { 2024-09-21T09:31:08.088 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 9172, 2024-09-21T09:31:08.088 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 537 2024-09-21T09:31:08.088 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.089 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_fsck": { 2024-09-21T09:31:08.089 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.089 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.089 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.089 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_inline_bl": { 2024-09-21T09:31:08.089 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 27400, 2024-09-21T09:31:08.089 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 185 2024-09-21T09:31:08.089 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.089 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_txc": { 2024-09-21T09:31:08.089 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 736, 2024-09-21T09:31:08.089 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 1 2024-09-21T09:31:08.089 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.090 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_writing": { 2024-09-21T09:31:08.090 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 447438, 2024-09-21T09:31:08.090 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 98 2024-09-21T09:31:08.090 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.090 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_writing_deferred": { 2024-09-21T09:31:08.090 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 12096, 2024-09-21T09:31:08.090 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 3 2024-09-21T09:31:08.090 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.090 INFO:tasks.workunit.client.0.smithi022.stdout: "buffer_anon": { 2024-09-21T09:31:08.090 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 2403228, 2024-09-21T09:31:08.090 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 178 2024-09-21T09:31:08.090 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.091 INFO:tasks.workunit.client.0.smithi022.stdout: "buffer_meta": { 2024-09-21T09:31:08.091 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 4840, 2024-09-21T09:31:08.091 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 55 2024-09-21T09:31:08.091 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.091 INFO:tasks.workunit.client.0.smithi022.stdout: "mds_co": { 2024-09-21T09:31:08.091 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.091 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.091 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.091 INFO:tasks.workunit.client.0.smithi022.stdout: "osd": { 2024-09-21T09:31:08.091 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 440192, 2024-09-21T09:31:08.091 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 38 2024-09-21T09:31:08.091 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.091 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_mapbl": { 2024-09-21T09:31:08.091 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.092 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.092 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.092 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_pglog": { 2024-09-21T09:31:08.092 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 8528, 2024-09-21T09:31:08.092 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 22 2024-09-21T09:31:08.092 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.092 INFO:tasks.workunit.client.0.smithi022.stdout: "osdmap": { 2024-09-21T09:31:08.092 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 236176, 2024-09-21T09:31:08.092 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 3516 2024-09-21T09:31:08.092 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.092 INFO:tasks.workunit.client.0.smithi022.stdout: "osdmap_mapping": { 2024-09-21T09:31:08.092 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.092 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.092 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.092 INFO:tasks.workunit.client.0.smithi022.stdout: "pgmap": { 2024-09-21T09:31:08.093 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.093 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.093 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.093 INFO:tasks.workunit.client.0.smithi022.stdout: "unittest_1": { 2024-09-21T09:31:08.093 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.093 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.093 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.093 INFO:tasks.workunit.client.0.smithi022.stdout: "unittest_2": { 2024-09-21T09:31:08.093 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.093 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.093 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.093 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.093 INFO:tasks.workunit.client.0.smithi022.stdout: "osd.2": { 2024-09-21T09:31:08.093 INFO:tasks.workunit.client.0.smithi022.stdout: "bloom_filter": { 2024-09-21T09:31:08.093 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.094 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.094 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.094 INFO:tasks.workunit.client.0.smithi022.stdout: "bluefs": { 2024-09-21T09:31:08.094 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 6304, 2024-09-21T09:31:08.094 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 115 2024-09-21T09:31:08.094 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.094 INFO:tasks.workunit.client.0.smithi022.stdout: "bluefs_file_reader": { 2024-09-21T09:31:08.094 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 16896, 2024-09-21T09:31:08.094 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 8 2024-09-21T09:31:08.094 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.094 INFO:tasks.workunit.client.0.smithi022.stdout: "bluefs_file_writer": { 2024-09-21T09:31:08.094 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 576, 2024-09-21T09:31:08.094 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 3 2024-09-21T09:31:08.095 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.095 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_Blob": { 2024-09-21T09:31:08.095 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 18816, 2024-09-21T09:31:08.095 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 168 2024-09-21T09:31:08.095 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.095 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_Buffer": { 2024-09-21T09:31:08.095 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 784, 2024-09-21T09:31:08.095 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 7 2024-09-21T09:31:08.095 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.095 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_Extent": { 2024-09-21T09:31:08.095 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 8064, 2024-09-21T09:31:08.095 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 168 2024-09-21T09:31:08.095 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.096 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_SharedBlob": { 2024-09-21T09:31:08.096 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 18816, 2024-09-21T09:31:08.096 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 168 2024-09-21T09:31:08.096 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.096 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_alloc": { 2024-09-21T09:31:08.096 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 4320, 2024-09-21T09:31:08.096 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 54 2024-09-21T09:31:08.096 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.096 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_cache_data": { 2024-09-21T09:31:08.096 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 8192, 2024-09-21T09:31:08.096 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 2 2024-09-21T09:31:08.096 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.096 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_cache_meta": { 2024-09-21T09:31:08.096 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 45210, 2024-09-21T09:31:08.097 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 8397 2024-09-21T09:31:08.097 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.097 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_cache_onode": { 2024-09-21T09:31:08.097 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 119000, 2024-09-21T09:31:08.097 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 175 2024-09-21T09:31:08.097 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.097 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_cache_other": { 2024-09-21T09:31:08.097 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 8216, 2024-09-21T09:31:08.097 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 523 2024-09-21T09:31:08.097 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.097 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_fsck": { 2024-09-21T09:31:08.097 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.097 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.097 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.098 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_inline_bl": { 2024-09-21T09:31:08.098 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 25344, 2024-09-21T09:31:08.098 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 175 2024-09-21T09:31:08.098 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.098 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_txc": { 2024-09-21T09:31:08.098 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 736, 2024-09-21T09:31:08.098 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 1 2024-09-21T09:31:08.098 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.098 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_writing": { 2024-09-21T09:31:08.098 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 369614, 2024-09-21T09:31:08.098 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 79 2024-09-21T09:31:08.098 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.098 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_writing_deferred": { 2024-09-21T09:31:08.098 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 12096, 2024-09-21T09:31:08.099 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 3 2024-09-21T09:31:08.099 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.099 INFO:tasks.workunit.client.0.smithi022.stdout: "buffer_anon": { 2024-09-21T09:31:08.099 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 2414434, 2024-09-21T09:31:08.099 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 179 2024-09-21T09:31:08.099 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.099 INFO:tasks.workunit.client.0.smithi022.stdout: "buffer_meta": { 2024-09-21T09:31:08.099 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 4224, 2024-09-21T09:31:08.099 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 48 2024-09-21T09:31:08.099 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.099 INFO:tasks.workunit.client.0.smithi022.stdout: "mds_co": { 2024-09-21T09:31:08.099 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.099 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.099 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.100 INFO:tasks.workunit.client.0.smithi022.stdout: "osd": { 2024-09-21T09:31:08.100 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 347520, 2024-09-21T09:31:08.100 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 30 2024-09-21T09:31:08.100 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.100 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_mapbl": { 2024-09-21T09:31:08.100 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.100 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.100 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.100 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_pglog": { 2024-09-21T09:31:08.100 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 8320, 2024-09-21T09:31:08.100 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 24 2024-09-21T09:31:08.100 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.100 INFO:tasks.workunit.client.0.smithi022.stdout: "osdmap": { 2024-09-21T09:31:08.100 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 236688, 2024-09-21T09:31:08.100 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 3540 2024-09-21T09:31:08.101 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.101 INFO:tasks.workunit.client.0.smithi022.stdout: "osdmap_mapping": { 2024-09-21T09:31:08.101 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.101 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.101 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.101 INFO:tasks.workunit.client.0.smithi022.stdout: "pgmap": { 2024-09-21T09:31:08.101 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.101 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.101 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.101 INFO:tasks.workunit.client.0.smithi022.stdout: "unittest_1": { 2024-09-21T09:31:08.101 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.101 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.101 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.101 INFO:tasks.workunit.client.0.smithi022.stdout: "unittest_2": { 2024-09-21T09:31:08.102 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.102 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.102 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.102 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.102 INFO:tasks.workunit.client.0.smithi022.stdout: "osd.3": { 2024-09-21T09:31:08.102 INFO:tasks.workunit.client.0.smithi022.stdout: "bloom_filter": { 2024-09-21T09:31:08.102 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.102 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.102 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.102 INFO:tasks.workunit.client.0.smithi022.stdout: "bluefs": { 2024-09-21T09:31:08.102 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 6304, 2024-09-21T09:31:08.102 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 115 2024-09-21T09:31:08.102 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.103 INFO:tasks.workunit.client.0.smithi022.stdout: "bluefs_file_reader": { 2024-09-21T09:31:08.103 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 16896, 2024-09-21T09:31:08.103 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 8 2024-09-21T09:31:08.103 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.103 INFO:tasks.workunit.client.0.smithi022.stdout: "bluefs_file_writer": { 2024-09-21T09:31:08.103 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 576, 2024-09-21T09:31:08.103 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 3 2024-09-21T09:31:08.103 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.103 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_Blob": { 2024-09-21T09:31:08.103 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 19376, 2024-09-21T09:31:08.103 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 173 2024-09-21T09:31:08.103 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.103 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_Buffer": { 2024-09-21T09:31:08.103 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 336, 2024-09-21T09:31:08.104 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 3 2024-09-21T09:31:08.104 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.104 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_Extent": { 2024-09-21T09:31:08.104 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 8304, 2024-09-21T09:31:08.104 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 173 2024-09-21T09:31:08.104 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.104 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_SharedBlob": { 2024-09-21T09:31:08.104 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 19376, 2024-09-21T09:31:08.104 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 173 2024-09-21T09:31:08.104 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.104 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_alloc": { 2024-09-21T09:31:08.104 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 3920, 2024-09-21T09:31:08.104 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 49 2024-09-21T09:31:08.104 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.105 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_cache_data": { 2024-09-21T09:31:08.105 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 4096, 2024-09-21T09:31:08.105 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 1 2024-09-21T09:31:08.105 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.105 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_cache_meta": { 2024-09-21T09:31:08.105 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 47228, 2024-09-21T09:31:08.105 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 8774 2024-09-21T09:31:08.105 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.105 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_cache_onode": { 2024-09-21T09:31:08.105 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 124440, 2024-09-21T09:31:08.105 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 183 2024-09-21T09:31:08.105 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.105 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_cache_other": { 2024-09-21T09:31:08.105 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 9132, 2024-09-21T09:31:08.106 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 542 2024-09-21T09:31:08.106 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.106 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_fsck": { 2024-09-21T09:31:08.106 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.106 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.106 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.106 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_inline_bl": { 2024-09-21T09:31:08.106 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 26552, 2024-09-21T09:31:08.106 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 183 2024-09-21T09:31:08.106 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.106 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_txc": { 2024-09-21T09:31:08.106 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 2208, 2024-09-21T09:31:08.106 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 3 2024-09-21T09:31:08.106 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.106 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_writing": { 2024-09-21T09:31:08.107 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 283359, 2024-09-21T09:31:08.107 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 58 2024-09-21T09:31:08.107 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.107 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_writing_deferred": { 2024-09-21T09:31:08.107 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 30637, 2024-09-21T09:31:08.107 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 9 2024-09-21T09:31:08.107 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.107 INFO:tasks.workunit.client.0.smithi022.stdout: "buffer_anon": { 2024-09-21T09:31:08.107 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 2445344, 2024-09-21T09:31:08.107 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 189 2024-09-21T09:31:08.107 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.107 INFO:tasks.workunit.client.0.smithi022.stdout: "buffer_meta": { 2024-09-21T09:31:08.107 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 4840, 2024-09-21T09:31:08.108 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 55 2024-09-21T09:31:08.108 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.108 INFO:tasks.workunit.client.0.smithi022.stdout: "mds_co": { 2024-09-21T09:31:08.108 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.108 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.108 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.108 INFO:tasks.workunit.client.0.smithi022.stdout: "osd": { 2024-09-21T09:31:08.108 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 428608, 2024-09-21T09:31:08.108 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 37 2024-09-21T09:31:08.108 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.108 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_mapbl": { 2024-09-21T09:31:08.108 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.108 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.108 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.109 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_pglog": { 2024-09-21T09:31:08.109 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 10400, 2024-09-21T09:31:08.109 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 20 2024-09-21T09:31:08.109 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.109 INFO:tasks.workunit.client.0.smithi022.stdout: "osdmap": { 2024-09-21T09:31:08.109 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 236176, 2024-09-21T09:31:08.109 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 3516 2024-09-21T09:31:08.109 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.109 INFO:tasks.workunit.client.0.smithi022.stdout: "osdmap_mapping": { 2024-09-21T09:31:08.109 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.109 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.109 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.109 INFO:tasks.workunit.client.0.smithi022.stdout: "pgmap": { 2024-09-21T09:31:08.110 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.110 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.110 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.110 INFO:tasks.workunit.client.0.smithi022.stdout: "unittest_1": { 2024-09-21T09:31:08.110 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.110 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.110 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.110 INFO:tasks.workunit.client.0.smithi022.stdout: "unittest_2": { 2024-09-21T09:31:08.110 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.110 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.110 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.110 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.110 INFO:tasks.workunit.client.0.smithi022.stdout: "osd.4": { 2024-09-21T09:31:08.110 INFO:tasks.workunit.client.0.smithi022.stdout: "bloom_filter": { 2024-09-21T09:31:08.111 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.111 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.111 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.111 INFO:tasks.workunit.client.0.smithi022.stdout: "bluefs": { 2024-09-21T09:31:08.111 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 6304, 2024-09-21T09:31:08.111 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 115 2024-09-21T09:31:08.111 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.111 INFO:tasks.workunit.client.0.smithi022.stdout: "bluefs_file_reader": { 2024-09-21T09:31:08.111 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 16896, 2024-09-21T09:31:08.111 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 8 2024-09-21T09:31:08.111 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.111 INFO:tasks.workunit.client.0.smithi022.stdout: "bluefs_file_writer": { 2024-09-21T09:31:08.111 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 576, 2024-09-21T09:31:08.111 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 3 2024-09-21T09:31:08.112 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.112 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_Blob": { 2024-09-21T09:31:08.112 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 19376, 2024-09-21T09:31:08.112 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 173 2024-09-21T09:31:08.112 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.112 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_Buffer": { 2024-09-21T09:31:08.112 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 784, 2024-09-21T09:31:08.112 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 7 2024-09-21T09:31:08.112 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.112 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_Extent": { 2024-09-21T09:31:08.112 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 8304, 2024-09-21T09:31:08.112 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 173 2024-09-21T09:31:08.112 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.112 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_SharedBlob": { 2024-09-21T09:31:08.113 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 19376, 2024-09-21T09:31:08.113 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 173 2024-09-21T09:31:08.113 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.113 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_alloc": { 2024-09-21T09:31:08.113 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 3440, 2024-09-21T09:31:08.113 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 43 2024-09-21T09:31:08.113 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.113 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_cache_data": { 2024-09-21T09:31:08.113 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 15086, 2024-09-21T09:31:08.113 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 3 2024-09-21T09:31:08.113 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.113 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_cache_meta": { 2024-09-21T09:31:08.113 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 48736, 2024-09-21T09:31:08.113 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 8972 2024-09-21T09:31:08.114 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.114 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_cache_onode": { 2024-09-21T09:31:08.114 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 127160, 2024-09-21T09:31:08.114 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 187 2024-09-21T09:31:08.114 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.114 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_cache_other": { 2024-09-21T09:31:08.114 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 9436, 2024-09-21T09:31:08.114 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 543 2024-09-21T09:31:08.114 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.114 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_fsck": { 2024-09-21T09:31:08.114 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.114 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.114 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.114 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_inline_bl": { 2024-09-21T09:31:08.114 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 27576, 2024-09-21T09:31:08.115 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 187 2024-09-21T09:31:08.115 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.115 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_txc": { 2024-09-21T09:31:08.115 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 736, 2024-09-21T09:31:08.115 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 1 2024-09-21T09:31:08.115 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.115 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_writing": { 2024-09-21T09:31:08.115 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 898068, 2024-09-21T09:31:08.115 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 66 2024-09-21T09:31:08.115 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.115 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_writing_deferred": { 2024-09-21T09:31:08.115 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 12096, 2024-09-21T09:31:08.115 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 3 2024-09-21T09:31:08.115 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.116 INFO:tasks.workunit.client.0.smithi022.stdout: "buffer_anon": { 2024-09-21T09:31:08.116 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 2405354, 2024-09-21T09:31:08.116 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 169 2024-09-21T09:31:08.116 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.116 INFO:tasks.workunit.client.0.smithi022.stdout: "buffer_meta": { 2024-09-21T09:31:08.116 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 5016, 2024-09-21T09:31:08.116 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 57 2024-09-21T09:31:08.116 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.116 INFO:tasks.workunit.client.0.smithi022.stdout: "mds_co": { 2024-09-21T09:31:08.116 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.116 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.116 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.116 INFO:tasks.workunit.client.0.smithi022.stdout: "osd": { 2024-09-21T09:31:08.117 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 463360, 2024-09-21T09:31:08.117 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 40 2024-09-21T09:31:08.117 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.117 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_mapbl": { 2024-09-21T09:31:08.117 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.117 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.117 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.117 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_pglog": { 2024-09-21T09:31:08.117 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 8944, 2024-09-21T09:31:08.117 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 18 2024-09-21T09:31:08.117 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.117 INFO:tasks.workunit.client.0.smithi022.stdout: "osdmap": { 2024-09-21T09:31:08.117 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 236432, 2024-09-21T09:31:08.117 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 3528 2024-09-21T09:31:08.118 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.118 INFO:tasks.workunit.client.0.smithi022.stdout: "osdmap_mapping": { 2024-09-21T09:31:08.118 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.118 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.118 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.118 INFO:tasks.workunit.client.0.smithi022.stdout: "pgmap": { 2024-09-21T09:31:08.118 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.118 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.118 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.118 INFO:tasks.workunit.client.0.smithi022.stdout: "unittest_1": { 2024-09-21T09:31:08.118 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.118 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.118 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.118 INFO:tasks.workunit.client.0.smithi022.stdout: "unittest_2": { 2024-09-21T09:31:08.119 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.119 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.119 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.119 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.119 INFO:tasks.workunit.client.0.smithi022.stdout: "osd.5": { 2024-09-21T09:31:08.119 INFO:tasks.workunit.client.0.smithi022.stdout: "bloom_filter": { 2024-09-21T09:31:08.119 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.119 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.119 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.119 INFO:tasks.workunit.client.0.smithi022.stdout: "bluefs": { 2024-09-21T09:31:08.119 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 6304, 2024-09-21T09:31:08.119 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 115 2024-09-21T09:31:08.119 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.119 INFO:tasks.workunit.client.0.smithi022.stdout: "bluefs_file_reader": { 2024-09-21T09:31:08.120 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 16896, 2024-09-21T09:31:08.120 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 8 2024-09-21T09:31:08.120 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.120 INFO:tasks.workunit.client.0.smithi022.stdout: "bluefs_file_writer": { 2024-09-21T09:31:08.120 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 576, 2024-09-21T09:31:08.120 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 3 2024-09-21T09:31:08.120 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.120 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_Blob": { 2024-09-21T09:31:08.120 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 19152, 2024-09-21T09:31:08.120 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 171 2024-09-21T09:31:08.120 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.120 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_Buffer": { 2024-09-21T09:31:08.120 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 224, 2024-09-21T09:31:08.120 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 2 2024-09-21T09:31:08.120 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.121 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_Extent": { 2024-09-21T09:31:08.121 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 8208, 2024-09-21T09:31:08.121 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 171 2024-09-21T09:31:08.121 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.121 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_SharedBlob": { 2024-09-21T09:31:08.121 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 19152, 2024-09-21T09:31:08.121 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 171 2024-09-21T09:31:08.121 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.121 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_alloc": { 2024-09-21T09:31:08.121 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 3040, 2024-09-21T09:31:08.121 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 38 2024-09-21T09:31:08.121 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.121 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_cache_data": { 2024-09-21T09:31:08.121 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.121 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.122 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.122 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_cache_meta": { 2024-09-21T09:31:08.122 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 49086, 2024-09-21T09:31:08.122 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 8963 2024-09-21T09:31:08.122 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.122 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_cache_onode": { 2024-09-21T09:31:08.122 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 127160, 2024-09-21T09:31:08.122 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 187 2024-09-21T09:31:08.122 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.122 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_cache_other": { 2024-09-21T09:31:08.122 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 9540, 2024-09-21T09:31:08.122 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 542 2024-09-21T09:31:08.122 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.123 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_fsck": { 2024-09-21T09:31:08.123 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.123 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.123 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.123 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_inline_bl": { 2024-09-21T09:31:08.123 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 27912, 2024-09-21T09:31:08.123 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 187 2024-09-21T09:31:08.123 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.123 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_txc": { 2024-09-21T09:31:08.123 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 2208, 2024-09-21T09:31:08.123 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 3 2024-09-21T09:31:08.123 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.123 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_writing": { 2024-09-21T09:31:08.123 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 830116, 2024-09-21T09:31:08.123 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 49 2024-09-21T09:31:08.124 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.124 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_writing_deferred": { 2024-09-21T09:31:08.124 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 30637, 2024-09-21T09:31:08.124 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 9 2024-09-21T09:31:08.124 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.124 INFO:tasks.workunit.client.0.smithi022.stdout: "buffer_anon": { 2024-09-21T09:31:08.124 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 2394760, 2024-09-21T09:31:08.124 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 164 2024-09-21T09:31:08.124 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.124 INFO:tasks.workunit.client.0.smithi022.stdout: "buffer_meta": { 2024-09-21T09:31:08.124 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 5456, 2024-09-21T09:31:08.124 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 62 2024-09-21T09:31:08.124 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.125 INFO:tasks.workunit.client.0.smithi022.stdout: "mds_co": { 2024-09-21T09:31:08.125 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.125 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.125 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.125 INFO:tasks.workunit.client.0.smithi022.stdout: "osd": { 2024-09-21T09:31:08.125 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 474944, 2024-09-21T09:31:08.125 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 41 2024-09-21T09:31:08.125 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.125 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_mapbl": { 2024-09-21T09:31:08.125 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.125 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.125 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.125 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_pglog": { 2024-09-21T09:31:08.125 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 13000, 2024-09-21T09:31:08.126 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 25 2024-09-21T09:31:08.126 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.126 INFO:tasks.workunit.client.0.smithi022.stdout: "osdmap": { 2024-09-21T09:31:08.126 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 236944, 2024-09-21T09:31:08.126 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 3552 2024-09-21T09:31:08.126 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.126 INFO:tasks.workunit.client.0.smithi022.stdout: "osdmap_mapping": { 2024-09-21T09:31:08.126 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.126 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.126 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.126 INFO:tasks.workunit.client.0.smithi022.stdout: "pgmap": { 2024-09-21T09:31:08.126 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.126 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.126 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.127 INFO:tasks.workunit.client.0.smithi022.stdout: "unittest_1": { 2024-09-21T09:31:08.127 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.127 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.127 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.127 INFO:tasks.workunit.client.0.smithi022.stdout: "unittest_2": { 2024-09-21T09:31:08.127 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.127 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.127 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.127 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.127 INFO:tasks.workunit.client.0.smithi022.stdout: "osd.6": { 2024-09-21T09:31:08.127 INFO:tasks.workunit.client.0.smithi022.stdout: "bloom_filter": { 2024-09-21T09:31:08.127 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.127 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.127 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.128 INFO:tasks.workunit.client.0.smithi022.stdout: "bluefs": { 2024-09-21T09:31:08.128 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 6304, 2024-09-21T09:31:08.128 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 115 2024-09-21T09:31:08.128 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.128 INFO:tasks.workunit.client.0.smithi022.stdout: "bluefs_file_reader": { 2024-09-21T09:31:08.128 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 16896, 2024-09-21T09:31:08.128 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 8 2024-09-21T09:31:08.128 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.128 INFO:tasks.workunit.client.0.smithi022.stdout: "bluefs_file_writer": { 2024-09-21T09:31:08.128 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 576, 2024-09-21T09:31:08.128 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 3 2024-09-21T09:31:08.128 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.128 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_Blob": { 2024-09-21T09:31:08.128 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 20160, 2024-09-21T09:31:08.128 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 180 2024-09-21T09:31:08.129 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.129 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_Buffer": { 2024-09-21T09:31:08.129 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 224, 2024-09-21T09:31:08.129 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 2 2024-09-21T09:31:08.129 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.129 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_Extent": { 2024-09-21T09:31:08.129 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 8640, 2024-09-21T09:31:08.129 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 180 2024-09-21T09:31:08.129 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.129 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_SharedBlob": { 2024-09-21T09:31:08.129 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 20160, 2024-09-21T09:31:08.129 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 180 2024-09-21T09:31:08.129 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.129 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_alloc": { 2024-09-21T09:31:08.130 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 2720, 2024-09-21T09:31:08.130 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 34 2024-09-21T09:31:08.130 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.130 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_cache_data": { 2024-09-21T09:31:08.130 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 15086, 2024-09-21T09:31:08.130 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 3 2024-09-21T09:31:08.130 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.130 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_cache_meta": { 2024-09-21T09:31:08.130 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 48793, 2024-09-21T09:31:08.130 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 9206 2024-09-21T09:31:08.130 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.130 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_cache_onode": { 2024-09-21T09:31:08.130 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 125800, 2024-09-21T09:31:08.130 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 185 2024-09-21T09:31:08.131 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.131 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_cache_other": { 2024-09-21T09:31:08.131 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 10564, 2024-09-21T09:31:08.131 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 699 2024-09-21T09:31:08.131 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.131 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_fsck": { 2024-09-21T09:31:08.131 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.131 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.131 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.131 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_inline_bl": { 2024-09-21T09:31:08.131 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 28428, 2024-09-21T09:31:08.131 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 185 2024-09-21T09:31:08.131 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.131 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_txc": { 2024-09-21T09:31:08.132 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.132 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.132 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.132 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_writing": { 2024-09-21T09:31:08.132 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 214192, 2024-09-21T09:31:08.132 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 42 2024-09-21T09:31:08.132 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.132 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_writing_deferred": { 2024-09-21T09:31:08.132 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.132 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.132 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.132 INFO:tasks.workunit.client.0.smithi022.stdout: "buffer_anon": { 2024-09-21T09:31:08.132 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 2400796, 2024-09-21T09:31:08.132 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 167 2024-09-21T09:31:08.132 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.133 INFO:tasks.workunit.client.0.smithi022.stdout: "buffer_meta": { 2024-09-21T09:31:08.133 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 4664, 2024-09-21T09:31:08.133 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 53 2024-09-21T09:31:08.133 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.133 INFO:tasks.workunit.client.0.smithi022.stdout: "mds_co": { 2024-09-21T09:31:08.133 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.133 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.133 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.133 INFO:tasks.workunit.client.0.smithi022.stdout: "osd": { 2024-09-21T09:31:08.133 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 451776, 2024-09-21T09:31:08.133 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 39 2024-09-21T09:31:08.133 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.133 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_mapbl": { 2024-09-21T09:31:08.133 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.134 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.134 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.134 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_pglog": { 2024-09-21T09:31:08.134 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 68952, 2024-09-21T09:31:08.134 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 143 2024-09-21T09:31:08.134 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.134 INFO:tasks.workunit.client.0.smithi022.stdout: "osdmap": { 2024-09-21T09:31:08.134 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 236944, 2024-09-21T09:31:08.134 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 3552 2024-09-21T09:31:08.134 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.134 INFO:tasks.workunit.client.0.smithi022.stdout: "osdmap_mapping": { 2024-09-21T09:31:08.134 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.134 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.134 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.135 INFO:tasks.workunit.client.0.smithi022.stdout: "pgmap": { 2024-09-21T09:31:08.135 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.135 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.135 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.135 INFO:tasks.workunit.client.0.smithi022.stdout: "unittest_1": { 2024-09-21T09:31:08.135 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.135 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.135 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.135 INFO:tasks.workunit.client.0.smithi022.stdout: "unittest_2": { 2024-09-21T09:31:08.135 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.135 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.135 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.135 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.135 INFO:tasks.workunit.client.0.smithi022.stdout: "osd.7": { 2024-09-21T09:31:08.136 INFO:tasks.workunit.client.0.smithi022.stdout: "bloom_filter": { 2024-09-21T09:31:08.136 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.136 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.136 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.136 INFO:tasks.workunit.client.0.smithi022.stdout: "bluefs": { 2024-09-21T09:31:08.136 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 6304, 2024-09-21T09:31:08.136 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 115 2024-09-21T09:31:08.136 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.136 INFO:tasks.workunit.client.0.smithi022.stdout: "bluefs_file_reader": { 2024-09-21T09:31:08.136 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 16896, 2024-09-21T09:31:08.136 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 8 2024-09-21T09:31:08.136 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.136 INFO:tasks.workunit.client.0.smithi022.stdout: "bluefs_file_writer": { 2024-09-21T09:31:08.136 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 576, 2024-09-21T09:31:08.137 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 3 2024-09-21T09:31:08.137 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.137 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_Blob": { 2024-09-21T09:31:08.137 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 20160, 2024-09-21T09:31:08.137 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 180 2024-09-21T09:31:08.137 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.137 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_Buffer": { 2024-09-21T09:31:08.137 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 1232, 2024-09-21T09:31:08.137 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 11 2024-09-21T09:31:08.137 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.137 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_Extent": { 2024-09-21T09:31:08.137 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 8640, 2024-09-21T09:31:08.137 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 180 2024-09-21T09:31:08.137 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.137 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_SharedBlob": { 2024-09-21T09:31:08.137 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 20160, 2024-09-21T09:31:08.138 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 180 2024-09-21T09:31:08.138 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.138 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_alloc": { 2024-09-21T09:31:08.138 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 2240, 2024-09-21T09:31:08.138 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 28 2024-09-21T09:31:08.138 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.138 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_cache_data": { 2024-09-21T09:31:08.138 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 286720, 2024-09-21T09:31:08.138 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 11 2024-09-21T09:31:08.138 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.138 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_cache_meta": { 2024-09-21T09:31:08.138 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 52758, 2024-09-21T09:31:08.138 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 9697 2024-09-21T09:31:08.138 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.139 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_cache_onode": { 2024-09-21T09:31:08.139 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 132600, 2024-09-21T09:31:08.139 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 195 2024-09-21T09:31:08.139 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.139 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_cache_other": { 2024-09-21T09:31:08.139 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 11348, 2024-09-21T09:31:08.139 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 706 2024-09-21T09:31:08.139 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.139 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_fsck": { 2024-09-21T09:31:08.139 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.139 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.139 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.139 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_inline_bl": { 2024-09-21T09:31:08.139 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 30988, 2024-09-21T09:31:08.139 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 195 2024-09-21T09:31:08.140 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.140 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_txc": { 2024-09-21T09:31:08.140 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.140 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.140 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.140 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_writing": { 2024-09-21T09:31:08.140 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 184048, 2024-09-21T09:31:08.140 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 35 2024-09-21T09:31:08.140 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.140 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore_writing_deferred": { 2024-09-21T09:31:08.140 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.140 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.140 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.140 INFO:tasks.workunit.client.0.smithi022.stdout: "buffer_anon": { 2024-09-21T09:31:08.140 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 2428044, 2024-09-21T09:31:08.141 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 221 2024-09-21T09:31:08.141 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.141 INFO:tasks.workunit.client.0.smithi022.stdout: "buffer_meta": { 2024-09-21T09:31:08.141 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 6336, 2024-09-21T09:31:08.141 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 72 2024-09-21T09:31:08.141 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.141 INFO:tasks.workunit.client.0.smithi022.stdout: "mds_co": { 2024-09-21T09:31:08.141 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.141 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.141 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.141 INFO:tasks.workunit.client.0.smithi022.stdout: "osd": { 2024-09-21T09:31:08.141 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 532864, 2024-09-21T09:31:08.141 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 46 2024-09-21T09:31:08.142 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.142 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_mapbl": { 2024-09-21T09:31:08.142 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.142 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.142 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.142 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_pglog": { 2024-09-21T09:31:08.142 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 71552, 2024-09-21T09:31:08.142 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 172 2024-09-21T09:31:08.142 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.142 INFO:tasks.workunit.client.0.smithi022.stdout: "osdmap": { 2024-09-21T09:31:08.142 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 236176, 2024-09-21T09:31:08.142 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 3516 2024-09-21T09:31:08.142 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.142 INFO:tasks.workunit.client.0.smithi022.stdout: "osdmap_mapping": { 2024-09-21T09:31:08.143 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.143 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.143 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.143 INFO:tasks.workunit.client.0.smithi022.stdout: "pgmap": { 2024-09-21T09:31:08.143 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.143 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.143 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.143 INFO:tasks.workunit.client.0.smithi022.stdout: "unittest_1": { 2024-09-21T09:31:08.143 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.143 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.143 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.143 INFO:tasks.workunit.client.0.smithi022.stdout: "unittest_2": { 2024-09-21T09:31:08.143 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes": 0, 2024-09-21T09:31:08.143 INFO:tasks.workunit.client.0.smithi022.stdout: "items": 0 2024-09-21T09:31:08.143 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.144 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.144 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.144 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.144 INFO:tasks.workunit.client.0.smithi022.stdout: "metadata": { 2024-09-21T09:31:08.144 INFO:tasks.workunit.client.0.smithi022.stdout: "mds": { 2024-09-21T09:31:08.144 INFO:tasks.workunit.client.0.smithi022.stdout: "arch": { 2024-09-21T09:31:08.144 INFO:tasks.workunit.client.0.smithi022.stdout: "x86_64": 2 2024-09-21T09:31:08.144 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.144 INFO:tasks.workunit.client.0.smithi022.stdout: "ceph_version": { 2024-09-21T09:31:08.144 INFO:tasks.workunit.client.0.smithi022.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 2 2024-09-21T09:31:08.144 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.144 INFO:tasks.workunit.client.0.smithi022.stdout: "cpu": { 2024-09-21T09:31:08.144 INFO:tasks.workunit.client.0.smithi022.stdout: "Intel(R) Xeon(R) CPU E5-1620 v3 @ 3.50GHz": 2 2024-09-21T09:31:08.144 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.144 INFO:tasks.workunit.client.0.smithi022.stdout: "distro": { 2024-09-21T09:31:08.145 INFO:tasks.workunit.client.0.smithi022.stdout: "centos": 2 2024-09-21T09:31:08.145 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.145 INFO:tasks.workunit.client.0.smithi022.stdout: "distro_description": { 2024-09-21T09:31:08.145 INFO:tasks.workunit.client.0.smithi022.stdout: "CentOS Stream 9": 2 2024-09-21T09:31:08.145 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.145 INFO:tasks.workunit.client.0.smithi022.stdout: "kernel_description": { 2024-09-21T09:31:08.145 INFO:tasks.workunit.client.0.smithi022.stdout: "#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024": 2 2024-09-21T09:31:08.145 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.145 INFO:tasks.workunit.client.0.smithi022.stdout: "kernel_version": { 2024-09-21T09:31:08.145 INFO:tasks.workunit.client.0.smithi022.stdout: "5.14.0-508.el9.x86_64": 2 2024-09-21T09:31:08.145 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.145 INFO:tasks.workunit.client.0.smithi022.stdout: "os": { 2024-09-21T09:31:08.145 INFO:tasks.workunit.client.0.smithi022.stdout: "Linux": 2 2024-09-21T09:31:08.146 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.146 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.146 INFO:tasks.workunit.client.0.smithi022.stdout: "mon": { 2024-09-21T09:31:08.146 INFO:tasks.workunit.client.0.smithi022.stdout: "arch": { 2024-09-21T09:31:08.146 INFO:tasks.workunit.client.0.smithi022.stdout: "x86_64": 3 2024-09-21T09:31:08.146 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.146 INFO:tasks.workunit.client.0.smithi022.stdout: "ceph_version": { 2024-09-21T09:31:08.146 INFO:tasks.workunit.client.0.smithi022.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 3 2024-09-21T09:31:08.146 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.146 INFO:tasks.workunit.client.0.smithi022.stdout: "cpu": { 2024-09-21T09:31:08.146 INFO:tasks.workunit.client.0.smithi022.stdout: "Intel(R) Xeon(R) CPU E5-1620 v3 @ 3.50GHz": 3 2024-09-21T09:31:08.146 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.146 INFO:tasks.workunit.client.0.smithi022.stdout: "distro": { 2024-09-21T09:31:08.146 INFO:tasks.workunit.client.0.smithi022.stdout: "centos": 3 2024-09-21T09:31:08.147 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.147 INFO:tasks.workunit.client.0.smithi022.stdout: "distro_description": { 2024-09-21T09:31:08.147 INFO:tasks.workunit.client.0.smithi022.stdout: "CentOS Stream 9": 3 2024-09-21T09:31:08.147 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.147 INFO:tasks.workunit.client.0.smithi022.stdout: "kernel_description": { 2024-09-21T09:31:08.147 INFO:tasks.workunit.client.0.smithi022.stdout: "#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024": 3 2024-09-21T09:31:08.147 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.147 INFO:tasks.workunit.client.0.smithi022.stdout: "kernel_version": { 2024-09-21T09:31:08.147 INFO:tasks.workunit.client.0.smithi022.stdout: "5.14.0-508.el9.x86_64": 3 2024-09-21T09:31:08.147 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.147 INFO:tasks.workunit.client.0.smithi022.stdout: "os": { 2024-09-21T09:31:08.147 INFO:tasks.workunit.client.0.smithi022.stdout: "Linux": 3 2024-09-21T09:31:08.147 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.147 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.148 INFO:tasks.workunit.client.0.smithi022.stdout: "osd": { 2024-09-21T09:31:08.148 INFO:tasks.workunit.client.0.smithi022.stdout: "arch": { 2024-09-21T09:31:08.148 INFO:tasks.workunit.client.0.smithi022.stdout: "x86_64": 8 2024-09-21T09:31:08.148 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.148 INFO:tasks.workunit.client.0.smithi022.stdout: "ceph_version": { 2024-09-21T09:31:08.148 INFO:tasks.workunit.client.0.smithi022.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 8 2024-09-21T09:31:08.148 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.148 INFO:tasks.workunit.client.0.smithi022.stdout: "cpu": { 2024-09-21T09:31:08.148 INFO:tasks.workunit.client.0.smithi022.stdout: "Intel(R) Xeon(R) CPU E5-1620 v3 @ 3.50GHz": 8 2024-09-21T09:31:08.148 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.148 INFO:tasks.workunit.client.0.smithi022.stdout: "distro": { 2024-09-21T09:31:08.148 INFO:tasks.workunit.client.0.smithi022.stdout: "centos": 8 2024-09-21T09:31:08.148 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.149 INFO:tasks.workunit.client.0.smithi022.stdout: "distro_description": { 2024-09-21T09:31:08.149 INFO:tasks.workunit.client.0.smithi022.stdout: "CentOS Stream 9": 8 2024-09-21T09:31:08.149 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.149 INFO:tasks.workunit.client.0.smithi022.stdout: "kernel_description": { 2024-09-21T09:31:08.149 INFO:tasks.workunit.client.0.smithi022.stdout: "#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024": 8 2024-09-21T09:31:08.149 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.149 INFO:tasks.workunit.client.0.smithi022.stdout: "kernel_version": { 2024-09-21T09:31:08.149 INFO:tasks.workunit.client.0.smithi022.stdout: "5.14.0-508.el9.x86_64": 8 2024-09-21T09:31:08.149 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.149 INFO:tasks.workunit.client.0.smithi022.stdout: "os": { 2024-09-21T09:31:08.149 INFO:tasks.workunit.client.0.smithi022.stdout: "Linux": 8 2024-09-21T09:31:08.149 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.149 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_objectstore": { 2024-09-21T09:31:08.149 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore": 8 2024-09-21T09:31:08.150 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.150 INFO:tasks.workunit.client.0.smithi022.stdout: "rotational": { 2024-09-21T09:31:08.150 INFO:tasks.workunit.client.0.smithi022.stdout: "0": 8 2024-09-21T09:31:08.150 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.150 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.150 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.150 INFO:tasks.workunit.client.0.smithi022.stdout: "mon": { 2024-09-21T09:31:08.150 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 3, 2024-09-21T09:31:08.150 INFO:tasks.workunit.client.0.smithi022.stdout: "features": { 2024-09-21T09:31:08.150 INFO:tasks.workunit.client.0.smithi022.stdout: "optional": [], 2024-09-21T09:31:08.150 INFO:tasks.workunit.client.0.smithi022.stdout: "persistent": [ 2024-09-21T09:31:08.150 INFO:tasks.workunit.client.0.smithi022.stdout: "kraken", 2024-09-21T09:31:08.150 INFO:tasks.workunit.client.0.smithi022.stdout: "luminous", 2024-09-21T09:31:08.150 INFO:tasks.workunit.client.0.smithi022.stdout: "mimic", 2024-09-21T09:31:08.151 INFO:tasks.workunit.client.0.smithi022.stdout: "osdmap-prune", 2024-09-21T09:31:08.151 INFO:tasks.workunit.client.0.smithi022.stdout: "nautilus", 2024-09-21T09:31:08.151 INFO:tasks.workunit.client.0.smithi022.stdout: "octopus", 2024-09-21T09:31:08.151 INFO:tasks.workunit.client.0.smithi022.stdout: "pacific", 2024-09-21T09:31:08.151 INFO:tasks.workunit.client.0.smithi022.stdout: "elector-pinging", 2024-09-21T09:31:08.151 INFO:tasks.workunit.client.0.smithi022.stdout: "quincy" 2024-09-21T09:31:08.151 INFO:tasks.workunit.client.0.smithi022.stdout: ] 2024-09-21T09:31:08.151 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.151 INFO:tasks.workunit.client.0.smithi022.stdout: "ipv4_addr_mons": 6, 2024-09-21T09:31:08.151 INFO:tasks.workunit.client.0.smithi022.stdout: "ipv6_addr_mons": 0, 2024-09-21T09:31:08.151 INFO:tasks.workunit.client.0.smithi022.stdout: "min_mon_release": 17, 2024-09-21T09:31:08.151 INFO:tasks.workunit.client.0.smithi022.stdout: "v1_addr_mons": 3, 2024-09-21T09:31:08.151 INFO:tasks.workunit.client.0.smithi022.stdout: "v2_addr_mons": 3 2024-09-21T09:31:08.151 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.151 INFO:tasks.workunit.client.0.smithi022.stdout: "organization": "ceph-qa", 2024-09-21T09:31:08.152 INFO:tasks.workunit.client.0.smithi022.stdout: "osd": { 2024-09-21T09:31:08.152 INFO:tasks.workunit.client.0.smithi022.stdout: "cluster_network": false, 2024-09-21T09:31:08.152 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 8, 2024-09-21T09:31:08.152 INFO:tasks.workunit.client.0.smithi022.stdout: "require_min_compat_client": "luminous", 2024-09-21T09:31:08.152 INFO:tasks.workunit.client.0.smithi022.stdout: "require_osd_release": "quincy" 2024-09-21T09:31:08.152 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.152 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_perf_histograms": [ 2024-09-21T09:31:08.152 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:08.152 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-21T09:31:08.152 INFO:tasks.workunit.client.0.smithi022.stdout: "axes": [ 2024-09-21T09:31:08.152 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:08.152 INFO:tasks.workunit.client.0.smithi022.stdout: "buckets": 32, 2024-09-21T09:31:08.152 INFO:tasks.workunit.client.0.smithi022.stdout: "min": 0, 2024-09-21T09:31:08.153 INFO:tasks.workunit.client.0.smithi022.stdout: "name": "Latency (usec)", 2024-09-21T09:31:08.153 INFO:tasks.workunit.client.0.smithi022.stdout: "quant_size": 100000, 2024-09-21T09:31:08.153 INFO:tasks.workunit.client.0.smithi022.stdout: "ranges": [ 2024-09-21T09:31:08.153 INFO:tasks.workunit.client.0.smithi022.stdout: "[None, -1]", 2024-09-21T09:31:08.153 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 99999]", 2024-09-21T09:31:08.153 INFO:tasks.workunit.client.0.smithi022.stdout: "[100000, 199999]", 2024-09-21T09:31:08.153 INFO:tasks.workunit.client.0.smithi022.stdout: "[200000, 399999]", 2024-09-21T09:31:08.153 INFO:tasks.workunit.client.0.smithi022.stdout: "[400000, 799999]", 2024-09-21T09:31:08.153 INFO:tasks.workunit.client.0.smithi022.stdout: "[800000, 1599999]", 2024-09-21T09:31:08.153 INFO:tasks.workunit.client.0.smithi022.stdout: "[1600000, 3199999]", 2024-09-21T09:31:08.153 INFO:tasks.workunit.client.0.smithi022.stdout: "[3200000, 6399999]", 2024-09-21T09:31:08.153 INFO:tasks.workunit.client.0.smithi022.stdout: "[6400000, 12799999]", 2024-09-21T09:31:08.153 INFO:tasks.workunit.client.0.smithi022.stdout: "[12800000, 25599999]", 2024-09-21T09:31:08.154 INFO:tasks.workunit.client.0.smithi022.stdout: "[25600000, 51199999]", 2024-09-21T09:31:08.154 INFO:tasks.workunit.client.0.smithi022.stdout: "[51200000, 102399999]", 2024-09-21T09:31:08.154 INFO:tasks.workunit.client.0.smithi022.stdout: "[102400000, 204799999]", 2024-09-21T09:31:08.154 INFO:tasks.workunit.client.0.smithi022.stdout: "[204800000, 409599999]", 2024-09-21T09:31:08.154 INFO:tasks.workunit.client.0.smithi022.stdout: "[409600000, 819199999]", 2024-09-21T09:31:08.154 INFO:tasks.workunit.client.0.smithi022.stdout: "[819200000, 1638399999]", 2024-09-21T09:31:08.154 INFO:tasks.workunit.client.0.smithi022.stdout: "[1638400000, 3276799999]", 2024-09-21T09:31:08.154 INFO:tasks.workunit.client.0.smithi022.stdout: "[3276800000, 6553599999]", 2024-09-21T09:31:08.154 INFO:tasks.workunit.client.0.smithi022.stdout: "[6553600000, 13107199999]", 2024-09-21T09:31:08.154 INFO:tasks.workunit.client.0.smithi022.stdout: "[13107200000, 26214399999]", 2024-09-21T09:31:08.154 INFO:tasks.workunit.client.0.smithi022.stdout: "[26214400000, 52428799999]", 2024-09-21T09:31:08.154 INFO:tasks.workunit.client.0.smithi022.stdout: "[52428800000, 104857599999]", 2024-09-21T09:31:08.154 INFO:tasks.workunit.client.0.smithi022.stdout: "[104857600000, 209715199999]", 2024-09-21T09:31:08.154 INFO:tasks.workunit.client.0.smithi022.stdout: "[209715200000, 419430399999]", 2024-09-21T09:31:08.154 INFO:tasks.workunit.client.0.smithi022.stdout: "[419430400000, 838860799999]", 2024-09-21T09:31:08.155 INFO:tasks.workunit.client.0.smithi022.stdout: "[838860800000, 1677721599999]", 2024-09-21T09:31:08.155 INFO:tasks.workunit.client.0.smithi022.stdout: "[1677721600000, 3355443199999]", 2024-09-21T09:31:08.155 INFO:tasks.workunit.client.0.smithi022.stdout: "[3355443200000, 6710886399999]", 2024-09-21T09:31:08.155 INFO:tasks.workunit.client.0.smithi022.stdout: "[6710886400000, 13421772799999]", 2024-09-21T09:31:08.155 INFO:tasks.workunit.client.0.smithi022.stdout: "[13421772800000, 26843545599999]", 2024-09-21T09:31:08.155 INFO:tasks.workunit.client.0.smithi022.stdout: "[26843545600000, 53687091199999]", 2024-09-21T09:31:08.155 INFO:tasks.workunit.client.0.smithi022.stdout: "[53687091200000, None]" 2024-09-21T09:31:08.155 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:08.155 INFO:tasks.workunit.client.0.smithi022.stdout: "scale_type": "log2" 2024-09-21T09:31:08.155 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.155 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:08.155 INFO:tasks.workunit.client.0.smithi022.stdout: "buckets": 32, 2024-09-21T09:31:08.155 INFO:tasks.workunit.client.0.smithi022.stdout: "min": 0, 2024-09-21T09:31:08.155 INFO:tasks.workunit.client.0.smithi022.stdout: "name": "Request size (bytes)", 2024-09-21T09:31:08.156 INFO:tasks.workunit.client.0.smithi022.stdout: "quant_size": 512, 2024-09-21T09:31:08.156 INFO:tasks.workunit.client.0.smithi022.stdout: "ranges": [ 2024-09-21T09:31:08.156 INFO:tasks.workunit.client.0.smithi022.stdout: "[None, -1]", 2024-09-21T09:31:08.156 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 511]", 2024-09-21T09:31:08.156 INFO:tasks.workunit.client.0.smithi022.stdout: "[512, 1023]", 2024-09-21T09:31:08.156 INFO:tasks.workunit.client.0.smithi022.stdout: "[1024, 2047]", 2024-09-21T09:31:08.156 INFO:tasks.workunit.client.0.smithi022.stdout: "[2048, 4095]", 2024-09-21T09:31:08.156 INFO:tasks.workunit.client.0.smithi022.stdout: "[4096, 8191]", 2024-09-21T09:31:08.156 INFO:tasks.workunit.client.0.smithi022.stdout: "[8192, 16383]", 2024-09-21T09:31:08.156 INFO:tasks.workunit.client.0.smithi022.stdout: "[16384, 32767]", 2024-09-21T09:31:08.156 INFO:tasks.workunit.client.0.smithi022.stdout: "[32768, 65535]", 2024-09-21T09:31:08.156 INFO:tasks.workunit.client.0.smithi022.stdout: "[65536, 131071]", 2024-09-21T09:31:08.156 INFO:tasks.workunit.client.0.smithi022.stdout: "[131072, 262143]", 2024-09-21T09:31:08.157 INFO:tasks.workunit.client.0.smithi022.stdout: "[262144, 524287]", 2024-09-21T09:31:08.157 INFO:tasks.workunit.client.0.smithi022.stdout: "[524288, 1048575]", 2024-09-21T09:31:08.157 INFO:tasks.workunit.client.0.smithi022.stdout: "[1048576, 2097151]", 2024-09-21T09:31:08.157 INFO:tasks.workunit.client.0.smithi022.stdout: "[2097152, 4194303]", 2024-09-21T09:31:08.157 INFO:tasks.workunit.client.0.smithi022.stdout: "[4194304, 8388607]", 2024-09-21T09:31:08.157 INFO:tasks.workunit.client.0.smithi022.stdout: "[8388608, 16777215]", 2024-09-21T09:31:08.157 INFO:tasks.workunit.client.0.smithi022.stdout: "[16777216, 33554431]", 2024-09-21T09:31:08.157 INFO:tasks.workunit.client.0.smithi022.stdout: "[33554432, 67108863]", 2024-09-21T09:31:08.157 INFO:tasks.workunit.client.0.smithi022.stdout: "[67108864, 134217727]", 2024-09-21T09:31:08.157 INFO:tasks.workunit.client.0.smithi022.stdout: "[134217728, 268435455]", 2024-09-21T09:31:08.157 INFO:tasks.workunit.client.0.smithi022.stdout: "[268435456, 536870911]", 2024-09-21T09:31:08.157 INFO:tasks.workunit.client.0.smithi022.stdout: "[536870912, 1073741823]", 2024-09-21T09:31:08.157 INFO:tasks.workunit.client.0.smithi022.stdout: "[1073741824, 2147483647]", 2024-09-21T09:31:08.157 INFO:tasks.workunit.client.0.smithi022.stdout: "[2147483648, 4294967295]", 2024-09-21T09:31:08.157 INFO:tasks.workunit.client.0.smithi022.stdout: "[4294967296, 8589934591]", 2024-09-21T09:31:08.158 INFO:tasks.workunit.client.0.smithi022.stdout: "[8589934592, 17179869183]", 2024-09-21T09:31:08.158 INFO:tasks.workunit.client.0.smithi022.stdout: "[17179869184, 34359738367]", 2024-09-21T09:31:08.158 INFO:tasks.workunit.client.0.smithi022.stdout: "[34359738368, 68719476735]", 2024-09-21T09:31:08.158 INFO:tasks.workunit.client.0.smithi022.stdout: "[68719476736, 137438953471]", 2024-09-21T09:31:08.158 INFO:tasks.workunit.client.0.smithi022.stdout: "[137438953472, 274877906943]", 2024-09-21T09:31:08.158 INFO:tasks.workunit.client.0.smithi022.stdout: "[274877906944, None]" 2024-09-21T09:31:08.158 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:08.158 INFO:tasks.workunit.client.0.smithi022.stdout: "scale_type": "log2" 2024-09-21T09:31:08.158 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.158 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:08.158 INFO:tasks.workunit.client.0.smithi022.stdout: "osds": [ 2024-09-21T09:31:08.158 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:08.158 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_id": 0, 2024-09-21T09:31:08.158 INFO:tasks.workunit.client.0.smithi022.stdout: "values": [ 2024-09-21T09:31:08.159 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.159 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.159 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.159 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.159 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.159 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.159 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.159 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.159 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.159 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.159 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.159 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.159 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.159 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.160 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.160 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.160 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.160 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.160 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.160 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.160 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.160 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.160 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.160 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.160 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.160 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.160 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.160 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.161 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.161 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.161 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.161 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:31:08.161 INFO:tasks.workunit.client.0.smithi022.stdout: ] 2024-09-21T09:31:08.161 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.161 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:08.161 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_id": 1, 2024-09-21T09:31:08.161 INFO:tasks.workunit.client.0.smithi022.stdout: "values": [ 2024-09-21T09:31:08.161 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.161 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.161 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.161 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.161 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.162 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.162 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.162 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.162 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.162 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.162 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.162 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.162 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.162 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.162 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.162 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.162 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.162 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.163 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.163 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.163 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.163 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.163 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.163 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.163 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.163 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.163 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.163 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.163 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.163 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.163 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.164 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:31:08.164 INFO:tasks.workunit.client.0.smithi022.stdout: ] 2024-09-21T09:31:08.164 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.164 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:08.164 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_id": 2, 2024-09-21T09:31:08.164 INFO:tasks.workunit.client.0.smithi022.stdout: "values": [ 2024-09-21T09:31:08.164 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.164 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.164 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.164 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.164 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.164 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.164 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.164 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.165 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.165 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.165 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.165 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.165 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.165 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.165 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.165 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.165 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.165 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.165 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.165 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.165 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.165 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.166 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.166 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.166 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.166 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.166 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.166 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.166 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.166 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.166 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.166 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:31:08.166 INFO:tasks.workunit.client.0.smithi022.stdout: ] 2024-09-21T09:31:08.166 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.166 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:08.166 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_id": 3, 2024-09-21T09:31:08.167 INFO:tasks.workunit.client.0.smithi022.stdout: "values": [ 2024-09-21T09:31:08.167 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.167 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.167 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.167 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.167 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.167 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.167 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.167 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.167 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.167 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.167 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.167 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.167 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.168 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.168 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.168 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.168 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.168 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.168 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.168 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.168 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.168 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.168 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.168 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.168 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.168 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.168 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.169 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.169 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.169 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.169 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.169 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:31:08.169 INFO:tasks.workunit.client.0.smithi022.stdout: ] 2024-09-21T09:31:08.169 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.169 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:08.169 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_id": 4, 2024-09-21T09:31:08.169 INFO:tasks.workunit.client.0.smithi022.stdout: "values": [ 2024-09-21T09:31:08.169 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.169 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.169 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.169 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.170 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.170 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.170 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.170 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.170 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.170 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.170 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.170 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.170 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.170 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.170 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.170 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.170 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.170 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.171 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.171 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.171 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.171 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.171 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.171 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.171 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.171 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.171 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.171 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.171 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.171 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.171 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.171 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:31:08.172 INFO:tasks.workunit.client.0.smithi022.stdout: ] 2024-09-21T09:31:08.172 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.172 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:08.172 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_id": 5, 2024-09-21T09:31:08.172 INFO:tasks.workunit.client.0.smithi022.stdout: "values": [ 2024-09-21T09:31:08.172 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.172 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.172 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.172 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.172 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.172 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.172 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.172 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.172 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.173 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.173 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.173 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.173 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.173 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.173 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.173 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.173 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.173 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.173 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.173 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.173 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.173 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.174 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.174 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.174 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.174 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.174 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.174 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.174 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.174 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.174 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.174 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:31:08.174 INFO:tasks.workunit.client.0.smithi022.stdout: ] 2024-09-21T09:31:08.174 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.174 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:08.175 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_id": 6, 2024-09-21T09:31:08.175 INFO:tasks.workunit.client.0.smithi022.stdout: "values": [ 2024-09-21T09:31:08.175 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.175 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.175 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.175 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.175 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.175 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.175 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.175 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.175 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.175 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.175 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.175 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.176 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.176 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.176 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.176 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.176 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.176 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.176 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.176 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.176 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.176 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.176 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.176 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.176 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.176 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.177 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.177 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.177 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.177 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.177 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.177 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:31:08.177 INFO:tasks.workunit.client.0.smithi022.stdout: ] 2024-09-21T09:31:08.177 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.177 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:08.177 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_id": 7, 2024-09-21T09:31:08.177 INFO:tasks.workunit.client.0.smithi022.stdout: "values": [ 2024-09-21T09:31:08.177 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.177 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.177 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.178 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 8, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.178 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.178 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.178 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.178 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.178 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.178 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.178 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.178 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.178 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.178 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.178 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.179 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.179 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.179 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.179 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.179 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.179 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.179 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.179 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.179 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.179 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.179 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.179 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.179 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.179 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.180 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.180 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.180 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:31:08.180 INFO:tasks.workunit.client.0.smithi022.stdout: ] 2024-09-21T09:31:08.180 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.180 INFO:tasks.workunit.client.0.smithi022.stdout: ] 2024-09-21T09:31:08.180 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.180 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-21T09:31:08.180 INFO:tasks.workunit.client.0.smithi022.stdout: "axes": [ 2024-09-21T09:31:08.180 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:08.181 INFO:tasks.workunit.client.0.smithi022.stdout: "buckets": 32, 2024-09-21T09:31:08.182 INFO:tasks.workunit.client.0.smithi022.stdout: "min": 0, 2024-09-21T09:31:08.182 INFO:tasks.workunit.client.0.smithi022.stdout: "name": "Latency (usec)", 2024-09-21T09:31:08.182 INFO:tasks.workunit.client.0.smithi022.stdout: "quant_size": 100000, 2024-09-21T09:31:08.182 INFO:tasks.workunit.client.0.smithi022.stdout: "ranges": [ 2024-09-21T09:31:08.182 INFO:tasks.workunit.client.0.smithi022.stdout: "[None, -1]", 2024-09-21T09:31:08.182 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 99999]", 2024-09-21T09:31:08.182 INFO:tasks.workunit.client.0.smithi022.stdout: "[100000, 199999]", 2024-09-21T09:31:08.182 INFO:tasks.workunit.client.0.smithi022.stdout: "[200000, 399999]", 2024-09-21T09:31:08.182 INFO:tasks.workunit.client.0.smithi022.stdout: "[400000, 799999]", 2024-09-21T09:31:08.182 INFO:tasks.workunit.client.0.smithi022.stdout: "[800000, 1599999]", 2024-09-21T09:31:08.182 INFO:tasks.workunit.client.0.smithi022.stdout: "[1600000, 3199999]", 2024-09-21T09:31:08.182 INFO:tasks.workunit.client.0.smithi022.stdout: "[3200000, 6399999]", 2024-09-21T09:31:08.182 INFO:tasks.workunit.client.0.smithi022.stdout: "[6400000, 12799999]", 2024-09-21T09:31:08.182 INFO:tasks.workunit.client.0.smithi022.stdout: "[12800000, 25599999]", 2024-09-21T09:31:08.183 INFO:tasks.workunit.client.0.smithi022.stdout: "[25600000, 51199999]", 2024-09-21T09:31:08.183 INFO:tasks.workunit.client.0.smithi022.stdout: "[51200000, 102399999]", 2024-09-21T09:31:08.183 INFO:tasks.workunit.client.0.smithi022.stdout: "[102400000, 204799999]", 2024-09-21T09:31:08.183 INFO:tasks.workunit.client.0.smithi022.stdout: "[204800000, 409599999]", 2024-09-21T09:31:08.183 INFO:tasks.workunit.client.0.smithi022.stdout: "[409600000, 819199999]", 2024-09-21T09:31:08.183 INFO:tasks.workunit.client.0.smithi022.stdout: "[819200000, 1638399999]", 2024-09-21T09:31:08.183 INFO:tasks.workunit.client.0.smithi022.stdout: "[1638400000, 3276799999]", 2024-09-21T09:31:08.183 INFO:tasks.workunit.client.0.smithi022.stdout: "[3276800000, 6553599999]", 2024-09-21T09:31:08.183 INFO:tasks.workunit.client.0.smithi022.stdout: "[6553600000, 13107199999]", 2024-09-21T09:31:08.183 INFO:tasks.workunit.client.0.smithi022.stdout: "[13107200000, 26214399999]", 2024-09-21T09:31:08.183 INFO:tasks.workunit.client.0.smithi022.stdout: "[26214400000, 52428799999]", 2024-09-21T09:31:08.183 INFO:tasks.workunit.client.0.smithi022.stdout: "[52428800000, 104857599999]", 2024-09-21T09:31:08.183 INFO:tasks.workunit.client.0.smithi022.stdout: "[104857600000, 209715199999]", 2024-09-21T09:31:08.183 INFO:tasks.workunit.client.0.smithi022.stdout: "[209715200000, 419430399999]", 2024-09-21T09:31:08.184 INFO:tasks.workunit.client.0.smithi022.stdout: "[419430400000, 838860799999]", 2024-09-21T09:31:08.184 INFO:tasks.workunit.client.0.smithi022.stdout: "[838860800000, 1677721599999]", 2024-09-21T09:31:08.184 INFO:tasks.workunit.client.0.smithi022.stdout: "[1677721600000, 3355443199999]", 2024-09-21T09:31:08.184 INFO:tasks.workunit.client.0.smithi022.stdout: "[3355443200000, 6710886399999]", 2024-09-21T09:31:08.184 INFO:tasks.workunit.client.0.smithi022.stdout: "[6710886400000, 13421772799999]", 2024-09-21T09:31:08.184 INFO:tasks.workunit.client.0.smithi022.stdout: "[13421772800000, 26843545599999]", 2024-09-21T09:31:08.184 INFO:tasks.workunit.client.0.smithi022.stdout: "[26843545600000, 53687091199999]", 2024-09-21T09:31:08.184 INFO:tasks.workunit.client.0.smithi022.stdout: "[53687091200000, None]" 2024-09-21T09:31:08.184 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:08.184 INFO:tasks.workunit.client.0.smithi022.stdout: "scale_type": "log2" 2024-09-21T09:31:08.184 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.184 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:08.184 INFO:tasks.workunit.client.0.smithi022.stdout: "buckets": 32, 2024-09-21T09:31:08.184 INFO:tasks.workunit.client.0.smithi022.stdout: "min": 0, 2024-09-21T09:31:08.184 INFO:tasks.workunit.client.0.smithi022.stdout: "name": "Request size (bytes)", 2024-09-21T09:31:08.185 INFO:tasks.workunit.client.0.smithi022.stdout: "quant_size": 512, 2024-09-21T09:31:08.185 INFO:tasks.workunit.client.0.smithi022.stdout: "ranges": [ 2024-09-21T09:31:08.185 INFO:tasks.workunit.client.0.smithi022.stdout: "[None, -1]", 2024-09-21T09:31:08.185 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 511]", 2024-09-21T09:31:08.185 INFO:tasks.workunit.client.0.smithi022.stdout: "[512, 1023]", 2024-09-21T09:31:08.185 INFO:tasks.workunit.client.0.smithi022.stdout: "[1024, 2047]", 2024-09-21T09:31:08.185 INFO:tasks.workunit.client.0.smithi022.stdout: "[2048, 4095]", 2024-09-21T09:31:08.185 INFO:tasks.workunit.client.0.smithi022.stdout: "[4096, 8191]", 2024-09-21T09:31:08.185 INFO:tasks.workunit.client.0.smithi022.stdout: "[8192, 16383]", 2024-09-21T09:31:08.185 INFO:tasks.workunit.client.0.smithi022.stdout: "[16384, 32767]", 2024-09-21T09:31:08.185 INFO:tasks.workunit.client.0.smithi022.stdout: "[32768, 65535]", 2024-09-21T09:31:08.185 INFO:tasks.workunit.client.0.smithi022.stdout: "[65536, 131071]", 2024-09-21T09:31:08.185 INFO:tasks.workunit.client.0.smithi022.stdout: "[131072, 262143]", 2024-09-21T09:31:08.185 INFO:tasks.workunit.client.0.smithi022.stdout: "[262144, 524287]", 2024-09-21T09:31:08.185 INFO:tasks.workunit.client.0.smithi022.stdout: "[524288, 1048575]", 2024-09-21T09:31:08.186 INFO:tasks.workunit.client.0.smithi022.stdout: "[1048576, 2097151]", 2024-09-21T09:31:08.186 INFO:tasks.workunit.client.0.smithi022.stdout: "[2097152, 4194303]", 2024-09-21T09:31:08.186 INFO:tasks.workunit.client.0.smithi022.stdout: "[4194304, 8388607]", 2024-09-21T09:31:08.186 INFO:tasks.workunit.client.0.smithi022.stdout: "[8388608, 16777215]", 2024-09-21T09:31:08.186 INFO:tasks.workunit.client.0.smithi022.stdout: "[16777216, 33554431]", 2024-09-21T09:31:08.186 INFO:tasks.workunit.client.0.smithi022.stdout: "[33554432, 67108863]", 2024-09-21T09:31:08.186 INFO:tasks.workunit.client.0.smithi022.stdout: "[67108864, 134217727]", 2024-09-21T09:31:08.186 INFO:tasks.workunit.client.0.smithi022.stdout: "[134217728, 268435455]", 2024-09-21T09:31:08.186 INFO:tasks.workunit.client.0.smithi022.stdout: "[268435456, 536870911]", 2024-09-21T09:31:08.186 INFO:tasks.workunit.client.0.smithi022.stdout: "[536870912, 1073741823]", 2024-09-21T09:31:08.186 INFO:tasks.workunit.client.0.smithi022.stdout: "[1073741824, 2147483647]", 2024-09-21T09:31:08.186 INFO:tasks.workunit.client.0.smithi022.stdout: "[2147483648, 4294967295]", 2024-09-21T09:31:08.186 INFO:tasks.workunit.client.0.smithi022.stdout: "[4294967296, 8589934591]", 2024-09-21T09:31:08.186 INFO:tasks.workunit.client.0.smithi022.stdout: "[8589934592, 17179869183]", 2024-09-21T09:31:08.187 INFO:tasks.workunit.client.0.smithi022.stdout: "[17179869184, 34359738367]", 2024-09-21T09:31:08.187 INFO:tasks.workunit.client.0.smithi022.stdout: "[34359738368, 68719476735]", 2024-09-21T09:31:08.187 INFO:tasks.workunit.client.0.smithi022.stdout: "[68719476736, 137438953471]", 2024-09-21T09:31:08.187 INFO:tasks.workunit.client.0.smithi022.stdout: "[137438953472, 274877906943]", 2024-09-21T09:31:08.187 INFO:tasks.workunit.client.0.smithi022.stdout: "[274877906944, None]" 2024-09-21T09:31:08.187 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:08.187 INFO:tasks.workunit.client.0.smithi022.stdout: "scale_type": "log2" 2024-09-21T09:31:08.187 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.187 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:08.187 INFO:tasks.workunit.client.0.smithi022.stdout: "osds": [ 2024-09-21T09:31:08.187 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:08.187 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_id": 0, 2024-09-21T09:31:08.187 INFO:tasks.workunit.client.0.smithi022.stdout: "values": [ 2024-09-21T09:31:08.188 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.188 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.188 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.188 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.188 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.188 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.188 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.188 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.188 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.188 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.188 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.188 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.188 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.188 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.189 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.189 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.189 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.189 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.189 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.189 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.189 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.189 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.189 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.189 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.189 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.189 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.189 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.189 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.189 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.190 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.190 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.190 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:31:08.190 INFO:tasks.workunit.client.0.smithi022.stdout: ] 2024-09-21T09:31:08.190 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.190 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:08.190 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_id": 1, 2024-09-21T09:31:08.190 INFO:tasks.workunit.client.0.smithi022.stdout: "values": [ 2024-09-21T09:31:08.190 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.190 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.190 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.190 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.190 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.191 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.191 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.191 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.191 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.191 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.191 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.191 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.191 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.191 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.191 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.191 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.191 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.191 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.191 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.192 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.192 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.192 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.192 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.192 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.192 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.192 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.192 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.192 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.192 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.192 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.192 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.192 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:31:08.192 INFO:tasks.workunit.client.0.smithi022.stdout: ] 2024-09-21T09:31:08.193 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.193 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:08.193 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_id": 2, 2024-09-21T09:31:08.193 INFO:tasks.workunit.client.0.smithi022.stdout: "values": [ 2024-09-21T09:31:08.193 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.193 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.193 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.193 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.193 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.193 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.193 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.193 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.193 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.193 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.193 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.194 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.194 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.194 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.194 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.194 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.194 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.194 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.194 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.194 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.194 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.194 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.194 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.194 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.194 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.195 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.195 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.195 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.195 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.195 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.195 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.195 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:31:08.195 INFO:tasks.workunit.client.0.smithi022.stdout: ] 2024-09-21T09:31:08.195 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.195 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:08.195 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_id": 3, 2024-09-21T09:31:08.195 INFO:tasks.workunit.client.0.smithi022.stdout: "values": [ 2024-09-21T09:31:08.195 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.195 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.196 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.196 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.196 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.196 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.196 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.196 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.196 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.196 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.196 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.196 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.196 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.196 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.196 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.197 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.197 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.197 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.197 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.197 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.197 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.197 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.197 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.197 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.197 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.197 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.197 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.197 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.197 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.198 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.198 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.198 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:31:08.198 INFO:tasks.workunit.client.0.smithi022.stdout: ] 2024-09-21T09:31:08.198 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.198 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:08.198 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_id": 4, 2024-09-21T09:31:08.198 INFO:tasks.workunit.client.0.smithi022.stdout: "values": [ 2024-09-21T09:31:08.198 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.198 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.198 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.198 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.198 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.198 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.199 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.199 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.199 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.199 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.199 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.199 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.199 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.199 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.199 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.199 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.199 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.199 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.199 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.200 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.200 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.200 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.200 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.200 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.200 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.200 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.200 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.200 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.200 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.200 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.200 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.200 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:31:08.200 INFO:tasks.workunit.client.0.smithi022.stdout: ] 2024-09-21T09:31:08.201 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.201 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:08.201 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_id": 5, 2024-09-21T09:31:08.201 INFO:tasks.workunit.client.0.smithi022.stdout: "values": [ 2024-09-21T09:31:08.201 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.201 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.201 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.201 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.201 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.201 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.201 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.201 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.201 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.201 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.202 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.202 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.202 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.202 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.202 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.202 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.202 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.202 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.202 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.202 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.202 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.202 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.202 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.203 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.203 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.203 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.203 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.203 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.203 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.203 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.203 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.203 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:31:08.203 INFO:tasks.workunit.client.0.smithi022.stdout: ] 2024-09-21T09:31:08.203 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.203 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:08.203 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_id": 6, 2024-09-21T09:31:08.203 INFO:tasks.workunit.client.0.smithi022.stdout: "values": [ 2024-09-21T09:31:08.204 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.204 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.204 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.204 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.204 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.204 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.204 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.204 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.204 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.204 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.204 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.204 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.204 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.204 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.205 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.205 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.205 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.205 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.205 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.205 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.205 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.205 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.205 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.205 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.205 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.205 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.205 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.205 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.206 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.206 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.206 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.206 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:31:08.206 INFO:tasks.workunit.client.0.smithi022.stdout: ] 2024-09-21T09:31:08.206 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.206 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:08.206 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_id": 7, 2024-09-21T09:31:08.206 INFO:tasks.workunit.client.0.smithi022.stdout: "values": [ 2024-09-21T09:31:08.206 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.206 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.206 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.206 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.206 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.207 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.207 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.207 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.207 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.207 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.207 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.207 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.207 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.207 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.207 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.207 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.207 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.207 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.208 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.208 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.208 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.208 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.208 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.208 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.208 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.208 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.208 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.208 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.208 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.208 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.208 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.208 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:31:08.208 INFO:tasks.workunit.client.0.smithi022.stdout: ] 2024-09-21T09:31:08.209 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.209 INFO:tasks.workunit.client.0.smithi022.stdout: ] 2024-09-21T09:31:08.209 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.209 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-21T09:31:08.209 INFO:tasks.workunit.client.0.smithi022.stdout: "axes": [ 2024-09-21T09:31:08.209 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:08.209 INFO:tasks.workunit.client.0.smithi022.stdout: "buckets": 32, 2024-09-21T09:31:08.209 INFO:tasks.workunit.client.0.smithi022.stdout: "min": 0, 2024-09-21T09:31:08.209 INFO:tasks.workunit.client.0.smithi022.stdout: "name": "Latency (usec)", 2024-09-21T09:31:08.209 INFO:tasks.workunit.client.0.smithi022.stdout: "quant_size": 100000, 2024-09-21T09:31:08.209 INFO:tasks.workunit.client.0.smithi022.stdout: "ranges": [ 2024-09-21T09:31:08.209 INFO:tasks.workunit.client.0.smithi022.stdout: "[None, -1]", 2024-09-21T09:31:08.209 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 99999]", 2024-09-21T09:31:08.209 INFO:tasks.workunit.client.0.smithi022.stdout: "[100000, 199999]", 2024-09-21T09:31:08.210 INFO:tasks.workunit.client.0.smithi022.stdout: "[200000, 399999]", 2024-09-21T09:31:08.210 INFO:tasks.workunit.client.0.smithi022.stdout: "[400000, 799999]", 2024-09-21T09:31:08.210 INFO:tasks.workunit.client.0.smithi022.stdout: "[800000, 1599999]", 2024-09-21T09:31:08.210 INFO:tasks.workunit.client.0.smithi022.stdout: "[1600000, 3199999]", 2024-09-21T09:31:08.210 INFO:tasks.workunit.client.0.smithi022.stdout: "[3200000, 6399999]", 2024-09-21T09:31:08.210 INFO:tasks.workunit.client.0.smithi022.stdout: "[6400000, 12799999]", 2024-09-21T09:31:08.210 INFO:tasks.workunit.client.0.smithi022.stdout: "[12800000, 25599999]", 2024-09-21T09:31:08.210 INFO:tasks.workunit.client.0.smithi022.stdout: "[25600000, 51199999]", 2024-09-21T09:31:08.210 INFO:tasks.workunit.client.0.smithi022.stdout: "[51200000, 102399999]", 2024-09-21T09:31:08.210 INFO:tasks.workunit.client.0.smithi022.stdout: "[102400000, 204799999]", 2024-09-21T09:31:08.210 INFO:tasks.workunit.client.0.smithi022.stdout: "[204800000, 409599999]", 2024-09-21T09:31:08.210 INFO:tasks.workunit.client.0.smithi022.stdout: "[409600000, 819199999]", 2024-09-21T09:31:08.210 INFO:tasks.workunit.client.0.smithi022.stdout: "[819200000, 1638399999]", 2024-09-21T09:31:08.210 INFO:tasks.workunit.client.0.smithi022.stdout: "[1638400000, 3276799999]", 2024-09-21T09:31:08.211 INFO:tasks.workunit.client.0.smithi022.stdout: "[3276800000, 6553599999]", 2024-09-21T09:31:08.211 INFO:tasks.workunit.client.0.smithi022.stdout: "[6553600000, 13107199999]", 2024-09-21T09:31:08.211 INFO:tasks.workunit.client.0.smithi022.stdout: "[13107200000, 26214399999]", 2024-09-21T09:31:08.211 INFO:tasks.workunit.client.0.smithi022.stdout: "[26214400000, 52428799999]", 2024-09-21T09:31:08.211 INFO:tasks.workunit.client.0.smithi022.stdout: "[52428800000, 104857599999]", 2024-09-21T09:31:08.211 INFO:tasks.workunit.client.0.smithi022.stdout: "[104857600000, 209715199999]", 2024-09-21T09:31:08.211 INFO:tasks.workunit.client.0.smithi022.stdout: "[209715200000, 419430399999]", 2024-09-21T09:31:08.211 INFO:tasks.workunit.client.0.smithi022.stdout: "[419430400000, 838860799999]", 2024-09-21T09:31:08.211 INFO:tasks.workunit.client.0.smithi022.stdout: "[838860800000, 1677721599999]", 2024-09-21T09:31:08.211 INFO:tasks.workunit.client.0.smithi022.stdout: "[1677721600000, 3355443199999]", 2024-09-21T09:31:08.211 INFO:tasks.workunit.client.0.smithi022.stdout: "[3355443200000, 6710886399999]", 2024-09-21T09:31:08.211 INFO:tasks.workunit.client.0.smithi022.stdout: "[6710886400000, 13421772799999]", 2024-09-21T09:31:08.211 INFO:tasks.workunit.client.0.smithi022.stdout: "[13421772800000, 26843545599999]", 2024-09-21T09:31:08.211 INFO:tasks.workunit.client.0.smithi022.stdout: "[26843545600000, 53687091199999]", 2024-09-21T09:31:08.212 INFO:tasks.workunit.client.0.smithi022.stdout: "[53687091200000, None]" 2024-09-21T09:31:08.212 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:08.212 INFO:tasks.workunit.client.0.smithi022.stdout: "scale_type": "log2" 2024-09-21T09:31:08.212 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.212 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:08.212 INFO:tasks.workunit.client.0.smithi022.stdout: "buckets": 32, 2024-09-21T09:31:08.212 INFO:tasks.workunit.client.0.smithi022.stdout: "min": 0, 2024-09-21T09:31:08.212 INFO:tasks.workunit.client.0.smithi022.stdout: "name": "Request size (bytes)", 2024-09-21T09:31:08.212 INFO:tasks.workunit.client.0.smithi022.stdout: "quant_size": 512, 2024-09-21T09:31:08.212 INFO:tasks.workunit.client.0.smithi022.stdout: "ranges": [ 2024-09-21T09:31:08.212 INFO:tasks.workunit.client.0.smithi022.stdout: "[None, -1]", 2024-09-21T09:31:08.212 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 511]", 2024-09-21T09:31:08.212 INFO:tasks.workunit.client.0.smithi022.stdout: "[512, 1023]", 2024-09-21T09:31:08.213 INFO:tasks.workunit.client.0.smithi022.stdout: "[1024, 2047]", 2024-09-21T09:31:08.213 INFO:tasks.workunit.client.0.smithi022.stdout: "[2048, 4095]", 2024-09-21T09:31:08.213 INFO:tasks.workunit.client.0.smithi022.stdout: "[4096, 8191]", 2024-09-21T09:31:08.213 INFO:tasks.workunit.client.0.smithi022.stdout: "[8192, 16383]", 2024-09-21T09:31:08.213 INFO:tasks.workunit.client.0.smithi022.stdout: "[16384, 32767]", 2024-09-21T09:31:08.213 INFO:tasks.workunit.client.0.smithi022.stdout: "[32768, 65535]", 2024-09-21T09:31:08.213 INFO:tasks.workunit.client.0.smithi022.stdout: "[65536, 131071]", 2024-09-21T09:31:08.213 INFO:tasks.workunit.client.0.smithi022.stdout: "[131072, 262143]", 2024-09-21T09:31:08.213 INFO:tasks.workunit.client.0.smithi022.stdout: "[262144, 524287]", 2024-09-21T09:31:08.213 INFO:tasks.workunit.client.0.smithi022.stdout: "[524288, 1048575]", 2024-09-21T09:31:08.213 INFO:tasks.workunit.client.0.smithi022.stdout: "[1048576, 2097151]", 2024-09-21T09:31:08.213 INFO:tasks.workunit.client.0.smithi022.stdout: "[2097152, 4194303]", 2024-09-21T09:31:08.213 INFO:tasks.workunit.client.0.smithi022.stdout: "[4194304, 8388607]", 2024-09-21T09:31:08.213 INFO:tasks.workunit.client.0.smithi022.stdout: "[8388608, 16777215]", 2024-09-21T09:31:08.214 INFO:tasks.workunit.client.0.smithi022.stdout: "[16777216, 33554431]", 2024-09-21T09:31:08.214 INFO:tasks.workunit.client.0.smithi022.stdout: "[33554432, 67108863]", 2024-09-21T09:31:08.214 INFO:tasks.workunit.client.0.smithi022.stdout: "[67108864, 134217727]", 2024-09-21T09:31:08.214 INFO:tasks.workunit.client.0.smithi022.stdout: "[134217728, 268435455]", 2024-09-21T09:31:08.214 INFO:tasks.workunit.client.0.smithi022.stdout: "[268435456, 536870911]", 2024-09-21T09:31:08.214 INFO:tasks.workunit.client.0.smithi022.stdout: "[536870912, 1073741823]", 2024-09-21T09:31:08.214 INFO:tasks.workunit.client.0.smithi022.stdout: "[1073741824, 2147483647]", 2024-09-21T09:31:08.214 INFO:tasks.workunit.client.0.smithi022.stdout: "[2147483648, 4294967295]", 2024-09-21T09:31:08.214 INFO:tasks.workunit.client.0.smithi022.stdout: "[4294967296, 8589934591]", 2024-09-21T09:31:08.214 INFO:tasks.workunit.client.0.smithi022.stdout: "[8589934592, 17179869183]", 2024-09-21T09:31:08.214 INFO:tasks.workunit.client.0.smithi022.stdout: "[17179869184, 34359738367]", 2024-09-21T09:31:08.214 INFO:tasks.workunit.client.0.smithi022.stdout: "[34359738368, 68719476735]", 2024-09-21T09:31:08.214 INFO:tasks.workunit.client.0.smithi022.stdout: "[68719476736, 137438953471]", 2024-09-21T09:31:08.214 INFO:tasks.workunit.client.0.smithi022.stdout: "[137438953472, 274877906943]", 2024-09-21T09:31:08.215 INFO:tasks.workunit.client.0.smithi022.stdout: "[274877906944, None]" 2024-09-21T09:31:08.215 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:08.215 INFO:tasks.workunit.client.0.smithi022.stdout: "scale_type": "log2" 2024-09-21T09:31:08.215 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.215 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:08.215 INFO:tasks.workunit.client.0.smithi022.stdout: "osds": [ 2024-09-21T09:31:08.215 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:08.215 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_id": 0, 2024-09-21T09:31:08.215 INFO:tasks.workunit.client.0.smithi022.stdout: "values": [ 2024-09-21T09:31:08.215 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.215 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.215 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.215 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.216 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.216 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.216 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.216 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.216 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.216 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.216 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.216 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.216 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.216 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.216 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.216 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.216 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.216 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.217 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.217 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.217 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.217 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.217 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.217 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.217 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.217 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.217 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.217 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.217 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.217 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.217 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.217 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:31:08.218 INFO:tasks.workunit.client.0.smithi022.stdout: ] 2024-09-21T09:31:08.218 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.218 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:08.218 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_id": 1, 2024-09-21T09:31:08.218 INFO:tasks.workunit.client.0.smithi022.stdout: "values": [ 2024-09-21T09:31:08.218 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.218 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.218 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.218 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.218 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.218 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.218 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.218 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.218 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.219 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.219 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.219 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.219 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.219 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.219 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.219 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.219 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.219 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.219 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.219 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.219 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.219 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.219 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.220 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.220 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.220 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.220 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.220 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.220 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.220 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.220 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.220 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:31:08.220 INFO:tasks.workunit.client.0.smithi022.stdout: ] 2024-09-21T09:31:08.220 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.220 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:08.220 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_id": 2, 2024-09-21T09:31:08.221 INFO:tasks.workunit.client.0.smithi022.stdout: "values": [ 2024-09-21T09:31:08.221 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.221 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.221 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.221 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.221 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.221 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.221 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.221 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.221 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.221 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.221 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.221 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.221 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.222 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.222 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.222 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.222 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.222 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.222 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.222 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.222 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.222 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.222 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.222 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.222 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.223 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.223 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.223 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.223 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.223 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.223 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.223 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:31:08.223 INFO:tasks.workunit.client.0.smithi022.stdout: ] 2024-09-21T09:31:08.223 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.223 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:08.223 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_id": 3, 2024-09-21T09:31:08.223 INFO:tasks.workunit.client.0.smithi022.stdout: "values": [ 2024-09-21T09:31:08.223 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.223 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.224 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.224 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.224 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.224 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.224 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.224 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.224 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.224 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.224 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.224 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.224 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.224 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.224 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.225 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.225 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.225 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.225 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.225 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.225 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.225 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.225 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.225 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.225 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.225 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.225 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.226 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.226 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.226 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.226 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.226 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:31:08.226 INFO:tasks.workunit.client.0.smithi022.stdout: ] 2024-09-21T09:31:08.226 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.226 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:08.226 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_id": 4, 2024-09-21T09:31:08.226 INFO:tasks.workunit.client.0.smithi022.stdout: "values": [ 2024-09-21T09:31:08.226 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.226 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.227 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.227 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.227 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.227 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.227 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.227 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.227 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.227 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.227 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.227 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.227 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.227 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.227 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.228 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.228 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.228 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.228 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.228 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.228 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.228 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.228 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.228 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.228 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.228 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.228 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.228 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.228 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.229 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.229 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.229 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:31:08.229 INFO:tasks.workunit.client.0.smithi022.stdout: ] 2024-09-21T09:31:08.229 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.229 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:08.229 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_id": 5, 2024-09-21T09:31:08.229 INFO:tasks.workunit.client.0.smithi022.stdout: "values": [ 2024-09-21T09:31:08.229 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.229 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.229 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.229 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.229 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.230 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.230 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.230 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.230 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.230 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.230 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.230 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.230 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.230 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.230 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.230 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.230 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.230 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.230 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.231 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.231 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.231 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.231 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.231 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.231 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.231 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.231 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.231 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.231 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.231 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.231 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.231 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:31:08.232 INFO:tasks.workunit.client.0.smithi022.stdout: ] 2024-09-21T09:31:08.232 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.232 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:08.232 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_id": 6, 2024-09-21T09:31:08.232 INFO:tasks.workunit.client.0.smithi022.stdout: "values": [ 2024-09-21T09:31:08.232 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.232 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.232 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.232 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.232 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.232 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.232 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.232 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.232 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.233 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.233 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.233 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.233 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.233 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.233 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.233 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.233 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.233 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.233 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.233 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.233 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.233 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.234 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.234 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.234 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.234 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.234 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.234 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.234 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.234 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.234 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.234 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:31:08.234 INFO:tasks.workunit.client.0.smithi022.stdout: ] 2024-09-21T09:31:08.234 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.234 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:08.234 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_id": 7, 2024-09-21T09:31:08.235 INFO:tasks.workunit.client.0.smithi022.stdout: "values": [ 2024-09-21T09:31:08.235 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.235 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.235 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.235 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.235 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.235 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.235 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.235 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.235 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.235 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.235 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.235 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.235 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.236 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.236 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.236 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.236 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.236 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.236 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.236 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.236 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.236 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.236 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.236 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.236 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.236 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.237 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.237 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.237 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.237 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.237 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.237 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:31:08.237 INFO:tasks.workunit.client.0.smithi022.stdout: ] 2024-09-21T09:31:08.237 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.237 INFO:tasks.workunit.client.0.smithi022.stdout: ] 2024-09-21T09:31:08.237 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.237 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-21T09:31:08.237 INFO:tasks.workunit.client.0.smithi022.stdout: "axes": [ 2024-09-21T09:31:08.237 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:08.237 INFO:tasks.workunit.client.0.smithi022.stdout: "buckets": 32, 2024-09-21T09:31:08.237 INFO:tasks.workunit.client.0.smithi022.stdout: "min": 0, 2024-09-21T09:31:08.238 INFO:tasks.workunit.client.0.smithi022.stdout: "name": "Latency (usec)", 2024-09-21T09:31:08.238 INFO:tasks.workunit.client.0.smithi022.stdout: "quant_size": 100000, 2024-09-21T09:31:08.238 INFO:tasks.workunit.client.0.smithi022.stdout: "ranges": [ 2024-09-21T09:31:08.238 INFO:tasks.workunit.client.0.smithi022.stdout: "[None, -1]", 2024-09-21T09:31:08.238 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 99999]", 2024-09-21T09:31:08.238 INFO:tasks.workunit.client.0.smithi022.stdout: "[100000, 199999]", 2024-09-21T09:31:08.238 INFO:tasks.workunit.client.0.smithi022.stdout: "[200000, 399999]", 2024-09-21T09:31:08.238 INFO:tasks.workunit.client.0.smithi022.stdout: "[400000, 799999]", 2024-09-21T09:31:08.238 INFO:tasks.workunit.client.0.smithi022.stdout: "[800000, 1599999]", 2024-09-21T09:31:08.238 INFO:tasks.workunit.client.0.smithi022.stdout: "[1600000, 3199999]", 2024-09-21T09:31:08.238 INFO:tasks.workunit.client.0.smithi022.stdout: "[3200000, 6399999]", 2024-09-21T09:31:08.238 INFO:tasks.workunit.client.0.smithi022.stdout: "[6400000, 12799999]", 2024-09-21T09:31:08.238 INFO:tasks.workunit.client.0.smithi022.stdout: "[12800000, 25599999]", 2024-09-21T09:31:08.238 INFO:tasks.workunit.client.0.smithi022.stdout: "[25600000, 51199999]", 2024-09-21T09:31:08.239 INFO:tasks.workunit.client.0.smithi022.stdout: "[51200000, 102399999]", 2024-09-21T09:31:08.239 INFO:tasks.workunit.client.0.smithi022.stdout: "[102400000, 204799999]", 2024-09-21T09:31:08.239 INFO:tasks.workunit.client.0.smithi022.stdout: "[204800000, 409599999]", 2024-09-21T09:31:08.239 INFO:tasks.workunit.client.0.smithi022.stdout: "[409600000, 819199999]", 2024-09-21T09:31:08.239 INFO:tasks.workunit.client.0.smithi022.stdout: "[819200000, 1638399999]", 2024-09-21T09:31:08.239 INFO:tasks.workunit.client.0.smithi022.stdout: "[1638400000, 3276799999]", 2024-09-21T09:31:08.239 INFO:tasks.workunit.client.0.smithi022.stdout: "[3276800000, 6553599999]", 2024-09-21T09:31:08.239 INFO:tasks.workunit.client.0.smithi022.stdout: "[6553600000, 13107199999]", 2024-09-21T09:31:08.239 INFO:tasks.workunit.client.0.smithi022.stdout: "[13107200000, 26214399999]", 2024-09-21T09:31:08.239 INFO:tasks.workunit.client.0.smithi022.stdout: "[26214400000, 52428799999]", 2024-09-21T09:31:08.239 INFO:tasks.workunit.client.0.smithi022.stdout: "[52428800000, 104857599999]", 2024-09-21T09:31:08.239 INFO:tasks.workunit.client.0.smithi022.stdout: "[104857600000, 209715199999]", 2024-09-21T09:31:08.239 INFO:tasks.workunit.client.0.smithi022.stdout: "[209715200000, 419430399999]", 2024-09-21T09:31:08.239 INFO:tasks.workunit.client.0.smithi022.stdout: "[419430400000, 838860799999]", 2024-09-21T09:31:08.240 INFO:tasks.workunit.client.0.smithi022.stdout: "[838860800000, 1677721599999]", 2024-09-21T09:31:08.240 INFO:tasks.workunit.client.0.smithi022.stdout: "[1677721600000, 3355443199999]", 2024-09-21T09:31:08.240 INFO:tasks.workunit.client.0.smithi022.stdout: "[3355443200000, 6710886399999]", 2024-09-21T09:31:08.240 INFO:tasks.workunit.client.0.smithi022.stdout: "[6710886400000, 13421772799999]", 2024-09-21T09:31:08.240 INFO:tasks.workunit.client.0.smithi022.stdout: "[13421772800000, 26843545599999]", 2024-09-21T09:31:08.240 INFO:tasks.workunit.client.0.smithi022.stdout: "[26843545600000, 53687091199999]", 2024-09-21T09:31:08.240 INFO:tasks.workunit.client.0.smithi022.stdout: "[53687091200000, None]" 2024-09-21T09:31:08.240 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:08.240 INFO:tasks.workunit.client.0.smithi022.stdout: "scale_type": "log2" 2024-09-21T09:31:08.240 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.240 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:08.240 INFO:tasks.workunit.client.0.smithi022.stdout: "buckets": 32, 2024-09-21T09:31:08.240 INFO:tasks.workunit.client.0.smithi022.stdout: "min": 0, 2024-09-21T09:31:08.240 INFO:tasks.workunit.client.0.smithi022.stdout: "name": "Request size (bytes)", 2024-09-21T09:31:08.241 INFO:tasks.workunit.client.0.smithi022.stdout: "quant_size": 512, 2024-09-21T09:31:08.241 INFO:tasks.workunit.client.0.smithi022.stdout: "ranges": [ 2024-09-21T09:31:08.241 INFO:tasks.workunit.client.0.smithi022.stdout: "[None, -1]", 2024-09-21T09:31:08.241 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 511]", 2024-09-21T09:31:08.241 INFO:tasks.workunit.client.0.smithi022.stdout: "[512, 1023]", 2024-09-21T09:31:08.241 INFO:tasks.workunit.client.0.smithi022.stdout: "[1024, 2047]", 2024-09-21T09:31:08.241 INFO:tasks.workunit.client.0.smithi022.stdout: "[2048, 4095]", 2024-09-21T09:31:08.241 INFO:tasks.workunit.client.0.smithi022.stdout: "[4096, 8191]", 2024-09-21T09:31:08.241 INFO:tasks.workunit.client.0.smithi022.stdout: "[8192, 16383]", 2024-09-21T09:31:08.241 INFO:tasks.workunit.client.0.smithi022.stdout: "[16384, 32767]", 2024-09-21T09:31:08.241 INFO:tasks.workunit.client.0.smithi022.stdout: "[32768, 65535]", 2024-09-21T09:31:08.241 INFO:tasks.workunit.client.0.smithi022.stdout: "[65536, 131071]", 2024-09-21T09:31:08.241 INFO:tasks.workunit.client.0.smithi022.stdout: "[131072, 262143]", 2024-09-21T09:31:08.241 INFO:tasks.workunit.client.0.smithi022.stdout: "[262144, 524287]", 2024-09-21T09:31:08.242 INFO:tasks.workunit.client.0.smithi022.stdout: "[524288, 1048575]", 2024-09-21T09:31:08.242 INFO:tasks.workunit.client.0.smithi022.stdout: "[1048576, 2097151]", 2024-09-21T09:31:08.242 INFO:tasks.workunit.client.0.smithi022.stdout: "[2097152, 4194303]", 2024-09-21T09:31:08.242 INFO:tasks.workunit.client.0.smithi022.stdout: "[4194304, 8388607]", 2024-09-21T09:31:08.242 INFO:tasks.workunit.client.0.smithi022.stdout: "[8388608, 16777215]", 2024-09-21T09:31:08.242 INFO:tasks.workunit.client.0.smithi022.stdout: "[16777216, 33554431]", 2024-09-21T09:31:08.242 INFO:tasks.workunit.client.0.smithi022.stdout: "[33554432, 67108863]", 2024-09-21T09:31:08.242 INFO:tasks.workunit.client.0.smithi022.stdout: "[67108864, 134217727]", 2024-09-21T09:31:08.242 INFO:tasks.workunit.client.0.smithi022.stdout: "[134217728, 268435455]", 2024-09-21T09:31:08.242 INFO:tasks.workunit.client.0.smithi022.stdout: "[268435456, 536870911]", 2024-09-21T09:31:08.242 INFO:tasks.workunit.client.0.smithi022.stdout: "[536870912, 1073741823]", 2024-09-21T09:31:08.242 INFO:tasks.workunit.client.0.smithi022.stdout: "[1073741824, 2147483647]", 2024-09-21T09:31:08.242 INFO:tasks.workunit.client.0.smithi022.stdout: "[2147483648, 4294967295]", 2024-09-21T09:31:08.242 INFO:tasks.workunit.client.0.smithi022.stdout: "[4294967296, 8589934591]", 2024-09-21T09:31:08.243 INFO:tasks.workunit.client.0.smithi022.stdout: "[8589934592, 17179869183]", 2024-09-21T09:31:08.243 INFO:tasks.workunit.client.0.smithi022.stdout: "[17179869184, 34359738367]", 2024-09-21T09:31:08.243 INFO:tasks.workunit.client.0.smithi022.stdout: "[34359738368, 68719476735]", 2024-09-21T09:31:08.243 INFO:tasks.workunit.client.0.smithi022.stdout: "[68719476736, 137438953471]", 2024-09-21T09:31:08.243 INFO:tasks.workunit.client.0.smithi022.stdout: "[137438953472, 274877906943]", 2024-09-21T09:31:08.243 INFO:tasks.workunit.client.0.smithi022.stdout: "[274877906944, None]" 2024-09-21T09:31:08.243 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:08.243 INFO:tasks.workunit.client.0.smithi022.stdout: "scale_type": "log2" 2024-09-21T09:31:08.243 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.243 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:08.243 INFO:tasks.workunit.client.0.smithi022.stdout: "osds": [ 2024-09-21T09:31:08.243 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:08.243 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_id": 0, 2024-09-21T09:31:08.243 INFO:tasks.workunit.client.0.smithi022.stdout: "values": [ 2024-09-21T09:31:08.243 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.244 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.244 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.244 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.244 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.244 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.244 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.244 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.244 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.244 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.244 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.244 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.244 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.244 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.244 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.245 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.245 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.245 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.245 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.245 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.245 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.245 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.245 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.245 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.245 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.245 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.245 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.245 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.246 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.246 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.246 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.246 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:31:08.246 INFO:tasks.workunit.client.0.smithi022.stdout: ] 2024-09-21T09:31:08.246 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.246 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:08.246 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_id": 1, 2024-09-21T09:31:08.246 INFO:tasks.workunit.client.0.smithi022.stdout: "values": [ 2024-09-21T09:31:08.246 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.246 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.246 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.246 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.247 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.247 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 6, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.247 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 7, 3, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.247 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 16, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.247 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 3, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.247 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.247 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.247 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.247 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.247 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.247 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.247 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.247 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.247 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.247 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.248 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.248 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.248 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.248 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.248 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.248 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.248 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.248 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.248 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.248 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.248 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.248 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.248 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:31:08.248 INFO:tasks.workunit.client.0.smithi022.stdout: ] 2024-09-21T09:31:08.248 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.249 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:08.249 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_id": 2, 2024-09-21T09:31:08.249 INFO:tasks.workunit.client.0.smithi022.stdout: "values": [ 2024-09-21T09:31:08.249 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.249 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.249 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.249 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.249 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.249 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.249 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.249 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.249 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.249 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.249 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.249 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.250 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.250 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.250 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.250 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.250 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.250 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.250 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.250 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.250 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.250 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.250 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.250 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.250 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.250 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.251 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.251 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.251 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.251 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.251 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.251 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:31:08.251 INFO:tasks.workunit.client.0.smithi022.stdout: ] 2024-09-21T09:31:08.251 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.251 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:08.251 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_id": 3, 2024-09-21T09:31:08.251 INFO:tasks.workunit.client.0.smithi022.stdout: "values": [ 2024-09-21T09:31:08.251 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.251 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.251 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.252 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.252 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.252 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.252 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.252 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.252 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.252 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.252 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.252 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.252 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.252 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.252 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.252 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.252 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.253 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.253 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.253 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.253 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.253 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.253 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.253 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.253 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.253 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.253 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.253 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.253 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.253 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.253 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.254 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:31:08.254 INFO:tasks.workunit.client.0.smithi022.stdout: ] 2024-09-21T09:31:08.254 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.254 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:08.254 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_id": 4, 2024-09-21T09:31:08.254 INFO:tasks.workunit.client.0.smithi022.stdout: "values": [ 2024-09-21T09:31:08.254 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.254 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.254 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.254 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.254 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.254 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.254 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.254 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.254 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.255 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.255 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.255 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.255 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.255 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.255 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.255 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.255 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.255 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.255 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.255 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.255 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.255 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.256 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.256 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.256 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.256 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.256 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.256 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.256 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.256 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.256 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.256 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:31:08.256 INFO:tasks.workunit.client.0.smithi022.stdout: ] 2024-09-21T09:31:08.256 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.256 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:08.256 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_id": 5, 2024-09-21T09:31:08.257 INFO:tasks.workunit.client.0.smithi022.stdout: "values": [ 2024-09-21T09:31:08.257 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.257 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.257 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.257 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.257 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.257 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.257 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.257 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.257 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.257 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.257 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.257 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.257 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.258 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.258 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.258 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.258 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.258 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.258 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.258 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.258 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.258 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.258 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.258 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.258 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.258 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.258 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.259 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.259 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.259 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.259 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.259 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:31:08.259 INFO:tasks.workunit.client.0.smithi022.stdout: ] 2024-09-21T09:31:08.259 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.259 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:08.259 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_id": 6, 2024-09-21T09:31:08.259 INFO:tasks.workunit.client.0.smithi022.stdout: "values": [ 2024-09-21T09:31:08.259 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.259 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.259 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.259 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.260 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.260 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.260 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.260 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.260 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.260 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.260 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.260 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.260 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.260 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.260 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.260 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.260 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.261 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.261 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.261 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.261 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.261 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.261 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.261 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.261 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.261 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.261 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.261 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.261 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.261 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.261 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.262 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:31:08.262 INFO:tasks.workunit.client.0.smithi022.stdout: ] 2024-09-21T09:31:08.262 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.262 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:08.262 INFO:tasks.workunit.client.0.smithi022.stdout: "osd_id": 7, 2024-09-21T09:31:08.262 INFO:tasks.workunit.client.0.smithi022.stdout: "values": [ 2024-09-21T09:31:08.262 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.262 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.262 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.262 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.262 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.262 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.262 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.262 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.263 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.263 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.263 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.263 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.263 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.263 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.263 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.263 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.263 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.263 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.263 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.263 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.263 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.263 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.263 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.264 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.264 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.264 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.264 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.264 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.264 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.264 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.264 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:31:08.264 INFO:tasks.workunit.client.0.smithi022.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:31:08.264 INFO:tasks.workunit.client.0.smithi022.stdout: ] 2024-09-21T09:31:08.264 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.264 INFO:tasks.workunit.client.0.smithi022.stdout: ] 2024-09-21T09:31:08.264 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.264 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.265 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:08.265 INFO:tasks.workunit.client.0.smithi022.stdout: "perf_counters": { 2024-09-21T09:31:08.265 INFO:tasks.workunit.client.0.smithi022.stdout: "mds.aa5e831b31d98e3163af5edfeb921f9af32463c5": { 2024-09-21T09:31:08.265 INFO:tasks.workunit.client.0.smithi022.stdout: "mds": { 2024-09-21T09:31:08.265 INFO:tasks.workunit.client.0.smithi022.stdout: "caps": { 2024-09-21T09:31:08.265 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.265 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.265 INFO:tasks.workunit.client.0.smithi022.stdout: "ceph_cap_op_flush_ack": { 2024-09-21T09:31:08.265 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.265 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.265 INFO:tasks.workunit.client.0.smithi022.stdout: "ceph_cap_op_flushsnap_ack": { 2024-09-21T09:31:08.265 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.265 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.266 INFO:tasks.workunit.client.0.smithi022.stdout: "ceph_cap_op_grant": { 2024-09-21T09:31:08.266 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.266 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.266 INFO:tasks.workunit.client.0.smithi022.stdout: "ceph_cap_op_revoke": { 2024-09-21T09:31:08.266 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.266 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.266 INFO:tasks.workunit.client.0.smithi022.stdout: "ceph_cap_op_trunc": { 2024-09-21T09:31:08.266 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.266 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.266 INFO:tasks.workunit.client.0.smithi022.stdout: "dir_commit": { 2024-09-21T09:31:08.266 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 12 2024-09-21T09:31:08.266 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.266 INFO:tasks.workunit.client.0.smithi022.stdout: "dir_fetch": { 2024-09-21T09:31:08.266 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.266 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.267 INFO:tasks.workunit.client.0.smithi022.stdout: "dir_merge": { 2024-09-21T09:31:08.267 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.267 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.267 INFO:tasks.workunit.client.0.smithi022.stdout: "dir_split": { 2024-09-21T09:31:08.267 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.267 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.267 INFO:tasks.workunit.client.0.smithi022.stdout: "exported_inodes": { 2024-09-21T09:31:08.267 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.267 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.267 INFO:tasks.workunit.client.0.smithi022.stdout: "forward": { 2024-09-21T09:31:08.267 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.267 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.267 INFO:tasks.workunit.client.0.smithi022.stdout: "handle_client_cap_release": { 2024-09-21T09:31:08.268 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.268 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.268 INFO:tasks.workunit.client.0.smithi022.stdout: "handle_client_caps": { 2024-09-21T09:31:08.268 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.268 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.268 INFO:tasks.workunit.client.0.smithi022.stdout: "handle_client_caps_dirty": { 2024-09-21T09:31:08.268 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.268 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.268 INFO:tasks.workunit.client.0.smithi022.stdout: "handle_inode_file_caps": { 2024-09-21T09:31:08.268 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.268 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.268 INFO:tasks.workunit.client.0.smithi022.stdout: "imported_inodes": { 2024-09-21T09:31:08.268 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.268 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.269 INFO:tasks.workunit.client.0.smithi022.stdout: "inodes": { 2024-09-21T09:31:08.269 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 10 2024-09-21T09:31:08.269 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.269 INFO:tasks.workunit.client.0.smithi022.stdout: "inodes_expired": { 2024-09-21T09:31:08.269 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.269 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.269 INFO:tasks.workunit.client.0.smithi022.stdout: "inodes_pinned": { 2024-09-21T09:31:08.269 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 10 2024-09-21T09:31:08.269 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.269 INFO:tasks.workunit.client.0.smithi022.stdout: "inodes_with_caps": { 2024-09-21T09:31:08.269 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.269 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.269 INFO:tasks.workunit.client.0.smithi022.stdout: "load_cent": { 2024-09-21T09:31:08.269 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.269 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.270 INFO:tasks.workunit.client.0.smithi022.stdout: "openino_dir_fetch": { 2024-09-21T09:31:08.270 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.270 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.270 INFO:tasks.workunit.client.0.smithi022.stdout: "process_request_cap_release": { 2024-09-21T09:31:08.270 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.270 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.270 INFO:tasks.workunit.client.0.smithi022.stdout: "reply_latency": { 2024-09-21T09:31:08.270 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 2, 2024-09-21T09:31:08.270 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 129410 2024-09-21T09:31:08.270 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.270 INFO:tasks.workunit.client.0.smithi022.stdout: "request": { 2024-09-21T09:31:08.270 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2 2024-09-21T09:31:08.270 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.270 INFO:tasks.workunit.client.0.smithi022.stdout: "root_rbytes": { 2024-09-21T09:31:08.271 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.271 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.271 INFO:tasks.workunit.client.0.smithi022.stdout: "root_rfiles": { 2024-09-21T09:31:08.271 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.271 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.271 INFO:tasks.workunit.client.0.smithi022.stdout: "root_rsnaps": { 2024-09-21T09:31:08.271 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.271 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.271 INFO:tasks.workunit.client.0.smithi022.stdout: "slow_reply": { 2024-09-21T09:31:08.271 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.271 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.271 INFO:tasks.workunit.client.0.smithi022.stdout: "subtrees": { 2024-09-21T09:31:08.271 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2 2024-09-21T09:31:08.271 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.272 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.272 INFO:tasks.workunit.client.0.smithi022.stdout: "mds_cache": { 2024-09-21T09:31:08.272 INFO:tasks.workunit.client.0.smithi022.stdout: "ireq_enqueue_scrub": { 2024-09-21T09:31:08.272 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.272 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.272 INFO:tasks.workunit.client.0.smithi022.stdout: "ireq_exportdir": { 2024-09-21T09:31:08.272 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.272 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.272 INFO:tasks.workunit.client.0.smithi022.stdout: "ireq_flush": { 2024-09-21T09:31:08.272 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.272 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.272 INFO:tasks.workunit.client.0.smithi022.stdout: "ireq_fragmentdir": { 2024-09-21T09:31:08.272 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.272 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.273 INFO:tasks.workunit.client.0.smithi022.stdout: "ireq_fragstats": { 2024-09-21T09:31:08.273 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.273 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.273 INFO:tasks.workunit.client.0.smithi022.stdout: "ireq_inodestats": { 2024-09-21T09:31:08.273 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.273 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.273 INFO:tasks.workunit.client.0.smithi022.stdout: "num_recovering_enqueued": { 2024-09-21T09:31:08.273 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.273 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.273 INFO:tasks.workunit.client.0.smithi022.stdout: "num_recovering_prioritized": { 2024-09-21T09:31:08.273 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.273 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.273 INFO:tasks.workunit.client.0.smithi022.stdout: "num_recovering_processing": { 2024-09-21T09:31:08.273 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.273 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.274 INFO:tasks.workunit.client.0.smithi022.stdout: "num_strays": { 2024-09-21T09:31:08.274 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.274 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.274 INFO:tasks.workunit.client.0.smithi022.stdout: "num_strays_delayed": { 2024-09-21T09:31:08.274 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.274 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.274 INFO:tasks.workunit.client.0.smithi022.stdout: "num_strays_enqueuing": { 2024-09-21T09:31:08.274 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.274 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.274 INFO:tasks.workunit.client.0.smithi022.stdout: "recovery_completed": { 2024-09-21T09:31:08.274 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.274 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.274 INFO:tasks.workunit.client.0.smithi022.stdout: "recovery_started": { 2024-09-21T09:31:08.274 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.275 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.275 INFO:tasks.workunit.client.0.smithi022.stdout: "strays_created": { 2024-09-21T09:31:08.275 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.275 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.275 INFO:tasks.workunit.client.0.smithi022.stdout: "strays_enqueued": { 2024-09-21T09:31:08.275 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.275 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.275 INFO:tasks.workunit.client.0.smithi022.stdout: "strays_migrated": { 2024-09-21T09:31:08.275 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.275 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.275 INFO:tasks.workunit.client.0.smithi022.stdout: "strays_reintegrated": { 2024-09-21T09:31:08.275 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.275 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.275 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.276 INFO:tasks.workunit.client.0.smithi022.stdout: "mds_log": { 2024-09-21T09:31:08.276 INFO:tasks.workunit.client.0.smithi022.stdout: "ev": { 2024-09-21T09:31:08.276 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 5 2024-09-21T09:31:08.276 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.276 INFO:tasks.workunit.client.0.smithi022.stdout: "evadd": { 2024-09-21T09:31:08.276 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 5 2024-09-21T09:31:08.276 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.276 INFO:tasks.workunit.client.0.smithi022.stdout: "evex": { 2024-09-21T09:31:08.276 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.276 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.276 INFO:tasks.workunit.client.0.smithi022.stdout: "evexd": { 2024-09-21T09:31:08.276 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.276 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.276 INFO:tasks.workunit.client.0.smithi022.stdout: "evexg": { 2024-09-21T09:31:08.277 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.277 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.277 INFO:tasks.workunit.client.0.smithi022.stdout: "evtrm": { 2024-09-21T09:31:08.277 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.277 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.277 INFO:tasks.workunit.client.0.smithi022.stdout: "jlat": { 2024-09-21T09:31:08.277 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 5, 2024-09-21T09:31:08.277 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 8856762 2024-09-21T09:31:08.277 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.277 INFO:tasks.workunit.client.0.smithi022.stdout: "replayed": { 2024-09-21T09:31:08.277 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.277 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.277 INFO:tasks.workunit.client.0.smithi022.stdout: "seg": { 2024-09-21T09:31:08.278 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1 2024-09-21T09:31:08.278 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.278 INFO:tasks.workunit.client.0.smithi022.stdout: "segadd": { 2024-09-21T09:31:08.278 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1 2024-09-21T09:31:08.278 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.278 INFO:tasks.workunit.client.0.smithi022.stdout: "segex": { 2024-09-21T09:31:08.278 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.278 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.278 INFO:tasks.workunit.client.0.smithi022.stdout: "segexd": { 2024-09-21T09:31:08.278 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.278 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.278 INFO:tasks.workunit.client.0.smithi022.stdout: "segexg": { 2024-09-21T09:31:08.278 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.278 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.279 INFO:tasks.workunit.client.0.smithi022.stdout: "segtrm": { 2024-09-21T09:31:08.279 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.279 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.279 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.279 INFO:tasks.workunit.client.0.smithi022.stdout: "mds_mem": { 2024-09-21T09:31:08.279 INFO:tasks.workunit.client.0.smithi022.stdout: "cap": { 2024-09-21T09:31:08.279 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.279 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.279 INFO:tasks.workunit.client.0.smithi022.stdout: "cap+": { 2024-09-21T09:31:08.279 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2 2024-09-21T09:31:08.279 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.279 INFO:tasks.workunit.client.0.smithi022.stdout: "cap-": { 2024-09-21T09:31:08.279 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2 2024-09-21T09:31:08.280 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.280 INFO:tasks.workunit.client.0.smithi022.stdout: "dir": { 2024-09-21T09:31:08.280 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 12 2024-09-21T09:31:08.280 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.280 INFO:tasks.workunit.client.0.smithi022.stdout: "dir+": { 2024-09-21T09:31:08.280 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 12 2024-09-21T09:31:08.280 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.280 INFO:tasks.workunit.client.0.smithi022.stdout: "dir-": { 2024-09-21T09:31:08.280 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.280 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.280 INFO:tasks.workunit.client.0.smithi022.stdout: "dn": { 2024-09-21T09:31:08.280 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 10 2024-09-21T09:31:08.280 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.280 INFO:tasks.workunit.client.0.smithi022.stdout: "dn+": { 2024-09-21T09:31:08.281 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 10 2024-09-21T09:31:08.281 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.281 INFO:tasks.workunit.client.0.smithi022.stdout: "dn-": { 2024-09-21T09:31:08.281 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.281 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.281 INFO:tasks.workunit.client.0.smithi022.stdout: "heap": { 2024-09-21T09:31:08.281 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 207068 2024-09-21T09:31:08.281 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.281 INFO:tasks.workunit.client.0.smithi022.stdout: "ino": { 2024-09-21T09:31:08.281 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 13 2024-09-21T09:31:08.281 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.281 INFO:tasks.workunit.client.0.smithi022.stdout: "ino+": { 2024-09-21T09:31:08.281 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 13 2024-09-21T09:31:08.281 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.281 INFO:tasks.workunit.client.0.smithi022.stdout: "ino-": { 2024-09-21T09:31:08.282 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.282 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.282 INFO:tasks.workunit.client.0.smithi022.stdout: "rss": { 2024-09-21T09:31:08.282 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 39020 2024-09-21T09:31:08.282 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.282 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.282 INFO:tasks.workunit.client.0.smithi022.stdout: "mds_server": { 2024-09-21T09:31:08.282 INFO:tasks.workunit.client.0.smithi022.stdout: "cap_acquisition_throttle": { 2024-09-21T09:31:08.282 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.282 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.282 INFO:tasks.workunit.client.0.smithi022.stdout: "cap_revoke_eviction": { 2024-09-21T09:31:08.282 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.282 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.282 INFO:tasks.workunit.client.0.smithi022.stdout: "handle_client_request": { 2024-09-21T09:31:08.283 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2 2024-09-21T09:31:08.283 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.283 INFO:tasks.workunit.client.0.smithi022.stdout: "handle_client_session": { 2024-09-21T09:31:08.283 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 16 2024-09-21T09:31:08.283 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.283 INFO:tasks.workunit.client.0.smithi022.stdout: "handle_peer_request": { 2024-09-21T09:31:08.283 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.283 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.283 INFO:tasks.workunit.client.0.smithi022.stdout: "req_create_latency": { 2024-09-21T09:31:08.283 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.283 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.283 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.283 INFO:tasks.workunit.client.0.smithi022.stdout: "req_getattr_latency": { 2024-09-21T09:31:08.283 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 2, 2024-09-21T09:31:08.284 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 129410 2024-09-21T09:31:08.284 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.284 INFO:tasks.workunit.client.0.smithi022.stdout: "req_getfilelock_latency": { 2024-09-21T09:31:08.284 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.284 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.284 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.284 INFO:tasks.workunit.client.0.smithi022.stdout: "req_getvxattr_latency": { 2024-09-21T09:31:08.284 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.284 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.284 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.284 INFO:tasks.workunit.client.0.smithi022.stdout: "req_link_latency": { 2024-09-21T09:31:08.284 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.285 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.285 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.285 INFO:tasks.workunit.client.0.smithi022.stdout: "req_lookup_latency": { 2024-09-21T09:31:08.285 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.285 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.285 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.285 INFO:tasks.workunit.client.0.smithi022.stdout: "req_lookuphash_latency": { 2024-09-21T09:31:08.285 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.285 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.285 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.285 INFO:tasks.workunit.client.0.smithi022.stdout: "req_lookupino_latency": { 2024-09-21T09:31:08.285 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.285 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.285 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.286 INFO:tasks.workunit.client.0.smithi022.stdout: "req_lookupname_latency": { 2024-09-21T09:31:08.286 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.286 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.286 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.286 INFO:tasks.workunit.client.0.smithi022.stdout: "req_lookupparent_latency": { 2024-09-21T09:31:08.286 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.286 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.286 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.286 INFO:tasks.workunit.client.0.smithi022.stdout: "req_lookupsnap_latency": { 2024-09-21T09:31:08.286 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.286 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.286 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.286 INFO:tasks.workunit.client.0.smithi022.stdout: "req_lssnap_latency": { 2024-09-21T09:31:08.286 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.287 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.287 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.287 INFO:tasks.workunit.client.0.smithi022.stdout: "req_mkdir_latency": { 2024-09-21T09:31:08.287 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.287 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.287 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.287 INFO:tasks.workunit.client.0.smithi022.stdout: "req_mknod_latency": { 2024-09-21T09:31:08.287 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.287 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.287 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.287 INFO:tasks.workunit.client.0.smithi022.stdout: "req_mksnap_latency": { 2024-09-21T09:31:08.287 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.287 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.287 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.288 INFO:tasks.workunit.client.0.smithi022.stdout: "req_open_latency": { 2024-09-21T09:31:08.288 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.288 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.288 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.288 INFO:tasks.workunit.client.0.smithi022.stdout: "req_readdir_latency": { 2024-09-21T09:31:08.288 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.288 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.288 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.288 INFO:tasks.workunit.client.0.smithi022.stdout: "req_rename_latency": { 2024-09-21T09:31:08.288 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.288 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.288 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.288 INFO:tasks.workunit.client.0.smithi022.stdout: "req_renamesnap_latency": { 2024-09-21T09:31:08.289 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.289 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.289 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.289 INFO:tasks.workunit.client.0.smithi022.stdout: "req_rmdir_latency": { 2024-09-21T09:31:08.289 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.289 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.289 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.289 INFO:tasks.workunit.client.0.smithi022.stdout: "req_rmsnap_latency": { 2024-09-21T09:31:08.289 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.289 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.289 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.289 INFO:tasks.workunit.client.0.smithi022.stdout: "req_rmxattr_latency": { 2024-09-21T09:31:08.289 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.289 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.290 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.290 INFO:tasks.workunit.client.0.smithi022.stdout: "req_setattr_latency": { 2024-09-21T09:31:08.290 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.290 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.290 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.290 INFO:tasks.workunit.client.0.smithi022.stdout: "req_setdirlayout_latency": { 2024-09-21T09:31:08.290 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.290 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.290 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.290 INFO:tasks.workunit.client.0.smithi022.stdout: "req_setfilelock_latency": { 2024-09-21T09:31:08.290 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.290 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.290 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.291 INFO:tasks.workunit.client.0.smithi022.stdout: "req_setlayout_latency": { 2024-09-21T09:31:08.291 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.291 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.291 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.291 INFO:tasks.workunit.client.0.smithi022.stdout: "req_setxattr_latency": { 2024-09-21T09:31:08.291 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.291 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.291 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.291 INFO:tasks.workunit.client.0.smithi022.stdout: "req_symlink_latency": { 2024-09-21T09:31:08.291 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.291 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.291 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.292 INFO:tasks.workunit.client.0.smithi022.stdout: "req_unlink_latency": { 2024-09-21T09:31:08.292 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.292 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.292 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.292 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.292 INFO:tasks.workunit.client.0.smithi022.stdout: "mds_sessions": { 2024-09-21T09:31:08.292 INFO:tasks.workunit.client.0.smithi022.stdout: "average_load": { 2024-09-21T09:31:08.292 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1 2024-09-21T09:31:08.292 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.292 INFO:tasks.workunit.client.0.smithi022.stdout: "avg_session_uptime": { 2024-09-21T09:31:08.292 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 89 2024-09-21T09:31:08.292 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.292 INFO:tasks.workunit.client.0.smithi022.stdout: "mdthresh_evicted": { 2024-09-21T09:31:08.293 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.293 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.293 INFO:tasks.workunit.client.0.smithi022.stdout: "session_add": { 2024-09-21T09:31:08.293 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2 2024-09-21T09:31:08.293 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.293 INFO:tasks.workunit.client.0.smithi022.stdout: "session_count": { 2024-09-21T09:31:08.293 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.293 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.293 INFO:tasks.workunit.client.0.smithi022.stdout: "session_remove": { 2024-09-21T09:31:08.293 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2 2024-09-21T09:31:08.293 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.293 INFO:tasks.workunit.client.0.smithi022.stdout: "sessions_open": { 2024-09-21T09:31:08.293 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.293 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.294 INFO:tasks.workunit.client.0.smithi022.stdout: "sessions_stale": { 2024-09-21T09:31:08.294 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.294 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.294 INFO:tasks.workunit.client.0.smithi022.stdout: "total_load": { 2024-09-21T09:31:08.294 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2 2024-09-21T09:31:08.294 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.294 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.294 INFO:tasks.workunit.client.0.smithi022.stdout: "objecter": { 2024-09-21T09:31:08.294 INFO:tasks.workunit.client.0.smithi022.stdout: "op_active": { 2024-09-21T09:31:08.294 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.294 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.294 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r": { 2024-09-21T09:31:08.294 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.294 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.295 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rmw": { 2024-09-21T09:31:08.295 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.295 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.295 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w": { 2024-09-21T09:31:08.295 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 45 2024-09-21T09:31:08.295 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.295 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.295 INFO:tasks.workunit.client.0.smithi022.stdout: "purge_queue": { 2024-09-21T09:31:08.295 INFO:tasks.workunit.client.0.smithi022.stdout: "pq_executed": { 2024-09-21T09:31:08.295 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.295 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.295 INFO:tasks.workunit.client.0.smithi022.stdout: "pq_executing": { 2024-09-21T09:31:08.295 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.295 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.296 INFO:tasks.workunit.client.0.smithi022.stdout: "pq_executing_high_water": { 2024-09-21T09:31:08.296 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.296 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.296 INFO:tasks.workunit.client.0.smithi022.stdout: "pq_executing_ops": { 2024-09-21T09:31:08.296 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.296 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.296 INFO:tasks.workunit.client.0.smithi022.stdout: "pq_executing_ops_high_water": { 2024-09-21T09:31:08.296 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.296 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.296 INFO:tasks.workunit.client.0.smithi022.stdout: "pq_item_in_journal": { 2024-09-21T09:31:08.296 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.296 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.296 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.297 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.297 INFO:tasks.workunit.client.0.smithi022.stdout: "mon.5349066dea191aa6e7969482c5114baabfb76f71": { 2024-09-21T09:31:08.297 INFO:tasks.workunit.client.0.smithi022.stdout: "mon": { 2024-09-21T09:31:08.297 INFO:tasks.workunit.client.0.smithi022.stdout: "election_call": { 2024-09-21T09:31:08.297 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1 2024-09-21T09:31:08.297 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.297 INFO:tasks.workunit.client.0.smithi022.stdout: "election_lose": { 2024-09-21T09:31:08.297 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1 2024-09-21T09:31:08.297 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.297 INFO:tasks.workunit.client.0.smithi022.stdout: "election_win": { 2024-09-21T09:31:08.297 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.297 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.297 INFO:tasks.workunit.client.0.smithi022.stdout: "num_elections": { 2024-09-21T09:31:08.298 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 3 2024-09-21T09:31:08.298 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.298 INFO:tasks.workunit.client.0.smithi022.stdout: "num_sessions": { 2024-09-21T09:31:08.298 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 11 2024-09-21T09:31:08.298 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.298 INFO:tasks.workunit.client.0.smithi022.stdout: "session_add": { 2024-09-21T09:31:08.298 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 5 2024-09-21T09:31:08.298 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.298 INFO:tasks.workunit.client.0.smithi022.stdout: "session_rm": { 2024-09-21T09:31:08.298 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 77 2024-09-21T09:31:08.298 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.298 INFO:tasks.workunit.client.0.smithi022.stdout: "session_trim": { 2024-09-21T09:31:08.298 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.299 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.299 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.299 INFO:tasks.workunit.client.0.smithi022.stdout: "paxos": { 2024-09-21T09:31:08.299 INFO:tasks.workunit.client.0.smithi022.stdout: "accept_timeout": { 2024-09-21T09:31:08.299 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.299 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.299 INFO:tasks.workunit.client.0.smithi022.stdout: "begin": { 2024-09-21T09:31:08.299 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 813 2024-09-21T09:31:08.299 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.299 INFO:tasks.workunit.client.0.smithi022.stdout: "begin_bytes": { 2024-09-21T09:31:08.299 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 813, 2024-09-21T09:31:08.299 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 3886043 2024-09-21T09:31:08.299 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.299 INFO:tasks.workunit.client.0.smithi022.stdout: "begin_keys": { 2024-09-21T09:31:08.300 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.300 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.300 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.300 INFO:tasks.workunit.client.0.smithi022.stdout: "begin_latency": { 2024-09-21T09:31:08.300 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 813, 2024-09-21T09:31:08.300 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 92998844 2024-09-21T09:31:08.300 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.300 INFO:tasks.workunit.client.0.smithi022.stdout: "collect": { 2024-09-21T09:31:08.300 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1 2024-09-21T09:31:08.300 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.300 INFO:tasks.workunit.client.0.smithi022.stdout: "collect_bytes": { 2024-09-21T09:31:08.300 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 1, 2024-09-21T09:31:08.300 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 73 2024-09-21T09:31:08.300 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.301 INFO:tasks.workunit.client.0.smithi022.stdout: "collect_keys": { 2024-09-21T09:31:08.301 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 1, 2024-09-21T09:31:08.301 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1 2024-09-21T09:31:08.301 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.301 INFO:tasks.workunit.client.0.smithi022.stdout: "collect_latency": { 2024-09-21T09:31:08.301 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 1, 2024-09-21T09:31:08.301 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.301 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.301 INFO:tasks.workunit.client.0.smithi022.stdout: "collect_timeout": { 2024-09-21T09:31:08.301 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.301 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.301 INFO:tasks.workunit.client.0.smithi022.stdout: "collect_uncommitted": { 2024-09-21T09:31:08.301 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.301 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.302 INFO:tasks.workunit.client.0.smithi022.stdout: "commit": { 2024-09-21T09:31:08.302 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 815 2024-09-21T09:31:08.302 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.302 INFO:tasks.workunit.client.0.smithi022.stdout: "commit_bytes": { 2024-09-21T09:31:08.302 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.302 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.302 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.302 INFO:tasks.workunit.client.0.smithi022.stdout: "commit_keys": { 2024-09-21T09:31:08.302 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.302 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.302 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.302 INFO:tasks.workunit.client.0.smithi022.stdout: "commit_latency": { 2024-09-21T09:31:08.303 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.303 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.303 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.303 INFO:tasks.workunit.client.0.smithi022.stdout: "lease_ack_timeout": { 2024-09-21T09:31:08.303 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.303 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.303 INFO:tasks.workunit.client.0.smithi022.stdout: "lease_timeout": { 2024-09-21T09:31:08.303 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.303 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.303 INFO:tasks.workunit.client.0.smithi022.stdout: "new_pn": { 2024-09-21T09:31:08.303 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.303 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.303 INFO:tasks.workunit.client.0.smithi022.stdout: "new_pn_latency": { 2024-09-21T09:31:08.303 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.304 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.304 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.304 INFO:tasks.workunit.client.0.smithi022.stdout: "refresh": { 2024-09-21T09:31:08.304 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 815 2024-09-21T09:31:08.304 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.304 INFO:tasks.workunit.client.0.smithi022.stdout: "refresh_latency": { 2024-09-21T09:31:08.304 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 815, 2024-09-21T09:31:08.304 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1849976914 2024-09-21T09:31:08.304 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.304 INFO:tasks.workunit.client.0.smithi022.stdout: "restart": { 2024-09-21T09:31:08.304 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 8 2024-09-21T09:31:08.304 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.304 INFO:tasks.workunit.client.0.smithi022.stdout: "share_state": { 2024-09-21T09:31:08.304 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.305 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.305 INFO:tasks.workunit.client.0.smithi022.stdout: "share_state_bytes": { 2024-09-21T09:31:08.305 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.305 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.305 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.305 INFO:tasks.workunit.client.0.smithi022.stdout: "share_state_keys": { 2024-09-21T09:31:08.305 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.305 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.305 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.305 INFO:tasks.workunit.client.0.smithi022.stdout: "start_leader": { 2024-09-21T09:31:08.305 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.305 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.305 INFO:tasks.workunit.client.0.smithi022.stdout: "start_peon": { 2024-09-21T09:31:08.306 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1 2024-09-21T09:31:08.306 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.306 INFO:tasks.workunit.client.0.smithi022.stdout: "store_state": { 2024-09-21T09:31:08.306 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 814 2024-09-21T09:31:08.306 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.306 INFO:tasks.workunit.client.0.smithi022.stdout: "store_state_bytes": { 2024-09-21T09:31:08.306 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 814, 2024-09-21T09:31:08.306 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 7619626 2024-09-21T09:31:08.306 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.306 INFO:tasks.workunit.client.0.smithi022.stdout: "store_state_keys": { 2024-09-21T09:31:08.306 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 814, 2024-09-21T09:31:08.306 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 5476 2024-09-21T09:31:08.306 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.307 INFO:tasks.workunit.client.0.smithi022.stdout: "store_state_latency": { 2024-09-21T09:31:08.307 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 814, 2024-09-21T09:31:08.307 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 99998748 2024-09-21T09:31:08.307 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.307 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.307 INFO:tasks.workunit.client.0.smithi022.stdout: "prioritycache": { 2024-09-21T09:31:08.307 INFO:tasks.workunit.client.0.smithi022.stdout: "cache_bytes": { 2024-09-21T09:31:08.307 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1020054731 2024-09-21T09:31:08.307 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.307 INFO:tasks.workunit.client.0.smithi022.stdout: "heap_bytes": { 2024-09-21T09:31:08.307 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 49078272 2024-09-21T09:31:08.307 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.307 INFO:tasks.workunit.client.0.smithi022.stdout: "mapped_bytes": { 2024-09-21T09:31:08.307 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 33382400 2024-09-21T09:31:08.308 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.308 INFO:tasks.workunit.client.0.smithi022.stdout: "target_bytes": { 2024-09-21T09:31:08.308 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2147483648 2024-09-21T09:31:08.308 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.308 INFO:tasks.workunit.client.0.smithi022.stdout: "unmapped_bytes": { 2024-09-21T09:31:08.308 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 15695872 2024-09-21T09:31:08.308 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.308 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.308 INFO:tasks.workunit.client.0.smithi022.stdout: "prioritycache:full": { 2024-09-21T09:31:08.308 INFO:tasks.workunit.client.0.smithi022.stdout: "committed_bytes": { 2024-09-21T09:31:08.308 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 348127232 2024-09-21T09:31:08.308 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.308 INFO:tasks.workunit.client.0.smithi022.stdout: "pri0_bytes": { 2024-09-21T09:31:08.308 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.309 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.309 INFO:tasks.workunit.client.0.smithi022.stdout: "pri10_bytes": { 2024-09-21T09:31:08.309 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.309 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.309 INFO:tasks.workunit.client.0.smithi022.stdout: "pri11_bytes": { 2024-09-21T09:31:08.309 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 277021337 2024-09-21T09:31:08.309 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.309 INFO:tasks.workunit.client.0.smithi022.stdout: "pri1_bytes": { 2024-09-21T09:31:08.309 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 271552 2024-09-21T09:31:08.309 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.309 INFO:tasks.workunit.client.0.smithi022.stdout: "pri2_bytes": { 2024-09-21T09:31:08.309 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.309 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.310 INFO:tasks.workunit.client.0.smithi022.stdout: "pri3_bytes": { 2024-09-21T09:31:08.310 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.310 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.310 INFO:tasks.workunit.client.0.smithi022.stdout: "pri4_bytes": { 2024-09-21T09:31:08.310 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.310 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.310 INFO:tasks.workunit.client.0.smithi022.stdout: "pri5_bytes": { 2024-09-21T09:31:08.310 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.310 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.310 INFO:tasks.workunit.client.0.smithi022.stdout: "pri6_bytes": { 2024-09-21T09:31:08.310 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.310 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.310 INFO:tasks.workunit.client.0.smithi022.stdout: "pri7_bytes": { 2024-09-21T09:31:08.310 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.310 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.311 INFO:tasks.workunit.client.0.smithi022.stdout: "pri8_bytes": { 2024-09-21T09:31:08.311 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.311 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.311 INFO:tasks.workunit.client.0.smithi022.stdout: "pri9_bytes": { 2024-09-21T09:31:08.311 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.311 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.311 INFO:tasks.workunit.client.0.smithi022.stdout: "reserved_bytes": { 2024-09-21T09:31:08.311 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 70834343 2024-09-21T09:31:08.311 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.311 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.311 INFO:tasks.workunit.client.0.smithi022.stdout: "prioritycache:inc": { 2024-09-21T09:31:08.311 INFO:tasks.workunit.client.0.smithi022.stdout: "committed_bytes": { 2024-09-21T09:31:08.311 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 348127232 2024-09-21T09:31:08.311 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.312 INFO:tasks.workunit.client.0.smithi022.stdout: "pri0_bytes": { 2024-09-21T09:31:08.312 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.312 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.312 INFO:tasks.workunit.client.0.smithi022.stdout: "pri10_bytes": { 2024-09-21T09:31:08.312 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.312 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.312 INFO:tasks.workunit.client.0.smithi022.stdout: "pri11_bytes": { 2024-09-21T09:31:08.312 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 277021337 2024-09-21T09:31:08.312 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.312 INFO:tasks.workunit.client.0.smithi022.stdout: "pri1_bytes": { 2024-09-21T09:31:08.312 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 41245 2024-09-21T09:31:08.312 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.312 INFO:tasks.workunit.client.0.smithi022.stdout: "pri2_bytes": { 2024-09-21T09:31:08.313 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.313 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.313 INFO:tasks.workunit.client.0.smithi022.stdout: "pri3_bytes": { 2024-09-21T09:31:08.313 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.313 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.313 INFO:tasks.workunit.client.0.smithi022.stdout: "pri4_bytes": { 2024-09-21T09:31:08.313 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.313 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.313 INFO:tasks.workunit.client.0.smithi022.stdout: "pri5_bytes": { 2024-09-21T09:31:08.313 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.313 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.313 INFO:tasks.workunit.client.0.smithi022.stdout: "pri6_bytes": { 2024-09-21T09:31:08.313 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.313 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.314 INFO:tasks.workunit.client.0.smithi022.stdout: "pri7_bytes": { 2024-09-21T09:31:08.314 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.314 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.314 INFO:tasks.workunit.client.0.smithi022.stdout: "pri8_bytes": { 2024-09-21T09:31:08.314 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.314 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.314 INFO:tasks.workunit.client.0.smithi022.stdout: "pri9_bytes": { 2024-09-21T09:31:08.314 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.314 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.314 INFO:tasks.workunit.client.0.smithi022.stdout: "reserved_bytes": { 2024-09-21T09:31:08.314 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 71064650 2024-09-21T09:31:08.314 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.314 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.314 INFO:tasks.workunit.client.0.smithi022.stdout: "prioritycache:kv": { 2024-09-21T09:31:08.315 INFO:tasks.workunit.client.0.smithi022.stdout: "committed_bytes": { 2024-09-21T09:31:08.315 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 322961408 2024-09-21T09:31:08.315 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.315 INFO:tasks.workunit.client.0.smithi022.stdout: "pri0_bytes": { 2024-09-21T09:31:08.315 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 67108864 2024-09-21T09:31:08.315 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.315 INFO:tasks.workunit.client.0.smithi022.stdout: "pri10_bytes": { 2024-09-21T09:31:08.315 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.315 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.315 INFO:tasks.workunit.client.0.smithi022.stdout: "pri11_bytes": { 2024-09-21T09:31:08.315 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 184680891 2024-09-21T09:31:08.315 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.315 INFO:tasks.workunit.client.0.smithi022.stdout: "pri1_bytes": { 2024-09-21T09:31:08.316 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.316 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.316 INFO:tasks.workunit.client.0.smithi022.stdout: "pri2_bytes": { 2024-09-21T09:31:08.316 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.316 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.316 INFO:tasks.workunit.client.0.smithi022.stdout: "pri3_bytes": { 2024-09-21T09:31:08.316 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.316 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.316 INFO:tasks.workunit.client.0.smithi022.stdout: "pri4_bytes": { 2024-09-21T09:31:08.316 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.316 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.316 INFO:tasks.workunit.client.0.smithi022.stdout: "pri5_bytes": { 2024-09-21T09:31:08.316 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.316 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.317 INFO:tasks.workunit.client.0.smithi022.stdout: "pri6_bytes": { 2024-09-21T09:31:08.317 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.317 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.317 INFO:tasks.workunit.client.0.smithi022.stdout: "pri7_bytes": { 2024-09-21T09:31:08.317 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.317 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.317 INFO:tasks.workunit.client.0.smithi022.stdout: "pri8_bytes": { 2024-09-21T09:31:08.317 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.317 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.317 INFO:tasks.workunit.client.0.smithi022.stdout: "pri9_bytes": { 2024-09-21T09:31:08.317 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.317 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.317 INFO:tasks.workunit.client.0.smithi022.stdout: "reserved_bytes": { 2024-09-21T09:31:08.317 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 71171653 2024-09-21T09:31:08.318 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.318 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.318 INFO:tasks.workunit.client.0.smithi022.stdout: "rocksdb": { 2024-09-21T09:31:08.318 INFO:tasks.workunit.client.0.smithi022.stdout: "compact": { 2024-09-21T09:31:08.318 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.318 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.318 INFO:tasks.workunit.client.0.smithi022.stdout: "compact_completed": { 2024-09-21T09:31:08.318 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1 2024-09-21T09:31:08.318 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.318 INFO:tasks.workunit.client.0.smithi022.stdout: "compact_lasted": { 2024-09-21T09:31:08.318 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 23852185 2024-09-21T09:31:08.318 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.318 INFO:tasks.workunit.client.0.smithi022.stdout: "compact_queue_len": { 2024-09-21T09:31:08.318 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.319 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.319 INFO:tasks.workunit.client.0.smithi022.stdout: "compact_queue_merge": { 2024-09-21T09:31:08.319 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.319 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.319 INFO:tasks.workunit.client.0.smithi022.stdout: "compact_running": { 2024-09-21T09:31:08.319 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.319 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.319 INFO:tasks.workunit.client.0.smithi022.stdout: "get_latency": { 2024-09-21T09:31:08.319 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 35045, 2024-09-21T09:31:08.319 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 85593023 2024-09-21T09:31:08.319 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.319 INFO:tasks.workunit.client.0.smithi022.stdout: "rocksdb_write_delay_time": { 2024-09-21T09:31:08.319 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.319 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.319 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.320 INFO:tasks.workunit.client.0.smithi022.stdout: "rocksdb_write_memtable_time": { 2024-09-21T09:31:08.320 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.320 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.320 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.320 INFO:tasks.workunit.client.0.smithi022.stdout: "rocksdb_write_pre_and_post_time": { 2024-09-21T09:31:08.320 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.320 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.320 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.321 INFO:tasks.workunit.client.0.smithi022.stdout: "rocksdb_write_wal_time": { 2024-09-21T09:31:08.321 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.321 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.321 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.321 INFO:tasks.workunit.client.0.smithi022.stdout: "submit_latency": { 2024-09-21T09:31:08.321 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.321 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.321 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.321 INFO:tasks.workunit.client.0.smithi022.stdout: "submit_sync_latency": { 2024-09-21T09:31:08.321 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 1747, 2024-09-21T09:31:08.321 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 209900578 2024-09-21T09:31:08.322 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.322 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.322 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.322 INFO:tasks.workunit.client.0.smithi022.stdout: "mon.aa5e831b31d98e3163af5edfeb921f9af32463c5": { 2024-09-21T09:31:08.322 INFO:tasks.workunit.client.0.smithi022.stdout: "mon": { 2024-09-21T09:31:08.322 INFO:tasks.workunit.client.0.smithi022.stdout: "election_call": { 2024-09-21T09:31:08.322 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2 2024-09-21T09:31:08.322 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.322 INFO:tasks.workunit.client.0.smithi022.stdout: "election_lose": { 2024-09-21T09:31:08.322 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2 2024-09-21T09:31:08.322 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.322 INFO:tasks.workunit.client.0.smithi022.stdout: "election_win": { 2024-09-21T09:31:08.322 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.322 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.323 INFO:tasks.workunit.client.0.smithi022.stdout: "num_elections": { 2024-09-21T09:31:08.323 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 7 2024-09-21T09:31:08.323 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.323 INFO:tasks.workunit.client.0.smithi022.stdout: "num_sessions": { 2024-09-21T09:31:08.323 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 10 2024-09-21T09:31:08.323 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.323 INFO:tasks.workunit.client.0.smithi022.stdout: "session_add": { 2024-09-21T09:31:08.323 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 4 2024-09-21T09:31:08.323 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.323 INFO:tasks.workunit.client.0.smithi022.stdout: "session_rm": { 2024-09-21T09:31:08.323 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 51 2024-09-21T09:31:08.323 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.323 INFO:tasks.workunit.client.0.smithi022.stdout: "session_trim": { 2024-09-21T09:31:08.324 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.324 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.324 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.324 INFO:tasks.workunit.client.0.smithi022.stdout: "paxos": { 2024-09-21T09:31:08.324 INFO:tasks.workunit.client.0.smithi022.stdout: "accept_timeout": { 2024-09-21T09:31:08.324 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.324 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.324 INFO:tasks.workunit.client.0.smithi022.stdout: "begin": { 2024-09-21T09:31:08.324 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 823 2024-09-21T09:31:08.324 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.324 INFO:tasks.workunit.client.0.smithi022.stdout: "begin_bytes": { 2024-09-21T09:31:08.324 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 823, 2024-09-21T09:31:08.325 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 3913382 2024-09-21T09:31:08.325 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.325 INFO:tasks.workunit.client.0.smithi022.stdout: "begin_keys": { 2024-09-21T09:31:08.325 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.325 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.325 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.325 INFO:tasks.workunit.client.0.smithi022.stdout: "begin_latency": { 2024-09-21T09:31:08.325 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 823, 2024-09-21T09:31:08.325 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 126997490 2024-09-21T09:31:08.325 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.325 INFO:tasks.workunit.client.0.smithi022.stdout: "collect": { 2024-09-21T09:31:08.325 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2 2024-09-21T09:31:08.325 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.325 INFO:tasks.workunit.client.0.smithi022.stdout: "collect_bytes": { 2024-09-21T09:31:08.326 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 2, 2024-09-21T09:31:08.326 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 146 2024-09-21T09:31:08.326 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.326 INFO:tasks.workunit.client.0.smithi022.stdout: "collect_keys": { 2024-09-21T09:31:08.326 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 2, 2024-09-21T09:31:08.326 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2 2024-09-21T09:31:08.326 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.326 INFO:tasks.workunit.client.0.smithi022.stdout: "collect_latency": { 2024-09-21T09:31:08.326 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 2, 2024-09-21T09:31:08.326 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.326 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.326 INFO:tasks.workunit.client.0.smithi022.stdout: "collect_timeout": { 2024-09-21T09:31:08.326 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.326 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.327 INFO:tasks.workunit.client.0.smithi022.stdout: "collect_uncommitted": { 2024-09-21T09:31:08.327 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.327 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.327 INFO:tasks.workunit.client.0.smithi022.stdout: "commit": { 2024-09-21T09:31:08.327 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 824 2024-09-21T09:31:08.327 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.327 INFO:tasks.workunit.client.0.smithi022.stdout: "commit_bytes": { 2024-09-21T09:31:08.327 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.327 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.327 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.327 INFO:tasks.workunit.client.0.smithi022.stdout: "commit_keys": { 2024-09-21T09:31:08.327 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.327 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.328 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.328 INFO:tasks.workunit.client.0.smithi022.stdout: "commit_latency": { 2024-09-21T09:31:08.328 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.328 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.328 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.328 INFO:tasks.workunit.client.0.smithi022.stdout: "lease_ack_timeout": { 2024-09-21T09:31:08.328 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.328 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.328 INFO:tasks.workunit.client.0.smithi022.stdout: "lease_timeout": { 2024-09-21T09:31:08.328 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.328 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.328 INFO:tasks.workunit.client.0.smithi022.stdout: "new_pn": { 2024-09-21T09:31:08.328 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.328 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.329 INFO:tasks.workunit.client.0.smithi022.stdout: "new_pn_latency": { 2024-09-21T09:31:08.329 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.329 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.329 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.329 INFO:tasks.workunit.client.0.smithi022.stdout: "refresh": { 2024-09-21T09:31:08.329 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 824 2024-09-21T09:31:08.329 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.329 INFO:tasks.workunit.client.0.smithi022.stdout: "refresh_latency": { 2024-09-21T09:31:08.329 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 824, 2024-09-21T09:31:08.329 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1861963206 2024-09-21T09:31:08.329 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.329 INFO:tasks.workunit.client.0.smithi022.stdout: "restart": { 2024-09-21T09:31:08.329 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 13 2024-09-21T09:31:08.329 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.330 INFO:tasks.workunit.client.0.smithi022.stdout: "share_state": { 2024-09-21T09:31:08.330 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.330 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.330 INFO:tasks.workunit.client.0.smithi022.stdout: "share_state_bytes": { 2024-09-21T09:31:08.330 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.330 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.330 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.330 INFO:tasks.workunit.client.0.smithi022.stdout: "share_state_keys": { 2024-09-21T09:31:08.330 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.330 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.330 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.330 INFO:tasks.workunit.client.0.smithi022.stdout: "start_leader": { 2024-09-21T09:31:08.330 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.330 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.331 INFO:tasks.workunit.client.0.smithi022.stdout: "start_peon": { 2024-09-21T09:31:08.331 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2 2024-09-21T09:31:08.331 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.331 INFO:tasks.workunit.client.0.smithi022.stdout: "store_state": { 2024-09-21T09:31:08.331 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 824 2024-09-21T09:31:08.331 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.331 INFO:tasks.workunit.client.0.smithi022.stdout: "store_state_bytes": { 2024-09-21T09:31:08.331 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 824, 2024-09-21T09:31:08.331 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 7672206 2024-09-21T09:31:08.331 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.331 INFO:tasks.workunit.client.0.smithi022.stdout: "store_state_keys": { 2024-09-21T09:31:08.331 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 824, 2024-09-21T09:31:08.331 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 5552 2024-09-21T09:31:08.331 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.331 INFO:tasks.workunit.client.0.smithi022.stdout: "store_state_latency": { 2024-09-21T09:31:08.332 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 824, 2024-09-21T09:31:08.332 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 343993214 2024-09-21T09:31:08.332 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.332 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.332 INFO:tasks.workunit.client.0.smithi022.stdout: "prioritycache": { 2024-09-21T09:31:08.332 INFO:tasks.workunit.client.0.smithi022.stdout: "cache_bytes": { 2024-09-21T09:31:08.332 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1020054731 2024-09-21T09:31:08.332 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.332 INFO:tasks.workunit.client.0.smithi022.stdout: "heap_bytes": { 2024-09-21T09:31:08.332 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 49086464 2024-09-21T09:31:08.332 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.332 INFO:tasks.workunit.client.0.smithi022.stdout: "mapped_bytes": { 2024-09-21T09:31:08.332 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 32604160 2024-09-21T09:31:08.333 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.333 INFO:tasks.workunit.client.0.smithi022.stdout: "target_bytes": { 2024-09-21T09:31:08.333 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2147483648 2024-09-21T09:31:08.333 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.333 INFO:tasks.workunit.client.0.smithi022.stdout: "unmapped_bytes": { 2024-09-21T09:31:08.333 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 16482304 2024-09-21T09:31:08.333 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.333 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.333 INFO:tasks.workunit.client.0.smithi022.stdout: "prioritycache:full": { 2024-09-21T09:31:08.333 INFO:tasks.workunit.client.0.smithi022.stdout: "committed_bytes": { 2024-09-21T09:31:08.333 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 348127232 2024-09-21T09:31:08.333 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.333 INFO:tasks.workunit.client.0.smithi022.stdout: "pri0_bytes": { 2024-09-21T09:31:08.333 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.334 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.334 INFO:tasks.workunit.client.0.smithi022.stdout: "pri10_bytes": { 2024-09-21T09:31:08.334 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.334 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.334 INFO:tasks.workunit.client.0.smithi022.stdout: "pri11_bytes": { 2024-09-21T09:31:08.334 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 277021337 2024-09-21T09:31:08.334 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.334 INFO:tasks.workunit.client.0.smithi022.stdout: "pri1_bytes": { 2024-09-21T09:31:08.334 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 271552 2024-09-21T09:31:08.334 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.334 INFO:tasks.workunit.client.0.smithi022.stdout: "pri2_bytes": { 2024-09-21T09:31:08.334 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.334 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.335 INFO:tasks.workunit.client.0.smithi022.stdout: "pri3_bytes": { 2024-09-21T09:31:08.335 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.335 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.335 INFO:tasks.workunit.client.0.smithi022.stdout: "pri4_bytes": { 2024-09-21T09:31:08.335 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.335 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.335 INFO:tasks.workunit.client.0.smithi022.stdout: "pri5_bytes": { 2024-09-21T09:31:08.335 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.335 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.335 INFO:tasks.workunit.client.0.smithi022.stdout: "pri6_bytes": { 2024-09-21T09:31:08.335 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.335 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.335 INFO:tasks.workunit.client.0.smithi022.stdout: "pri7_bytes": { 2024-09-21T09:31:08.335 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.336 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.336 INFO:tasks.workunit.client.0.smithi022.stdout: "pri8_bytes": { 2024-09-21T09:31:08.336 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.336 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.336 INFO:tasks.workunit.client.0.smithi022.stdout: "pri9_bytes": { 2024-09-21T09:31:08.336 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.336 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.336 INFO:tasks.workunit.client.0.smithi022.stdout: "reserved_bytes": { 2024-09-21T09:31:08.336 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 70834343 2024-09-21T09:31:08.336 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.336 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.336 INFO:tasks.workunit.client.0.smithi022.stdout: "prioritycache:inc": { 2024-09-21T09:31:08.336 INFO:tasks.workunit.client.0.smithi022.stdout: "committed_bytes": { 2024-09-21T09:31:08.336 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 348127232 2024-09-21T09:31:08.337 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.337 INFO:tasks.workunit.client.0.smithi022.stdout: "pri0_bytes": { 2024-09-21T09:31:08.337 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.337 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.337 INFO:tasks.workunit.client.0.smithi022.stdout: "pri10_bytes": { 2024-09-21T09:31:08.337 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.337 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.337 INFO:tasks.workunit.client.0.smithi022.stdout: "pri11_bytes": { 2024-09-21T09:31:08.337 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 277021337 2024-09-21T09:31:08.337 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.337 INFO:tasks.workunit.client.0.smithi022.stdout: "pri1_bytes": { 2024-09-21T09:31:08.337 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 41245 2024-09-21T09:31:08.337 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.337 INFO:tasks.workunit.client.0.smithi022.stdout: "pri2_bytes": { 2024-09-21T09:31:08.338 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.338 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.338 INFO:tasks.workunit.client.0.smithi022.stdout: "pri3_bytes": { 2024-09-21T09:31:08.338 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.338 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.338 INFO:tasks.workunit.client.0.smithi022.stdout: "pri4_bytes": { 2024-09-21T09:31:08.338 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.338 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.338 INFO:tasks.workunit.client.0.smithi022.stdout: "pri5_bytes": { 2024-09-21T09:31:08.338 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.338 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.338 INFO:tasks.workunit.client.0.smithi022.stdout: "pri6_bytes": { 2024-09-21T09:31:08.339 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.339 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.339 INFO:tasks.workunit.client.0.smithi022.stdout: "pri7_bytes": { 2024-09-21T09:31:08.339 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.339 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.339 INFO:tasks.workunit.client.0.smithi022.stdout: "pri8_bytes": { 2024-09-21T09:31:08.339 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.339 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.339 INFO:tasks.workunit.client.0.smithi022.stdout: "pri9_bytes": { 2024-09-21T09:31:08.339 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.339 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.339 INFO:tasks.workunit.client.0.smithi022.stdout: "reserved_bytes": { 2024-09-21T09:31:08.340 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 71064650 2024-09-21T09:31:08.340 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.340 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.340 INFO:tasks.workunit.client.0.smithi022.stdout: "prioritycache:kv": { 2024-09-21T09:31:08.340 INFO:tasks.workunit.client.0.smithi022.stdout: "committed_bytes": { 2024-09-21T09:31:08.340 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 322961408 2024-09-21T09:31:08.340 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.340 INFO:tasks.workunit.client.0.smithi022.stdout: "pri0_bytes": { 2024-09-21T09:31:08.340 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 67108864 2024-09-21T09:31:08.340 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.340 INFO:tasks.workunit.client.0.smithi022.stdout: "pri10_bytes": { 2024-09-21T09:31:08.340 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.340 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.340 INFO:tasks.workunit.client.0.smithi022.stdout: "pri11_bytes": { 2024-09-21T09:31:08.341 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 184680891 2024-09-21T09:31:08.341 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.341 INFO:tasks.workunit.client.0.smithi022.stdout: "pri1_bytes": { 2024-09-21T09:31:08.341 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.341 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.341 INFO:tasks.workunit.client.0.smithi022.stdout: "pri2_bytes": { 2024-09-21T09:31:08.341 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.341 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.341 INFO:tasks.workunit.client.0.smithi022.stdout: "pri3_bytes": { 2024-09-21T09:31:08.341 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.341 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.341 INFO:tasks.workunit.client.0.smithi022.stdout: "pri4_bytes": { 2024-09-21T09:31:08.341 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.341 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.342 INFO:tasks.workunit.client.0.smithi022.stdout: "pri5_bytes": { 2024-09-21T09:31:08.342 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.342 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.342 INFO:tasks.workunit.client.0.smithi022.stdout: "pri6_bytes": { 2024-09-21T09:31:08.342 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.342 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.342 INFO:tasks.workunit.client.0.smithi022.stdout: "pri7_bytes": { 2024-09-21T09:31:08.342 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.342 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.342 INFO:tasks.workunit.client.0.smithi022.stdout: "pri8_bytes": { 2024-09-21T09:31:08.342 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.342 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.342 INFO:tasks.workunit.client.0.smithi022.stdout: "pri9_bytes": { 2024-09-21T09:31:08.342 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.343 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.343 INFO:tasks.workunit.client.0.smithi022.stdout: "reserved_bytes": { 2024-09-21T09:31:08.343 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 71171653 2024-09-21T09:31:08.343 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.343 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.343 INFO:tasks.workunit.client.0.smithi022.stdout: "rocksdb": { 2024-09-21T09:31:08.343 INFO:tasks.workunit.client.0.smithi022.stdout: "compact": { 2024-09-21T09:31:08.343 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.343 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.343 INFO:tasks.workunit.client.0.smithi022.stdout: "compact_completed": { 2024-09-21T09:31:08.343 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1 2024-09-21T09:31:08.343 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.343 INFO:tasks.workunit.client.0.smithi022.stdout: "compact_lasted": { 2024-09-21T09:31:08.344 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 29092821 2024-09-21T09:31:08.344 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.344 INFO:tasks.workunit.client.0.smithi022.stdout: "compact_queue_len": { 2024-09-21T09:31:08.344 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.344 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.344 INFO:tasks.workunit.client.0.smithi022.stdout: "compact_queue_merge": { 2024-09-21T09:31:08.344 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.344 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.344 INFO:tasks.workunit.client.0.smithi022.stdout: "compact_running": { 2024-09-21T09:31:08.344 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.344 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.344 INFO:tasks.workunit.client.0.smithi022.stdout: "get_latency": { 2024-09-21T09:31:08.344 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 35234, 2024-09-21T09:31:08.344 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 86559894 2024-09-21T09:31:08.344 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.345 INFO:tasks.workunit.client.0.smithi022.stdout: "rocksdb_write_delay_time": { 2024-09-21T09:31:08.345 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.345 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.345 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.345 INFO:tasks.workunit.client.0.smithi022.stdout: "rocksdb_write_memtable_time": { 2024-09-21T09:31:08.345 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.345 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.345 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.345 INFO:tasks.workunit.client.0.smithi022.stdout: "rocksdb_write_pre_and_post_time": { 2024-09-21T09:31:08.345 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.345 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.345 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.345 INFO:tasks.workunit.client.0.smithi022.stdout: "rocksdb_write_wal_time": { 2024-09-21T09:31:08.346 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.346 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.346 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.346 INFO:tasks.workunit.client.0.smithi022.stdout: "submit_latency": { 2024-09-21T09:31:08.346 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.346 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.346 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.346 INFO:tasks.workunit.client.0.smithi022.stdout: "submit_sync_latency": { 2024-09-21T09:31:08.346 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 1794, 2024-09-21T09:31:08.346 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 577083552 2024-09-21T09:31:08.346 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.346 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.346 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.346 INFO:tasks.workunit.client.0.smithi022.stdout: "mon.f85c107c3119ea450f9f41ee1907af1822a71ba3": { 2024-09-21T09:31:08.347 INFO:tasks.workunit.client.0.smithi022.stdout: "mon": { 2024-09-21T09:31:08.347 INFO:tasks.workunit.client.0.smithi022.stdout: "election_call": { 2024-09-21T09:31:08.347 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2 2024-09-21T09:31:08.347 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.347 INFO:tasks.workunit.client.0.smithi022.stdout: "election_lose": { 2024-09-21T09:31:08.347 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.347 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.347 INFO:tasks.workunit.client.0.smithi022.stdout: "election_win": { 2024-09-21T09:31:08.347 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 3 2024-09-21T09:31:08.347 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.347 INFO:tasks.workunit.client.0.smithi022.stdout: "num_elections": { 2024-09-21T09:31:08.347 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 9 2024-09-21T09:31:08.347 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.347 INFO:tasks.workunit.client.0.smithi022.stdout: "num_sessions": { 2024-09-21T09:31:08.348 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 8 2024-09-21T09:31:08.348 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.348 INFO:tasks.workunit.client.0.smithi022.stdout: "session_add": { 2024-09-21T09:31:08.348 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 3 2024-09-21T09:31:08.348 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.348 INFO:tasks.workunit.client.0.smithi022.stdout: "session_rm": { 2024-09-21T09:31:08.348 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 177 2024-09-21T09:31:08.348 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.348 INFO:tasks.workunit.client.0.smithi022.stdout: "session_trim": { 2024-09-21T09:31:08.348 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.348 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.348 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.348 INFO:tasks.workunit.client.0.smithi022.stdout: "paxos": { 2024-09-21T09:31:08.348 INFO:tasks.workunit.client.0.smithi022.stdout: "accept_timeout": { 2024-09-21T09:31:08.349 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.349 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.349 INFO:tasks.workunit.client.0.smithi022.stdout: "begin": { 2024-09-21T09:31:08.349 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 951 2024-09-21T09:31:08.349 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.349 INFO:tasks.workunit.client.0.smithi022.stdout: "begin_bytes": { 2024-09-21T09:31:08.349 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 951, 2024-09-21T09:31:08.349 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 4963289 2024-09-21T09:31:08.349 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.349 INFO:tasks.workunit.client.0.smithi022.stdout: "begin_keys": { 2024-09-21T09:31:08.349 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 951, 2024-09-21T09:31:08.349 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2853 2024-09-21T09:31:08.349 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.350 INFO:tasks.workunit.client.0.smithi022.stdout: "begin_latency": { 2024-09-21T09:31:08.350 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 951, 2024-09-21T09:31:08.350 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 138998270 2024-09-21T09:31:08.350 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.350 INFO:tasks.workunit.client.0.smithi022.stdout: "collect": { 2024-09-21T09:31:08.350 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.350 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.350 INFO:tasks.workunit.client.0.smithi022.stdout: "collect_bytes": { 2024-09-21T09:31:08.350 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.350 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.350 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.350 INFO:tasks.workunit.client.0.smithi022.stdout: "collect_keys": { 2024-09-21T09:31:08.350 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.350 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.351 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.351 INFO:tasks.workunit.client.0.smithi022.stdout: "collect_latency": { 2024-09-21T09:31:08.351 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.351 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.351 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.351 INFO:tasks.workunit.client.0.smithi022.stdout: "collect_timeout": { 2024-09-21T09:31:08.351 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.351 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.351 INFO:tasks.workunit.client.0.smithi022.stdout: "collect_uncommitted": { 2024-09-21T09:31:08.351 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.351 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.351 INFO:tasks.workunit.client.0.smithi022.stdout: "commit": { 2024-09-21T09:31:08.351 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 951 2024-09-21T09:31:08.351 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.352 INFO:tasks.workunit.client.0.smithi022.stdout: "commit_bytes": { 2024-09-21T09:31:08.352 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 951, 2024-09-21T09:31:08.352 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 4894875 2024-09-21T09:31:08.352 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.352 INFO:tasks.workunit.client.0.smithi022.stdout: "commit_keys": { 2024-09-21T09:31:08.352 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 951, 2024-09-21T09:31:08.352 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 5416 2024-09-21T09:31:08.352 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.352 INFO:tasks.workunit.client.0.smithi022.stdout: "commit_latency": { 2024-09-21T09:31:08.352 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 951, 2024-09-21T09:31:08.352 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 133493004 2024-09-21T09:31:08.352 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.352 INFO:tasks.workunit.client.0.smithi022.stdout: "lease_ack_timeout": { 2024-09-21T09:31:08.352 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.353 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.353 INFO:tasks.workunit.client.0.smithi022.stdout: "lease_timeout": { 2024-09-21T09:31:08.353 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.353 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.353 INFO:tasks.workunit.client.0.smithi022.stdout: "new_pn": { 2024-09-21T09:31:08.353 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2 2024-09-21T09:31:08.353 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.353 INFO:tasks.workunit.client.0.smithi022.stdout: "new_pn_latency": { 2024-09-21T09:31:08.353 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 2, 2024-09-21T09:31:08.353 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.353 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.353 INFO:tasks.workunit.client.0.smithi022.stdout: "refresh": { 2024-09-21T09:31:08.353 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 953 2024-09-21T09:31:08.354 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.354 INFO:tasks.workunit.client.0.smithi022.stdout: "refresh_latency": { 2024-09-21T09:31:08.354 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 953, 2024-09-21T09:31:08.354 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2276971617 2024-09-21T09:31:08.354 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.354 INFO:tasks.workunit.client.0.smithi022.stdout: "restart": { 2024-09-21T09:31:08.354 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 12 2024-09-21T09:31:08.354 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.354 INFO:tasks.workunit.client.0.smithi022.stdout: "share_state": { 2024-09-21T09:31:08.354 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 3 2024-09-21T09:31:08.354 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.354 INFO:tasks.workunit.client.0.smithi022.stdout: "share_state_bytes": { 2024-09-21T09:31:08.354 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 3, 2024-09-21T09:31:08.354 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1627 2024-09-21T09:31:08.354 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.355 INFO:tasks.workunit.client.0.smithi022.stdout: "share_state_keys": { 2024-09-21T09:31:08.355 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 3, 2024-09-21T09:31:08.355 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 3 2024-09-21T09:31:08.355 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.355 INFO:tasks.workunit.client.0.smithi022.stdout: "start_leader": { 2024-09-21T09:31:08.355 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 3 2024-09-21T09:31:08.355 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.355 INFO:tasks.workunit.client.0.smithi022.stdout: "start_peon": { 2024-09-21T09:31:08.355 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.355 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.355 INFO:tasks.workunit.client.0.smithi022.stdout: "store_state": { 2024-09-21T09:31:08.355 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.355 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.355 INFO:tasks.workunit.client.0.smithi022.stdout: "store_state_bytes": { 2024-09-21T09:31:08.356 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.356 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.356 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.356 INFO:tasks.workunit.client.0.smithi022.stdout: "store_state_keys": { 2024-09-21T09:31:08.356 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.356 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.356 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.356 INFO:tasks.workunit.client.0.smithi022.stdout: "store_state_latency": { 2024-09-21T09:31:08.356 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.356 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.356 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.356 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.356 INFO:tasks.workunit.client.0.smithi022.stdout: "prioritycache": { 2024-09-21T09:31:08.357 INFO:tasks.workunit.client.0.smithi022.stdout: "cache_bytes": { 2024-09-21T09:31:08.357 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1020054731 2024-09-21T09:31:08.357 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.357 INFO:tasks.workunit.client.0.smithi022.stdout: "heap_bytes": { 2024-09-21T09:31:08.357 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 48979968 2024-09-21T09:31:08.357 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.357 INFO:tasks.workunit.client.0.smithi022.stdout: "mapped_bytes": { 2024-09-21T09:31:08.357 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 40542208 2024-09-21T09:31:08.357 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.357 INFO:tasks.workunit.client.0.smithi022.stdout: "target_bytes": { 2024-09-21T09:31:08.357 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2147483648 2024-09-21T09:31:08.357 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.357 INFO:tasks.workunit.client.0.smithi022.stdout: "unmapped_bytes": { 2024-09-21T09:31:08.358 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 8437760 2024-09-21T09:31:08.358 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.358 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.358 INFO:tasks.workunit.client.0.smithi022.stdout: "prioritycache:full": { 2024-09-21T09:31:08.358 INFO:tasks.workunit.client.0.smithi022.stdout: "committed_bytes": { 2024-09-21T09:31:08.358 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 348127232 2024-09-21T09:31:08.358 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.358 INFO:tasks.workunit.client.0.smithi022.stdout: "pri0_bytes": { 2024-09-21T09:31:08.358 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.358 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.358 INFO:tasks.workunit.client.0.smithi022.stdout: "pri10_bytes": { 2024-09-21T09:31:08.358 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.358 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.358 INFO:tasks.workunit.client.0.smithi022.stdout: "pri11_bytes": { 2024-09-21T09:31:08.359 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 277022399 2024-09-21T09:31:08.359 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.359 INFO:tasks.workunit.client.0.smithi022.stdout: "pri1_bytes": { 2024-09-21T09:31:08.359 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 271122 2024-09-21T09:31:08.359 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.359 INFO:tasks.workunit.client.0.smithi022.stdout: "pri2_bytes": { 2024-09-21T09:31:08.359 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.359 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.359 INFO:tasks.workunit.client.0.smithi022.stdout: "pri3_bytes": { 2024-09-21T09:31:08.359 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.359 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.359 INFO:tasks.workunit.client.0.smithi022.stdout: "pri4_bytes": { 2024-09-21T09:31:08.360 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.360 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.360 INFO:tasks.workunit.client.0.smithi022.stdout: "pri5_bytes": { 2024-09-21T09:31:08.360 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.360 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.360 INFO:tasks.workunit.client.0.smithi022.stdout: "pri6_bytes": { 2024-09-21T09:31:08.360 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.360 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.360 INFO:tasks.workunit.client.0.smithi022.stdout: "pri7_bytes": { 2024-09-21T09:31:08.360 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.360 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.360 INFO:tasks.workunit.client.0.smithi022.stdout: "pri8_bytes": { 2024-09-21T09:31:08.360 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.360 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.361 INFO:tasks.workunit.client.0.smithi022.stdout: "pri9_bytes": { 2024-09-21T09:31:08.361 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.361 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.361 INFO:tasks.workunit.client.0.smithi022.stdout: "reserved_bytes": { 2024-09-21T09:31:08.361 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 70833711 2024-09-21T09:31:08.361 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.361 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.361 INFO:tasks.workunit.client.0.smithi022.stdout: "prioritycache:inc": { 2024-09-21T09:31:08.361 INFO:tasks.workunit.client.0.smithi022.stdout: "committed_bytes": { 2024-09-21T09:31:08.361 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 348127232 2024-09-21T09:31:08.361 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.361 INFO:tasks.workunit.client.0.smithi022.stdout: "pri0_bytes": { 2024-09-21T09:31:08.361 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.361 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.362 INFO:tasks.workunit.client.0.smithi022.stdout: "pri10_bytes": { 2024-09-21T09:31:08.362 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.362 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.362 INFO:tasks.workunit.client.0.smithi022.stdout: "pri11_bytes": { 2024-09-21T09:31:08.362 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 277022399 2024-09-21T09:31:08.362 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.362 INFO:tasks.workunit.client.0.smithi022.stdout: "pri1_bytes": { 2024-09-21T09:31:08.362 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 38842 2024-09-21T09:31:08.362 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.362 INFO:tasks.workunit.client.0.smithi022.stdout: "pri2_bytes": { 2024-09-21T09:31:08.362 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.362 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.362 INFO:tasks.workunit.client.0.smithi022.stdout: "pri3_bytes": { 2024-09-21T09:31:08.362 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.363 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.363 INFO:tasks.workunit.client.0.smithi022.stdout: "pri4_bytes": { 2024-09-21T09:31:08.363 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.363 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.363 INFO:tasks.workunit.client.0.smithi022.stdout: "pri5_bytes": { 2024-09-21T09:31:08.363 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.363 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.363 INFO:tasks.workunit.client.0.smithi022.stdout: "pri6_bytes": { 2024-09-21T09:31:08.363 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.363 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.363 INFO:tasks.workunit.client.0.smithi022.stdout: "pri7_bytes": { 2024-09-21T09:31:08.363 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.363 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.363 INFO:tasks.workunit.client.0.smithi022.stdout: "pri8_bytes": { 2024-09-21T09:31:08.364 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.364 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.364 INFO:tasks.workunit.client.0.smithi022.stdout: "pri9_bytes": { 2024-09-21T09:31:08.364 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.364 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.364 INFO:tasks.workunit.client.0.smithi022.stdout: "reserved_bytes": { 2024-09-21T09:31:08.364 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 71065991 2024-09-21T09:31:08.364 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.364 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.364 INFO:tasks.workunit.client.0.smithi022.stdout: "prioritycache:kv": { 2024-09-21T09:31:08.364 INFO:tasks.workunit.client.0.smithi022.stdout: "committed_bytes": { 2024-09-21T09:31:08.364 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 322961408 2024-09-21T09:31:08.364 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.364 INFO:tasks.workunit.client.0.smithi022.stdout: "pri0_bytes": { 2024-09-21T09:31:08.365 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 67108864 2024-09-21T09:31:08.365 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.365 INFO:tasks.workunit.client.0.smithi022.stdout: "pri10_bytes": { 2024-09-21T09:31:08.365 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.365 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.365 INFO:tasks.workunit.client.0.smithi022.stdout: "pri11_bytes": { 2024-09-21T09:31:08.365 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 184681599 2024-09-21T09:31:08.365 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.365 INFO:tasks.workunit.client.0.smithi022.stdout: "pri1_bytes": { 2024-09-21T09:31:08.365 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.365 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.365 INFO:tasks.workunit.client.0.smithi022.stdout: "pri2_bytes": { 2024-09-21T09:31:08.365 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.366 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.366 INFO:tasks.workunit.client.0.smithi022.stdout: "pri3_bytes": { 2024-09-21T09:31:08.366 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.366 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.366 INFO:tasks.workunit.client.0.smithi022.stdout: "pri4_bytes": { 2024-09-21T09:31:08.366 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.366 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.366 INFO:tasks.workunit.client.0.smithi022.stdout: "pri5_bytes": { 2024-09-21T09:31:08.366 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.366 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.366 INFO:tasks.workunit.client.0.smithi022.stdout: "pri6_bytes": { 2024-09-21T09:31:08.366 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.366 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.366 INFO:tasks.workunit.client.0.smithi022.stdout: "pri7_bytes": { 2024-09-21T09:31:08.367 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.367 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.367 INFO:tasks.workunit.client.0.smithi022.stdout: "pri8_bytes": { 2024-09-21T09:31:08.367 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.367 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.367 INFO:tasks.workunit.client.0.smithi022.stdout: "pri9_bytes": { 2024-09-21T09:31:08.367 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.367 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.367 INFO:tasks.workunit.client.0.smithi022.stdout: "reserved_bytes": { 2024-09-21T09:31:08.367 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 71170945 2024-09-21T09:31:08.367 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.367 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.367 INFO:tasks.workunit.client.0.smithi022.stdout: "rocksdb": { 2024-09-21T09:31:08.367 INFO:tasks.workunit.client.0.smithi022.stdout: "compact": { 2024-09-21T09:31:08.367 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.368 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.368 INFO:tasks.workunit.client.0.smithi022.stdout: "compact_completed": { 2024-09-21T09:31:08.368 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1 2024-09-21T09:31:08.368 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.368 INFO:tasks.workunit.client.0.smithi022.stdout: "compact_lasted": { 2024-09-21T09:31:08.368 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 24725695 2024-09-21T09:31:08.368 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.368 INFO:tasks.workunit.client.0.smithi022.stdout: "compact_queue_len": { 2024-09-21T09:31:08.368 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.368 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.368 INFO:tasks.workunit.client.0.smithi022.stdout: "compact_queue_merge": { 2024-09-21T09:31:08.368 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.368 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.369 INFO:tasks.workunit.client.0.smithi022.stdout: "compact_running": { 2024-09-21T09:31:08.369 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.369 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.369 INFO:tasks.workunit.client.0.smithi022.stdout: "get_latency": { 2024-09-21T09:31:08.369 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 42345, 2024-09-21T09:31:08.369 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 170203915 2024-09-21T09:31:08.369 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.369 INFO:tasks.workunit.client.0.smithi022.stdout: "rocksdb_write_delay_time": { 2024-09-21T09:31:08.369 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.369 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.369 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.369 INFO:tasks.workunit.client.0.smithi022.stdout: "rocksdb_write_memtable_time": { 2024-09-21T09:31:08.369 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.369 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.369 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.370 INFO:tasks.workunit.client.0.smithi022.stdout: "rocksdb_write_pre_and_post_time": { 2024-09-21T09:31:08.370 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.370 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.370 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.370 INFO:tasks.workunit.client.0.smithi022.stdout: "rocksdb_write_wal_time": { 2024-09-21T09:31:08.370 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.370 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.370 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.370 INFO:tasks.workunit.client.0.smithi022.stdout: "submit_latency": { 2024-09-21T09:31:08.370 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.370 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.370 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.370 INFO:tasks.workunit.client.0.smithi022.stdout: "submit_sync_latency": { 2024-09-21T09:31:08.370 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 2069, 2024-09-21T09:31:08.371 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 279929127 2024-09-21T09:31:08.371 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.371 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.371 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.371 INFO:tasks.workunit.client.0.smithi022.stdout: "osd.0": { 2024-09-21T09:31:08.371 INFO:tasks.workunit.client.0.smithi022.stdout: "bluefs": { 2024-09-21T09:31:08.371 INFO:tasks.workunit.client.0.smithi022.stdout: "alloc_slow_fallback": { 2024-09-21T09:31:08.371 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.371 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.371 INFO:tasks.workunit.client.0.smithi022.stdout: "alloc_slow_size_fallback": { 2024-09-21T09:31:08.371 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.371 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.371 INFO:tasks.workunit.client.0.smithi022.stdout: "alloc_unit_db": { 2024-09-21T09:31:08.371 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 65536 2024-09-21T09:31:08.372 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.372 INFO:tasks.workunit.client.0.smithi022.stdout: "alloc_unit_main": { 2024-09-21T09:31:08.372 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.372 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.372 INFO:tasks.workunit.client.0.smithi022.stdout: "alloc_unit_wal": { 2024-09-21T09:31:08.372 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.372 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.372 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes_written_slow": { 2024-09-21T09:31:08.372 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.372 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.372 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes_written_sst": { 2024-09-21T09:31:08.372 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 12288 2024-09-21T09:31:08.372 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.372 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes_written_wal": { 2024-09-21T09:31:08.373 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2215936 2024-09-21T09:31:08.373 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.373 INFO:tasks.workunit.client.0.smithi022.stdout: "compact_lat": { 2024-09-21T09:31:08.373 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.373 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.373 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.373 INFO:tasks.workunit.client.0.smithi022.stdout: "compact_lock_lat": { 2024-09-21T09:31:08.373 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.373 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.373 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.373 INFO:tasks.workunit.client.0.smithi022.stdout: "db_total_bytes": { 2024-09-21T09:31:08.373 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 95995035648 2024-09-21T09:31:08.373 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.373 INFO:tasks.workunit.client.0.smithi022.stdout: "db_used_bytes": { 2024-09-21T09:31:08.373 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 304480256 2024-09-21T09:31:08.374 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.374 INFO:tasks.workunit.client.0.smithi022.stdout: "log_bytes": { 2024-09-21T09:31:08.374 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1470464 2024-09-21T09:31:08.374 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.374 INFO:tasks.workunit.client.0.smithi022.stdout: "logged_bytes": { 2024-09-21T09:31:08.374 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1331200 2024-09-21T09:31:08.374 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.374 INFO:tasks.workunit.client.0.smithi022.stdout: "max_bytes_db": { 2024-09-21T09:31:08.374 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 312868864 2024-09-21T09:31:08.374 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.374 INFO:tasks.workunit.client.0.smithi022.stdout: "max_bytes_slow": { 2024-09-21T09:31:08.374 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.374 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.374 INFO:tasks.workunit.client.0.smithi022.stdout: "max_bytes_wal": { 2024-09-21T09:31:08.375 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.375 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.375 INFO:tasks.workunit.client.0.smithi022.stdout: "num_files": { 2024-09-21T09:31:08.375 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 14 2024-09-21T09:31:08.375 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.375 INFO:tasks.workunit.client.0.smithi022.stdout: "read_bytes": { 2024-09-21T09:31:08.375 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 195697 2024-09-21T09:31:08.375 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.375 INFO:tasks.workunit.client.0.smithi022.stdout: "read_count": { 2024-09-21T09:31:08.375 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 23 2024-09-21T09:31:08.375 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.375 INFO:tasks.workunit.client.0.smithi022.stdout: "read_disk_bytes": { 2024-09-21T09:31:08.375 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1269760 2024-09-21T09:31:08.376 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.376 INFO:tasks.workunit.client.0.smithi022.stdout: "read_disk_bytes_db": { 2024-09-21T09:31:08.376 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1273856 2024-09-21T09:31:08.376 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.376 INFO:tasks.workunit.client.0.smithi022.stdout: "read_disk_bytes_slow": { 2024-09-21T09:31:08.376 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.376 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.376 INFO:tasks.workunit.client.0.smithi022.stdout: "read_disk_bytes_wal": { 2024-09-21T09:31:08.376 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.376 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.376 INFO:tasks.workunit.client.0.smithi022.stdout: "read_disk_count": { 2024-09-21T09:31:08.376 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 15 2024-09-21T09:31:08.376 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.376 INFO:tasks.workunit.client.0.smithi022.stdout: "read_prefetch_bytes": { 2024-09-21T09:31:08.376 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 4789 2024-09-21T09:31:08.377 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.377 INFO:tasks.workunit.client.0.smithi022.stdout: "read_prefetch_count": { 2024-09-21T09:31:08.377 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 4 2024-09-21T09:31:08.377 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.377 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_buffer_bytes": { 2024-09-21T09:31:08.377 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 5589 2024-09-21T09:31:08.377 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.377 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_buffer_count": { 2024-09-21T09:31:08.377 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 24 2024-09-21T09:31:08.377 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.377 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_bytes": { 2024-09-21T09:31:08.377 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 5589 2024-09-21T09:31:08.377 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.377 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_count": { 2024-09-21T09:31:08.378 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 24 2024-09-21T09:31:08.378 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.378 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_disk_bytes": { 2024-09-21T09:31:08.378 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.378 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.378 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_disk_bytes_db": { 2024-09-21T09:31:08.378 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.378 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.378 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_disk_bytes_slow": { 2024-09-21T09:31:08.378 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.378 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.378 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_disk_bytes_wal": { 2024-09-21T09:31:08.378 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.379 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.379 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_disk_count": { 2024-09-21T09:31:08.379 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.379 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.379 INFO:tasks.workunit.client.0.smithi022.stdout: "slow_total_bytes": { 2024-09-21T09:31:08.379 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.379 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.379 INFO:tasks.workunit.client.0.smithi022.stdout: "slow_used_bytes": { 2024-09-21T09:31:08.379 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.379 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.379 INFO:tasks.workunit.client.0.smithi022.stdout: "wal_total_bytes": { 2024-09-21T09:31:08.379 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.380 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.380 INFO:tasks.workunit.client.0.smithi022.stdout: "wal_used_bytes": { 2024-09-21T09:31:08.380 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.380 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.380 INFO:tasks.workunit.client.0.smithi022.stdout: "write_bytes": { 2024-09-21T09:31:08.380 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 3620864 2024-09-21T09:31:08.380 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.380 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.380 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore": { 2024-09-21T09:31:08.380 INFO:tasks.workunit.client.0.smithi022.stdout: "alloc_unit": { 2024-09-21T09:31:08.380 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 4096 2024-09-21T09:31:08.380 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.380 INFO:tasks.workunit.client.0.smithi022.stdout: "allocated": { 2024-09-21T09:31:08.380 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1294336 2024-09-21T09:31:08.381 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.381 INFO:tasks.workunit.client.0.smithi022.stdout: "clist_lat": { 2024-09-21T09:31:08.381 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 6, 2024-09-21T09:31:08.381 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 268543 2024-09-21T09:31:08.381 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.381 INFO:tasks.workunit.client.0.smithi022.stdout: "compress_lat": { 2024-09-21T09:31:08.381 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.381 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.381 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.381 INFO:tasks.workunit.client.0.smithi022.stdout: "compressed": { 2024-09-21T09:31:08.381 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.381 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.381 INFO:tasks.workunit.client.0.smithi022.stdout: "compressed_allocated": { 2024-09-21T09:31:08.381 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.382 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.382 INFO:tasks.workunit.client.0.smithi022.stdout: "compressed_original": { 2024-09-21T09:31:08.382 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.382 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.382 INFO:tasks.workunit.client.0.smithi022.stdout: "csum_lat": { 2024-09-21T09:31:08.382 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 24, 2024-09-21T09:31:08.382 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 78025 2024-09-21T09:31:08.382 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.382 INFO:tasks.workunit.client.0.smithi022.stdout: "decompress_lat": { 2024-09-21T09:31:08.382 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.382 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.382 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.382 INFO:tasks.workunit.client.0.smithi022.stdout: "kv_commit_lat": { 2024-09-21T09:31:08.382 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 308, 2024-09-21T09:31:08.383 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 52145765 2024-09-21T09:31:08.383 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.383 INFO:tasks.workunit.client.0.smithi022.stdout: "kv_final_lat": { 2024-09-21T09:31:08.383 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 301, 2024-09-21T09:31:08.383 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 12656371 2024-09-21T09:31:08.383 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.383 INFO:tasks.workunit.client.0.smithi022.stdout: "kv_flush_lat": { 2024-09-21T09:31:08.383 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 308, 2024-09-21T09:31:08.383 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 394762 2024-09-21T09:31:08.383 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.383 INFO:tasks.workunit.client.0.smithi022.stdout: "kv_sync_lat": { 2024-09-21T09:31:08.383 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 308, 2024-09-21T09:31:08.383 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 52540527 2024-09-21T09:31:08.383 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.384 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_get_keys_lat": { 2024-09-21T09:31:08.384 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.384 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.384 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.384 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_get_values_lat": { 2024-09-21T09:31:08.384 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 1, 2024-09-21T09:31:08.384 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 49657 2024-09-21T09:31:08.384 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.384 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_lower_bound_lat": { 2024-09-21T09:31:08.384 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 5, 2024-09-21T09:31:08.384 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 13525 2024-09-21T09:31:08.384 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.384 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_next_lat": { 2024-09-21T09:31:08.384 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.385 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.385 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.385 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_seek_to_first_lat": { 2024-09-21T09:31:08.385 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.385 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.385 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.385 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_upper_bound_lat": { 2024-09-21T09:31:08.385 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 4, 2024-09-21T09:31:08.385 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 12081 2024-09-21T09:31:08.385 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.385 INFO:tasks.workunit.client.0.smithi022.stdout: "onode_hits": { 2024-09-21T09:31:08.385 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 672 2024-09-21T09:31:08.385 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.386 INFO:tasks.workunit.client.0.smithi022.stdout: "onode_misses": { 2024-09-21T09:31:08.386 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 220 2024-09-21T09:31:08.386 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.386 INFO:tasks.workunit.client.0.smithi022.stdout: "read_lat": { 2024-09-21T09:31:08.386 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 105, 2024-09-21T09:31:08.386 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1662758 2024-09-21T09:31:08.386 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.386 INFO:tasks.workunit.client.0.smithi022.stdout: "read_onode_meta_lat": { 2024-09-21T09:31:08.386 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 253, 2024-09-21T09:31:08.386 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 242000 2024-09-21T09:31:08.386 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.386 INFO:tasks.workunit.client.0.smithi022.stdout: "read_wait_aio_lat": { 2024-09-21T09:31:08.386 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 148, 2024-09-21T09:31:08.387 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1037316 2024-09-21T09:31:08.387 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.387 INFO:tasks.workunit.client.0.smithi022.stdout: "reads_with_retries": { 2024-09-21T09:31:08.387 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.387 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.387 INFO:tasks.workunit.client.0.smithi022.stdout: "remove_lat": { 2024-09-21T09:31:08.387 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 3, 2024-09-21T09:31:08.387 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 63010 2024-09-21T09:31:08.387 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.387 INFO:tasks.workunit.client.0.smithi022.stdout: "state_aio_wait_lat": { 2024-09-21T09:31:08.387 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 417, 2024-09-21T09:31:08.387 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 5786609 2024-09-21T09:31:08.387 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.387 INFO:tasks.workunit.client.0.smithi022.stdout: "state_deferred_aio_wait_lat": { 2024-09-21T09:31:08.388 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 98, 2024-09-21T09:31:08.388 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 7343840 2024-09-21T09:31:08.388 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.388 INFO:tasks.workunit.client.0.smithi022.stdout: "state_deferred_cleanup_lat": { 2024-09-21T09:31:08.388 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 97, 2024-09-21T09:31:08.388 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 416582490984 2024-09-21T09:31:08.388 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.388 INFO:tasks.workunit.client.0.smithi022.stdout: "state_deferred_queued_lat": { 2024-09-21T09:31:08.388 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 98, 2024-09-21T09:31:08.388 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 157672872424 2024-09-21T09:31:08.388 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.388 INFO:tasks.workunit.client.0.smithi022.stdout: "state_done_lat": { 2024-09-21T09:31:08.388 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 416, 2024-09-21T09:31:08.388 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 77598915152 2024-09-21T09:31:08.389 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.389 INFO:tasks.workunit.client.0.smithi022.stdout: "state_finishing_lat": { 2024-09-21T09:31:08.389 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 416, 2024-09-21T09:31:08.389 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 43715 2024-09-21T09:31:08.389 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.389 INFO:tasks.workunit.client.0.smithi022.stdout: "state_io_done_lat": { 2024-09-21T09:31:08.389 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 417, 2024-09-21T09:31:08.389 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 422297 2024-09-21T09:31:08.389 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.389 INFO:tasks.workunit.client.0.smithi022.stdout: "state_kv_commiting_lat": { 2024-09-21T09:31:08.389 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 417, 2024-09-21T09:31:08.389 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 131736946 2024-09-21T09:31:08.389 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.390 INFO:tasks.workunit.client.0.smithi022.stdout: "state_kv_done_lat": { 2024-09-21T09:31:08.390 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 417, 2024-09-21T09:31:08.390 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 144824 2024-09-21T09:31:08.390 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.390 INFO:tasks.workunit.client.0.smithi022.stdout: "state_kv_queued_lat": { 2024-09-21T09:31:08.390 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 417, 2024-09-21T09:31:08.390 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 102226352 2024-09-21T09:31:08.390 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.390 INFO:tasks.workunit.client.0.smithi022.stdout: "state_prepare_lat": { 2024-09-21T09:31:08.390 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 417, 2024-09-21T09:31:08.390 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 20572631 2024-09-21T09:31:08.390 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.390 INFO:tasks.workunit.client.0.smithi022.stdout: "stored": { 2024-09-21T09:31:08.390 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 900748 2024-09-21T09:31:08.391 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.391 INFO:tasks.workunit.client.0.smithi022.stdout: "truncate_lat": { 2024-09-21T09:31:08.391 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 58, 2024-09-21T09:31:08.391 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 305527 2024-09-21T09:31:08.391 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.391 INFO:tasks.workunit.client.0.smithi022.stdout: "txc_commit_lat": { 2024-09-21T09:31:08.391 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 417, 2024-09-21T09:31:08.391 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 260803531 2024-09-21T09:31:08.391 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.391 INFO:tasks.workunit.client.0.smithi022.stdout: "txc_submit_lat": { 2024-09-21T09:31:08.391 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 417, 2024-09-21T09:31:08.391 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 24307691 2024-09-21T09:31:08.391 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.391 INFO:tasks.workunit.client.0.smithi022.stdout: "txc_throttle_lat": { 2024-09-21T09:31:08.391 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 417, 2024-09-21T09:31:08.392 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 3885809 2024-09-21T09:31:08.392 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.392 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.392 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore-pricache": { 2024-09-21T09:31:08.392 INFO:tasks.workunit.client.0.smithi022.stdout: "cache_bytes": { 2024-09-21T09:31:08.392 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1862400347 2024-09-21T09:31:08.392 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.392 INFO:tasks.workunit.client.0.smithi022.stdout: "heap_bytes": { 2024-09-21T09:31:08.392 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 270909440 2024-09-21T09:31:08.392 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.392 INFO:tasks.workunit.client.0.smithi022.stdout: "mapped_bytes": { 2024-09-21T09:31:08.392 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 270516224 2024-09-21T09:31:08.392 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.393 INFO:tasks.workunit.client.0.smithi022.stdout: "target_bytes": { 2024-09-21T09:31:08.393 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 3138478489 2024-09-21T09:31:08.393 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.393 INFO:tasks.workunit.client.0.smithi022.stdout: "unmapped_bytes": { 2024-09-21T09:31:08.393 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 393216 2024-09-21T09:31:08.393 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.393 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.393 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore-pricache:data": { 2024-09-21T09:31:08.393 INFO:tasks.workunit.client.0.smithi022.stdout: "committed_bytes": { 2024-09-21T09:31:08.393 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 159383552 2024-09-21T09:31:08.393 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.393 INFO:tasks.workunit.client.0.smithi022.stdout: "pri0_bytes": { 2024-09-21T09:31:08.393 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.393 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.394 INFO:tasks.workunit.client.0.smithi022.stdout: "pri10_bytes": { 2024-09-21T09:31:08.394 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.394 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.394 INFO:tasks.workunit.client.0.smithi022.stdout: "pri11_bytes": { 2024-09-21T09:31:08.394 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 85554152 2024-09-21T09:31:08.394 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.394 INFO:tasks.workunit.client.0.smithi022.stdout: "pri1_bytes": { 2024-09-21T09:31:08.394 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.394 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.394 INFO:tasks.workunit.client.0.smithi022.stdout: "pri2_bytes": { 2024-09-21T09:31:08.394 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.394 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.394 INFO:tasks.workunit.client.0.smithi022.stdout: "pri3_bytes": { 2024-09-21T09:31:08.394 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.395 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.395 INFO:tasks.workunit.client.0.smithi022.stdout: "pri4_bytes": { 2024-09-21T09:31:08.395 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.395 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.395 INFO:tasks.workunit.client.0.smithi022.stdout: "pri5_bytes": { 2024-09-21T09:31:08.395 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.395 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.395 INFO:tasks.workunit.client.0.smithi022.stdout: "pri6_bytes": { 2024-09-21T09:31:08.395 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 8192 2024-09-21T09:31:08.395 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.395 INFO:tasks.workunit.client.0.smithi022.stdout: "pri7_bytes": { 2024-09-21T09:31:08.395 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.395 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.395 INFO:tasks.workunit.client.0.smithi022.stdout: "pri8_bytes": { 2024-09-21T09:31:08.396 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.396 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.396 INFO:tasks.workunit.client.0.smithi022.stdout: "pri9_bytes": { 2024-09-21T09:31:08.396 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.396 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.396 INFO:tasks.workunit.client.0.smithi022.stdout: "reserved_bytes": { 2024-09-21T09:31:08.396 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 73821208 2024-09-21T09:31:08.396 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.396 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.396 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore-pricache:kv": { 2024-09-21T09:31:08.396 INFO:tasks.workunit.client.0.smithi022.stdout: "committed_bytes": { 2024-09-21T09:31:08.396 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 780140544 2024-09-21T09:31:08.396 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.396 INFO:tasks.workunit.client.0.smithi022.stdout: "pri0_bytes": { 2024-09-21T09:31:08.397 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 67108864 2024-09-21T09:31:08.397 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.397 INFO:tasks.workunit.client.0.smithi022.stdout: "pri10_bytes": { 2024-09-21T09:31:08.397 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.397 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.397 INFO:tasks.workunit.client.0.smithi022.stdout: "pri11_bytes": { 2024-09-21T09:31:08.397 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 641656144 2024-09-21T09:31:08.397 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.397 INFO:tasks.workunit.client.0.smithi022.stdout: "pri1_bytes": { 2024-09-21T09:31:08.397 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.397 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.397 INFO:tasks.workunit.client.0.smithi022.stdout: "pri2_bytes": { 2024-09-21T09:31:08.397 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.397 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.398 INFO:tasks.workunit.client.0.smithi022.stdout: "pri3_bytes": { 2024-09-21T09:31:08.398 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.398 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.398 INFO:tasks.workunit.client.0.smithi022.stdout: "pri4_bytes": { 2024-09-21T09:31:08.398 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.398 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.398 INFO:tasks.workunit.client.0.smithi022.stdout: "pri5_bytes": { 2024-09-21T09:31:08.398 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.398 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.398 INFO:tasks.workunit.client.0.smithi022.stdout: "pri6_bytes": { 2024-09-21T09:31:08.398 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2128 2024-09-21T09:31:08.398 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.398 INFO:tasks.workunit.client.0.smithi022.stdout: "pri7_bytes": { 2024-09-21T09:31:08.399 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.399 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.399 INFO:tasks.workunit.client.0.smithi022.stdout: "pri8_bytes": { 2024-09-21T09:31:08.399 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.399 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.399 INFO:tasks.workunit.client.0.smithi022.stdout: "pri9_bytes": { 2024-09-21T09:31:08.399 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.399 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.399 INFO:tasks.workunit.client.0.smithi022.stdout: "reserved_bytes": { 2024-09-21T09:31:08.399 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 71373408 2024-09-21T09:31:08.399 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.399 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.399 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore-pricache:kv_onode": { 2024-09-21T09:31:08.399 INFO:tasks.workunit.client.0.smithi022.stdout: "committed_bytes": { 2024-09-21T09:31:08.400 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 192937984 2024-09-21T09:31:08.400 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.400 INFO:tasks.workunit.client.0.smithi022.stdout: "pri0_bytes": { 2024-09-21T09:31:08.400 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 67108864 2024-09-21T09:31:08.400 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.400 INFO:tasks.workunit.client.0.smithi022.stdout: "pri10_bytes": { 2024-09-21T09:31:08.400 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 464 2024-09-21T09:31:08.400 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.400 INFO:tasks.workunit.client.0.smithi022.stdout: "pri11_bytes": { 2024-09-21T09:31:08.400 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 57036101 2024-09-21T09:31:08.400 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.400 INFO:tasks.workunit.client.0.smithi022.stdout: "pri1_bytes": { 2024-09-21T09:31:08.400 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.401 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.401 INFO:tasks.workunit.client.0.smithi022.stdout: "pri2_bytes": { 2024-09-21T09:31:08.401 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.401 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.401 INFO:tasks.workunit.client.0.smithi022.stdout: "pri3_bytes": { 2024-09-21T09:31:08.401 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.401 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.401 INFO:tasks.workunit.client.0.smithi022.stdout: "pri4_bytes": { 2024-09-21T09:31:08.401 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.401 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.401 INFO:tasks.workunit.client.0.smithi022.stdout: "pri5_bytes": { 2024-09-21T09:31:08.401 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.401 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.402 INFO:tasks.workunit.client.0.smithi022.stdout: "pri6_bytes": { 2024-09-21T09:31:08.402 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.402 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.402 INFO:tasks.workunit.client.0.smithi022.stdout: "pri7_bytes": { 2024-09-21T09:31:08.402 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.402 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.402 INFO:tasks.workunit.client.0.smithi022.stdout: "pri8_bytes": { 2024-09-21T09:31:08.402 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.402 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.402 INFO:tasks.workunit.client.0.smithi022.stdout: "pri9_bytes": { 2024-09-21T09:31:08.402 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.402 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.402 INFO:tasks.workunit.client.0.smithi022.stdout: "reserved_bytes": { 2024-09-21T09:31:08.403 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 68792555 2024-09-21T09:31:08.403 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.403 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.403 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore-pricache:meta": { 2024-09-21T09:31:08.403 INFO:tasks.workunit.client.0.smithi022.stdout: "committed_bytes": { 2024-09-21T09:31:08.403 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 713031680 2024-09-21T09:31:08.403 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.403 INFO:tasks.workunit.client.0.smithi022.stdout: "pri0_bytes": { 2024-09-21T09:31:08.403 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.403 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.403 INFO:tasks.workunit.client.0.smithi022.stdout: "pri10_bytes": { 2024-09-21T09:31:08.403 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.403 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.403 INFO:tasks.workunit.client.0.smithi022.stdout: "pri11_bytes": { 2024-09-21T09:31:08.404 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 641656144 2024-09-21T09:31:08.404 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.404 INFO:tasks.workunit.client.0.smithi022.stdout: "pri1_bytes": { 2024-09-21T09:31:08.404 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.404 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.404 INFO:tasks.workunit.client.0.smithi022.stdout: "pri2_bytes": { 2024-09-21T09:31:08.404 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.404 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.404 INFO:tasks.workunit.client.0.smithi022.stdout: "pri3_bytes": { 2024-09-21T09:31:08.404 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.404 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.404 INFO:tasks.workunit.client.0.smithi022.stdout: "pri4_bytes": { 2024-09-21T09:31:08.404 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.404 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.404 INFO:tasks.workunit.client.0.smithi022.stdout: "pri5_bytes": { 2024-09-21T09:31:08.405 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.405 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.405 INFO:tasks.workunit.client.0.smithi022.stdout: "pri6_bytes": { 2024-09-21T09:31:08.405 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 276551 2024-09-21T09:31:08.405 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.405 INFO:tasks.workunit.client.0.smithi022.stdout: "pri7_bytes": { 2024-09-21T09:31:08.405 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.405 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.405 INFO:tasks.workunit.client.0.smithi022.stdout: "pri8_bytes": { 2024-09-21T09:31:08.405 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.405 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.405 INFO:tasks.workunit.client.0.smithi022.stdout: "pri9_bytes": { 2024-09-21T09:31:08.405 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.405 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.406 INFO:tasks.workunit.client.0.smithi022.stdout: "reserved_bytes": { 2024-09-21T09:31:08.406 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 71098985 2024-09-21T09:31:08.406 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.406 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.406 INFO:tasks.workunit.client.0.smithi022.stdout: "objecter": { 2024-09-21T09:31:08.406 INFO:tasks.workunit.client.0.smithi022.stdout: "op_active": { 2024-09-21T09:31:08.406 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.406 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.406 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r": { 2024-09-21T09:31:08.406 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.406 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.406 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rmw": { 2024-09-21T09:31:08.406 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.406 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.407 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w": { 2024-09-21T09:31:08.407 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.407 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.407 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.407 INFO:tasks.workunit.client.0.smithi022.stdout: "osd": { 2024-09-21T09:31:08.407 INFO:tasks.workunit.client.0.smithi022.stdout: "numpg": { 2024-09-21T09:31:08.407 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 44 2024-09-21T09:31:08.407 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.407 INFO:tasks.workunit.client.0.smithi022.stdout: "numpg_removing": { 2024-09-21T09:31:08.407 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.407 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.407 INFO:tasks.workunit.client.0.smithi022.stdout: "op": { 2024-09-21T09:31:08.407 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 6 2024-09-21T09:31:08.407 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.408 INFO:tasks.workunit.client.0.smithi022.stdout: "op_delayed_degraded": { 2024-09-21T09:31:08.408 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.408 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.408 INFO:tasks.workunit.client.0.smithi022.stdout: "op_delayed_unreadable": { 2024-09-21T09:31:08.408 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.408 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.408 INFO:tasks.workunit.client.0.smithi022.stdout: "op_in_bytes": { 2024-09-21T09:31:08.408 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 604 2024-09-21T09:31:08.408 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.408 INFO:tasks.workunit.client.0.smithi022.stdout: "op_latency": { 2024-09-21T09:31:08.408 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 6, 2024-09-21T09:31:08.408 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 43758824 2024-09-21T09:31:08.408 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.408 INFO:tasks.workunit.client.0.smithi022.stdout: "op_out_bytes": { 2024-09-21T09:31:08.409 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.409 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.409 INFO:tasks.workunit.client.0.smithi022.stdout: "op_prepare_latency": { 2024-09-21T09:31:08.409 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 7, 2024-09-21T09:31:08.409 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 5335765 2024-09-21T09:31:08.409 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.409 INFO:tasks.workunit.client.0.smithi022.stdout: "op_process_latency": { 2024-09-21T09:31:08.409 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 6, 2024-09-21T09:31:08.409 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 13562427 2024-09-21T09:31:08.409 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.409 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r": { 2024-09-21T09:31:08.409 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.409 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.409 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r_latency": { 2024-09-21T09:31:08.409 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.410 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.410 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.410 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-21T09:31:08.410 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.410 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.410 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r_out_bytes": { 2024-09-21T09:31:08.410 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.410 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.410 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r_prepare_latency": { 2024-09-21T09:31:08.410 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.410 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.410 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.410 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r_process_latency": { 2024-09-21T09:31:08.411 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.411 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.411 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.411 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw": { 2024-09-21T09:31:08.411 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.411 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.411 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_in_bytes": { 2024-09-21T09:31:08.411 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.411 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.411 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_latency": { 2024-09-21T09:31:08.411 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.411 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.411 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.411 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-21T09:31:08.412 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.412 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.412 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-21T09:31:08.412 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.412 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.412 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_out_bytes": { 2024-09-21T09:31:08.412 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.412 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.412 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_prepare_latency": { 2024-09-21T09:31:08.412 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.412 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.412 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.412 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_process_latency": { 2024-09-21T09:31:08.412 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.413 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.413 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.413 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w": { 2024-09-21T09:31:08.413 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 6 2024-09-21T09:31:08.413 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.413 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w_in_bytes": { 2024-09-21T09:31:08.413 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 604 2024-09-21T09:31:08.413 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.413 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w_latency": { 2024-09-21T09:31:08.413 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 6, 2024-09-21T09:31:08.413 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 43758824 2024-09-21T09:31:08.413 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.413 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-21T09:31:08.413 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.414 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.414 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w_prepare_latency": { 2024-09-21T09:31:08.414 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 7, 2024-09-21T09:31:08.414 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 5335765 2024-09-21T09:31:08.414 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.414 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w_process_latency": { 2024-09-21T09:31:08.414 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 6, 2024-09-21T09:31:08.414 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 13562427 2024-09-21T09:31:08.414 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.414 INFO:tasks.workunit.client.0.smithi022.stdout: "op_wip": { 2024-09-21T09:31:08.414 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.414 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.414 INFO:tasks.workunit.client.0.smithi022.stdout: "recovery_bytes": { 2024-09-21T09:31:08.415 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.415 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.415 INFO:tasks.workunit.client.0.smithi022.stdout: "recovery_ops": { 2024-09-21T09:31:08.415 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.415 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.415 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_bytes": { 2024-09-21T09:31:08.415 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 95995035648 2024-09-21T09:31:08.415 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.415 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_bytes_used": { 2024-09-21T09:31:08.415 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 305786880 2024-09-21T09:31:08.415 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.415 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.415 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.415 INFO:tasks.workunit.client.0.smithi022.stdout: "osd.1": { 2024-09-21T09:31:08.415 INFO:tasks.workunit.client.0.smithi022.stdout: "bluefs": { 2024-09-21T09:31:08.416 INFO:tasks.workunit.client.0.smithi022.stdout: "alloc_slow_fallback": { 2024-09-21T09:31:08.416 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.416 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.416 INFO:tasks.workunit.client.0.smithi022.stdout: "alloc_slow_size_fallback": { 2024-09-21T09:31:08.416 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.416 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.416 INFO:tasks.workunit.client.0.smithi022.stdout: "alloc_unit_db": { 2024-09-21T09:31:08.416 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 65536 2024-09-21T09:31:08.416 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.416 INFO:tasks.workunit.client.0.smithi022.stdout: "alloc_unit_main": { 2024-09-21T09:31:08.416 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.416 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.416 INFO:tasks.workunit.client.0.smithi022.stdout: "alloc_unit_wal": { 2024-09-21T09:31:08.417 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.417 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.417 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes_written_slow": { 2024-09-21T09:31:08.417 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.417 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.417 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes_written_sst": { 2024-09-21T09:31:08.417 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 12288 2024-09-21T09:31:08.417 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.417 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes_written_wal": { 2024-09-21T09:31:08.417 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2113536 2024-09-21T09:31:08.417 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.417 INFO:tasks.workunit.client.0.smithi022.stdout: "compact_lat": { 2024-09-21T09:31:08.417 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.417 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.418 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.418 INFO:tasks.workunit.client.0.smithi022.stdout: "compact_lock_lat": { 2024-09-21T09:31:08.418 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.418 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.418 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.418 INFO:tasks.workunit.client.0.smithi022.stdout: "db_total_bytes": { 2024-09-21T09:31:08.418 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 95995035648 2024-09-21T09:31:08.418 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.418 INFO:tasks.workunit.client.0.smithi022.stdout: "db_used_bytes": { 2024-09-21T09:31:08.418 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 304480256 2024-09-21T09:31:08.418 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.418 INFO:tasks.workunit.client.0.smithi022.stdout: "log_bytes": { 2024-09-21T09:31:08.418 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1458176 2024-09-21T09:31:08.419 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.419 INFO:tasks.workunit.client.0.smithi022.stdout: "logged_bytes": { 2024-09-21T09:31:08.419 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1318912 2024-09-21T09:31:08.419 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.419 INFO:tasks.workunit.client.0.smithi022.stdout: "max_bytes_db": { 2024-09-21T09:31:08.419 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 312868864 2024-09-21T09:31:08.419 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.419 INFO:tasks.workunit.client.0.smithi022.stdout: "max_bytes_slow": { 2024-09-21T09:31:08.419 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.419 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.419 INFO:tasks.workunit.client.0.smithi022.stdout: "max_bytes_wal": { 2024-09-21T09:31:08.419 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.420 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.420 INFO:tasks.workunit.client.0.smithi022.stdout: "num_files": { 2024-09-21T09:31:08.420 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 14 2024-09-21T09:31:08.420 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.420 INFO:tasks.workunit.client.0.smithi022.stdout: "read_bytes": { 2024-09-21T09:31:08.420 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 195697 2024-09-21T09:31:08.420 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.420 INFO:tasks.workunit.client.0.smithi022.stdout: "read_count": { 2024-09-21T09:31:08.420 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 23 2024-09-21T09:31:08.420 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.420 INFO:tasks.workunit.client.0.smithi022.stdout: "read_disk_bytes": { 2024-09-21T09:31:08.420 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1269760 2024-09-21T09:31:08.420 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.420 INFO:tasks.workunit.client.0.smithi022.stdout: "read_disk_bytes_db": { 2024-09-21T09:31:08.421 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1273856 2024-09-21T09:31:08.421 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.421 INFO:tasks.workunit.client.0.smithi022.stdout: "read_disk_bytes_slow": { 2024-09-21T09:31:08.421 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.421 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.421 INFO:tasks.workunit.client.0.smithi022.stdout: "read_disk_bytes_wal": { 2024-09-21T09:31:08.421 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.421 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.421 INFO:tasks.workunit.client.0.smithi022.stdout: "read_disk_count": { 2024-09-21T09:31:08.421 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 15 2024-09-21T09:31:08.421 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.421 INFO:tasks.workunit.client.0.smithi022.stdout: "read_prefetch_bytes": { 2024-09-21T09:31:08.421 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 4789 2024-09-21T09:31:08.421 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.422 INFO:tasks.workunit.client.0.smithi022.stdout: "read_prefetch_count": { 2024-09-21T09:31:08.422 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 4 2024-09-21T09:31:08.422 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.422 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_buffer_bytes": { 2024-09-21T09:31:08.422 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 5589 2024-09-21T09:31:08.422 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.422 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_buffer_count": { 2024-09-21T09:31:08.422 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 24 2024-09-21T09:31:08.422 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.422 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_bytes": { 2024-09-21T09:31:08.422 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 5589 2024-09-21T09:31:08.422 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.422 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_count": { 2024-09-21T09:31:08.422 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 24 2024-09-21T09:31:08.423 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.423 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_disk_bytes": { 2024-09-21T09:31:08.423 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.423 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.423 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_disk_bytes_db": { 2024-09-21T09:31:08.423 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.423 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.423 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_disk_bytes_slow": { 2024-09-21T09:31:08.423 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.423 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.423 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_disk_bytes_wal": { 2024-09-21T09:31:08.423 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.423 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.423 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_disk_count": { 2024-09-21T09:31:08.424 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.424 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.424 INFO:tasks.workunit.client.0.smithi022.stdout: "slow_total_bytes": { 2024-09-21T09:31:08.424 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.424 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.424 INFO:tasks.workunit.client.0.smithi022.stdout: "slow_used_bytes": { 2024-09-21T09:31:08.424 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.424 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.424 INFO:tasks.workunit.client.0.smithi022.stdout: "wal_total_bytes": { 2024-09-21T09:31:08.424 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.424 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.424 INFO:tasks.workunit.client.0.smithi022.stdout: "wal_used_bytes": { 2024-09-21T09:31:08.424 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.424 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.425 INFO:tasks.workunit.client.0.smithi022.stdout: "write_bytes": { 2024-09-21T09:31:08.425 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 3506176 2024-09-21T09:31:08.425 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.425 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.425 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore": { 2024-09-21T09:31:08.425 INFO:tasks.workunit.client.0.smithi022.stdout: "alloc_unit": { 2024-09-21T09:31:08.425 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 4096 2024-09-21T09:31:08.425 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.425 INFO:tasks.workunit.client.0.smithi022.stdout: "allocated": { 2024-09-21T09:31:08.425 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 700416 2024-09-21T09:31:08.425 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.425 INFO:tasks.workunit.client.0.smithi022.stdout: "clist_lat": { 2024-09-21T09:31:08.425 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 5, 2024-09-21T09:31:08.425 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 212127 2024-09-21T09:31:08.425 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.426 INFO:tasks.workunit.client.0.smithi022.stdout: "compress_lat": { 2024-09-21T09:31:08.426 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.426 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.426 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.426 INFO:tasks.workunit.client.0.smithi022.stdout: "compressed": { 2024-09-21T09:31:08.426 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.426 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.426 INFO:tasks.workunit.client.0.smithi022.stdout: "compressed_allocated": { 2024-09-21T09:31:08.426 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.426 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.426 INFO:tasks.workunit.client.0.smithi022.stdout: "compressed_original": { 2024-09-21T09:31:08.426 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.426 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.426 INFO:tasks.workunit.client.0.smithi022.stdout: "csum_lat": { 2024-09-21T09:31:08.427 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 23, 2024-09-21T09:31:08.427 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 73883 2024-09-21T09:31:08.427 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.427 INFO:tasks.workunit.client.0.smithi022.stdout: "decompress_lat": { 2024-09-21T09:31:08.427 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.427 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.427 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.427 INFO:tasks.workunit.client.0.smithi022.stdout: "kv_commit_lat": { 2024-09-21T09:31:08.427 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 305, 2024-09-21T09:31:08.427 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 59745722 2024-09-21T09:31:08.427 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.427 INFO:tasks.workunit.client.0.smithi022.stdout: "kv_final_lat": { 2024-09-21T09:31:08.427 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 302, 2024-09-21T09:31:08.427 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 10574229 2024-09-21T09:31:08.428 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.428 INFO:tasks.workunit.client.0.smithi022.stdout: "kv_flush_lat": { 2024-09-21T09:31:08.428 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 305, 2024-09-21T09:31:08.428 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 434879 2024-09-21T09:31:08.428 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.428 INFO:tasks.workunit.client.0.smithi022.stdout: "kv_sync_lat": { 2024-09-21T09:31:08.428 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 305, 2024-09-21T09:31:08.428 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 60180601 2024-09-21T09:31:08.428 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.428 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_get_keys_lat": { 2024-09-21T09:31:08.428 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.428 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.428 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.428 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_get_values_lat": { 2024-09-21T09:31:08.429 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 3, 2024-09-21T09:31:08.429 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 38458 2024-09-21T09:31:08.429 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.429 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_lower_bound_lat": { 2024-09-21T09:31:08.429 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 5, 2024-09-21T09:31:08.429 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 10824 2024-09-21T09:31:08.429 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.429 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_next_lat": { 2024-09-21T09:31:08.429 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.429 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.429 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.429 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_seek_to_first_lat": { 2024-09-21T09:31:08.429 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.429 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.430 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.430 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_upper_bound_lat": { 2024-09-21T09:31:08.430 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 5, 2024-09-21T09:31:08.430 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 14996 2024-09-21T09:31:08.430 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.430 INFO:tasks.workunit.client.0.smithi022.stdout: "onode_hits": { 2024-09-21T09:31:08.430 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 685 2024-09-21T09:31:08.430 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.430 INFO:tasks.workunit.client.0.smithi022.stdout: "onode_misses": { 2024-09-21T09:31:08.430 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 224 2024-09-21T09:31:08.430 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.430 INFO:tasks.workunit.client.0.smithi022.stdout: "read_lat": { 2024-09-21T09:31:08.430 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 125, 2024-09-21T09:31:08.431 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1976819 2024-09-21T09:31:08.431 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.431 INFO:tasks.workunit.client.0.smithi022.stdout: "read_onode_meta_lat": { 2024-09-21T09:31:08.431 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 291, 2024-09-21T09:31:08.431 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 280425 2024-09-21T09:31:08.431 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.431 INFO:tasks.workunit.client.0.smithi022.stdout: "read_wait_aio_lat": { 2024-09-21T09:31:08.431 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 166, 2024-09-21T09:31:08.431 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1109474 2024-09-21T09:31:08.431 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.431 INFO:tasks.workunit.client.0.smithi022.stdout: "reads_with_retries": { 2024-09-21T09:31:08.431 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.431 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.431 INFO:tasks.workunit.client.0.smithi022.stdout: "remove_lat": { 2024-09-21T09:31:08.432 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 3, 2024-09-21T09:31:08.432 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 105215 2024-09-21T09:31:08.432 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.432 INFO:tasks.workunit.client.0.smithi022.stdout: "state_aio_wait_lat": { 2024-09-21T09:31:08.432 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 360, 2024-09-21T09:31:08.432 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 5779123 2024-09-21T09:31:08.432 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.432 INFO:tasks.workunit.client.0.smithi022.stdout: "state_deferred_aio_wait_lat": { 2024-09-21T09:31:08.432 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 96, 2024-09-21T09:31:08.432 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 6923216 2024-09-21T09:31:08.432 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.432 INFO:tasks.workunit.client.0.smithi022.stdout: "state_deferred_cleanup_lat": { 2024-09-21T09:31:08.432 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 95, 2024-09-21T09:31:08.432 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 405290222809 2024-09-21T09:31:08.432 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.433 INFO:tasks.workunit.client.0.smithi022.stdout: "state_deferred_queued_lat": { 2024-09-21T09:31:08.433 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 96, 2024-09-21T09:31:08.433 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 156744314046 2024-09-21T09:31:08.433 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.433 INFO:tasks.workunit.client.0.smithi022.stdout: "state_done_lat": { 2024-09-21T09:31:08.433 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 359, 2024-09-21T09:31:08.433 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 77402288748 2024-09-21T09:31:08.433 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.433 INFO:tasks.workunit.client.0.smithi022.stdout: "state_finishing_lat": { 2024-09-21T09:31:08.433 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 359, 2024-09-21T09:31:08.433 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 39606 2024-09-21T09:31:08.433 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.433 INFO:tasks.workunit.client.0.smithi022.stdout: "state_io_done_lat": { 2024-09-21T09:31:08.433 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 360, 2024-09-21T09:31:08.434 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 389361 2024-09-21T09:31:08.434 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.434 INFO:tasks.workunit.client.0.smithi022.stdout: "state_kv_commiting_lat": { 2024-09-21T09:31:08.434 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 360, 2024-09-21T09:31:08.434 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 95465222 2024-09-21T09:31:08.434 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.434 INFO:tasks.workunit.client.0.smithi022.stdout: "state_kv_done_lat": { 2024-09-21T09:31:08.434 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 360, 2024-09-21T09:31:08.434 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 149571 2024-09-21T09:31:08.434 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.434 INFO:tasks.workunit.client.0.smithi022.stdout: "state_kv_queued_lat": { 2024-09-21T09:31:08.434 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 360, 2024-09-21T09:31:08.434 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 46182165 2024-09-21T09:31:08.435 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.435 INFO:tasks.workunit.client.0.smithi022.stdout: "state_prepare_lat": { 2024-09-21T09:31:08.435 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 360, 2024-09-21T09:31:08.435 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 21284805 2024-09-21T09:31:08.435 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.435 INFO:tasks.workunit.client.0.smithi022.stdout: "stored": { 2024-09-21T09:31:08.435 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 310158 2024-09-21T09:31:08.435 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.435 INFO:tasks.workunit.client.0.smithi022.stdout: "truncate_lat": { 2024-09-21T09:31:08.435 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 55, 2024-09-21T09:31:08.435 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 318589 2024-09-21T09:31:08.435 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.435 INFO:tasks.workunit.client.0.smithi022.stdout: "txc_commit_lat": { 2024-09-21T09:31:08.435 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 360, 2024-09-21T09:31:08.435 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 169171709 2024-09-21T09:31:08.436 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.436 INFO:tasks.workunit.client.0.smithi022.stdout: "txc_submit_lat": { 2024-09-21T09:31:08.436 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 360, 2024-09-21T09:31:08.436 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 25744325 2024-09-21T09:31:08.436 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.436 INFO:tasks.workunit.client.0.smithi022.stdout: "txc_throttle_lat": { 2024-09-21T09:31:08.436 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 360, 2024-09-21T09:31:08.436 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 4315358 2024-09-21T09:31:08.436 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.436 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.436 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore-pricache": { 2024-09-21T09:31:08.436 INFO:tasks.workunit.client.0.smithi022.stdout: "cache_bytes": { 2024-09-21T09:31:08.436 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1862400347 2024-09-21T09:31:08.436 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.437 INFO:tasks.workunit.client.0.smithi022.stdout: "heap_bytes": { 2024-09-21T09:31:08.437 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 270974976 2024-09-21T09:31:08.437 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.437 INFO:tasks.workunit.client.0.smithi022.stdout: "mapped_bytes": { 2024-09-21T09:31:08.437 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 269991936 2024-09-21T09:31:08.437 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.437 INFO:tasks.workunit.client.0.smithi022.stdout: "target_bytes": { 2024-09-21T09:31:08.437 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 3138478489 2024-09-21T09:31:08.437 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.437 INFO:tasks.workunit.client.0.smithi022.stdout: "unmapped_bytes": { 2024-09-21T09:31:08.437 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 983040 2024-09-21T09:31:08.437 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.437 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.437 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore-pricache:data": { 2024-09-21T09:31:08.437 INFO:tasks.workunit.client.0.smithi022.stdout: "committed_bytes": { 2024-09-21T09:31:08.437 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 159383552 2024-09-21T09:31:08.438 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.438 INFO:tasks.workunit.client.0.smithi022.stdout: "pri0_bytes": { 2024-09-21T09:31:08.438 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.438 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.438 INFO:tasks.workunit.client.0.smithi022.stdout: "pri10_bytes": { 2024-09-21T09:31:08.438 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.438 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.438 INFO:tasks.workunit.client.0.smithi022.stdout: "pri11_bytes": { 2024-09-21T09:31:08.438 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 85555426 2024-09-21T09:31:08.438 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.438 INFO:tasks.workunit.client.0.smithi022.stdout: "pri1_bytes": { 2024-09-21T09:31:08.438 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.438 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.439 INFO:tasks.workunit.client.0.smithi022.stdout: "pri2_bytes": { 2024-09-21T09:31:08.439 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.439 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.439 INFO:tasks.workunit.client.0.smithi022.stdout: "pri3_bytes": { 2024-09-21T09:31:08.439 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.439 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.439 INFO:tasks.workunit.client.0.smithi022.stdout: "pri4_bytes": { 2024-09-21T09:31:08.439 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.439 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.439 INFO:tasks.workunit.client.0.smithi022.stdout: "pri5_bytes": { 2024-09-21T09:31:08.439 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.439 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.439 INFO:tasks.workunit.client.0.smithi022.stdout: "pri6_bytes": { 2024-09-21T09:31:08.440 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 8192 2024-09-21T09:31:08.440 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.440 INFO:tasks.workunit.client.0.smithi022.stdout: "pri7_bytes": { 2024-09-21T09:31:08.440 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.440 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.440 INFO:tasks.workunit.client.0.smithi022.stdout: "pri8_bytes": { 2024-09-21T09:31:08.440 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.440 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.440 INFO:tasks.workunit.client.0.smithi022.stdout: "pri9_bytes": { 2024-09-21T09:31:08.440 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.440 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.440 INFO:tasks.workunit.client.0.smithi022.stdout: "reserved_bytes": { 2024-09-21T09:31:08.440 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 73819934 2024-09-21T09:31:08.440 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.441 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.441 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore-pricache:kv": { 2024-09-21T09:31:08.441 INFO:tasks.workunit.client.0.smithi022.stdout: "committed_bytes": { 2024-09-21T09:31:08.441 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 780140544 2024-09-21T09:31:08.441 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.441 INFO:tasks.workunit.client.0.smithi022.stdout: "pri0_bytes": { 2024-09-21T09:31:08.441 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 67108864 2024-09-21T09:31:08.441 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.441 INFO:tasks.workunit.client.0.smithi022.stdout: "pri10_bytes": { 2024-09-21T09:31:08.441 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.441 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.441 INFO:tasks.workunit.client.0.smithi022.stdout: "pri11_bytes": { 2024-09-21T09:31:08.441 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 641665696 2024-09-21T09:31:08.441 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.441 INFO:tasks.workunit.client.0.smithi022.stdout: "pri1_bytes": { 2024-09-21T09:31:08.442 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.442 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.442 INFO:tasks.workunit.client.0.smithi022.stdout: "pri2_bytes": { 2024-09-21T09:31:08.442 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.442 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.442 INFO:tasks.workunit.client.0.smithi022.stdout: "pri3_bytes": { 2024-09-21T09:31:08.442 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.442 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.442 INFO:tasks.workunit.client.0.smithi022.stdout: "pri4_bytes": { 2024-09-21T09:31:08.442 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.442 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.442 INFO:tasks.workunit.client.0.smithi022.stdout: "pri5_bytes": { 2024-09-21T09:31:08.442 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.442 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.443 INFO:tasks.workunit.client.0.smithi022.stdout: "pri6_bytes": { 2024-09-21T09:31:08.443 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2128 2024-09-21T09:31:08.443 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.443 INFO:tasks.workunit.client.0.smithi022.stdout: "pri7_bytes": { 2024-09-21T09:31:08.443 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.443 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.443 INFO:tasks.workunit.client.0.smithi022.stdout: "pri8_bytes": { 2024-09-21T09:31:08.443 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.443 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.443 INFO:tasks.workunit.client.0.smithi022.stdout: "pri9_bytes": { 2024-09-21T09:31:08.443 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.443 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.443 INFO:tasks.workunit.client.0.smithi022.stdout: "reserved_bytes": { 2024-09-21T09:31:08.443 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 71363856 2024-09-21T09:31:08.443 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.444 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.444 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore-pricache:kv_onode": { 2024-09-21T09:31:08.444 INFO:tasks.workunit.client.0.smithi022.stdout: "committed_bytes": { 2024-09-21T09:31:08.444 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 192937984 2024-09-21T09:31:08.444 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.444 INFO:tasks.workunit.client.0.smithi022.stdout: "pri0_bytes": { 2024-09-21T09:31:08.444 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 67108864 2024-09-21T09:31:08.444 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.444 INFO:tasks.workunit.client.0.smithi022.stdout: "pri10_bytes": { 2024-09-21T09:31:08.444 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 464 2024-09-21T09:31:08.444 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.444 INFO:tasks.workunit.client.0.smithi022.stdout: "pri11_bytes": { 2024-09-21T09:31:08.444 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 57036950 2024-09-21T09:31:08.444 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.445 INFO:tasks.workunit.client.0.smithi022.stdout: "pri1_bytes": { 2024-09-21T09:31:08.445 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.445 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.445 INFO:tasks.workunit.client.0.smithi022.stdout: "pri2_bytes": { 2024-09-21T09:31:08.445 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.445 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.445 INFO:tasks.workunit.client.0.smithi022.stdout: "pri3_bytes": { 2024-09-21T09:31:08.445 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.445 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.445 INFO:tasks.workunit.client.0.smithi022.stdout: "pri4_bytes": { 2024-09-21T09:31:08.445 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.445 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.445 INFO:tasks.workunit.client.0.smithi022.stdout: "pri5_bytes": { 2024-09-21T09:31:08.446 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.446 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.446 INFO:tasks.workunit.client.0.smithi022.stdout: "pri6_bytes": { 2024-09-21T09:31:08.446 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.446 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.446 INFO:tasks.workunit.client.0.smithi022.stdout: "pri7_bytes": { 2024-09-21T09:31:08.446 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.446 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.446 INFO:tasks.workunit.client.0.smithi022.stdout: "pri8_bytes": { 2024-09-21T09:31:08.446 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.446 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.446 INFO:tasks.workunit.client.0.smithi022.stdout: "pri9_bytes": { 2024-09-21T09:31:08.446 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.446 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.447 INFO:tasks.workunit.client.0.smithi022.stdout: "reserved_bytes": { 2024-09-21T09:31:08.447 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 68791706 2024-09-21T09:31:08.447 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.447 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.447 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore-pricache:meta": { 2024-09-21T09:31:08.447 INFO:tasks.workunit.client.0.smithi022.stdout: "committed_bytes": { 2024-09-21T09:31:08.447 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 713031680 2024-09-21T09:31:08.447 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.447 INFO:tasks.workunit.client.0.smithi022.stdout: "pri0_bytes": { 2024-09-21T09:31:08.447 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.447 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.447 INFO:tasks.workunit.client.0.smithi022.stdout: "pri10_bytes": { 2024-09-21T09:31:08.447 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.447 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.448 INFO:tasks.workunit.client.0.smithi022.stdout: "pri11_bytes": { 2024-09-21T09:31:08.448 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 641665696 2024-09-21T09:31:08.448 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.448 INFO:tasks.workunit.client.0.smithi022.stdout: "pri1_bytes": { 2024-09-21T09:31:08.448 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.448 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.448 INFO:tasks.workunit.client.0.smithi022.stdout: "pri2_bytes": { 2024-09-21T09:31:08.448 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.448 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.448 INFO:tasks.workunit.client.0.smithi022.stdout: "pri3_bytes": { 2024-09-21T09:31:08.448 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.448 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.448 INFO:tasks.workunit.client.0.smithi022.stdout: "pri4_bytes": { 2024-09-21T09:31:08.448 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.449 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.449 INFO:tasks.workunit.client.0.smithi022.stdout: "pri5_bytes": { 2024-09-21T09:31:08.449 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.449 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.449 INFO:tasks.workunit.client.0.smithi022.stdout: "pri6_bytes": { 2024-09-21T09:31:08.449 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 255384 2024-09-21T09:31:08.449 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.449 INFO:tasks.workunit.client.0.smithi022.stdout: "pri7_bytes": { 2024-09-21T09:31:08.449 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.449 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.449 INFO:tasks.workunit.client.0.smithi022.stdout: "pri8_bytes": { 2024-09-21T09:31:08.449 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.449 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.449 INFO:tasks.workunit.client.0.smithi022.stdout: "pri9_bytes": { 2024-09-21T09:31:08.449 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.450 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.450 INFO:tasks.workunit.client.0.smithi022.stdout: "reserved_bytes": { 2024-09-21T09:31:08.450 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 71110600 2024-09-21T09:31:08.450 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.450 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.450 INFO:tasks.workunit.client.0.smithi022.stdout: "objecter": { 2024-09-21T09:31:08.450 INFO:tasks.workunit.client.0.smithi022.stdout: "op_active": { 2024-09-21T09:31:08.450 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.450 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.450 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r": { 2024-09-21T09:31:08.450 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.450 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.450 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rmw": { 2024-09-21T09:31:08.451 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.451 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.451 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w": { 2024-09-21T09:31:08.451 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.451 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.451 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.451 INFO:tasks.workunit.client.0.smithi022.stdout: "osd": { 2024-09-21T09:31:08.451 INFO:tasks.workunit.client.0.smithi022.stdout: "numpg": { 2024-09-21T09:31:08.451 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 38 2024-09-21T09:31:08.451 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.451 INFO:tasks.workunit.client.0.smithi022.stdout: "numpg_removing": { 2024-09-21T09:31:08.451 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.451 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.451 INFO:tasks.workunit.client.0.smithi022.stdout: "op": { 2024-09-21T09:31:08.451 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 97 2024-09-21T09:31:08.452 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.452 INFO:tasks.workunit.client.0.smithi022.stdout: "op_delayed_degraded": { 2024-09-21T09:31:08.452 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.452 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.452 INFO:tasks.workunit.client.0.smithi022.stdout: "op_delayed_unreadable": { 2024-09-21T09:31:08.452 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.452 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.452 INFO:tasks.workunit.client.0.smithi022.stdout: "op_in_bytes": { 2024-09-21T09:31:08.452 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1640827 2024-09-21T09:31:08.452 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.452 INFO:tasks.workunit.client.0.smithi022.stdout: "op_latency": { 2024-09-21T09:31:08.452 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 97, 2024-09-21T09:31:08.452 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 368569962 2024-09-21T09:31:08.452 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.453 INFO:tasks.workunit.client.0.smithi022.stdout: "op_out_bytes": { 2024-09-21T09:31:08.453 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 602 2024-09-21T09:31:08.453 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.453 INFO:tasks.workunit.client.0.smithi022.stdout: "op_prepare_latency": { 2024-09-21T09:31:08.453 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 97, 2024-09-21T09:31:08.453 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 33419604 2024-09-21T09:31:08.453 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.453 INFO:tasks.workunit.client.0.smithi022.stdout: "op_process_latency": { 2024-09-21T09:31:08.453 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 97, 2024-09-21T09:31:08.453 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 106861826 2024-09-21T09:31:08.453 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.453 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r": { 2024-09-21T09:31:08.453 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 18 2024-09-21T09:31:08.453 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.453 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r_latency": { 2024-09-21T09:31:08.454 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 18, 2024-09-21T09:31:08.454 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2701008 2024-09-21T09:31:08.454 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.454 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-21T09:31:08.454 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.454 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.454 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r_out_bytes": { 2024-09-21T09:31:08.454 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 602 2024-09-21T09:31:08.454 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.454 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r_prepare_latency": { 2024-09-21T09:31:08.454 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 18, 2024-09-21T09:31:08.454 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2305491 2024-09-21T09:31:08.454 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.454 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r_process_latency": { 2024-09-21T09:31:08.455 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 18, 2024-09-21T09:31:08.455 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1860842 2024-09-21T09:31:08.455 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.455 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw": { 2024-09-21T09:31:08.455 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 12 2024-09-21T09:31:08.455 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.455 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_in_bytes": { 2024-09-21T09:31:08.455 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.455 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.455 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_latency": { 2024-09-21T09:31:08.455 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 12, 2024-09-21T09:31:08.455 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 10826824 2024-09-21T09:31:08.455 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.455 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-21T09:31:08.456 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.456 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.456 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-21T09:31:08.456 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.456 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.456 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_out_bytes": { 2024-09-21T09:31:08.456 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.456 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.456 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_prepare_latency": { 2024-09-21T09:31:08.456 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 12, 2024-09-21T09:31:08.456 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 4695971 2024-09-21T09:31:08.456 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.456 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_process_latency": { 2024-09-21T09:31:08.456 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 12, 2024-09-21T09:31:08.456 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 10130626 2024-09-21T09:31:08.457 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.457 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w": { 2024-09-21T09:31:08.457 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 67 2024-09-21T09:31:08.457 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.457 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w_in_bytes": { 2024-09-21T09:31:08.457 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1640827 2024-09-21T09:31:08.457 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.457 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w_latency": { 2024-09-21T09:31:08.457 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 67, 2024-09-21T09:31:08.457 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 355042130 2024-09-21T09:31:08.457 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.457 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-21T09:31:08.457 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.457 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.458 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w_prepare_latency": { 2024-09-21T09:31:08.458 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 67, 2024-09-21T09:31:08.458 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 26418142 2024-09-21T09:31:08.458 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.458 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w_process_latency": { 2024-09-21T09:31:08.458 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 67, 2024-09-21T09:31:08.458 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 94870358 2024-09-21T09:31:08.458 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.458 INFO:tasks.workunit.client.0.smithi022.stdout: "op_wip": { 2024-09-21T09:31:08.458 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.458 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.458 INFO:tasks.workunit.client.0.smithi022.stdout: "recovery_bytes": { 2024-09-21T09:31:08.458 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 590368 2024-09-21T09:31:08.458 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.458 INFO:tasks.workunit.client.0.smithi022.stdout: "recovery_ops": { 2024-09-21T09:31:08.459 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2 2024-09-21T09:31:08.459 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.459 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_bytes": { 2024-09-21T09:31:08.459 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 95995035648 2024-09-21T09:31:08.459 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.459 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_bytes_used": { 2024-09-21T09:31:08.459 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 305192960 2024-09-21T09:31:08.459 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.459 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.459 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.459 INFO:tasks.workunit.client.0.smithi022.stdout: "osd.2": { 2024-09-21T09:31:08.459 INFO:tasks.workunit.client.0.smithi022.stdout: "bluefs": { 2024-09-21T09:31:08.459 INFO:tasks.workunit.client.0.smithi022.stdout: "alloc_slow_fallback": { 2024-09-21T09:31:08.459 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.459 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.460 INFO:tasks.workunit.client.0.smithi022.stdout: "alloc_slow_size_fallback": { 2024-09-21T09:31:08.460 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.460 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.460 INFO:tasks.workunit.client.0.smithi022.stdout: "alloc_unit_db": { 2024-09-21T09:31:08.460 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 65536 2024-09-21T09:31:08.460 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.460 INFO:tasks.workunit.client.0.smithi022.stdout: "alloc_unit_main": { 2024-09-21T09:31:08.460 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.460 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.460 INFO:tasks.workunit.client.0.smithi022.stdout: "alloc_unit_wal": { 2024-09-21T09:31:08.460 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.460 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.460 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes_written_slow": { 2024-09-21T09:31:08.460 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.461 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.461 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes_written_sst": { 2024-09-21T09:31:08.461 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 12288 2024-09-21T09:31:08.461 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.461 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes_written_wal": { 2024-09-21T09:31:08.461 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1703936 2024-09-21T09:31:08.461 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.461 INFO:tasks.workunit.client.0.smithi022.stdout: "compact_lat": { 2024-09-21T09:31:08.461 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.461 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.461 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.461 INFO:tasks.workunit.client.0.smithi022.stdout: "compact_lock_lat": { 2024-09-21T09:31:08.462 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.462 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.462 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.462 INFO:tasks.workunit.client.0.smithi022.stdout: "db_total_bytes": { 2024-09-21T09:31:08.462 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 95995035648 2024-09-21T09:31:08.462 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.462 INFO:tasks.workunit.client.0.smithi022.stdout: "db_used_bytes": { 2024-09-21T09:31:08.462 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 304480256 2024-09-21T09:31:08.462 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.462 INFO:tasks.workunit.client.0.smithi022.stdout: "log_bytes": { 2024-09-21T09:31:08.462 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1146880 2024-09-21T09:31:08.462 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.462 INFO:tasks.workunit.client.0.smithi022.stdout: "logged_bytes": { 2024-09-21T09:31:08.462 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1007616 2024-09-21T09:31:08.463 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.463 INFO:tasks.workunit.client.0.smithi022.stdout: "max_bytes_db": { 2024-09-21T09:31:08.463 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 312868864 2024-09-21T09:31:08.463 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.463 INFO:tasks.workunit.client.0.smithi022.stdout: "max_bytes_slow": { 2024-09-21T09:31:08.463 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.463 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.463 INFO:tasks.workunit.client.0.smithi022.stdout: "max_bytes_wal": { 2024-09-21T09:31:08.463 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.463 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.463 INFO:tasks.workunit.client.0.smithi022.stdout: "num_files": { 2024-09-21T09:31:08.463 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 14 2024-09-21T09:31:08.463 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.463 INFO:tasks.workunit.client.0.smithi022.stdout: "read_bytes": { 2024-09-21T09:31:08.464 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 195697 2024-09-21T09:31:08.464 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.464 INFO:tasks.workunit.client.0.smithi022.stdout: "read_count": { 2024-09-21T09:31:08.464 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 23 2024-09-21T09:31:08.464 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.464 INFO:tasks.workunit.client.0.smithi022.stdout: "read_disk_bytes": { 2024-09-21T09:31:08.464 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1269760 2024-09-21T09:31:08.464 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.464 INFO:tasks.workunit.client.0.smithi022.stdout: "read_disk_bytes_db": { 2024-09-21T09:31:08.464 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1273856 2024-09-21T09:31:08.464 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.464 INFO:tasks.workunit.client.0.smithi022.stdout: "read_disk_bytes_slow": { 2024-09-21T09:31:08.464 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.465 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.465 INFO:tasks.workunit.client.0.smithi022.stdout: "read_disk_bytes_wal": { 2024-09-21T09:31:08.465 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.465 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.465 INFO:tasks.workunit.client.0.smithi022.stdout: "read_disk_count": { 2024-09-21T09:31:08.465 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 15 2024-09-21T09:31:08.465 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.465 INFO:tasks.workunit.client.0.smithi022.stdout: "read_prefetch_bytes": { 2024-09-21T09:31:08.465 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 4789 2024-09-21T09:31:08.465 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.465 INFO:tasks.workunit.client.0.smithi022.stdout: "read_prefetch_count": { 2024-09-21T09:31:08.465 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 4 2024-09-21T09:31:08.465 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.466 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_buffer_bytes": { 2024-09-21T09:31:08.466 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 5589 2024-09-21T09:31:08.466 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.466 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_buffer_count": { 2024-09-21T09:31:08.466 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 24 2024-09-21T09:31:08.466 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.466 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_bytes": { 2024-09-21T09:31:08.466 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 5589 2024-09-21T09:31:08.466 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.466 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_count": { 2024-09-21T09:31:08.466 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 24 2024-09-21T09:31:08.466 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.466 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_disk_bytes": { 2024-09-21T09:31:08.466 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.467 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.467 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_disk_bytes_db": { 2024-09-21T09:31:08.467 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.467 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.467 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_disk_bytes_slow": { 2024-09-21T09:31:08.467 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.467 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.467 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_disk_bytes_wal": { 2024-09-21T09:31:08.467 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.467 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.467 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_disk_count": { 2024-09-21T09:31:08.467 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.467 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.467 INFO:tasks.workunit.client.0.smithi022.stdout: "slow_total_bytes": { 2024-09-21T09:31:08.468 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.468 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.468 INFO:tasks.workunit.client.0.smithi022.stdout: "slow_used_bytes": { 2024-09-21T09:31:08.468 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.468 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.468 INFO:tasks.workunit.client.0.smithi022.stdout: "wal_total_bytes": { 2024-09-21T09:31:08.468 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.468 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.468 INFO:tasks.workunit.client.0.smithi022.stdout: "wal_used_bytes": { 2024-09-21T09:31:08.468 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.468 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.468 INFO:tasks.workunit.client.0.smithi022.stdout: "write_bytes": { 2024-09-21T09:31:08.468 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2785280 2024-09-21T09:31:08.468 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.468 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.469 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore": { 2024-09-21T09:31:08.469 INFO:tasks.workunit.client.0.smithi022.stdout: "alloc_unit": { 2024-09-21T09:31:08.469 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 4096 2024-09-21T09:31:08.469 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.469 INFO:tasks.workunit.client.0.smithi022.stdout: "allocated": { 2024-09-21T09:31:08.469 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 688128 2024-09-21T09:31:08.469 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.469 INFO:tasks.workunit.client.0.smithi022.stdout: "clist_lat": { 2024-09-21T09:31:08.469 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 4, 2024-09-21T09:31:08.469 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 197871 2024-09-21T09:31:08.469 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.469 INFO:tasks.workunit.client.0.smithi022.stdout: "compress_lat": { 2024-09-21T09:31:08.469 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.470 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.470 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.470 INFO:tasks.workunit.client.0.smithi022.stdout: "compressed": { 2024-09-21T09:31:08.470 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.470 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.470 INFO:tasks.workunit.client.0.smithi022.stdout: "compressed_allocated": { 2024-09-21T09:31:08.470 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.470 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.470 INFO:tasks.workunit.client.0.smithi022.stdout: "compressed_original": { 2024-09-21T09:31:08.470 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.470 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.470 INFO:tasks.workunit.client.0.smithi022.stdout: "csum_lat": { 2024-09-21T09:31:08.470 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 10, 2024-09-21T09:31:08.470 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 11569 2024-09-21T09:31:08.471 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.471 INFO:tasks.workunit.client.0.smithi022.stdout: "decompress_lat": { 2024-09-21T09:31:08.471 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.471 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.471 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.471 INFO:tasks.workunit.client.0.smithi022.stdout: "kv_commit_lat": { 2024-09-21T09:31:08.471 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 229, 2024-09-21T09:31:08.471 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 40494224 2024-09-21T09:31:08.471 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.471 INFO:tasks.workunit.client.0.smithi022.stdout: "kv_final_lat": { 2024-09-21T09:31:08.471 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 225, 2024-09-21T09:31:08.471 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 8386751 2024-09-21T09:31:08.471 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.471 INFO:tasks.workunit.client.0.smithi022.stdout: "kv_flush_lat": { 2024-09-21T09:31:08.472 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 229, 2024-09-21T09:31:08.472 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 340445 2024-09-21T09:31:08.472 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.472 INFO:tasks.workunit.client.0.smithi022.stdout: "kv_sync_lat": { 2024-09-21T09:31:08.472 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 229, 2024-09-21T09:31:08.472 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 40834669 2024-09-21T09:31:08.472 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.472 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_get_keys_lat": { 2024-09-21T09:31:08.472 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.472 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.472 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.472 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_get_values_lat": { 2024-09-21T09:31:08.472 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 3, 2024-09-21T09:31:08.472 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 40544 2024-09-21T09:31:08.472 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.473 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_lower_bound_lat": { 2024-09-21T09:31:08.473 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 2, 2024-09-21T09:31:08.473 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1415 2024-09-21T09:31:08.473 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.473 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_next_lat": { 2024-09-21T09:31:08.473 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.473 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.473 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.473 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_seek_to_first_lat": { 2024-09-21T09:31:08.473 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.473 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.473 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.473 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_upper_bound_lat": { 2024-09-21T09:31:08.473 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 2, 2024-09-21T09:31:08.474 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 7390 2024-09-21T09:31:08.474 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.474 INFO:tasks.workunit.client.0.smithi022.stdout: "onode_hits": { 2024-09-21T09:31:08.474 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 509 2024-09-21T09:31:08.474 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.474 INFO:tasks.workunit.client.0.smithi022.stdout: "onode_misses": { 2024-09-21T09:31:08.474 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 205 2024-09-21T09:31:08.474 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.474 INFO:tasks.workunit.client.0.smithi022.stdout: "read_lat": { 2024-09-21T09:31:08.474 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 96, 2024-09-21T09:31:08.474 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 901168 2024-09-21T09:31:08.474 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.474 INFO:tasks.workunit.client.0.smithi022.stdout: "read_onode_meta_lat": { 2024-09-21T09:31:08.474 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 227, 2024-09-21T09:31:08.475 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 237836 2024-09-21T09:31:08.475 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.475 INFO:tasks.workunit.client.0.smithi022.stdout: "read_wait_aio_lat": { 2024-09-21T09:31:08.475 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 131, 2024-09-21T09:31:08.475 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 472466 2024-09-21T09:31:08.475 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.475 INFO:tasks.workunit.client.0.smithi022.stdout: "reads_with_retries": { 2024-09-21T09:31:08.475 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.475 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.475 INFO:tasks.workunit.client.0.smithi022.stdout: "remove_lat": { 2024-09-21T09:31:08.475 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 3, 2024-09-21T09:31:08.475 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 105977 2024-09-21T09:31:08.475 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.475 INFO:tasks.workunit.client.0.smithi022.stdout: "state_aio_wait_lat": { 2024-09-21T09:31:08.475 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 287, 2024-09-21T09:31:08.476 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 5234531 2024-09-21T09:31:08.476 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.476 INFO:tasks.workunit.client.0.smithi022.stdout: "state_deferred_aio_wait_lat": { 2024-09-21T09:31:08.476 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 89, 2024-09-21T09:31:08.476 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 6506780 2024-09-21T09:31:08.476 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.476 INFO:tasks.workunit.client.0.smithi022.stdout: "state_deferred_cleanup_lat": { 2024-09-21T09:31:08.476 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 88, 2024-09-21T09:31:08.476 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 330243082590 2024-09-21T09:31:08.476 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.476 INFO:tasks.workunit.client.0.smithi022.stdout: "state_deferred_queued_lat": { 2024-09-21T09:31:08.476 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 89, 2024-09-21T09:31:08.476 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 119707374385 2024-09-21T09:31:08.476 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.476 INFO:tasks.workunit.client.0.smithi022.stdout: "state_done_lat": { 2024-09-21T09:31:08.477 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 286, 2024-09-21T09:31:08.477 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 77969329092 2024-09-21T09:31:08.477 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.477 INFO:tasks.workunit.client.0.smithi022.stdout: "state_finishing_lat": { 2024-09-21T09:31:08.477 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 286, 2024-09-21T09:31:08.477 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 29304 2024-09-21T09:31:08.477 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.477 INFO:tasks.workunit.client.0.smithi022.stdout: "state_io_done_lat": { 2024-09-21T09:31:08.477 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 287, 2024-09-21T09:31:08.477 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 352169 2024-09-21T09:31:08.477 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.477 INFO:tasks.workunit.client.0.smithi022.stdout: "state_kv_commiting_lat": { 2024-09-21T09:31:08.477 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 287, 2024-09-21T09:31:08.477 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 95585029 2024-09-21T09:31:08.477 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.478 INFO:tasks.workunit.client.0.smithi022.stdout: "state_kv_done_lat": { 2024-09-21T09:31:08.478 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 287, 2024-09-21T09:31:08.478 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 105994 2024-09-21T09:31:08.478 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.478 INFO:tasks.workunit.client.0.smithi022.stdout: "state_kv_queued_lat": { 2024-09-21T09:31:08.478 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 287, 2024-09-21T09:31:08.478 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 79771479 2024-09-21T09:31:08.478 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.478 INFO:tasks.workunit.client.0.smithi022.stdout: "state_prepare_lat": { 2024-09-21T09:31:08.478 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 287, 2024-09-21T09:31:08.478 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 16479816 2024-09-21T09:31:08.478 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.478 INFO:tasks.workunit.client.0.smithi022.stdout: "stored": { 2024-09-21T09:31:08.478 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 309979 2024-09-21T09:31:08.478 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.479 INFO:tasks.workunit.client.0.smithi022.stdout: "truncate_lat": { 2024-09-21T09:31:08.479 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 49, 2024-09-21T09:31:08.479 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 232193 2024-09-21T09:31:08.479 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.479 INFO:tasks.workunit.client.0.smithi022.stdout: "txc_commit_lat": { 2024-09-21T09:31:08.479 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 287, 2024-09-21T09:31:08.479 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 197473775 2024-09-21T09:31:08.479 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.479 INFO:tasks.workunit.client.0.smithi022.stdout: "txc_submit_lat": { 2024-09-21T09:31:08.479 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 287, 2024-09-21T09:31:08.479 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 19985041 2024-09-21T09:31:08.479 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.479 INFO:tasks.workunit.client.0.smithi022.stdout: "txc_throttle_lat": { 2024-09-21T09:31:08.479 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 287, 2024-09-21T09:31:08.479 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1943254 2024-09-21T09:31:08.480 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.480 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.480 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore-pricache": { 2024-09-21T09:31:08.480 INFO:tasks.workunit.client.0.smithi022.stdout: "cache_bytes": { 2024-09-21T09:31:08.480 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1862400347 2024-09-21T09:31:08.480 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.480 INFO:tasks.workunit.client.0.smithi022.stdout: "heap_bytes": { 2024-09-21T09:31:08.480 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 270139392 2024-09-21T09:31:08.480 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.480 INFO:tasks.workunit.client.0.smithi022.stdout: "mapped_bytes": { 2024-09-21T09:31:08.480 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 269344768 2024-09-21T09:31:08.480 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.480 INFO:tasks.workunit.client.0.smithi022.stdout: "target_bytes": { 2024-09-21T09:31:08.480 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 3138478489 2024-09-21T09:31:08.480 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.480 INFO:tasks.workunit.client.0.smithi022.stdout: "unmapped_bytes": { 2024-09-21T09:31:08.481 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 794624 2024-09-21T09:31:08.481 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.481 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.481 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore-pricache:data": { 2024-09-21T09:31:08.481 INFO:tasks.workunit.client.0.smithi022.stdout: "committed_bytes": { 2024-09-21T09:31:08.481 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 159383552 2024-09-21T09:31:08.481 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.481 INFO:tasks.workunit.client.0.smithi022.stdout: "pri0_bytes": { 2024-09-21T09:31:08.481 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.481 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.481 INFO:tasks.workunit.client.0.smithi022.stdout: "pri10_bytes": { 2024-09-21T09:31:08.481 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.481 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.482 INFO:tasks.workunit.client.0.smithi022.stdout: "pri11_bytes": { 2024-09-21T09:31:08.482 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 85556261 2024-09-21T09:31:08.482 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.482 INFO:tasks.workunit.client.0.smithi022.stdout: "pri1_bytes": { 2024-09-21T09:31:08.482 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.482 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.482 INFO:tasks.workunit.client.0.smithi022.stdout: "pri2_bytes": { 2024-09-21T09:31:08.482 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.482 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.482 INFO:tasks.workunit.client.0.smithi022.stdout: "pri3_bytes": { 2024-09-21T09:31:08.482 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.482 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.482 INFO:tasks.workunit.client.0.smithi022.stdout: "pri4_bytes": { 2024-09-21T09:31:08.483 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.483 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.483 INFO:tasks.workunit.client.0.smithi022.stdout: "pri5_bytes": { 2024-09-21T09:31:08.483 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.483 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.483 INFO:tasks.workunit.client.0.smithi022.stdout: "pri6_bytes": { 2024-09-21T09:31:08.483 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 8192 2024-09-21T09:31:08.483 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.483 INFO:tasks.workunit.client.0.smithi022.stdout: "pri7_bytes": { 2024-09-21T09:31:08.483 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.483 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.483 INFO:tasks.workunit.client.0.smithi022.stdout: "pri8_bytes": { 2024-09-21T09:31:08.483 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.483 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.484 INFO:tasks.workunit.client.0.smithi022.stdout: "pri9_bytes": { 2024-09-21T09:31:08.484 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.484 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.484 INFO:tasks.workunit.client.0.smithi022.stdout: "reserved_bytes": { 2024-09-21T09:31:08.484 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 73819099 2024-09-21T09:31:08.484 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.484 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.484 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore-pricache:kv": { 2024-09-21T09:31:08.484 INFO:tasks.workunit.client.0.smithi022.stdout: "committed_bytes": { 2024-09-21T09:31:08.484 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 780140544 2024-09-21T09:31:08.484 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.484 INFO:tasks.workunit.client.0.smithi022.stdout: "pri0_bytes": { 2024-09-21T09:31:08.484 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 67108864 2024-09-21T09:31:08.484 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.485 INFO:tasks.workunit.client.0.smithi022.stdout: "pri10_bytes": { 2024-09-21T09:31:08.485 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.485 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.485 INFO:tasks.workunit.client.0.smithi022.stdout: "pri11_bytes": { 2024-09-21T09:31:08.485 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 641671963 2024-09-21T09:31:08.485 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.485 INFO:tasks.workunit.client.0.smithi022.stdout: "pri1_bytes": { 2024-09-21T09:31:08.485 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.485 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.485 INFO:tasks.workunit.client.0.smithi022.stdout: "pri2_bytes": { 2024-09-21T09:31:08.485 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.485 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.485 INFO:tasks.workunit.client.0.smithi022.stdout: "pri3_bytes": { 2024-09-21T09:31:08.485 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.486 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.486 INFO:tasks.workunit.client.0.smithi022.stdout: "pri4_bytes": { 2024-09-21T09:31:08.486 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.486 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.486 INFO:tasks.workunit.client.0.smithi022.stdout: "pri5_bytes": { 2024-09-21T09:31:08.486 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.486 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.486 INFO:tasks.workunit.client.0.smithi022.stdout: "pri6_bytes": { 2024-09-21T09:31:08.486 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2128 2024-09-21T09:31:08.486 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.486 INFO:tasks.workunit.client.0.smithi022.stdout: "pri7_bytes": { 2024-09-21T09:31:08.486 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.486 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.486 INFO:tasks.workunit.client.0.smithi022.stdout: "pri8_bytes": { 2024-09-21T09:31:08.487 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.487 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.487 INFO:tasks.workunit.client.0.smithi022.stdout: "pri9_bytes": { 2024-09-21T09:31:08.487 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.487 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.487 INFO:tasks.workunit.client.0.smithi022.stdout: "reserved_bytes": { 2024-09-21T09:31:08.487 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 71357589 2024-09-21T09:31:08.487 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.487 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.487 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore-pricache:kv_onode": { 2024-09-21T09:31:08.487 INFO:tasks.workunit.client.0.smithi022.stdout: "committed_bytes": { 2024-09-21T09:31:08.487 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 192937984 2024-09-21T09:31:08.487 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.487 INFO:tasks.workunit.client.0.smithi022.stdout: "pri0_bytes": { 2024-09-21T09:31:08.488 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 67108864 2024-09-21T09:31:08.488 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.488 INFO:tasks.workunit.client.0.smithi022.stdout: "pri10_bytes": { 2024-09-21T09:31:08.488 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 464 2024-09-21T09:31:08.488 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.488 INFO:tasks.workunit.client.0.smithi022.stdout: "pri11_bytes": { 2024-09-21T09:31:08.488 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 57037507 2024-09-21T09:31:08.488 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.488 INFO:tasks.workunit.client.0.smithi022.stdout: "pri1_bytes": { 2024-09-21T09:31:08.488 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.488 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.488 INFO:tasks.workunit.client.0.smithi022.stdout: "pri2_bytes": { 2024-09-21T09:31:08.488 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.488 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.489 INFO:tasks.workunit.client.0.smithi022.stdout: "pri3_bytes": { 2024-09-21T09:31:08.489 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.489 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.489 INFO:tasks.workunit.client.0.smithi022.stdout: "pri4_bytes": { 2024-09-21T09:31:08.489 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.489 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.489 INFO:tasks.workunit.client.0.smithi022.stdout: "pri5_bytes": { 2024-09-21T09:31:08.489 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.489 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.489 INFO:tasks.workunit.client.0.smithi022.stdout: "pri6_bytes": { 2024-09-21T09:31:08.489 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.489 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.489 INFO:tasks.workunit.client.0.smithi022.stdout: "pri7_bytes": { 2024-09-21T09:31:08.489 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.490 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.490 INFO:tasks.workunit.client.0.smithi022.stdout: "pri8_bytes": { 2024-09-21T09:31:08.490 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.490 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.490 INFO:tasks.workunit.client.0.smithi022.stdout: "pri9_bytes": { 2024-09-21T09:31:08.490 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.490 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.490 INFO:tasks.workunit.client.0.smithi022.stdout: "reserved_bytes": { 2024-09-21T09:31:08.490 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 68791149 2024-09-21T09:31:08.490 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.490 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.490 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore-pricache:meta": { 2024-09-21T09:31:08.490 INFO:tasks.workunit.client.0.smithi022.stdout: "committed_bytes": { 2024-09-21T09:31:08.490 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 713031680 2024-09-21T09:31:08.490 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.491 INFO:tasks.workunit.client.0.smithi022.stdout: "pri0_bytes": { 2024-09-21T09:31:08.491 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.491 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.491 INFO:tasks.workunit.client.0.smithi022.stdout: "pri10_bytes": { 2024-09-21T09:31:08.491 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.491 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.491 INFO:tasks.workunit.client.0.smithi022.stdout: "pri11_bytes": { 2024-09-21T09:31:08.491 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 641671963 2024-09-21T09:31:08.491 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.491 INFO:tasks.workunit.client.0.smithi022.stdout: "pri1_bytes": { 2024-09-21T09:31:08.491 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.491 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.491 INFO:tasks.workunit.client.0.smithi022.stdout: "pri2_bytes": { 2024-09-21T09:31:08.491 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.492 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.492 INFO:tasks.workunit.client.0.smithi022.stdout: "pri3_bytes": { 2024-09-21T09:31:08.492 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.492 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.492 INFO:tasks.workunit.client.0.smithi022.stdout: "pri4_bytes": { 2024-09-21T09:31:08.492 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.492 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.492 INFO:tasks.workunit.client.0.smithi022.stdout: "pri5_bytes": { 2024-09-21T09:31:08.492 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.492 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.492 INFO:tasks.workunit.client.0.smithi022.stdout: "pri6_bytes": { 2024-09-21T09:31:08.492 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 241458 2024-09-21T09:31:08.492 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.492 INFO:tasks.workunit.client.0.smithi022.stdout: "pri7_bytes": { 2024-09-21T09:31:08.493 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.493 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.493 INFO:tasks.workunit.client.0.smithi022.stdout: "pri8_bytes": { 2024-09-21T09:31:08.493 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.493 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.493 INFO:tasks.workunit.client.0.smithi022.stdout: "pri9_bytes": { 2024-09-21T09:31:08.493 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.493 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.493 INFO:tasks.workunit.client.0.smithi022.stdout: "reserved_bytes": { 2024-09-21T09:31:08.493 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 71118259 2024-09-21T09:31:08.493 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.493 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.493 INFO:tasks.workunit.client.0.smithi022.stdout: "objecter": { 2024-09-21T09:31:08.493 INFO:tasks.workunit.client.0.smithi022.stdout: "op_active": { 2024-09-21T09:31:08.493 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.494 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.494 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r": { 2024-09-21T09:31:08.494 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.494 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.494 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rmw": { 2024-09-21T09:31:08.494 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.494 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.494 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w": { 2024-09-21T09:31:08.494 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.494 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.494 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.494 INFO:tasks.workunit.client.0.smithi022.stdout: "osd": { 2024-09-21T09:31:08.494 INFO:tasks.workunit.client.0.smithi022.stdout: "numpg": { 2024-09-21T09:31:08.494 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 30 2024-09-21T09:31:08.495 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.495 INFO:tasks.workunit.client.0.smithi022.stdout: "numpg_removing": { 2024-09-21T09:31:08.495 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.495 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.495 INFO:tasks.workunit.client.0.smithi022.stdout: "op": { 2024-09-21T09:31:08.495 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 10 2024-09-21T09:31:08.495 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.495 INFO:tasks.workunit.client.0.smithi022.stdout: "op_delayed_degraded": { 2024-09-21T09:31:08.495 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.495 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.495 INFO:tasks.workunit.client.0.smithi022.stdout: "op_delayed_unreadable": { 2024-09-21T09:31:08.495 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.495 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.495 INFO:tasks.workunit.client.0.smithi022.stdout: "op_in_bytes": { 2024-09-21T09:31:08.496 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.496 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.496 INFO:tasks.workunit.client.0.smithi022.stdout: "op_latency": { 2024-09-21T09:31:08.496 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 10, 2024-09-21T09:31:08.496 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 54020340 2024-09-21T09:31:08.496 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.496 INFO:tasks.workunit.client.0.smithi022.stdout: "op_out_bytes": { 2024-09-21T09:31:08.496 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.496 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.496 INFO:tasks.workunit.client.0.smithi022.stdout: "op_prepare_latency": { 2024-09-21T09:31:08.496 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 10, 2024-09-21T09:31:08.496 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 3890283 2024-09-21T09:31:08.496 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.496 INFO:tasks.workunit.client.0.smithi022.stdout: "op_process_latency": { 2024-09-21T09:31:08.497 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 10, 2024-09-21T09:31:08.497 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 42756099 2024-09-21T09:31:08.497 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.497 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r": { 2024-09-21T09:31:08.497 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.497 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.497 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r_latency": { 2024-09-21T09:31:08.497 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.497 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.497 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.497 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-21T09:31:08.497 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.497 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.498 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r_out_bytes": { 2024-09-21T09:31:08.498 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.498 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.498 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r_prepare_latency": { 2024-09-21T09:31:08.498 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.498 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.498 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.498 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r_process_latency": { 2024-09-21T09:31:08.498 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.498 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.498 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.498 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw": { 2024-09-21T09:31:08.498 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.498 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.499 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_in_bytes": { 2024-09-21T09:31:08.499 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.499 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.499 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_latency": { 2024-09-21T09:31:08.499 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.499 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.499 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.499 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-21T09:31:08.499 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.499 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.499 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-21T09:31:08.499 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.499 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.499 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_out_bytes": { 2024-09-21T09:31:08.500 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.500 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.500 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_prepare_latency": { 2024-09-21T09:31:08.500 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.500 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.500 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.500 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_process_latency": { 2024-09-21T09:31:08.500 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.500 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.500 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.500 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w": { 2024-09-21T09:31:08.500 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 10 2024-09-21T09:31:08.500 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.500 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w_in_bytes": { 2024-09-21T09:31:08.500 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.501 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.501 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w_latency": { 2024-09-21T09:31:08.501 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 10, 2024-09-21T09:31:08.501 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 54020340 2024-09-21T09:31:08.501 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.501 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-21T09:31:08.501 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.501 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.501 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w_prepare_latency": { 2024-09-21T09:31:08.501 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 10, 2024-09-21T09:31:08.501 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 3890283 2024-09-21T09:31:08.501 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.501 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w_process_latency": { 2024-09-21T09:31:08.501 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 10, 2024-09-21T09:31:08.502 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 42756099 2024-09-21T09:31:08.502 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.502 INFO:tasks.workunit.client.0.smithi022.stdout: "op_wip": { 2024-09-21T09:31:08.502 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.502 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.502 INFO:tasks.workunit.client.0.smithi022.stdout: "recovery_bytes": { 2024-09-21T09:31:08.502 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.502 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.502 INFO:tasks.workunit.client.0.smithi022.stdout: "recovery_ops": { 2024-09-21T09:31:08.502 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.502 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.502 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_bytes": { 2024-09-21T09:31:08.502 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 95995035648 2024-09-21T09:31:08.502 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.503 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_bytes_used": { 2024-09-21T09:31:08.503 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 305180672 2024-09-21T09:31:08.503 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.503 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.503 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.503 INFO:tasks.workunit.client.0.smithi022.stdout: "osd.3": { 2024-09-21T09:31:08.503 INFO:tasks.workunit.client.0.smithi022.stdout: "bluefs": { 2024-09-21T09:31:08.503 INFO:tasks.workunit.client.0.smithi022.stdout: "alloc_slow_fallback": { 2024-09-21T09:31:08.503 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.503 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.503 INFO:tasks.workunit.client.0.smithi022.stdout: "alloc_slow_size_fallback": { 2024-09-21T09:31:08.503 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.503 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.503 INFO:tasks.workunit.client.0.smithi022.stdout: "alloc_unit_db": { 2024-09-21T09:31:08.503 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 65536 2024-09-21T09:31:08.504 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.504 INFO:tasks.workunit.client.0.smithi022.stdout: "alloc_unit_main": { 2024-09-21T09:31:08.504 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.504 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.504 INFO:tasks.workunit.client.0.smithi022.stdout: "alloc_unit_wal": { 2024-09-21T09:31:08.504 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.504 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.504 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes_written_slow": { 2024-09-21T09:31:08.504 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.504 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.504 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes_written_sst": { 2024-09-21T09:31:08.504 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 12288 2024-09-21T09:31:08.504 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.504 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes_written_wal": { 2024-09-21T09:31:08.505 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1228800 2024-09-21T09:31:08.505 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.505 INFO:tasks.workunit.client.0.smithi022.stdout: "compact_lat": { 2024-09-21T09:31:08.505 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.505 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.505 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.505 INFO:tasks.workunit.client.0.smithi022.stdout: "compact_lock_lat": { 2024-09-21T09:31:08.505 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.505 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.505 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.505 INFO:tasks.workunit.client.0.smithi022.stdout: "db_total_bytes": { 2024-09-21T09:31:08.505 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 95995035648 2024-09-21T09:31:08.505 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.505 INFO:tasks.workunit.client.0.smithi022.stdout: "db_used_bytes": { 2024-09-21T09:31:08.505 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 304480256 2024-09-21T09:31:08.506 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.506 INFO:tasks.workunit.client.0.smithi022.stdout: "log_bytes": { 2024-09-21T09:31:08.506 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 937984 2024-09-21T09:31:08.506 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.506 INFO:tasks.workunit.client.0.smithi022.stdout: "logged_bytes": { 2024-09-21T09:31:08.506 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 798720 2024-09-21T09:31:08.506 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.506 INFO:tasks.workunit.client.0.smithi022.stdout: "max_bytes_db": { 2024-09-21T09:31:08.506 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 312868864 2024-09-21T09:31:08.507 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.507 INFO:tasks.workunit.client.0.smithi022.stdout: "max_bytes_slow": { 2024-09-21T09:31:08.507 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.507 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.507 INFO:tasks.workunit.client.0.smithi022.stdout: "max_bytes_wal": { 2024-09-21T09:31:08.507 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.507 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.507 INFO:tasks.workunit.client.0.smithi022.stdout: "num_files": { 2024-09-21T09:31:08.507 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 14 2024-09-21T09:31:08.507 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.508 INFO:tasks.workunit.client.0.smithi022.stdout: "read_bytes": { 2024-09-21T09:31:08.508 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 195697 2024-09-21T09:31:08.508 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.508 INFO:tasks.workunit.client.0.smithi022.stdout: "read_count": { 2024-09-21T09:31:08.508 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 23 2024-09-21T09:31:08.508 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.508 INFO:tasks.workunit.client.0.smithi022.stdout: "read_disk_bytes": { 2024-09-21T09:31:08.508 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1269760 2024-09-21T09:31:08.508 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.508 INFO:tasks.workunit.client.0.smithi022.stdout: "read_disk_bytes_db": { 2024-09-21T09:31:08.508 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1273856 2024-09-21T09:31:08.508 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.508 INFO:tasks.workunit.client.0.smithi022.stdout: "read_disk_bytes_slow": { 2024-09-21T09:31:08.508 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.508 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.509 INFO:tasks.workunit.client.0.smithi022.stdout: "read_disk_bytes_wal": { 2024-09-21T09:31:08.509 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.509 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.509 INFO:tasks.workunit.client.0.smithi022.stdout: "read_disk_count": { 2024-09-21T09:31:08.509 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 15 2024-09-21T09:31:08.509 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.509 INFO:tasks.workunit.client.0.smithi022.stdout: "read_prefetch_bytes": { 2024-09-21T09:31:08.509 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 4789 2024-09-21T09:31:08.509 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.509 INFO:tasks.workunit.client.0.smithi022.stdout: "read_prefetch_count": { 2024-09-21T09:31:08.509 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 4 2024-09-21T09:31:08.510 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.510 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_buffer_bytes": { 2024-09-21T09:31:08.510 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 5589 2024-09-21T09:31:08.510 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.510 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_buffer_count": { 2024-09-21T09:31:08.510 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 24 2024-09-21T09:31:08.510 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.510 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_bytes": { 2024-09-21T09:31:08.510 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 5589 2024-09-21T09:31:08.510 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.510 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_count": { 2024-09-21T09:31:08.510 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 24 2024-09-21T09:31:08.510 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.511 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_disk_bytes": { 2024-09-21T09:31:08.511 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.511 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.511 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_disk_bytes_db": { 2024-09-21T09:31:08.511 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.511 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.511 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_disk_bytes_slow": { 2024-09-21T09:31:08.511 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.511 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.511 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_disk_bytes_wal": { 2024-09-21T09:31:08.511 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.511 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.511 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_disk_count": { 2024-09-21T09:31:08.511 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.512 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.512 INFO:tasks.workunit.client.0.smithi022.stdout: "slow_total_bytes": { 2024-09-21T09:31:08.512 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.512 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.512 INFO:tasks.workunit.client.0.smithi022.stdout: "slow_used_bytes": { 2024-09-21T09:31:08.512 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.512 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.512 INFO:tasks.workunit.client.0.smithi022.stdout: "wal_total_bytes": { 2024-09-21T09:31:08.512 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.512 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.512 INFO:tasks.workunit.client.0.smithi022.stdout: "wal_used_bytes": { 2024-09-21T09:31:08.512 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.512 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.512 INFO:tasks.workunit.client.0.smithi022.stdout: "write_bytes": { 2024-09-21T09:31:08.512 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2101248 2024-09-21T09:31:08.513 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.513 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.513 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore": { 2024-09-21T09:31:08.513 INFO:tasks.workunit.client.0.smithi022.stdout: "alloc_unit": { 2024-09-21T09:31:08.513 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 4096 2024-09-21T09:31:08.513 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.513 INFO:tasks.workunit.client.0.smithi022.stdout: "allocated": { 2024-09-21T09:31:08.513 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 708608 2024-09-21T09:31:08.513 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.513 INFO:tasks.workunit.client.0.smithi022.stdout: "clist_lat": { 2024-09-21T09:31:08.513 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.513 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.513 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.514 INFO:tasks.workunit.client.0.smithi022.stdout: "compress_lat": { 2024-09-21T09:31:08.514 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.514 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.514 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.514 INFO:tasks.workunit.client.0.smithi022.stdout: "compressed": { 2024-09-21T09:31:08.514 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.514 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.514 INFO:tasks.workunit.client.0.smithi022.stdout: "compressed_allocated": { 2024-09-21T09:31:08.514 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.514 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.514 INFO:tasks.workunit.client.0.smithi022.stdout: "compressed_original": { 2024-09-21T09:31:08.514 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.514 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.514 INFO:tasks.workunit.client.0.smithi022.stdout: "csum_lat": { 2024-09-21T09:31:08.514 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 4, 2024-09-21T09:31:08.515 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 7157 2024-09-21T09:31:08.515 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.515 INFO:tasks.workunit.client.0.smithi022.stdout: "decompress_lat": { 2024-09-21T09:31:08.515 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.515 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.515 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.515 INFO:tasks.workunit.client.0.smithi022.stdout: "kv_commit_lat": { 2024-09-21T09:31:08.515 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 178, 2024-09-21T09:31:08.515 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 35605293 2024-09-21T09:31:08.515 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.515 INFO:tasks.workunit.client.0.smithi022.stdout: "kv_final_lat": { 2024-09-21T09:31:08.515 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 172, 2024-09-21T09:31:08.515 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 9751410 2024-09-21T09:31:08.516 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.516 INFO:tasks.workunit.client.0.smithi022.stdout: "kv_flush_lat": { 2024-09-21T09:31:08.516 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 178, 2024-09-21T09:31:08.516 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 244331 2024-09-21T09:31:08.516 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.516 INFO:tasks.workunit.client.0.smithi022.stdout: "kv_sync_lat": { 2024-09-21T09:31:08.516 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 178, 2024-09-21T09:31:08.516 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 35849624 2024-09-21T09:31:08.516 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.516 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_get_keys_lat": { 2024-09-21T09:31:08.516 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.516 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.516 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.516 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_get_values_lat": { 2024-09-21T09:31:08.516 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 1, 2024-09-21T09:31:08.517 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 28513 2024-09-21T09:31:08.517 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.517 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_lower_bound_lat": { 2024-09-21T09:31:08.517 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 2, 2024-09-21T09:31:08.517 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 916 2024-09-21T09:31:08.517 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.517 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_next_lat": { 2024-09-21T09:31:08.517 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.517 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.517 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.517 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_seek_to_first_lat": { 2024-09-21T09:31:08.517 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.517 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.517 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.517 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_upper_bound_lat": { 2024-09-21T09:31:08.518 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 3, 2024-09-21T09:31:08.518 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 7611 2024-09-21T09:31:08.518 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.518 INFO:tasks.workunit.client.0.smithi022.stdout: "onode_hits": { 2024-09-21T09:31:08.518 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 352 2024-09-21T09:31:08.518 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.518 INFO:tasks.workunit.client.0.smithi022.stdout: "onode_misses": { 2024-09-21T09:31:08.518 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 214 2024-09-21T09:31:08.518 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.518 INFO:tasks.workunit.client.0.smithi022.stdout: "read_lat": { 2024-09-21T09:31:08.518 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 85, 2024-09-21T09:31:08.518 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 738958 2024-09-21T09:31:08.518 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.518 INFO:tasks.workunit.client.0.smithi022.stdout: "read_onode_meta_lat": { 2024-09-21T09:31:08.519 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 149, 2024-09-21T09:31:08.519 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 245550 2024-09-21T09:31:08.519 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.519 INFO:tasks.workunit.client.0.smithi022.stdout: "read_wait_aio_lat": { 2024-09-21T09:31:08.519 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 64, 2024-09-21T09:31:08.519 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 222496 2024-09-21T09:31:08.519 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.519 INFO:tasks.workunit.client.0.smithi022.stdout: "reads_with_retries": { 2024-09-21T09:31:08.519 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.519 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.519 INFO:tasks.workunit.client.0.smithi022.stdout: "remove_lat": { 2024-09-21T09:31:08.519 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.519 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.520 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.520 INFO:tasks.workunit.client.0.smithi022.stdout: "state_aio_wait_lat": { 2024-09-21T09:31:08.520 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 251, 2024-09-21T09:31:08.520 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 3272554 2024-09-21T09:31:08.520 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.520 INFO:tasks.workunit.client.0.smithi022.stdout: "state_deferred_aio_wait_lat": { 2024-09-21T09:31:08.520 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 51, 2024-09-21T09:31:08.520 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 3656446 2024-09-21T09:31:08.520 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.520 INFO:tasks.workunit.client.0.smithi022.stdout: "state_deferred_cleanup_lat": { 2024-09-21T09:31:08.520 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 48, 2024-09-21T09:31:08.520 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 348209618281 2024-09-21T09:31:08.520 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.520 INFO:tasks.workunit.client.0.smithi022.stdout: "state_deferred_queued_lat": { 2024-09-21T09:31:08.521 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 51, 2024-09-21T09:31:08.521 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 60751115424 2024-09-21T09:31:08.521 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.521 INFO:tasks.workunit.client.0.smithi022.stdout: "state_done_lat": { 2024-09-21T09:31:08.521 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 248, 2024-09-21T09:31:08.521 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 509483 2024-09-21T09:31:08.521 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.521 INFO:tasks.workunit.client.0.smithi022.stdout: "state_finishing_lat": { 2024-09-21T09:31:08.521 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 248, 2024-09-21T09:31:08.521 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 25639 2024-09-21T09:31:08.521 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.521 INFO:tasks.workunit.client.0.smithi022.stdout: "state_io_done_lat": { 2024-09-21T09:31:08.521 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 251, 2024-09-21T09:31:08.521 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 233596 2024-09-21T09:31:08.522 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.522 INFO:tasks.workunit.client.0.smithi022.stdout: "state_kv_commiting_lat": { 2024-09-21T09:31:08.522 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 251, 2024-09-21T09:31:08.522 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 85587384 2024-09-21T09:31:08.522 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.522 INFO:tasks.workunit.client.0.smithi022.stdout: "state_kv_done_lat": { 2024-09-21T09:31:08.522 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 251, 2024-09-21T09:31:08.522 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 78983 2024-09-21T09:31:08.522 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.522 INFO:tasks.workunit.client.0.smithi022.stdout: "state_kv_queued_lat": { 2024-09-21T09:31:08.522 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 251, 2024-09-21T09:31:08.522 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 38798576 2024-09-21T09:31:08.522 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.523 INFO:tasks.workunit.client.0.smithi022.stdout: "state_prepare_lat": { 2024-09-21T09:31:08.523 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 251, 2024-09-21T09:31:08.523 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 11159142 2024-09-21T09:31:08.523 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.523 INFO:tasks.workunit.client.0.smithi022.stdout: "stored": { 2024-09-21T09:31:08.523 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 312707 2024-09-21T09:31:08.523 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.523 INFO:tasks.workunit.client.0.smithi022.stdout: "truncate_lat": { 2024-09-21T09:31:08.523 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 42, 2024-09-21T09:31:08.523 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 217112 2024-09-21T09:31:08.523 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.523 INFO:tasks.workunit.client.0.smithi022.stdout: "txc_commit_lat": { 2024-09-21T09:31:08.523 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 251, 2024-09-21T09:31:08.523 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 139086793 2024-09-21T09:31:08.524 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.524 INFO:tasks.workunit.client.0.smithi022.stdout: "txc_submit_lat": { 2024-09-21T09:31:08.524 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 251, 2024-09-21T09:31:08.524 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 13699222 2024-09-21T09:31:08.524 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.524 INFO:tasks.workunit.client.0.smithi022.stdout: "txc_throttle_lat": { 2024-09-21T09:31:08.524 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 251, 2024-09-21T09:31:08.524 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1770625 2024-09-21T09:31:08.524 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.524 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.524 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore-pricache": { 2024-09-21T09:31:08.524 INFO:tasks.workunit.client.0.smithi022.stdout: "cache_bytes": { 2024-09-21T09:31:08.524 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1862400347 2024-09-21T09:31:08.524 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.525 INFO:tasks.workunit.client.0.smithi022.stdout: "heap_bytes": { 2024-09-21T09:31:08.525 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 268877824 2024-09-21T09:31:08.525 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.525 INFO:tasks.workunit.client.0.smithi022.stdout: "mapped_bytes": { 2024-09-21T09:31:08.525 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 268582912 2024-09-21T09:31:08.525 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.525 INFO:tasks.workunit.client.0.smithi022.stdout: "target_bytes": { 2024-09-21T09:31:08.525 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 3138478489 2024-09-21T09:31:08.525 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.525 INFO:tasks.workunit.client.0.smithi022.stdout: "unmapped_bytes": { 2024-09-21T09:31:08.525 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 294912 2024-09-21T09:31:08.525 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.525 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.525 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore-pricache:data": { 2024-09-21T09:31:08.526 INFO:tasks.workunit.client.0.smithi022.stdout: "committed_bytes": { 2024-09-21T09:31:08.526 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 159383552 2024-09-21T09:31:08.526 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.526 INFO:tasks.workunit.client.0.smithi022.stdout: "pri0_bytes": { 2024-09-21T09:31:08.526 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.526 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.526 INFO:tasks.workunit.client.0.smithi022.stdout: "pri10_bytes": { 2024-09-21T09:31:08.526 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.526 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.526 INFO:tasks.workunit.client.0.smithi022.stdout: "pri11_bytes": { 2024-09-21T09:31:08.526 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 85555878 2024-09-21T09:31:08.526 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.526 INFO:tasks.workunit.client.0.smithi022.stdout: "pri1_bytes": { 2024-09-21T09:31:08.527 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.527 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.527 INFO:tasks.workunit.client.0.smithi022.stdout: "pri2_bytes": { 2024-09-21T09:31:08.527 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.527 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.527 INFO:tasks.workunit.client.0.smithi022.stdout: "pri3_bytes": { 2024-09-21T09:31:08.527 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.527 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.527 INFO:tasks.workunit.client.0.smithi022.stdout: "pri4_bytes": { 2024-09-21T09:31:08.527 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.527 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.527 INFO:tasks.workunit.client.0.smithi022.stdout: "pri5_bytes": { 2024-09-21T09:31:08.527 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.527 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.528 INFO:tasks.workunit.client.0.smithi022.stdout: "pri6_bytes": { 2024-09-21T09:31:08.528 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 4096 2024-09-21T09:31:08.528 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.528 INFO:tasks.workunit.client.0.smithi022.stdout: "pri7_bytes": { 2024-09-21T09:31:08.528 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.528 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.528 INFO:tasks.workunit.client.0.smithi022.stdout: "pri8_bytes": { 2024-09-21T09:31:08.528 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.528 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.528 INFO:tasks.workunit.client.0.smithi022.stdout: "pri9_bytes": { 2024-09-21T09:31:08.528 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.528 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.528 INFO:tasks.workunit.client.0.smithi022.stdout: "reserved_bytes": { 2024-09-21T09:31:08.528 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 73823578 2024-09-21T09:31:08.529 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.529 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.529 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore-pricache:kv": { 2024-09-21T09:31:08.529 INFO:tasks.workunit.client.0.smithi022.stdout: "committed_bytes": { 2024-09-21T09:31:08.529 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 780140544 2024-09-21T09:31:08.529 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.529 INFO:tasks.workunit.client.0.smithi022.stdout: "pri0_bytes": { 2024-09-21T09:31:08.529 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 67108864 2024-09-21T09:31:08.529 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.529 INFO:tasks.workunit.client.0.smithi022.stdout: "pri10_bytes": { 2024-09-21T09:31:08.529 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.529 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.529 INFO:tasks.workunit.client.0.smithi022.stdout: "pri11_bytes": { 2024-09-21T09:31:08.529 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 641669085 2024-09-21T09:31:08.530 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.530 INFO:tasks.workunit.client.0.smithi022.stdout: "pri1_bytes": { 2024-09-21T09:31:08.530 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.530 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.530 INFO:tasks.workunit.client.0.smithi022.stdout: "pri2_bytes": { 2024-09-21T09:31:08.530 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.530 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.530 INFO:tasks.workunit.client.0.smithi022.stdout: "pri3_bytes": { 2024-09-21T09:31:08.530 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.530 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.530 INFO:tasks.workunit.client.0.smithi022.stdout: "pri4_bytes": { 2024-09-21T09:31:08.530 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.531 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.531 INFO:tasks.workunit.client.0.smithi022.stdout: "pri5_bytes": { 2024-09-21T09:31:08.531 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.531 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.531 INFO:tasks.workunit.client.0.smithi022.stdout: "pri6_bytes": { 2024-09-21T09:31:08.531 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2128 2024-09-21T09:31:08.531 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.531 INFO:tasks.workunit.client.0.smithi022.stdout: "pri7_bytes": { 2024-09-21T09:31:08.531 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.531 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.531 INFO:tasks.workunit.client.0.smithi022.stdout: "pri8_bytes": { 2024-09-21T09:31:08.531 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.531 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.531 INFO:tasks.workunit.client.0.smithi022.stdout: "pri9_bytes": { 2024-09-21T09:31:08.531 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.532 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.532 INFO:tasks.workunit.client.0.smithi022.stdout: "reserved_bytes": { 2024-09-21T09:31:08.532 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 71360467 2024-09-21T09:31:08.532 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.532 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.532 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore-pricache:kv_onode": { 2024-09-21T09:31:08.532 INFO:tasks.workunit.client.0.smithi022.stdout: "committed_bytes": { 2024-09-21T09:31:08.532 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 192937984 2024-09-21T09:31:08.532 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.532 INFO:tasks.workunit.client.0.smithi022.stdout: "pri0_bytes": { 2024-09-21T09:31:08.532 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 67108864 2024-09-21T09:31:08.532 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.532 INFO:tasks.workunit.client.0.smithi022.stdout: "pri10_bytes": { 2024-09-21T09:31:08.532 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 464 2024-09-21T09:31:08.533 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.533 INFO:tasks.workunit.client.0.smithi022.stdout: "pri11_bytes": { 2024-09-21T09:31:08.533 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 57037252 2024-09-21T09:31:08.533 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.533 INFO:tasks.workunit.client.0.smithi022.stdout: "pri1_bytes": { 2024-09-21T09:31:08.533 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.533 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.533 INFO:tasks.workunit.client.0.smithi022.stdout: "pri2_bytes": { 2024-09-21T09:31:08.533 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.533 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.533 INFO:tasks.workunit.client.0.smithi022.stdout: "pri3_bytes": { 2024-09-21T09:31:08.533 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.533 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.533 INFO:tasks.workunit.client.0.smithi022.stdout: "pri4_bytes": { 2024-09-21T09:31:08.534 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.534 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.534 INFO:tasks.workunit.client.0.smithi022.stdout: "pri5_bytes": { 2024-09-21T09:31:08.534 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.534 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.534 INFO:tasks.workunit.client.0.smithi022.stdout: "pri6_bytes": { 2024-09-21T09:31:08.534 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.534 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.534 INFO:tasks.workunit.client.0.smithi022.stdout: "pri7_bytes": { 2024-09-21T09:31:08.534 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.534 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.534 INFO:tasks.workunit.client.0.smithi022.stdout: "pri8_bytes": { 2024-09-21T09:31:08.534 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.534 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.535 INFO:tasks.workunit.client.0.smithi022.stdout: "pri9_bytes": { 2024-09-21T09:31:08.535 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.535 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.535 INFO:tasks.workunit.client.0.smithi022.stdout: "reserved_bytes": { 2024-09-21T09:31:08.535 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 68791404 2024-09-21T09:31:08.535 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.535 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.535 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore-pricache:meta": { 2024-09-21T09:31:08.535 INFO:tasks.workunit.client.0.smithi022.stdout: "committed_bytes": { 2024-09-21T09:31:08.535 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 713031680 2024-09-21T09:31:08.535 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.535 INFO:tasks.workunit.client.0.smithi022.stdout: "pri0_bytes": { 2024-09-21T09:31:08.535 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.535 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.536 INFO:tasks.workunit.client.0.smithi022.stdout: "pri10_bytes": { 2024-09-21T09:31:08.536 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.536 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.536 INFO:tasks.workunit.client.0.smithi022.stdout: "pri11_bytes": { 2024-09-21T09:31:08.536 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 641669085 2024-09-21T09:31:08.536 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.536 INFO:tasks.workunit.client.0.smithi022.stdout: "pri1_bytes": { 2024-09-21T09:31:08.536 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.536 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.536 INFO:tasks.workunit.client.0.smithi022.stdout: "pri2_bytes": { 2024-09-21T09:31:08.536 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.536 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.536 INFO:tasks.workunit.client.0.smithi022.stdout: "pri3_bytes": { 2024-09-21T09:31:08.536 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.537 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.537 INFO:tasks.workunit.client.0.smithi022.stdout: "pri4_bytes": { 2024-09-21T09:31:08.537 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.537 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.537 INFO:tasks.workunit.client.0.smithi022.stdout: "pri5_bytes": { 2024-09-21T09:31:08.537 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 6960 2024-09-21T09:31:08.537 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.537 INFO:tasks.workunit.client.0.smithi022.stdout: "pri6_bytes": { 2024-09-21T09:31:08.537 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 247783 2024-09-21T09:31:08.537 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.537 INFO:tasks.workunit.client.0.smithi022.stdout: "pri7_bytes": { 2024-09-21T09:31:08.537 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.537 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.537 INFO:tasks.workunit.client.0.smithi022.stdout: "pri8_bytes": { 2024-09-21T09:31:08.537 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.538 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.538 INFO:tasks.workunit.client.0.smithi022.stdout: "pri9_bytes": { 2024-09-21T09:31:08.538 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.538 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.538 INFO:tasks.workunit.client.0.smithi022.stdout: "reserved_bytes": { 2024-09-21T09:31:08.538 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 71107852 2024-09-21T09:31:08.538 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.538 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.538 INFO:tasks.workunit.client.0.smithi022.stdout: "objecter": { 2024-09-21T09:31:08.538 INFO:tasks.workunit.client.0.smithi022.stdout: "op_active": { 2024-09-21T09:31:08.538 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.538 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.538 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r": { 2024-09-21T09:31:08.538 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.539 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.539 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rmw": { 2024-09-21T09:31:08.539 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.539 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.539 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w": { 2024-09-21T09:31:08.539 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.539 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.539 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.539 INFO:tasks.workunit.client.0.smithi022.stdout: "osd": { 2024-09-21T09:31:08.539 INFO:tasks.workunit.client.0.smithi022.stdout: "numpg": { 2024-09-21T09:31:08.539 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 37 2024-09-21T09:31:08.539 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.539 INFO:tasks.workunit.client.0.smithi022.stdout: "numpg_removing": { 2024-09-21T09:31:08.539 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.540 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.540 INFO:tasks.workunit.client.0.smithi022.stdout: "op": { 2024-09-21T09:31:08.540 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.540 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.540 INFO:tasks.workunit.client.0.smithi022.stdout: "op_delayed_degraded": { 2024-09-21T09:31:08.540 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.540 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.540 INFO:tasks.workunit.client.0.smithi022.stdout: "op_delayed_unreadable": { 2024-09-21T09:31:08.540 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.540 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.540 INFO:tasks.workunit.client.0.smithi022.stdout: "op_in_bytes": { 2024-09-21T09:31:08.540 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.540 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.540 INFO:tasks.workunit.client.0.smithi022.stdout: "op_latency": { 2024-09-21T09:31:08.541 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.541 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.541 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.541 INFO:tasks.workunit.client.0.smithi022.stdout: "op_out_bytes": { 2024-09-21T09:31:08.541 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.541 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.541 INFO:tasks.workunit.client.0.smithi022.stdout: "op_prepare_latency": { 2024-09-21T09:31:08.541 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 2, 2024-09-21T09:31:08.541 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 731032 2024-09-21T09:31:08.541 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.541 INFO:tasks.workunit.client.0.smithi022.stdout: "op_process_latency": { 2024-09-21T09:31:08.541 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.541 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.541 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.542 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r": { 2024-09-21T09:31:08.542 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.542 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.542 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r_latency": { 2024-09-21T09:31:08.542 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.542 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.542 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.542 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-21T09:31:08.542 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.542 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.542 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r_out_bytes": { 2024-09-21T09:31:08.542 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.542 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.542 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r_prepare_latency": { 2024-09-21T09:31:08.542 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.543 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.543 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.543 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r_process_latency": { 2024-09-21T09:31:08.543 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.543 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.543 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.543 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw": { 2024-09-21T09:31:08.543 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.543 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.543 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_in_bytes": { 2024-09-21T09:31:08.543 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.543 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.543 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_latency": { 2024-09-21T09:31:08.543 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.544 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.544 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.544 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-21T09:31:08.544 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.544 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.544 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-21T09:31:08.544 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.544 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.544 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_out_bytes": { 2024-09-21T09:31:08.544 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.544 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.544 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_prepare_latency": { 2024-09-21T09:31:08.544 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.544 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.545 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.545 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_process_latency": { 2024-09-21T09:31:08.545 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.545 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.545 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.545 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w": { 2024-09-21T09:31:08.545 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.545 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.545 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w_in_bytes": { 2024-09-21T09:31:08.545 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.545 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.545 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w_latency": { 2024-09-21T09:31:08.545 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.545 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.545 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.546 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-21T09:31:08.546 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.546 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.546 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w_prepare_latency": { 2024-09-21T09:31:08.546 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 2, 2024-09-21T09:31:08.546 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 731032 2024-09-21T09:31:08.546 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.546 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w_process_latency": { 2024-09-21T09:31:08.546 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.546 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.546 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.546 INFO:tasks.workunit.client.0.smithi022.stdout: "op_wip": { 2024-09-21T09:31:08.546 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.546 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.547 INFO:tasks.workunit.client.0.smithi022.stdout: "recovery_bytes": { 2024-09-21T09:31:08.547 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.547 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.547 INFO:tasks.workunit.client.0.smithi022.stdout: "recovery_ops": { 2024-09-21T09:31:08.547 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.547 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.547 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_bytes": { 2024-09-21T09:31:08.547 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 95995035648 2024-09-21T09:31:08.547 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.547 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_bytes_used": { 2024-09-21T09:31:08.547 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 305197056 2024-09-21T09:31:08.547 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.547 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.547 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.548 INFO:tasks.workunit.client.0.smithi022.stdout: "osd.4": { 2024-09-21T09:31:08.548 INFO:tasks.workunit.client.0.smithi022.stdout: "bluefs": { 2024-09-21T09:31:08.548 INFO:tasks.workunit.client.0.smithi022.stdout: "alloc_slow_fallback": { 2024-09-21T09:31:08.548 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.548 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.548 INFO:tasks.workunit.client.0.smithi022.stdout: "alloc_slow_size_fallback": { 2024-09-21T09:31:08.548 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.548 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.548 INFO:tasks.workunit.client.0.smithi022.stdout: "alloc_unit_db": { 2024-09-21T09:31:08.548 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 65536 2024-09-21T09:31:08.548 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.548 INFO:tasks.workunit.client.0.smithi022.stdout: "alloc_unit_main": { 2024-09-21T09:31:08.548 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.548 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.549 INFO:tasks.workunit.client.0.smithi022.stdout: "alloc_unit_wal": { 2024-09-21T09:31:08.549 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.549 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.549 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes_written_slow": { 2024-09-21T09:31:08.549 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.549 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.549 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes_written_sst": { 2024-09-21T09:31:08.549 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 12288 2024-09-21T09:31:08.549 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.549 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes_written_wal": { 2024-09-21T09:31:08.549 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1212416 2024-09-21T09:31:08.549 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.549 INFO:tasks.workunit.client.0.smithi022.stdout: "compact_lat": { 2024-09-21T09:31:08.549 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.549 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.550 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.550 INFO:tasks.workunit.client.0.smithi022.stdout: "compact_lock_lat": { 2024-09-21T09:31:08.550 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.550 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.550 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.550 INFO:tasks.workunit.client.0.smithi022.stdout: "db_total_bytes": { 2024-09-21T09:31:08.550 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 95995035648 2024-09-21T09:31:08.550 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.550 INFO:tasks.workunit.client.0.smithi022.stdout: "db_used_bytes": { 2024-09-21T09:31:08.550 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 304480256 2024-09-21T09:31:08.550 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.550 INFO:tasks.workunit.client.0.smithi022.stdout: "log_bytes": { 2024-09-21T09:31:08.550 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 917504 2024-09-21T09:31:08.550 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.551 INFO:tasks.workunit.client.0.smithi022.stdout: "logged_bytes": { 2024-09-21T09:31:08.551 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 778240 2024-09-21T09:31:08.551 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.551 INFO:tasks.workunit.client.0.smithi022.stdout: "max_bytes_db": { 2024-09-21T09:31:08.551 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 312868864 2024-09-21T09:31:08.551 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.551 INFO:tasks.workunit.client.0.smithi022.stdout: "max_bytes_slow": { 2024-09-21T09:31:08.551 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.551 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.551 INFO:tasks.workunit.client.0.smithi022.stdout: "max_bytes_wal": { 2024-09-21T09:31:08.551 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.551 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.551 INFO:tasks.workunit.client.0.smithi022.stdout: "num_files": { 2024-09-21T09:31:08.551 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 14 2024-09-21T09:31:08.552 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.552 INFO:tasks.workunit.client.0.smithi022.stdout: "read_bytes": { 2024-09-21T09:31:08.552 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 195697 2024-09-21T09:31:08.552 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.552 INFO:tasks.workunit.client.0.smithi022.stdout: "read_count": { 2024-09-21T09:31:08.552 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 23 2024-09-21T09:31:08.552 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.552 INFO:tasks.workunit.client.0.smithi022.stdout: "read_disk_bytes": { 2024-09-21T09:31:08.552 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1269760 2024-09-21T09:31:08.552 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.552 INFO:tasks.workunit.client.0.smithi022.stdout: "read_disk_bytes_db": { 2024-09-21T09:31:08.552 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1273856 2024-09-21T09:31:08.552 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.552 INFO:tasks.workunit.client.0.smithi022.stdout: "read_disk_bytes_slow": { 2024-09-21T09:31:08.553 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.553 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.553 INFO:tasks.workunit.client.0.smithi022.stdout: "read_disk_bytes_wal": { 2024-09-21T09:31:08.553 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.553 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.553 INFO:tasks.workunit.client.0.smithi022.stdout: "read_disk_count": { 2024-09-21T09:31:08.553 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 15 2024-09-21T09:31:08.553 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.553 INFO:tasks.workunit.client.0.smithi022.stdout: "read_prefetch_bytes": { 2024-09-21T09:31:08.553 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 4789 2024-09-21T09:31:08.553 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.553 INFO:tasks.workunit.client.0.smithi022.stdout: "read_prefetch_count": { 2024-09-21T09:31:08.553 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 4 2024-09-21T09:31:08.553 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.554 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_buffer_bytes": { 2024-09-21T09:31:08.554 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 5589 2024-09-21T09:31:08.554 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.554 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_buffer_count": { 2024-09-21T09:31:08.554 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 24 2024-09-21T09:31:08.554 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.554 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_bytes": { 2024-09-21T09:31:08.554 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 5589 2024-09-21T09:31:08.554 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.554 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_count": { 2024-09-21T09:31:08.554 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 24 2024-09-21T09:31:08.554 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.554 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_disk_bytes": { 2024-09-21T09:31:08.554 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.555 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.555 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_disk_bytes_db": { 2024-09-21T09:31:08.555 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.555 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.555 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_disk_bytes_slow": { 2024-09-21T09:31:08.555 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.555 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.555 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_disk_bytes_wal": { 2024-09-21T09:31:08.555 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.555 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.555 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_disk_count": { 2024-09-21T09:31:08.555 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.555 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.555 INFO:tasks.workunit.client.0.smithi022.stdout: "slow_total_bytes": { 2024-09-21T09:31:08.556 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.556 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.556 INFO:tasks.workunit.client.0.smithi022.stdout: "slow_used_bytes": { 2024-09-21T09:31:08.556 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.556 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.556 INFO:tasks.workunit.client.0.smithi022.stdout: "wal_total_bytes": { 2024-09-21T09:31:08.556 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.556 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.556 INFO:tasks.workunit.client.0.smithi022.stdout: "wal_used_bytes": { 2024-09-21T09:31:08.556 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.556 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.556 INFO:tasks.workunit.client.0.smithi022.stdout: "write_bytes": { 2024-09-21T09:31:08.556 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2064384 2024-09-21T09:31:08.556 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.557 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.557 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore": { 2024-09-21T09:31:08.557 INFO:tasks.workunit.client.0.smithi022.stdout: "alloc_unit": { 2024-09-21T09:31:08.557 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 4096 2024-09-21T09:31:08.557 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.557 INFO:tasks.workunit.client.0.smithi022.stdout: "allocated": { 2024-09-21T09:31:08.557 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 708608 2024-09-21T09:31:08.557 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.557 INFO:tasks.workunit.client.0.smithi022.stdout: "clist_lat": { 2024-09-21T09:31:08.557 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 8, 2024-09-21T09:31:08.557 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 332492 2024-09-21T09:31:08.557 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.557 INFO:tasks.workunit.client.0.smithi022.stdout: "compress_lat": { 2024-09-21T09:31:08.557 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.557 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.558 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.558 INFO:tasks.workunit.client.0.smithi022.stdout: "compressed": { 2024-09-21T09:31:08.558 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.558 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.558 INFO:tasks.workunit.client.0.smithi022.stdout: "compressed_allocated": { 2024-09-21T09:31:08.558 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.558 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.558 INFO:tasks.workunit.client.0.smithi022.stdout: "compressed_original": { 2024-09-21T09:31:08.558 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.558 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.558 INFO:tasks.workunit.client.0.smithi022.stdout: "csum_lat": { 2024-09-21T09:31:08.558 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 8, 2024-09-21T09:31:08.558 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 19002 2024-09-21T09:31:08.558 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.559 INFO:tasks.workunit.client.0.smithi022.stdout: "decompress_lat": { 2024-09-21T09:31:08.559 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.559 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.559 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.559 INFO:tasks.workunit.client.0.smithi022.stdout: "kv_commit_lat": { 2024-09-21T09:31:08.559 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 173, 2024-09-21T09:31:08.559 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 48290848 2024-09-21T09:31:08.559 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.559 INFO:tasks.workunit.client.0.smithi022.stdout: "kv_final_lat": { 2024-09-21T09:31:08.559 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 163, 2024-09-21T09:31:08.559 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 10875385 2024-09-21T09:31:08.559 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.559 INFO:tasks.workunit.client.0.smithi022.stdout: "kv_flush_lat": { 2024-09-21T09:31:08.559 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 173, 2024-09-21T09:31:08.559 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 273861 2024-09-21T09:31:08.560 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.560 INFO:tasks.workunit.client.0.smithi022.stdout: "kv_sync_lat": { 2024-09-21T09:31:08.560 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 173, 2024-09-21T09:31:08.560 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 48564709 2024-09-21T09:31:08.560 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.560 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_get_keys_lat": { 2024-09-21T09:31:08.560 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.560 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.560 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.560 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_get_values_lat": { 2024-09-21T09:31:08.560 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 3, 2024-09-21T09:31:08.560 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 45531 2024-09-21T09:31:08.560 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.560 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_lower_bound_lat": { 2024-09-21T09:31:08.561 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 3, 2024-09-21T09:31:08.561 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 4564 2024-09-21T09:31:08.561 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.561 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_next_lat": { 2024-09-21T09:31:08.561 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.561 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.561 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.561 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_seek_to_first_lat": { 2024-09-21T09:31:08.561 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.561 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.561 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.561 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_upper_bound_lat": { 2024-09-21T09:31:08.561 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 3, 2024-09-21T09:31:08.561 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 8263 2024-09-21T09:31:08.561 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.562 INFO:tasks.workunit.client.0.smithi022.stdout: "onode_hits": { 2024-09-21T09:31:08.562 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 364 2024-09-21T09:31:08.562 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.562 INFO:tasks.workunit.client.0.smithi022.stdout: "onode_misses": { 2024-09-21T09:31:08.562 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 233 2024-09-21T09:31:08.562 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.562 INFO:tasks.workunit.client.0.smithi022.stdout: "read_lat": { 2024-09-21T09:31:08.562 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 93, 2024-09-21T09:31:08.562 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1467917 2024-09-21T09:31:08.562 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.562 INFO:tasks.workunit.client.0.smithi022.stdout: "read_onode_meta_lat": { 2024-09-21T09:31:08.562 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 156, 2024-09-21T09:31:08.562 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 259382 2024-09-21T09:31:08.562 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.563 INFO:tasks.workunit.client.0.smithi022.stdout: "read_wait_aio_lat": { 2024-09-21T09:31:08.563 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 63, 2024-09-21T09:31:08.563 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 720054 2024-09-21T09:31:08.563 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.563 INFO:tasks.workunit.client.0.smithi022.stdout: "reads_with_retries": { 2024-09-21T09:31:08.563 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.563 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.563 INFO:tasks.workunit.client.0.smithi022.stdout: "remove_lat": { 2024-09-21T09:31:08.563 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 5, 2024-09-21T09:31:08.563 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 120139 2024-09-21T09:31:08.563 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.563 INFO:tasks.workunit.client.0.smithi022.stdout: "state_aio_wait_lat": { 2024-09-21T09:31:08.563 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 261, 2024-09-21T09:31:08.563 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 3155241 2024-09-21T09:31:08.564 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.564 INFO:tasks.workunit.client.0.smithi022.stdout: "state_deferred_aio_wait_lat": { 2024-09-21T09:31:08.564 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 41, 2024-09-21T09:31:08.564 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2664064 2024-09-21T09:31:08.564 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.564 INFO:tasks.workunit.client.0.smithi022.stdout: "state_deferred_cleanup_lat": { 2024-09-21T09:31:08.564 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 40, 2024-09-21T09:31:08.564 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 234961781205 2024-09-21T09:31:08.564 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.564 INFO:tasks.workunit.client.0.smithi022.stdout: "state_deferred_queued_lat": { 2024-09-21T09:31:08.564 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 41, 2024-09-21T09:31:08.564 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 59734430380 2024-09-21T09:31:08.564 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.564 INFO:tasks.workunit.client.0.smithi022.stdout: "state_done_lat": { 2024-09-21T09:31:08.564 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 260, 2024-09-21T09:31:08.565 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 539644 2024-09-21T09:31:08.565 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.565 INFO:tasks.workunit.client.0.smithi022.stdout: "state_finishing_lat": { 2024-09-21T09:31:08.565 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 260, 2024-09-21T09:31:08.565 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 30665 2024-09-21T09:31:08.565 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.565 INFO:tasks.workunit.client.0.smithi022.stdout: "state_io_done_lat": { 2024-09-21T09:31:08.565 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 261, 2024-09-21T09:31:08.565 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 262379 2024-09-21T09:31:08.565 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.565 INFO:tasks.workunit.client.0.smithi022.stdout: "state_kv_commiting_lat": { 2024-09-21T09:31:08.565 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 261, 2024-09-21T09:31:08.565 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 145402039 2024-09-21T09:31:08.565 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.566 INFO:tasks.workunit.client.0.smithi022.stdout: "state_kv_done_lat": { 2024-09-21T09:31:08.566 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 261, 2024-09-21T09:31:08.566 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 85011 2024-09-21T09:31:08.566 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.566 INFO:tasks.workunit.client.0.smithi022.stdout: "state_kv_queued_lat": { 2024-09-21T09:31:08.566 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 261, 2024-09-21T09:31:08.566 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 83345383 2024-09-21T09:31:08.566 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.566 INFO:tasks.workunit.client.0.smithi022.stdout: "state_prepare_lat": { 2024-09-21T09:31:08.566 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 261, 2024-09-21T09:31:08.566 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 13784998 2024-09-21T09:31:08.566 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.566 INFO:tasks.workunit.client.0.smithi022.stdout: "stored": { 2024-09-21T09:31:08.566 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 310694 2024-09-21T09:31:08.567 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.567 INFO:tasks.workunit.client.0.smithi022.stdout: "truncate_lat": { 2024-09-21T09:31:08.567 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 39, 2024-09-21T09:31:08.567 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 176771 2024-09-21T09:31:08.567 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.567 INFO:tasks.workunit.client.0.smithi022.stdout: "txc_commit_lat": { 2024-09-21T09:31:08.567 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 261, 2024-09-21T09:31:08.567 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 245987319 2024-09-21T09:31:08.567 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.567 INFO:tasks.workunit.client.0.smithi022.stdout: "txc_submit_lat": { 2024-09-21T09:31:08.567 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 261, 2024-09-21T09:31:08.567 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 16421168 2024-09-21T09:31:08.567 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.567 INFO:tasks.workunit.client.0.smithi022.stdout: "txc_throttle_lat": { 2024-09-21T09:31:08.568 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 261, 2024-09-21T09:31:08.568 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1920806 2024-09-21T09:31:08.568 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.568 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.568 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore-pricache": { 2024-09-21T09:31:08.568 INFO:tasks.workunit.client.0.smithi022.stdout: "cache_bytes": { 2024-09-21T09:31:08.568 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1861463884 2024-09-21T09:31:08.568 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.568 INFO:tasks.workunit.client.0.smithi022.stdout: "heap_bytes": { 2024-09-21T09:31:08.568 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 269860864 2024-09-21T09:31:08.568 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.568 INFO:tasks.workunit.client.0.smithi022.stdout: "mapped_bytes": { 2024-09-21T09:31:08.568 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 269139968 2024-09-21T09:31:08.568 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.568 INFO:tasks.workunit.client.0.smithi022.stdout: "target_bytes": { 2024-09-21T09:31:08.569 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 3137376768 2024-09-21T09:31:08.569 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.569 INFO:tasks.workunit.client.0.smithi022.stdout: "unmapped_bytes": { 2024-09-21T09:31:08.569 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 720896 2024-09-21T09:31:08.569 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.569 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.569 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore-pricache:data": { 2024-09-21T09:31:08.569 INFO:tasks.workunit.client.0.smithi022.stdout: "committed_bytes": { 2024-09-21T09:31:08.569 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 159383552 2024-09-21T09:31:08.569 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.569 INFO:tasks.workunit.client.0.smithi022.stdout: "pri0_bytes": { 2024-09-21T09:31:08.569 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.569 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.569 INFO:tasks.workunit.client.0.smithi022.stdout: "pri10_bytes": { 2024-09-21T09:31:08.570 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.570 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.570 INFO:tasks.workunit.client.0.smithi022.stdout: "pri11_bytes": { 2024-09-21T09:31:08.570 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 85499084 2024-09-21T09:31:08.570 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.570 INFO:tasks.workunit.client.0.smithi022.stdout: "pri1_bytes": { 2024-09-21T09:31:08.570 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.570 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.570 INFO:tasks.workunit.client.0.smithi022.stdout: "pri2_bytes": { 2024-09-21T09:31:08.570 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.570 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.570 INFO:tasks.workunit.client.0.smithi022.stdout: "pri3_bytes": { 2024-09-21T09:31:08.570 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.570 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.571 INFO:tasks.workunit.client.0.smithi022.stdout: "pri4_bytes": { 2024-09-21T09:31:08.571 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.571 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.571 INFO:tasks.workunit.client.0.smithi022.stdout: "pri5_bytes": { 2024-09-21T09:31:08.571 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.571 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.571 INFO:tasks.workunit.client.0.smithi022.stdout: "pri6_bytes": { 2024-09-21T09:31:08.571 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 8192 2024-09-21T09:31:08.571 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.571 INFO:tasks.workunit.client.0.smithi022.stdout: "pri7_bytes": { 2024-09-21T09:31:08.571 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.571 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.571 INFO:tasks.workunit.client.0.smithi022.stdout: "pri8_bytes": { 2024-09-21T09:31:08.571 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.571 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.572 INFO:tasks.workunit.client.0.smithi022.stdout: "pri9_bytes": { 2024-09-21T09:31:08.572 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.572 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.572 INFO:tasks.workunit.client.0.smithi022.stdout: "reserved_bytes": { 2024-09-21T09:31:08.572 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 73876276 2024-09-21T09:31:08.572 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.572 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.572 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore-pricache:kv": { 2024-09-21T09:31:08.572 INFO:tasks.workunit.client.0.smithi022.stdout: "committed_bytes": { 2024-09-21T09:31:08.572 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 780140544 2024-09-21T09:31:08.572 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.572 INFO:tasks.workunit.client.0.smithi022.stdout: "pri0_bytes": { 2024-09-21T09:31:08.572 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 67108864 2024-09-21T09:31:08.572 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.573 INFO:tasks.workunit.client.0.smithi022.stdout: "pri10_bytes": { 2024-09-21T09:31:08.573 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.573 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.573 INFO:tasks.workunit.client.0.smithi022.stdout: "pri11_bytes": { 2024-09-21T09:31:08.573 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 641243131 2024-09-21T09:31:08.573 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.573 INFO:tasks.workunit.client.0.smithi022.stdout: "pri1_bytes": { 2024-09-21T09:31:08.573 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.573 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.573 INFO:tasks.workunit.client.0.smithi022.stdout: "pri2_bytes": { 2024-09-21T09:31:08.573 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.573 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.573 INFO:tasks.workunit.client.0.smithi022.stdout: "pri3_bytes": { 2024-09-21T09:31:08.573 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.574 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.574 INFO:tasks.workunit.client.0.smithi022.stdout: "pri4_bytes": { 2024-09-21T09:31:08.574 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.574 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.574 INFO:tasks.workunit.client.0.smithi022.stdout: "pri5_bytes": { 2024-09-21T09:31:08.574 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.574 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.574 INFO:tasks.workunit.client.0.smithi022.stdout: "pri6_bytes": { 2024-09-21T09:31:08.574 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2128 2024-09-21T09:31:08.574 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.574 INFO:tasks.workunit.client.0.smithi022.stdout: "pri7_bytes": { 2024-09-21T09:31:08.574 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.574 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.574 INFO:tasks.workunit.client.0.smithi022.stdout: "pri8_bytes": { 2024-09-21T09:31:08.574 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.575 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.575 INFO:tasks.workunit.client.0.smithi022.stdout: "pri9_bytes": { 2024-09-21T09:31:08.575 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.575 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.575 INFO:tasks.workunit.client.0.smithi022.stdout: "reserved_bytes": { 2024-09-21T09:31:08.575 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 71786421 2024-09-21T09:31:08.575 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.575 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.575 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore-pricache:kv_onode": { 2024-09-21T09:31:08.575 INFO:tasks.workunit.client.0.smithi022.stdout: "committed_bytes": { 2024-09-21T09:31:08.575 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 192937984 2024-09-21T09:31:08.575 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.575 INFO:tasks.workunit.client.0.smithi022.stdout: "pri0_bytes": { 2024-09-21T09:31:08.575 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 67108864 2024-09-21T09:31:08.576 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.576 INFO:tasks.workunit.client.0.smithi022.stdout: "pri10_bytes": { 2024-09-21T09:31:08.576 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 464 2024-09-21T09:31:08.576 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.576 INFO:tasks.workunit.client.0.smithi022.stdout: "pri11_bytes": { 2024-09-21T09:31:08.576 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 56999389 2024-09-21T09:31:08.576 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.576 INFO:tasks.workunit.client.0.smithi022.stdout: "pri1_bytes": { 2024-09-21T09:31:08.576 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.576 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.576 INFO:tasks.workunit.client.0.smithi022.stdout: "pri2_bytes": { 2024-09-21T09:31:08.576 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.576 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.576 INFO:tasks.workunit.client.0.smithi022.stdout: "pri3_bytes": { 2024-09-21T09:31:08.576 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.577 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.577 INFO:tasks.workunit.client.0.smithi022.stdout: "pri4_bytes": { 2024-09-21T09:31:08.577 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.577 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.577 INFO:tasks.workunit.client.0.smithi022.stdout: "pri5_bytes": { 2024-09-21T09:31:08.577 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.577 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.577 INFO:tasks.workunit.client.0.smithi022.stdout: "pri6_bytes": { 2024-09-21T09:31:08.577 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.577 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.577 INFO:tasks.workunit.client.0.smithi022.stdout: "pri7_bytes": { 2024-09-21T09:31:08.577 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.577 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.577 INFO:tasks.workunit.client.0.smithi022.stdout: "pri8_bytes": { 2024-09-21T09:31:08.577 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.578 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.578 INFO:tasks.workunit.client.0.smithi022.stdout: "pri9_bytes": { 2024-09-21T09:31:08.578 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.578 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.578 INFO:tasks.workunit.client.0.smithi022.stdout: "reserved_bytes": { 2024-09-21T09:31:08.578 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 68829267 2024-09-21T09:31:08.578 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.578 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.578 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore-pricache:meta": { 2024-09-21T09:31:08.578 INFO:tasks.workunit.client.0.smithi022.stdout: "committed_bytes": { 2024-09-21T09:31:08.578 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 713031680 2024-09-21T09:31:08.578 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.578 INFO:tasks.workunit.client.0.smithi022.stdout: "pri0_bytes": { 2024-09-21T09:31:08.578 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.579 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.579 INFO:tasks.workunit.client.0.smithi022.stdout: "pri10_bytes": { 2024-09-21T09:31:08.579 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.579 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.579 INFO:tasks.workunit.client.0.smithi022.stdout: "pri11_bytes": { 2024-09-21T09:31:08.579 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 641243131 2024-09-21T09:31:08.579 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.579 INFO:tasks.workunit.client.0.smithi022.stdout: "pri1_bytes": { 2024-09-21T09:31:08.579 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.579 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.579 INFO:tasks.workunit.client.0.smithi022.stdout: "pri2_bytes": { 2024-09-21T09:31:08.579 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.579 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.579 INFO:tasks.workunit.client.0.smithi022.stdout: "pri3_bytes": { 2024-09-21T09:31:08.580 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.580 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.580 INFO:tasks.workunit.client.0.smithi022.stdout: "pri4_bytes": { 2024-09-21T09:31:08.580 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.580 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.580 INFO:tasks.workunit.client.0.smithi022.stdout: "pri5_bytes": { 2024-09-21T09:31:08.580 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.580 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.580 INFO:tasks.workunit.client.0.smithi022.stdout: "pri6_bytes": { 2024-09-21T09:31:08.580 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 257959 2024-09-21T09:31:08.580 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.580 INFO:tasks.workunit.client.0.smithi022.stdout: "pri7_bytes": { 2024-09-21T09:31:08.580 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.580 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.580 INFO:tasks.workunit.client.0.smithi022.stdout: "pri8_bytes": { 2024-09-21T09:31:08.581 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.581 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.581 INFO:tasks.workunit.client.0.smithi022.stdout: "pri9_bytes": { 2024-09-21T09:31:08.581 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.581 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.581 INFO:tasks.workunit.client.0.smithi022.stdout: "reserved_bytes": { 2024-09-21T09:31:08.581 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 71530590 2024-09-21T09:31:08.581 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.581 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.581 INFO:tasks.workunit.client.0.smithi022.stdout: "objecter": { 2024-09-21T09:31:08.581 INFO:tasks.workunit.client.0.smithi022.stdout: "op_active": { 2024-09-21T09:31:08.581 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.581 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.581 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r": { 2024-09-21T09:31:08.581 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.582 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.582 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rmw": { 2024-09-21T09:31:08.582 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.582 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.582 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w": { 2024-09-21T09:31:08.582 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.582 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.582 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.582 INFO:tasks.workunit.client.0.smithi022.stdout: "osd": { 2024-09-21T09:31:08.582 INFO:tasks.workunit.client.0.smithi022.stdout: "numpg": { 2024-09-21T09:31:08.582 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 40 2024-09-21T09:31:08.582 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.582 INFO:tasks.workunit.client.0.smithi022.stdout: "numpg_removing": { 2024-09-21T09:31:08.582 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.583 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.583 INFO:tasks.workunit.client.0.smithi022.stdout: "op": { 2024-09-21T09:31:08.583 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1 2024-09-21T09:31:08.583 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.583 INFO:tasks.workunit.client.0.smithi022.stdout: "op_delayed_degraded": { 2024-09-21T09:31:08.583 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.583 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.583 INFO:tasks.workunit.client.0.smithi022.stdout: "op_delayed_unreadable": { 2024-09-21T09:31:08.583 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.583 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.583 INFO:tasks.workunit.client.0.smithi022.stdout: "op_in_bytes": { 2024-09-21T09:31:08.583 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 22 2024-09-21T09:31:08.583 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.583 INFO:tasks.workunit.client.0.smithi022.stdout: "op_latency": { 2024-09-21T09:31:08.584 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 1, 2024-09-21T09:31:08.584 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 7870434 2024-09-21T09:31:08.584 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.584 INFO:tasks.workunit.client.0.smithi022.stdout: "op_out_bytes": { 2024-09-21T09:31:08.584 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.584 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.584 INFO:tasks.workunit.client.0.smithi022.stdout: "op_prepare_latency": { 2024-09-21T09:31:08.584 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 1, 2024-09-21T09:31:08.584 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 786116 2024-09-21T09:31:08.584 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.584 INFO:tasks.workunit.client.0.smithi022.stdout: "op_process_latency": { 2024-09-21T09:31:08.584 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 1, 2024-09-21T09:31:08.584 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 7748154 2024-09-21T09:31:08.584 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.585 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r": { 2024-09-21T09:31:08.585 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.585 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.585 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r_latency": { 2024-09-21T09:31:08.585 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.585 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.585 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.585 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-21T09:31:08.585 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.585 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.585 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r_out_bytes": { 2024-09-21T09:31:08.585 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.585 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.585 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r_prepare_latency": { 2024-09-21T09:31:08.586 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.586 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.586 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.586 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r_process_latency": { 2024-09-21T09:31:08.586 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.586 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.586 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.586 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw": { 2024-09-21T09:31:08.586 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.586 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.586 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_in_bytes": { 2024-09-21T09:31:08.586 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.586 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.586 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_latency": { 2024-09-21T09:31:08.587 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.587 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.587 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.587 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-21T09:31:08.587 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.587 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.587 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-21T09:31:08.587 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.587 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.587 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_out_bytes": { 2024-09-21T09:31:08.587 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.587 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.587 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_prepare_latency": { 2024-09-21T09:31:08.587 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.588 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.588 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.588 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_process_latency": { 2024-09-21T09:31:08.588 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.588 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.588 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.588 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w": { 2024-09-21T09:31:08.588 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1 2024-09-21T09:31:08.588 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.588 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w_in_bytes": { 2024-09-21T09:31:08.588 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 22 2024-09-21T09:31:08.588 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.588 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w_latency": { 2024-09-21T09:31:08.588 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 1, 2024-09-21T09:31:08.589 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 7870434 2024-09-21T09:31:08.589 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.589 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-21T09:31:08.589 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.589 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.589 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w_prepare_latency": { 2024-09-21T09:31:08.589 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 1, 2024-09-21T09:31:08.589 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 786116 2024-09-21T09:31:08.589 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.589 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w_process_latency": { 2024-09-21T09:31:08.589 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 1, 2024-09-21T09:31:08.589 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 7748154 2024-09-21T09:31:08.589 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.589 INFO:tasks.workunit.client.0.smithi022.stdout: "op_wip": { 2024-09-21T09:31:08.589 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.590 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.590 INFO:tasks.workunit.client.0.smithi022.stdout: "recovery_bytes": { 2024-09-21T09:31:08.590 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.590 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.590 INFO:tasks.workunit.client.0.smithi022.stdout: "recovery_ops": { 2024-09-21T09:31:08.590 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.590 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.590 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_bytes": { 2024-09-21T09:31:08.590 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 95995035648 2024-09-21T09:31:08.590 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.590 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_bytes_used": { 2024-09-21T09:31:08.590 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 305201152 2024-09-21T09:31:08.590 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.590 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.591 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.591 INFO:tasks.workunit.client.0.smithi022.stdout: "osd.5": { 2024-09-21T09:31:08.591 INFO:tasks.workunit.client.0.smithi022.stdout: "bluefs": { 2024-09-21T09:31:08.591 INFO:tasks.workunit.client.0.smithi022.stdout: "alloc_slow_fallback": { 2024-09-21T09:31:08.591 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.591 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.591 INFO:tasks.workunit.client.0.smithi022.stdout: "alloc_slow_size_fallback": { 2024-09-21T09:31:08.591 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.591 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.591 INFO:tasks.workunit.client.0.smithi022.stdout: "alloc_unit_db": { 2024-09-21T09:31:08.591 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 65536 2024-09-21T09:31:08.591 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.591 INFO:tasks.workunit.client.0.smithi022.stdout: "alloc_unit_main": { 2024-09-21T09:31:08.591 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.592 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.592 INFO:tasks.workunit.client.0.smithi022.stdout: "alloc_unit_wal": { 2024-09-21T09:31:08.592 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.592 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.592 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes_written_slow": { 2024-09-21T09:31:08.592 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.592 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.592 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes_written_sst": { 2024-09-21T09:31:08.592 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 12288 2024-09-21T09:31:08.592 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.592 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes_written_wal": { 2024-09-21T09:31:08.592 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1298432 2024-09-21T09:31:08.592 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.592 INFO:tasks.workunit.client.0.smithi022.stdout: "compact_lat": { 2024-09-21T09:31:08.593 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.593 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.593 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.593 INFO:tasks.workunit.client.0.smithi022.stdout: "compact_lock_lat": { 2024-09-21T09:31:08.593 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.593 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.593 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.593 INFO:tasks.workunit.client.0.smithi022.stdout: "db_total_bytes": { 2024-09-21T09:31:08.593 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 95995035648 2024-09-21T09:31:08.593 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.593 INFO:tasks.workunit.client.0.smithi022.stdout: "db_used_bytes": { 2024-09-21T09:31:08.593 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 304480256 2024-09-21T09:31:08.593 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.594 INFO:tasks.workunit.client.0.smithi022.stdout: "log_bytes": { 2024-09-21T09:31:08.594 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 991232 2024-09-21T09:31:08.594 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.594 INFO:tasks.workunit.client.0.smithi022.stdout: "logged_bytes": { 2024-09-21T09:31:08.594 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 851968 2024-09-21T09:31:08.594 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.594 INFO:tasks.workunit.client.0.smithi022.stdout: "max_bytes_db": { 2024-09-21T09:31:08.594 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 312868864 2024-09-21T09:31:08.594 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.594 INFO:tasks.workunit.client.0.smithi022.stdout: "max_bytes_slow": { 2024-09-21T09:31:08.594 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.594 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.594 INFO:tasks.workunit.client.0.smithi022.stdout: "max_bytes_wal": { 2024-09-21T09:31:08.595 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.595 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.595 INFO:tasks.workunit.client.0.smithi022.stdout: "num_files": { 2024-09-21T09:31:08.595 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 14 2024-09-21T09:31:08.595 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.595 INFO:tasks.workunit.client.0.smithi022.stdout: "read_bytes": { 2024-09-21T09:31:08.595 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 195697 2024-09-21T09:31:08.595 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.595 INFO:tasks.workunit.client.0.smithi022.stdout: "read_count": { 2024-09-21T09:31:08.595 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 23 2024-09-21T09:31:08.595 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.595 INFO:tasks.workunit.client.0.smithi022.stdout: "read_disk_bytes": { 2024-09-21T09:31:08.595 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1269760 2024-09-21T09:31:08.595 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.596 INFO:tasks.workunit.client.0.smithi022.stdout: "read_disk_bytes_db": { 2024-09-21T09:31:08.596 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1273856 2024-09-21T09:31:08.596 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.596 INFO:tasks.workunit.client.0.smithi022.stdout: "read_disk_bytes_slow": { 2024-09-21T09:31:08.596 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.596 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.596 INFO:tasks.workunit.client.0.smithi022.stdout: "read_disk_bytes_wal": { 2024-09-21T09:31:08.596 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.596 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.596 INFO:tasks.workunit.client.0.smithi022.stdout: "read_disk_count": { 2024-09-21T09:31:08.596 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 15 2024-09-21T09:31:08.596 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.596 INFO:tasks.workunit.client.0.smithi022.stdout: "read_prefetch_bytes": { 2024-09-21T09:31:08.597 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 4789 2024-09-21T09:31:08.597 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.597 INFO:tasks.workunit.client.0.smithi022.stdout: "read_prefetch_count": { 2024-09-21T09:31:08.597 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 4 2024-09-21T09:31:08.597 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.597 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_buffer_bytes": { 2024-09-21T09:31:08.597 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 5589 2024-09-21T09:31:08.597 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.597 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_buffer_count": { 2024-09-21T09:31:08.597 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 24 2024-09-21T09:31:08.597 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.597 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_bytes": { 2024-09-21T09:31:08.597 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 5589 2024-09-21T09:31:08.598 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.598 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_count": { 2024-09-21T09:31:08.598 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 24 2024-09-21T09:31:08.598 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.598 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_disk_bytes": { 2024-09-21T09:31:08.598 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.598 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.598 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_disk_bytes_db": { 2024-09-21T09:31:08.598 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.598 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.598 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_disk_bytes_slow": { 2024-09-21T09:31:08.598 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.598 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.598 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_disk_bytes_wal": { 2024-09-21T09:31:08.599 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.599 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.599 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_disk_count": { 2024-09-21T09:31:08.599 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.599 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.599 INFO:tasks.workunit.client.0.smithi022.stdout: "slow_total_bytes": { 2024-09-21T09:31:08.599 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.599 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.599 INFO:tasks.workunit.client.0.smithi022.stdout: "slow_used_bytes": { 2024-09-21T09:31:08.599 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.599 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.599 INFO:tasks.workunit.client.0.smithi022.stdout: "wal_total_bytes": { 2024-09-21T09:31:08.599 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.599 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.600 INFO:tasks.workunit.client.0.smithi022.stdout: "wal_used_bytes": { 2024-09-21T09:31:08.600 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.600 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.600 INFO:tasks.workunit.client.0.smithi022.stdout: "write_bytes": { 2024-09-21T09:31:08.600 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2224128 2024-09-21T09:31:08.600 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.600 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.600 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore": { 2024-09-21T09:31:08.600 INFO:tasks.workunit.client.0.smithi022.stdout: "alloc_unit": { 2024-09-21T09:31:08.600 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 4096 2024-09-21T09:31:08.600 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.600 INFO:tasks.workunit.client.0.smithi022.stdout: "allocated": { 2024-09-21T09:31:08.600 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 700416 2024-09-21T09:31:08.600 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.600 INFO:tasks.workunit.client.0.smithi022.stdout: "clist_lat": { 2024-09-21T09:31:08.600 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 6, 2024-09-21T09:31:08.600 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 223263 2024-09-21T09:31:08.601 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.601 INFO:tasks.workunit.client.0.smithi022.stdout: "compress_lat": { 2024-09-21T09:31:08.601 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.601 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.601 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.601 INFO:tasks.workunit.client.0.smithi022.stdout: "compressed": { 2024-09-21T09:31:08.601 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.601 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.601 INFO:tasks.workunit.client.0.smithi022.stdout: "compressed_allocated": { 2024-09-21T09:31:08.601 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.601 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.601 INFO:tasks.workunit.client.0.smithi022.stdout: "compressed_original": { 2024-09-21T09:31:08.601 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.602 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.602 INFO:tasks.workunit.client.0.smithi022.stdout: "csum_lat": { 2024-09-21T09:31:08.602 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 8, 2024-09-21T09:31:08.602 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 21257 2024-09-21T09:31:08.602 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.602 INFO:tasks.workunit.client.0.smithi022.stdout: "decompress_lat": { 2024-09-21T09:31:08.602 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.602 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.602 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.602 INFO:tasks.workunit.client.0.smithi022.stdout: "kv_commit_lat": { 2024-09-21T09:31:08.602 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 191, 2024-09-21T09:31:08.602 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 57204104 2024-09-21T09:31:08.602 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.602 INFO:tasks.workunit.client.0.smithi022.stdout: "kv_final_lat": { 2024-09-21T09:31:08.603 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 185, 2024-09-21T09:31:08.603 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 7085387 2024-09-21T09:31:08.603 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.603 INFO:tasks.workunit.client.0.smithi022.stdout: "kv_flush_lat": { 2024-09-21T09:31:08.603 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 191, 2024-09-21T09:31:08.603 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 202500 2024-09-21T09:31:08.603 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.603 INFO:tasks.workunit.client.0.smithi022.stdout: "kv_sync_lat": { 2024-09-21T09:31:08.603 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 191, 2024-09-21T09:31:08.603 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 57406604 2024-09-21T09:31:08.603 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.603 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_get_keys_lat": { 2024-09-21T09:31:08.603 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.603 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.604 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.604 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_get_values_lat": { 2024-09-21T09:31:08.604 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 3, 2024-09-21T09:31:08.604 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 45534 2024-09-21T09:31:08.604 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.604 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_lower_bound_lat": { 2024-09-21T09:31:08.604 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 3, 2024-09-21T09:31:08.604 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 4226 2024-09-21T09:31:08.604 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.604 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_next_lat": { 2024-09-21T09:31:08.604 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.604 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.604 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.604 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_seek_to_first_lat": { 2024-09-21T09:31:08.605 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.605 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.605 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.605 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_upper_bound_lat": { 2024-09-21T09:31:08.605 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 2, 2024-09-21T09:31:08.605 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 5556 2024-09-21T09:31:08.605 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.605 INFO:tasks.workunit.client.0.smithi022.stdout: "onode_hits": { 2024-09-21T09:31:08.605 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 355 2024-09-21T09:31:08.605 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.605 INFO:tasks.workunit.client.0.smithi022.stdout: "onode_misses": { 2024-09-21T09:31:08.605 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 237 2024-09-21T09:31:08.605 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.605 INFO:tasks.workunit.client.0.smithi022.stdout: "read_lat": { 2024-09-21T09:31:08.605 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 87, 2024-09-21T09:31:08.606 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1264385 2024-09-21T09:31:08.606 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.606 INFO:tasks.workunit.client.0.smithi022.stdout: "read_onode_meta_lat": { 2024-09-21T09:31:08.606 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 140, 2024-09-21T09:31:08.606 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 265863 2024-09-21T09:31:08.606 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.606 INFO:tasks.workunit.client.0.smithi022.stdout: "read_wait_aio_lat": { 2024-09-21T09:31:08.606 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 53, 2024-09-21T09:31:08.606 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 902729 2024-09-21T09:31:08.606 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.606 INFO:tasks.workunit.client.0.smithi022.stdout: "reads_with_retries": { 2024-09-21T09:31:08.606 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.606 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.606 INFO:tasks.workunit.client.0.smithi022.stdout: "remove_lat": { 2024-09-21T09:31:08.607 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 4, 2024-09-21T09:31:08.607 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 84353 2024-09-21T09:31:08.607 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.607 INFO:tasks.workunit.client.0.smithi022.stdout: "state_aio_wait_lat": { 2024-09-21T09:31:08.607 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 284, 2024-09-21T09:31:08.607 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2665786 2024-09-21T09:31:08.607 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.607 INFO:tasks.workunit.client.0.smithi022.stdout: "state_deferred_aio_wait_lat": { 2024-09-21T09:31:08.607 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 41, 2024-09-21T09:31:08.607 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2599677 2024-09-21T09:31:08.607 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.607 INFO:tasks.workunit.client.0.smithi022.stdout: "state_deferred_cleanup_lat": { 2024-09-21T09:31:08.607 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 38, 2024-09-21T09:31:08.608 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 294532670170 2024-09-21T09:31:08.608 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.608 INFO:tasks.workunit.client.0.smithi022.stdout: "state_deferred_queued_lat": { 2024-09-21T09:31:08.608 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 41, 2024-09-21T09:31:08.608 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 66574400736 2024-09-21T09:31:08.608 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.608 INFO:tasks.workunit.client.0.smithi022.stdout: "state_done_lat": { 2024-09-21T09:31:08.608 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 281, 2024-09-21T09:31:08.608 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 7312017343 2024-09-21T09:31:08.608 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.608 INFO:tasks.workunit.client.0.smithi022.stdout: "state_finishing_lat": { 2024-09-21T09:31:08.608 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 281, 2024-09-21T09:31:08.608 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 33542 2024-09-21T09:31:08.608 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.608 INFO:tasks.workunit.client.0.smithi022.stdout: "state_io_done_lat": { 2024-09-21T09:31:08.609 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 284, 2024-09-21T09:31:08.609 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 245343 2024-09-21T09:31:08.609 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.609 INFO:tasks.workunit.client.0.smithi022.stdout: "state_kv_commiting_lat": { 2024-09-21T09:31:08.609 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 284, 2024-09-21T09:31:08.609 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 151544224 2024-09-21T09:31:08.609 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.609 INFO:tasks.workunit.client.0.smithi022.stdout: "state_kv_done_lat": { 2024-09-21T09:31:08.609 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 284, 2024-09-21T09:31:08.609 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 87321 2024-09-21T09:31:08.609 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.609 INFO:tasks.workunit.client.0.smithi022.stdout: "state_kv_queued_lat": { 2024-09-21T09:31:08.609 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 284, 2024-09-21T09:31:08.609 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 115059209 2024-09-21T09:31:08.609 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.609 INFO:tasks.workunit.client.0.smithi022.stdout: "state_prepare_lat": { 2024-09-21T09:31:08.610 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 284, 2024-09-21T09:31:08.610 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 14646137 2024-09-21T09:31:08.610 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.610 INFO:tasks.workunit.client.0.smithi022.stdout: "stored": { 2024-09-21T09:31:08.610 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 312082 2024-09-21T09:31:08.610 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.610 INFO:tasks.workunit.client.0.smithi022.stdout: "truncate_lat": { 2024-09-21T09:31:08.610 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 33, 2024-09-21T09:31:08.610 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 153003 2024-09-21T09:31:08.610 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.610 INFO:tasks.workunit.client.0.smithi022.stdout: "txc_commit_lat": { 2024-09-21T09:31:08.610 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 284, 2024-09-21T09:31:08.610 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 284197396 2024-09-21T09:31:08.611 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.611 INFO:tasks.workunit.client.0.smithi022.stdout: "txc_submit_lat": { 2024-09-21T09:31:08.611 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 284, 2024-09-21T09:31:08.611 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 17576248 2024-09-21T09:31:08.611 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.611 INFO:tasks.workunit.client.0.smithi022.stdout: "txc_throttle_lat": { 2024-09-21T09:31:08.611 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 284, 2024-09-21T09:31:08.611 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2153215 2024-09-21T09:31:08.611 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.611 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.611 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore-pricache": { 2024-09-21T09:31:08.611 INFO:tasks.workunit.client.0.smithi022.stdout: "cache_bytes": { 2024-09-21T09:31:08.611 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1861463884 2024-09-21T09:31:08.612 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.612 INFO:tasks.workunit.client.0.smithi022.stdout: "heap_bytes": { 2024-09-21T09:31:08.612 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 270974976 2024-09-21T09:31:08.612 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.612 INFO:tasks.workunit.client.0.smithi022.stdout: "mapped_bytes": { 2024-09-21T09:31:08.612 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 270041088 2024-09-21T09:31:08.612 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.612 INFO:tasks.workunit.client.0.smithi022.stdout: "target_bytes": { 2024-09-21T09:31:08.612 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 3137376768 2024-09-21T09:31:08.612 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.612 INFO:tasks.workunit.client.0.smithi022.stdout: "unmapped_bytes": { 2024-09-21T09:31:08.612 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 933888 2024-09-21T09:31:08.613 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.613 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.613 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore-pricache:data": { 2024-09-21T09:31:08.613 INFO:tasks.workunit.client.0.smithi022.stdout: "committed_bytes": { 2024-09-21T09:31:08.613 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 159383552 2024-09-21T09:31:08.613 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.613 INFO:tasks.workunit.client.0.smithi022.stdout: "pri0_bytes": { 2024-09-21T09:31:08.613 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.613 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.613 INFO:tasks.workunit.client.0.smithi022.stdout: "pri10_bytes": { 2024-09-21T09:31:08.613 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.613 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.613 INFO:tasks.workunit.client.0.smithi022.stdout: "pri11_bytes": { 2024-09-21T09:31:08.613 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 85499594 2024-09-21T09:31:08.613 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.614 INFO:tasks.workunit.client.0.smithi022.stdout: "pri1_bytes": { 2024-09-21T09:31:08.614 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.614 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.614 INFO:tasks.workunit.client.0.smithi022.stdout: "pri2_bytes": { 2024-09-21T09:31:08.614 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.614 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.614 INFO:tasks.workunit.client.0.smithi022.stdout: "pri3_bytes": { 2024-09-21T09:31:08.614 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.614 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.614 INFO:tasks.workunit.client.0.smithi022.stdout: "pri4_bytes": { 2024-09-21T09:31:08.614 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.614 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.614 INFO:tasks.workunit.client.0.smithi022.stdout: "pri5_bytes": { 2024-09-21T09:31:08.615 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.615 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.615 INFO:tasks.workunit.client.0.smithi022.stdout: "pri6_bytes": { 2024-09-21T09:31:08.615 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.615 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.615 INFO:tasks.workunit.client.0.smithi022.stdout: "pri7_bytes": { 2024-09-21T09:31:08.615 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.615 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.615 INFO:tasks.workunit.client.0.smithi022.stdout: "pri8_bytes": { 2024-09-21T09:31:08.615 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.615 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.616 INFO:tasks.workunit.client.0.smithi022.stdout: "pri9_bytes": { 2024-09-21T09:31:08.616 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.616 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.616 INFO:tasks.workunit.client.0.smithi022.stdout: "reserved_bytes": { 2024-09-21T09:31:08.616 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 73883958 2024-09-21T09:31:08.616 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.616 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.616 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore-pricache:kv": { 2024-09-21T09:31:08.616 INFO:tasks.workunit.client.0.smithi022.stdout: "committed_bytes": { 2024-09-21T09:31:08.616 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 780140544 2024-09-21T09:31:08.616 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.616 INFO:tasks.workunit.client.0.smithi022.stdout: "pri0_bytes": { 2024-09-21T09:31:08.616 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 67108864 2024-09-21T09:31:08.617 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.617 INFO:tasks.workunit.client.0.smithi022.stdout: "pri10_bytes": { 2024-09-21T09:31:08.617 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.617 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.617 INFO:tasks.workunit.client.0.smithi022.stdout: "pri11_bytes": { 2024-09-21T09:31:08.617 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 641246959 2024-09-21T09:31:08.617 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.617 INFO:tasks.workunit.client.0.smithi022.stdout: "pri1_bytes": { 2024-09-21T09:31:08.617 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.617 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.617 INFO:tasks.workunit.client.0.smithi022.stdout: "pri2_bytes": { 2024-09-21T09:31:08.617 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.617 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.617 INFO:tasks.workunit.client.0.smithi022.stdout: "pri3_bytes": { 2024-09-21T09:31:08.617 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.617 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.618 INFO:tasks.workunit.client.0.smithi022.stdout: "pri4_bytes": { 2024-09-21T09:31:08.618 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.618 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.618 INFO:tasks.workunit.client.0.smithi022.stdout: "pri5_bytes": { 2024-09-21T09:31:08.618 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.618 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.618 INFO:tasks.workunit.client.0.smithi022.stdout: "pri6_bytes": { 2024-09-21T09:31:08.618 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2128 2024-09-21T09:31:08.618 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.618 INFO:tasks.workunit.client.0.smithi022.stdout: "pri7_bytes": { 2024-09-21T09:31:08.618 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.618 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.618 INFO:tasks.workunit.client.0.smithi022.stdout: "pri8_bytes": { 2024-09-21T09:31:08.618 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.619 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.619 INFO:tasks.workunit.client.0.smithi022.stdout: "pri9_bytes": { 2024-09-21T09:31:08.619 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.619 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.619 INFO:tasks.workunit.client.0.smithi022.stdout: "reserved_bytes": { 2024-09-21T09:31:08.619 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 71782593 2024-09-21T09:31:08.619 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.619 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.619 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore-pricache:kv_onode": { 2024-09-21T09:31:08.619 INFO:tasks.workunit.client.0.smithi022.stdout: "committed_bytes": { 2024-09-21T09:31:08.619 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 192937984 2024-09-21T09:31:08.619 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.619 INFO:tasks.workunit.client.0.smithi022.stdout: "pri0_bytes": { 2024-09-21T09:31:08.620 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 67108864 2024-09-21T09:31:08.620 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.620 INFO:tasks.workunit.client.0.smithi022.stdout: "pri10_bytes": { 2024-09-21T09:31:08.620 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 464 2024-09-21T09:31:08.620 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.620 INFO:tasks.workunit.client.0.smithi022.stdout: "pri11_bytes": { 2024-09-21T09:31:08.620 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 56999729 2024-09-21T09:31:08.620 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.620 INFO:tasks.workunit.client.0.smithi022.stdout: "pri1_bytes": { 2024-09-21T09:31:08.620 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.620 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.620 INFO:tasks.workunit.client.0.smithi022.stdout: "pri2_bytes": { 2024-09-21T09:31:08.620 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.621 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.621 INFO:tasks.workunit.client.0.smithi022.stdout: "pri3_bytes": { 2024-09-21T09:31:08.621 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.621 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.621 INFO:tasks.workunit.client.0.smithi022.stdout: "pri4_bytes": { 2024-09-21T09:31:08.621 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.621 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.621 INFO:tasks.workunit.client.0.smithi022.stdout: "pri5_bytes": { 2024-09-21T09:31:08.621 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.621 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.621 INFO:tasks.workunit.client.0.smithi022.stdout: "pri6_bytes": { 2024-09-21T09:31:08.621 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.621 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.621 INFO:tasks.workunit.client.0.smithi022.stdout: "pri7_bytes": { 2024-09-21T09:31:08.622 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.622 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.622 INFO:tasks.workunit.client.0.smithi022.stdout: "pri8_bytes": { 2024-09-21T09:31:08.622 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.622 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.622 INFO:tasks.workunit.client.0.smithi022.stdout: "pri9_bytes": { 2024-09-21T09:31:08.622 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.622 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.622 INFO:tasks.workunit.client.0.smithi022.stdout: "reserved_bytes": { 2024-09-21T09:31:08.622 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 68828927 2024-09-21T09:31:08.622 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.622 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.623 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore-pricache:meta": { 2024-09-21T09:31:08.623 INFO:tasks.workunit.client.0.smithi022.stdout: "committed_bytes": { 2024-09-21T09:31:08.623 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 713031680 2024-09-21T09:31:08.623 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.623 INFO:tasks.workunit.client.0.smithi022.stdout: "pri0_bytes": { 2024-09-21T09:31:08.623 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.623 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.623 INFO:tasks.workunit.client.0.smithi022.stdout: "pri10_bytes": { 2024-09-21T09:31:08.623 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.623 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.623 INFO:tasks.workunit.client.0.smithi022.stdout: "pri11_bytes": { 2024-09-21T09:31:08.623 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 641246959 2024-09-21T09:31:08.623 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.624 INFO:tasks.workunit.client.0.smithi022.stdout: "pri1_bytes": { 2024-09-21T09:31:08.624 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.624 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.624 INFO:tasks.workunit.client.0.smithi022.stdout: "pri2_bytes": { 2024-09-21T09:31:08.624 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.624 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.624 INFO:tasks.workunit.client.0.smithi022.stdout: "pri3_bytes": { 2024-09-21T09:31:08.624 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.624 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.624 INFO:tasks.workunit.client.0.smithi022.stdout: "pri4_bytes": { 2024-09-21T09:31:08.624 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.624 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.624 INFO:tasks.workunit.client.0.smithi022.stdout: "pri5_bytes": { 2024-09-21T09:31:08.624 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 6963 2024-09-21T09:31:08.625 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.625 INFO:tasks.workunit.client.0.smithi022.stdout: "pri6_bytes": { 2024-09-21T09:31:08.625 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 253470 2024-09-21T09:31:08.625 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.625 INFO:tasks.workunit.client.0.smithi022.stdout: "pri7_bytes": { 2024-09-21T09:31:08.625 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.625 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.625 INFO:tasks.workunit.client.0.smithi022.stdout: "pri8_bytes": { 2024-09-21T09:31:08.625 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.625 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.625 INFO:tasks.workunit.client.0.smithi022.stdout: "pri9_bytes": { 2024-09-21T09:31:08.625 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.626 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.626 INFO:tasks.workunit.client.0.smithi022.stdout: "reserved_bytes": { 2024-09-21T09:31:08.626 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 71524288 2024-09-21T09:31:08.626 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.626 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.626 INFO:tasks.workunit.client.0.smithi022.stdout: "objecter": { 2024-09-21T09:31:08.626 INFO:tasks.workunit.client.0.smithi022.stdout: "op_active": { 2024-09-21T09:31:08.626 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.626 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.626 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r": { 2024-09-21T09:31:08.626 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.626 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.626 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rmw": { 2024-09-21T09:31:08.627 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.627 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.627 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w": { 2024-09-21T09:31:08.627 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.627 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.627 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.627 INFO:tasks.workunit.client.0.smithi022.stdout: "osd": { 2024-09-21T09:31:08.627 INFO:tasks.workunit.client.0.smithi022.stdout: "numpg": { 2024-09-21T09:31:08.627 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 41 2024-09-21T09:31:08.627 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.627 INFO:tasks.workunit.client.0.smithi022.stdout: "numpg_removing": { 2024-09-21T09:31:08.627 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.627 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.628 INFO:tasks.workunit.client.0.smithi022.stdout: "op": { 2024-09-21T09:31:08.628 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.628 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.628 INFO:tasks.workunit.client.0.smithi022.stdout: "op_delayed_degraded": { 2024-09-21T09:31:08.628 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.628 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.628 INFO:tasks.workunit.client.0.smithi022.stdout: "op_delayed_unreadable": { 2024-09-21T09:31:08.628 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.628 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.628 INFO:tasks.workunit.client.0.smithi022.stdout: "op_in_bytes": { 2024-09-21T09:31:08.628 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.628 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.628 INFO:tasks.workunit.client.0.smithi022.stdout: "op_latency": { 2024-09-21T09:31:08.629 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.629 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.629 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.629 INFO:tasks.workunit.client.0.smithi022.stdout: "op_out_bytes": { 2024-09-21T09:31:08.629 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.629 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.629 INFO:tasks.workunit.client.0.smithi022.stdout: "op_prepare_latency": { 2024-09-21T09:31:08.629 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.629 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.629 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.629 INFO:tasks.workunit.client.0.smithi022.stdout: "op_process_latency": { 2024-09-21T09:31:08.629 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.629 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.629 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.629 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r": { 2024-09-21T09:31:08.630 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.630 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.630 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r_latency": { 2024-09-21T09:31:08.630 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.630 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.630 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.630 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-21T09:31:08.630 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.630 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.630 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r_out_bytes": { 2024-09-21T09:31:08.630 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.630 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.630 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r_prepare_latency": { 2024-09-21T09:31:08.630 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.631 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.631 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.631 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r_process_latency": { 2024-09-21T09:31:08.631 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.631 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.631 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.631 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw": { 2024-09-21T09:31:08.631 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.631 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.631 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_in_bytes": { 2024-09-21T09:31:08.631 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.631 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.631 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_latency": { 2024-09-21T09:31:08.631 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.632 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.632 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.632 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-21T09:31:08.632 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.632 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.632 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-21T09:31:08.632 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.632 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.632 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_out_bytes": { 2024-09-21T09:31:08.632 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.632 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.632 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_prepare_latency": { 2024-09-21T09:31:08.632 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.632 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.632 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.632 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_process_latency": { 2024-09-21T09:31:08.632 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.633 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.633 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.633 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w": { 2024-09-21T09:31:08.633 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.633 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.633 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w_in_bytes": { 2024-09-21T09:31:08.633 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.633 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.633 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w_latency": { 2024-09-21T09:31:08.633 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.633 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.633 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.633 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-21T09:31:08.633 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.633 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.633 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w_prepare_latency": { 2024-09-21T09:31:08.634 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.634 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.634 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.634 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w_process_latency": { 2024-09-21T09:31:08.634 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.634 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.634 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.634 INFO:tasks.workunit.client.0.smithi022.stdout: "op_wip": { 2024-09-21T09:31:08.634 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.634 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.634 INFO:tasks.workunit.client.0.smithi022.stdout: "recovery_bytes": { 2024-09-21T09:31:08.634 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 590368 2024-09-21T09:31:08.634 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.634 INFO:tasks.workunit.client.0.smithi022.stdout: "recovery_ops": { 2024-09-21T09:31:08.635 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2 2024-09-21T09:31:08.635 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.635 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_bytes": { 2024-09-21T09:31:08.635 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 95995035648 2024-09-21T09:31:08.635 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.635 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_bytes_used": { 2024-09-21T09:31:08.635 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 305188864 2024-09-21T09:31:08.635 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.635 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.635 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.635 INFO:tasks.workunit.client.0.smithi022.stdout: "osd.6": { 2024-09-21T09:31:08.635 INFO:tasks.workunit.client.0.smithi022.stdout: "bluefs": { 2024-09-21T09:31:08.635 INFO:tasks.workunit.client.0.smithi022.stdout: "alloc_slow_fallback": { 2024-09-21T09:31:08.635 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.635 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.636 INFO:tasks.workunit.client.0.smithi022.stdout: "alloc_slow_size_fallback": { 2024-09-21T09:31:08.636 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.636 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.636 INFO:tasks.workunit.client.0.smithi022.stdout: "alloc_unit_db": { 2024-09-21T09:31:08.636 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 65536 2024-09-21T09:31:08.636 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.636 INFO:tasks.workunit.client.0.smithi022.stdout: "alloc_unit_main": { 2024-09-21T09:31:08.636 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.636 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.636 INFO:tasks.workunit.client.0.smithi022.stdout: "alloc_unit_wal": { 2024-09-21T09:31:08.636 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.636 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.636 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes_written_slow": { 2024-09-21T09:31:08.636 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.636 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.637 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes_written_sst": { 2024-09-21T09:31:08.637 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 12288 2024-09-21T09:31:08.637 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.637 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes_written_wal": { 2024-09-21T09:31:08.637 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1167360 2024-09-21T09:31:08.637 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.637 INFO:tasks.workunit.client.0.smithi022.stdout: "compact_lat": { 2024-09-21T09:31:08.637 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.637 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.637 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.637 INFO:tasks.workunit.client.0.smithi022.stdout: "compact_lock_lat": { 2024-09-21T09:31:08.637 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.637 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.637 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.638 INFO:tasks.workunit.client.0.smithi022.stdout: "db_total_bytes": { 2024-09-21T09:31:08.638 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 95995035648 2024-09-21T09:31:08.638 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.638 INFO:tasks.workunit.client.0.smithi022.stdout: "db_used_bytes": { 2024-09-21T09:31:08.638 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 304480256 2024-09-21T09:31:08.638 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.638 INFO:tasks.workunit.client.0.smithi022.stdout: "log_bytes": { 2024-09-21T09:31:08.638 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 917504 2024-09-21T09:31:08.638 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.638 INFO:tasks.workunit.client.0.smithi022.stdout: "logged_bytes": { 2024-09-21T09:31:08.638 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 778240 2024-09-21T09:31:08.638 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.638 INFO:tasks.workunit.client.0.smithi022.stdout: "max_bytes_db": { 2024-09-21T09:31:08.638 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 312868864 2024-09-21T09:31:08.638 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.639 INFO:tasks.workunit.client.0.smithi022.stdout: "max_bytes_slow": { 2024-09-21T09:31:08.639 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.639 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.639 INFO:tasks.workunit.client.0.smithi022.stdout: "max_bytes_wal": { 2024-09-21T09:31:08.639 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.639 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.639 INFO:tasks.workunit.client.0.smithi022.stdout: "num_files": { 2024-09-21T09:31:08.639 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 14 2024-09-21T09:31:08.639 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.639 INFO:tasks.workunit.client.0.smithi022.stdout: "read_bytes": { 2024-09-21T09:31:08.639 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 195697 2024-09-21T09:31:08.639 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.639 INFO:tasks.workunit.client.0.smithi022.stdout: "read_count": { 2024-09-21T09:31:08.639 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 23 2024-09-21T09:31:08.640 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.640 INFO:tasks.workunit.client.0.smithi022.stdout: "read_disk_bytes": { 2024-09-21T09:31:08.640 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1269760 2024-09-21T09:31:08.640 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.640 INFO:tasks.workunit.client.0.smithi022.stdout: "read_disk_bytes_db": { 2024-09-21T09:31:08.640 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1273856 2024-09-21T09:31:08.640 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.640 INFO:tasks.workunit.client.0.smithi022.stdout: "read_disk_bytes_slow": { 2024-09-21T09:31:08.640 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.640 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.640 INFO:tasks.workunit.client.0.smithi022.stdout: "read_disk_bytes_wal": { 2024-09-21T09:31:08.640 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.641 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.641 INFO:tasks.workunit.client.0.smithi022.stdout: "read_disk_count": { 2024-09-21T09:31:08.641 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 15 2024-09-21T09:31:08.641 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.641 INFO:tasks.workunit.client.0.smithi022.stdout: "read_prefetch_bytes": { 2024-09-21T09:31:08.641 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 4789 2024-09-21T09:31:08.641 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.641 INFO:tasks.workunit.client.0.smithi022.stdout: "read_prefetch_count": { 2024-09-21T09:31:08.641 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 4 2024-09-21T09:31:08.641 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.641 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_buffer_bytes": { 2024-09-21T09:31:08.641 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 5589 2024-09-21T09:31:08.641 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.641 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_buffer_count": { 2024-09-21T09:31:08.641 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 24 2024-09-21T09:31:08.641 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.642 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_bytes": { 2024-09-21T09:31:08.642 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 5589 2024-09-21T09:31:08.642 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.642 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_count": { 2024-09-21T09:31:08.642 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 24 2024-09-21T09:31:08.642 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.642 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_disk_bytes": { 2024-09-21T09:31:08.642 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.642 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.642 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_disk_bytes_db": { 2024-09-21T09:31:08.642 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.642 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.642 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_disk_bytes_slow": { 2024-09-21T09:31:08.642 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.643 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.643 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_disk_bytes_wal": { 2024-09-21T09:31:08.643 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.643 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.643 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_disk_count": { 2024-09-21T09:31:08.643 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.643 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.643 INFO:tasks.workunit.client.0.smithi022.stdout: "slow_total_bytes": { 2024-09-21T09:31:08.643 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.643 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.643 INFO:tasks.workunit.client.0.smithi022.stdout: "slow_used_bytes": { 2024-09-21T09:31:08.643 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.643 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.643 INFO:tasks.workunit.client.0.smithi022.stdout: "wal_total_bytes": { 2024-09-21T09:31:08.644 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.644 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.644 INFO:tasks.workunit.client.0.smithi022.stdout: "wal_used_bytes": { 2024-09-21T09:31:08.644 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.644 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.644 INFO:tasks.workunit.client.0.smithi022.stdout: "write_bytes": { 2024-09-21T09:31:08.644 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2019328 2024-09-21T09:31:08.644 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.644 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.644 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore": { 2024-09-21T09:31:08.644 INFO:tasks.workunit.client.0.smithi022.stdout: "alloc_unit": { 2024-09-21T09:31:08.644 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 4096 2024-09-21T09:31:08.644 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.644 INFO:tasks.workunit.client.0.smithi022.stdout: "allocated": { 2024-09-21T09:31:08.644 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1290240 2024-09-21T09:31:08.644 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.645 INFO:tasks.workunit.client.0.smithi022.stdout: "clist_lat": { 2024-09-21T09:31:08.645 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.645 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.645 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.645 INFO:tasks.workunit.client.0.smithi022.stdout: "compress_lat": { 2024-09-21T09:31:08.645 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.645 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.645 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.645 INFO:tasks.workunit.client.0.smithi022.stdout: "compressed": { 2024-09-21T09:31:08.645 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.645 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.645 INFO:tasks.workunit.client.0.smithi022.stdout: "compressed_allocated": { 2024-09-21T09:31:08.645 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.646 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.646 INFO:tasks.workunit.client.0.smithi022.stdout: "compressed_original": { 2024-09-21T09:31:08.646 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.646 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.646 INFO:tasks.workunit.client.0.smithi022.stdout: "csum_lat": { 2024-09-21T09:31:08.646 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 5, 2024-09-21T09:31:08.646 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 8640 2024-09-21T09:31:08.646 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.646 INFO:tasks.workunit.client.0.smithi022.stdout: "decompress_lat": { 2024-09-21T09:31:08.646 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.646 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.646 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.646 INFO:tasks.workunit.client.0.smithi022.stdout: "kv_commit_lat": { 2024-09-21T09:31:08.646 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 173, 2024-09-21T09:31:08.647 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 49130440 2024-09-21T09:31:08.647 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.647 INFO:tasks.workunit.client.0.smithi022.stdout: "kv_final_lat": { 2024-09-21T09:31:08.647 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 167, 2024-09-21T09:31:08.647 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 4765920 2024-09-21T09:31:08.647 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.647 INFO:tasks.workunit.client.0.smithi022.stdout: "kv_flush_lat": { 2024-09-21T09:31:08.647 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 173, 2024-09-21T09:31:08.647 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 172237 2024-09-21T09:31:08.647 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.647 INFO:tasks.workunit.client.0.smithi022.stdout: "kv_sync_lat": { 2024-09-21T09:31:08.647 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 173, 2024-09-21T09:31:08.647 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 49302677 2024-09-21T09:31:08.647 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.648 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_get_keys_lat": { 2024-09-21T09:31:08.648 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.648 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.648 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.648 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_get_values_lat": { 2024-09-21T09:31:08.648 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 1, 2024-09-21T09:31:08.648 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 39943 2024-09-21T09:31:08.648 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.648 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_lower_bound_lat": { 2024-09-21T09:31:08.648 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 2, 2024-09-21T09:31:08.648 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 12870 2024-09-21T09:31:08.648 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.648 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_next_lat": { 2024-09-21T09:31:08.648 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.649 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.649 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.649 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_seek_to_first_lat": { 2024-09-21T09:31:08.649 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.649 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.649 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.649 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_upper_bound_lat": { 2024-09-21T09:31:08.649 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 3, 2024-09-21T09:31:08.649 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 8352 2024-09-21T09:31:08.649 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.649 INFO:tasks.workunit.client.0.smithi022.stdout: "onode_hits": { 2024-09-21T09:31:08.649 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 308 2024-09-21T09:31:08.649 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.649 INFO:tasks.workunit.client.0.smithi022.stdout: "onode_misses": { 2024-09-21T09:31:08.650 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 241 2024-09-21T09:31:08.650 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.650 INFO:tasks.workunit.client.0.smithi022.stdout: "read_lat": { 2024-09-21T09:31:08.650 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 86, 2024-09-21T09:31:08.650 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 900908 2024-09-21T09:31:08.650 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.650 INFO:tasks.workunit.client.0.smithi022.stdout: "read_onode_meta_lat": { 2024-09-21T09:31:08.650 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 134, 2024-09-21T09:31:08.650 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 251268 2024-09-21T09:31:08.650 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.650 INFO:tasks.workunit.client.0.smithi022.stdout: "read_wait_aio_lat": { 2024-09-21T09:31:08.650 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 48, 2024-09-21T09:31:08.650 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 612055 2024-09-21T09:31:08.650 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.651 INFO:tasks.workunit.client.0.smithi022.stdout: "reads_with_retries": { 2024-09-21T09:31:08.651 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.651 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.651 INFO:tasks.workunit.client.0.smithi022.stdout: "remove_lat": { 2024-09-21T09:31:08.651 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.651 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.651 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.651 INFO:tasks.workunit.client.0.smithi022.stdout: "state_aio_wait_lat": { 2024-09-21T09:31:08.651 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 245, 2024-09-21T09:31:08.651 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2747998 2024-09-21T09:31:08.651 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.651 INFO:tasks.workunit.client.0.smithi022.stdout: "state_deferred_aio_wait_lat": { 2024-09-21T09:31:08.651 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 36, 2024-09-21T09:31:08.651 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2510672 2024-09-21T09:31:08.652 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.652 INFO:tasks.workunit.client.0.smithi022.stdout: "state_deferred_cleanup_lat": { 2024-09-21T09:31:08.652 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 33, 2024-09-21T09:31:08.652 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 251855406390 2024-09-21T09:31:08.652 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.652 INFO:tasks.workunit.client.0.smithi022.stdout: "state_deferred_queued_lat": { 2024-09-21T09:31:08.652 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 36, 2024-09-21T09:31:08.652 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 54118203799 2024-09-21T09:31:08.652 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.652 INFO:tasks.workunit.client.0.smithi022.stdout: "state_done_lat": { 2024-09-21T09:31:08.652 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 242, 2024-09-21T09:31:08.652 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 460560 2024-09-21T09:31:08.652 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.652 INFO:tasks.workunit.client.0.smithi022.stdout: "state_finishing_lat": { 2024-09-21T09:31:08.653 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 242, 2024-09-21T09:31:08.653 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 27951 2024-09-21T09:31:08.653 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.653 INFO:tasks.workunit.client.0.smithi022.stdout: "state_io_done_lat": { 2024-09-21T09:31:08.653 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 245, 2024-09-21T09:31:08.653 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 200260 2024-09-21T09:31:08.653 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.653 INFO:tasks.workunit.client.0.smithi022.stdout: "state_kv_commiting_lat": { 2024-09-21T09:31:08.653 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 245, 2024-09-21T09:31:08.653 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 99524523 2024-09-21T09:31:08.653 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.653 INFO:tasks.workunit.client.0.smithi022.stdout: "state_kv_done_lat": { 2024-09-21T09:31:08.653 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 245, 2024-09-21T09:31:08.653 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 79625 2024-09-21T09:31:08.654 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.654 INFO:tasks.workunit.client.0.smithi022.stdout: "state_kv_queued_lat": { 2024-09-21T09:31:08.654 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 245, 2024-09-21T09:31:08.654 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 67615661 2024-09-21T09:31:08.654 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.654 INFO:tasks.workunit.client.0.smithi022.stdout: "state_prepare_lat": { 2024-09-21T09:31:08.654 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 245, 2024-09-21T09:31:08.654 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 17205604 2024-09-21T09:31:08.654 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.654 INFO:tasks.workunit.client.0.smithi022.stdout: "stored": { 2024-09-21T09:31:08.654 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 902656 2024-09-21T09:31:08.654 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.654 INFO:tasks.workunit.client.0.smithi022.stdout: "truncate_lat": { 2024-09-21T09:31:08.654 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 29, 2024-09-21T09:31:08.654 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 128923 2024-09-21T09:31:08.655 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.655 INFO:tasks.workunit.client.0.smithi022.stdout: "txc_commit_lat": { 2024-09-21T09:31:08.655 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 245, 2024-09-21T09:31:08.655 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 187328793 2024-09-21T09:31:08.655 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.655 INFO:tasks.workunit.client.0.smithi022.stdout: "txc_submit_lat": { 2024-09-21T09:31:08.655 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 245, 2024-09-21T09:31:08.655 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 19285473 2024-09-21T09:31:08.655 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.655 INFO:tasks.workunit.client.0.smithi022.stdout: "txc_throttle_lat": { 2024-09-21T09:31:08.655 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 245, 2024-09-21T09:31:08.655 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 5521377 2024-09-21T09:31:08.655 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.655 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.656 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore-pricache": { 2024-09-21T09:31:08.656 INFO:tasks.workunit.client.0.smithi022.stdout: "cache_bytes": { 2024-09-21T09:31:08.656 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1861463884 2024-09-21T09:31:08.656 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.656 INFO:tasks.workunit.client.0.smithi022.stdout: "heap_bytes": { 2024-09-21T09:31:08.656 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 269090816 2024-09-21T09:31:08.656 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.656 INFO:tasks.workunit.client.0.smithi022.stdout: "mapped_bytes": { 2024-09-21T09:31:08.656 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 268804096 2024-09-21T09:31:08.656 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.656 INFO:tasks.workunit.client.0.smithi022.stdout: "target_bytes": { 2024-09-21T09:31:08.656 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 3137376768 2024-09-21T09:31:08.656 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.656 INFO:tasks.workunit.client.0.smithi022.stdout: "unmapped_bytes": { 2024-09-21T09:31:08.657 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 286720 2024-09-21T09:31:08.657 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.657 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.657 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore-pricache:data": { 2024-09-21T09:31:08.657 INFO:tasks.workunit.client.0.smithi022.stdout: "committed_bytes": { 2024-09-21T09:31:08.657 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 159383552 2024-09-21T09:31:08.657 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.657 INFO:tasks.workunit.client.0.smithi022.stdout: "pri0_bytes": { 2024-09-21T09:31:08.657 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.657 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.657 INFO:tasks.workunit.client.0.smithi022.stdout: "pri10_bytes": { 2024-09-21T09:31:08.657 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.657 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.657 INFO:tasks.workunit.client.0.smithi022.stdout: "pri11_bytes": { 2024-09-21T09:31:08.657 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 85498940 2024-09-21T09:31:08.658 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.658 INFO:tasks.workunit.client.0.smithi022.stdout: "pri1_bytes": { 2024-09-21T09:31:08.658 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.658 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.658 INFO:tasks.workunit.client.0.smithi022.stdout: "pri2_bytes": { 2024-09-21T09:31:08.658 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.658 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.658 INFO:tasks.workunit.client.0.smithi022.stdout: "pri3_bytes": { 2024-09-21T09:31:08.658 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.658 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.658 INFO:tasks.workunit.client.0.smithi022.stdout: "pri4_bytes": { 2024-09-21T09:31:08.658 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.658 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.659 INFO:tasks.workunit.client.0.smithi022.stdout: "pri5_bytes": { 2024-09-21T09:31:08.659 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.659 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.659 INFO:tasks.workunit.client.0.smithi022.stdout: "pri6_bytes": { 2024-09-21T09:31:08.659 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 8192 2024-09-21T09:31:08.659 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.659 INFO:tasks.workunit.client.0.smithi022.stdout: "pri7_bytes": { 2024-09-21T09:31:08.659 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.659 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.659 INFO:tasks.workunit.client.0.smithi022.stdout: "pri8_bytes": { 2024-09-21T09:31:08.659 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.659 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.659 INFO:tasks.workunit.client.0.smithi022.stdout: "pri9_bytes": { 2024-09-21T09:31:08.659 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.660 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.660 INFO:tasks.workunit.client.0.smithi022.stdout: "reserved_bytes": { 2024-09-21T09:31:08.660 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 73876420 2024-09-21T09:31:08.660 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.660 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.660 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore-pricache:kv": { 2024-09-21T09:31:08.660 INFO:tasks.workunit.client.0.smithi022.stdout: "committed_bytes": { 2024-09-21T09:31:08.660 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 780140544 2024-09-21T09:31:08.660 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.660 INFO:tasks.workunit.client.0.smithi022.stdout: "pri0_bytes": { 2024-09-21T09:31:08.660 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 67108864 2024-09-21T09:31:08.660 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.660 INFO:tasks.workunit.client.0.smithi022.stdout: "pri10_bytes": { 2024-09-21T09:31:08.660 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.661 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.661 INFO:tasks.workunit.client.0.smithi022.stdout: "pri11_bytes": { 2024-09-21T09:31:08.661 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 641242056 2024-09-21T09:31:08.661 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.661 INFO:tasks.workunit.client.0.smithi022.stdout: "pri1_bytes": { 2024-09-21T09:31:08.661 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.661 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.661 INFO:tasks.workunit.client.0.smithi022.stdout: "pri2_bytes": { 2024-09-21T09:31:08.661 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.661 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.661 INFO:tasks.workunit.client.0.smithi022.stdout: "pri3_bytes": { 2024-09-21T09:31:08.661 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.661 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.661 INFO:tasks.workunit.client.0.smithi022.stdout: "pri4_bytes": { 2024-09-21T09:31:08.661 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.662 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.662 INFO:tasks.workunit.client.0.smithi022.stdout: "pri5_bytes": { 2024-09-21T09:31:08.662 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.662 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.662 INFO:tasks.workunit.client.0.smithi022.stdout: "pri6_bytes": { 2024-09-21T09:31:08.662 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2128 2024-09-21T09:31:08.662 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.662 INFO:tasks.workunit.client.0.smithi022.stdout: "pri7_bytes": { 2024-09-21T09:31:08.662 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.662 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.662 INFO:tasks.workunit.client.0.smithi022.stdout: "pri8_bytes": { 2024-09-21T09:31:08.662 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.662 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.662 INFO:tasks.workunit.client.0.smithi022.stdout: "pri9_bytes": { 2024-09-21T09:31:08.663 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.663 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.663 INFO:tasks.workunit.client.0.smithi022.stdout: "reserved_bytes": { 2024-09-21T09:31:08.663 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 71787496 2024-09-21T09:31:08.663 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.663 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.663 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore-pricache:kv_onode": { 2024-09-21T09:31:08.663 INFO:tasks.workunit.client.0.smithi022.stdout: "committed_bytes": { 2024-09-21T09:31:08.663 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 192937984 2024-09-21T09:31:08.663 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.663 INFO:tasks.workunit.client.0.smithi022.stdout: "pri0_bytes": { 2024-09-21T09:31:08.663 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 67108864 2024-09-21T09:31:08.663 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.663 INFO:tasks.workunit.client.0.smithi022.stdout: "pri10_bytes": { 2024-09-21T09:31:08.663 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 464 2024-09-21T09:31:08.664 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.664 INFO:tasks.workunit.client.0.smithi022.stdout: "pri11_bytes": { 2024-09-21T09:31:08.664 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 56999293 2024-09-21T09:31:08.664 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.664 INFO:tasks.workunit.client.0.smithi022.stdout: "pri1_bytes": { 2024-09-21T09:31:08.664 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.664 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.664 INFO:tasks.workunit.client.0.smithi022.stdout: "pri2_bytes": { 2024-09-21T09:31:08.664 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.664 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.664 INFO:tasks.workunit.client.0.smithi022.stdout: "pri3_bytes": { 2024-09-21T09:31:08.664 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.664 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.664 INFO:tasks.workunit.client.0.smithi022.stdout: "pri4_bytes": { 2024-09-21T09:31:08.665 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.665 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.665 INFO:tasks.workunit.client.0.smithi022.stdout: "pri5_bytes": { 2024-09-21T09:31:08.665 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.665 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.665 INFO:tasks.workunit.client.0.smithi022.stdout: "pri6_bytes": { 2024-09-21T09:31:08.665 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.665 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.665 INFO:tasks.workunit.client.0.smithi022.stdout: "pri7_bytes": { 2024-09-21T09:31:08.665 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.665 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.665 INFO:tasks.workunit.client.0.smithi022.stdout: "pri8_bytes": { 2024-09-21T09:31:08.665 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.665 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.665 INFO:tasks.workunit.client.0.smithi022.stdout: "pri9_bytes": { 2024-09-21T09:31:08.666 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.666 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.666 INFO:tasks.workunit.client.0.smithi022.stdout: "reserved_bytes": { 2024-09-21T09:31:08.666 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 68829363 2024-09-21T09:31:08.666 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.666 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.666 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore-pricache:meta": { 2024-09-21T09:31:08.666 INFO:tasks.workunit.client.0.smithi022.stdout: "committed_bytes": { 2024-09-21T09:31:08.666 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 713031680 2024-09-21T09:31:08.666 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.666 INFO:tasks.workunit.client.0.smithi022.stdout: "pri0_bytes": { 2024-09-21T09:31:08.666 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.666 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.666 INFO:tasks.workunit.client.0.smithi022.stdout: "pri10_bytes": { 2024-09-21T09:31:08.666 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.667 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.667 INFO:tasks.workunit.client.0.smithi022.stdout: "pri11_bytes": { 2024-09-21T09:31:08.667 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 641242056 2024-09-21T09:31:08.667 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.667 INFO:tasks.workunit.client.0.smithi022.stdout: "pri1_bytes": { 2024-09-21T09:31:08.667 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.667 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.667 INFO:tasks.workunit.client.0.smithi022.stdout: "pri2_bytes": { 2024-09-21T09:31:08.667 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.667 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.667 INFO:tasks.workunit.client.0.smithi022.stdout: "pri3_bytes": { 2024-09-21T09:31:08.667 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.667 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.668 INFO:tasks.workunit.client.0.smithi022.stdout: "pri4_bytes": { 2024-09-21T09:31:08.668 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.668 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.668 INFO:tasks.workunit.client.0.smithi022.stdout: "pri5_bytes": { 2024-09-21T09:31:08.668 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 7111 2024-09-21T09:31:08.668 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.668 INFO:tasks.workunit.client.0.smithi022.stdout: "pri6_bytes": { 2024-09-21T09:31:08.668 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 256025 2024-09-21T09:31:08.668 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.668 INFO:tasks.workunit.client.0.smithi022.stdout: "pri7_bytes": { 2024-09-21T09:31:08.668 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.668 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.668 INFO:tasks.workunit.client.0.smithi022.stdout: "pri8_bytes": { 2024-09-21T09:31:08.668 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.669 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.669 INFO:tasks.workunit.client.0.smithi022.stdout: "pri9_bytes": { 2024-09-21T09:31:08.669 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.669 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.669 INFO:tasks.workunit.client.0.smithi022.stdout: "reserved_bytes": { 2024-09-21T09:31:08.669 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 71526488 2024-09-21T09:31:08.669 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.669 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.669 INFO:tasks.workunit.client.0.smithi022.stdout: "objecter": { 2024-09-21T09:31:08.669 INFO:tasks.workunit.client.0.smithi022.stdout: "op_active": { 2024-09-21T09:31:08.669 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.669 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.669 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r": { 2024-09-21T09:31:08.670 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.670 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.670 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rmw": { 2024-09-21T09:31:08.670 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.670 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.670 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w": { 2024-09-21T09:31:08.670 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.670 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.670 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.670 INFO:tasks.workunit.client.0.smithi022.stdout: "osd": { 2024-09-21T09:31:08.670 INFO:tasks.workunit.client.0.smithi022.stdout: "numpg": { 2024-09-21T09:31:08.670 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 39 2024-09-21T09:31:08.670 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.671 INFO:tasks.workunit.client.0.smithi022.stdout: "numpg_removing": { 2024-09-21T09:31:08.671 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.671 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.671 INFO:tasks.workunit.client.0.smithi022.stdout: "op": { 2024-09-21T09:31:08.671 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 13 2024-09-21T09:31:08.671 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.671 INFO:tasks.workunit.client.0.smithi022.stdout: "op_delayed_degraded": { 2024-09-21T09:31:08.671 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.671 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.671 INFO:tasks.workunit.client.0.smithi022.stdout: "op_delayed_unreadable": { 2024-09-21T09:31:08.671 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.671 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.671 INFO:tasks.workunit.client.0.smithi022.stdout: "op_in_bytes": { 2024-09-21T09:31:08.672 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2804 2024-09-21T09:31:08.672 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.672 INFO:tasks.workunit.client.0.smithi022.stdout: "op_latency": { 2024-09-21T09:31:08.672 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 13, 2024-09-21T09:31:08.672 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 68095634 2024-09-21T09:31:08.672 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.672 INFO:tasks.workunit.client.0.smithi022.stdout: "op_out_bytes": { 2024-09-21T09:31:08.672 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.672 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.672 INFO:tasks.workunit.client.0.smithi022.stdout: "op_prepare_latency": { 2024-09-21T09:31:08.672 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 15, 2024-09-21T09:31:08.672 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 9750663 2024-09-21T09:31:08.672 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.672 INFO:tasks.workunit.client.0.smithi022.stdout: "op_process_latency": { 2024-09-21T09:31:08.673 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 13, 2024-09-21T09:31:08.673 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 27964068 2024-09-21T09:31:08.673 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.673 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r": { 2024-09-21T09:31:08.673 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.673 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.673 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r_latency": { 2024-09-21T09:31:08.673 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.673 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.673 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.673 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-21T09:31:08.673 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.673 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.673 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r_out_bytes": { 2024-09-21T09:31:08.674 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.674 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.674 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r_prepare_latency": { 2024-09-21T09:31:08.674 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.674 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.674 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.674 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r_process_latency": { 2024-09-21T09:31:08.674 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.674 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.674 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.674 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw": { 2024-09-21T09:31:08.674 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.674 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.675 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_in_bytes": { 2024-09-21T09:31:08.675 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.675 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.675 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_latency": { 2024-09-21T09:31:08.675 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.675 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.675 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.675 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-21T09:31:08.675 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.675 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.675 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-21T09:31:08.675 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.675 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.676 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_out_bytes": { 2024-09-21T09:31:08.676 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.676 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.676 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_prepare_latency": { 2024-09-21T09:31:08.676 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.676 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.676 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.676 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_process_latency": { 2024-09-21T09:31:08.676 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.676 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.676 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.676 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w": { 2024-09-21T09:31:08.676 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 13 2024-09-21T09:31:08.677 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.677 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w_in_bytes": { 2024-09-21T09:31:08.677 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2804 2024-09-21T09:31:08.677 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.677 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w_latency": { 2024-09-21T09:31:08.677 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 13, 2024-09-21T09:31:08.677 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 68095634 2024-09-21T09:31:08.677 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.677 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-21T09:31:08.677 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.677 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.677 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w_prepare_latency": { 2024-09-21T09:31:08.677 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 15, 2024-09-21T09:31:08.677 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 9750663 2024-09-21T09:31:08.677 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.678 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w_process_latency": { 2024-09-21T09:31:08.678 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 13, 2024-09-21T09:31:08.678 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 27964068 2024-09-21T09:31:08.678 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.678 INFO:tasks.workunit.client.0.smithi022.stdout: "op_wip": { 2024-09-21T09:31:08.678 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.678 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.678 INFO:tasks.workunit.client.0.smithi022.stdout: "recovery_bytes": { 2024-09-21T09:31:08.678 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 590368 2024-09-21T09:31:08.678 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.678 INFO:tasks.workunit.client.0.smithi022.stdout: "recovery_ops": { 2024-09-21T09:31:08.678 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2 2024-09-21T09:31:08.678 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.678 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_bytes": { 2024-09-21T09:31:08.679 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 95995035648 2024-09-21T09:31:08.679 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.679 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_bytes_used": { 2024-09-21T09:31:08.679 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 305778688 2024-09-21T09:31:08.679 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.679 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.679 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.679 INFO:tasks.workunit.client.0.smithi022.stdout: "osd.7": { 2024-09-21T09:31:08.679 INFO:tasks.workunit.client.0.smithi022.stdout: "bluefs": { 2024-09-21T09:31:08.679 INFO:tasks.workunit.client.0.smithi022.stdout: "alloc_slow_fallback": { 2024-09-21T09:31:08.679 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.679 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.679 INFO:tasks.workunit.client.0.smithi022.stdout: "alloc_slow_size_fallback": { 2024-09-21T09:31:08.680 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.680 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.680 INFO:tasks.workunit.client.0.smithi022.stdout: "alloc_unit_db": { 2024-09-21T09:31:08.680 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 65536 2024-09-21T09:31:08.680 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.680 INFO:tasks.workunit.client.0.smithi022.stdout: "alloc_unit_main": { 2024-09-21T09:31:08.680 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.680 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.680 INFO:tasks.workunit.client.0.smithi022.stdout: "alloc_unit_wal": { 2024-09-21T09:31:08.680 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.680 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.680 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes_written_slow": { 2024-09-21T09:31:08.680 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.680 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.680 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes_written_sst": { 2024-09-21T09:31:08.681 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 12288 2024-09-21T09:31:08.681 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.681 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes_written_wal": { 2024-09-21T09:31:08.681 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1101824 2024-09-21T09:31:08.681 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.681 INFO:tasks.workunit.client.0.smithi022.stdout: "compact_lat": { 2024-09-21T09:31:08.681 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.681 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.681 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.681 INFO:tasks.workunit.client.0.smithi022.stdout: "compact_lock_lat": { 2024-09-21T09:31:08.681 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.681 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.681 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.681 INFO:tasks.workunit.client.0.smithi022.stdout: "db_total_bytes": { 2024-09-21T09:31:08.682 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 95995035648 2024-09-21T09:31:08.682 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.682 INFO:tasks.workunit.client.0.smithi022.stdout: "db_used_bytes": { 2024-09-21T09:31:08.682 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 304480256 2024-09-21T09:31:08.682 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.682 INFO:tasks.workunit.client.0.smithi022.stdout: "log_bytes": { 2024-09-21T09:31:08.682 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 872448 2024-09-21T09:31:08.682 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.682 INFO:tasks.workunit.client.0.smithi022.stdout: "logged_bytes": { 2024-09-21T09:31:08.682 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 733184 2024-09-21T09:31:08.682 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.682 INFO:tasks.workunit.client.0.smithi022.stdout: "max_bytes_db": { 2024-09-21T09:31:08.682 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 312868864 2024-09-21T09:31:08.682 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.683 INFO:tasks.workunit.client.0.smithi022.stdout: "max_bytes_slow": { 2024-09-21T09:31:08.683 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.683 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.683 INFO:tasks.workunit.client.0.smithi022.stdout: "max_bytes_wal": { 2024-09-21T09:31:08.683 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.683 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.683 INFO:tasks.workunit.client.0.smithi022.stdout: "num_files": { 2024-09-21T09:31:08.683 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 14 2024-09-21T09:31:08.683 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.683 INFO:tasks.workunit.client.0.smithi022.stdout: "read_bytes": { 2024-09-21T09:31:08.683 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 195697 2024-09-21T09:31:08.683 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.683 INFO:tasks.workunit.client.0.smithi022.stdout: "read_count": { 2024-09-21T09:31:08.683 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 23 2024-09-21T09:31:08.683 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.684 INFO:tasks.workunit.client.0.smithi022.stdout: "read_disk_bytes": { 2024-09-21T09:31:08.684 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1269760 2024-09-21T09:31:08.684 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.684 INFO:tasks.workunit.client.0.smithi022.stdout: "read_disk_bytes_db": { 2024-09-21T09:31:08.684 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1273856 2024-09-21T09:31:08.684 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.684 INFO:tasks.workunit.client.0.smithi022.stdout: "read_disk_bytes_slow": { 2024-09-21T09:31:08.684 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.684 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.684 INFO:tasks.workunit.client.0.smithi022.stdout: "read_disk_bytes_wal": { 2024-09-21T09:31:08.684 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.684 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.684 INFO:tasks.workunit.client.0.smithi022.stdout: "read_disk_count": { 2024-09-21T09:31:08.684 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 15 2024-09-21T09:31:08.685 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.685 INFO:tasks.workunit.client.0.smithi022.stdout: "read_prefetch_bytes": { 2024-09-21T09:31:08.685 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 4789 2024-09-21T09:31:08.685 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.685 INFO:tasks.workunit.client.0.smithi022.stdout: "read_prefetch_count": { 2024-09-21T09:31:08.685 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 4 2024-09-21T09:31:08.685 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.685 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_buffer_bytes": { 2024-09-21T09:31:08.685 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 5589 2024-09-21T09:31:08.685 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.685 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_buffer_count": { 2024-09-21T09:31:08.685 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 24 2024-09-21T09:31:08.685 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.685 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_bytes": { 2024-09-21T09:31:08.686 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 5589 2024-09-21T09:31:08.686 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.686 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_count": { 2024-09-21T09:31:08.686 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 24 2024-09-21T09:31:08.686 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.686 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_disk_bytes": { 2024-09-21T09:31:08.686 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.686 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.686 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_disk_bytes_db": { 2024-09-21T09:31:08.686 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.686 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.686 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_disk_bytes_slow": { 2024-09-21T09:31:08.686 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.686 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.687 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_disk_bytes_wal": { 2024-09-21T09:31:08.687 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.687 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.687 INFO:tasks.workunit.client.0.smithi022.stdout: "read_random_disk_count": { 2024-09-21T09:31:08.687 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.687 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.687 INFO:tasks.workunit.client.0.smithi022.stdout: "slow_total_bytes": { 2024-09-21T09:31:08.687 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.687 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.687 INFO:tasks.workunit.client.0.smithi022.stdout: "slow_used_bytes": { 2024-09-21T09:31:08.687 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.687 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.687 INFO:tasks.workunit.client.0.smithi022.stdout: "wal_total_bytes": { 2024-09-21T09:31:08.688 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.688 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.688 INFO:tasks.workunit.client.0.smithi022.stdout: "wal_used_bytes": { 2024-09-21T09:31:08.688 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.688 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.688 INFO:tasks.workunit.client.0.smithi022.stdout: "write_bytes": { 2024-09-21T09:31:08.688 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1908736 2024-09-21T09:31:08.688 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.688 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.688 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore": { 2024-09-21T09:31:08.688 INFO:tasks.workunit.client.0.smithi022.stdout: "alloc_unit": { 2024-09-21T09:31:08.688 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 4096 2024-09-21T09:31:08.688 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.688 INFO:tasks.workunit.client.0.smithi022.stdout: "allocated": { 2024-09-21T09:31:08.688 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1290240 2024-09-21T09:31:08.689 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.689 INFO:tasks.workunit.client.0.smithi022.stdout: "clist_lat": { 2024-09-21T09:31:08.689 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 3, 2024-09-21T09:31:08.689 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 137214 2024-09-21T09:31:08.689 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.689 INFO:tasks.workunit.client.0.smithi022.stdout: "compress_lat": { 2024-09-21T09:31:08.689 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.689 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.689 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.689 INFO:tasks.workunit.client.0.smithi022.stdout: "compressed": { 2024-09-21T09:31:08.689 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.689 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.689 INFO:tasks.workunit.client.0.smithi022.stdout: "compressed_allocated": { 2024-09-21T09:31:08.689 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.690 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.690 INFO:tasks.workunit.client.0.smithi022.stdout: "compressed_original": { 2024-09-21T09:31:08.690 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.690 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.690 INFO:tasks.workunit.client.0.smithi022.stdout: "csum_lat": { 2024-09-21T09:31:08.690 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 6, 2024-09-21T09:31:08.690 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 18916 2024-09-21T09:31:08.690 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.690 INFO:tasks.workunit.client.0.smithi022.stdout: "decompress_lat": { 2024-09-21T09:31:08.690 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.690 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.690 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.690 INFO:tasks.workunit.client.0.smithi022.stdout: "kv_commit_lat": { 2024-09-21T09:31:08.690 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 162, 2024-09-21T09:31:08.691 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 44052102 2024-09-21T09:31:08.691 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.691 INFO:tasks.workunit.client.0.smithi022.stdout: "kv_final_lat": { 2024-09-21T09:31:08.691 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 148, 2024-09-21T09:31:08.691 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 9993526 2024-09-21T09:31:08.691 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.691 INFO:tasks.workunit.client.0.smithi022.stdout: "kv_flush_lat": { 2024-09-21T09:31:08.691 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 162, 2024-09-21T09:31:08.691 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 138154 2024-09-21T09:31:08.691 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.691 INFO:tasks.workunit.client.0.smithi022.stdout: "kv_sync_lat": { 2024-09-21T09:31:08.691 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 162, 2024-09-21T09:31:08.691 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 44190256 2024-09-21T09:31:08.691 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.691 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_get_keys_lat": { 2024-09-21T09:31:08.692 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.692 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.692 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.692 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_get_values_lat": { 2024-09-21T09:31:08.692 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 1, 2024-09-21T09:31:08.692 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 35500 2024-09-21T09:31:08.692 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.692 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_lower_bound_lat": { 2024-09-21T09:31:08.692 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 2, 2024-09-21T09:31:08.692 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1026 2024-09-21T09:31:08.692 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.692 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_next_lat": { 2024-09-21T09:31:08.692 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.692 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.693 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.693 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_seek_to_first_lat": { 2024-09-21T09:31:08.693 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 0, 2024-09-21T09:31:08.693 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.693 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.693 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_upper_bound_lat": { 2024-09-21T09:31:08.693 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 2, 2024-09-21T09:31:08.693 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 4201 2024-09-21T09:31:08.693 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.693 INFO:tasks.workunit.client.0.smithi022.stdout: "onode_hits": { 2024-09-21T09:31:08.693 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 360 2024-09-21T09:31:08.693 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.693 INFO:tasks.workunit.client.0.smithi022.stdout: "onode_misses": { 2024-09-21T09:31:08.693 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 257 2024-09-21T09:31:08.693 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.694 INFO:tasks.workunit.client.0.smithi022.stdout: "read_lat": { 2024-09-21T09:31:08.694 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 90, 2024-09-21T09:31:08.694 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1583211 2024-09-21T09:31:08.694 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.694 INFO:tasks.workunit.client.0.smithi022.stdout: "read_onode_meta_lat": { 2024-09-21T09:31:08.694 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 134, 2024-09-21T09:31:08.694 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 239391 2024-09-21T09:31:08.694 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.694 INFO:tasks.workunit.client.0.smithi022.stdout: "read_wait_aio_lat": { 2024-09-21T09:31:08.694 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 44, 2024-09-21T09:31:08.694 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1027252 2024-09-21T09:31:08.694 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.694 INFO:tasks.workunit.client.0.smithi022.stdout: "reads_with_retries": { 2024-09-21T09:31:08.695 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.695 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.695 INFO:tasks.workunit.client.0.smithi022.stdout: "remove_lat": { 2024-09-21T09:31:08.695 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 1, 2024-09-21T09:31:08.695 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 15458 2024-09-21T09:31:08.695 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.695 INFO:tasks.workunit.client.0.smithi022.stdout: "state_aio_wait_lat": { 2024-09-21T09:31:08.695 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 273, 2024-09-21T09:31:08.695 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2128426 2024-09-21T09:31:08.695 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.695 INFO:tasks.workunit.client.0.smithi022.stdout: "state_deferred_aio_wait_lat": { 2024-09-21T09:31:08.695 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 24, 2024-09-21T09:31:08.695 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1767705 2024-09-21T09:31:08.695 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.696 INFO:tasks.workunit.client.0.smithi022.stdout: "state_deferred_cleanup_lat": { 2024-09-21T09:31:08.696 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 23, 2024-09-21T09:31:08.696 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 153476156123 2024-09-21T09:31:08.696 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.696 INFO:tasks.workunit.client.0.smithi022.stdout: "state_deferred_queued_lat": { 2024-09-21T09:31:08.696 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 24, 2024-09-21T09:31:08.696 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 43631242718 2024-09-21T09:31:08.696 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.696 INFO:tasks.workunit.client.0.smithi022.stdout: "state_done_lat": { 2024-09-21T09:31:08.696 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 272, 2024-09-21T09:31:08.696 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 424514 2024-09-21T09:31:08.696 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.696 INFO:tasks.workunit.client.0.smithi022.stdout: "state_finishing_lat": { 2024-09-21T09:31:08.696 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 272, 2024-09-21T09:31:08.696 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 32163 2024-09-21T09:31:08.697 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.697 INFO:tasks.workunit.client.0.smithi022.stdout: "state_io_done_lat": { 2024-09-21T09:31:08.697 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 273, 2024-09-21T09:31:08.697 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 200068 2024-09-21T09:31:08.697 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.697 INFO:tasks.workunit.client.0.smithi022.stdout: "state_kv_commiting_lat": { 2024-09-21T09:31:08.697 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 273, 2024-09-21T09:31:08.697 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 172372448 2024-09-21T09:31:08.697 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.697 INFO:tasks.workunit.client.0.smithi022.stdout: "state_kv_done_lat": { 2024-09-21T09:31:08.697 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 273, 2024-09-21T09:31:08.697 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 81417 2024-09-21T09:31:08.697 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.698 INFO:tasks.workunit.client.0.smithi022.stdout: "state_kv_queued_lat": { 2024-09-21T09:31:08.698 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 273, 2024-09-21T09:31:08.698 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 105292381 2024-09-21T09:31:08.698 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.698 INFO:tasks.workunit.client.0.smithi022.stdout: "state_prepare_lat": { 2024-09-21T09:31:08.698 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 273, 2024-09-21T09:31:08.698 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 12978219 2024-09-21T09:31:08.698 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.698 INFO:tasks.workunit.client.0.smithi022.stdout: "stored": { 2024-09-21T09:31:08.698 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 900939 2024-09-21T09:31:08.698 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.698 INFO:tasks.workunit.client.0.smithi022.stdout: "truncate_lat": { 2024-09-21T09:31:08.698 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 23, 2024-09-21T09:31:08.699 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 110508 2024-09-21T09:31:08.699 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.699 INFO:tasks.workunit.client.0.smithi022.stdout: "txc_commit_lat": { 2024-09-21T09:31:08.699 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 273, 2024-09-21T09:31:08.699 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 293007041 2024-09-21T09:31:08.699 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.699 INFO:tasks.workunit.client.0.smithi022.stdout: "txc_submit_lat": { 2024-09-21T09:31:08.699 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 273, 2024-09-21T09:31:08.699 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 15401487 2024-09-21T09:31:08.699 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.699 INFO:tasks.workunit.client.0.smithi022.stdout: "txc_throttle_lat": { 2024-09-21T09:31:08.699 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 273, 2024-09-21T09:31:08.699 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2362761 2024-09-21T09:31:08.699 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.700 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.700 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore-pricache": { 2024-09-21T09:31:08.700 INFO:tasks.workunit.client.0.smithi022.stdout: "cache_bytes": { 2024-09-21T09:31:08.700 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 1861463884 2024-09-21T09:31:08.700 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.700 INFO:tasks.workunit.client.0.smithi022.stdout: "heap_bytes": { 2024-09-21T09:31:08.700 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 270974976 2024-09-21T09:31:08.700 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.700 INFO:tasks.workunit.client.0.smithi022.stdout: "mapped_bytes": { 2024-09-21T09:31:08.700 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 270049280 2024-09-21T09:31:08.700 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.700 INFO:tasks.workunit.client.0.smithi022.stdout: "target_bytes": { 2024-09-21T09:31:08.700 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 3137376768 2024-09-21T09:31:08.701 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.701 INFO:tasks.workunit.client.0.smithi022.stdout: "unmapped_bytes": { 2024-09-21T09:31:08.701 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 925696 2024-09-21T09:31:08.701 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.701 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.701 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore-pricache:data": { 2024-09-21T09:31:08.701 INFO:tasks.workunit.client.0.smithi022.stdout: "committed_bytes": { 2024-09-21T09:31:08.701 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 159383552 2024-09-21T09:31:08.701 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.701 INFO:tasks.workunit.client.0.smithi022.stdout: "pri0_bytes": { 2024-09-21T09:31:08.701 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.701 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.701 INFO:tasks.workunit.client.0.smithi022.stdout: "pri10_bytes": { 2024-09-21T09:31:08.701 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.702 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.702 INFO:tasks.workunit.client.0.smithi022.stdout: "pri11_bytes": { 2024-09-21T09:31:08.702 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 85490200 2024-09-21T09:31:08.702 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.702 INFO:tasks.workunit.client.0.smithi022.stdout: "pri1_bytes": { 2024-09-21T09:31:08.702 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.702 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.702 INFO:tasks.workunit.client.0.smithi022.stdout: "pri2_bytes": { 2024-09-21T09:31:08.702 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.702 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.702 INFO:tasks.workunit.client.0.smithi022.stdout: "pri3_bytes": { 2024-09-21T09:31:08.702 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.702 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.703 INFO:tasks.workunit.client.0.smithi022.stdout: "pri4_bytes": { 2024-09-21T09:31:08.703 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.703 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.703 INFO:tasks.workunit.client.0.smithi022.stdout: "pri5_bytes": { 2024-09-21T09:31:08.703 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.703 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.703 INFO:tasks.workunit.client.0.smithi022.stdout: "pri6_bytes": { 2024-09-21T09:31:08.703 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 139264 2024-09-21T09:31:08.703 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.703 INFO:tasks.workunit.client.0.smithi022.stdout: "pri7_bytes": { 2024-09-21T09:31:08.703 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.703 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.703 INFO:tasks.workunit.client.0.smithi022.stdout: "pri8_bytes": { 2024-09-21T09:31:08.703 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.704 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.704 INFO:tasks.workunit.client.0.smithi022.stdout: "pri9_bytes": { 2024-09-21T09:31:08.704 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.704 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.704 INFO:tasks.workunit.client.0.smithi022.stdout: "reserved_bytes": { 2024-09-21T09:31:08.704 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 73754088 2024-09-21T09:31:08.704 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.704 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.704 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore-pricache:kv": { 2024-09-21T09:31:08.704 INFO:tasks.workunit.client.0.smithi022.stdout: "committed_bytes": { 2024-09-21T09:31:08.704 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 780140544 2024-09-21T09:31:08.704 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.705 INFO:tasks.workunit.client.0.smithi022.stdout: "pri0_bytes": { 2024-09-21T09:31:08.705 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 67108864 2024-09-21T09:31:08.705 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.705 INFO:tasks.workunit.client.0.smithi022.stdout: "pri10_bytes": { 2024-09-21T09:31:08.705 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.705 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.705 INFO:tasks.workunit.client.0.smithi022.stdout: "pri11_bytes": { 2024-09-21T09:31:08.705 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 641176501 2024-09-21T09:31:08.705 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.705 INFO:tasks.workunit.client.0.smithi022.stdout: "pri1_bytes": { 2024-09-21T09:31:08.705 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.705 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.705 INFO:tasks.workunit.client.0.smithi022.stdout: "pri2_bytes": { 2024-09-21T09:31:08.705 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.706 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.706 INFO:tasks.workunit.client.0.smithi022.stdout: "pri3_bytes": { 2024-09-21T09:31:08.706 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.706 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.706 INFO:tasks.workunit.client.0.smithi022.stdout: "pri4_bytes": { 2024-09-21T09:31:08.706 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.706 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.706 INFO:tasks.workunit.client.0.smithi022.stdout: "pri5_bytes": { 2024-09-21T09:31:08.706 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.706 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.706 INFO:tasks.workunit.client.0.smithi022.stdout: "pri6_bytes": { 2024-09-21T09:31:08.706 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2128 2024-09-21T09:31:08.706 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.707 INFO:tasks.workunit.client.0.smithi022.stdout: "pri7_bytes": { 2024-09-21T09:31:08.707 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.707 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.707 INFO:tasks.workunit.client.0.smithi022.stdout: "pri8_bytes": { 2024-09-21T09:31:08.707 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.707 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.707 INFO:tasks.workunit.client.0.smithi022.stdout: "pri9_bytes": { 2024-09-21T09:31:08.707 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.707 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.707 INFO:tasks.workunit.client.0.smithi022.stdout: "reserved_bytes": { 2024-09-21T09:31:08.707 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 71853051 2024-09-21T09:31:08.707 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.707 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.708 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore-pricache:kv_onode": { 2024-09-21T09:31:08.708 INFO:tasks.workunit.client.0.smithi022.stdout: "committed_bytes": { 2024-09-21T09:31:08.708 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 192937984 2024-09-21T09:31:08.708 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.708 INFO:tasks.workunit.client.0.smithi022.stdout: "pri0_bytes": { 2024-09-21T09:31:08.708 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 67108864 2024-09-21T09:31:08.708 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.708 INFO:tasks.workunit.client.0.smithi022.stdout: "pri10_bytes": { 2024-09-21T09:31:08.708 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 464 2024-09-21T09:31:08.708 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.708 INFO:tasks.workunit.client.0.smithi022.stdout: "pri11_bytes": { 2024-09-21T09:31:08.708 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 56993466 2024-09-21T09:31:08.709 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.709 INFO:tasks.workunit.client.0.smithi022.stdout: "pri1_bytes": { 2024-09-21T09:31:08.709 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.709 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.709 INFO:tasks.workunit.client.0.smithi022.stdout: "pri2_bytes": { 2024-09-21T09:31:08.709 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.709 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.709 INFO:tasks.workunit.client.0.smithi022.stdout: "pri3_bytes": { 2024-09-21T09:31:08.709 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.709 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.709 INFO:tasks.workunit.client.0.smithi022.stdout: "pri4_bytes": { 2024-09-21T09:31:08.709 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.709 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.709 INFO:tasks.workunit.client.0.smithi022.stdout: "pri5_bytes": { 2024-09-21T09:31:08.709 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.710 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.710 INFO:tasks.workunit.client.0.smithi022.stdout: "pri6_bytes": { 2024-09-21T09:31:08.710 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.710 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.710 INFO:tasks.workunit.client.0.smithi022.stdout: "pri7_bytes": { 2024-09-21T09:31:08.710 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.710 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.710 INFO:tasks.workunit.client.0.smithi022.stdout: "pri8_bytes": { 2024-09-21T09:31:08.710 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.710 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.710 INFO:tasks.workunit.client.0.smithi022.stdout: "pri9_bytes": { 2024-09-21T09:31:08.710 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.710 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.711 INFO:tasks.workunit.client.0.smithi022.stdout: "reserved_bytes": { 2024-09-21T09:31:08.711 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 68835190 2024-09-21T09:31:08.711 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.711 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.711 INFO:tasks.workunit.client.0.smithi022.stdout: "bluestore-pricache:meta": { 2024-09-21T09:31:08.711 INFO:tasks.workunit.client.0.smithi022.stdout: "committed_bytes": { 2024-09-21T09:31:08.711 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 713031680 2024-09-21T09:31:08.711 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.711 INFO:tasks.workunit.client.0.smithi022.stdout: "pri0_bytes": { 2024-09-21T09:31:08.711 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.711 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.711 INFO:tasks.workunit.client.0.smithi022.stdout: "pri10_bytes": { 2024-09-21T09:31:08.711 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.712 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.712 INFO:tasks.workunit.client.0.smithi022.stdout: "pri11_bytes": { 2024-09-21T09:31:08.712 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 641176501 2024-09-21T09:31:08.712 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.712 INFO:tasks.workunit.client.0.smithi022.stdout: "pri1_bytes": { 2024-09-21T09:31:08.712 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.712 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.712 INFO:tasks.workunit.client.0.smithi022.stdout: "pri2_bytes": { 2024-09-21T09:31:08.712 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.712 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.712 INFO:tasks.workunit.client.0.smithi022.stdout: "pri3_bytes": { 2024-09-21T09:31:08.712 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.712 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.712 INFO:tasks.workunit.client.0.smithi022.stdout: "pri4_bytes": { 2024-09-21T09:31:08.713 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.713 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.713 INFO:tasks.workunit.client.0.smithi022.stdout: "pri5_bytes": { 2024-09-21T09:31:08.713 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.713 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.713 INFO:tasks.workunit.client.0.smithi022.stdout: "pri6_bytes": { 2024-09-21T09:31:08.713 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 272044 2024-09-21T09:31:08.713 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.713 INFO:tasks.workunit.client.0.smithi022.stdout: "pri7_bytes": { 2024-09-21T09:31:08.713 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.713 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.713 INFO:tasks.workunit.client.0.smithi022.stdout: "pri8_bytes": { 2024-09-21T09:31:08.713 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.713 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.713 INFO:tasks.workunit.client.0.smithi022.stdout: "pri9_bytes": { 2024-09-21T09:31:08.714 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.714 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.714 INFO:tasks.workunit.client.0.smithi022.stdout: "reserved_bytes": { 2024-09-21T09:31:08.714 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 71583135 2024-09-21T09:31:08.714 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.714 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.714 INFO:tasks.workunit.client.0.smithi022.stdout: "objecter": { 2024-09-21T09:31:08.714 INFO:tasks.workunit.client.0.smithi022.stdout: "op_active": { 2024-09-21T09:31:08.714 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.714 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.714 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r": { 2024-09-21T09:31:08.714 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.714 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.715 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rmw": { 2024-09-21T09:31:08.715 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.715 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.715 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w": { 2024-09-21T09:31:08.715 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.715 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.715 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.715 INFO:tasks.workunit.client.0.smithi022.stdout: "osd": { 2024-09-21T09:31:08.715 INFO:tasks.workunit.client.0.smithi022.stdout: "numpg": { 2024-09-21T09:31:08.715 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 46 2024-09-21T09:31:08.715 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.715 INFO:tasks.workunit.client.0.smithi022.stdout: "numpg_removing": { 2024-09-21T09:31:08.715 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.715 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.716 INFO:tasks.workunit.client.0.smithi022.stdout: "op": { 2024-09-21T09:31:08.716 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 29 2024-09-21T09:31:08.716 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.716 INFO:tasks.workunit.client.0.smithi022.stdout: "op_delayed_degraded": { 2024-09-21T09:31:08.716 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.716 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.716 INFO:tasks.workunit.client.0.smithi022.stdout: "op_delayed_unreadable": { 2024-09-21T09:31:08.716 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.716 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.716 INFO:tasks.workunit.client.0.smithi022.stdout: "op_in_bytes": { 2024-09-21T09:31:08.716 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.716 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.716 INFO:tasks.workunit.client.0.smithi022.stdout: "op_latency": { 2024-09-21T09:31:08.716 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 29, 2024-09-21T09:31:08.717 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 47302505 2024-09-21T09:31:08.717 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.717 INFO:tasks.workunit.client.0.smithi022.stdout: "op_out_bytes": { 2024-09-21T09:31:08.717 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 131762 2024-09-21T09:31:08.717 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.717 INFO:tasks.workunit.client.0.smithi022.stdout: "op_prepare_latency": { 2024-09-21T09:31:08.717 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 31, 2024-09-21T09:31:08.717 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 8253325 2024-09-21T09:31:08.717 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.717 INFO:tasks.workunit.client.0.smithi022.stdout: "op_process_latency": { 2024-09-21T09:31:08.717 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 29, 2024-09-21T09:31:08.717 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 29473970 2024-09-21T09:31:08.717 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.717 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r": { 2024-09-21T09:31:08.718 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 18 2024-09-21T09:31:08.718 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.718 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r_latency": { 2024-09-21T09:31:08.718 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 18, 2024-09-21T09:31:08.718 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 3096894 2024-09-21T09:31:08.718 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.718 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-21T09:31:08.718 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.718 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.718 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r_out_bytes": { 2024-09-21T09:31:08.718 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 131762 2024-09-21T09:31:08.719 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.719 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r_prepare_latency": { 2024-09-21T09:31:08.719 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 18, 2024-09-21T09:31:08.719 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2895370 2024-09-21T09:31:08.719 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.719 INFO:tasks.workunit.client.0.smithi022.stdout: "op_r_process_latency": { 2024-09-21T09:31:08.719 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 18, 2024-09-21T09:31:08.719 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2472706 2024-09-21T09:31:08.719 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.720 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw": { 2024-09-21T09:31:08.720 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 6 2024-09-21T09:31:08.720 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.720 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_in_bytes": { 2024-09-21T09:31:08.720 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.720 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.720 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_latency": { 2024-09-21T09:31:08.720 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 6, 2024-09-21T09:31:08.720 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 5010170 2024-09-21T09:31:08.720 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.721 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-21T09:31:08.721 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.721 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.721 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-21T09:31:08.721 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.721 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.721 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_out_bytes": { 2024-09-21T09:31:08.721 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.721 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.721 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_prepare_latency": { 2024-09-21T09:31:08.721 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 6, 2024-09-21T09:31:08.721 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2293509 2024-09-21T09:31:08.721 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.721 INFO:tasks.workunit.client.0.smithi022.stdout: "op_rw_process_latency": { 2024-09-21T09:31:08.722 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 6, 2024-09-21T09:31:08.722 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 4778960 2024-09-21T09:31:08.722 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.722 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w": { 2024-09-21T09:31:08.722 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 5 2024-09-21T09:31:08.722 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.722 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w_in_bytes": { 2024-09-21T09:31:08.722 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.722 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.722 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w_latency": { 2024-09-21T09:31:08.722 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 5, 2024-09-21T09:31:08.722 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 39195441 2024-09-21T09:31:08.722 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.722 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-21T09:31:08.723 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.723 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.723 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w_prepare_latency": { 2024-09-21T09:31:08.723 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 7, 2024-09-21T09:31:08.723 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 3064446 2024-09-21T09:31:08.723 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.723 INFO:tasks.workunit.client.0.smithi022.stdout: "op_w_process_latency": { 2024-09-21T09:31:08.723 INFO:tasks.workunit.client.0.smithi022.stdout: "count": 5, 2024-09-21T09:31:08.723 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 22222304 2024-09-21T09:31:08.723 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.723 INFO:tasks.workunit.client.0.smithi022.stdout: "op_wip": { 2024-09-21T09:31:08.723 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 0 2024-09-21T09:31:08.723 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.723 INFO:tasks.workunit.client.0.smithi022.stdout: "recovery_bytes": { 2024-09-21T09:31:08.723 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 590368 2024-09-21T09:31:08.724 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.724 INFO:tasks.workunit.client.0.smithi022.stdout: "recovery_ops": { 2024-09-21T09:31:08.724 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 2 2024-09-21T09:31:08.724 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.724 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_bytes": { 2024-09-21T09:31:08.724 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 95995035648 2024-09-21T09:31:08.724 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.724 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_bytes_used": { 2024-09-21T09:31:08.724 INFO:tasks.workunit.client.0.smithi022.stdout: "value": 305782784 2024-09-21T09:31:08.724 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.724 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.724 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.724 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.724 INFO:tasks.workunit.client.0.smithi022.stdout: "pools": [ 2024-09-21T09:31:08.724 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:08.724 INFO:tasks.workunit.client.0.smithi022.stdout: "application": [ 2024-09-21T09:31:08.725 INFO:tasks.workunit.client.0.smithi022.stdout: "mgr" 2024-09-21T09:31:08.725 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:08.725 INFO:tasks.workunit.client.0.smithi022.stdout: "cache_mode": "none", 2024-09-21T09:31:08.725 INFO:tasks.workunit.client.0.smithi022.stdout: "erasure_code_profile": {}, 2024-09-21T09:31:08.725 INFO:tasks.workunit.client.0.smithi022.stdout: "min_size": 2, 2024-09-21T09:31:08.725 INFO:tasks.workunit.client.0.smithi022.stdout: "options": {}, 2024-09-21T09:31:08.725 INFO:tasks.workunit.client.0.smithi022.stdout: "pg_autoscale_mode": "off", 2024-09-21T09:31:08.725 INFO:tasks.workunit.client.0.smithi022.stdout: "pg_num": 1, 2024-09-21T09:31:08.725 INFO:tasks.workunit.client.0.smithi022.stdout: "pgp_num": 1, 2024-09-21T09:31:08.725 INFO:tasks.workunit.client.0.smithi022.stdout: "pool": 1, 2024-09-21T09:31:08.725 INFO:tasks.workunit.client.0.smithi022.stdout: "size": 3, 2024-09-21T09:31:08.725 INFO:tasks.workunit.client.0.smithi022.stdout: "stats": { 2024-09-21T09:31:08.725 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_raw": 727115966730, 2024-09-21T09:31:08.725 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes_used": 1781760, 2024-09-21T09:31:08.726 INFO:tasks.workunit.client.0.smithi022.stdout: "compress_bytes_used": 0, 2024-09-21T09:31:08.726 INFO:tasks.workunit.client.0.smithi022.stdout: "compress_under_bytes": 0, 2024-09-21T09:31:08.726 INFO:tasks.workunit.client.0.smithi022.stdout: "data_bytes_used": 1781760, 2024-09-21T09:31:08.726 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty": 0, 2024-09-21T09:31:08.726 INFO:tasks.workunit.client.0.smithi022.stdout: "max_avail": 242371985408, 2024-09-21T09:31:08.726 INFO:tasks.workunit.client.0.smithi022.stdout: "objects": 2, 2024-09-21T09:31:08.726 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_bytes_used": 0, 2024-09-21T09:31:08.726 INFO:tasks.workunit.client.0.smithi022.stdout: "percent_used": 2.4504422526661074e-06, 2024-09-21T09:31:08.726 INFO:tasks.workunit.client.0.smithi022.stdout: "quota_bytes": 0, 2024-09-21T09:31:08.726 INFO:tasks.workunit.client.0.smithi022.stdout: "quota_objects": 0, 2024-09-21T09:31:08.726 INFO:tasks.workunit.client.0.smithi022.stdout: "rd": 126, 2024-09-21T09:31:08.726 INFO:tasks.workunit.client.0.smithi022.stdout: "rd_bytes": 236544, 2024-09-21T09:31:08.726 INFO:tasks.workunit.client.0.smithi022.stdout: "stored": 590368, 2024-09-21T09:31:08.726 INFO:tasks.workunit.client.0.smithi022.stdout: "stored_data": 590368, 2024-09-21T09:31:08.726 INFO:tasks.workunit.client.0.smithi022.stdout: "stored_omap": 0, 2024-09-21T09:31:08.727 INFO:tasks.workunit.client.0.smithi022.stdout: "stored_raw": 1771104, 2024-09-21T09:31:08.727 INFO:tasks.workunit.client.0.smithi022.stdout: "wr": 113, 2024-09-21T09:31:08.727 INFO:tasks.workunit.client.0.smithi022.stdout: "wr_bytes": 1667072 2024-09-21T09:31:08.727 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.727 INFO:tasks.workunit.client.0.smithi022.stdout: "target_max_bytes": 0, 2024-09-21T09:31:08.727 INFO:tasks.workunit.client.0.smithi022.stdout: "target_max_objects": 0, 2024-09-21T09:31:08.727 INFO:tasks.workunit.client.0.smithi022.stdout: "type": "replicated" 2024-09-21T09:31:08.727 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.727 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:08.727 INFO:tasks.workunit.client.0.smithi022.stdout: "application": [ 2024-09-21T09:31:08.727 INFO:tasks.workunit.client.0.smithi022.stdout: "cephfs" 2024-09-21T09:31:08.727 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:08.727 INFO:tasks.workunit.client.0.smithi022.stdout: "cache_mode": "none", 2024-09-21T09:31:08.727 INFO:tasks.workunit.client.0.smithi022.stdout: "erasure_code_profile": {}, 2024-09-21T09:31:08.727 INFO:tasks.workunit.client.0.smithi022.stdout: "min_size": 2, 2024-09-21T09:31:08.728 INFO:tasks.workunit.client.0.smithi022.stdout: "options": {}, 2024-09-21T09:31:08.728 INFO:tasks.workunit.client.0.smithi022.stdout: "pg_autoscale_mode": "off", 2024-09-21T09:31:08.728 INFO:tasks.workunit.client.0.smithi022.stdout: "pg_num": 32, 2024-09-21T09:31:08.728 INFO:tasks.workunit.client.0.smithi022.stdout: "pgp_num": 32, 2024-09-21T09:31:08.728 INFO:tasks.workunit.client.0.smithi022.stdout: "pool": 2, 2024-09-21T09:31:08.728 INFO:tasks.workunit.client.0.smithi022.stdout: "size": 3, 2024-09-21T09:31:08.728 INFO:tasks.workunit.client.0.smithi022.stdout: "stats": { 2024-09-21T09:31:08.728 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_raw": 727115966730, 2024-09-21T09:31:08.728 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes_used": 98304, 2024-09-21T09:31:08.728 INFO:tasks.workunit.client.0.smithi022.stdout: "compress_bytes_used": 0, 2024-09-21T09:31:08.728 INFO:tasks.workunit.client.0.smithi022.stdout: "compress_under_bytes": 0, 2024-09-21T09:31:08.728 INFO:tasks.workunit.client.0.smithi022.stdout: "data_bytes_used": 98304, 2024-09-21T09:31:08.728 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty": 0, 2024-09-21T09:31:08.728 INFO:tasks.workunit.client.0.smithi022.stdout: "max_avail": 242371985408, 2024-09-21T09:31:08.729 INFO:tasks.workunit.client.0.smithi022.stdout: "objects": 22, 2024-09-21T09:31:08.729 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_bytes_used": 0, 2024-09-21T09:31:08.729 INFO:tasks.workunit.client.0.smithi022.stdout: "percent_used": 1.3519712638299097e-07, 2024-09-21T09:31:08.729 INFO:tasks.workunit.client.0.smithi022.stdout: "quota_bytes": 0, 2024-09-21T09:31:08.729 INFO:tasks.workunit.client.0.smithi022.stdout: "quota_objects": 0, 2024-09-21T09:31:08.729 INFO:tasks.workunit.client.0.smithi022.stdout: "rd": 0, 2024-09-21T09:31:08.729 INFO:tasks.workunit.client.0.smithi022.stdout: "rd_bytes": 0, 2024-09-21T09:31:08.729 INFO:tasks.workunit.client.0.smithi022.stdout: "stored": 3410, 2024-09-21T09:31:08.729 INFO:tasks.workunit.client.0.smithi022.stdout: "stored_data": 3410, 2024-09-21T09:31:08.729 INFO:tasks.workunit.client.0.smithi022.stdout: "stored_omap": 0, 2024-09-21T09:31:08.729 INFO:tasks.workunit.client.0.smithi022.stdout: "stored_raw": 10230, 2024-09-21T09:31:08.729 INFO:tasks.workunit.client.0.smithi022.stdout: "wr": 51, 2024-09-21T09:31:08.729 INFO:tasks.workunit.client.0.smithi022.stdout: "wr_bytes": 19456 2024-09-21T09:31:08.729 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.730 INFO:tasks.workunit.client.0.smithi022.stdout: "target_max_bytes": 0, 2024-09-21T09:31:08.730 INFO:tasks.workunit.client.0.smithi022.stdout: "target_max_objects": 0, 2024-09-21T09:31:08.730 INFO:tasks.workunit.client.0.smithi022.stdout: "type": "replicated" 2024-09-21T09:31:08.730 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.730 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:08.730 INFO:tasks.workunit.client.0.smithi022.stdout: "application": [ 2024-09-21T09:31:08.730 INFO:tasks.workunit.client.0.smithi022.stdout: "cephfs" 2024-09-21T09:31:08.730 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:08.730 INFO:tasks.workunit.client.0.smithi022.stdout: "cache_mode": "none", 2024-09-21T09:31:08.730 INFO:tasks.workunit.client.0.smithi022.stdout: "erasure_code_profile": {}, 2024-09-21T09:31:08.730 INFO:tasks.workunit.client.0.smithi022.stdout: "min_size": 2, 2024-09-21T09:31:08.730 INFO:tasks.workunit.client.0.smithi022.stdout: "options": {}, 2024-09-21T09:31:08.730 INFO:tasks.workunit.client.0.smithi022.stdout: "pg_autoscale_mode": "off", 2024-09-21T09:31:08.731 INFO:tasks.workunit.client.0.smithi022.stdout: "pg_num": 64, 2024-09-21T09:31:08.731 INFO:tasks.workunit.client.0.smithi022.stdout: "pgp_num": 64, 2024-09-21T09:31:08.731 INFO:tasks.workunit.client.0.smithi022.stdout: "pool": 3, 2024-09-21T09:31:08.731 INFO:tasks.workunit.client.0.smithi022.stdout: "size": 3, 2024-09-21T09:31:08.731 INFO:tasks.workunit.client.0.smithi022.stdout: "stats": { 2024-09-21T09:31:08.731 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_raw": 727115966730, 2024-09-21T09:31:08.731 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes_used": 0, 2024-09-21T09:31:08.731 INFO:tasks.workunit.client.0.smithi022.stdout: "compress_bytes_used": 0, 2024-09-21T09:31:08.731 INFO:tasks.workunit.client.0.smithi022.stdout: "compress_under_bytes": 0, 2024-09-21T09:31:08.731 INFO:tasks.workunit.client.0.smithi022.stdout: "data_bytes_used": 0, 2024-09-21T09:31:08.731 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty": 0, 2024-09-21T09:31:08.731 INFO:tasks.workunit.client.0.smithi022.stdout: "max_avail": 242371985408, 2024-09-21T09:31:08.731 INFO:tasks.workunit.client.0.smithi022.stdout: "objects": 0, 2024-09-21T09:31:08.731 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_bytes_used": 0, 2024-09-21T09:31:08.731 INFO:tasks.workunit.client.0.smithi022.stdout: "percent_used": 0.0, 2024-09-21T09:31:08.732 INFO:tasks.workunit.client.0.smithi022.stdout: "quota_bytes": 0, 2024-09-21T09:31:08.732 INFO:tasks.workunit.client.0.smithi022.stdout: "quota_objects": 0, 2024-09-21T09:31:08.732 INFO:tasks.workunit.client.0.smithi022.stdout: "rd": 0, 2024-09-21T09:31:08.732 INFO:tasks.workunit.client.0.smithi022.stdout: "rd_bytes": 0, 2024-09-21T09:31:08.732 INFO:tasks.workunit.client.0.smithi022.stdout: "stored": 0, 2024-09-21T09:31:08.732 INFO:tasks.workunit.client.0.smithi022.stdout: "stored_data": 0, 2024-09-21T09:31:08.732 INFO:tasks.workunit.client.0.smithi022.stdout: "stored_omap": 0, 2024-09-21T09:31:08.732 INFO:tasks.workunit.client.0.smithi022.stdout: "stored_raw": 0, 2024-09-21T09:31:08.732 INFO:tasks.workunit.client.0.smithi022.stdout: "wr": 0, 2024-09-21T09:31:08.732 INFO:tasks.workunit.client.0.smithi022.stdout: "wr_bytes": 0 2024-09-21T09:31:08.732 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.732 INFO:tasks.workunit.client.0.smithi022.stdout: "target_max_bytes": 0, 2024-09-21T09:31:08.732 INFO:tasks.workunit.client.0.smithi022.stdout: "target_max_objects": 0, 2024-09-21T09:31:08.732 INFO:tasks.workunit.client.0.smithi022.stdout: "type": "replicated" 2024-09-21T09:31:08.732 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.732 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:08.733 INFO:tasks.workunit.client.0.smithi022.stdout: "application": [ 2024-09-21T09:31:08.733 INFO:tasks.workunit.client.0.smithi022.stdout: "rbd" 2024-09-21T09:31:08.733 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:08.733 INFO:tasks.workunit.client.0.smithi022.stdout: "cache_mode": "none", 2024-09-21T09:31:08.733 INFO:tasks.workunit.client.0.smithi022.stdout: "erasure_code_profile": {}, 2024-09-21T09:31:08.733 INFO:tasks.workunit.client.0.smithi022.stdout: "min_size": 2, 2024-09-21T09:31:08.733 INFO:tasks.workunit.client.0.smithi022.stdout: "options": {}, 2024-09-21T09:31:08.733 INFO:tasks.workunit.client.0.smithi022.stdout: "pg_autoscale_mode": "off", 2024-09-21T09:31:08.733 INFO:tasks.workunit.client.0.smithi022.stdout: "pg_num": 8, 2024-09-21T09:31:08.733 INFO:tasks.workunit.client.0.smithi022.stdout: "pgp_num": 8, 2024-09-21T09:31:08.733 INFO:tasks.workunit.client.0.smithi022.stdout: "pool": 4, 2024-09-21T09:31:08.733 INFO:tasks.workunit.client.0.smithi022.stdout: "size": 3, 2024-09-21T09:31:08.733 INFO:tasks.workunit.client.0.smithi022.stdout: "stats": { 2024-09-21T09:31:08.733 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_raw": 727115966730, 2024-09-21T09:31:08.733 INFO:tasks.workunit.client.0.smithi022.stdout: "bytes_used": 12288, 2024-09-21T09:31:08.734 INFO:tasks.workunit.client.0.smithi022.stdout: "compress_bytes_used": 0, 2024-09-21T09:31:08.734 INFO:tasks.workunit.client.0.smithi022.stdout: "compress_under_bytes": 0, 2024-09-21T09:31:08.734 INFO:tasks.workunit.client.0.smithi022.stdout: "data_bytes_used": 12288, 2024-09-21T09:31:08.734 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty": 0, 2024-09-21T09:31:08.734 INFO:tasks.workunit.client.0.smithi022.stdout: "max_avail": 242371985408, 2024-09-21T09:31:08.734 INFO:tasks.workunit.client.0.smithi022.stdout: "objects": 2, 2024-09-21T09:31:08.734 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_bytes_used": 0, 2024-09-21T09:31:08.734 INFO:tasks.workunit.client.0.smithi022.stdout: "percent_used": 1.689964257423071e-08, 2024-09-21T09:31:08.734 INFO:tasks.workunit.client.0.smithi022.stdout: "quota_bytes": 0, 2024-09-21T09:31:08.734 INFO:tasks.workunit.client.0.smithi022.stdout: "quota_objects": 0, 2024-09-21T09:31:08.734 INFO:tasks.workunit.client.0.smithi022.stdout: "rd": 0, 2024-09-21T09:31:08.734 INFO:tasks.workunit.client.0.smithi022.stdout: "rd_bytes": 0, 2024-09-21T09:31:08.734 INFO:tasks.workunit.client.0.smithi022.stdout: "stored": 19, 2024-09-21T09:31:08.735 INFO:tasks.workunit.client.0.smithi022.stdout: "stored_data": 19, 2024-09-21T09:31:08.735 INFO:tasks.workunit.client.0.smithi022.stdout: "stored_omap": 0, 2024-09-21T09:31:08.735 INFO:tasks.workunit.client.0.smithi022.stdout: "stored_raw": 57, 2024-09-21T09:31:08.735 INFO:tasks.workunit.client.0.smithi022.stdout: "wr": 2, 2024-09-21T09:31:08.735 INFO:tasks.workunit.client.0.smithi022.stdout: "wr_bytes": 2048 2024-09-21T09:31:08.735 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.735 INFO:tasks.workunit.client.0.smithi022.stdout: "target_max_bytes": 0, 2024-09-21T09:31:08.735 INFO:tasks.workunit.client.0.smithi022.stdout: "target_max_objects": 0, 2024-09-21T09:31:08.735 INFO:tasks.workunit.client.0.smithi022.stdout: "type": "replicated" 2024-09-21T09:31:08.735 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.735 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:08.735 INFO:tasks.workunit.client.0.smithi022.stdout: "rbd": { 2024-09-21T09:31:08.736 INFO:tasks.workunit.client.0.smithi022.stdout: "mirroring_by_pool": [ 2024-09-21T09:31:08.736 INFO:tasks.workunit.client.0.smithi022.stdout: false 2024-09-21T09:31:08.736 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:08.736 INFO:tasks.workunit.client.0.smithi022.stdout: "num_images_by_pool": [ 2024-09-21T09:31:08.736 INFO:tasks.workunit.client.0.smithi022.stdout: 0 2024-09-21T09:31:08.736 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:08.736 INFO:tasks.workunit.client.0.smithi022.stdout: "num_pools": 1 2024-09-21T09:31:08.736 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.736 INFO:tasks.workunit.client.0.smithi022.stdout: "report_id": "77addb0d-e829-4757-884e-46deb5965591", 2024-09-21T09:31:08.736 INFO:tasks.workunit.client.0.smithi022.stdout: "report_timestamp": "2024-09-21T09:31:07.352134", 2024-09-21T09:31:08.736 INFO:tasks.workunit.client.0.smithi022.stdout: "report_version": 1, 2024-09-21T09:31:08.736 INFO:tasks.workunit.client.0.smithi022.stdout: "rocksdb_stats": { 2024-09-21T09:31:08.736 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "6.15.5" 2024-09-21T09:31:08.736 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.737 INFO:tasks.workunit.client.0.smithi022.stdout: "rook": { 2024-09-21T09:31:08.737 INFO:tasks.workunit.client.0.smithi022.stdout: "cluster": { 2024-09-21T09:31:08.737 INFO:tasks.workunit.client.0.smithi022.stdout: "external-mode": null, 2024-09-21T09:31:08.737 INFO:tasks.workunit.client.0.smithi022.stdout: "mon": { 2024-09-21T09:31:08.737 INFO:tasks.workunit.client.0.smithi022.stdout: "allow-multiple-per-node": null, 2024-09-21T09:31:08.737 INFO:tasks.workunit.client.0.smithi022.stdout: "count": null, 2024-09-21T09:31:08.737 INFO:tasks.workunit.client.0.smithi022.stdout: "max-id": null, 2024-09-21T09:31:08.737 INFO:tasks.workunit.client.0.smithi022.stdout: "pvc": { 2024-09-21T09:31:08.737 INFO:tasks.workunit.client.0.smithi022.stdout: "enabled": null 2024-09-21T09:31:08.737 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.737 INFO:tasks.workunit.client.0.smithi022.stdout: "stretch": { 2024-09-21T09:31:08.737 INFO:tasks.workunit.client.0.smithi022.stdout: "enabled": null 2024-09-21T09:31:08.737 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.737 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.737 INFO:tasks.workunit.client.0.smithi022.stdout: "network": { 2024-09-21T09:31:08.738 INFO:tasks.workunit.client.0.smithi022.stdout: "provider": null 2024-09-21T09:31:08.738 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.738 INFO:tasks.workunit.client.0.smithi022.stdout: "storage": { 2024-09-21T09:31:08.738 INFO:tasks.workunit.client.0.smithi022.stdout: "device-set": { 2024-09-21T09:31:08.738 INFO:tasks.workunit.client.0.smithi022.stdout: "count": { 2024-09-21T09:31:08.738 INFO:tasks.workunit.client.0.smithi022.stdout: "non-portable": null, 2024-09-21T09:31:08.738 INFO:tasks.workunit.client.0.smithi022.stdout: "portable": null, 2024-09-21T09:31:08.738 INFO:tasks.workunit.client.0.smithi022.stdout: "total": null 2024-09-21T09:31:08.738 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.738 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.738 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.738 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.738 INFO:tasks.workunit.client.0.smithi022.stdout: "csi": { 2024-09-21T09:31:08.739 INFO:tasks.workunit.client.0.smithi022.stdout: "version": null 2024-09-21T09:31:08.739 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.739 INFO:tasks.workunit.client.0.smithi022.stdout: "kubernetes": { 2024-09-21T09:31:08.739 INFO:tasks.workunit.client.0.smithi022.stdout: "version": null 2024-09-21T09:31:08.739 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.739 INFO:tasks.workunit.client.0.smithi022.stdout: "node": { 2024-09-21T09:31:08.739 INFO:tasks.workunit.client.0.smithi022.stdout: "count": { 2024-09-21T09:31:08.739 INFO:tasks.workunit.client.0.smithi022.stdout: "kubernetes-total": null, 2024-09-21T09:31:08.739 INFO:tasks.workunit.client.0.smithi022.stdout: "with-ceph-daemons": null, 2024-09-21T09:31:08.739 INFO:tasks.workunit.client.0.smithi022.stdout: "with-csi-cephfs-plugin": null, 2024-09-21T09:31:08.739 INFO:tasks.workunit.client.0.smithi022.stdout: "with-csi-nfs-plugin": null, 2024-09-21T09:31:08.739 INFO:tasks.workunit.client.0.smithi022.stdout: "with-csi-rbd-plugin": null 2024-09-21T09:31:08.739 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.739 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.740 INFO:tasks.workunit.client.0.smithi022.stdout: "usage": { 2024-09-21T09:31:08.740 INFO:tasks.workunit.client.0.smithi022.stdout: "storage-class": { 2024-09-21T09:31:08.740 INFO:tasks.workunit.client.0.smithi022.stdout: "count": { 2024-09-21T09:31:08.740 INFO:tasks.workunit.client.0.smithi022.stdout: "bucket": null, 2024-09-21T09:31:08.740 INFO:tasks.workunit.client.0.smithi022.stdout: "cephfs": null, 2024-09-21T09:31:08.740 INFO:tasks.workunit.client.0.smithi022.stdout: "nfs": null, 2024-09-21T09:31:08.740 INFO:tasks.workunit.client.0.smithi022.stdout: "rbd": null, 2024-09-21T09:31:08.740 INFO:tasks.workunit.client.0.smithi022.stdout: "total": null 2024-09-21T09:31:08.740 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.740 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:08.740 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.740 INFO:tasks.workunit.client.0.smithi022.stdout: "version": null 2024-09-21T09:31:08.740 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.740 INFO:tasks.workunit.client.0.smithi022.stdout: "services": {}, 2024-09-21T09:31:08.741 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_per_pg": [ 2024-09-21T09:31:08.741 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:08.741 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:08.741 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:08.741 INFO:tasks.workunit.client.0.smithi022.stdout: 6, 2024-09-21T09:31:08.741 INFO:tasks.workunit.client.0.smithi022.stdout: 0 2024-09-21T09:31:08.741 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:08.741 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 7, 2024-09-21T09:31:08.741 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:08.741 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:08.741 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:08.741 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:08.742 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:08.742 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:08.742 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.359261+0000", 2024-09-21T09:31:08.742 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.635255+0000", 2024-09-21T09:31:08.742 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.635255+0000", 2024-09-21T09:31:08.742 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.635695+0000", 2024-09-21T09:31:08.742 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.359261+0000", 2024-09-21T09:31:08.742 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:08.742 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:08.742 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:08.742 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:08.742 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.359261+0000", 2024-09-21T09:31:08.742 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.359261+0000", 2024-09-21T09:31:08.742 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.359261+0000", 2024-09-21T09:31:08.743 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:08.743 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:08.743 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:08.743 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.359261+0000", 2024-09-21T09:31:08.743 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.359261+0000", 2024-09-21T09:31:08.743 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:08.743 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:08.743 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:08.743 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:08.743 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:08.743 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:08.743 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:08.744 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:08.744 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:08.744 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:08.744 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:08.744 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:08.744 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:08.744 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.3f", 2024-09-21T09:31:08.744 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:08.744 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:08.744 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:08.744 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:08.744 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:08.744 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T16:15:50.156743+0000", 2024-09-21T09:31:08.744 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:08.745 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:08.745 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:08.745 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:08.745 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:08.745 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:08.745 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:08.745 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:08.745 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:08.745 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:08.745 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:08.745 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:08.745 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:08.745 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:08.745 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:08.746 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:08.746 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:08.746 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:08.746 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:08.746 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:08.746 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:08.746 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:08.746 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:08.746 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:08.746 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:08.746 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:08.746 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:08.746 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:08.746 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:08.746 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:08.747 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:08.747 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:08.747 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:08.747 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:08.747 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:08.747 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:08.747 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:08.747 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:08.747 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:08.747 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:08.747 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:08.747 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:08.747 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:08.748 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.748 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:08.748 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:08.748 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:08.748 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:08.748 INFO:tasks.workunit.client.0.smithi022.stdout: 6, 2024-09-21T09:31:08.748 INFO:tasks.workunit.client.0.smithi022.stdout: 0 2024-09-21T09:31:08.748 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:08.748 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 7, 2024-09-21T09:31:08.748 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:08.748 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.748 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:08.748 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:08.748 INFO:tasks.workunit.client.0.smithi022.stdout: 1, 2024-09-21T09:31:08.749 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:08.749 INFO:tasks.workunit.client.0.smithi022.stdout: 6 2024-09-21T09:31:08.749 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:08.749 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 1, 2024-09-21T09:31:08.749 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:08.749 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:08.749 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:08.749 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:08.749 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:08.749 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:08.749 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.062366+0000", 2024-09-21T09:31:08.749 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.635778+0000", 2024-09-21T09:31:08.749 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.635778+0000", 2024-09-21T09:31:08.749 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.636115+0000", 2024-09-21T09:31:08.750 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.062366+0000", 2024-09-21T09:31:08.750 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:08.750 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:08.750 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:08.750 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:08.750 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.062366+0000", 2024-09-21T09:31:08.750 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.062366+0000", 2024-09-21T09:31:08.750 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.062366+0000", 2024-09-21T09:31:08.750 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:08.750 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:08.750 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:08.750 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.062366+0000", 2024-09-21T09:31:08.750 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.062366+0000", 2024-09-21T09:31:08.750 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:08.751 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:08.751 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:08.751 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:08.751 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:08.751 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:08.751 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:08.751 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:08.751 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:08.751 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:08.751 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:08.751 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:08.751 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:08.751 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.3e", 2024-09-21T09:31:08.751 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:08.752 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:08.752 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:08.752 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:08.752 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:08.752 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T11:36:35.607057+0000", 2024-09-21T09:31:08.752 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:08.752 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:08.752 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:08.752 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:08.752 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:08.752 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:08.752 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:08.752 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:08.752 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:08.752 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:08.752 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:08.753 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:08.753 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:08.753 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:08.753 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:08.753 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:08.753 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:08.753 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:08.753 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:08.753 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:08.753 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:08.753 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:08.753 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:08.753 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:08.753 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:08.754 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:08.754 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:08.754 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:08.754 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:08.754 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:08.754 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:08.754 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:08.754 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:08.754 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:08.754 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:08.754 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:08.754 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:08.754 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:08.754 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:08.754 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:08.755 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:08.755 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:08.755 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:08.755 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.755 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:08.755 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:08.755 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:08.755 INFO:tasks.workunit.client.0.smithi022.stdout: 1, 2024-09-21T09:31:08.755 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:08.755 INFO:tasks.workunit.client.0.smithi022.stdout: 6 2024-09-21T09:31:08.755 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:08.755 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 1, 2024-09-21T09:31:08.755 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:08.755 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.755 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:08.756 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:08.756 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:08.756 INFO:tasks.workunit.client.0.smithi022.stdout: 2, 2024-09-21T09:31:08.756 INFO:tasks.workunit.client.0.smithi022.stdout: 1 2024-09-21T09:31:08.756 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:08.756 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 0, 2024-09-21T09:31:08.756 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:08.756 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:08.756 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:08.756 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:08.756 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:08.756 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:08.756 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.141182+0000", 2024-09-21T09:31:08.756 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.640415+0000", 2024-09-21T09:31:08.756 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.640415+0000", 2024-09-21T09:31:08.757 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.640689+0000", 2024-09-21T09:31:08.757 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.141182+0000", 2024-09-21T09:31:08.757 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:08.757 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:08.757 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:08.757 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:08.757 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.141182+0000", 2024-09-21T09:31:08.757 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.141182+0000", 2024-09-21T09:31:08.757 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.141182+0000", 2024-09-21T09:31:08.757 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:08.757 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:08.757 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:08.757 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.141182+0000", 2024-09-21T09:31:08.758 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.141182+0000", 2024-09-21T09:31:08.758 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:08.758 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:08.758 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:08.758 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:08.758 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:08.758 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:08.758 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:08.758 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:08.758 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:08.758 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:08.758 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:08.758 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:08.758 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:08.759 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.3d", 2024-09-21T09:31:08.759 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:08.759 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:08.759 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:08.759 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:08.759 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:08.759 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T21:12:22.314199+0000", 2024-09-21T09:31:08.759 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:08.759 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:08.759 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:08.759 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:08.759 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:08.759 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:08.760 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:08.760 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:08.760 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:08.760 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:08.760 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:08.760 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:08.760 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:08.760 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:08.760 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:08.760 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:08.760 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:08.760 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:08.760 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:08.760 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:08.761 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:08.761 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:08.761 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:08.761 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:08.761 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:08.761 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:08.761 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:08.761 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:08.761 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:08.761 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:08.761 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:08.761 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:08.761 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:08.762 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:08.762 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:08.762 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:08.762 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:08.762 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:08.762 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:08.762 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:08.762 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:08.762 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:08.762 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:08.762 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.762 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:08.762 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:08.763 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:08.763 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:08.763 INFO:tasks.workunit.client.0.smithi022.stdout: 2, 2024-09-21T09:31:08.763 INFO:tasks.workunit.client.0.smithi022.stdout: 1 2024-09-21T09:31:08.763 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:08.763 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 0, 2024-09-21T09:31:08.763 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:08.763 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.763 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:08.763 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:08.763 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:08.763 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:08.763 INFO:tasks.workunit.client.0.smithi022.stdout: 7 2024-09-21T09:31:08.763 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:08.764 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 5, 2024-09-21T09:31:08.764 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:08.764 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:08.764 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:08.764 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:08.764 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:08.764 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:08.764 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.066315+0000", 2024-09-21T09:31:08.764 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.644246+0000", 2024-09-21T09:31:08.764 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.644246+0000", 2024-09-21T09:31:08.764 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.644536+0000", 2024-09-21T09:31:08.764 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.066315+0000", 2024-09-21T09:31:08.764 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:08.764 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:08.764 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:08.765 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:08.765 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.066315+0000", 2024-09-21T09:31:08.765 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.066315+0000", 2024-09-21T09:31:08.765 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.066315+0000", 2024-09-21T09:31:08.765 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:08.765 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:08.765 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:08.765 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.066315+0000", 2024-09-21T09:31:08.765 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.066315+0000", 2024-09-21T09:31:08.765 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:08.765 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:08.765 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:08.765 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:08.765 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:08.765 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:08.766 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:08.766 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:08.766 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:08.766 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:08.766 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:08.766 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:08.766 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:08.766 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.3c", 2024-09-21T09:31:08.766 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:08.766 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:08.766 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:08.766 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:08.766 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:08.766 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T17:31:37.886172+0000", 2024-09-21T09:31:08.767 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:08.767 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:08.767 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:08.767 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:08.767 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:08.767 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:08.767 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:08.767 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:08.767 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:08.767 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:08.767 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:08.767 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:08.767 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:08.767 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:08.767 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:08.768 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:08.768 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:08.768 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:08.768 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:08.768 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:08.768 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:08.768 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:08.768 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:08.768 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:08.768 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:08.768 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:08.768 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:08.768 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:08.768 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:08.768 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:08.769 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:08.769 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:08.769 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:08.769 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:08.769 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:08.769 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:08.769 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:08.770 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:08.770 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:08.770 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:08.770 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:08.770 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:08.770 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:08.770 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.770 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:08.770 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:08.770 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:08.770 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:08.770 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:08.770 INFO:tasks.workunit.client.0.smithi022.stdout: 7 2024-09-21T09:31:08.770 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:08.770 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 5, 2024-09-21T09:31:08.771 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:08.771 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.771 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:08.771 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:08.771 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:08.771 INFO:tasks.workunit.client.0.smithi022.stdout: 2, 2024-09-21T09:31:08.771 INFO:tasks.workunit.client.0.smithi022.stdout: 4 2024-09-21T09:31:08.771 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:08.771 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 7, 2024-09-21T09:31:08.771 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:08.771 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:08.771 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:08.771 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:08.771 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:08.772 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:08.772 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.358990+0000", 2024-09-21T09:31:08.772 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.642487+0000", 2024-09-21T09:31:08.772 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.642487+0000", 2024-09-21T09:31:08.772 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.642779+0000", 2024-09-21T09:31:08.772 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.358990+0000", 2024-09-21T09:31:08.772 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:08.772 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:08.772 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:08.772 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:08.772 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.358990+0000", 2024-09-21T09:31:08.772 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.358990+0000", 2024-09-21T09:31:08.772 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.358990+0000", 2024-09-21T09:31:08.773 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:08.773 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:08.773 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:08.773 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.358990+0000", 2024-09-21T09:31:08.773 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.358990+0000", 2024-09-21T09:31:08.773 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:08.773 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:08.773 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:08.773 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:08.773 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:08.773 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:08.773 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:08.773 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:08.773 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:08.773 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:08.774 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:08.774 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:08.774 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:08.774 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.3b", 2024-09-21T09:31:08.774 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:08.774 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:08.774 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:08.774 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:08.774 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:08.774 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T11:17:29.631740+0000", 2024-09-21T09:31:08.774 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:08.774 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:08.774 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:08.774 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:08.775 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:08.775 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:08.775 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:08.775 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:08.775 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:08.775 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:08.775 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:08.775 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:08.775 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:08.775 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:08.775 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:08.775 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:08.775 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:08.775 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:08.776 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:08.776 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:08.776 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:08.776 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:08.776 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:08.776 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:08.776 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:08.776 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:08.776 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:08.776 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:08.776 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:08.776 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:08.776 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:08.776 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:08.776 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:08.777 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:08.777 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:08.777 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:08.777 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:08.777 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:08.777 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:08.777 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:08.777 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:08.777 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:08.777 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:08.777 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.777 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:08.777 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:08.777 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:08.777 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:08.777 INFO:tasks.workunit.client.0.smithi022.stdout: 2, 2024-09-21T09:31:08.778 INFO:tasks.workunit.client.0.smithi022.stdout: 4 2024-09-21T09:31:08.778 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:08.778 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 7, 2024-09-21T09:31:08.778 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:08.778 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.778 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:08.778 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:08.778 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:08.778 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:08.778 INFO:tasks.workunit.client.0.smithi022.stdout: 1 2024-09-21T09:31:08.778 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:08.778 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 5, 2024-09-21T09:31:08.778 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:08.779 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:08.779 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:08.779 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:08.779 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:08.779 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:08.779 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.066399+0000", 2024-09-21T09:31:08.779 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.650742+0000", 2024-09-21T09:31:08.779 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.650742+0000", 2024-09-21T09:31:08.779 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.651016+0000", 2024-09-21T09:31:08.779 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.066399+0000", 2024-09-21T09:31:08.779 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:08.779 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:08.779 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:08.779 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:08.779 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.066399+0000", 2024-09-21T09:31:08.780 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.066399+0000", 2024-09-21T09:31:08.780 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.066399+0000", 2024-09-21T09:31:08.780 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:08.780 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:08.780 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:08.780 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.066399+0000", 2024-09-21T09:31:08.780 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.066399+0000", 2024-09-21T09:31:08.780 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:08.780 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:08.780 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:08.780 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:08.780 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:08.780 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:08.780 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:08.780 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:08.781 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:08.781 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:08.781 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:08.781 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:08.781 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:08.781 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.3a", 2024-09-21T09:31:08.781 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:08.781 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:08.781 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:08.781 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:08.781 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:08.781 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T17:11:11.493990+0000", 2024-09-21T09:31:08.781 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:08.781 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:08.781 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:08.781 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:08.781 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:08.782 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:08.782 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:08.782 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:08.782 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:08.782 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:08.782 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:08.782 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:08.782 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:08.782 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:08.782 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:08.782 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:08.782 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:08.782 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:08.783 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:08.783 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:08.783 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:08.783 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:08.783 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:08.783 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:08.783 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:08.783 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:08.783 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:08.783 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:08.783 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:08.783 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:08.783 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:08.783 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:08.784 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:08.784 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:08.784 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:08.784 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:08.784 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:08.784 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:08.784 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:08.784 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:08.784 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:08.784 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:08.784 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:08.784 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.784 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:08.784 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:08.785 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:08.785 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:08.785 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:08.785 INFO:tasks.workunit.client.0.smithi022.stdout: 1 2024-09-21T09:31:08.785 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:08.785 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 5, 2024-09-21T09:31:08.785 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:08.785 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.785 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:08.785 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:08.785 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:08.785 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:08.785 INFO:tasks.workunit.client.0.smithi022.stdout: 4 2024-09-21T09:31:08.785 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:08.786 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 7, 2024-09-21T09:31:08.786 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:08.786 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:08.786 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:08.786 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:08.786 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:08.786 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:08.786 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.358835+0000", 2024-09-21T09:31:08.786 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.646890+0000", 2024-09-21T09:31:08.786 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.646890+0000", 2024-09-21T09:31:08.786 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.647235+0000", 2024-09-21T09:31:08.786 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.358835+0000", 2024-09-21T09:31:08.786 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:08.787 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:08.787 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:08.787 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:08.787 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.358835+0000", 2024-09-21T09:31:08.787 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.358835+0000", 2024-09-21T09:31:08.787 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.358835+0000", 2024-09-21T09:31:08.787 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:08.787 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:08.787 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:08.787 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.358835+0000", 2024-09-21T09:31:08.787 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.358835+0000", 2024-09-21T09:31:08.787 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:08.787 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:08.787 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:08.787 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:08.788 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:08.788 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:08.788 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:08.788 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:08.788 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:08.788 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:08.788 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:08.788 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:08.788 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:08.788 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.39", 2024-09-21T09:31:08.788 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:08.788 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:08.788 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:08.788 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:08.789 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:08.789 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T11:21:45.995387+0000", 2024-09-21T09:31:08.789 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:08.789 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:08.789 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:08.789 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:08.789 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:08.789 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:08.789 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:08.789 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:08.789 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:08.789 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:08.789 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:08.789 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:08.790 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:08.790 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:08.790 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:08.790 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:08.790 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:08.790 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:08.790 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:08.790 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:08.790 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:08.790 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:08.790 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:08.790 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:08.790 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:08.791 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:08.791 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:08.791 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:08.791 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:08.791 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:08.791 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:08.791 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:08.791 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:08.791 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:08.791 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:08.791 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:08.791 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:08.791 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:08.791 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:08.792 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:08.792 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:08.792 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:08.792 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:08.792 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.792 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:08.792 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:08.792 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:08.792 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:08.792 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:08.792 INFO:tasks.workunit.client.0.smithi022.stdout: 4 2024-09-21T09:31:08.792 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:08.792 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 7, 2024-09-21T09:31:08.792 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:08.792 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.793 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:08.793 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:08.793 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:08.793 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:08.793 INFO:tasks.workunit.client.0.smithi022.stdout: 6 2024-09-21T09:31:08.793 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:08.793 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 3, 2024-09-21T09:31:08.793 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:08.793 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:08.793 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:08.793 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:08.793 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:08.793 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:08.793 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.124976+0000", 2024-09-21T09:31:08.793 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.637751+0000", 2024-09-21T09:31:08.794 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.637751+0000", 2024-09-21T09:31:08.794 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.638161+0000", 2024-09-21T09:31:08.794 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.124976+0000", 2024-09-21T09:31:08.794 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:08.794 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:08.794 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:08.794 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:08.794 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.124976+0000", 2024-09-21T09:31:08.794 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.124976+0000", 2024-09-21T09:31:08.794 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.124976+0000", 2024-09-21T09:31:08.794 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:08.794 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:08.794 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:08.794 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.124976+0000", 2024-09-21T09:31:08.794 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.124976+0000", 2024-09-21T09:31:08.795 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:08.795 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:08.795 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:08.795 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:08.795 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:08.795 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:08.795 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:08.795 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:08.795 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:08.795 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:08.795 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:08.795 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:08.795 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:08.795 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.38", 2024-09-21T09:31:08.795 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:08.795 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:08.796 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:08.796 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:08.796 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:08.796 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T15:07:40.751242+0000", 2024-09-21T09:31:08.796 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:08.796 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:08.796 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:08.796 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:08.796 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:08.796 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:08.796 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:08.796 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:08.796 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:08.796 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:08.796 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:08.797 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:08.797 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:08.797 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:08.797 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:08.797 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:08.797 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:08.797 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:08.797 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:08.797 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:08.797 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:08.797 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:08.797 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:08.797 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:08.797 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:08.797 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:08.797 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:08.798 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:08.798 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:08.798 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:08.798 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:08.798 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:08.798 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:08.798 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:08.798 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:08.798 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:08.798 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:08.798 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:08.798 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:08.798 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:08.798 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:08.799 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:08.799 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:08.799 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.799 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:08.799 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:08.799 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:08.799 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:08.799 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:08.799 INFO:tasks.workunit.client.0.smithi022.stdout: 6 2024-09-21T09:31:08.799 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:08.799 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 3, 2024-09-21T09:31:08.799 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:08.799 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.799 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:08.799 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:08.799 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:08.800 INFO:tasks.workunit.client.0.smithi022.stdout: 1, 2024-09-21T09:31:08.800 INFO:tasks.workunit.client.0.smithi022.stdout: 0 2024-09-21T09:31:08.800 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:08.800 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 7, 2024-09-21T09:31:08.800 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:08.800 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:08.800 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:08.800 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:08.800 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:08.800 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:08.800 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.359502+0000", 2024-09-21T09:31:08.800 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:52.079352+0000", 2024-09-21T09:31:08.800 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:52.079352+0000", 2024-09-21T09:31:08.800 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:52.079760+0000", 2024-09-21T09:31:08.800 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.359502+0000", 2024-09-21T09:31:08.801 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:08.801 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:08.801 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:08.801 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 59, 2024-09-21T09:31:08.801 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.359502+0000", 2024-09-21T09:31:08.801 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.359502+0000", 2024-09-21T09:31:08.801 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.359502+0000", 2024-09-21T09:31:08.801 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:08.801 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:08.801 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:08.801 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.359502+0000", 2024-09-21T09:31:08.801 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.359502+0000", 2024-09-21T09:31:08.801 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:08.801 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:08.801 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:08.801 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:08.802 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 58, 2024-09-21T09:31:08.802 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:08.802 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:08.802 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:08.802 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:08.802 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:08.802 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:08.802 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:08.802 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:08.802 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.37", 2024-09-21T09:31:08.802 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:08.802 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:08.802 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:08.802 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 31, 2024-09-21T09:31:08.803 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:08.803 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T11:57:57.194876+0000", 2024-09-21T09:31:08.803 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:08.803 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:08.803 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:08.803 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:08.803 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:08.803 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:08.803 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:08.803 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:08.803 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:08.803 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:08.803 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:08.803 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:08.804 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:08.804 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:08.804 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:08.804 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:08.804 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:08.804 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:08.804 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:08.804 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:08.804 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:08.804 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:08.804 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:08.804 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:08.804 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:08.804 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:08.804 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:08.804 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:08.805 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:08.805 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:08.805 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:08.805 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:08.805 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:08.805 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:08.805 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:08.805 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:08.805 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:08.805 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:08.805 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:08.805 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:08.805 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:08.805 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:08.805 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:08.806 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.806 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:08.806 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:08.806 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:08.806 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:08.806 INFO:tasks.workunit.client.0.smithi022.stdout: 1, 2024-09-21T09:31:08.806 INFO:tasks.workunit.client.0.smithi022.stdout: 0 2024-09-21T09:31:08.806 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:08.806 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 7, 2024-09-21T09:31:08.806 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:08.806 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.806 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:08.806 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:08.806 INFO:tasks.workunit.client.0.smithi022.stdout: 6, 2024-09-21T09:31:08.807 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:08.807 INFO:tasks.workunit.client.0.smithi022.stdout: 1 2024-09-21T09:31:08.807 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:08.807 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 6, 2024-09-21T09:31:08.807 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:08.807 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:08.807 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:08.807 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:08.807 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:08.807 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:08.807 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.484301+0000", 2024-09-21T09:31:08.807 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.635770+0000", 2024-09-21T09:31:08.807 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.635770+0000", 2024-09-21T09:31:08.807 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.636094+0000", 2024-09-21T09:31:08.807 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.484301+0000", 2024-09-21T09:31:08.808 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:08.808 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:08.808 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:08.808 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:08.808 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.484301+0000", 2024-09-21T09:31:08.808 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.484301+0000", 2024-09-21T09:31:08.808 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.484301+0000", 2024-09-21T09:31:08.808 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:08.808 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:08.808 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:08.808 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.484301+0000", 2024-09-21T09:31:08.808 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.484301+0000", 2024-09-21T09:31:08.809 INFO:tasks.workunit.client.0.smithi022.stderr:+ echo OK 2024-09-21T09:31:08.812 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:08.812 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:08.813 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:08.813 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:08.813 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:08.813 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:08.813 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:08.813 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:08.813 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:08.813 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:08.813 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:08.813 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:08.813 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:08.813 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.36", 2024-09-21T09:31:08.813 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:08.814 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:08.814 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:08.814 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:08.814 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:08.814 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T12:41:52.909549+0000", 2024-09-21T09:31:08.814 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:08.814 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:08.814 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:08.814 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:08.814 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:08.814 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:08.814 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:08.814 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:08.815 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:08.815 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:08.815 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:08.815 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:08.815 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:08.815 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:08.815 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:08.815 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:08.815 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:08.815 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:08.815 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:08.815 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:08.815 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:08.816 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:08.816 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:08.816 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:08.816 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:08.816 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:08.816 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:08.816 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:08.816 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:08.816 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:08.816 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:08.816 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:08.816 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:08.816 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:08.816 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:08.816 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:08.817 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:08.817 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:08.817 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:08.817 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:08.817 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:08.817 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:08.817 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:08.817 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.817 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:08.817 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:08.817 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:08.817 INFO:tasks.workunit.client.0.smithi022.stdout: 6, 2024-09-21T09:31:08.817 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:08.817 INFO:tasks.workunit.client.0.smithi022.stdout: 1 2024-09-21T09:31:08.818 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:08.818 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 6, 2024-09-21T09:31:08.818 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:08.818 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.818 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:08.818 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:08.818 INFO:tasks.workunit.client.0.smithi022.stdout: 6, 2024-09-21T09:31:08.818 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:08.818 INFO:tasks.workunit.client.0.smithi022.stdout: 0 2024-09-21T09:31:08.818 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:08.818 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 6, 2024-09-21T09:31:08.818 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:08.818 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:08.818 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:08.819 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:08.819 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:08.819 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:08.819 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.484371+0000", 2024-09-21T09:31:08.819 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.640253+0000", 2024-09-21T09:31:08.819 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.640253+0000", 2024-09-21T09:31:08.819 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.640571+0000", 2024-09-21T09:31:08.819 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.484371+0000", 2024-09-21T09:31:08.819 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:08.819 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:08.819 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:08.819 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:08.819 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.484371+0000", 2024-09-21T09:31:08.820 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.484371+0000", 2024-09-21T09:31:08.820 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.484371+0000", 2024-09-21T09:31:08.820 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:08.820 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:08.820 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:08.820 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.484371+0000", 2024-09-21T09:31:08.820 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.484371+0000", 2024-09-21T09:31:08.820 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:08.820 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:08.820 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:08.820 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:08.820 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:08.820 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:08.820 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:08.820 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:08.821 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:08.821 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:08.821 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:08.821 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:08.821 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:08.821 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.35", 2024-09-21T09:31:08.821 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:08.821 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:08.821 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:08.821 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:08.821 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:08.821 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T12:40:44.721446+0000", 2024-09-21T09:31:08.821 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:08.822 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:08.822 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:08.822 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:08.822 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:08.822 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:08.822 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:08.822 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:08.822 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:08.822 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:08.822 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:08.822 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:08.822 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:08.822 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:08.823 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:08.823 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:08.823 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:08.823 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:08.823 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:08.823 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:08.823 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:08.823 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:08.823 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:08.823 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:08.823 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:08.823 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:08.823 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:08.824 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:08.824 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:08.824 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:08.824 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:08.824 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:08.824 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:08.824 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:08.824 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:08.824 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:08.824 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:08.824 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:08.824 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:08.824 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:08.824 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:08.824 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:08.825 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:08.825 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.825 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:08.825 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:08.825 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:08.825 INFO:tasks.workunit.client.0.smithi022.stdout: 6, 2024-09-21T09:31:08.825 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:08.825 INFO:tasks.workunit.client.0.smithi022.stdout: 0 2024-09-21T09:31:08.825 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:08.825 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 6, 2024-09-21T09:31:08.825 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:08.825 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.825 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:08.826 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:08.826 INFO:tasks.workunit.client.0.smithi022.stdout: 4, 2024-09-21T09:31:08.826 INFO:tasks.workunit.client.0.smithi022.stdout: 6, 2024-09-21T09:31:08.826 INFO:tasks.workunit.client.0.smithi022.stdout: 0 2024-09-21T09:31:08.826 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:08.826 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 4, 2024-09-21T09:31:08.826 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:08.826 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:08.826 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:08.826 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:08.826 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:08.826 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:08.826 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.062533+0000", 2024-09-21T09:31:08.826 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.639056+0000", 2024-09-21T09:31:08.826 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.639056+0000", 2024-09-21T09:31:08.827 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.639471+0000", 2024-09-21T09:31:08.827 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.062533+0000", 2024-09-21T09:31:08.827 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:08.827 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:08.827 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:08.827 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:08.827 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.062533+0000", 2024-09-21T09:31:08.827 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.062533+0000", 2024-09-21T09:31:08.827 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.062533+0000", 2024-09-21T09:31:08.827 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:08.827 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:08.827 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:08.827 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.062533+0000", 2024-09-21T09:31:08.828 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.062533+0000", 2024-09-21T09:31:08.828 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:08.828 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:08.828 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:08.828 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:08.828 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:08.828 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:08.828 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:08.828 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:08.828 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:08.828 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:08.828 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:08.828 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:08.829 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:08.829 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.34", 2024-09-21T09:31:08.829 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:08.829 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:08.829 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:08.829 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:08.829 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:08.829 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T21:16:15.724781+0000", 2024-09-21T09:31:08.829 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:08.829 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:08.829 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:08.829 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:08.829 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:08.829 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:08.829 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:08.830 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:08.830 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:08.830 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:08.830 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:08.830 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:08.830 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:08.830 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:08.830 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:08.830 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:08.830 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:08.830 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:08.830 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:08.830 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:08.831 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:08.831 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:08.831 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:08.831 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:08.831 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:08.831 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:08.831 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:08.831 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:08.831 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:08.831 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:08.831 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:08.831 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:08.831 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:08.831 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:08.832 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:08.832 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:08.832 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:08.832 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:08.832 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:08.832 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:08.832 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:08.832 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:08.832 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:08.832 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.832 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:08.832 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:08.832 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:08.833 INFO:tasks.workunit.client.0.smithi022.stdout: 4, 2024-09-21T09:31:08.833 INFO:tasks.workunit.client.0.smithi022.stdout: 6, 2024-09-21T09:31:08.833 INFO:tasks.workunit.client.0.smithi022.stdout: 0 2024-09-21T09:31:08.833 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:08.833 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 4, 2024-09-21T09:31:08.833 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:08.833 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.833 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:08.833 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:08.833 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:08.833 INFO:tasks.workunit.client.0.smithi022.stdout: 6, 2024-09-21T09:31:08.833 INFO:tasks.workunit.client.0.smithi022.stdout: 1 2024-09-21T09:31:08.833 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:08.834 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 3, 2024-09-21T09:31:08.834 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:08.834 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:08.834 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:08.834 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:08.834 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:08.834 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:08.834 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.125281+0000", 2024-09-21T09:31:08.834 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.637068+0000", 2024-09-21T09:31:08.834 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.637068+0000", 2024-09-21T09:31:08.834 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.637331+0000", 2024-09-21T09:31:08.834 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.125281+0000", 2024-09-21T09:31:08.834 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:08.834 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:08.834 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:08.835 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:08.835 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.125281+0000", 2024-09-21T09:31:08.835 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.125281+0000", 2024-09-21T09:31:08.835 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.125281+0000", 2024-09-21T09:31:08.835 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:08.835 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:08.835 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:08.835 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.125281+0000", 2024-09-21T09:31:08.835 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.125281+0000", 2024-09-21T09:31:08.835 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:08.835 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:08.835 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:08.835 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:08.836 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:08.836 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:08.836 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:08.836 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:08.836 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:08.836 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:08.836 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:08.836 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:08.836 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:08.836 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.33", 2024-09-21T09:31:08.836 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:08.836 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:08.836 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:08.836 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:08.836 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:08.837 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T15:24:52.176104+0000", 2024-09-21T09:31:08.837 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:08.837 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:08.837 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:08.837 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:08.837 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:08.837 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:08.837 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:08.837 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:08.837 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:08.837 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:08.837 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:08.837 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:08.837 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:08.837 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:08.838 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:08.838 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:08.838 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:08.838 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:08.838 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:08.838 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:08.838 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:08.838 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:08.838 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:08.838 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:08.838 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:08.838 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:08.838 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:08.838 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:08.838 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:08.839 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:08.839 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:08.839 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:08.839 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:08.839 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:08.839 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:08.839 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:08.839 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:08.839 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:08.839 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:08.839 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:08.839 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:08.839 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:08.839 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:08.840 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.840 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:08.840 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:08.840 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:08.840 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:08.840 INFO:tasks.workunit.client.0.smithi022.stdout: 6, 2024-09-21T09:31:08.840 INFO:tasks.workunit.client.0.smithi022.stdout: 1 2024-09-21T09:31:08.840 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:08.840 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 3, 2024-09-21T09:31:08.840 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:08.840 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.840 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:08.841 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:08.841 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:08.841 INFO:tasks.workunit.client.0.smithi022.stdout: 1, 2024-09-21T09:31:08.841 INFO:tasks.workunit.client.0.smithi022.stdout: 5 2024-09-21T09:31:08.841 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:08.841 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 0, 2024-09-21T09:31:08.841 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:08.841 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:08.841 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:08.841 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:08.841 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:08.841 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:08.841 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.140751+0000", 2024-09-21T09:31:08.841 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.644114+0000", 2024-09-21T09:31:08.842 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.644114+0000", 2024-09-21T09:31:08.842 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.645565+0000", 2024-09-21T09:31:08.842 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.140751+0000", 2024-09-21T09:31:08.842 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:08.842 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:08.842 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:08.842 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:08.842 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.140751+0000", 2024-09-21T09:31:08.842 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.140751+0000", 2024-09-21T09:31:08.842 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.140751+0000", 2024-09-21T09:31:08.842 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:08.842 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:08.842 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:08.842 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.140751+0000", 2024-09-21T09:31:08.843 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.140751+0000", 2024-09-21T09:31:08.843 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:08.843 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:08.843 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:08.843 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:08.843 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:08.843 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:08.843 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:08.843 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:08.843 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:08.843 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:08.843 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:08.843 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:08.843 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:08.844 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.32", 2024-09-21T09:31:08.844 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:08.844 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:08.844 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:08.844 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:08.844 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:08.844 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T10:27:18.008605+0000", 2024-09-21T09:31:08.844 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:08.844 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:08.844 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:08.844 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:08.844 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:08.844 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:08.845 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:08.845 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:08.845 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:08.845 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:08.845 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:08.845 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:08.845 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:08.845 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:08.845 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:08.845 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:08.845 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:08.845 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:08.845 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:08.846 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:08.846 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:08.846 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:08.846 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:08.846 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:08.846 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:08.846 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:08.846 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:08.846 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:08.846 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:08.846 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:08.846 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:08.846 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:08.847 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:08.847 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:08.847 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:08.847 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:08.847 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:08.847 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:08.847 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:08.847 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:08.847 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:08.847 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:08.847 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:08.847 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.847 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:08.848 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:08.848 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:08.848 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:08.848 INFO:tasks.workunit.client.0.smithi022.stdout: 1, 2024-09-21T09:31:08.848 INFO:tasks.workunit.client.0.smithi022.stdout: 5 2024-09-21T09:31:08.848 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:08.848 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 0, 2024-09-21T09:31:08.848 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:08.848 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.848 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:08.848 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:08.848 INFO:tasks.workunit.client.0.smithi022.stdout: 1, 2024-09-21T09:31:08.848 INFO:tasks.workunit.client.0.smithi022.stdout: 4, 2024-09-21T09:31:08.848 INFO:tasks.workunit.client.0.smithi022.stdout: 5 2024-09-21T09:31:08.849 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:08.849 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 1, 2024-09-21T09:31:08.849 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:08.849 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:08.849 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:08.849 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:08.849 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:08.849 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:08.849 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.062608+0000", 2024-09-21T09:31:08.849 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.634089+0000", 2024-09-21T09:31:08.849 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.634089+0000", 2024-09-21T09:31:08.849 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.634448+0000", 2024-09-21T09:31:08.849 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.062608+0000", 2024-09-21T09:31:08.850 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:08.850 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:08.850 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:08.850 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:08.850 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.062608+0000", 2024-09-21T09:31:08.850 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.062608+0000", 2024-09-21T09:31:08.850 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.062608+0000", 2024-09-21T09:31:08.850 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:08.850 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:08.850 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:08.850 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.062608+0000", 2024-09-21T09:31:08.850 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.062608+0000", 2024-09-21T09:31:08.850 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:08.851 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:08.851 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:08.851 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:08.851 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:08.851 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:08.851 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:08.851 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:08.851 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:08.851 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:08.851 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:08.851 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:08.851 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:08.851 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.31", 2024-09-21T09:31:08.852 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:08.852 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:08.852 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:08.852 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:08.852 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:08.852 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T10:14:14.726492+0000", 2024-09-21T09:31:08.852 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:08.852 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:08.852 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:08.852 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:08.852 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:08.852 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:08.852 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:08.852 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:08.853 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:08.853 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:08.853 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:08.853 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:08.853 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:08.853 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:08.853 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:08.853 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:08.853 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:08.853 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:08.853 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:08.853 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:08.853 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:08.853 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:08.854 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:08.854 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:08.854 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:08.854 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:08.854 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:08.854 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:08.854 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:08.854 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:08.854 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:08.854 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:08.854 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:08.854 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:08.854 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:08.855 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:08.855 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:08.855 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:08.855 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:08.855 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:08.855 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:08.855 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:08.855 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:08.855 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.855 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:08.855 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:08.855 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:08.855 INFO:tasks.workunit.client.0.smithi022.stdout: 1, 2024-09-21T09:31:08.855 INFO:tasks.workunit.client.0.smithi022.stdout: 4, 2024-09-21T09:31:08.856 INFO:tasks.workunit.client.0.smithi022.stdout: 5 2024-09-21T09:31:08.856 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:08.856 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 1, 2024-09-21T09:31:08.856 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:08.856 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.856 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:08.856 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:08.856 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:08.856 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:08.856 INFO:tasks.workunit.client.0.smithi022.stdout: 7 2024-09-21T09:31:08.856 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:08.856 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 5, 2024-09-21T09:31:08.856 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:08.856 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:08.857 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:08.857 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:08.857 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:08.857 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:08.857 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.066102+0000", 2024-09-21T09:31:08.857 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.651698+0000", 2024-09-21T09:31:08.857 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.651698+0000", 2024-09-21T09:31:08.857 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.651997+0000", 2024-09-21T09:31:08.857 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.066102+0000", 2024-09-21T09:31:08.857 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:08.857 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:08.857 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:08.857 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:08.857 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.066102+0000", 2024-09-21T09:31:08.857 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.066102+0000", 2024-09-21T09:31:08.858 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.066102+0000", 2024-09-21T09:31:08.858 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:08.858 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:08.858 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:08.858 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.066102+0000", 2024-09-21T09:31:08.858 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.066102+0000", 2024-09-21T09:31:08.858 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:08.858 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:08.858 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:08.858 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:08.858 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:08.858 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:08.858 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:08.858 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:08.859 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:08.859 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:08.859 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:08.859 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:08.859 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:08.859 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.30", 2024-09-21T09:31:08.859 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:08.859 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:08.859 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:08.859 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:08.859 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:08.859 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T09:43:39.574788+0000", 2024-09-21T09:31:08.859 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:08.859 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:08.860 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:08.860 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:08.860 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:08.860 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:08.860 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:08.860 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:08.860 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:08.860 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:08.860 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:08.860 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:08.860 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:08.860 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:08.860 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:08.860 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:08.860 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:08.861 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:08.861 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:08.861 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:08.861 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:08.861 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:08.861 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:08.861 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:08.861 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:08.861 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:08.861 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:08.861 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:08.861 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:08.861 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:08.861 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:08.861 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:08.862 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:08.862 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:08.862 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:08.862 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:08.862 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:08.862 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:08.862 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:08.862 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:08.862 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:08.862 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:08.862 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:08.862 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.862 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:08.862 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:08.863 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:08.863 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:08.863 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:08.863 INFO:tasks.workunit.client.0.smithi022.stdout: 7 2024-09-21T09:31:08.863 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:08.863 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 5, 2024-09-21T09:31:08.863 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:08.863 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.863 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:08.863 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:08.863 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:08.863 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:08.863 INFO:tasks.workunit.client.0.smithi022.stdout: 4 2024-09-21T09:31:08.863 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:08.863 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 5, 2024-09-21T09:31:08.864 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:08.864 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:08.864 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:08.864 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:08.864 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:08.864 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:08.864 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.066495+0000", 2024-09-21T09:31:08.864 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.646084+0000", 2024-09-21T09:31:08.864 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.646084+0000", 2024-09-21T09:31:08.864 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.646363+0000", 2024-09-21T09:31:08.864 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.066495+0000", 2024-09-21T09:31:08.864 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:08.864 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:08.864 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:08.864 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:08.865 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.066495+0000", 2024-09-21T09:31:08.865 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.066495+0000", 2024-09-21T09:31:08.865 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.066495+0000", 2024-09-21T09:31:08.865 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:08.865 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:08.865 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:08.865 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.066495+0000", 2024-09-21T09:31:08.865 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.066495+0000", 2024-09-21T09:31:08.865 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:08.865 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:08.865 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:08.865 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:08.865 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:08.865 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:08.865 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:08.866 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:08.866 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:08.866 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:08.866 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:08.866 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:08.866 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:08.866 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.2f", 2024-09-21T09:31:08.866 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:08.866 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:08.866 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:08.866 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:08.866 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:08.866 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T15:35:38.711926+0000", 2024-09-21T09:31:08.866 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:08.866 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:08.867 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:08.867 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:08.867 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:08.867 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:08.867 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:08.867 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:08.867 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:08.867 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:08.867 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:08.867 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:08.867 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:08.867 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:08.867 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:08.867 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:08.868 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:08.868 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:08.868 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:08.868 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:08.868 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:08.868 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:08.868 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:08.868 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:08.868 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:08.868 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:08.868 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:08.868 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:08.869 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:08.869 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:08.869 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:08.869 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:08.869 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:08.869 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:08.869 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:08.869 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:08.869 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:08.869 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:08.869 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:08.869 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:08.869 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:08.870 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:08.870 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:08.870 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.870 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:08.870 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:08.870 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:08.870 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:08.870 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:08.870 INFO:tasks.workunit.client.0.smithi022.stdout: 4 2024-09-21T09:31:08.870 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:08.870 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 5, 2024-09-21T09:31:08.870 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:08.870 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.870 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:08.871 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:08.871 INFO:tasks.workunit.client.0.smithi022.stdout: 2, 2024-09-21T09:31:08.871 INFO:tasks.workunit.client.0.smithi022.stdout: 6, 2024-09-21T09:31:08.871 INFO:tasks.workunit.client.0.smithi022.stdout: 7 2024-09-21T09:31:08.871 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:08.871 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 2, 2024-09-21T09:31:08.871 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:08.871 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:08.871 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:08.871 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:08.871 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:08.871 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:08.871 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.061977+0000", 2024-09-21T09:31:08.872 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.636438+0000", 2024-09-21T09:31:08.872 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.636438+0000", 2024-09-21T09:31:08.872 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.636697+0000", 2024-09-21T09:31:08.872 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.061977+0000", 2024-09-21T09:31:08.872 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:08.872 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:08.872 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:08.872 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:08.872 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.061977+0000", 2024-09-21T09:31:08.872 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.061977+0000", 2024-09-21T09:31:08.872 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.061977+0000", 2024-09-21T09:31:08.872 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:08.872 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:08.872 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:08.872 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.061977+0000", 2024-09-21T09:31:08.873 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.061977+0000", 2024-09-21T09:31:08.873 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:08.873 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:08.873 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:08.873 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:08.873 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:08.873 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:08.873 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:08.873 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:08.873 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:08.873 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:08.873 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:08.873 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:08.873 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:08.874 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.2e", 2024-09-21T09:31:08.874 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:08.874 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:08.874 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:08.874 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:08.874 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:08.874 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T15:55:44.287544+0000", 2024-09-21T09:31:08.874 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:08.874 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:08.874 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:08.874 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:08.874 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:08.874 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:08.874 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:08.874 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:08.875 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:08.875 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:08.875 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:08.875 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:08.875 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:08.875 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:08.875 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:08.875 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:08.875 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:08.875 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:08.875 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:08.875 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:08.875 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:08.875 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:08.876 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:08.876 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:08.876 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:08.876 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:08.876 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:08.876 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:08.876 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:08.876 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:08.876 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:08.876 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:08.876 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:08.876 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:08.877 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:08.877 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:08.877 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:08.877 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:08.877 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:08.877 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:08.877 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:08.877 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:08.877 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:08.877 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.877 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:08.877 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:08.877 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:08.877 INFO:tasks.workunit.client.0.smithi022.stdout: 2, 2024-09-21T09:31:08.878 INFO:tasks.workunit.client.0.smithi022.stdout: 6, 2024-09-21T09:31:08.878 INFO:tasks.workunit.client.0.smithi022.stdout: 7 2024-09-21T09:31:08.878 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:08.878 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 2, 2024-09-21T09:31:08.878 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:08.878 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.878 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:08.878 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:08.878 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:08.878 INFO:tasks.workunit.client.0.smithi022.stdout: 2, 2024-09-21T09:31:08.878 INFO:tasks.workunit.client.0.smithi022.stdout: 6 2024-09-21T09:31:08.878 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:08.878 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 7, 2024-09-21T09:31:08.878 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:08.878 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:08.879 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:08.879 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:08.879 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:08.879 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:08.879 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.359395+0000", 2024-09-21T09:31:08.879 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.635807+0000", 2024-09-21T09:31:08.879 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.635807+0000", 2024-09-21T09:31:08.879 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.636110+0000", 2024-09-21T09:31:08.879 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.359395+0000", 2024-09-21T09:31:08.879 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:08.879 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:08.879 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:08.879 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:08.879 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.359395+0000", 2024-09-21T09:31:08.879 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.359395+0000", 2024-09-21T09:31:08.880 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.359395+0000", 2024-09-21T09:31:08.880 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:08.880 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:08.880 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:08.880 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.359395+0000", 2024-09-21T09:31:08.880 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.359395+0000", 2024-09-21T09:31:08.880 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:08.880 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:08.880 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:08.880 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:08.880 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:08.880 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:08.880 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:08.880 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:08.880 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:08.880 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:08.881 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:08.881 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:08.881 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:08.881 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.2d", 2024-09-21T09:31:08.881 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:08.881 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:08.881 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:08.881 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:08.881 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:08.881 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T16:22:43.942811+0000", 2024-09-21T09:31:08.881 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:08.881 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:08.881 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:08.881 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:08.881 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:08.882 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:08.882 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:08.882 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:08.882 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:08.882 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:08.882 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:08.882 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:08.882 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:08.882 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:08.882 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:08.882 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:08.882 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:08.882 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:08.882 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:08.883 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:08.883 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:08.883 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:08.883 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:08.883 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:08.883 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:08.883 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:08.883 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:08.883 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:08.883 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:08.883 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:08.883 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:08.883 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:08.883 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:08.884 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:08.884 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:08.884 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:08.884 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:08.884 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:08.884 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:08.884 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:08.884 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:08.884 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:08.884 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:08.884 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.884 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:08.884 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:08.884 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:08.884 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:08.884 INFO:tasks.workunit.client.0.smithi022.stdout: 2, 2024-09-21T09:31:08.885 INFO:tasks.workunit.client.0.smithi022.stdout: 6 2024-09-21T09:31:08.885 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:08.885 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 7, 2024-09-21T09:31:08.885 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:08.885 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.885 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:08.885 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:08.885 INFO:tasks.workunit.client.0.smithi022.stdout: 6, 2024-09-21T09:31:08.885 INFO:tasks.workunit.client.0.smithi022.stdout: 4, 2024-09-21T09:31:08.885 INFO:tasks.workunit.client.0.smithi022.stdout: 0 2024-09-21T09:31:08.885 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:08.885 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 6, 2024-09-21T09:31:08.885 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:08.885 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:08.885 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:08.886 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:08.886 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:08.886 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:08.886 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.484182+0000", 2024-09-21T09:31:08.886 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.649905+0000", 2024-09-21T09:31:08.886 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.649905+0000", 2024-09-21T09:31:08.886 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.650351+0000", 2024-09-21T09:31:08.886 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.484182+0000", 2024-09-21T09:31:08.886 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:08.886 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:08.886 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:08.886 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:08.886 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.484182+0000", 2024-09-21T09:31:08.887 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.484182+0000", 2024-09-21T09:31:08.887 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.484182+0000", 2024-09-21T09:31:08.887 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:08.887 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:08.887 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:08.887 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.484182+0000", 2024-09-21T09:31:08.887 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.484182+0000", 2024-09-21T09:31:08.887 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:08.887 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:08.887 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:08.887 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:08.887 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:08.887 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:08.887 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:08.887 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:08.887 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:08.888 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:08.888 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:08.888 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:08.888 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:08.888 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.2c", 2024-09-21T09:31:08.888 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:08.888 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:08.888 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:08.888 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:08.888 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:08.888 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T13:12:31.390666+0000", 2024-09-21T09:31:08.888 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:08.888 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:08.888 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:08.888 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:08.889 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:08.889 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:08.889 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:08.889 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:08.889 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:08.889 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:08.889 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:08.889 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:08.889 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:08.889 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:08.889 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:08.889 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:08.889 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:08.889 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:08.889 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:08.889 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:08.889 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:08.890 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:08.890 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:08.890 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:08.890 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:08.890 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:08.890 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:08.890 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:08.890 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:08.890 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:08.890 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:08.890 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:08.890 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:08.890 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:08.891 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:08.891 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:08.891 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:08.891 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:08.891 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:08.891 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:08.891 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:08.891 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:08.891 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:08.891 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.891 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:08.891 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:08.891 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:08.891 INFO:tasks.workunit.client.0.smithi022.stdout: 6, 2024-09-21T09:31:08.891 INFO:tasks.workunit.client.0.smithi022.stdout: 4, 2024-09-21T09:31:08.892 INFO:tasks.workunit.client.0.smithi022.stdout: 0 2024-09-21T09:31:08.892 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:08.892 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 6, 2024-09-21T09:31:08.892 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:08.892 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.892 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:08.892 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:08.892 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:08.892 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:08.892 INFO:tasks.workunit.client.0.smithi022.stdout: 1 2024-09-21T09:31:08.892 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:08.892 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 0, 2024-09-21T09:31:08.892 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:08.892 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:08.892 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:08.893 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:08.893 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:08.893 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:08.893 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.141020+0000", 2024-09-21T09:31:08.893 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.642408+0000", 2024-09-21T09:31:08.893 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.642408+0000", 2024-09-21T09:31:08.893 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.643345+0000", 2024-09-21T09:31:08.893 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.141020+0000", 2024-09-21T09:31:08.893 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:08.893 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:08.893 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:08.893 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:08.893 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.141020+0000", 2024-09-21T09:31:08.893 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.141020+0000", 2024-09-21T09:31:08.893 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.141020+0000", 2024-09-21T09:31:08.893 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:08.894 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:08.894 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:08.894 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.141020+0000", 2024-09-21T09:31:08.894 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.141020+0000", 2024-09-21T09:31:08.894 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:08.894 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:08.894 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:08.894 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:08.894 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:08.894 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:08.894 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:08.894 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:08.894 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:08.894 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:08.895 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:08.895 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:08.895 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:08.895 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.2b", 2024-09-21T09:31:08.895 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:08.895 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:08.895 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:08.895 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:08.895 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:08.895 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T17:34:59.698334+0000", 2024-09-21T09:31:08.895 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:08.895 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:08.895 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:08.895 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:08.896 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:08.896 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:08.896 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:08.896 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:08.896 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:08.896 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:08.896 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:08.896 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:08.896 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:08.896 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:08.896 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:08.896 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:08.896 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:08.896 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:08.896 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:08.896 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:08.897 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:08.897 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:08.897 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:08.897 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:08.897 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:08.897 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:08.897 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:08.897 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:08.897 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:08.897 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:08.897 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:08.897 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:08.897 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:08.897 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:08.897 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:08.898 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:08.898 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:08.898 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:08.898 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:08.898 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:08.898 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:08.898 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:08.898 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:08.898 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.898 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:08.898 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:08.898 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:08.898 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:08.898 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:08.899 INFO:tasks.workunit.client.0.smithi022.stdout: 1 2024-09-21T09:31:08.899 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:08.899 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 0, 2024-09-21T09:31:08.899 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:08.899 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.899 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:08.899 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:08.899 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:08.899 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:08.899 INFO:tasks.workunit.client.0.smithi022.stdout: 6 2024-09-21T09:31:08.899 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:08.899 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 5, 2024-09-21T09:31:08.899 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:08.899 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:08.899 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:08.899 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:08.900 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:08.900 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:08.900 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.066015+0000", 2024-09-21T09:31:08.900 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.644322+0000", 2024-09-21T09:31:08.900 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.644322+0000", 2024-09-21T09:31:08.900 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.645032+0000", 2024-09-21T09:31:08.900 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.066015+0000", 2024-09-21T09:31:08.900 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:08.900 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:08.900 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:08.900 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:08.900 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.066015+0000", 2024-09-21T09:31:08.900 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.066015+0000", 2024-09-21T09:31:08.900 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.066015+0000", 2024-09-21T09:31:08.900 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:08.901 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:08.901 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:08.901 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.066015+0000", 2024-09-21T09:31:08.901 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.066015+0000", 2024-09-21T09:31:08.901 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:08.901 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:08.901 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:08.901 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:08.901 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:08.901 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:08.901 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:08.901 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:08.901 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:08.901 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:08.901 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:08.902 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:08.902 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:08.902 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.2a", 2024-09-21T09:31:08.902 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:08.902 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:08.902 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:08.902 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:08.902 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:08.902 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T16:10:14.511772+0000", 2024-09-21T09:31:08.902 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:08.902 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:08.902 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:08.902 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:08.902 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:08.902 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:08.903 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:08.903 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:08.903 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:08.903 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:08.903 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:08.903 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:08.903 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:08.903 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:08.903 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:08.903 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:08.903 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:08.903 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:08.903 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:08.903 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:08.903 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:08.903 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:08.904 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:08.904 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:08.904 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:08.904 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:08.904 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:08.904 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:08.904 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:08.904 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:08.904 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:08.904 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:08.904 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:08.904 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:08.904 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:08.904 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:08.904 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:08.904 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:08.905 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:08.905 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:08.905 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:08.905 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:08.905 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:08.905 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.905 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:08.905 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:08.905 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:08.905 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:08.905 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:08.905 INFO:tasks.workunit.client.0.smithi022.stdout: 6 2024-09-21T09:31:08.905 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:08.905 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 5, 2024-09-21T09:31:08.905 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:08.905 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.906 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:08.906 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:08.906 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:08.906 INFO:tasks.workunit.client.0.smithi022.stdout: 4, 2024-09-21T09:31:08.906 INFO:tasks.workunit.client.0.smithi022.stdout: 1 2024-09-21T09:31:08.906 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:08.906 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 3, 2024-09-21T09:31:08.906 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:08.906 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:08.906 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:08.906 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:08.906 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:08.906 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:08.906 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.125084+0000", 2024-09-21T09:31:08.907 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.638036+0000", 2024-09-21T09:31:08.907 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.638036+0000", 2024-09-21T09:31:08.907 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.638305+0000", 2024-09-21T09:31:08.907 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.125084+0000", 2024-09-21T09:31:08.907 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:08.907 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:08.907 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:08.907 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:08.907 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.125084+0000", 2024-09-21T09:31:08.907 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.125084+0000", 2024-09-21T09:31:08.907 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.125084+0000", 2024-09-21T09:31:08.907 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:08.907 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:08.907 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:08.907 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.125084+0000", 2024-09-21T09:31:08.908 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.125084+0000", 2024-09-21T09:31:08.908 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:08.908 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:08.908 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:08.908 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:08.908 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:08.908 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:08.908 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:08.908 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:08.908 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:08.908 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:08.908 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:08.908 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:08.908 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:08.909 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.29", 2024-09-21T09:31:08.909 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:08.909 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:08.909 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:08.909 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:08.909 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:08.909 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T10:38:54.627840+0000", 2024-09-21T09:31:08.909 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:08.909 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:08.909 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:08.909 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:08.909 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:08.909 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:08.909 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:08.910 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:08.910 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:08.910 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:08.910 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:08.910 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:08.910 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:08.910 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:08.910 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:08.910 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:08.910 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:08.910 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:08.910 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:08.910 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:08.910 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:08.910 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:08.911 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:08.911 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:08.911 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:08.911 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:08.911 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:08.911 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:08.911 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:08.911 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:08.911 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:08.911 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:08.911 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:08.911 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:08.911 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:08.911 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:08.911 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:08.912 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:08.912 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:08.912 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:08.912 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:08.912 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:08.912 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:08.912 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.912 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:08.912 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:08.912 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:08.912 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:08.912 INFO:tasks.workunit.client.0.smithi022.stdout: 4, 2024-09-21T09:31:08.912 INFO:tasks.workunit.client.0.smithi022.stdout: 1 2024-09-21T09:31:08.913 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:08.913 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 3, 2024-09-21T09:31:08.913 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:08.913 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.913 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:08.913 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:08.913 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:08.913 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:08.913 INFO:tasks.workunit.client.0.smithi022.stdout: 2 2024-09-21T09:31:08.913 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:08.913 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 5, 2024-09-21T09:31:08.913 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:08.913 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:08.913 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:08.913 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:08.914 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:08.914 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:08.914 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.066181+0000", 2024-09-21T09:31:08.914 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.636541+0000", 2024-09-21T09:31:08.914 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.636541+0000", 2024-09-21T09:31:08.914 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.637277+0000", 2024-09-21T09:31:08.914 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.066181+0000", 2024-09-21T09:31:08.914 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:08.914 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:08.914 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:08.914 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:08.914 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.066181+0000", 2024-09-21T09:31:08.914 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.066181+0000", 2024-09-21T09:31:08.914 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.066181+0000", 2024-09-21T09:31:08.914 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:08.914 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:08.915 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:08.915 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.066181+0000", 2024-09-21T09:31:08.915 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.066181+0000", 2024-09-21T09:31:08.915 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:08.915 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:08.915 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:08.915 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:08.915 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:08.915 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:08.915 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:08.915 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:08.915 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:08.916 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:08.916 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:08.916 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:08.916 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:08.916 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.28", 2024-09-21T09:31:08.916 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:08.916 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:08.916 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:08.916 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:08.916 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:08.916 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T09:45:47.090531+0000", 2024-09-21T09:31:08.916 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:08.916 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:08.917 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:08.917 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:08.917 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:08.917 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:08.917 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:08.917 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:08.917 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:08.917 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:08.917 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:08.917 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:08.917 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:08.917 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:08.917 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:08.917 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:08.917 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:08.917 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:08.918 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:08.918 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:08.918 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:08.918 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:08.918 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:08.918 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:08.918 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:08.918 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:08.918 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:08.918 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:08.918 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:08.918 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:08.918 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:08.919 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:08.919 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:08.919 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:08.919 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:08.919 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:08.919 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:08.919 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:08.919 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:08.919 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:08.919 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:08.919 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:08.919 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:08.919 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.920 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:08.920 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:08.920 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:08.920 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:08.920 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:08.920 INFO:tasks.workunit.client.0.smithi022.stdout: 2 2024-09-21T09:31:08.920 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:08.920 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 5, 2024-09-21T09:31:08.920 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:08.920 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.920 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:08.920 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:08.921 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:08.921 INFO:tasks.workunit.client.0.smithi022.stdout: 4, 2024-09-21T09:31:08.921 INFO:tasks.workunit.client.0.smithi022.stdout: 2 2024-09-21T09:31:08.921 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:08.921 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 3, 2024-09-21T09:31:08.921 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:08.921 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:08.921 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:08.921 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:08.921 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:08.921 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:08.921 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.125608+0000", 2024-09-21T09:31:08.921 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.638663+0000", 2024-09-21T09:31:08.921 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.638663+0000", 2024-09-21T09:31:08.922 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.638946+0000", 2024-09-21T09:31:08.922 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.125608+0000", 2024-09-21T09:31:08.922 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:08.922 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:08.922 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:08.922 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:08.922 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.125608+0000", 2024-09-21T09:31:08.922 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.125608+0000", 2024-09-21T09:31:08.922 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.125608+0000", 2024-09-21T09:31:08.922 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:08.922 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:08.922 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:08.922 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.125608+0000", 2024-09-21T09:31:08.923 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.125608+0000", 2024-09-21T09:31:08.923 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:08.923 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:08.923 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:08.923 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:08.923 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:08.923 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:08.923 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:08.923 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:08.923 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:08.923 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:08.923 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:08.923 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:08.923 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:08.924 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.27", 2024-09-21T09:31:08.924 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:08.924 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:08.924 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:08.924 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:08.924 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:08.924 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T12:49:57.591531+0000", 2024-09-21T09:31:08.924 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:08.924 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:08.924 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:08.924 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:08.924 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:08.924 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:08.924 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:08.924 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:08.925 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:08.925 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:08.925 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:08.925 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:08.925 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:08.925 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:08.925 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:08.925 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:08.925 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:08.925 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:08.925 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:08.925 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:08.925 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:08.925 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:08.925 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:08.925 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:08.926 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:08.926 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:08.926 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:08.926 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:08.926 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:08.926 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:08.926 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:08.926 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:08.926 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:08.926 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:08.926 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:08.926 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:08.926 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:08.926 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:08.926 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:08.927 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:08.927 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:08.927 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:08.927 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:08.927 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.927 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:08.927 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:08.927 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:08.927 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:08.927 INFO:tasks.workunit.client.0.smithi022.stdout: 4, 2024-09-21T09:31:08.927 INFO:tasks.workunit.client.0.smithi022.stdout: 2 2024-09-21T09:31:08.927 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:08.927 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 3, 2024-09-21T09:31:08.927 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:08.928 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.928 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:08.928 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:08.928 INFO:tasks.workunit.client.0.smithi022.stdout: 6, 2024-09-21T09:31:08.928 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:08.928 INFO:tasks.workunit.client.0.smithi022.stdout: 4 2024-09-21T09:31:08.928 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:08.928 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 6, 2024-09-21T09:31:08.928 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:08.928 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:08.928 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:08.928 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:08.928 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:08.928 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:08.928 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.484321+0000", 2024-09-21T09:31:08.928 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.638383+0000", 2024-09-21T09:31:08.929 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.638383+0000", 2024-09-21T09:31:08.929 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.638679+0000", 2024-09-21T09:31:08.929 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.484321+0000", 2024-09-21T09:31:08.929 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:08.929 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:08.929 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:08.929 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:08.929 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.484321+0000", 2024-09-21T09:31:08.929 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.484321+0000", 2024-09-21T09:31:08.929 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.484321+0000", 2024-09-21T09:31:08.929 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:08.929 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:08.929 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:08.929 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.484321+0000", 2024-09-21T09:31:08.929 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.484321+0000", 2024-09-21T09:31:08.930 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:08.930 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:08.930 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:08.930 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:08.930 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:08.930 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:08.930 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:08.930 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:08.930 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:08.930 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:08.930 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:08.930 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:08.930 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:08.930 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.26", 2024-09-21T09:31:08.931 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:08.931 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:08.931 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:08.931 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:08.931 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:08.931 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T18:16:11.663503+0000", 2024-09-21T09:31:08.931 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:08.931 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:08.931 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:08.931 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:08.931 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:08.931 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:08.931 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:08.932 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:08.932 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:08.932 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:08.932 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:08.932 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:08.932 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:08.932 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:08.932 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:08.932 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:08.932 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:08.932 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:08.932 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:08.932 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:08.932 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:08.933 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:08.933 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:08.933 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:08.933 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:08.933 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:08.933 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:08.933 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:08.933 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:08.933 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:08.933 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:08.933 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:08.933 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:08.934 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:08.934 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:08.934 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:08.934 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:08.934 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:08.934 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:08.934 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:08.934 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:08.934 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:08.934 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:08.934 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.934 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:08.934 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:08.934 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:08.935 INFO:tasks.workunit.client.0.smithi022.stdout: 6, 2024-09-21T09:31:08.935 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:08.935 INFO:tasks.workunit.client.0.smithi022.stdout: 4 2024-09-21T09:31:08.935 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:08.935 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 6, 2024-09-21T09:31:08.935 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:08.935 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.935 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:08.935 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:08.935 INFO:tasks.workunit.client.0.smithi022.stdout: 1, 2024-09-21T09:31:08.935 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:08.935 INFO:tasks.workunit.client.0.smithi022.stdout: 2 2024-09-21T09:31:08.935 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:08.936 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 1, 2024-09-21T09:31:08.936 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:08.936 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:08.936 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:08.936 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:08.936 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:08.936 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:08.936 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.061992+0000", 2024-09-21T09:31:08.936 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.632569+0000", 2024-09-21T09:31:08.936 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.632569+0000", 2024-09-21T09:31:08.936 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.632894+0000", 2024-09-21T09:31:08.936 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.061992+0000", 2024-09-21T09:31:08.936 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:08.936 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:08.936 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:08.937 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:08.937 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.061992+0000", 2024-09-21T09:31:08.937 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.061992+0000", 2024-09-21T09:31:08.937 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.061992+0000", 2024-09-21T09:31:08.937 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:08.937 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:08.937 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:08.937 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.061992+0000", 2024-09-21T09:31:08.937 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.061992+0000", 2024-09-21T09:31:08.937 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:08.937 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:08.937 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:08.937 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:08.937 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:08.937 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:08.937 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:08.938 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:08.938 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:08.938 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:08.938 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:08.938 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:08.938 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:08.938 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.25", 2024-09-21T09:31:08.938 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:08.938 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:08.938 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:08.938 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:08.938 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:08.938 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T13:25:22.833535+0000", 2024-09-21T09:31:08.939 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:08.939 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:08.939 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:08.939 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:08.939 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:08.939 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:08.939 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:08.939 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:08.939 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:08.939 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:08.939 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:08.939 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:08.939 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:08.939 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:08.939 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:08.940 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:08.940 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:08.940 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:08.940 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:08.940 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:08.940 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:08.940 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:08.940 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:08.940 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:08.940 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:08.940 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:08.940 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:08.940 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:08.940 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:08.940 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:08.941 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:08.941 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:08.941 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:08.941 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:08.941 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:08.941 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:08.941 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:08.941 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:08.941 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:08.941 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:08.941 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:08.941 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:08.941 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:08.941 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.941 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:08.941 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:08.942 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:08.942 INFO:tasks.workunit.client.0.smithi022.stdout: 1, 2024-09-21T09:31:08.942 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:08.942 INFO:tasks.workunit.client.0.smithi022.stdout: 2 2024-09-21T09:31:08.942 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:08.942 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 1, 2024-09-21T09:31:08.942 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:08.942 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.942 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:08.942 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:08.942 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:08.942 INFO:tasks.workunit.client.0.smithi022.stdout: 2, 2024-09-21T09:31:08.942 INFO:tasks.workunit.client.0.smithi022.stdout: 0 2024-09-21T09:31:08.943 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:08.943 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 3, 2024-09-21T09:31:08.943 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:08.943 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:08.943 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:08.943 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:08.943 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:08.943 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:08.943 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.125399+0000", 2024-09-21T09:31:08.943 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:52.079395+0000", 2024-09-21T09:31:08.943 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:52.079395+0000", 2024-09-21T09:31:08.943 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:52.079786+0000", 2024-09-21T09:31:08.943 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.125399+0000", 2024-09-21T09:31:08.943 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:08.944 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:08.944 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:08.944 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 59, 2024-09-21T09:31:08.944 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.125399+0000", 2024-09-21T09:31:08.944 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.125399+0000", 2024-09-21T09:31:08.944 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.125399+0000", 2024-09-21T09:31:08.944 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:08.944 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:08.944 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:08.944 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.125399+0000", 2024-09-21T09:31:08.944 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.125399+0000", 2024-09-21T09:31:08.944 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:08.944 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:08.945 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:08.945 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:08.945 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 58, 2024-09-21T09:31:08.945 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:08.945 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:08.945 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:08.945 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:08.945 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:08.945 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:08.945 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:08.945 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:08.945 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.24", 2024-09-21T09:31:08.945 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:08.945 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:08.946 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:08.946 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 31, 2024-09-21T09:31:08.946 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:08.946 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T12:17:07.388900+0000", 2024-09-21T09:31:08.946 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:08.946 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:08.946 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:08.946 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:08.946 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:08.946 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:08.946 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:08.946 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:08.947 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:08.947 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:08.947 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:08.947 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:08.947 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:08.947 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:08.947 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:08.947 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:08.947 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:08.947 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:08.947 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:08.948 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:08.948 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:08.948 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:08.948 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:08.948 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:08.948 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:08.948 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:08.948 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:08.948 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:08.948 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:08.948 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:08.948 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:08.948 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:08.949 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:08.949 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:08.949 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:08.949 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:08.949 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:08.949 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:08.949 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:08.949 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:08.949 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:08.949 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:08.949 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:08.949 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.949 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:08.950 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:08.950 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:08.950 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:08.950 INFO:tasks.workunit.client.0.smithi022.stdout: 2, 2024-09-21T09:31:08.950 INFO:tasks.workunit.client.0.smithi022.stdout: 0 2024-09-21T09:31:08.950 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:08.950 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 3, 2024-09-21T09:31:08.950 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:08.950 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.950 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:08.950 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:08.950 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:08.950 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:08.951 INFO:tasks.workunit.client.0.smithi022.stdout: 4 2024-09-21T09:31:08.951 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:08.951 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 0, 2024-09-21T09:31:08.951 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:08.951 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:08.951 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:08.951 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:08.951 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:08.951 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:08.951 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.141121+0000", 2024-09-21T09:31:08.951 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.634065+0000", 2024-09-21T09:31:08.951 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.634065+0000", 2024-09-21T09:31:08.951 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.634785+0000", 2024-09-21T09:31:08.951 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.141121+0000", 2024-09-21T09:31:08.951 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:08.952 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:08.952 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:08.952 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:08.952 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.141121+0000", 2024-09-21T09:31:08.952 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.141121+0000", 2024-09-21T09:31:08.952 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.141121+0000", 2024-09-21T09:31:08.952 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:08.952 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:08.952 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:08.952 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.141121+0000", 2024-09-21T09:31:08.952 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.141121+0000", 2024-09-21T09:31:08.952 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:08.952 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:08.952 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:08.953 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:08.953 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:08.953 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:08.953 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:08.953 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:08.953 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:08.953 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:08.953 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:08.953 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:08.953 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:08.953 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.23", 2024-09-21T09:31:08.953 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:08.953 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:08.953 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:08.954 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:08.954 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:08.954 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T19:54:21.482132+0000", 2024-09-21T09:31:08.954 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:08.954 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:08.954 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:08.954 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:08.954 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:08.954 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:08.954 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:08.954 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:08.954 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:08.954 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:08.954 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:08.955 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:08.955 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:08.955 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:08.955 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:08.955 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:08.955 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:08.955 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:08.955 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:08.955 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:08.955 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:08.955 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:08.955 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:08.955 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:08.955 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:08.955 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:08.956 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:08.956 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:08.956 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:08.956 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:08.956 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:08.956 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:08.956 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:08.956 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:08.956 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:08.956 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:08.956 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:08.956 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:08.956 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:08.956 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:08.957 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:08.957 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:08.957 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:08.957 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.957 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:08.957 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:08.957 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:08.957 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:08.957 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:08.957 INFO:tasks.workunit.client.0.smithi022.stdout: 4 2024-09-21T09:31:08.957 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:08.957 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 0, 2024-09-21T09:31:08.957 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:08.957 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.958 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:08.958 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:08.958 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:08.958 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:08.958 INFO:tasks.workunit.client.0.smithi022.stdout: 1 2024-09-21T09:31:08.958 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:08.958 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 5, 2024-09-21T09:31:08.958 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:08.958 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:08.958 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:08.958 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:08.958 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:08.958 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:08.958 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.066111+0000", 2024-09-21T09:31:08.958 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:52.048885+0000", 2024-09-21T09:31:08.959 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:52.048885+0000", 2024-09-21T09:31:08.959 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:52.049243+0000", 2024-09-21T09:31:08.959 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.066111+0000", 2024-09-21T09:31:08.959 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:08.959 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:08.959 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:08.959 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 59, 2024-09-21T09:31:08.959 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.066111+0000", 2024-09-21T09:31:08.959 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.066111+0000", 2024-09-21T09:31:08.959 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.066111+0000", 2024-09-21T09:31:08.959 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:08.959 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:08.959 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:08.959 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.066111+0000", 2024-09-21T09:31:08.959 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.066111+0000", 2024-09-21T09:31:08.959 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:08.960 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:08.960 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:08.960 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:08.960 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 58, 2024-09-21T09:31:08.960 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:08.960 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:08.960 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:08.960 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:08.960 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:08.960 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:08.960 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:08.960 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:08.960 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.22", 2024-09-21T09:31:08.960 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:08.960 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:08.961 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:08.961 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 31, 2024-09-21T09:31:08.961 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:08.961 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T10:44:22.262414+0000", 2024-09-21T09:31:08.961 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:08.961 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:08.961 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:08.961 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:08.961 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:08.961 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:08.961 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:08.961 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:08.961 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:08.962 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:08.962 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:08.962 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:08.962 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:08.962 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:08.962 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:08.962 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:08.962 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:08.962 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:08.962 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:08.962 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:08.962 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:08.962 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:08.963 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:08.963 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:08.963 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:08.963 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:08.963 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:08.963 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:08.963 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:08.963 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:08.963 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:08.963 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:08.963 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:08.963 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:08.963 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:08.963 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:08.963 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:08.963 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:08.964 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:08.964 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:08.964 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:08.964 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:08.964 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:08.964 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.964 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:08.964 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:08.964 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:08.964 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:08.964 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:08.964 INFO:tasks.workunit.client.0.smithi022.stdout: 1 2024-09-21T09:31:08.964 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:08.964 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 5, 2024-09-21T09:31:08.964 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:08.965 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.965 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:08.965 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:08.965 INFO:tasks.workunit.client.0.smithi022.stdout: 2, 2024-09-21T09:31:08.965 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:08.965 INFO:tasks.workunit.client.0.smithi022.stdout: 6 2024-09-21T09:31:08.965 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:08.965 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 2, 2024-09-21T09:31:08.965 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:08.965 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:08.965 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:08.965 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:08.965 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:08.965 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:08.965 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.061749+0000", 2024-09-21T09:31:08.965 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.635935+0000", 2024-09-21T09:31:08.966 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.635935+0000", 2024-09-21T09:31:08.966 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.636222+0000", 2024-09-21T09:31:08.966 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.061749+0000", 2024-09-21T09:31:08.966 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:08.966 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:08.966 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:08.966 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:08.966 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.061749+0000", 2024-09-21T09:31:08.966 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.061749+0000", 2024-09-21T09:31:08.966 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.061749+0000", 2024-09-21T09:31:08.966 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:08.966 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:08.966 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:08.966 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.061749+0000", 2024-09-21T09:31:08.966 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.061749+0000", 2024-09-21T09:31:08.967 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:08.967 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:08.967 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:08.967 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:08.967 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:08.967 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:08.967 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:08.967 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:08.967 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:08.967 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:08.967 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:08.967 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:08.967 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:08.967 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.21", 2024-09-21T09:31:08.967 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:08.967 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:08.968 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:08.968 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:08.968 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:08.968 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T14:05:50.402224+0000", 2024-09-21T09:31:08.968 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:08.968 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:08.968 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:08.968 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:08.968 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:08.968 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:08.968 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:08.968 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:08.968 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:08.968 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:08.968 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:08.969 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:08.969 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:08.969 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:08.969 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:08.969 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:08.969 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:08.969 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:08.969 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:08.969 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:08.969 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:08.969 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:08.969 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:08.969 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:08.969 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:08.970 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:08.970 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:08.970 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:08.970 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:08.970 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:08.970 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:08.970 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:08.970 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:08.970 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:08.970 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:08.970 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:08.970 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:08.971 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:08.971 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:08.971 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:08.971 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:08.971 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:08.971 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:08.971 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.971 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:08.971 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:08.971 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:08.971 INFO:tasks.workunit.client.0.smithi022.stdout: 2, 2024-09-21T09:31:08.971 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:08.971 INFO:tasks.workunit.client.0.smithi022.stdout: 6 2024-09-21T09:31:08.971 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:08.971 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 2, 2024-09-21T09:31:08.972 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:08.972 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.972 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:08.972 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:08.972 INFO:tasks.workunit.client.0.smithi022.stdout: 6, 2024-09-21T09:31:08.972 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:08.972 INFO:tasks.workunit.client.0.smithi022.stdout: 5 2024-09-21T09:31:08.972 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:08.972 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 6, 2024-09-21T09:31:08.972 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:08.972 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:08.972 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:08.972 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:08.972 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:08.973 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:08.973 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.483839+0000", 2024-09-21T09:31:08.973 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.641409+0000", 2024-09-21T09:31:08.973 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.641409+0000", 2024-09-21T09:31:08.973 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.641740+0000", 2024-09-21T09:31:08.973 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.483839+0000", 2024-09-21T09:31:08.973 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:08.973 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:08.973 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:08.973 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:08.973 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.483839+0000", 2024-09-21T09:31:08.973 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.483839+0000", 2024-09-21T09:31:08.973 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.483839+0000", 2024-09-21T09:31:08.973 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:08.974 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:08.974 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:08.974 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.483839+0000", 2024-09-21T09:31:08.974 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.483839+0000", 2024-09-21T09:31:08.974 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:08.974 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:08.974 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:08.974 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:08.974 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:08.974 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:08.974 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:08.974 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:08.975 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:08.975 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:08.975 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:08.975 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:08.975 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:08.975 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.20", 2024-09-21T09:31:08.975 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:08.975 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:08.975 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:08.975 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:08.975 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:08.975 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T18:59:30.487730+0000", 2024-09-21T09:31:08.975 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:08.975 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:08.976 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:08.976 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:08.976 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:08.976 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:08.976 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:08.976 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:08.976 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:08.976 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:08.976 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:08.976 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:08.976 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:08.976 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:08.976 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:08.977 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:08.977 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:08.977 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:08.977 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:08.977 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:08.977 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:08.977 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:08.977 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:08.977 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:08.977 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:08.977 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:08.977 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:08.977 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:08.977 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:08.978 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:08.978 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:08.978 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:08.978 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:08.978 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:08.978 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:08.978 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:08.978 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:08.978 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:08.978 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:08.978 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:08.978 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:08.978 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:08.978 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:08.979 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.979 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:08.979 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:08.979 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:08.979 INFO:tasks.workunit.client.0.smithi022.stdout: 6, 2024-09-21T09:31:08.979 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:08.979 INFO:tasks.workunit.client.0.smithi022.stdout: 5 2024-09-21T09:31:08.979 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:08.979 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 6, 2024-09-21T09:31:08.979 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:08.979 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.979 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:08.979 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:08.979 INFO:tasks.workunit.client.0.smithi022.stdout: 4, 2024-09-21T09:31:08.979 INFO:tasks.workunit.client.0.smithi022.stdout: 1, 2024-09-21T09:31:08.980 INFO:tasks.workunit.client.0.smithi022.stdout: 2 2024-09-21T09:31:08.980 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:08.980 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 4, 2024-09-21T09:31:08.980 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:08.980 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:08.980 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:08.980 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:08.980 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:08.980 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:08.980 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.062273+0000", 2024-09-21T09:31:08.980 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.629167+0000", 2024-09-21T09:31:08.980 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.629167+0000", 2024-09-21T09:31:08.980 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.629525+0000", 2024-09-21T09:31:08.980 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.062273+0000", 2024-09-21T09:31:08.980 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:08.980 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:08.981 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:08.981 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:08.981 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.062273+0000", 2024-09-21T09:31:08.981 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.062273+0000", 2024-09-21T09:31:08.981 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.062273+0000", 2024-09-21T09:31:08.981 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:08.981 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:08.981 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:08.981 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.062273+0000", 2024-09-21T09:31:08.981 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.062273+0000", 2024-09-21T09:31:08.981 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:08.981 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:08.981 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:08.981 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:08.981 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:08.981 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:08.982 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:08.982 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:08.982 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:08.982 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:08.982 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:08.982 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:08.982 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:08.982 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.1a", 2024-09-21T09:31:08.982 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:08.982 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:08.982 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:08.982 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:08.982 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:08.982 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T21:06:24.323825+0000", 2024-09-21T09:31:08.983 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:08.983 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:08.983 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:08.983 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:08.983 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:08.983 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:08.983 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:08.983 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:08.983 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:08.983 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:08.983 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:08.983 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:08.983 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:08.983 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:08.983 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:08.984 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:08.984 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:08.984 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:08.984 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:08.984 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:08.984 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:08.984 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:08.984 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:08.984 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:08.984 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:08.984 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:08.984 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:08.984 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:08.984 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:08.984 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:08.985 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:08.985 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:08.985 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:08.985 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:08.985 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:08.985 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:08.985 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:08.985 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:08.985 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:08.985 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:08.985 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:08.985 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:08.985 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:08.985 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.985 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:08.985 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:08.986 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:08.986 INFO:tasks.workunit.client.0.smithi022.stdout: 4, 2024-09-21T09:31:08.986 INFO:tasks.workunit.client.0.smithi022.stdout: 1, 2024-09-21T09:31:08.986 INFO:tasks.workunit.client.0.smithi022.stdout: 2 2024-09-21T09:31:08.986 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:08.986 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 4, 2024-09-21T09:31:08.986 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:08.986 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.986 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:08.986 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:08.986 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:08.986 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:08.986 INFO:tasks.workunit.client.0.smithi022.stdout: 6 2024-09-21T09:31:08.986 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:08.987 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 3, 2024-09-21T09:31:08.987 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:08.987 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:08.987 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 51, 2024-09-21T09:31:08.987 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:08.987 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:08.987 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:08.987 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.125391+0000", 2024-09-21T09:31:08.987 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:32.264963+0000", 2024-09-21T09:31:08.987 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:32.264963+0000", 2024-09-21T09:31:08.987 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:32.265235+0000", 2024-09-21T09:31:08.987 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.125391+0000", 2024-09-21T09:31:08.987 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:08.987 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:08.988 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:08.988 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 52, 2024-09-21T09:31:08.988 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.125391+0000", 2024-09-21T09:31:08.988 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.125391+0000", 2024-09-21T09:31:08.988 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.125391+0000", 2024-09-21T09:31:08.988 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:08.988 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:08.988 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:08.988 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.125391+0000", 2024-09-21T09:31:08.988 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.125391+0000", 2024-09-21T09:31:08.988 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:08.988 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 1, 2024-09-21T09:31:08.988 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:08.988 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:08.989 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 51, 2024-09-21T09:31:08.989 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:08.989 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:08.989 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:08.989 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:08.989 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 1, 2024-09-21T09:31:08.989 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:08.989 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:08.989 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:08.989 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "2.1b", 2024-09-21T09:31:08.989 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:08.989 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:08.989 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:08.989 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 25, 2024-09-21T09:31:08.989 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:08.989 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T18:09:03.927341+0000", 2024-09-21T09:31:08.990 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:08.990 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:08.990 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:08.990 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:08.990 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:08.990 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:08.990 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:08.990 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:08.990 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:08.990 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:08.990 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:08.990 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:08.990 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:08.990 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:08.990 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:08.990 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:08.991 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:08.991 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:08.991 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:08.991 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:08.991 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:08.991 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:08.991 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:08.991 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:08.991 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:08.991 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:08.991 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:08.991 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:08.991 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:08.991 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:08.991 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:08.991 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:08.992 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:08.992 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:08.992 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:08.992 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:08.992 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:08.992 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:08.992 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:08.992 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:08.992 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:08.992 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:08.992 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:08.992 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.992 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:08.992 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:08.992 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:08.993 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:08.993 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:08.993 INFO:tasks.workunit.client.0.smithi022.stdout: 6 2024-09-21T09:31:08.993 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:08.993 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 3, 2024-09-21T09:31:08.993 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "57'1" 2024-09-21T09:31:08.993 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.993 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:08.993 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:08.993 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:08.993 INFO:tasks.workunit.client.0.smithi022.stdout: 4, 2024-09-21T09:31:08.993 INFO:tasks.workunit.client.0.smithi022.stdout: 7 2024-09-21T09:31:08.993 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:08.993 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 0, 2024-09-21T09:31:08.993 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:08.994 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:08.994 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:08.994 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:08.994 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:08.994 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:08.994 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.141127+0000", 2024-09-21T09:31:08.994 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.641801+0000", 2024-09-21T09:31:08.994 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.641801+0000", 2024-09-21T09:31:08.994 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.642154+0000", 2024-09-21T09:31:08.994 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.141127+0000", 2024-09-21T09:31:08.994 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:08.994 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:08.994 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:08.994 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:08.995 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.141127+0000", 2024-09-21T09:31:08.995 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.141127+0000", 2024-09-21T09:31:08.995 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.141127+0000", 2024-09-21T09:31:08.995 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:08.995 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:08.995 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:08.995 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.141127+0000", 2024-09-21T09:31:08.995 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.141127+0000", 2024-09-21T09:31:08.995 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:08.995 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:08.995 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:08.995 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:08.995 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:08.995 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:08.995 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:08.995 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:08.996 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:08.996 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:08.996 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:08.996 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:08.996 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:08.996 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.1b", 2024-09-21T09:31:08.996 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:08.996 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:08.996 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:08.996 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:08.996 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:08.996 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T10:50:45.075506+0000", 2024-09-21T09:31:08.996 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:08.996 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:08.996 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:08.996 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:08.997 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:08.997 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:08.997 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:08.997 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:08.997 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:08.997 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:08.997 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:08.997 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:08.997 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:08.997 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:08.997 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:08.997 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:08.997 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:08.997 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:08.997 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:08.998 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:08.998 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:08.998 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:08.998 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:08.998 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:08.998 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:08.998 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:08.998 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:08.998 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:08.998 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:08.998 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:08.998 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:08.998 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:08.999 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:08.999 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:08.999 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:08.999 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:08.999 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:08.999 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:08.999 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:08.999 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:08.999 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:08.999 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:08.999 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:08.999 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:08.999 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:08.999 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:08.999 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:09.000 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:09.000 INFO:tasks.workunit.client.0.smithi022.stdout: 4, 2024-09-21T09:31:09.000 INFO:tasks.workunit.client.0.smithi022.stdout: 7 2024-09-21T09:31:09.000 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.000 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 0, 2024-09-21T09:31:09.000 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:09.000 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.000 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:09.000 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:09.000 INFO:tasks.workunit.client.0.smithi022.stdout: 6, 2024-09-21T09:31:09.000 INFO:tasks.workunit.client.0.smithi022.stdout: 4, 2024-09-21T09:31:09.000 INFO:tasks.workunit.client.0.smithi022.stdout: 7 2024-09-21T09:31:09.000 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.000 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 6, 2024-09-21T09:31:09.000 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:09.001 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:09.001 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 51, 2024-09-21T09:31:09.001 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:09.001 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:09.001 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:09.001 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.484044+0000", 2024-09-21T09:31:09.001 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:32.269026+0000", 2024-09-21T09:31:09.001 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:32.269026+0000", 2024-09-21T09:31:09.001 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:32.269358+0000", 2024-09-21T09:31:09.001 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.484044+0000", 2024-09-21T09:31:09.001 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:09.001 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:09.001 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:09.001 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 52, 2024-09-21T09:31:09.001 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.484044+0000", 2024-09-21T09:31:09.001 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.484044+0000", 2024-09-21T09:31:09.002 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.484044+0000", 2024-09-21T09:31:09.002 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:09.002 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:09.002 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:09.002 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.484044+0000", 2024-09-21T09:31:09.002 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.484044+0000", 2024-09-21T09:31:09.002 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:09.002 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 1, 2024-09-21T09:31:09.002 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:09.002 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:09.002 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 51, 2024-09-21T09:31:09.002 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:09.002 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:09.002 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:09.003 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:09.003 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 1, 2024-09-21T09:31:09.003 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:09.003 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:09.003 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:09.003 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "2.1a", 2024-09-21T09:31:09.003 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:09.003 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:09.003 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:09.003 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 25, 2024-09-21T09:31:09.003 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:09.003 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T09:59:17.916370+0000", 2024-09-21T09:31:09.003 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:09.003 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:09.003 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:09.003 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:09.004 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:09.004 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:09.004 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:09.004 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:09.004 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:09.004 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:09.004 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:09.004 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:09.004 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:09.004 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:09.004 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:09.004 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:09.004 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:09.004 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:09.004 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:09.004 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:09.005 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:09.005 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:09.005 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:09.005 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:09.005 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:09.005 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:09.005 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:09.005 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:09.005 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:09.005 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:09.005 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:09.005 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:09.005 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:09.005 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:09.006 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:09.006 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:09.006 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:09.006 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:09.006 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:09.006 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:09.006 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:09.006 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:09.006 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:09.006 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.006 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:09.006 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:09.006 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:09.006 INFO:tasks.workunit.client.0.smithi022.stdout: 6, 2024-09-21T09:31:09.007 INFO:tasks.workunit.client.0.smithi022.stdout: 4, 2024-09-21T09:31:09.007 INFO:tasks.workunit.client.0.smithi022.stdout: 7 2024-09-21T09:31:09.007 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.007 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 6, 2024-09-21T09:31:09.007 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "57'1" 2024-09-21T09:31:09.007 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.007 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:09.007 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:09.007 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:09.007 INFO:tasks.workunit.client.0.smithi022.stdout: 6, 2024-09-21T09:31:09.007 INFO:tasks.workunit.client.0.smithi022.stdout: 0 2024-09-21T09:31:09.007 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.007 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 3, 2024-09-21T09:31:09.007 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:09.007 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:09.007 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 51, 2024-09-21T09:31:09.008 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:09.008 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:09.008 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:09.008 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.125195+0000", 2024-09-21T09:31:09.008 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:32.264381+0000", 2024-09-21T09:31:09.008 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:32.264381+0000", 2024-09-21T09:31:09.008 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:32.264601+0000", 2024-09-21T09:31:09.008 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.125195+0000", 2024-09-21T09:31:09.008 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:09.008 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:09.008 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:09.008 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 52, 2024-09-21T09:31:09.008 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.125195+0000", 2024-09-21T09:31:09.008 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.125195+0000", 2024-09-21T09:31:09.008 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.125195+0000", 2024-09-21T09:31:09.008 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:09.008 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:09.009 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:09.009 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.125195+0000", 2024-09-21T09:31:09.009 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.125195+0000", 2024-09-21T09:31:09.009 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:09.009 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:09.009 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:09.009 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:09.009 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 51, 2024-09-21T09:31:09.009 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:09.009 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:09.009 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:09.009 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:09.009 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:09.009 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:09.009 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:09.010 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:09.010 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "2.19", 2024-09-21T09:31:09.010 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:09.010 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:09.010 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:09.010 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 24, 2024-09-21T09:31:09.010 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:09.010 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T19:35:57.463156+0000", 2024-09-21T09:31:09.010 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:09.010 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:09.010 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:09.010 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:09.010 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:09.010 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:09.010 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:09.010 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:09.011 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:09.011 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:09.011 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:09.011 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:09.011 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:09.011 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:09.011 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:09.011 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:09.011 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:09.011 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:09.011 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:09.011 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:09.011 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:09.011 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:09.011 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:09.011 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:09.012 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:09.012 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:09.012 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:09.012 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:09.012 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:09.012 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:09.012 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:09.012 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:09.012 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:09.012 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:09.012 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:09.012 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:09.012 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:09.013 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:09.013 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:09.013 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:09.013 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:09.013 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:09.013 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:09.013 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.013 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:09.013 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:09.013 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:09.013 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:09.013 INFO:tasks.workunit.client.0.smithi022.stdout: 6, 2024-09-21T09:31:09.013 INFO:tasks.workunit.client.0.smithi022.stdout: 0 2024-09-21T09:31:09.013 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.014 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 3, 2024-09-21T09:31:09.014 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:09.014 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.014 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:09.014 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:09.014 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:09.014 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:09.014 INFO:tasks.workunit.client.0.smithi022.stdout: 1 2024-09-21T09:31:09.014 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.014 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 3, 2024-09-21T09:31:09.014 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:09.014 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:09.014 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:09.014 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:09.015 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:09.015 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:09.015 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.125067+0000", 2024-09-21T09:31:09.015 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.634224+0000", 2024-09-21T09:31:09.015 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.634224+0000", 2024-09-21T09:31:09.015 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.634475+0000", 2024-09-21T09:31:09.015 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.125067+0000", 2024-09-21T09:31:09.015 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:09.015 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:09.015 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:09.015 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:09.015 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.125067+0000", 2024-09-21T09:31:09.015 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.125067+0000", 2024-09-21T09:31:09.015 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.125067+0000", 2024-09-21T09:31:09.016 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:09.016 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:09.016 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:09.016 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.125067+0000", 2024-09-21T09:31:09.016 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.125067+0000", 2024-09-21T09:31:09.016 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:09.016 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:09.016 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:09.016 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:09.016 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:09.016 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:09.016 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:09.016 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:09.016 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:09.017 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:09.017 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:09.017 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:09.017 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:09.017 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.18", 2024-09-21T09:31:09.017 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:09.017 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:09.017 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:09.017 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:09.017 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:09.017 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T16:38:21.241590+0000", 2024-09-21T09:31:09.017 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:09.017 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:09.017 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:09.018 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:09.018 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:09.018 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:09.018 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:09.018 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:09.018 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:09.018 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:09.018 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:09.018 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:09.018 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:09.018 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:09.018 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:09.018 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:09.018 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:09.019 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:09.019 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:09.019 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:09.019 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:09.019 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:09.019 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:09.019 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:09.019 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:09.019 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:09.019 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:09.019 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:09.019 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:09.019 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:09.019 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:09.019 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:09.020 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:09.020 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:09.020 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:09.020 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:09.020 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:09.020 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:09.020 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:09.020 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:09.020 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:09.020 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:09.020 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:09.020 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.021 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:09.021 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:09.021 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:09.021 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:09.021 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:09.021 INFO:tasks.workunit.client.0.smithi022.stdout: 1 2024-09-21T09:31:09.022 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.022 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 3, 2024-09-21T09:31:09.022 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:09.022 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.022 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:09.022 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:09.022 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:09.022 INFO:tasks.workunit.client.0.smithi022.stdout: 4, 2024-09-21T09:31:09.022 INFO:tasks.workunit.client.0.smithi022.stdout: 7 2024-09-21T09:31:09.022 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.022 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 5, 2024-09-21T09:31:09.022 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:09.022 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:09.022 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 51, 2024-09-21T09:31:09.023 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:09.023 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:09.023 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:09.023 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.066263+0000", 2024-09-21T09:31:09.023 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:32.267484+0000", 2024-09-21T09:31:09.023 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:32.267484+0000", 2024-09-21T09:31:09.023 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:32.267805+0000", 2024-09-21T09:31:09.023 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.066263+0000", 2024-09-21T09:31:09.023 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:09.023 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:09.023 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:09.023 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 52, 2024-09-21T09:31:09.023 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.066263+0000", 2024-09-21T09:31:09.023 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.066263+0000", 2024-09-21T09:31:09.023 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.066263+0000", 2024-09-21T09:31:09.024 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:09.024 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:09.024 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:09.024 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.066263+0000", 2024-09-21T09:31:09.024 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.066263+0000", 2024-09-21T09:31:09.024 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:09.024 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:09.024 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:09.024 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:09.024 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 51, 2024-09-21T09:31:09.024 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:09.024 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:09.024 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:09.024 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:09.024 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:09.025 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:09.025 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:09.025 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:09.025 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "2.18", 2024-09-21T09:31:09.025 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:09.025 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:09.025 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:09.025 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 24, 2024-09-21T09:31:09.025 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:09.025 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T19:00:43.105362+0000", 2024-09-21T09:31:09.025 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:09.025 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:09.025 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:09.025 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:09.026 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:09.026 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:09.026 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:09.026 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:09.026 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:09.026 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:09.026 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:09.026 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:09.026 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:09.026 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:09.026 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:09.026 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:09.026 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:09.026 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:09.027 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:09.027 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:09.027 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:09.027 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:09.027 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:09.027 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:09.027 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:09.027 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:09.027 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:09.027 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:09.027 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:09.027 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:09.027 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:09.027 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:09.028 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:09.028 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:09.028 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:09.028 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:09.028 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:09.028 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:09.028 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:09.028 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:09.028 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:09.028 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:09.028 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:09.028 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.028 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:09.028 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:09.029 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:09.029 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:09.029 INFO:tasks.workunit.client.0.smithi022.stdout: 4, 2024-09-21T09:31:09.029 INFO:tasks.workunit.client.0.smithi022.stdout: 7 2024-09-21T09:31:09.029 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.029 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 5, 2024-09-21T09:31:09.029 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:09.029 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.029 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:09.029 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:09.029 INFO:tasks.workunit.client.0.smithi022.stdout: 1, 2024-09-21T09:31:09.029 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:09.029 INFO:tasks.workunit.client.0.smithi022.stdout: 4 2024-09-21T09:31:09.030 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.030 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 1, 2024-09-21T09:31:09.030 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:09.030 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:09.030 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:09.030 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:09.030 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:09.030 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:09.030 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.062486+0000", 2024-09-21T09:31:09.030 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.630723+0000", 2024-09-21T09:31:09.030 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.630723+0000", 2024-09-21T09:31:09.030 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.631350+0000", 2024-09-21T09:31:09.030 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.062486+0000", 2024-09-21T09:31:09.031 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:09.031 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:09.031 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:09.031 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:09.031 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.062486+0000", 2024-09-21T09:31:09.031 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.062486+0000", 2024-09-21T09:31:09.031 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.062486+0000", 2024-09-21T09:31:09.031 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:09.031 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:09.031 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:09.031 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.062486+0000", 2024-09-21T09:31:09.031 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.062486+0000", 2024-09-21T09:31:09.031 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:09.032 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:09.032 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:09.032 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:09.032 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:09.032 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:09.032 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:09.032 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:09.032 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:09.032 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:09.032 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:09.032 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:09.032 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:09.032 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.19", 2024-09-21T09:31:09.032 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:09.032 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:09.033 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:09.033 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:09.033 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:09.033 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T15:46:32.135037+0000", 2024-09-21T09:31:09.033 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:09.033 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:09.033 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:09.033 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:09.033 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:09.033 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:09.033 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:09.033 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:09.033 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:09.034 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:09.034 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:09.034 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:09.034 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:09.034 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:09.034 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:09.034 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:09.034 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:09.034 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:09.034 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:09.034 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:09.034 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:09.034 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:09.034 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:09.035 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:09.035 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:09.035 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:09.035 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:09.035 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:09.035 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:09.035 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:09.035 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:09.035 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:09.035 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:09.035 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:09.035 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:09.035 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:09.035 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:09.035 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:09.036 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:09.036 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:09.036 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:09.036 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:09.036 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:09.036 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.036 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:09.036 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:09.036 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:09.036 INFO:tasks.workunit.client.0.smithi022.stdout: 1, 2024-09-21T09:31:09.036 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:09.036 INFO:tasks.workunit.client.0.smithi022.stdout: 4 2024-09-21T09:31:09.036 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.036 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 1, 2024-09-21T09:31:09.037 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:09.037 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.037 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:09.037 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:09.037 INFO:tasks.workunit.client.0.smithi022.stdout: 6, 2024-09-21T09:31:09.037 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:09.037 INFO:tasks.workunit.client.0.smithi022.stdout: 2 2024-09-21T09:31:09.037 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.037 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 6, 2024-09-21T09:31:09.037 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:09.037 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:09.037 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 51, 2024-09-21T09:31:09.037 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:09.037 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:09.037 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:09.038 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.484390+0000", 2024-09-21T09:31:09.038 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:32.265995+0000", 2024-09-21T09:31:09.038 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:32.265995+0000", 2024-09-21T09:31:09.038 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:32.266307+0000", 2024-09-21T09:31:09.038 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.484390+0000", 2024-09-21T09:31:09.038 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:09.038 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:09.038 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:09.038 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 52, 2024-09-21T09:31:09.038 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.484390+0000", 2024-09-21T09:31:09.038 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.484390+0000", 2024-09-21T09:31:09.038 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.484390+0000", 2024-09-21T09:31:09.038 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:09.038 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:09.039 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:09.039 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.484390+0000", 2024-09-21T09:31:09.039 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.484390+0000", 2024-09-21T09:31:09.039 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:09.039 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:09.039 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:09.039 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:09.039 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 51, 2024-09-21T09:31:09.039 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:09.039 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:09.039 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:09.039 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:09.039 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:09.039 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:09.040 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:09.040 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:09.040 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "2.17", 2024-09-21T09:31:09.040 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:09.040 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:09.040 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:09.040 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 24, 2024-09-21T09:31:09.040 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:09.040 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T16:24:28.036534+0000", 2024-09-21T09:31:09.040 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:09.040 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:09.040 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:09.040 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:09.040 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:09.041 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:09.041 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:09.041 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:09.041 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:09.041 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:09.041 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:09.041 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:09.041 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:09.041 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:09.041 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:09.041 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:09.041 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:09.041 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:09.042 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:09.042 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:09.042 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:09.042 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:09.042 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:09.042 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:09.042 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:09.042 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:09.042 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:09.042 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:09.042 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:09.042 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:09.042 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:09.042 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:09.042 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:09.043 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:09.043 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:09.043 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:09.043 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:09.043 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:09.043 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:09.043 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:09.043 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:09.043 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:09.043 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:09.043 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.043 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:09.043 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:09.044 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:09.044 INFO:tasks.workunit.client.0.smithi022.stdout: 6, 2024-09-21T09:31:09.044 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:09.044 INFO:tasks.workunit.client.0.smithi022.stdout: 2 2024-09-21T09:31:09.044 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.044 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 6, 2024-09-21T09:31:09.044 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:09.044 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.044 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:09.044 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:09.044 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:09.044 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:09.044 INFO:tasks.workunit.client.0.smithi022.stdout: 1 2024-09-21T09:31:09.044 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.045 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 5, 2024-09-21T09:31:09.045 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:09.045 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:09.045 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:09.045 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:09.045 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:09.045 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:09.045 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.066174+0000", 2024-09-21T09:31:09.045 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.647649+0000", 2024-09-21T09:31:09.045 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.647649+0000", 2024-09-21T09:31:09.045 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.647952+0000", 2024-09-21T09:31:09.045 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.066174+0000", 2024-09-21T09:31:09.045 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:09.046 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:09.046 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:09.046 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:09.046 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.066174+0000", 2024-09-21T09:31:09.046 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.066174+0000", 2024-09-21T09:31:09.046 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.066174+0000", 2024-09-21T09:31:09.046 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:09.046 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:09.046 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:09.046 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.066174+0000", 2024-09-21T09:31:09.046 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.066174+0000", 2024-09-21T09:31:09.046 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:09.046 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:09.047 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:09.047 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:09.047 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:09.047 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:09.047 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:09.047 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:09.047 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:09.047 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:09.047 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:09.047 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:09.047 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:09.047 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.16", 2024-09-21T09:31:09.047 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:09.047 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:09.048 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:09.048 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:09.048 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:09.048 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T12:12:29.850835+0000", 2024-09-21T09:31:09.048 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:09.048 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:09.048 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:09.048 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:09.048 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:09.048 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:09.048 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:09.048 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:09.048 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:09.049 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:09.049 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:09.049 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:09.049 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:09.049 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:09.049 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:09.049 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:09.049 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:09.049 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:09.049 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:09.049 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:09.049 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:09.049 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:09.049 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:09.050 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:09.050 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:09.050 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:09.050 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:09.050 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:09.050 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:09.050 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:09.050 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:09.050 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:09.050 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:09.050 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:09.050 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:09.050 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:09.051 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:09.051 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:09.051 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:09.051 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:09.051 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:09.051 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:09.051 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:09.051 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.051 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:09.051 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:09.051 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:09.051 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:09.051 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:09.052 INFO:tasks.workunit.client.0.smithi022.stdout: 1 2024-09-21T09:31:09.052 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.052 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 5, 2024-09-21T09:31:09.052 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:09.052 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.052 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:09.052 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:09.052 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:09.052 INFO:tasks.workunit.client.0.smithi022.stdout: 6, 2024-09-21T09:31:09.052 INFO:tasks.workunit.client.0.smithi022.stdout: 2 2024-09-21T09:31:09.052 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.052 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 5, 2024-09-21T09:31:09.052 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:09.053 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:09.053 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 51, 2024-09-21T09:31:09.053 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:09.053 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:09.053 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:09.053 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.066325+0000", 2024-09-21T09:31:09.053 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:32.268832+0000", 2024-09-21T09:31:09.053 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:32.268832+0000", 2024-09-21T09:31:09.053 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:32.269137+0000", 2024-09-21T09:31:09.053 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.066325+0000", 2024-09-21T09:31:09.053 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:09.053 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:09.053 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:09.054 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 52, 2024-09-21T09:31:09.054 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.066325+0000", 2024-09-21T09:31:09.054 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.066325+0000", 2024-09-21T09:31:09.054 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.066325+0000", 2024-09-21T09:31:09.054 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:09.054 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:09.054 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:09.054 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.066325+0000", 2024-09-21T09:31:09.054 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.066325+0000", 2024-09-21T09:31:09.054 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:09.054 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:09.054 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:09.054 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:09.055 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 51, 2024-09-21T09:31:09.055 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:09.055 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:09.055 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:09.055 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:09.055 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:09.055 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:09.055 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:09.055 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:09.055 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "2.16", 2024-09-21T09:31:09.055 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:09.055 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:09.055 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:09.055 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 24, 2024-09-21T09:31:09.056 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:09.056 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T15:51:58.668390+0000", 2024-09-21T09:31:09.056 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:09.056 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:09.056 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:09.056 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:09.056 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:09.056 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:09.056 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:09.056 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:09.056 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:09.056 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:09.056 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:09.056 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:09.056 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:09.057 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:09.057 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:09.057 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:09.057 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:09.057 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:09.057 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:09.057 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:09.057 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:09.057 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:09.057 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:09.057 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:09.057 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:09.057 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:09.057 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:09.057 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:09.058 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:09.058 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:09.058 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:09.058 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:09.058 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:09.058 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:09.058 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:09.058 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:09.058 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:09.058 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:09.058 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:09.058 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:09.058 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:09.058 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:09.059 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:09.059 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.059 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:09.059 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:09.059 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:09.059 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:09.059 INFO:tasks.workunit.client.0.smithi022.stdout: 6, 2024-09-21T09:31:09.059 INFO:tasks.workunit.client.0.smithi022.stdout: 2 2024-09-21T09:31:09.059 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.059 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 5, 2024-09-21T09:31:09.059 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:09.059 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.059 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:09.059 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:09.060 INFO:tasks.workunit.client.0.smithi022.stdout: 2, 2024-09-21T09:31:09.060 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:09.060 INFO:tasks.workunit.client.0.smithi022.stdout: 3 2024-09-21T09:31:09.060 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.060 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 2, 2024-09-21T09:31:09.060 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:09.060 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:09.060 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:09.060 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:09.060 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:09.060 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:09.060 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.062462+0000", 2024-09-21T09:31:09.060 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:52.049549+0000", 2024-09-21T09:31:09.060 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:52.049549+0000", 2024-09-21T09:31:09.060 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:52.050121+0000", 2024-09-21T09:31:09.060 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.062462+0000", 2024-09-21T09:31:09.061 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:09.061 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:09.061 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:09.061 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 59, 2024-09-21T09:31:09.061 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.062462+0000", 2024-09-21T09:31:09.061 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.062462+0000", 2024-09-21T09:31:09.061 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.062462+0000", 2024-09-21T09:31:09.061 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:09.061 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:09.061 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:09.061 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.062462+0000", 2024-09-21T09:31:09.061 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.062462+0000", 2024-09-21T09:31:09.061 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:09.061 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:09.061 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:09.062 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:09.062 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 58, 2024-09-21T09:31:09.062 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:09.062 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:09.062 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:09.062 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:09.062 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:09.062 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:09.062 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:09.062 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:09.062 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.17", 2024-09-21T09:31:09.062 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:09.062 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:09.062 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:09.063 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 18, 2024-09-21T09:31:09.063 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:09.063 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T11:25:27.303500+0000", 2024-09-21T09:31:09.063 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:09.063 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:09.063 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:09.063 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:09.063 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:09.063 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:09.063 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:09.063 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:09.063 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:09.063 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:09.063 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:09.064 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:09.064 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:09.064 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:09.064 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:09.064 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:09.064 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:09.064 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:09.064 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:09.064 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:09.064 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:09.064 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:09.064 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:09.064 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:09.064 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:09.064 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:09.065 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:09.065 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:09.065 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:09.065 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:09.065 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:09.065 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:09.065 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:09.065 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:09.065 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:09.065 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:09.065 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:09.065 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:09.065 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:09.065 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:09.065 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:09.066 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:09.066 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:09.066 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.066 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:09.066 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:09.066 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:09.066 INFO:tasks.workunit.client.0.smithi022.stdout: 2, 2024-09-21T09:31:09.066 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:09.066 INFO:tasks.workunit.client.0.smithi022.stdout: 3 2024-09-21T09:31:09.066 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.066 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 2, 2024-09-21T09:31:09.066 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:09.066 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.067 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:09.067 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:09.067 INFO:tasks.workunit.client.0.smithi022.stdout: 1, 2024-09-21T09:31:09.067 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:09.067 INFO:tasks.workunit.client.0.smithi022.stdout: 0 2024-09-21T09:31:09.067 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.067 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 1, 2024-09-21T09:31:09.067 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:09.067 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:09.067 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 51, 2024-09-21T09:31:09.067 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:09.067 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:09.067 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:09.067 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.062137+0000", 2024-09-21T09:31:09.067 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:32.269808+0000", 2024-09-21T09:31:09.068 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:32.269808+0000", 2024-09-21T09:31:09.068 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:32.270145+0000", 2024-09-21T09:31:09.068 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.062137+0000", 2024-09-21T09:31:09.068 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:09.068 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:09.068 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:09.068 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 52, 2024-09-21T09:31:09.068 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.062137+0000", 2024-09-21T09:31:09.068 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.062137+0000", 2024-09-21T09:31:09.068 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.062137+0000", 2024-09-21T09:31:09.068 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:09.068 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:09.068 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:09.068 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.062137+0000", 2024-09-21T09:31:09.068 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.062137+0000", 2024-09-21T09:31:09.069 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:09.069 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 3, 2024-09-21T09:31:09.069 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:09.069 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:09.069 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 51, 2024-09-21T09:31:09.069 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:09.069 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:09.069 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:09.069 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:09.069 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 3, 2024-09-21T09:31:09.069 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:09.069 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:09.069 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:09.069 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "2.15", 2024-09-21T09:31:09.070 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:09.070 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:09.070 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:09.070 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 27, 2024-09-21T09:31:09.070 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:09.070 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T15:35:53.397343+0000", 2024-09-21T09:31:09.070 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:09.070 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:09.070 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:09.070 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 90, 2024-09-21T09:31:09.070 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:09.070 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:09.070 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:09.070 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:09.071 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:09.071 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:09.071 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:09.071 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:09.071 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:09.071 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:09.071 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:09.071 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:09.071 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:09.071 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:09.071 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:09.071 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 6, 2024-09-21T09:31:09.071 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 2, 2024-09-21T09:31:09.071 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:09.072 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 2, 2024-09-21T09:31:09.072 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:09.072 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:09.072 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:09.072 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:09.072 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:09.072 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 1, 2024-09-21T09:31:09.072 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:09.072 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:09.072 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:09.072 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:09.072 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:09.072 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:09.072 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:09.072 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:09.073 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:09.073 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:09.073 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:09.073 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:09.073 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 4, 2024-09-21T09:31:09.073 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 1 2024-09-21T09:31:09.073 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.073 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:09.073 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:09.073 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:09.073 INFO:tasks.workunit.client.0.smithi022.stdout: 1, 2024-09-21T09:31:09.073 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:09.073 INFO:tasks.workunit.client.0.smithi022.stdout: 0 2024-09-21T09:31:09.073 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.074 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 1, 2024-09-21T09:31:09.074 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "57'3" 2024-09-21T09:31:09.074 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.074 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:09.074 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:09.074 INFO:tasks.workunit.client.0.smithi022.stdout: 4, 2024-09-21T09:31:09.074 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:09.074 INFO:tasks.workunit.client.0.smithi022.stdout: 6 2024-09-21T09:31:09.074 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.074 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 4, 2024-09-21T09:31:09.074 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:09.074 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:09.074 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:09.074 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:09.074 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:09.075 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:09.075 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.062595+0000", 2024-09-21T09:31:09.075 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.639976+0000", 2024-09-21T09:31:09.075 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.639976+0000", 2024-09-21T09:31:09.075 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.640314+0000", 2024-09-21T09:31:09.075 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.062595+0000", 2024-09-21T09:31:09.075 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:09.075 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:09.075 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:09.075 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:09.075 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.062595+0000", 2024-09-21T09:31:09.075 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.062595+0000", 2024-09-21T09:31:09.075 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.062595+0000", 2024-09-21T09:31:09.075 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:09.075 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:09.076 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:09.076 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.062595+0000", 2024-09-21T09:31:09.076 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.062595+0000", 2024-09-21T09:31:09.076 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:09.076 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:09.076 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:09.076 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:09.076 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:09.076 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:09.076 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:09.076 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:09.076 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:09.076 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:09.076 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:09.077 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:09.077 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:09.077 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.14", 2024-09-21T09:31:09.077 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:09.077 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:09.077 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:09.077 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:09.077 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:09.077 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T15:55:21.229874+0000", 2024-09-21T09:31:09.077 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:09.077 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:09.077 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:09.077 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:09.077 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:09.077 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:09.078 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:09.078 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:09.078 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:09.078 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:09.078 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:09.078 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:09.078 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:09.078 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:09.078 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:09.078 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:09.078 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:09.078 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:09.078 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:09.078 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:09.079 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:09.079 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:09.079 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:09.079 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:09.079 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:09.079 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:09.079 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:09.079 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:09.079 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:09.079 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:09.079 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:09.079 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:09.079 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:09.079 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:09.079 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:09.080 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:09.080 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:09.080 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:09.080 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:09.080 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:09.080 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:09.080 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:09.080 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:09.080 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.080 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:09.080 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:09.080 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:09.080 INFO:tasks.workunit.client.0.smithi022.stdout: 4, 2024-09-21T09:31:09.080 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:09.080 INFO:tasks.workunit.client.0.smithi022.stdout: 6 2024-09-21T09:31:09.081 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.081 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 4, 2024-09-21T09:31:09.081 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:09.081 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.081 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:09.081 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:09.081 INFO:tasks.workunit.client.0.smithi022.stdout: 6, 2024-09-21T09:31:09.081 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:09.081 INFO:tasks.workunit.client.0.smithi022.stdout: 5 2024-09-21T09:31:09.081 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.081 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 6, 2024-09-21T09:31:09.081 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:09.081 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:09.081 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 51, 2024-09-21T09:31:09.081 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:09.082 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:09.082 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:09.082 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:29:56.877634+0000", 2024-09-21T09:31:09.082 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:32.265712+0000", 2024-09-21T09:31:09.082 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:32.265712+0000", 2024-09-21T09:31:09.082 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:32.266499+0000", 2024-09-21T09:31:09.082 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:29:56.877634+0000", 2024-09-21T09:31:09.082 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:09.082 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:09.082 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:09.082 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 52, 2024-09-21T09:31:09.082 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:29:56.877634+0000", 2024-09-21T09:31:09.082 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:29:56.877634+0000", 2024-09-21T09:31:09.082 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:29:56.877634+0000", 2024-09-21T09:31:09.083 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:09.083 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:09.083 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:09.083 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:29:56.877634+0000", 2024-09-21T09:31:09.083 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:29:56.877634+0000", 2024-09-21T09:31:09.083 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:09.083 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 12, 2024-09-21T09:31:09.083 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:09.083 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:09.083 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 51, 2024-09-21T09:31:09.083 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:09.083 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:09.083 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:09.083 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:09.083 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 12, 2024-09-21T09:31:09.084 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:09.084 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:09.084 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:09.084 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "2.14", 2024-09-21T09:31:09.084 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:09.084 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:09.084 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 67, 2024-09-21T09:31:09.084 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 36, 2024-09-21T09:31:09.084 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:09.084 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T14:19:17.014990+0000", 2024-09-21T09:31:09.084 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:09.084 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:09.084 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:09.084 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 2624, 2024-09-21T09:31:09.085 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:09.085 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:09.085 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:09.085 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:09.085 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:09.085 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:09.085 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:09.085 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:09.085 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:09.085 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:09.085 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:09.085 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:09.085 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:09.085 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:09.085 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:09.086 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 12, 2024-09-21T09:31:09.086 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 4, 2024-09-21T09:31:09.086 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:09.086 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 4, 2024-09-21T09:31:09.086 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:09.086 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:09.086 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:09.086 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:09.086 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:09.086 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 1, 2024-09-21T09:31:09.086 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:09.086 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:09.086 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:09.086 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:09.087 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:09.087 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:09.087 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:09.087 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:09.087 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:09.087 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:09.087 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:09.087 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:09.087 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 12, 2024-09-21T09:31:09.087 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 9 2024-09-21T09:31:09.087 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.087 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:09.087 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:09.087 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:09.088 INFO:tasks.workunit.client.0.smithi022.stdout: 6, 2024-09-21T09:31:09.088 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:09.088 INFO:tasks.workunit.client.0.smithi022.stdout: 5 2024-09-21T09:31:09.088 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.088 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 6, 2024-09-21T09:31:09.088 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "67'12" 2024-09-21T09:31:09.088 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.088 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:09.088 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:09.088 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:09.088 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:09.088 INFO:tasks.workunit.client.0.smithi022.stdout: 4 2024-09-21T09:31:09.088 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.088 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 7, 2024-09-21T09:31:09.089 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:09.089 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:09.089 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:09.089 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:09.089 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:09.089 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:09.089 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.359220+0000", 2024-09-21T09:31:09.089 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.641272+0000", 2024-09-21T09:31:09.089 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.641272+0000", 2024-09-21T09:31:09.089 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.641626+0000", 2024-09-21T09:31:09.089 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.359220+0000", 2024-09-21T09:31:09.089 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:09.089 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:09.089 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:09.089 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:09.090 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.359220+0000", 2024-09-21T09:31:09.090 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.359220+0000", 2024-09-21T09:31:09.090 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.359220+0000", 2024-09-21T09:31:09.090 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:09.090 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:09.090 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:09.090 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.359220+0000", 2024-09-21T09:31:09.090 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.359220+0000", 2024-09-21T09:31:09.090 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:09.090 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:09.090 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:09.090 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:09.090 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:09.090 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:09.090 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:09.091 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:09.091 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:09.091 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:09.091 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:09.091 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:09.091 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:09.091 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.15", 2024-09-21T09:31:09.091 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:09.091 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:09.091 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:09.091 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:09.091 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:09.091 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T17:15:05.108072+0000", 2024-09-21T09:31:09.091 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:09.091 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:09.092 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:09.092 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:09.092 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:09.092 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:09.092 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:09.092 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:09.092 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:09.092 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:09.092 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:09.092 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:09.092 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:09.092 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:09.092 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:09.092 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:09.093 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:09.093 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:09.093 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:09.093 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:09.093 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:09.093 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:09.093 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:09.093 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:09.093 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:09.093 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:09.093 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:09.093 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:09.093 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:09.093 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:09.093 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:09.094 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:09.094 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:09.094 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:09.094 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:09.094 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:09.094 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:09.094 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:09.094 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:09.094 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:09.094 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:09.094 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:09.094 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:09.094 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.095 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:09.095 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:09.095 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:09.095 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:09.095 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:09.095 INFO:tasks.workunit.client.0.smithi022.stdout: 4 2024-09-21T09:31:09.095 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.095 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 7, 2024-09-21T09:31:09.095 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:09.095 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.095 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:09.095 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:09.095 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:09.095 INFO:tasks.workunit.client.0.smithi022.stdout: 4, 2024-09-21T09:31:09.096 INFO:tasks.workunit.client.0.smithi022.stdout: 3 2024-09-21T09:31:09.096 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.096 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 0, 2024-09-21T09:31:09.096 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:09.096 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:09.096 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 51, 2024-09-21T09:31:09.096 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:09.096 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:09.096 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:09.096 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.141220+0000", 2024-09-21T09:31:09.096 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:32.267108+0000", 2024-09-21T09:31:09.096 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:32.267108+0000", 2024-09-21T09:31:09.096 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:32.267412+0000", 2024-09-21T09:31:09.096 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.141220+0000", 2024-09-21T09:31:09.096 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:09.097 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:09.097 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:09.097 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 52, 2024-09-21T09:31:09.097 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.141220+0000", 2024-09-21T09:31:09.097 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.141220+0000", 2024-09-21T09:31:09.097 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.141220+0000", 2024-09-21T09:31:09.097 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:09.097 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:09.097 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:09.097 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.141220+0000", 2024-09-21T09:31:09.097 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.141220+0000", 2024-09-21T09:31:09.097 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:09.097 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 2, 2024-09-21T09:31:09.097 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:09.098 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:09.098 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 51, 2024-09-21T09:31:09.098 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:09.098 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:09.098 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:09.098 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:09.098 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 2, 2024-09-21T09:31:09.098 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:09.098 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:09.098 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:09.098 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "2.13", 2024-09-21T09:31:09.098 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:09.098 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:09.098 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:09.098 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 26, 2024-09-21T09:31:09.099 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:09.099 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T17:54:11.228702+0000", 2024-09-21T09:31:09.099 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:09.099 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:09.099 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:09.099 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 34, 2024-09-21T09:31:09.099 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:09.099 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:09.099 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:09.099 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:09.099 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:09.099 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:09.099 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:09.099 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:09.100 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:09.100 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:09.100 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:09.100 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:09.100 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:09.100 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:09.100 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:09.100 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 6, 2024-09-21T09:31:09.100 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 2, 2024-09-21T09:31:09.100 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:09.100 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 2, 2024-09-21T09:31:09.100 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:09.100 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:09.100 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:09.101 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:09.101 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:09.101 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 1, 2024-09-21T09:31:09.101 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:09.101 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:09.101 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:09.101 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:09.101 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:09.101 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:09.101 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:09.101 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:09.101 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:09.101 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:09.101 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:09.102 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:09.102 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 3, 2024-09-21T09:31:09.102 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 6 2024-09-21T09:31:09.102 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.102 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:09.102 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:09.102 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:09.102 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:09.102 INFO:tasks.workunit.client.0.smithi022.stdout: 4, 2024-09-21T09:31:09.102 INFO:tasks.workunit.client.0.smithi022.stdout: 3 2024-09-21T09:31:09.102 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.102 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 0, 2024-09-21T09:31:09.103 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "57'2" 2024-09-21T09:31:09.103 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.103 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:09.103 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:09.103 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:09.103 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:09.103 INFO:tasks.workunit.client.0.smithi022.stdout: 3 2024-09-21T09:31:09.103 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.103 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 0, 2024-09-21T09:31:09.103 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:09.104 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:09.104 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:09.104 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:09.104 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:09.104 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:09.104 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.140829+0000", 2024-09-21T09:31:09.104 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.641185+0000", 2024-09-21T09:31:09.104 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.641185+0000", 2024-09-21T09:31:09.104 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.641572+0000", 2024-09-21T09:31:09.104 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.140829+0000", 2024-09-21T09:31:09.104 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:09.104 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:09.104 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:09.104 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:09.104 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.140829+0000", 2024-09-21T09:31:09.105 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.140829+0000", 2024-09-21T09:31:09.105 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.140829+0000", 2024-09-21T09:31:09.105 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:09.105 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:09.105 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:09.105 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.140829+0000", 2024-09-21T09:31:09.105 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.140829+0000", 2024-09-21T09:31:09.105 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:09.105 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:09.105 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:09.105 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:09.105 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:09.105 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:09.105 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:09.105 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:09.106 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:09.106 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:09.106 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:09.106 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:09.106 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:09.106 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.12", 2024-09-21T09:31:09.106 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:09.106 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:09.106 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:09.106 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:09.106 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:09.106 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T09:52:30.993491+0000", 2024-09-21T09:31:09.106 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:09.106 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:09.107 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:09.107 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:09.107 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:09.107 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:09.107 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:09.107 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:09.107 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:09.107 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:09.107 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:09.107 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:09.107 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:09.107 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:09.107 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:09.107 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:09.108 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:09.108 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:09.108 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:09.108 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:09.108 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:09.108 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:09.108 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:09.108 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:09.108 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:09.108 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:09.108 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:09.108 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:09.108 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:09.108 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:09.109 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:09.109 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:09.109 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:09.109 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:09.109 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:09.109 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:09.109 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:09.109 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:09.109 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:09.109 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:09.109 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:09.109 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:09.109 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:09.109 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.109 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:09.110 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:09.110 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:09.110 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:09.110 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:09.110 INFO:tasks.workunit.client.0.smithi022.stdout: 3 2024-09-21T09:31:09.110 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.110 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 0, 2024-09-21T09:31:09.110 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:09.110 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.110 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:09.110 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:09.110 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:09.110 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:09.111 INFO:tasks.workunit.client.0.smithi022.stdout: 7 2024-09-21T09:31:09.111 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.111 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 5, 2024-09-21T09:31:09.111 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:09.111 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:09.111 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 51, 2024-09-21T09:31:09.111 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:09.111 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:09.111 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:09.111 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.066197+0000", 2024-09-21T09:31:09.111 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:32.267232+0000", 2024-09-21T09:31:09.111 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:32.267232+0000", 2024-09-21T09:31:09.111 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:32.267644+0000", 2024-09-21T09:31:09.111 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.066197+0000", 2024-09-21T09:31:09.111 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:09.112 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:09.112 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:09.112 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 52, 2024-09-21T09:31:09.112 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.066197+0000", 2024-09-21T09:31:09.112 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.066197+0000", 2024-09-21T09:31:09.112 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.066197+0000", 2024-09-21T09:31:09.112 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:09.112 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:09.112 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:09.112 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.066197+0000", 2024-09-21T09:31:09.112 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.066197+0000", 2024-09-21T09:31:09.112 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:09.112 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:09.112 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:09.113 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:09.113 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 51, 2024-09-21T09:31:09.113 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:09.113 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:09.113 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:09.113 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:09.113 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:09.113 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:09.113 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:09.113 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:09.113 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "2.12", 2024-09-21T09:31:09.113 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:09.113 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:09.113 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:09.114 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 24, 2024-09-21T09:31:09.114 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:09.114 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T19:14:19.578564+0000", 2024-09-21T09:31:09.114 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:09.114 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:09.114 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:09.114 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:09.114 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:09.114 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:09.114 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:09.114 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:09.114 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:09.114 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:09.114 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:09.114 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:09.115 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:09.115 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:09.115 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:09.115 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:09.115 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:09.115 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:09.115 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:09.115 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:09.115 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:09.115 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:09.115 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:09.115 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:09.115 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:09.115 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:09.116 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:09.116 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:09.116 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:09.116 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:09.116 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:09.116 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:09.116 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:09.116 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:09.116 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:09.116 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:09.116 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:09.116 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:09.116 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:09.116 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:09.116 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:09.116 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:09.117 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:09.117 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.117 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:09.117 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:09.117 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:09.117 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:09.117 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:09.117 INFO:tasks.workunit.client.0.smithi022.stdout: 7 2024-09-21T09:31:09.117 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.117 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 5, 2024-09-21T09:31:09.117 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:09.117 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.117 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:09.117 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:09.118 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:09.118 INFO:tasks.workunit.client.0.smithi022.stdout: 4, 2024-09-21T09:31:09.118 INFO:tasks.workunit.client.0.smithi022.stdout: 2 2024-09-21T09:31:09.118 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.118 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 7, 2024-09-21T09:31:09.118 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:09.118 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:09.118 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:09.118 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:09.118 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:09.118 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:09.118 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.358995+0000", 2024-09-21T09:31:09.118 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.643681+0000", 2024-09-21T09:31:09.118 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.643681+0000", 2024-09-21T09:31:09.119 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.644217+0000", 2024-09-21T09:31:09.119 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.358995+0000", 2024-09-21T09:31:09.119 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:09.119 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:09.119 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:09.119 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:09.119 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.358995+0000", 2024-09-21T09:31:09.119 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.358995+0000", 2024-09-21T09:31:09.119 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.358995+0000", 2024-09-21T09:31:09.119 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:09.119 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:09.119 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:09.119 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.358995+0000", 2024-09-21T09:31:09.119 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.358995+0000", 2024-09-21T09:31:09.119 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:09.120 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:09.120 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:09.120 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:09.120 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:09.120 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:09.120 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:09.120 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:09.120 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:09.120 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:09.120 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:09.120 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:09.120 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:09.120 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.13", 2024-09-21T09:31:09.120 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:09.121 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:09.121 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:09.121 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:09.121 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:09.121 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T14:07:58.418951+0000", 2024-09-21T09:31:09.121 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:09.121 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:09.121 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:09.121 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:09.121 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:09.121 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:09.121 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:09.121 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:09.121 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:09.122 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:09.122 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:09.122 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:09.122 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:09.122 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:09.122 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:09.122 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:09.122 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:09.122 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:09.122 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:09.122 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:09.122 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:09.122 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:09.122 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:09.123 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:09.123 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:09.123 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:09.123 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:09.123 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:09.123 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:09.123 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:09.123 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:09.123 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:09.123 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:09.123 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:09.123 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:09.123 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:09.123 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:09.123 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:09.124 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:09.124 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:09.124 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:09.124 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:09.124 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:09.124 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.124 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:09.124 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:09.124 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:09.124 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:09.124 INFO:tasks.workunit.client.0.smithi022.stdout: 4, 2024-09-21T09:31:09.124 INFO:tasks.workunit.client.0.smithi022.stdout: 2 2024-09-21T09:31:09.124 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.125 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 7, 2024-09-21T09:31:09.125 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:09.125 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.125 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:09.125 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:09.125 INFO:tasks.workunit.client.0.smithi022.stdout: 6, 2024-09-21T09:31:09.125 INFO:tasks.workunit.client.0.smithi022.stdout: 4, 2024-09-21T09:31:09.125 INFO:tasks.workunit.client.0.smithi022.stdout: 1 2024-09-21T09:31:09.125 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.125 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 6, 2024-09-21T09:31:09.125 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:09.125 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:09.125 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 51, 2024-09-21T09:31:09.125 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:09.126 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:09.126 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:09.126 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.483959+0000", 2024-09-21T09:31:09.126 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:32.269808+0000", 2024-09-21T09:31:09.126 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:32.269808+0000", 2024-09-21T09:31:09.126 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:32.270501+0000", 2024-09-21T09:31:09.126 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.483959+0000", 2024-09-21T09:31:09.126 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:09.126 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:09.126 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:09.126 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 52, 2024-09-21T09:31:09.126 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.483959+0000", 2024-09-21T09:31:09.126 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.483959+0000", 2024-09-21T09:31:09.126 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.483959+0000", 2024-09-21T09:31:09.127 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:09.127 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:09.127 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:09.127 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.483959+0000", 2024-09-21T09:31:09.127 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.483959+0000", 2024-09-21T09:31:09.127 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:09.127 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:09.127 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:09.127 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:09.127 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 51, 2024-09-21T09:31:09.127 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:09.127 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:09.127 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:09.127 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:09.128 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:09.128 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:09.128 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:09.128 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:09.128 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "2.11", 2024-09-21T09:31:09.128 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:09.128 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:09.128 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:09.128 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 24, 2024-09-21T09:31:09.128 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:09.128 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T11:28:03.705631+0000", 2024-09-21T09:31:09.128 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:09.128 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:09.128 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:09.129 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:09.129 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:09.129 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:09.129 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:09.129 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:09.129 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:09.129 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:09.129 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:09.129 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:09.129 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:09.129 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:09.129 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:09.129 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:09.129 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:09.129 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:09.129 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:09.130 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:09.130 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:09.130 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:09.130 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:09.130 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:09.130 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:09.130 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:09.130 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:09.130 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:09.130 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:09.130 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:09.130 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:09.130 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:09.130 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:09.131 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:09.131 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:09.131 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:09.131 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:09.131 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:09.131 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:09.131 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:09.131 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:09.131 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:09.131 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:09.131 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.131 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:09.131 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:09.131 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:09.132 INFO:tasks.workunit.client.0.smithi022.stdout: 6, 2024-09-21T09:31:09.132 INFO:tasks.workunit.client.0.smithi022.stdout: 4, 2024-09-21T09:31:09.132 INFO:tasks.workunit.client.0.smithi022.stdout: 1 2024-09-21T09:31:09.132 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.132 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 6, 2024-09-21T09:31:09.132 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:09.132 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.132 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:09.132 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:09.132 INFO:tasks.workunit.client.0.smithi022.stdout: 6, 2024-09-21T09:31:09.132 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:09.132 INFO:tasks.workunit.client.0.smithi022.stdout: 5 2024-09-21T09:31:09.132 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.132 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 6, 2024-09-21T09:31:09.132 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:09.133 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:09.133 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:09.133 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:09.133 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:09.133 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:09.133 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.483855+0000", 2024-09-21T09:31:09.133 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.640996+0000", 2024-09-21T09:31:09.133 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.640996+0000", 2024-09-21T09:31:09.133 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.641508+0000", 2024-09-21T09:31:09.133 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.483855+0000", 2024-09-21T09:31:09.133 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:09.133 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:09.133 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:09.133 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:09.133 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.483855+0000", 2024-09-21T09:31:09.134 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.483855+0000", 2024-09-21T09:31:09.134 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.483855+0000", 2024-09-21T09:31:09.134 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:09.134 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:09.134 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:09.134 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.483855+0000", 2024-09-21T09:31:09.134 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.483855+0000", 2024-09-21T09:31:09.134 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:09.134 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:09.134 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:09.134 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:09.134 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:09.134 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:09.135 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:09.135 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:09.135 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:09.135 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:09.135 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:09.135 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:09.135 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:09.135 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.10", 2024-09-21T09:31:09.135 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:09.135 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:09.135 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:09.135 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:09.135 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:09.136 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T12:02:24.674295+0000", 2024-09-21T09:31:09.136 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:09.136 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:09.136 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:09.136 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:09.136 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:09.136 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:09.136 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:09.136 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:09.137 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:09.137 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:09.137 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:09.137 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:09.137 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:09.137 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:09.137 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:09.137 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:09.137 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:09.137 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:09.137 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:09.137 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:09.137 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:09.137 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:09.138 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:09.138 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:09.138 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:09.138 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:09.138 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:09.138 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:09.138 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:09.138 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:09.138 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:09.138 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:09.138 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:09.138 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:09.139 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:09.139 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:09.139 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:09.139 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:09.139 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:09.139 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:09.139 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:09.139 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:09.139 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:09.139 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.139 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:09.139 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:09.139 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:09.139 INFO:tasks.workunit.client.0.smithi022.stdout: 6, 2024-09-21T09:31:09.139 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:09.140 INFO:tasks.workunit.client.0.smithi022.stdout: 5 2024-09-21T09:31:09.140 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.140 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 6, 2024-09-21T09:31:09.140 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:09.140 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.140 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:09.140 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:09.140 INFO:tasks.workunit.client.0.smithi022.stdout: 2, 2024-09-21T09:31:09.140 INFO:tasks.workunit.client.0.smithi022.stdout: 1, 2024-09-21T09:31:09.140 INFO:tasks.workunit.client.0.smithi022.stdout: 0 2024-09-21T09:31:09.140 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.140 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 2, 2024-09-21T09:31:09.140 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:09.141 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:09.141 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 51, 2024-09-21T09:31:09.141 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:09.141 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:09.141 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:09.141 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.061786+0000", 2024-09-21T09:31:09.141 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:32.261324+0000", 2024-09-21T09:31:09.141 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:32.261324+0000", 2024-09-21T09:31:09.141 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:32.264620+0000", 2024-09-21T09:31:09.141 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.061786+0000", 2024-09-21T09:31:09.141 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:09.141 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:09.141 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:09.142 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 52, 2024-09-21T09:31:09.142 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.061786+0000", 2024-09-21T09:31:09.142 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.061786+0000", 2024-09-21T09:31:09.142 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.061786+0000", 2024-09-21T09:31:09.142 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:09.142 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:09.142 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:09.142 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.061786+0000", 2024-09-21T09:31:09.142 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.061786+0000", 2024-09-21T09:31:09.142 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:09.142 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 8, 2024-09-21T09:31:09.142 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:09.142 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:09.142 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 51, 2024-09-21T09:31:09.143 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:09.143 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:09.143 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:09.143 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:09.143 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 8, 2024-09-21T09:31:09.143 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:09.143 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:09.143 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:09.143 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "2.10", 2024-09-21T09:31:09.143 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:09.143 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:09.143 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:09.143 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 32, 2024-09-21T09:31:09.143 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:09.143 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T14:11:04.933873+0000", 2024-09-21T09:31:09.144 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:09.144 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:09.144 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:09.144 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:09.144 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:09.144 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:09.144 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:09.144 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:09.144 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:09.144 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:09.144 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:09.144 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:09.144 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:09.144 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:09.145 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:09.145 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:09.145 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:09.145 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:09.145 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:09.145 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 12, 2024-09-21T09:31:09.145 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 4, 2024-09-21T09:31:09.145 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:09.145 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 4, 2024-09-21T09:31:09.145 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:09.145 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:09.145 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:09.145 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:09.145 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:09.146 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 4, 2024-09-21T09:31:09.146 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:09.146 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:09.146 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:09.146 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:09.146 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:09.146 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:09.146 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:09.146 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:09.146 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:09.146 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:09.146 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:09.146 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:09.146 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 12, 2024-09-21T09:31:09.147 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:09.147 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.147 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:09.147 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:09.147 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:09.147 INFO:tasks.workunit.client.0.smithi022.stdout: 2, 2024-09-21T09:31:09.147 INFO:tasks.workunit.client.0.smithi022.stdout: 1, 2024-09-21T09:31:09.147 INFO:tasks.workunit.client.0.smithi022.stdout: 0 2024-09-21T09:31:09.147 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.147 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 2, 2024-09-21T09:31:09.147 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "57'8" 2024-09-21T09:31:09.147 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.147 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:09.147 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:09.148 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:09.148 INFO:tasks.workunit.client.0.smithi022.stdout: 4, 2024-09-21T09:31:09.148 INFO:tasks.workunit.client.0.smithi022.stdout: 6 2024-09-21T09:31:09.148 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.148 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 7, 2024-09-21T09:31:09.148 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:09.148 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:09.148 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:09.148 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:09.148 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:09.148 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:09.148 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.358860+0000", 2024-09-21T09:31:09.148 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.641261+0000", 2024-09-21T09:31:09.148 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.641261+0000", 2024-09-21T09:31:09.149 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.641576+0000", 2024-09-21T09:31:09.149 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.358860+0000", 2024-09-21T09:31:09.149 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:09.149 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:09.149 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:09.149 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:09.149 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.358860+0000", 2024-09-21T09:31:09.149 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.358860+0000", 2024-09-21T09:31:09.149 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.358860+0000", 2024-09-21T09:31:09.149 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:09.149 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:09.149 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:09.149 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.358860+0000", 2024-09-21T09:31:09.149 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.358860+0000", 2024-09-21T09:31:09.149 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:09.150 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:09.150 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:09.150 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:09.150 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:09.150 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:09.150 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:09.150 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:09.150 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:09.150 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:09.150 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:09.150 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:09.150 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:09.150 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.11", 2024-09-21T09:31:09.150 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:09.150 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:09.151 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:09.151 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:09.151 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:09.151 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T14:59:48.868788+0000", 2024-09-21T09:31:09.151 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:09.151 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:09.151 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:09.151 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:09.151 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:09.151 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:09.151 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:09.151 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:09.151 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:09.151 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:09.152 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:09.152 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:09.152 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:09.152 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:09.152 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:09.152 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:09.152 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:09.152 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:09.152 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:09.152 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:09.152 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:09.152 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:09.152 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:09.153 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:09.153 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:09.153 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:09.153 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:09.153 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:09.153 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:09.153 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:09.153 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:09.153 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:09.153 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:09.153 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:09.153 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:09.153 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:09.154 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:09.154 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:09.154 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:09.154 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:09.154 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:09.154 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:09.154 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:09.154 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.154 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:09.154 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:09.154 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:09.154 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:09.154 INFO:tasks.workunit.client.0.smithi022.stdout: 4, 2024-09-21T09:31:09.155 INFO:tasks.workunit.client.0.smithi022.stdout: 6 2024-09-21T09:31:09.155 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.155 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 7, 2024-09-21T09:31:09.155 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:09.155 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.155 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:09.155 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:09.155 INFO:tasks.workunit.client.0.smithi022.stdout: 4, 2024-09-21T09:31:09.155 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:09.155 INFO:tasks.workunit.client.0.smithi022.stdout: 7 2024-09-21T09:31:09.155 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.155 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 4, 2024-09-21T09:31:09.155 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:09.155 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:09.156 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 51, 2024-09-21T09:31:09.156 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:09.156 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:09.156 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:09.156 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.062869+0000", 2024-09-21T09:31:09.156 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:32.269748+0000", 2024-09-21T09:31:09.156 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:32.269748+0000", 2024-09-21T09:31:09.156 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:32.270055+0000", 2024-09-21T09:31:09.156 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.062869+0000", 2024-09-21T09:31:09.156 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:09.156 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:09.156 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:09.156 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 52, 2024-09-21T09:31:09.156 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.062869+0000", 2024-09-21T09:31:09.157 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.062869+0000", 2024-09-21T09:31:09.157 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.062869+0000", 2024-09-21T09:31:09.157 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:09.157 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:09.157 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:09.157 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.062869+0000", 2024-09-21T09:31:09.157 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.062869+0000", 2024-09-21T09:31:09.157 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:09.157 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 1, 2024-09-21T09:31:09.157 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:09.157 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:09.157 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 51, 2024-09-21T09:31:09.157 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:09.158 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:09.158 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:09.158 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:09.158 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 1, 2024-09-21T09:31:09.158 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:09.158 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:09.158 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:09.158 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "2.f", 2024-09-21T09:31:09.158 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:09.158 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:09.158 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:09.158 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 25, 2024-09-21T09:31:09.158 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:09.158 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T14:50:54.853319+0000", 2024-09-21T09:31:09.159 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:09.159 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:09.159 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:09.159 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 22, 2024-09-21T09:31:09.159 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:09.159 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:09.159 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:09.159 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:09.159 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:09.159 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:09.159 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:09.159 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:09.159 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:09.159 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:09.160 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:09.160 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:09.160 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:09.160 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:09.160 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:09.160 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 3, 2024-09-21T09:31:09.160 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 1, 2024-09-21T09:31:09.160 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:09.160 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 1, 2024-09-21T09:31:09.160 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:09.160 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:09.160 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:09.160 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:09.160 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:09.161 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:09.161 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:09.161 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:09.161 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:09.161 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:09.161 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:09.161 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:09.161 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:09.161 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:09.161 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:09.161 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:09.161 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:09.161 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:09.161 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 1, 2024-09-21T09:31:09.161 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 1 2024-09-21T09:31:09.162 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.162 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:09.162 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:09.162 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:09.162 INFO:tasks.workunit.client.0.smithi022.stdout: 4, 2024-09-21T09:31:09.162 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:09.162 INFO:tasks.workunit.client.0.smithi022.stdout: 7 2024-09-21T09:31:09.162 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.162 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 4, 2024-09-21T09:31:09.162 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "57'1" 2024-09-21T09:31:09.162 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.162 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:09.162 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:09.162 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:09.163 INFO:tasks.workunit.client.0.smithi022.stdout: 4, 2024-09-21T09:31:09.163 INFO:tasks.workunit.client.0.smithi022.stdout: 1 2024-09-21T09:31:09.163 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.163 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 7, 2024-09-21T09:31:09.163 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:09.163 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:09.163 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:09.163 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:09.163 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:09.163 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:09.163 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.359401+0000", 2024-09-21T09:31:09.163 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.641348+0000", 2024-09-21T09:31:09.163 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.641348+0000", 2024-09-21T09:31:09.163 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.641790+0000", 2024-09-21T09:31:09.164 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.359401+0000", 2024-09-21T09:31:09.164 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:09.164 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:09.164 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:09.164 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:09.164 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.359401+0000", 2024-09-21T09:31:09.164 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.359401+0000", 2024-09-21T09:31:09.164 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.359401+0000", 2024-09-21T09:31:09.164 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:09.164 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:09.164 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:09.164 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.359401+0000", 2024-09-21T09:31:09.164 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.359401+0000", 2024-09-21T09:31:09.164 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:09.165 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:09.165 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:09.165 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:09.165 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:09.165 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:09.165 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:09.165 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:09.165 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:09.165 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:09.165 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:09.165 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:09.165 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:09.165 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.e", 2024-09-21T09:31:09.165 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:09.166 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:09.166 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:09.166 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:09.166 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:09.166 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T18:25:10.635353+0000", 2024-09-21T09:31:09.166 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:09.166 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:09.166 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:09.166 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:09.166 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:09.166 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:09.166 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:09.166 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:09.166 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:09.167 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:09.167 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:09.167 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:09.167 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:09.167 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:09.167 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:09.167 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:09.167 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:09.167 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:09.167 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:09.167 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:09.167 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:09.167 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:09.168 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:09.168 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:09.168 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:09.168 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:09.168 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:09.168 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:09.168 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:09.168 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:09.168 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:09.168 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:09.169 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:09.169 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:09.169 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:09.169 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:09.169 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:09.169 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:09.169 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:09.169 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:09.169 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:09.169 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:09.169 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:09.169 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.169 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:09.170 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:09.170 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:09.170 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:09.170 INFO:tasks.workunit.client.0.smithi022.stdout: 4, 2024-09-21T09:31:09.170 INFO:tasks.workunit.client.0.smithi022.stdout: 1 2024-09-21T09:31:09.170 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.170 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 7, 2024-09-21T09:31:09.170 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:09.170 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.170 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:09.170 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:09.170 INFO:tasks.workunit.client.0.smithi022.stdout: 2, 2024-09-21T09:31:09.170 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:09.170 INFO:tasks.workunit.client.0.smithi022.stdout: 7 2024-09-21T09:31:09.171 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.171 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 2, 2024-09-21T09:31:09.171 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:09.171 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:09.171 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 51, 2024-09-21T09:31:09.171 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:09.171 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:09.171 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:09.171 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.062124+0000", 2024-09-21T09:31:09.171 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:32.263984+0000", 2024-09-21T09:31:09.171 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:32.263984+0000", 2024-09-21T09:31:09.171 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:32.264276+0000", 2024-09-21T09:31:09.171 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.062124+0000", 2024-09-21T09:31:09.171 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:09.172 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:09.172 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:09.172 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 52, 2024-09-21T09:31:09.172 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.062124+0000", 2024-09-21T09:31:09.172 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.062124+0000", 2024-09-21T09:31:09.172 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.062124+0000", 2024-09-21T09:31:09.172 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:09.172 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:09.172 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:09.172 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.062124+0000", 2024-09-21T09:31:09.172 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.062124+0000", 2024-09-21T09:31:09.172 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:09.172 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 2, 2024-09-21T09:31:09.173 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:09.173 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:09.173 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 51, 2024-09-21T09:31:09.173 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:09.173 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:09.173 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:09.173 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:09.173 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 2, 2024-09-21T09:31:09.173 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:09.173 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:09.173 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:09.173 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "2.e", 2024-09-21T09:31:09.173 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:09.173 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:09.174 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:09.174 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 26, 2024-09-21T09:31:09.174 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:09.174 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T17:18:50.045457+0000", 2024-09-21T09:31:09.174 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:09.174 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:09.174 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:09.174 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:09.174 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:09.174 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:09.174 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:09.174 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:09.174 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:09.174 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:09.175 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:09.175 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:09.175 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:09.175 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:09.175 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:09.175 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:09.175 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:09.175 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:09.175 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:09.175 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 3, 2024-09-21T09:31:09.175 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 1, 2024-09-21T09:31:09.175 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:09.175 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 1, 2024-09-21T09:31:09.176 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:09.176 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:09.176 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:09.176 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:09.176 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:09.176 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 1, 2024-09-21T09:31:09.176 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:09.176 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:09.176 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:09.176 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:09.176 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:09.176 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:09.177 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:09.177 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:09.177 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:09.177 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:09.177 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:09.177 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:09.177 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 3, 2024-09-21T09:31:09.177 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:09.178 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.178 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:09.178 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:09.178 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:09.178 INFO:tasks.workunit.client.0.smithi022.stdout: 2, 2024-09-21T09:31:09.178 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:09.178 INFO:tasks.workunit.client.0.smithi022.stdout: 7 2024-09-21T09:31:09.178 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.179 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 2, 2024-09-21T09:31:09.179 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "57'2" 2024-09-21T09:31:09.179 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.179 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:09.179 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:09.179 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:09.179 INFO:tasks.workunit.client.0.smithi022.stdout: 4, 2024-09-21T09:31:09.179 INFO:tasks.workunit.client.0.smithi022.stdout: 0 2024-09-21T09:31:09.179 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.179 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 7, 2024-09-21T09:31:09.179 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:09.179 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:09.179 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:09.179 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:09.180 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:09.180 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:09.180 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.359338+0000", 2024-09-21T09:31:09.180 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.649033+0000", 2024-09-21T09:31:09.180 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.649033+0000", 2024-09-21T09:31:09.180 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.649353+0000", 2024-09-21T09:31:09.180 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.359338+0000", 2024-09-21T09:31:09.180 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:09.180 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:09.180 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:09.180 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:09.180 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.359338+0000", 2024-09-21T09:31:09.180 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.359338+0000", 2024-09-21T09:31:09.180 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.359338+0000", 2024-09-21T09:31:09.181 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:09.181 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:09.181 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:09.181 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.359338+0000", 2024-09-21T09:31:09.181 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.359338+0000", 2024-09-21T09:31:09.181 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:09.181 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:09.181 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:09.181 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:09.181 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:09.181 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:09.181 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:09.181 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:09.181 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:09.181 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:09.182 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:09.182 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:09.182 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:09.182 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.f", 2024-09-21T09:31:09.182 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:09.182 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:09.182 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:09.182 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:09.182 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:09.182 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T15:13:12.611019+0000", 2024-09-21T09:31:09.182 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:09.182 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:09.182 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:09.183 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:09.183 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:09.183 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:09.183 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:09.183 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:09.183 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:09.183 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:09.183 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:09.183 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:09.183 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:09.183 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:09.183 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:09.183 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:09.183 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:09.184 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:09.184 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:09.184 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:09.184 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:09.184 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:09.184 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:09.184 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:09.184 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:09.184 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:09.184 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:09.184 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:09.184 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:09.184 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:09.184 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:09.184 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:09.185 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:09.185 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:09.185 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:09.185 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:09.185 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:09.185 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:09.185 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:09.185 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:09.185 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:09.185 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:09.185 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:09.185 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.185 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:09.185 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:09.185 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:09.186 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:09.186 INFO:tasks.workunit.client.0.smithi022.stdout: 4, 2024-09-21T09:31:09.186 INFO:tasks.workunit.client.0.smithi022.stdout: 0 2024-09-21T09:31:09.186 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.186 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 7, 2024-09-21T09:31:09.186 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:09.186 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.186 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:09.186 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:09.186 INFO:tasks.workunit.client.0.smithi022.stdout: 1, 2024-09-21T09:31:09.186 INFO:tasks.workunit.client.0.smithi022.stdout: 4, 2024-09-21T09:31:09.186 INFO:tasks.workunit.client.0.smithi022.stdout: 3 2024-09-21T09:31:09.186 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.186 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 1, 2024-09-21T09:31:09.187 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:09.187 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:09.187 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 51, 2024-09-21T09:31:09.187 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:09.187 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:09.187 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:09.187 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.062217+0000", 2024-09-21T09:31:09.187 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:32.259209+0000", 2024-09-21T09:31:09.187 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:32.259209+0000", 2024-09-21T09:31:09.187 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:32.260033+0000", 2024-09-21T09:31:09.187 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.062217+0000", 2024-09-21T09:31:09.187 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:09.187 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:09.187 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:09.188 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 52, 2024-09-21T09:31:09.188 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.062217+0000", 2024-09-21T09:31:09.188 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.062217+0000", 2024-09-21T09:31:09.188 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.062217+0000", 2024-09-21T09:31:09.188 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:09.188 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:09.188 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:09.188 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.062217+0000", 2024-09-21T09:31:09.188 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.062217+0000", 2024-09-21T09:31:09.188 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:09.188 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 1, 2024-09-21T09:31:09.188 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:09.188 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:09.188 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 51, 2024-09-21T09:31:09.189 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:09.189 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:09.189 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:09.189 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:09.189 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 1, 2024-09-21T09:31:09.189 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:09.189 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:09.189 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:09.189 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "2.d", 2024-09-21T09:31:09.189 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:09.189 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:09.189 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:09.189 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 25, 2024-09-21T09:31:09.189 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:09.189 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T21:16:09.835879+0000", 2024-09-21T09:31:09.190 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:09.190 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:09.190 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:09.190 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 70, 2024-09-21T09:31:09.190 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:09.190 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:09.190 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:09.190 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:09.190 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:09.190 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:09.190 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:09.190 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:09.190 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:09.190 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:09.191 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:09.191 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:09.191 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:09.191 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:09.191 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:09.191 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 3, 2024-09-21T09:31:09.191 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 1, 2024-09-21T09:31:09.191 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:09.191 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 1, 2024-09-21T09:31:09.191 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:09.191 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:09.191 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:09.191 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:09.191 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:09.192 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:09.192 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:09.192 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:09.192 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:09.192 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:09.192 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:09.192 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:09.192 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:09.192 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:09.192 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:09.192 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:09.192 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:09.192 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:09.192 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 1, 2024-09-21T09:31:09.192 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 1 2024-09-21T09:31:09.193 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.193 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:09.193 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:09.193 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:09.193 INFO:tasks.workunit.client.0.smithi022.stdout: 1, 2024-09-21T09:31:09.193 INFO:tasks.workunit.client.0.smithi022.stdout: 4, 2024-09-21T09:31:09.193 INFO:tasks.workunit.client.0.smithi022.stdout: 3 2024-09-21T09:31:09.193 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.193 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 1, 2024-09-21T09:31:09.193 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "57'1" 2024-09-21T09:31:09.193 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.193 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:09.193 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:09.193 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:09.194 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:09.194 INFO:tasks.workunit.client.0.smithi022.stdout: 6 2024-09-21T09:31:09.194 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.194 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 5, 2024-09-21T09:31:09.194 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:09.194 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:09.194 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:09.194 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:09.194 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:09.194 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:09.194 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.066397+0000", 2024-09-21T09:31:09.194 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.638802+0000", 2024-09-21T09:31:09.194 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.638802+0000", 2024-09-21T09:31:09.194 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.639076+0000", 2024-09-21T09:31:09.195 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.066397+0000", 2024-09-21T09:31:09.195 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:09.195 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:09.195 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:09.195 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:09.195 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.066397+0000", 2024-09-21T09:31:09.195 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.066397+0000", 2024-09-21T09:31:09.195 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.066397+0000", 2024-09-21T09:31:09.195 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:09.195 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:09.195 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:09.195 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.066397+0000", 2024-09-21T09:31:09.195 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.066397+0000", 2024-09-21T09:31:09.196 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:09.196 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:09.196 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:09.196 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:09.196 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:09.196 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:09.196 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:09.196 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:09.196 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:09.196 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:09.196 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:09.196 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:09.196 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:09.196 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.c", 2024-09-21T09:31:09.196 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:09.197 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:09.197 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:09.197 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:09.197 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:09.197 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T17:36:18.524320+0000", 2024-09-21T09:31:09.197 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:09.197 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:09.197 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:09.197 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:09.197 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:09.197 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:09.197 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:09.197 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:09.197 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:09.197 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:09.198 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:09.198 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:09.198 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:09.198 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:09.198 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:09.198 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:09.198 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:09.198 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:09.198 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:09.198 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:09.198 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:09.198 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:09.198 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:09.198 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:09.199 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:09.199 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:09.199 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:09.199 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:09.199 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:09.199 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:09.199 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:09.199 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:09.199 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:09.199 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:09.199 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:09.199 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:09.199 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:09.199 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:09.199 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:09.200 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:09.200 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:09.200 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:09.200 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:09.200 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.200 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:09.200 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:09.200 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:09.200 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:09.200 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:09.200 INFO:tasks.workunit.client.0.smithi022.stdout: 6 2024-09-21T09:31:09.200 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.200 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 5, 2024-09-21T09:31:09.200 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:09.201 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.201 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:09.201 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:09.201 INFO:tasks.workunit.client.0.smithi022.stdout: 2, 2024-09-21T09:31:09.201 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:09.201 INFO:tasks.workunit.client.0.smithi022.stdout: 0 2024-09-21T09:31:09.201 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.201 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 2, 2024-09-21T09:31:09.201 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:09.201 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:09.201 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 51, 2024-09-21T09:31:09.201 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:09.201 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:09.201 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:09.202 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.062156+0000", 2024-09-21T09:31:09.202 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:32.269717+0000", 2024-09-21T09:31:09.202 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:32.269717+0000", 2024-09-21T09:31:09.202 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:32.270060+0000", 2024-09-21T09:31:09.202 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.062156+0000", 2024-09-21T09:31:09.202 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:09.202 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:09.202 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:09.202 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 52, 2024-09-21T09:31:09.202 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.062156+0000", 2024-09-21T09:31:09.202 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.062156+0000", 2024-09-21T09:31:09.202 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.062156+0000", 2024-09-21T09:31:09.202 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:09.202 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:09.203 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:09.203 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.062156+0000", 2024-09-21T09:31:09.203 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.062156+0000", 2024-09-21T09:31:09.203 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:09.203 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:09.203 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:09.203 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:09.203 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 51, 2024-09-21T09:31:09.203 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:09.203 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:09.203 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:09.203 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:09.203 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:09.203 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:09.204 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:09.204 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:09.204 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "2.c", 2024-09-21T09:31:09.204 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:09.204 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:09.204 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:09.204 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 24, 2024-09-21T09:31:09.204 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:09.204 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T19:59:27.725031+0000", 2024-09-21T09:31:09.204 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:09.204 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:09.204 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:09.204 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:09.204 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:09.205 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:09.205 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:09.205 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:09.205 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:09.205 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:09.205 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:09.205 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:09.205 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:09.205 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:09.205 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:09.205 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:09.205 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:09.205 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:09.205 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:09.205 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:09.206 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:09.206 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:09.206 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:09.206 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:09.206 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:09.206 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:09.206 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:09.206 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:09.206 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:09.206 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:09.206 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:09.206 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:09.206 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:09.206 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:09.207 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:09.207 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:09.207 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:09.207 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:09.207 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:09.207 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:09.207 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:09.207 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:09.207 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:09.207 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.207 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:09.207 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:09.207 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:09.207 INFO:tasks.workunit.client.0.smithi022.stdout: 2, 2024-09-21T09:31:09.208 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:09.208 INFO:tasks.workunit.client.0.smithi022.stdout: 0 2024-09-21T09:31:09.208 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.208 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 2, 2024-09-21T09:31:09.208 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:09.208 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.208 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:09.208 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:09.208 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:09.208 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:09.208 INFO:tasks.workunit.client.0.smithi022.stdout: 6 2024-09-21T09:31:09.208 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.208 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 7, 2024-09-21T09:31:09.208 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:09.208 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:09.209 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:09.209 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:09.209 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:09.209 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:09.209 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.359314+0000", 2024-09-21T09:31:09.209 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.639746+0000", 2024-09-21T09:31:09.209 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.639746+0000", 2024-09-21T09:31:09.209 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.640059+0000", 2024-09-21T09:31:09.209 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.359314+0000", 2024-09-21T09:31:09.209 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:09.209 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:09.209 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:09.209 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:09.209 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.359314+0000", 2024-09-21T09:31:09.210 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.359314+0000", 2024-09-21T09:31:09.210 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.359314+0000", 2024-09-21T09:31:09.210 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:09.210 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:09.210 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:09.210 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.359314+0000", 2024-09-21T09:31:09.210 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.359314+0000", 2024-09-21T09:31:09.210 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:09.210 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:09.210 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:09.210 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:09.210 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:09.210 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:09.211 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:09.211 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:09.211 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:09.211 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:09.211 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:09.211 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:09.211 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:09.211 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.d", 2024-09-21T09:31:09.211 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:09.211 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:09.211 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:09.211 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:09.211 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:09.211 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T11:33:15.402410+0000", 2024-09-21T09:31:09.211 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:09.212 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:09.212 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:09.212 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:09.212 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:09.212 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:09.212 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:09.212 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:09.212 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:09.212 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:09.212 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:09.212 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:09.212 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:09.212 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:09.213 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:09.213 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:09.213 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:09.213 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:09.213 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:09.213 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:09.213 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:09.213 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:09.213 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:09.213 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:09.213 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:09.213 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:09.213 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:09.213 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:09.213 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:09.214 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:09.214 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:09.214 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:09.214 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:09.214 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:09.214 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:09.214 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:09.214 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:09.214 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:09.214 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:09.214 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:09.214 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:09.214 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:09.214 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:09.215 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.215 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:09.215 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:09.215 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:09.215 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:09.215 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:09.215 INFO:tasks.workunit.client.0.smithi022.stdout: 6 2024-09-21T09:31:09.215 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.215 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 7, 2024-09-21T09:31:09.215 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:09.215 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.215 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:09.215 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:09.215 INFO:tasks.workunit.client.0.smithi022.stdout: 6, 2024-09-21T09:31:09.215 INFO:tasks.workunit.client.0.smithi022.stdout: 4, 2024-09-21T09:31:09.216 INFO:tasks.workunit.client.0.smithi022.stdout: 7 2024-09-21T09:31:09.216 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.216 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 6, 2024-09-21T09:31:09.216 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:09.216 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:09.216 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 61, 2024-09-21T09:31:09.216 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:09.216 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:09.216 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:09.216 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:41.371073+0000", 2024-09-21T09:31:09.216 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:28:28.844168+0000", 2024-09-21T09:31:09.216 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:28:28.844168+0000", 2024-09-21T09:31:09.216 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:28:33.484488+0000", 2024-09-21T09:31:09.216 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:41.371073+0000", 2024-09-21T09:31:09.217 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:28:27.830318+0000", 2024-09-21T09:31:09.217 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:09.217 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:28:27.830318+0000", 2024-09-21T09:31:09.217 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 62, 2024-09-21T09:31:09.217 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:41.371073+0000", 2024-09-21T09:31:09.217 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:41.371073+0000", 2024-09-21T09:31:09.217 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:41.371073+0000", 2024-09-21T09:31:09.217 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:09.217 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:09.217 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:28:27.830318+0000", 2024-09-21T09:31:09.217 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:41.371073+0000", 2024-09-21T09:31:09.217 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:41.371073+0000", 2024-09-21T09:31:09.217 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:09.217 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:09.218 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:09.218 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:09.218 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 61, 2024-09-21T09:31:09.218 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:09.218 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:09.218 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:09.218 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:09.218 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:09.218 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:09.218 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:09.218 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:09.218 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "4.5", 2024-09-21T09:31:09.218 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:09.219 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:09.219 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 67, 2024-09-21T09:31:09.219 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 19, 2024-09-21T09:31:09.219 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:09.219 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T14:07:12.698450+0000", 2024-09-21T09:31:09.219 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.000304273, 2024-09-21T09:31:09.219 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:09.219 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:09.219 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:09.219 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:09.219 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:09.219 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:09.219 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:09.219 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:09.219 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:09.220 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:09.220 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:09.220 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:09.220 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:09.220 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:09.220 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:09.220 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:09.220 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:09.220 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:09.220 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:09.220 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:09.220 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:09.220 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:09.220 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:09.221 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:09.221 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:09.221 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:09.221 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:09.221 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:09.221 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:09.221 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:09.221 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:09.221 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:09.221 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:09.221 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:09.221 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:09.221 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:09.221 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:09.222 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:09.222 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:09.222 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:09.222 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:09.222 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:09.222 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.222 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:09.222 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:09.222 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:09.222 INFO:tasks.workunit.client.0.smithi022.stdout: 6, 2024-09-21T09:31:09.222 INFO:tasks.workunit.client.0.smithi022.stdout: 4, 2024-09-21T09:31:09.222 INFO:tasks.workunit.client.0.smithi022.stdout: 7 2024-09-21T09:31:09.222 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.223 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 6, 2024-09-21T09:31:09.223 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:09.223 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.223 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:09.223 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:09.223 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:09.223 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:09.223 INFO:tasks.workunit.client.0.smithi022.stdout: 6 2024-09-21T09:31:09.223 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.223 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 7, 2024-09-21T09:31:09.223 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:09.223 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:09.223 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 19, 2024-09-21T09:31:09.223 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:09.224 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:09.224 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:09.224 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.359151+0000", 2024-09-21T09:31:09.224 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:08.454770+0000", 2024-09-21T09:31:09.224 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:08.454770+0000", 2024-09-21T09:31:09.224 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:08.460023+0000", 2024-09-21T09:31:09.224 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.359151+0000", 2024-09-21T09:31:09.224 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:25:07.646659+0000", 2024-09-21T09:31:09.224 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:09.224 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:25:07.646659+0000", 2024-09-21T09:31:09.224 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 50, 2024-09-21T09:31:09.224 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.359151+0000", 2024-09-21T09:31:09.224 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.359151+0000", 2024-09-21T09:31:09.224 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.359151+0000", 2024-09-21T09:31:09.225 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:09.225 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:09.225 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:25:07.646659+0000", 2024-09-21T09:31:09.225 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.359151+0000", 2024-09-21T09:31:09.225 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.359151+0000", 2024-09-21T09:31:09.225 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:09.225 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 79, 2024-09-21T09:31:09.225 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:09.225 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:09.225 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 49, 2024-09-21T09:31:09.225 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:09.225 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:09.226 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:09.226 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:09.226 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 79, 2024-09-21T09:31:09.226 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:09.226 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:09.226 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:09.226 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "1.0", 2024-09-21T09:31:09.226 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:09.226 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:09.226 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:09.226 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 58, 2024-09-21T09:31:09.227 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:09.227 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T19:08:45.658466+0000", 2024-09-21T09:31:09.227 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:09.227 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:09.227 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:09.227 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 590368, 2024-09-21T09:31:09.227 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:09.227 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 590368, 2024-09-21T09:31:09.227 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:09.227 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:09.227 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:09.227 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:09.227 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:09.228 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:09.228 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:09.228 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:09.228 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:09.228 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:09.228 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:09.228 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:09.228 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:09.228 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 6, 2024-09-21T09:31:09.228 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 2, 2024-09-21T09:31:09.228 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:09.228 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 2, 2024-09-21T09:31:09.228 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:09.228 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:09.229 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:09.229 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:09.229 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:09.229 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:09.229 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:09.229 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 2, 2024-09-21T09:31:09.229 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:09.229 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:09.229 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:09.229 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:09.229 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:09.229 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 126, 2024-09-21T09:31:09.229 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 231, 2024-09-21T09:31:09.229 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:09.229 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:09.230 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:09.230 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 113, 2024-09-21T09:31:09.230 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 1628 2024-09-21T09:31:09.230 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.230 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:09.230 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:09.230 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:09.230 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:09.230 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:09.230 INFO:tasks.workunit.client.0.smithi022.stdout: 6 2024-09-21T09:31:09.230 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.230 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 7, 2024-09-21T09:31:09.230 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "62'79" 2024-09-21T09:31:09.230 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.231 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:09.231 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:09.231 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:09.231 INFO:tasks.workunit.client.0.smithi022.stdout: 2, 2024-09-21T09:31:09.231 INFO:tasks.workunit.client.0.smithi022.stdout: 4 2024-09-21T09:31:09.231 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.231 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 5, 2024-09-21T09:31:09.231 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:09.231 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:09.231 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 51, 2024-09-21T09:31:09.231 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:09.231 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:09.231 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:09.231 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.066105+0000", 2024-09-21T09:31:09.231 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:52.080680+0000", 2024-09-21T09:31:09.232 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:52.080680+0000", 2024-09-21T09:31:09.232 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:52.081088+0000", 2024-09-21T09:31:09.232 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.066105+0000", 2024-09-21T09:31:09.232 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:09.232 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:09.232 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:09.232 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 59, 2024-09-21T09:31:09.232 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.066105+0000", 2024-09-21T09:31:09.232 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.066105+0000", 2024-09-21T09:31:09.232 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.066105+0000", 2024-09-21T09:31:09.232 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:09.232 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:09.232 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:09.233 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.066105+0000", 2024-09-21T09:31:09.233 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.066105+0000", 2024-09-21T09:31:09.233 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:09.233 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:09.233 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:09.233 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:09.233 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 58, 2024-09-21T09:31:09.233 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:09.233 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:09.233 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:09.233 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:09.233 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:09.233 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:09.233 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:09.233 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:09.234 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "2.3", 2024-09-21T09:31:09.234 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:09.234 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:09.234 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:09.234 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 34, 2024-09-21T09:31:09.234 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:09.234 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T14:01:04.317396+0000", 2024-09-21T09:31:09.234 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:09.234 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:09.234 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:09.234 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:09.234 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:09.234 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:09.235 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:09.235 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:09.235 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:09.235 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:09.235 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:09.235 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:09.235 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:09.235 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:09.235 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:09.235 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:09.235 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:09.235 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:09.235 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:09.235 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:09.236 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:09.236 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:09.236 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:09.236 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:09.236 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:09.236 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:09.236 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:09.236 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:09.236 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:09.236 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:09.236 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:09.236 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:09.236 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:09.237 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:09.237 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:09.237 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:09.237 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:09.237 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:09.237 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:09.237 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:09.237 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:09.237 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:09.237 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:09.237 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.237 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:09.237 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:09.237 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:09.238 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:09.238 INFO:tasks.workunit.client.0.smithi022.stdout: 2, 2024-09-21T09:31:09.238 INFO:tasks.workunit.client.0.smithi022.stdout: 4 2024-09-21T09:31:09.238 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.238 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 5, 2024-09-21T09:31:09.238 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:09.238 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.238 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:09.238 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:09.238 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:09.238 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:09.238 INFO:tasks.workunit.client.0.smithi022.stdout: 6 2024-09-21T09:31:09.238 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.239 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 3, 2024-09-21T09:31:09.239 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:09.239 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:09.239 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:09.239 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:09.239 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:09.239 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:09.239 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.125141+0000", 2024-09-21T09:31:09.239 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.635939+0000", 2024-09-21T09:31:09.239 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.635939+0000", 2024-09-21T09:31:09.239 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.636200+0000", 2024-09-21T09:31:09.239 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.125141+0000", 2024-09-21T09:31:09.239 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:09.240 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:09.240 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:09.240 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:09.240 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.125141+0000", 2024-09-21T09:31:09.240 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.125141+0000", 2024-09-21T09:31:09.240 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.125141+0000", 2024-09-21T09:31:09.240 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:09.240 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:09.240 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:09.240 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.125141+0000", 2024-09-21T09:31:09.240 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.125141+0000", 2024-09-21T09:31:09.240 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:09.240 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:09.240 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:09.240 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:09.241 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:09.241 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:09.241 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:09.241 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:09.241 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:09.241 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:09.241 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:09.241 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:09.241 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:09.241 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.2", 2024-09-21T09:31:09.241 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:09.241 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:09.241 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:09.242 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:09.242 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:09.242 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T12:02:06.551561+0000", 2024-09-21T09:31:09.242 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:09.242 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:09.242 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:09.242 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:09.242 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:09.242 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:09.242 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:09.242 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:09.242 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:09.242 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:09.243 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:09.243 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:09.243 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:09.243 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:09.243 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:09.243 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:09.243 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:09.243 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:09.243 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:09.243 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:09.243 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:09.243 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:09.243 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:09.244 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:09.244 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:09.244 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:09.244 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:09.244 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:09.244 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:09.244 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:09.244 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:09.244 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:09.244 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:09.244 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:09.244 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:09.245 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:09.245 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:09.245 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:09.245 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:09.245 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:09.245 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:09.245 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:09.245 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:09.245 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.245 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:09.245 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:09.245 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:09.245 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:09.245 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:09.246 INFO:tasks.workunit.client.0.smithi022.stdout: 6 2024-09-21T09:31:09.246 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.246 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 3, 2024-09-21T09:31:09.246 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:09.246 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.246 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:09.246 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:09.246 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:09.246 INFO:tasks.workunit.client.0.smithi022.stdout: 1, 2024-09-21T09:31:09.246 INFO:tasks.workunit.client.0.smithi022.stdout: 2 2024-09-21T09:31:09.246 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.246 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 0, 2024-09-21T09:31:09.246 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:09.246 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:09.247 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 61, 2024-09-21T09:31:09.247 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:09.247 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:09.247 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:09.247 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:40.923121+0000", 2024-09-21T09:31:09.247 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:28:28.846894+0000", 2024-09-21T09:31:09.247 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:28:28.846894+0000", 2024-09-21T09:31:09.247 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:28:33.141556+0000", 2024-09-21T09:31:09.247 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:40.923121+0000", 2024-09-21T09:31:09.247 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:28:27.830318+0000", 2024-09-21T09:31:09.247 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:09.247 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:28:27.830318+0000", 2024-09-21T09:31:09.247 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 62, 2024-09-21T09:31:09.248 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:40.923121+0000", 2024-09-21T09:31:09.248 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:40.923121+0000", 2024-09-21T09:31:09.248 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:40.923121+0000", 2024-09-21T09:31:09.248 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:09.248 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:09.248 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:28:27.830318+0000", 2024-09-21T09:31:09.248 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:40.923121+0000", 2024-09-21T09:31:09.248 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:40.923121+0000", 2024-09-21T09:31:09.248 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:09.248 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:09.248 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:09.248 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:09.248 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 61, 2024-09-21T09:31:09.249 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:09.249 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:09.249 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:09.249 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:09.249 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:09.249 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:09.249 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:09.249 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:09.249 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "4.6", 2024-09-21T09:31:09.249 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:09.249 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:09.249 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 67, 2024-09-21T09:31:09.249 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 19, 2024-09-21T09:31:09.249 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:09.249 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T10:54:49.861215+0000", 2024-09-21T09:31:09.250 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.000276941, 2024-09-21T09:31:09.250 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:09.250 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:09.250 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:09.250 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:09.250 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:09.250 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:09.250 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:09.250 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:09.250 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:09.250 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:09.250 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:09.250 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:09.250 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:09.250 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:09.251 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:09.251 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:09.251 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:09.251 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:09.251 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:09.251 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:09.251 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:09.251 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:09.251 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:09.251 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:09.251 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:09.251 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:09.251 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:09.251 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:09.252 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:09.252 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:09.252 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:09.252 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:09.252 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:09.252 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:09.252 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:09.252 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:09.252 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:09.252 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:09.252 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:09.252 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:09.252 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:09.252 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:09.253 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.253 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:09.253 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:09.253 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:09.253 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:09.253 INFO:tasks.workunit.client.0.smithi022.stdout: 1, 2024-09-21T09:31:09.253 INFO:tasks.workunit.client.0.smithi022.stdout: 2 2024-09-21T09:31:09.253 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.253 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 0, 2024-09-21T09:31:09.253 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:09.253 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.253 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:09.253 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:09.253 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:09.254 INFO:tasks.workunit.client.0.smithi022.stdout: 1, 2024-09-21T09:31:09.254 INFO:tasks.workunit.client.0.smithi022.stdout: 0 2024-09-21T09:31:09.254 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.254 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 7, 2024-09-21T09:31:09.254 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:09.254 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:09.254 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 51, 2024-09-21T09:31:09.254 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:09.254 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:09.254 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:09.254 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.359086+0000", 2024-09-21T09:31:09.254 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:32.268026+0000", 2024-09-21T09:31:09.254 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:32.268026+0000", 2024-09-21T09:31:09.254 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:32.268500+0000", 2024-09-21T09:31:09.255 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.359086+0000", 2024-09-21T09:31:09.255 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:09.255 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:09.255 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:09.255 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 52, 2024-09-21T09:31:09.255 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.359086+0000", 2024-09-21T09:31:09.255 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.359086+0000", 2024-09-21T09:31:09.255 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.359086+0000", 2024-09-21T09:31:09.255 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:09.255 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:09.255 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:09.255 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.359086+0000", 2024-09-21T09:31:09.255 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.359086+0000", 2024-09-21T09:31:09.256 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:09.256 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:09.256 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:09.256 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:09.256 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 51, 2024-09-21T09:31:09.256 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:09.256 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:09.256 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:09.256 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:09.256 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:09.256 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:09.256 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:09.256 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:09.257 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "2.0", 2024-09-21T09:31:09.257 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:09.257 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:09.257 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:09.257 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 24, 2024-09-21T09:31:09.257 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:09.257 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T13:44:43.463552+0000", 2024-09-21T09:31:09.257 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:09.257 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:09.257 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:09.257 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:09.257 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:09.257 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:09.258 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:09.258 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:09.258 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:09.258 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:09.258 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:09.258 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:09.258 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:09.258 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:09.258 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:09.258 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:09.258 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:09.259 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:09.259 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:09.259 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:09.259 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:09.259 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:09.259 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:09.259 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:09.259 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:09.259 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:09.259 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:09.259 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:09.259 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:09.259 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:09.259 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:09.260 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:09.260 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:09.260 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:09.260 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:09.260 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:09.260 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:09.260 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:09.260 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:09.260 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:09.260 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:09.260 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:09.260 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:09.260 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.260 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:09.261 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:09.261 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:09.261 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:09.261 INFO:tasks.workunit.client.0.smithi022.stdout: 1, 2024-09-21T09:31:09.261 INFO:tasks.workunit.client.0.smithi022.stdout: 0 2024-09-21T09:31:09.261 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.261 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 7, 2024-09-21T09:31:09.261 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:09.261 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.261 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:09.261 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:09.261 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:09.261 INFO:tasks.workunit.client.0.smithi022.stdout: 4, 2024-09-21T09:31:09.262 INFO:tasks.workunit.client.0.smithi022.stdout: 3 2024-09-21T09:31:09.262 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.262 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 0, 2024-09-21T09:31:09.262 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:09.262 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:09.262 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:09.262 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:09.262 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:09.262 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:09.262 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.140711+0000", 2024-09-21T09:31:09.262 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.633746+0000", 2024-09-21T09:31:09.262 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.633746+0000", 2024-09-21T09:31:09.262 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.634025+0000", 2024-09-21T09:31:09.263 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.140711+0000", 2024-09-21T09:31:09.263 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:09.263 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:09.263 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:09.263 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:09.263 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.140711+0000", 2024-09-21T09:31:09.263 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.140711+0000", 2024-09-21T09:31:09.263 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.140711+0000", 2024-09-21T09:31:09.263 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:09.263 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:09.263 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:09.263 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.140711+0000", 2024-09-21T09:31:09.263 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.140711+0000", 2024-09-21T09:31:09.263 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:09.264 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:09.264 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:09.264 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:09.264 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:09.264 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:09.264 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:09.264 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:09.264 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:09.264 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:09.264 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:09.264 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:09.264 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:09.264 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.1", 2024-09-21T09:31:09.264 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:09.265 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:09.265 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:09.265 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:09.265 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:09.265 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T15:18:40.511292+0000", 2024-09-21T09:31:09.265 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:09.265 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:09.265 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:09.265 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:09.265 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:09.265 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:09.265 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:09.265 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:09.266 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:09.266 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:09.266 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:09.266 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:09.266 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:09.266 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:09.266 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:09.266 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:09.266 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:09.266 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:09.266 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:09.266 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:09.266 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:09.266 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:09.267 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:09.267 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:09.267 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:09.267 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:09.267 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:09.267 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:09.267 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:09.267 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:09.267 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:09.267 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:09.267 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:09.267 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:09.267 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:09.267 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:09.268 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:09.268 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:09.268 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:09.268 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:09.268 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:09.268 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:09.268 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:09.268 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.268 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:09.268 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:09.268 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:09.268 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:09.268 INFO:tasks.workunit.client.0.smithi022.stdout: 4, 2024-09-21T09:31:09.268 INFO:tasks.workunit.client.0.smithi022.stdout: 3 2024-09-21T09:31:09.268 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.269 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 0, 2024-09-21T09:31:09.269 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:09.269 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.269 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:09.269 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:09.269 INFO:tasks.workunit.client.0.smithi022.stdout: 1, 2024-09-21T09:31:09.269 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:09.269 INFO:tasks.workunit.client.0.smithi022.stdout: 0 2024-09-21T09:31:09.269 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.269 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 1, 2024-09-21T09:31:09.269 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:09.269 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:09.269 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 61, 2024-09-21T09:31:09.269 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:09.270 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:09.270 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:09.270 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:40.375561+0000", 2024-09-21T09:31:09.270 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:28:28.924106+0000", 2024-09-21T09:31:09.270 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:28:28.924106+0000", 2024-09-21T09:31:09.270 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:28:33.062917+0000", 2024-09-21T09:31:09.270 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:40.375561+0000", 2024-09-21T09:31:09.270 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:28:27.830318+0000", 2024-09-21T09:31:09.270 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:09.270 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:28:27.830318+0000", 2024-09-21T09:31:09.270 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 62, 2024-09-21T09:31:09.270 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:40.375561+0000", 2024-09-21T09:31:09.270 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:40.375561+0000", 2024-09-21T09:31:09.270 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:40.375561+0000", 2024-09-21T09:31:09.271 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:09.271 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:09.271 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:28:27.830318+0000", 2024-09-21T09:31:09.271 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:40.375561+0000", 2024-09-21T09:31:09.271 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:40.375561+0000", 2024-09-21T09:31:09.271 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:09.271 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:09.271 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:09.271 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:09.271 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 61, 2024-09-21T09:31:09.271 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:09.271 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:09.271 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:09.271 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:09.272 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:09.272 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:09.272 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:09.272 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:09.272 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "4.7", 2024-09-21T09:31:09.272 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:09.272 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:09.272 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 67, 2024-09-21T09:31:09.272 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 19, 2024-09-21T09:31:09.272 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:09.272 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T12:40:32.351197+0000", 2024-09-21T09:31:09.272 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.000580449, 2024-09-21T09:31:09.272 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:09.272 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:09.273 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:09.273 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:09.273 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:09.273 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:09.273 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:09.273 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:09.273 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:09.273 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:09.273 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:09.273 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:09.273 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:09.273 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:09.273 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:09.273 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:09.274 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:09.274 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:09.274 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:09.274 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:09.274 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:09.274 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:09.274 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:09.274 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:09.274 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:09.274 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:09.274 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:09.274 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:09.274 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:09.274 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:09.275 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:09.275 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:09.275 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:09.275 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:09.275 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:09.275 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:09.275 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:09.275 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:09.275 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:09.275 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:09.275 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:09.275 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:09.275 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.275 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:09.276 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:09.276 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:09.276 INFO:tasks.workunit.client.0.smithi022.stdout: 1, 2024-09-21T09:31:09.276 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:09.276 INFO:tasks.workunit.client.0.smithi022.stdout: 0 2024-09-21T09:31:09.276 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.276 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 1, 2024-09-21T09:31:09.276 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:09.276 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.276 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:09.276 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:09.276 INFO:tasks.workunit.client.0.smithi022.stdout: 2, 2024-09-21T09:31:09.277 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:09.277 INFO:tasks.workunit.client.0.smithi022.stdout: 0 2024-09-21T09:31:09.277 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.277 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 2, 2024-09-21T09:31:09.277 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:09.277 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:09.277 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 51, 2024-09-21T09:31:09.277 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:09.277 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:09.277 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:09.277 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.061847+0000", 2024-09-21T09:31:09.277 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:32.262373+0000", 2024-09-21T09:31:09.278 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:32.262373+0000", 2024-09-21T09:31:09.278 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:32.262645+0000", 2024-09-21T09:31:09.278 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.061847+0000", 2024-09-21T09:31:09.278 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:09.278 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:09.278 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:09.278 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 52, 2024-09-21T09:31:09.278 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.061847+0000", 2024-09-21T09:31:09.278 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.061847+0000", 2024-09-21T09:31:09.278 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.061847+0000", 2024-09-21T09:31:09.278 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:09.278 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:09.278 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:09.278 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.061847+0000", 2024-09-21T09:31:09.279 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.061847+0000", 2024-09-21T09:31:09.279 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:09.279 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:09.279 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:09.279 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:09.279 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 51, 2024-09-21T09:31:09.279 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:09.279 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:09.279 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:09.279 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:09.279 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:09.279 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:09.279 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:09.279 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:09.280 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "2.1", 2024-09-21T09:31:09.280 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:09.280 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:09.280 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:09.280 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 24, 2024-09-21T09:31:09.280 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:09.280 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T10:37:23.684561+0000", 2024-09-21T09:31:09.280 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:09.280 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:09.280 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:09.280 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:09.280 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:09.280 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:09.280 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:09.280 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:09.281 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:09.281 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:09.281 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:09.281 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:09.281 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:09.281 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:09.281 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:09.281 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:09.281 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:09.281 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:09.281 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:09.281 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:09.281 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:09.281 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:09.282 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:09.282 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:09.282 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:09.282 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:09.282 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:09.282 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:09.282 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:09.282 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:09.282 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:09.282 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:09.282 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:09.282 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:09.282 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:09.282 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:09.283 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:09.283 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:09.283 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:09.283 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:09.283 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:09.283 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:09.283 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:09.283 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.283 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:09.283 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:09.283 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:09.283 INFO:tasks.workunit.client.0.smithi022.stdout: 2, 2024-09-21T09:31:09.283 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:09.284 INFO:tasks.workunit.client.0.smithi022.stdout: 0 2024-09-21T09:31:09.284 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.284 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 2, 2024-09-21T09:31:09.284 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:09.284 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.284 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:09.284 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:09.284 INFO:tasks.workunit.client.0.smithi022.stdout: 1, 2024-09-21T09:31:09.284 INFO:tasks.workunit.client.0.smithi022.stdout: 2, 2024-09-21T09:31:09.284 INFO:tasks.workunit.client.0.smithi022.stdout: 6 2024-09-21T09:31:09.284 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.284 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 1, 2024-09-21T09:31:09.284 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:09.284 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:09.284 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:09.285 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:09.285 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:09.285 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:09.285 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.061597+0000", 2024-09-21T09:31:09.285 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.635805+0000", 2024-09-21T09:31:09.285 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.635805+0000", 2024-09-21T09:31:09.285 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.636105+0000", 2024-09-21T09:31:09.285 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.061597+0000", 2024-09-21T09:31:09.285 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:09.285 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:09.285 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:09.285 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:09.285 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.061597+0000", 2024-09-21T09:31:09.285 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.061597+0000", 2024-09-21T09:31:09.286 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.061597+0000", 2024-09-21T09:31:09.286 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:09.286 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:09.286 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:09.286 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.061597+0000", 2024-09-21T09:31:09.286 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.061597+0000", 2024-09-21T09:31:09.286 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:09.286 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:09.286 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:09.286 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:09.286 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:09.286 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:09.286 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:09.287 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:09.287 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:09.287 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:09.287 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:09.287 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:09.287 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:09.287 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.0", 2024-09-21T09:31:09.287 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:09.287 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:09.287 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:09.287 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:09.287 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:09.287 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T13:22:53.711130+0000", 2024-09-21T09:31:09.287 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:09.288 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:09.288 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:09.288 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:09.288 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:09.288 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:09.288 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:09.288 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:09.288 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:09.288 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:09.288 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:09.288 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:09.288 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:09.288 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:09.288 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:09.288 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:09.289 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:09.289 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:09.289 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:09.289 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:09.289 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:09.289 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:09.289 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:09.289 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:09.289 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:09.289 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:09.289 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:09.289 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:09.289 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:09.290 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:09.290 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:09.290 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:09.290 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:09.290 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:09.290 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:09.290 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:09.290 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:09.290 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:09.290 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:09.290 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:09.290 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:09.290 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:09.291 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:09.291 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.291 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:09.291 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:09.291 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:09.291 INFO:tasks.workunit.client.0.smithi022.stdout: 1, 2024-09-21T09:31:09.291 INFO:tasks.workunit.client.0.smithi022.stdout: 2, 2024-09-21T09:31:09.291 INFO:tasks.workunit.client.0.smithi022.stdout: 6 2024-09-21T09:31:09.291 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.291 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 1, 2024-09-21T09:31:09.291 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:09.291 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.291 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:09.291 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:09.292 INFO:tasks.workunit.client.0.smithi022.stdout: 1, 2024-09-21T09:31:09.292 INFO:tasks.workunit.client.0.smithi022.stdout: 2, 2024-09-21T09:31:09.292 INFO:tasks.workunit.client.0.smithi022.stdout: 3 2024-09-21T09:31:09.292 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.292 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 1, 2024-09-21T09:31:09.292 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:09.292 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:09.292 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 61, 2024-09-21T09:31:09.292 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:09.292 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:09.292 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:09.292 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:40.376066+0000", 2024-09-21T09:31:09.292 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:28:28.844698+0000", 2024-09-21T09:31:09.293 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:28:28.844698+0000", 2024-09-21T09:31:09.293 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:28:33.062726+0000", 2024-09-21T09:31:09.293 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:40.376066+0000", 2024-09-21T09:31:09.293 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:28:27.830318+0000", 2024-09-21T09:31:09.293 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:09.293 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:28:27.830318+0000", 2024-09-21T09:31:09.293 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 62, 2024-09-21T09:31:09.293 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:40.376066+0000", 2024-09-21T09:31:09.293 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:40.376066+0000", 2024-09-21T09:31:09.293 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:40.376066+0000", 2024-09-21T09:31:09.293 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:09.293 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:09.293 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:28:27.830318+0000", 2024-09-21T09:31:09.293 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:40.376066+0000", 2024-09-21T09:31:09.293 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:40.376066+0000", 2024-09-21T09:31:09.294 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:09.294 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:09.294 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:09.294 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:09.294 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 61, 2024-09-21T09:31:09.294 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:09.294 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:09.294 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:09.294 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:09.294 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:09.294 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:09.294 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:09.294 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:09.295 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "4.4", 2024-09-21T09:31:09.295 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:09.295 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:09.295 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 67, 2024-09-21T09:31:09.295 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 19, 2024-09-21T09:31:09.295 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:09.295 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T14:24:39.284345+0000", 2024-09-21T09:31:09.295 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.000725062, 2024-09-21T09:31:09.295 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:09.295 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:09.295 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:09.295 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:09.296 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:09.296 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:09.296 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:09.296 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:09.296 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:09.296 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:09.296 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:09.296 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:09.296 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:09.296 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:09.296 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:09.296 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:09.296 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:09.296 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:09.297 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:09.297 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:09.297 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:09.297 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:09.297 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:09.297 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:09.297 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:09.297 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:09.297 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:09.297 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:09.297 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:09.297 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:09.297 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:09.297 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:09.297 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:09.298 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:09.298 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:09.298 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:09.298 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:09.298 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:09.298 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:09.298 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:09.298 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:09.298 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:09.298 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.298 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:09.298 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:09.298 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:09.299 INFO:tasks.workunit.client.0.smithi022.stdout: 1, 2024-09-21T09:31:09.299 INFO:tasks.workunit.client.0.smithi022.stdout: 2, 2024-09-21T09:31:09.299 INFO:tasks.workunit.client.0.smithi022.stdout: 3 2024-09-21T09:31:09.299 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.299 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 1, 2024-09-21T09:31:09.299 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:09.299 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.299 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:09.299 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:09.299 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:09.299 INFO:tasks.workunit.client.0.smithi022.stdout: 1, 2024-09-21T09:31:09.299 INFO:tasks.workunit.client.0.smithi022.stdout: 6 2024-09-21T09:31:09.299 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.300 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 5, 2024-09-21T09:31:09.300 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:09.300 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:09.300 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 51, 2024-09-21T09:31:09.300 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:09.300 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:09.300 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:09.300 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.066308+0000", 2024-09-21T09:31:09.300 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:32.264740+0000", 2024-09-21T09:31:09.300 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:32.264740+0000", 2024-09-21T09:31:09.300 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:32.265210+0000", 2024-09-21T09:31:09.300 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.066308+0000", 2024-09-21T09:31:09.300 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:09.300 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:09.301 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:09.301 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 52, 2024-09-21T09:31:09.301 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.066308+0000", 2024-09-21T09:31:09.301 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.066308+0000", 2024-09-21T09:31:09.301 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.066308+0000", 2024-09-21T09:31:09.301 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:09.301 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:09.301 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:09.301 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.066308+0000", 2024-09-21T09:31:09.301 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.066308+0000", 2024-09-21T09:31:09.301 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:09.301 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:09.301 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:09.302 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:09.302 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 51, 2024-09-21T09:31:09.302 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:09.302 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:09.302 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:09.302 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:09.302 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:09.302 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:09.302 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:09.302 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:09.302 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "2.2", 2024-09-21T09:31:09.302 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:09.302 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:09.303 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:09.303 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 24, 2024-09-21T09:31:09.303 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:09.303 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T19:14:01.491316+0000", 2024-09-21T09:31:09.303 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:09.303 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:09.303 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:09.303 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:09.303 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:09.303 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:09.303 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:09.303 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:09.303 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:09.303 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:09.304 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:09.304 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:09.304 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:09.304 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:09.304 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:09.304 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:09.304 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:09.304 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:09.304 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:09.304 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:09.304 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:09.304 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:09.304 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:09.304 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:09.304 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:09.305 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:09.305 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:09.305 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:09.305 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:09.305 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:09.305 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:09.305 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:09.305 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:09.305 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:09.305 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:09.305 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:09.305 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:09.305 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:09.306 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:09.306 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:09.306 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:09.306 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:09.306 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:09.306 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.306 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:09.306 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:09.306 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:09.306 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:09.306 INFO:tasks.workunit.client.0.smithi022.stdout: 1, 2024-09-21T09:31:09.306 INFO:tasks.workunit.client.0.smithi022.stdout: 6 2024-09-21T09:31:09.306 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.307 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 5, 2024-09-21T09:31:09.307 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:09.307 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.307 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:09.307 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:09.307 INFO:tasks.workunit.client.0.smithi022.stdout: 4, 2024-09-21T09:31:09.307 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:09.307 INFO:tasks.workunit.client.0.smithi022.stdout: 6 2024-09-21T09:31:09.307 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.307 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 4, 2024-09-21T09:31:09.307 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:09.307 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:09.307 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:09.307 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:09.308 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:09.308 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:09.308 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.062810+0000", 2024-09-21T09:31:09.308 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.638269+0000", 2024-09-21T09:31:09.308 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.638269+0000", 2024-09-21T09:31:09.308 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.638582+0000", 2024-09-21T09:31:09.308 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.062810+0000", 2024-09-21T09:31:09.308 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:09.308 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:09.308 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:09.308 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:09.308 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.062810+0000", 2024-09-21T09:31:09.308 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.062810+0000", 2024-09-21T09:31:09.309 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.062810+0000", 2024-09-21T09:31:09.309 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:09.309 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:09.309 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:09.309 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.062810+0000", 2024-09-21T09:31:09.309 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.062810+0000", 2024-09-21T09:31:09.309 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:09.309 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:09.309 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:09.309 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:09.309 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:09.309 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:09.309 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:09.309 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:09.309 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:09.310 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:09.310 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:09.310 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:09.310 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:09.310 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.3", 2024-09-21T09:31:09.310 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:09.310 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:09.310 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:09.310 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:09.310 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:09.310 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T20:52:20.849743+0000", 2024-09-21T09:31:09.310 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:09.310 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:09.310 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:09.311 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:09.311 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:09.311 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:09.311 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:09.311 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:09.311 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:09.311 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:09.311 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:09.311 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:09.311 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:09.311 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:09.311 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:09.311 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:09.311 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:09.312 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:09.312 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:09.312 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:09.312 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:09.312 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:09.312 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:09.312 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:09.312 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:09.312 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:09.312 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:09.312 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:09.312 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:09.312 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:09.312 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:09.313 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:09.313 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:09.313 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:09.313 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:09.313 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:09.313 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:09.313 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:09.313 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:09.313 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:09.313 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:09.313 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:09.313 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:09.313 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.313 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:09.314 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:09.314 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:09.314 INFO:tasks.workunit.client.0.smithi022.stdout: 4, 2024-09-21T09:31:09.314 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:09.314 INFO:tasks.workunit.client.0.smithi022.stdout: 6 2024-09-21T09:31:09.314 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.314 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 4, 2024-09-21T09:31:09.314 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:09.314 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.314 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:09.314 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:09.314 INFO:tasks.workunit.client.0.smithi022.stdout: 1, 2024-09-21T09:31:09.314 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:09.314 INFO:tasks.workunit.client.0.smithi022.stdout: 4 2024-09-21T09:31:09.315 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.315 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 1, 2024-09-21T09:31:09.315 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:09.315 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:09.315 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 61, 2024-09-21T09:31:09.315 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:09.315 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:09.315 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:09.315 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:40.375501+0000", 2024-09-21T09:31:09.315 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:28:28.924195+0000", 2024-09-21T09:31:09.315 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:28:28.924195+0000", 2024-09-21T09:31:09.315 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:28:33.063343+0000", 2024-09-21T09:31:09.315 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:40.375501+0000", 2024-09-21T09:31:09.316 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:28:27.830318+0000", 2024-09-21T09:31:09.316 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:09.316 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:28:27.830318+0000", 2024-09-21T09:31:09.316 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 62, 2024-09-21T09:31:09.316 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:40.375501+0000", 2024-09-21T09:31:09.316 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:40.375501+0000", 2024-09-21T09:31:09.316 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:40.375501+0000", 2024-09-21T09:31:09.316 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:09.316 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:09.316 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:28:27.830318+0000", 2024-09-21T09:31:09.316 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:40.375501+0000", 2024-09-21T09:31:09.316 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:40.375501+0000", 2024-09-21T09:31:09.316 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:09.316 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 2, 2024-09-21T09:31:09.317 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:09.317 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:09.317 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 61, 2024-09-21T09:31:09.317 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:09.317 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:09.317 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:09.317 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:09.317 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 2, 2024-09-21T09:31:09.317 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:09.317 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:09.317 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:09.317 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "4.2", 2024-09-21T09:31:09.317 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:09.317 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:09.318 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 67, 2024-09-21T09:31:09.318 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:09.318 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:09.318 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T10:33:35.119949+0000", 2024-09-21T09:31:09.318 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.000800248, 2024-09-21T09:31:09.318 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:09.318 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:09.318 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 19, 2024-09-21T09:31:09.318 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:09.318 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:09.318 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:09.318 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:09.319 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:09.319 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:09.319 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:09.319 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:09.319 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:09.319 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:09.319 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:09.319 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:09.319 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:09.319 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:09.319 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:09.319 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 3, 2024-09-21T09:31:09.319 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 1, 2024-09-21T09:31:09.319 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:09.320 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 1, 2024-09-21T09:31:09.320 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:09.320 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:09.320 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:09.320 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:09.320 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:09.320 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:09.320 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:09.320 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:09.320 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:09.320 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:09.320 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:09.320 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:09.321 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:09.321 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:09.321 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:09.321 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:09.321 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:09.321 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:09.321 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 2, 2024-09-21T09:31:09.321 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 2 2024-09-21T09:31:09.321 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.321 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:09.321 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:09.321 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:09.321 INFO:tasks.workunit.client.0.smithi022.stdout: 1, 2024-09-21T09:31:09.321 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:09.321 INFO:tasks.workunit.client.0.smithi022.stdout: 4 2024-09-21T09:31:09.322 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.322 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 1, 2024-09-21T09:31:09.322 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "66'2" 2024-09-21T09:31:09.322 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.322 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:09.322 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:09.322 INFO:tasks.workunit.client.0.smithi022.stdout: 1, 2024-09-21T09:31:09.322 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:09.322 INFO:tasks.workunit.client.0.smithi022.stdout: 7 2024-09-21T09:31:09.322 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.322 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 1, 2024-09-21T09:31:09.322 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:09.322 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:09.323 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 51, 2024-09-21T09:31:09.323 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:09.323 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:09.323 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:09.323 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.062271+0000", 2024-09-21T09:31:09.323 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:32.264201+0000", 2024-09-21T09:31:09.323 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:32.264201+0000", 2024-09-21T09:31:09.323 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:32.264482+0000", 2024-09-21T09:31:09.323 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.062271+0000", 2024-09-21T09:31:09.323 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:09.323 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:09.323 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:09.323 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 52, 2024-09-21T09:31:09.324 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.062271+0000", 2024-09-21T09:31:09.324 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.062271+0000", 2024-09-21T09:31:09.324 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.062271+0000", 2024-09-21T09:31:09.324 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:09.324 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:09.324 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:09.324 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.062271+0000", 2024-09-21T09:31:09.324 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.062271+0000", 2024-09-21T09:31:09.324 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:09.324 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:09.324 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:09.324 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:09.324 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 51, 2024-09-21T09:31:09.324 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:09.324 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:09.325 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:09.325 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:09.325 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:09.325 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:09.325 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:09.325 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:09.325 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "2.4", 2024-09-21T09:31:09.325 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:09.325 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:09.325 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:09.325 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 24, 2024-09-21T09:31:09.325 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:09.325 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T20:37:15.565839+0000", 2024-09-21T09:31:09.325 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:09.326 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:09.326 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:09.326 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:09.326 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:09.326 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:09.326 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:09.326 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:09.326 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:09.326 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:09.326 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:09.326 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:09.326 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:09.326 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:09.327 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:09.327 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:09.327 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:09.327 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:09.327 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:09.327 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:09.327 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:09.327 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:09.327 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:09.327 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:09.327 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:09.327 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:09.327 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:09.327 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:09.328 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:09.328 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:09.328 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:09.328 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:09.328 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:09.328 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:09.328 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:09.328 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:09.328 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:09.328 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:09.328 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:09.328 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:09.328 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:09.328 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:09.329 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:09.329 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.329 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:09.329 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:09.329 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:09.329 INFO:tasks.workunit.client.0.smithi022.stdout: 1, 2024-09-21T09:31:09.329 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:09.329 INFO:tasks.workunit.client.0.smithi022.stdout: 7 2024-09-21T09:31:09.329 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.329 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 1, 2024-09-21T09:31:09.329 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:09.329 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.329 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:09.330 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:09.330 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:09.330 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:09.330 INFO:tasks.workunit.client.0.smithi022.stdout: 2 2024-09-21T09:31:09.330 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.330 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 5, 2024-09-21T09:31:09.330 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:09.330 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:09.330 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:09.330 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:09.330 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:09.330 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:09.330 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.066122+0000", 2024-09-21T09:31:09.331 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.634594+0000", 2024-09-21T09:31:09.331 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.634594+0000", 2024-09-21T09:31:09.331 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.634935+0000", 2024-09-21T09:31:09.331 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.066122+0000", 2024-09-21T09:31:09.331 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:09.331 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:09.331 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:09.331 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:09.331 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.066122+0000", 2024-09-21T09:31:09.331 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.066122+0000", 2024-09-21T09:31:09.331 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.066122+0000", 2024-09-21T09:31:09.331 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:09.331 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:09.332 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:09.332 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.066122+0000", 2024-09-21T09:31:09.332 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.066122+0000", 2024-09-21T09:31:09.332 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:09.332 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:09.332 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:09.332 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:09.332 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:09.332 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:09.332 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:09.332 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:09.332 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:09.332 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:09.333 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:09.333 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:09.333 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:09.333 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.5", 2024-09-21T09:31:09.333 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:09.333 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:09.333 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:09.333 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:09.333 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:09.333 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T11:53:03.606929+0000", 2024-09-21T09:31:09.333 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:09.333 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:09.333 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:09.333 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:09.334 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:09.334 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:09.334 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:09.334 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:09.334 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:09.334 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:09.334 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:09.334 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:09.334 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:09.334 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:09.334 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:09.334 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:09.334 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:09.335 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:09.335 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:09.335 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:09.335 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:09.335 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:09.335 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:09.335 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:09.335 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:09.335 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:09.335 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:09.335 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:09.335 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:09.335 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:09.335 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:09.336 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:09.336 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:09.336 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:09.336 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:09.336 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:09.336 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:09.336 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:09.336 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:09.336 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:09.336 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:09.336 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:09.336 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:09.336 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.336 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:09.337 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:09.337 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:09.337 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:09.337 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:09.337 INFO:tasks.workunit.client.0.smithi022.stdout: 2 2024-09-21T09:31:09.337 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.337 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 5, 2024-09-21T09:31:09.337 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:09.337 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.337 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:09.337 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:09.337 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:09.337 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:09.337 INFO:tasks.workunit.client.0.smithi022.stdout: 7 2024-09-21T09:31:09.338 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.338 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 0, 2024-09-21T09:31:09.338 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:09.338 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:09.338 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 61, 2024-09-21T09:31:09.338 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:09.338 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:09.338 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:09.338 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:40.922745+0000", 2024-09-21T09:31:09.338 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:28:28.923619+0000", 2024-09-21T09:31:09.338 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:28:28.923619+0000", 2024-09-21T09:31:09.338 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:28:33.141355+0000", 2024-09-21T09:31:09.338 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:40.922745+0000", 2024-09-21T09:31:09.338 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:28:27.830318+0000", 2024-09-21T09:31:09.339 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:09.339 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:28:27.830318+0000", 2024-09-21T09:31:09.339 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 62, 2024-09-21T09:31:09.339 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:40.922745+0000", 2024-09-21T09:31:09.339 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:40.922745+0000", 2024-09-21T09:31:09.339 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:40.922745+0000", 2024-09-21T09:31:09.339 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:09.339 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:09.339 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:28:27.830318+0000", 2024-09-21T09:31:09.339 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:40.922745+0000", 2024-09-21T09:31:09.339 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:40.922745+0000", 2024-09-21T09:31:09.339 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:09.339 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 1, 2024-09-21T09:31:09.339 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:09.340 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:09.340 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 61, 2024-09-21T09:31:09.340 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:09.340 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:09.340 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:09.340 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:09.340 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 1, 2024-09-21T09:31:09.340 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:09.340 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:09.340 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:09.340 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "4.3", 2024-09-21T09:31:09.340 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:09.340 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:09.340 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 67, 2024-09-21T09:31:09.341 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 20, 2024-09-21T09:31:09.341 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:09.341 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T10:40:50.990669+0000", 2024-09-21T09:31:09.341 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.000417457, 2024-09-21T09:31:09.341 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:09.341 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:09.341 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:09.341 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:09.341 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:09.341 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:09.341 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:09.341 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:09.341 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:09.341 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:09.341 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:09.342 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:09.342 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:09.342 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:09.342 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:09.342 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:09.342 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:09.342 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:09.342 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 3, 2024-09-21T09:31:09.342 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 1, 2024-09-21T09:31:09.342 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:09.342 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 1, 2024-09-21T09:31:09.342 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:09.342 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:09.342 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:09.343 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:09.343 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:09.343 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:09.343 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:09.343 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:09.343 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:09.343 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:09.343 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:09.343 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:09.343 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:09.343 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:09.343 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:09.343 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:09.344 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:09.344 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:09.344 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:09.344 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:09.344 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.344 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:09.344 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:09.344 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:09.344 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:09.344 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:09.344 INFO:tasks.workunit.client.0.smithi022.stdout: 7 2024-09-21T09:31:09.344 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.344 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 0, 2024-09-21T09:31:09.345 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "64'1" 2024-09-21T09:31:09.345 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.345 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:09.345 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:09.345 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:09.345 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:09.345 INFO:tasks.workunit.client.0.smithi022.stdout: 4 2024-09-21T09:31:09.345 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.345 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 7, 2024-09-21T09:31:09.345 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:09.345 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:09.345 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 51, 2024-09-21T09:31:09.345 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:09.345 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:09.346 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:09.346 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.359440+0000", 2024-09-21T09:31:09.346 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:32.274058+0000", 2024-09-21T09:31:09.346 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:32.274058+0000", 2024-09-21T09:31:09.346 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:32.274427+0000", 2024-09-21T09:31:09.346 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.359440+0000", 2024-09-21T09:31:09.346 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:09.346 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:09.346 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:09.346 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 52, 2024-09-21T09:31:09.346 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.359440+0000", 2024-09-21T09:31:09.346 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.359440+0000", 2024-09-21T09:31:09.346 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.359440+0000", 2024-09-21T09:31:09.346 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:09.347 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:09.347 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:09.347 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.359440+0000", 2024-09-21T09:31:09.347 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.359440+0000", 2024-09-21T09:31:09.347 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:09.347 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:09.347 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:09.347 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:09.347 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 51, 2024-09-21T09:31:09.347 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:09.347 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:09.347 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:09.347 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:09.348 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:09.348 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:09.348 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:09.348 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:09.348 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "2.5", 2024-09-21T09:31:09.348 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:09.348 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:09.348 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:09.348 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 24, 2024-09-21T09:31:09.348 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:09.348 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T16:09:38.714482+0000", 2024-09-21T09:31:09.348 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:09.348 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:09.348 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:09.348 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:09.349 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:09.349 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:09.349 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:09.349 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:09.349 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:09.349 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:09.349 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:09.349 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:09.349 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:09.349 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:09.349 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:09.349 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:09.349 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:09.349 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:09.350 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:09.350 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:09.350 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:09.350 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:09.350 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:09.350 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:09.350 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:09.350 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:09.350 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:09.350 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:09.350 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:09.350 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:09.350 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:09.350 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:09.350 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:09.351 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:09.351 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:09.351 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:09.351 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:09.351 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:09.351 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:09.351 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:09.351 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:09.351 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:09.351 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:09.351 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.351 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:09.351 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:09.351 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:09.352 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:09.352 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:09.352 INFO:tasks.workunit.client.0.smithi022.stdout: 4 2024-09-21T09:31:09.352 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.352 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 7, 2024-09-21T09:31:09.352 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:09.352 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.352 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:09.352 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:09.352 INFO:tasks.workunit.client.0.smithi022.stdout: 1, 2024-09-21T09:31:09.352 INFO:tasks.workunit.client.0.smithi022.stdout: 2, 2024-09-21T09:31:09.352 INFO:tasks.workunit.client.0.smithi022.stdout: 5 2024-09-21T09:31:09.352 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.352 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 1, 2024-09-21T09:31:09.352 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:09.353 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:09.353 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:09.353 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:09.353 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:09.353 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:09.353 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.062188+0000", 2024-09-21T09:31:09.353 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.642792+0000", 2024-09-21T09:31:09.353 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.642792+0000", 2024-09-21T09:31:09.353 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.643081+0000", 2024-09-21T09:31:09.353 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.062188+0000", 2024-09-21T09:31:09.353 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:09.353 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:09.353 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:09.353 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:09.354 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.062188+0000", 2024-09-21T09:31:09.354 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.062188+0000", 2024-09-21T09:31:09.354 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.062188+0000", 2024-09-21T09:31:09.354 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:09.354 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:09.354 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:09.354 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.062188+0000", 2024-09-21T09:31:09.354 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.062188+0000", 2024-09-21T09:31:09.354 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:09.354 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:09.354 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:09.354 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:09.354 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:09.354 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:09.355 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:09.355 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:09.355 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:09.355 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:09.355 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:09.355 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:09.355 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:09.355 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.4", 2024-09-21T09:31:09.355 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:09.355 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:09.355 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:09.355 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:09.355 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:09.355 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T17:05:03.157102+0000", 2024-09-21T09:31:09.355 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:09.356 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:09.356 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:09.356 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:09.356 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:09.356 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:09.356 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:09.356 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:09.356 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:09.356 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:09.356 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:09.356 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:09.356 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:09.356 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:09.356 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:09.356 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:09.357 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:09.357 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:09.357 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:09.357 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:09.357 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:09.357 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:09.357 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:09.357 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:09.357 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:09.357 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:09.357 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:09.357 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:09.357 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:09.357 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:09.358 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:09.358 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:09.358 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:09.358 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:09.358 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:09.358 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:09.358 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:09.358 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:09.358 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:09.358 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:09.358 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:09.358 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:09.358 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:09.358 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.358 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:09.359 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:09.359 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:09.359 INFO:tasks.workunit.client.0.smithi022.stdout: 1, 2024-09-21T09:31:09.359 INFO:tasks.workunit.client.0.smithi022.stdout: 2, 2024-09-21T09:31:09.359 INFO:tasks.workunit.client.0.smithi022.stdout: 5 2024-09-21T09:31:09.359 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.359 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 1, 2024-09-21T09:31:09.359 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:09.359 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.359 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:09.359 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:09.359 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:09.359 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:09.359 INFO:tasks.workunit.client.0.smithi022.stdout: 0 2024-09-21T09:31:09.359 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.360 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 3, 2024-09-21T09:31:09.360 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:09.360 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:09.360 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 61, 2024-09-21T09:31:09.360 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:09.360 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:09.360 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:09.360 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:40.484507+0000", 2024-09-21T09:31:09.360 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:28:28.845163+0000", 2024-09-21T09:31:09.360 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:28:28.845163+0000", 2024-09-21T09:31:09.360 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:28:33.125263+0000", 2024-09-21T09:31:09.360 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:40.484507+0000", 2024-09-21T09:31:09.360 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:28:27.830318+0000", 2024-09-21T09:31:09.360 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:09.361 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:28:27.830318+0000", 2024-09-21T09:31:09.361 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 62, 2024-09-21T09:31:09.361 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:40.484507+0000", 2024-09-21T09:31:09.361 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:40.484507+0000", 2024-09-21T09:31:09.361 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:40.484507+0000", 2024-09-21T09:31:09.361 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:09.361 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:09.361 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:28:27.830318+0000", 2024-09-21T09:31:09.361 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:40.484507+0000", 2024-09-21T09:31:09.361 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:40.484507+0000", 2024-09-21T09:31:09.361 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:09.361 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:09.361 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:09.361 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:09.361 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 61, 2024-09-21T09:31:09.362 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:09.362 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:09.362 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:09.362 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:09.362 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:09.362 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:09.362 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:09.362 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:09.362 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "4.0", 2024-09-21T09:31:09.362 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:09.362 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:09.362 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 67, 2024-09-21T09:31:09.362 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 19, 2024-09-21T09:31:09.362 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:09.362 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T13:56:39.868147+0000", 2024-09-21T09:31:09.363 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.000345912, 2024-09-21T09:31:09.363 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:09.363 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:09.363 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:09.363 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:09.363 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:09.363 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:09.363 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:09.363 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:09.363 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:09.363 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:09.363 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:09.363 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:09.363 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:09.364 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:09.364 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:09.364 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:09.364 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:09.364 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:09.364 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:09.364 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:09.364 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:09.364 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:09.364 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:09.364 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:09.364 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:09.364 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:09.364 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:09.364 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:09.365 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:09.365 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:09.365 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:09.365 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:09.365 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:09.365 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:09.365 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:09.365 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:09.365 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:09.365 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:09.365 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:09.365 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:09.365 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:09.365 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:09.365 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.366 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:09.366 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:09.366 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:09.366 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:09.366 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:09.366 INFO:tasks.workunit.client.0.smithi022.stdout: 0 2024-09-21T09:31:09.366 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.366 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 3, 2024-09-21T09:31:09.366 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:09.366 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.366 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:09.366 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:09.367 INFO:tasks.workunit.client.0.smithi022.stdout: 1, 2024-09-21T09:31:09.367 INFO:tasks.workunit.client.0.smithi022.stdout: 6, 2024-09-21T09:31:09.367 INFO:tasks.workunit.client.0.smithi022.stdout: 4 2024-09-21T09:31:09.367 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.367 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 1, 2024-09-21T09:31:09.367 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:09.367 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:09.367 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 51, 2024-09-21T09:31:09.367 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:09.367 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:09.367 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:09.367 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.062459+0000", 2024-09-21T09:31:09.367 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:32.265171+0000", 2024-09-21T09:31:09.367 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:32.265171+0000", 2024-09-21T09:31:09.368 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:32.265476+0000", 2024-09-21T09:31:09.368 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.062459+0000", 2024-09-21T09:31:09.368 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:09.368 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:09.368 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:09.368 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 52, 2024-09-21T09:31:09.368 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.062459+0000", 2024-09-21T09:31:09.368 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.062459+0000", 2024-09-21T09:31:09.368 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.062459+0000", 2024-09-21T09:31:09.368 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:09.368 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:09.368 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:09.368 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.062459+0000", 2024-09-21T09:31:09.368 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.062459+0000", 2024-09-21T09:31:09.368 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:09.369 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 1, 2024-09-21T09:31:09.369 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:09.369 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:09.369 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 51, 2024-09-21T09:31:09.369 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:09.369 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:09.369 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:09.369 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:09.369 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 1, 2024-09-21T09:31:09.369 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:09.369 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:09.369 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:09.369 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "2.6", 2024-09-21T09:31:09.369 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:09.369 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:09.370 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:09.370 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 25, 2024-09-21T09:31:09.370 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:09.370 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T12:42:48.279357+0000", 2024-09-21T09:31:09.370 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:09.370 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:09.370 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:09.370 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:09.370 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:09.370 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:09.370 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:09.370 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:09.370 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:09.371 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:09.371 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:09.371 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:09.371 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:09.371 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:09.371 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:09.371 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:09.371 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:09.371 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:09.371 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:09.371 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 3, 2024-09-21T09:31:09.371 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 1, 2024-09-21T09:31:09.371 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:09.371 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 1, 2024-09-21T09:31:09.372 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:09.372 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:09.372 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:09.372 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:09.372 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:09.372 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 1, 2024-09-21T09:31:09.372 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:09.372 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:09.372 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:09.372 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:09.372 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:09.372 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:09.372 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:09.373 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:09.373 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:09.373 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:09.373 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:09.373 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:09.373 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 1, 2024-09-21T09:31:09.373 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:09.373 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.373 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:09.373 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:09.373 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:09.373 INFO:tasks.workunit.client.0.smithi022.stdout: 1, 2024-09-21T09:31:09.373 INFO:tasks.workunit.client.0.smithi022.stdout: 6, 2024-09-21T09:31:09.373 INFO:tasks.workunit.client.0.smithi022.stdout: 4 2024-09-21T09:31:09.374 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.374 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 1, 2024-09-21T09:31:09.374 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "57'1" 2024-09-21T09:31:09.374 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.374 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:09.374 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:09.374 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:09.374 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:09.374 INFO:tasks.workunit.client.0.smithi022.stdout: 0 2024-09-21T09:31:09.374 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.374 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 3, 2024-09-21T09:31:09.374 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:09.374 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:09.375 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:09.375 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:09.375 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:09.375 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:09.375 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.125628+0000", 2024-09-21T09:31:09.375 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.629345+0000", 2024-09-21T09:31:09.375 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.629345+0000", 2024-09-21T09:31:09.375 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.629856+0000", 2024-09-21T09:31:09.375 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.125628+0000", 2024-09-21T09:31:09.375 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:09.375 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:09.375 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:09.375 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:09.375 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.125628+0000", 2024-09-21T09:31:09.376 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.125628+0000", 2024-09-21T09:31:09.376 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.125628+0000", 2024-09-21T09:31:09.376 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:09.376 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:09.376 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:09.376 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.125628+0000", 2024-09-21T09:31:09.376 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.125628+0000", 2024-09-21T09:31:09.376 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:09.376 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:09.376 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:09.376 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:09.376 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:09.377 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:09.377 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:09.377 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:09.377 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:09.377 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:09.377 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:09.377 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:09.377 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:09.377 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.7", 2024-09-21T09:31:09.377 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:09.377 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:09.377 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:09.377 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:09.377 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:09.377 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T17:30:14.361278+0000", 2024-09-21T09:31:09.378 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:09.378 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:09.378 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:09.378 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:09.378 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:09.378 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:09.378 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:09.378 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:09.378 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:09.378 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:09.378 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:09.378 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:09.378 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:09.378 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:09.378 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:09.379 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:09.379 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:09.379 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:09.379 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:09.379 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:09.379 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:09.379 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:09.379 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:09.379 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:09.379 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:09.379 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:09.379 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:09.379 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:09.379 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:09.379 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:09.380 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:09.380 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:09.380 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:09.380 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:09.380 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:09.380 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:09.380 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:09.380 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:09.380 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:09.380 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:09.380 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:09.380 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:09.380 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:09.380 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.381 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:09.381 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:09.381 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:09.381 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:09.381 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:09.381 INFO:tasks.workunit.client.0.smithi022.stdout: 0 2024-09-21T09:31:09.381 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.381 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 3, 2024-09-21T09:31:09.381 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:09.381 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.381 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:09.381 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:09.381 INFO:tasks.workunit.client.0.smithi022.stdout: 4, 2024-09-21T09:31:09.381 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:09.382 INFO:tasks.workunit.client.0.smithi022.stdout: 6 2024-09-21T09:31:09.382 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.382 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 4, 2024-09-21T09:31:09.382 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:09.382 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:09.382 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 61, 2024-09-21T09:31:09.382 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:09.382 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:09.382 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:09.382 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:40.749241+0000", 2024-09-21T09:31:09.382 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:28:28.924207+0000", 2024-09-21T09:31:09.382 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:28:28.924207+0000", 2024-09-21T09:31:09.382 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:28:33.062764+0000", 2024-09-21T09:31:09.382 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:40.749241+0000", 2024-09-21T09:31:09.382 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:28:27.830318+0000", 2024-09-21T09:31:09.383 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:09.383 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:28:27.830318+0000", 2024-09-21T09:31:09.383 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 62, 2024-09-21T09:31:09.383 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:40.749241+0000", 2024-09-21T09:31:09.383 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:40.749241+0000", 2024-09-21T09:31:09.383 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:40.749241+0000", 2024-09-21T09:31:09.383 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:09.383 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:09.383 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:28:27.830318+0000", 2024-09-21T09:31:09.383 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:40.749241+0000", 2024-09-21T09:31:09.383 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:40.749241+0000", 2024-09-21T09:31:09.383 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:09.383 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:09.384 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:09.384 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:09.384 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 61, 2024-09-21T09:31:09.384 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:09.384 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:09.384 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:09.384 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:09.384 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:09.384 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:09.384 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:09.384 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:09.384 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "4.1", 2024-09-21T09:31:09.384 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:09.384 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:09.384 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 67, 2024-09-21T09:31:09.385 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 19, 2024-09-21T09:31:09.385 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:09.385 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T11:08:48.406053+0000", 2024-09-21T09:31:09.385 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.000555972, 2024-09-21T09:31:09.385 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:09.385 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:09.385 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:09.385 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:09.385 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:09.385 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:09.385 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:09.385 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:09.385 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:09.386 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:09.386 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:09.386 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:09.386 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:09.386 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:09.386 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:09.386 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:09.386 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:09.386 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:09.386 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:09.386 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:09.386 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:09.386 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:09.387 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:09.387 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:09.387 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:09.387 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:09.387 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:09.387 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:09.387 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:09.387 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:09.387 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:09.387 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:09.387 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:09.387 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:09.387 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:09.388 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:09.388 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:09.388 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:09.388 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:09.388 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:09.388 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:09.388 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:09.388 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.388 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:09.388 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:09.388 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:09.388 INFO:tasks.workunit.client.0.smithi022.stdout: 4, 2024-09-21T09:31:09.388 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:09.388 INFO:tasks.workunit.client.0.smithi022.stdout: 6 2024-09-21T09:31:09.389 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.389 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 4, 2024-09-21T09:31:09.389 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:09.389 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.389 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:09.389 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:09.389 INFO:tasks.workunit.client.0.smithi022.stdout: 6, 2024-09-21T09:31:09.389 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:09.389 INFO:tasks.workunit.client.0.smithi022.stdout: 2 2024-09-21T09:31:09.389 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.389 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 6, 2024-09-21T09:31:09.389 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:09.389 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:09.390 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 51, 2024-09-21T09:31:09.390 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:09.390 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:09.390 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:09.390 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.484453+0000", 2024-09-21T09:31:09.390 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:32.264444+0000", 2024-09-21T09:31:09.390 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:32.264444+0000", 2024-09-21T09:31:09.390 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:32.264763+0000", 2024-09-21T09:31:09.390 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.484453+0000", 2024-09-21T09:31:09.390 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:09.390 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:09.390 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:09.390 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 52, 2024-09-21T09:31:09.390 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.484453+0000", 2024-09-21T09:31:09.391 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.484453+0000", 2024-09-21T09:31:09.391 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.484453+0000", 2024-09-21T09:31:09.391 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:09.391 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:09.391 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:09.391 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.484453+0000", 2024-09-21T09:31:09.391 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.484453+0000", 2024-09-21T09:31:09.391 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:09.391 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 2, 2024-09-21T09:31:09.391 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:09.391 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:09.391 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 51, 2024-09-21T09:31:09.391 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:09.392 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:09.392 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:09.392 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:09.392 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 2, 2024-09-21T09:31:09.392 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:09.392 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:09.392 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:09.392 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "2.7", 2024-09-21T09:31:09.392 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:09.392 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:09.392 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:09.392 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 26, 2024-09-21T09:31:09.392 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:09.392 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T12:01:52.871773+0000", 2024-09-21T09:31:09.392 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:09.393 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:09.393 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:09.393 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:09.393 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:09.393 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:09.393 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:09.393 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:09.393 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:09.393 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:09.393 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:09.393 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:09.393 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:09.393 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:09.393 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:09.394 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:09.394 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:09.394 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:09.394 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:09.394 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 3, 2024-09-21T09:31:09.394 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 1, 2024-09-21T09:31:09.394 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:09.394 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 1, 2024-09-21T09:31:09.394 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:09.394 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:09.394 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:09.394 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:09.394 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:09.394 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 1, 2024-09-21T09:31:09.395 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:09.395 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:09.395 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:09.395 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:09.395 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:09.395 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:09.395 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:09.395 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:09.395 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:09.395 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:09.395 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:09.395 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:09.395 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 3, 2024-09-21T09:31:09.395 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:09.395 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.396 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:09.396 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:09.396 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:09.396 INFO:tasks.workunit.client.0.smithi022.stdout: 6, 2024-09-21T09:31:09.396 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:09.396 INFO:tasks.workunit.client.0.smithi022.stdout: 2 2024-09-21T09:31:09.396 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.396 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 6, 2024-09-21T09:31:09.396 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "57'2" 2024-09-21T09:31:09.396 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.396 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:09.396 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:09.396 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:09.397 INFO:tasks.workunit.client.0.smithi022.stdout: 1, 2024-09-21T09:31:09.397 INFO:tasks.workunit.client.0.smithi022.stdout: 4 2024-09-21T09:31:09.397 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.397 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 0, 2024-09-21T09:31:09.397 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:09.397 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:09.397 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:09.397 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:09.397 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:09.397 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:09.397 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.141267+0000", 2024-09-21T09:31:09.397 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.634680+0000", 2024-09-21T09:31:09.397 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.634680+0000", 2024-09-21T09:31:09.397 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.637964+0000", 2024-09-21T09:31:09.397 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.141267+0000", 2024-09-21T09:31:09.398 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:09.398 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:09.398 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:09.398 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:09.398 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.141267+0000", 2024-09-21T09:31:09.398 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.141267+0000", 2024-09-21T09:31:09.398 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.141267+0000", 2024-09-21T09:31:09.398 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:09.398 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:09.398 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:09.398 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.141267+0000", 2024-09-21T09:31:09.398 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.141267+0000", 2024-09-21T09:31:09.398 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:09.398 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:09.399 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:09.399 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:09.399 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:09.399 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:09.399 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:09.399 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:09.399 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:09.399 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:09.399 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:09.399 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:09.399 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:09.399 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.6", 2024-09-21T09:31:09.399 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:09.399 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:09.400 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:09.400 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:09.400 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:09.400 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T19:50:36.642394+0000", 2024-09-21T09:31:09.400 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:09.400 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:09.400 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:09.400 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:09.400 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:09.400 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:09.400 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:09.400 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:09.401 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:09.401 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:09.401 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:09.401 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:09.401 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:09.401 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:09.401 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:09.401 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:09.401 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:09.401 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:09.401 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:09.401 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:09.401 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:09.402 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:09.402 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:09.402 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:09.402 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:09.402 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:09.402 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:09.402 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:09.402 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:09.402 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:09.402 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:09.402 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:09.402 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:09.402 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:09.402 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:09.403 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:09.403 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:09.403 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:09.403 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:09.403 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:09.403 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:09.403 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:09.403 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:09.403 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.403 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:09.403 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:09.403 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:09.403 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:09.403 INFO:tasks.workunit.client.0.smithi022.stdout: 1, 2024-09-21T09:31:09.404 INFO:tasks.workunit.client.0.smithi022.stdout: 4 2024-09-21T09:31:09.404 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.404 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 0, 2024-09-21T09:31:09.404 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:09.404 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.404 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:09.404 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:09.404 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:09.404 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:09.404 INFO:tasks.workunit.client.0.smithi022.stdout: 1 2024-09-21T09:31:09.404 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.404 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 7, 2024-09-21T09:31:09.404 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:09.404 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:09.405 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 51, 2024-09-21T09:31:09.405 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:09.405 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:09.405 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:09.405 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.359011+0000", 2024-09-21T09:31:09.405 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:32.268720+0000", 2024-09-21T09:31:09.405 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:32.268720+0000", 2024-09-21T09:31:09.405 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:32.270810+0000", 2024-09-21T09:31:09.405 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.359011+0000", 2024-09-21T09:31:09.405 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:09.405 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:09.405 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:09.405 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 52, 2024-09-21T09:31:09.405 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.359011+0000", 2024-09-21T09:31:09.405 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.359011+0000", 2024-09-21T09:31:09.406 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.359011+0000", 2024-09-21T09:31:09.406 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:09.406 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:09.406 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:09.406 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.359011+0000", 2024-09-21T09:31:09.406 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.359011+0000", 2024-09-21T09:31:09.406 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:09.406 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:09.406 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:09.406 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:09.406 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 51, 2024-09-21T09:31:09.406 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:09.406 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:09.407 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:09.407 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:09.407 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:09.407 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:09.407 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:09.407 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:09.407 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "2.8", 2024-09-21T09:31:09.407 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:09.407 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:09.407 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:09.407 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 24, 2024-09-21T09:31:09.407 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:09.407 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T13:58:54.820070+0000", 2024-09-21T09:31:09.407 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:09.408 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:09.408 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:09.408 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:09.408 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:09.408 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:09.408 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:09.408 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:09.408 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:09.408 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:09.408 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:09.408 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:09.408 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:09.408 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:09.408 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:09.408 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:09.409 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:09.409 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:09.409 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:09.409 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:09.409 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:09.409 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:09.409 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:09.409 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:09.409 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:09.409 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:09.409 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:09.409 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:09.409 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:09.409 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:09.410 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:09.410 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:09.410 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:09.410 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:09.410 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:09.410 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:09.410 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:09.410 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:09.410 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:09.410 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:09.410 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:09.410 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:09.410 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:09.411 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.411 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:09.411 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:09.411 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:09.411 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:09.411 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:09.411 INFO:tasks.workunit.client.0.smithi022.stdout: 1 2024-09-21T09:31:09.411 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.411 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 7, 2024-09-21T09:31:09.411 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:09.411 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.411 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:09.411 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:09.412 INFO:tasks.workunit.client.0.smithi022.stdout: 4, 2024-09-21T09:31:09.412 INFO:tasks.workunit.client.0.smithi022.stdout: 2, 2024-09-21T09:31:09.412 INFO:tasks.workunit.client.0.smithi022.stdout: 7 2024-09-21T09:31:09.412 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.412 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 4, 2024-09-21T09:31:09.412 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:09.412 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:09.412 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:09.412 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:09.412 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:09.412 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:09.412 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.062395+0000", 2024-09-21T09:31:09.412 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.637064+0000", 2024-09-21T09:31:09.412 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.637064+0000", 2024-09-21T09:31:09.413 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.637942+0000", 2024-09-21T09:31:09.413 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.062395+0000", 2024-09-21T09:31:09.413 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:09.413 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:09.413 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:09.413 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:09.413 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.062395+0000", 2024-09-21T09:31:09.413 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.062395+0000", 2024-09-21T09:31:09.413 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.062395+0000", 2024-09-21T09:31:09.413 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:09.413 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:09.413 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:09.413 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.062395+0000", 2024-09-21T09:31:09.413 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.062395+0000", 2024-09-21T09:31:09.414 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:09.414 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:09.414 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:09.414 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:09.414 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:09.414 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:09.414 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:09.414 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:09.414 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:09.414 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:09.414 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:09.414 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:09.414 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:09.414 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.9", 2024-09-21T09:31:09.415 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:09.415 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:09.415 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:09.415 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:09.415 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:09.415 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T10:16:53.911245+0000", 2024-09-21T09:31:09.415 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:09.415 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:09.415 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:09.415 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:09.415 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:09.415 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:09.415 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:09.415 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:09.416 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:09.416 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:09.416 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:09.416 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:09.416 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:09.416 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:09.416 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:09.416 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:09.416 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:09.416 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:09.416 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:09.416 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:09.416 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:09.417 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:09.417 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:09.417 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:09.417 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:09.417 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:09.417 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:09.417 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:09.417 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:09.417 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:09.417 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:09.417 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:09.417 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:09.417 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:09.417 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:09.417 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:09.418 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:09.418 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:09.418 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:09.418 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:09.418 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:09.418 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:09.418 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:09.418 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.418 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:09.418 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:09.418 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:09.418 INFO:tasks.workunit.client.0.smithi022.stdout: 4, 2024-09-21T09:31:09.418 INFO:tasks.workunit.client.0.smithi022.stdout: 2, 2024-09-21T09:31:09.418 INFO:tasks.workunit.client.0.smithi022.stdout: 7 2024-09-21T09:31:09.419 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.419 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 4, 2024-09-21T09:31:09.419 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:09.419 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.419 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:09.419 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:09.419 INFO:tasks.workunit.client.0.smithi022.stdout: 1, 2024-09-21T09:31:09.419 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:09.419 INFO:tasks.workunit.client.0.smithi022.stdout: 3 2024-09-21T09:31:09.419 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.419 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 1, 2024-09-21T09:31:09.419 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:09.419 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:09.419 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 51, 2024-09-21T09:31:09.420 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:09.420 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:09.420 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:09.420 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.062682+0000", 2024-09-21T09:31:09.420 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:32.262555+0000", 2024-09-21T09:31:09.420 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:32.262555+0000", 2024-09-21T09:31:09.420 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:32.262848+0000", 2024-09-21T09:31:09.420 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.062682+0000", 2024-09-21T09:31:09.420 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:09.420 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:09.420 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:09.420 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 52, 2024-09-21T09:31:09.421 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.062682+0000", 2024-09-21T09:31:09.421 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.062682+0000", 2024-09-21T09:31:09.421 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.062682+0000", 2024-09-21T09:31:09.421 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:09.421 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:09.421 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:09.421 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.062682+0000", 2024-09-21T09:31:09.421 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.062682+0000", 2024-09-21T09:31:09.421 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:09.421 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:09.421 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:09.421 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:09.422 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 51, 2024-09-21T09:31:09.422 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:09.422 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:09.422 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:09.422 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:09.422 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:09.422 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:09.422 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:09.422 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:09.422 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "2.9", 2024-09-21T09:31:09.422 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:09.422 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:09.422 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:09.423 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 24, 2024-09-21T09:31:09.423 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:09.423 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T20:27:02.670310+0000", 2024-09-21T09:31:09.423 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:09.423 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:09.423 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:09.423 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:09.423 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:09.423 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:09.423 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:09.423 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:09.423 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:09.424 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:09.424 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:09.424 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:09.424 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:09.424 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:09.424 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:09.424 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:09.424 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:09.424 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:09.424 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:09.424 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:09.424 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:09.424 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:09.424 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:09.425 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:09.425 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:09.425 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:09.425 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:09.425 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:09.425 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:09.425 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:09.425 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:09.425 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:09.425 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:09.425 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:09.425 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:09.425 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:09.426 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:09.426 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:09.426 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:09.426 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:09.426 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:09.426 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:09.426 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:09.426 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.426 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:09.426 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:09.426 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:09.426 INFO:tasks.workunit.client.0.smithi022.stdout: 1, 2024-09-21T09:31:09.426 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:09.427 INFO:tasks.workunit.client.0.smithi022.stdout: 3 2024-09-21T09:31:09.427 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.427 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 1, 2024-09-21T09:31:09.427 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:09.427 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.427 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:09.427 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:09.427 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:09.427 INFO:tasks.workunit.client.0.smithi022.stdout: 1, 2024-09-21T09:31:09.427 INFO:tasks.workunit.client.0.smithi022.stdout: 7 2024-09-21T09:31:09.427 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.427 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 3, 2024-09-21T09:31:09.427 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:09.427 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:09.428 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:09.428 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:09.428 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:09.428 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:09.428 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.125068+0000", 2024-09-21T09:31:09.428 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.625619+0000", 2024-09-21T09:31:09.428 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.625619+0000", 2024-09-21T09:31:09.428 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.625899+0000", 2024-09-21T09:31:09.428 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.125068+0000", 2024-09-21T09:31:09.428 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:09.428 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:09.428 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:09.428 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:09.429 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.125068+0000", 2024-09-21T09:31:09.429 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.125068+0000", 2024-09-21T09:31:09.429 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.125068+0000", 2024-09-21T09:31:09.429 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:09.429 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:09.429 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:09.429 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.125068+0000", 2024-09-21T09:31:09.429 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.125068+0000", 2024-09-21T09:31:09.429 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:09.429 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:09.429 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:09.429 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:09.429 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:09.429 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:09.430 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:09.430 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:09.430 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:09.430 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:09.430 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:09.430 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:09.430 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:09.430 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.8", 2024-09-21T09:31:09.430 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:09.430 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:09.430 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:09.430 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:09.430 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:09.430 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T14:38:48.577776+0000", 2024-09-21T09:31:09.431 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:09.431 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:09.431 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:09.431 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:09.431 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:09.431 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:09.431 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:09.431 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:09.431 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:09.431 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:09.431 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:09.431 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:09.432 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:09.432 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:09.432 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:09.432 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:09.432 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:09.432 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:09.432 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:09.432 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:09.432 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:09.432 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:09.432 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:09.432 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:09.432 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:09.432 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:09.433 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:09.433 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:09.433 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:09.433 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:09.433 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:09.433 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:09.433 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:09.433 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:09.433 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:09.433 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:09.433 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:09.433 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:09.433 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:09.433 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:09.434 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:09.434 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:09.434 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:09.434 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.434 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:09.434 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:09.434 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:09.434 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:09.434 INFO:tasks.workunit.client.0.smithi022.stdout: 1, 2024-09-21T09:31:09.434 INFO:tasks.workunit.client.0.smithi022.stdout: 7 2024-09-21T09:31:09.434 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.434 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 3, 2024-09-21T09:31:09.434 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:09.435 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.435 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:09.435 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:09.435 INFO:tasks.workunit.client.0.smithi022.stdout: 1, 2024-09-21T09:31:09.435 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:09.435 INFO:tasks.workunit.client.0.smithi022.stdout: 7 2024-09-21T09:31:09.435 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.435 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 1, 2024-09-21T09:31:09.435 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:09.435 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:09.435 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 51, 2024-09-21T09:31:09.435 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:09.435 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:09.435 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:09.436 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.063179+0000", 2024-09-21T09:31:09.436 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:32.264353+0000", 2024-09-21T09:31:09.436 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:32.264353+0000", 2024-09-21T09:31:09.436 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:32.264667+0000", 2024-09-21T09:31:09.436 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.063179+0000", 2024-09-21T09:31:09.436 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:09.436 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:09.436 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:09.436 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 52, 2024-09-21T09:31:09.436 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.063179+0000", 2024-09-21T09:31:09.436 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.063179+0000", 2024-09-21T09:31:09.436 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.063179+0000", 2024-09-21T09:31:09.436 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:09.436 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:09.437 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:09.437 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.063179+0000", 2024-09-21T09:31:09.437 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.063179+0000", 2024-09-21T09:31:09.437 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:09.437 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:09.437 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:09.437 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:09.437 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 51, 2024-09-21T09:31:09.437 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:09.437 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:09.437 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:09.437 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:09.437 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:09.437 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:09.438 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:09.438 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:09.438 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "2.a", 2024-09-21T09:31:09.438 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:09.438 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:09.438 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:09.438 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 24, 2024-09-21T09:31:09.438 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:09.438 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T17:21:53.938525+0000", 2024-09-21T09:31:09.438 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:09.438 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:09.438 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:09.438 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:09.438 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:09.439 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:09.439 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:09.439 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:09.439 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:09.439 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:09.439 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:09.439 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:09.439 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:09.439 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:09.439 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:09.439 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:09.439 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:09.439 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:09.439 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:09.440 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:09.440 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:09.440 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:09.440 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:09.440 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:09.440 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:09.440 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:09.440 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:09.440 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:09.440 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:09.440 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:09.440 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:09.440 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:09.440 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:09.440 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:09.441 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:09.441 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:09.441 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:09.441 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:09.441 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:09.441 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:09.441 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:09.441 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:09.441 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:09.441 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.441 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:09.441 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:09.441 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:09.441 INFO:tasks.workunit.client.0.smithi022.stdout: 1, 2024-09-21T09:31:09.442 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:09.442 INFO:tasks.workunit.client.0.smithi022.stdout: 7 2024-09-21T09:31:09.442 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.442 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 1, 2024-09-21T09:31:09.442 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:09.442 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.442 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:09.442 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:09.442 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:09.442 INFO:tasks.workunit.client.0.smithi022.stdout: 2, 2024-09-21T09:31:09.442 INFO:tasks.workunit.client.0.smithi022.stdout: 4 2024-09-21T09:31:09.442 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.442 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 3, 2024-09-21T09:31:09.443 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:09.443 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:09.443 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:09.443 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:09.443 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:09.443 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:09.443 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.125251+0000", 2024-09-21T09:31:09.443 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:52.080726+0000", 2024-09-21T09:31:09.443 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:52.080726+0000", 2024-09-21T09:31:09.443 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:52.081164+0000", 2024-09-21T09:31:09.443 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.125251+0000", 2024-09-21T09:31:09.443 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:09.443 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:09.443 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:09.444 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 59, 2024-09-21T09:31:09.444 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.125251+0000", 2024-09-21T09:31:09.444 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.125251+0000", 2024-09-21T09:31:09.444 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.125251+0000", 2024-09-21T09:31:09.444 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:09.444 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:09.444 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:09.444 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.125251+0000", 2024-09-21T09:31:09.444 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.125251+0000", 2024-09-21T09:31:09.444 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:09.444 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:09.444 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:09.444 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:09.444 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 58, 2024-09-21T09:31:09.445 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:09.445 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:09.445 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:09.445 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:09.445 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:09.445 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:09.445 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:09.445 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:09.445 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.b", 2024-09-21T09:31:09.445 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:09.445 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:09.445 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:09.445 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 31, 2024-09-21T09:31:09.445 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:09.446 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T17:37:01.620726+0000", 2024-09-21T09:31:09.446 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:09.446 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:09.446 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:09.446 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:09.446 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:09.446 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:09.446 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:09.446 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:09.446 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:09.446 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:09.446 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:09.446 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:09.447 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:09.447 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:09.447 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:09.447 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:09.447 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:09.447 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:09.447 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:09.447 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:09.447 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:09.447 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:09.447 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:09.447 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:09.447 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:09.447 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:09.448 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:09.448 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:09.448 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:09.448 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:09.448 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:09.448 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:09.448 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:09.448 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:09.448 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:09.448 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:09.448 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:09.448 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:09.448 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:09.449 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:09.449 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:09.449 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:09.449 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:09.449 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.449 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:09.449 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:09.449 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:09.449 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:09.449 INFO:tasks.workunit.client.0.smithi022.stdout: 2, 2024-09-21T09:31:09.449 INFO:tasks.workunit.client.0.smithi022.stdout: 4 2024-09-21T09:31:09.449 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.449 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 3, 2024-09-21T09:31:09.450 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:09.450 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.450 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:09.450 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:09.450 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:09.450 INFO:tasks.workunit.client.0.smithi022.stdout: 4, 2024-09-21T09:31:09.450 INFO:tasks.workunit.client.0.smithi022.stdout: 5 2024-09-21T09:31:09.450 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.450 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 7, 2024-09-21T09:31:09.450 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:09.450 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:09.450 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 51, 2024-09-21T09:31:09.450 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:09.450 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:09.450 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:09.451 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.359154+0000", 2024-09-21T09:31:09.451 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:32.274109+0000", 2024-09-21T09:31:09.451 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:32.274109+0000", 2024-09-21T09:31:09.451 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:32.274404+0000", 2024-09-21T09:31:09.451 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.359154+0000", 2024-09-21T09:31:09.451 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:09.451 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:09.451 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:09.451 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 52, 2024-09-21T09:31:09.451 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.359154+0000", 2024-09-21T09:31:09.451 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.359154+0000", 2024-09-21T09:31:09.451 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.359154+0000", 2024-09-21T09:31:09.451 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:09.451 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:09.452 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:09.452 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.359154+0000", 2024-09-21T09:31:09.452 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.359154+0000", 2024-09-21T09:31:09.452 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:09.452 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 2, 2024-09-21T09:31:09.452 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:09.452 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:09.452 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 51, 2024-09-21T09:31:09.452 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:09.452 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:09.452 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:09.452 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:09.452 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 2, 2024-09-21T09:31:09.452 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:09.452 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:09.453 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:09.453 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "2.b", 2024-09-21T09:31:09.453 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:09.453 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:09.453 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:09.453 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 26, 2024-09-21T09:31:09.453 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:09.453 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T14:24:06.607203+0000", 2024-09-21T09:31:09.453 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:09.453 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:09.453 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:09.453 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:09.453 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:09.454 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:09.454 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:09.454 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:09.454 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:09.454 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:09.454 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:09.454 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:09.454 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:09.454 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:09.454 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:09.454 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:09.454 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:09.454 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:09.454 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:09.455 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 3, 2024-09-21T09:31:09.455 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 1, 2024-09-21T09:31:09.455 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:09.455 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 1, 2024-09-21T09:31:09.455 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:09.455 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:09.455 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:09.455 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:09.455 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:09.455 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 1, 2024-09-21T09:31:09.455 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:09.455 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:09.455 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:09.456 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:09.456 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:09.456 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:09.456 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:09.456 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:09.456 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:09.456 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:09.456 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:09.456 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:09.456 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 1, 2024-09-21T09:31:09.456 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:09.456 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.456 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:09.456 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:09.457 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:09.457 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:09.457 INFO:tasks.workunit.client.0.smithi022.stdout: 4, 2024-09-21T09:31:09.457 INFO:tasks.workunit.client.0.smithi022.stdout: 5 2024-09-21T09:31:09.457 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.457 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 7, 2024-09-21T09:31:09.457 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "57'2" 2024-09-21T09:31:09.457 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.457 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:09.457 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:09.457 INFO:tasks.workunit.client.0.smithi022.stdout: 6, 2024-09-21T09:31:09.457 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:09.457 INFO:tasks.workunit.client.0.smithi022.stdout: 1 2024-09-21T09:31:09.457 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.457 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 6, 2024-09-21T09:31:09.458 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:09.458 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:09.458 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:09.458 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:09.458 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:09.458 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:09.458 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.483953+0000", 2024-09-21T09:31:09.458 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:52.050052+0000", 2024-09-21T09:31:09.458 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:52.050052+0000", 2024-09-21T09:31:09.458 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:52.050465+0000", 2024-09-21T09:31:09.458 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.483953+0000", 2024-09-21T09:31:09.458 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:09.458 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:09.458 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:09.459 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 59, 2024-09-21T09:31:09.459 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.483953+0000", 2024-09-21T09:31:09.459 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.483953+0000", 2024-09-21T09:31:09.459 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.483953+0000", 2024-09-21T09:31:09.459 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:09.459 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:09.459 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:09.459 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.483953+0000", 2024-09-21T09:31:09.459 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.483953+0000", 2024-09-21T09:31:09.459 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:09.459 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:09.459 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:09.459 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:09.459 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 58, 2024-09-21T09:31:09.459 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:09.460 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:09.460 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:09.460 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:09.460 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:09.460 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:09.460 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:09.460 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:09.460 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.a", 2024-09-21T09:31:09.460 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:09.460 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:09.460 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:09.460 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 31, 2024-09-21T09:31:09.460 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:09.460 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T17:58:35.370217+0000", 2024-09-21T09:31:09.460 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:09.461 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:09.461 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:09.461 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:09.461 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:09.461 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:09.461 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:09.461 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:09.461 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:09.461 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:09.461 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:09.461 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:09.461 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:09.461 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:09.461 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:09.462 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:09.462 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:09.462 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:09.462 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:09.462 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:09.462 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:09.462 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:09.462 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:09.462 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:09.462 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:09.462 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:09.462 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:09.462 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:09.462 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:09.462 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:09.463 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:09.463 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:09.463 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:09.463 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:09.463 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:09.463 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:09.463 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:09.463 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:09.463 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:09.463 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:09.463 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:09.463 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:09.463 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:09.463 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.464 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:09.464 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:09.464 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:09.464 INFO:tasks.workunit.client.0.smithi022.stdout: 6, 2024-09-21T09:31:09.464 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:09.464 INFO:tasks.workunit.client.0.smithi022.stdout: 1 2024-09-21T09:31:09.464 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.464 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 6, 2024-09-21T09:31:09.464 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:09.464 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.464 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:09.464 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:09.464 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:09.464 INFO:tasks.workunit.client.0.smithi022.stdout: 4, 2024-09-21T09:31:09.464 INFO:tasks.workunit.client.0.smithi022.stdout: 6 2024-09-21T09:31:09.465 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.465 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 5, 2024-09-21T09:31:09.465 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:09.465 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:09.465 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:09.465 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:09.465 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:09.465 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:09.465 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.066042+0000", 2024-09-21T09:31:09.465 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.637664+0000", 2024-09-21T09:31:09.465 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.637664+0000", 2024-09-21T09:31:09.465 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.637946+0000", 2024-09-21T09:31:09.465 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.066042+0000", 2024-09-21T09:31:09.465 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:09.466 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:09.466 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:09.466 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:09.466 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.066042+0000", 2024-09-21T09:31:09.466 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.066042+0000", 2024-09-21T09:31:09.466 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.066042+0000", 2024-09-21T09:31:09.466 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:09.466 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:09.466 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:09.466 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.066042+0000", 2024-09-21T09:31:09.466 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.066042+0000", 2024-09-21T09:31:09.466 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:09.466 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:09.466 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:09.467 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:09.467 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:09.467 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:09.467 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:09.467 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:09.467 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:09.467 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:09.467 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:09.467 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:09.467 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:09.467 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.1d", 2024-09-21T09:31:09.467 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:09.467 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:09.467 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:09.467 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:09.468 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:09.468 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T18:22:17.260192+0000", 2024-09-21T09:31:09.468 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:09.468 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:09.468 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:09.468 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:09.468 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:09.468 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:09.468 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:09.468 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:09.468 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:09.468 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:09.468 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:09.468 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:09.468 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:09.469 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:09.469 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:09.469 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:09.469 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:09.469 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:09.469 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:09.469 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:09.469 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:09.469 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:09.469 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:09.469 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:09.469 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:09.469 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:09.469 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:09.469 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:09.470 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:09.470 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:09.470 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:09.470 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:09.470 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:09.470 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:09.470 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:09.470 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:09.470 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:09.470 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:09.470 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:09.470 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:09.470 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:09.470 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:09.470 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:09.471 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.471 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:09.471 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:09.471 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:09.471 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:09.471 INFO:tasks.workunit.client.0.smithi022.stdout: 4, 2024-09-21T09:31:09.471 INFO:tasks.workunit.client.0.smithi022.stdout: 6 2024-09-21T09:31:09.471 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.471 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 5, 2024-09-21T09:31:09.471 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:09.471 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.471 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:09.471 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:09.471 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:09.472 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:09.472 INFO:tasks.workunit.client.0.smithi022.stdout: 2 2024-09-21T09:31:09.472 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.472 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 7, 2024-09-21T09:31:09.472 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:09.472 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:09.472 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 51, 2024-09-21T09:31:09.472 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:09.472 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:09.472 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:09.472 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.359224+0000", 2024-09-21T09:31:09.472 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:32.268038+0000", 2024-09-21T09:31:09.473 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:32.268038+0000", 2024-09-21T09:31:09.473 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:32.268336+0000", 2024-09-21T09:31:09.473 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.359224+0000", 2024-09-21T09:31:09.473 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:09.473 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:09.473 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:09.473 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 52, 2024-09-21T09:31:09.473 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.359224+0000", 2024-09-21T09:31:09.473 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.359224+0000", 2024-09-21T09:31:09.473 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.359224+0000", 2024-09-21T09:31:09.473 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:09.473 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:09.474 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:09.474 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.359224+0000", 2024-09-21T09:31:09.474 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.359224+0000", 2024-09-21T09:31:09.474 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:09.474 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 2, 2024-09-21T09:31:09.474 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:09.474 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:09.474 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 51, 2024-09-21T09:31:09.474 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:09.474 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:09.474 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:09.474 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:09.474 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 2, 2024-09-21T09:31:09.474 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:09.474 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:09.475 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:09.475 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "2.1c", 2024-09-21T09:31:09.475 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:09.475 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:09.475 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:09.475 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 26, 2024-09-21T09:31:09.475 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:09.475 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T16:57:13.991593+0000", 2024-09-21T09:31:09.475 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:09.475 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:09.475 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:09.475 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:09.475 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:09.476 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:09.476 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:09.476 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:09.476 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:09.476 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:09.476 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:09.476 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:09.476 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:09.476 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:09.476 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:09.476 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:09.476 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:09.476 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:09.477 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:09.477 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 3, 2024-09-21T09:31:09.477 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 1, 2024-09-21T09:31:09.477 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:09.477 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 1, 2024-09-21T09:31:09.477 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:09.477 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:09.477 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:09.477 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:09.477 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:09.477 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 1, 2024-09-21T09:31:09.477 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:09.478 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:09.478 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:09.478 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:09.478 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:09.478 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:09.478 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:09.478 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:09.478 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:09.478 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:09.478 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:09.478 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:09.478 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 3, 2024-09-21T09:31:09.478 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:09.479 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.479 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:09.479 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:09.479 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:09.479 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:09.479 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:09.479 INFO:tasks.workunit.client.0.smithi022.stdout: 2 2024-09-21T09:31:09.479 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.479 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 7, 2024-09-21T09:31:09.479 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "57'2" 2024-09-21T09:31:09.479 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.479 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:09.479 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:09.479 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:09.480 INFO:tasks.workunit.client.0.smithi022.stdout: 4, 2024-09-21T09:31:09.480 INFO:tasks.workunit.client.0.smithi022.stdout: 1 2024-09-21T09:31:09.480 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.480 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 5, 2024-09-21T09:31:09.480 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:09.480 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:09.480 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:09.480 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:09.480 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:09.480 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:09.480 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.066636+0000", 2024-09-21T09:31:09.480 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.642487+0000", 2024-09-21T09:31:09.480 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.642487+0000", 2024-09-21T09:31:09.480 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.642780+0000", 2024-09-21T09:31:09.481 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.066636+0000", 2024-09-21T09:31:09.481 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:09.481 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:09.481 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:09.481 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:09.481 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.066636+0000", 2024-09-21T09:31:09.481 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.066636+0000", 2024-09-21T09:31:09.481 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.066636+0000", 2024-09-21T09:31:09.481 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:09.481 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:09.481 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:09.481 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.066636+0000", 2024-09-21T09:31:09.481 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.066636+0000", 2024-09-21T09:31:09.481 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:09.482 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:09.482 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:09.482 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:09.482 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:09.482 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:09.482 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:09.482 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:09.482 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:09.482 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:09.482 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:09.482 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:09.482 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:09.482 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.1c", 2024-09-21T09:31:09.482 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:09.482 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:09.483 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:09.483 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:09.483 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:09.483 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T15:28:54.864609+0000", 2024-09-21T09:31:09.483 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:09.483 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:09.483 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:09.483 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:09.483 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:09.483 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:09.483 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:09.483 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:09.483 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:09.484 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:09.484 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:09.484 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:09.484 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:09.484 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:09.484 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:09.484 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:09.484 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:09.484 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:09.484 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:09.484 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:09.484 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:09.484 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:09.485 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:09.485 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:09.485 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:09.485 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:09.485 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:09.485 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:09.485 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:09.485 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:09.485 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:09.485 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:09.485 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:09.485 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:09.485 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:09.485 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:09.485 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:09.486 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:09.486 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:09.486 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:09.486 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:09.486 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:09.486 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:09.486 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.486 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:09.486 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:09.486 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:09.486 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:09.486 INFO:tasks.workunit.client.0.smithi022.stdout: 4, 2024-09-21T09:31:09.487 INFO:tasks.workunit.client.0.smithi022.stdout: 1 2024-09-21T09:31:09.487 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.487 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 5, 2024-09-21T09:31:09.487 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:09.487 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.487 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:09.487 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:09.487 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:09.487 INFO:tasks.workunit.client.0.smithi022.stdout: 6, 2024-09-21T09:31:09.487 INFO:tasks.workunit.client.0.smithi022.stdout: 0 2024-09-21T09:31:09.487 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.487 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 7, 2024-09-21T09:31:09.487 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:09.488 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:09.488 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 51, 2024-09-21T09:31:09.488 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:09.488 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:09.488 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:09.488 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.359353+0000", 2024-09-21T09:31:09.488 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:32.269469+0000", 2024-09-21T09:31:09.488 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:32.269469+0000", 2024-09-21T09:31:09.488 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:32.270398+0000", 2024-09-21T09:31:09.488 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.359353+0000", 2024-09-21T09:31:09.488 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:09.488 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:09.488 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:09.489 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 52, 2024-09-21T09:31:09.489 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.359353+0000", 2024-09-21T09:31:09.489 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.359353+0000", 2024-09-21T09:31:09.489 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.359353+0000", 2024-09-21T09:31:09.489 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:09.489 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:09.489 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:09.489 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.359353+0000", 2024-09-21T09:31:09.489 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.359353+0000", 2024-09-21T09:31:09.489 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:09.489 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 2, 2024-09-21T09:31:09.489 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:09.489 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:09.490 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 51, 2024-09-21T09:31:09.490 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:09.490 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:09.490 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:09.490 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:09.490 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 2, 2024-09-21T09:31:09.490 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:09.490 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:09.490 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:09.490 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "2.1d", 2024-09-21T09:31:09.490 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:09.490 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:09.490 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:09.491 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 26, 2024-09-21T09:31:09.491 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:09.491 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T17:11:17.381607+0000", 2024-09-21T09:31:09.491 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:09.491 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:09.491 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:09.491 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:09.491 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:09.491 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:09.491 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:09.491 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:09.491 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:09.491 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:09.491 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:09.491 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:09.492 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:09.492 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:09.492 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:09.492 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:09.492 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:09.492 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:09.492 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:09.492 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 3, 2024-09-21T09:31:09.492 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 1, 2024-09-21T09:31:09.492 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:09.492 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 1, 2024-09-21T09:31:09.492 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:09.492 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:09.492 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:09.493 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:09.493 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:09.493 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 1, 2024-09-21T09:31:09.493 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:09.493 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:09.493 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:09.493 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:09.493 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:09.493 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:09.493 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:09.493 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:09.493 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:09.493 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:09.493 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:09.494 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:09.494 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 3, 2024-09-21T09:31:09.494 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:09.494 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.494 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:09.494 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:09.494 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:09.494 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:09.494 INFO:tasks.workunit.client.0.smithi022.stdout: 6, 2024-09-21T09:31:09.494 INFO:tasks.workunit.client.0.smithi022.stdout: 0 2024-09-21T09:31:09.494 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.494 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 7, 2024-09-21T09:31:09.494 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "57'2" 2024-09-21T09:31:09.495 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.495 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:09.495 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:09.495 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:09.495 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:09.495 INFO:tasks.workunit.client.0.smithi022.stdout: 2 2024-09-21T09:31:09.495 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.495 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 0, 2024-09-21T09:31:09.495 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:09.495 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:09.495 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:09.495 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:09.495 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:09.496 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:09.496 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.141385+0000", 2024-09-21T09:31:09.496 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.646141+0000", 2024-09-21T09:31:09.496 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.646141+0000", 2024-09-21T09:31:09.496 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.646662+0000", 2024-09-21T09:31:09.496 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.141385+0000", 2024-09-21T09:31:09.496 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:09.496 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:09.496 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:09.496 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:09.496 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.141385+0000", 2024-09-21T09:31:09.496 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.141385+0000", 2024-09-21T09:31:09.496 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.141385+0000", 2024-09-21T09:31:09.496 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:09.497 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:09.497 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:09.497 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.141385+0000", 2024-09-21T09:31:09.497 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.141385+0000", 2024-09-21T09:31:09.497 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:09.497 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:09.497 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:09.497 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:09.497 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:09.497 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:09.497 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:09.497 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:09.498 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:09.498 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:09.498 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:09.498 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:09.498 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:09.498 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.1f", 2024-09-21T09:31:09.498 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:09.498 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:09.498 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:09.498 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:09.498 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:09.498 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T19:22:15.834922+0000", 2024-09-21T09:31:09.498 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:09.498 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:09.499 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:09.499 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:09.499 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:09.499 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:09.499 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:09.499 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:09.499 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:09.499 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:09.499 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:09.499 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:09.499 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:09.499 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:09.499 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:09.500 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:09.500 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:09.500 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:09.500 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:09.500 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:09.500 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:09.500 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:09.500 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:09.500 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:09.500 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:09.500 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:09.501 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:09.501 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:09.501 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:09.501 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:09.501 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:09.501 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:09.501 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:09.501 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:09.501 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:09.502 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:09.502 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:09.502 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:09.502 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:09.502 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:09.502 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:09.502 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:09.502 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:09.502 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.502 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:09.502 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:09.502 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:09.502 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:09.503 INFO:tasks.workunit.client.0.smithi022.stdout: 5, 2024-09-21T09:31:09.503 INFO:tasks.workunit.client.0.smithi022.stdout: 2 2024-09-21T09:31:09.503 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.503 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 0, 2024-09-21T09:31:09.503 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:09.503 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.503 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:09.503 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:09.503 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:09.503 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:09.503 INFO:tasks.workunit.client.0.smithi022.stdout: 5 2024-09-21T09:31:09.503 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.503 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 3, 2024-09-21T09:31:09.504 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:09.504 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:09.504 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 51, 2024-09-21T09:31:09.504 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:09.504 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:09.504 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:09.504 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.125593+0000", 2024-09-21T09:31:09.504 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:32.269813+0000", 2024-09-21T09:31:09.504 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:32.269813+0000", 2024-09-21T09:31:09.504 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:32.270481+0000", 2024-09-21T09:31:09.504 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.125593+0000", 2024-09-21T09:31:09.504 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:09.504 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:09.504 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:09.505 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 52, 2024-09-21T09:31:09.505 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.125593+0000", 2024-09-21T09:31:09.505 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.125593+0000", 2024-09-21T09:31:09.505 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.125593+0000", 2024-09-21T09:31:09.505 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:09.505 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:09.505 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:09.505 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.125593+0000", 2024-09-21T09:31:09.505 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.125593+0000", 2024-09-21T09:31:09.505 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:09.505 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 1, 2024-09-21T09:31:09.505 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:09.505 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:09.505 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 51, 2024-09-21T09:31:09.506 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:09.506 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:09.506 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:09.506 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:09.506 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 1, 2024-09-21T09:31:09.506 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:09.506 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:09.506 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:09.506 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "2.1e", 2024-09-21T09:31:09.506 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:09.506 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:09.506 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:09.506 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 25, 2024-09-21T09:31:09.506 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:09.507 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T15:28:41.266917+0000", 2024-09-21T09:31:09.507 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:09.507 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:09.507 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:09.507 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:09.507 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:09.507 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:09.507 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:09.507 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:09.507 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:09.507 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:09.507 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:09.507 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:09.507 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:09.507 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:09.508 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:09.508 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:09.508 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:09.508 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:09.508 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:09.508 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:09.508 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:09.508 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:09.508 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:09.508 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:09.508 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:09.508 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:09.508 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:09.508 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:09.508 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:09.509 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:09.509 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:09.509 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:09.509 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:09.509 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:09.509 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:09.509 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:09.509 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:09.509 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:09.509 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:09.509 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:09.509 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:09.509 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:09.510 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:09.510 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.510 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:09.510 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:09.510 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:09.510 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:09.510 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:09.510 INFO:tasks.workunit.client.0.smithi022.stdout: 5 2024-09-21T09:31:09.510 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.510 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 3, 2024-09-21T09:31:09.510 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "57'1" 2024-09-21T09:31:09.510 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.510 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:09.510 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:09.511 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:09.511 INFO:tasks.workunit.client.0.smithi022.stdout: 6, 2024-09-21T09:31:09.511 INFO:tasks.workunit.client.0.smithi022.stdout: 2 2024-09-21T09:31:09.511 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.511 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 3, 2024-09-21T09:31:09.511 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:09.511 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:09.511 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 54, 2024-09-21T09:31:09.511 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:09.511 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:09.511 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:09.511 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.125527+0000", 2024-09-21T09:31:09.511 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:35.636625+0000", 2024-09-21T09:31:09.512 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:35.636625+0000", 2024-09-21T09:31:09.512 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:35.636905+0000", 2024-09-21T09:31:09.512 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.125527+0000", 2024-09-21T09:31:09.512 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:09.512 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:09.512 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:09.512 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 55, 2024-09-21T09:31:09.512 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.125527+0000", 2024-09-21T09:31:09.512 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.125527+0000", 2024-09-21T09:31:09.512 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.125527+0000", 2024-09-21T09:31:09.512 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:09.512 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:09.512 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:34.609070+0000", 2024-09-21T09:31:09.513 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.125527+0000", 2024-09-21T09:31:09.513 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.125527+0000", 2024-09-21T09:31:09.513 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:09.513 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:09.513 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:09.513 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:09.513 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 54, 2024-09-21T09:31:09.513 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:09.513 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:09.513 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:09.513 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:09.513 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:09.513 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:09.513 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:09.514 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:09.514 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "3.1e", 2024-09-21T09:31:09.514 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:09.514 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:09.514 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:09.514 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 21, 2024-09-21T09:31:09.514 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:09.514 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T12:19:39.022799+0000", 2024-09-21T09:31:09.514 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:09.514 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:09.514 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:09.514 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:09.514 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:09.514 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:09.515 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:09.515 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:09.515 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:09.515 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:09.515 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:09.515 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:09.515 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:09.515 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:09.515 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:09.515 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:09.515 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:09.515 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:09.516 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:09.516 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:09.516 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:09.516 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:09.516 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:09.516 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:09.516 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:09.516 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:09.516 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:09.516 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:09.516 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:09.516 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:09.516 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:09.516 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:09.517 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:09.517 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:09.517 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:09.517 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:09.517 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:09.517 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:09.517 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:09.517 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:09.517 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:09.517 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:09.517 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:09.517 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.517 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:09.517 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:09.517 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:09.518 INFO:tasks.workunit.client.0.smithi022.stdout: 3, 2024-09-21T09:31:09.518 INFO:tasks.workunit.client.0.smithi022.stdout: 6, 2024-09-21T09:31:09.518 INFO:tasks.workunit.client.0.smithi022.stdout: 2 2024-09-21T09:31:09.518 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.518 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 3, 2024-09-21T09:31:09.518 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "0'0" 2024-09-21T09:31:09.518 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.518 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:09.518 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": [ 2024-09-21T09:31:09.518 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:09.518 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:09.518 INFO:tasks.workunit.client.0.smithi022.stdout: 4 2024-09-21T09:31:09.518 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.518 INFO:tasks.workunit.client.0.smithi022.stdout: "acting_primary": 0, 2024-09-21T09:31:09.519 INFO:tasks.workunit.client.0.smithi022.stdout: "avail_no_missing": [], 2024-09-21T09:31:09.519 INFO:tasks.workunit.client.0.smithi022.stdout: "blocked_by": [], 2024-09-21T09:31:09.519 INFO:tasks.workunit.client.0.smithi022.stdout: "created": 51, 2024-09-21T09:31:09.519 INFO:tasks.workunit.client.0.smithi022.stdout: "dirty_stats_invalid": false, 2024-09-21T09:31:09.519 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:31:09.519 INFO:tasks.workunit.client.0.smithi022.stdout: "hitset_stats_invalid": false, 2024-09-21T09:31:09.519 INFO:tasks.workunit.client.0.smithi022.stdout: "last_active": "2024-09-21T09:28:33.141529+0000", 2024-09-21T09:31:09.519 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_active": "2024-09-21T09:27:32.267583+0000", 2024-09-21T09:31:09.519 INFO:tasks.workunit.client.0.smithi022.stdout: "last_became_peered": "2024-09-21T09:27:32.267583+0000", 2024-09-21T09:31:09.519 INFO:tasks.workunit.client.0.smithi022.stdout: "last_change": "2024-09-21T09:27:32.267938+0000", 2024-09-21T09:31:09.519 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean": "2024-09-21T09:28:33.141529+0000", 2024-09-21T09:31:09.519 INFO:tasks.workunit.client.0.smithi022.stdout: "last_clean_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:09.519 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:31:09.519 INFO:tasks.workunit.client.0.smithi022.stdout: "last_deep_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:09.520 INFO:tasks.workunit.client.0.smithi022.stdout: "last_epoch_clean": 52, 2024-09-21T09:31:09.520 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fresh": "2024-09-21T09:28:33.141529+0000", 2024-09-21T09:31:09.520 INFO:tasks.workunit.client.0.smithi022.stdout: "last_fullsized": "2024-09-21T09:28:33.141529+0000", 2024-09-21T09:31:09.520 INFO:tasks.workunit.client.0.smithi022.stdout: "last_peered": "2024-09-21T09:28:33.141529+0000", 2024-09-21T09:31:09.520 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub": "0'0", 2024-09-21T09:31:09.520 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_duration": 0, 2024-09-21T09:31:09.520 INFO:tasks.workunit.client.0.smithi022.stdout: "last_scrub_stamp": "2024-09-21T09:27:31.247150+0000", 2024-09-21T09:31:09.520 INFO:tasks.workunit.client.0.smithi022.stdout: "last_undegraded": "2024-09-21T09:28:33.141529+0000", 2024-09-21T09:31:09.520 INFO:tasks.workunit.client.0.smithi022.stdout: "last_unstale": "2024-09-21T09:28:33.141529+0000", 2024-09-21T09:31:09.520 INFO:tasks.workunit.client.0.smithi022.stdout: "log_dups_size": 0, 2024-09-21T09:31:09.520 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 4, 2024-09-21T09:31:09.520 INFO:tasks.workunit.client.0.smithi022.stdout: "log_start": "0'0", 2024-09-21T09:31:09.520 INFO:tasks.workunit.client.0.smithi022.stdout: "manifest_stats_invalid": false, 2024-09-21T09:31:09.520 INFO:tasks.workunit.client.0.smithi022.stdout: "mapping_epoch": 51, 2024-09-21T09:31:09.520 INFO:tasks.workunit.client.0.smithi022.stdout: "object_location_counts": [], 2024-09-21T09:31:09.521 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_scrubbed": 0, 2024-09-21T09:31:09.521 INFO:tasks.workunit.client.0.smithi022.stdout: "objects_trimmed": 0, 2024-09-21T09:31:09.521 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_stats_invalid": false, 2024-09-21T09:31:09.521 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 4, 2024-09-21T09:31:09.521 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:31:09.521 INFO:tasks.workunit.client.0.smithi022.stdout: "parent": "0.0", 2024-09-21T09:31:09.521 INFO:tasks.workunit.client.0.smithi022.stdout: "parent_split_bits": 0, 2024-09-21T09:31:09.521 INFO:tasks.workunit.client.0.smithi022.stdout: "pgid": "2.1f", 2024-09-21T09:31:09.521 INFO:tasks.workunit.client.0.smithi022.stdout: "pin_stats_invalid": false, 2024-09-21T09:31:09.521 INFO:tasks.workunit.client.0.smithi022.stdout: "purged_snaps": [], 2024-09-21T09:31:09.521 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_epoch": 66, 2024-09-21T09:31:09.521 INFO:tasks.workunit.client.0.smithi022.stdout: "reported_seq": 28, 2024-09-21T09:31:09.521 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_duration": 0.0, 2024-09-21T09:31:09.521 INFO:tasks.workunit.client.0.smithi022.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T12:35:00.591469+0000", 2024-09-21T09:31:09.522 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:31:09.522 INFO:tasks.workunit.client.0.smithi022.stdout: "snaptrimq_len": 0, 2024-09-21T09:31:09.522 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:09.522 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 570, 2024-09-21T09:31:09.522 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:09.522 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:09.522 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:09.522 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:09.522 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:09.522 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:09.522 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:09.522 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:09.522 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:09.523 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:09.523 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:09.523 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:09.523 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:09.523 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:09.523 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:09.523 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 6, 2024-09-21T09:31:09.523 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 2, 2024-09-21T09:31:09.523 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:09.523 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 2, 2024-09-21T09:31:09.523 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:09.523 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:09.523 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:09.523 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:09.524 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:09.524 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 1, 2024-09-21T09:31:09.524 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:09.524 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:09.524 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:09.524 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:09.524 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:09.524 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:09.524 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:09.524 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:09.524 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:09.524 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:09.524 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:09.524 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:09.525 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 4, 2024-09-21T09:31:09.525 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 1 2024-09-21T09:31:09.525 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.525 INFO:tasks.workunit.client.0.smithi022.stdout: "state": "active+clean", 2024-09-21T09:31:09.525 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_invalid": false, 2024-09-21T09:31:09.525 INFO:tasks.workunit.client.0.smithi022.stdout: "up": [ 2024-09-21T09:31:09.525 INFO:tasks.workunit.client.0.smithi022.stdout: 0, 2024-09-21T09:31:09.525 INFO:tasks.workunit.client.0.smithi022.stdout: 7, 2024-09-21T09:31:09.525 INFO:tasks.workunit.client.0.smithi022.stdout: 4 2024-09-21T09:31:09.525 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.525 INFO:tasks.workunit.client.0.smithi022.stdout: "up_primary": 0, 2024-09-21T09:31:09.525 INFO:tasks.workunit.client.0.smithi022.stdout: "version": "57'4" 2024-09-21T09:31:09.525 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:09.525 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.526 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_per_pool": [ 2024-09-21T09:31:09.526 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:09.526 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": 24, 2024-09-21T09:31:09.526 INFO:tasks.workunit.client.0.smithi022.stdout: "application": [ 2024-09-21T09:31:09.526 INFO:tasks.workunit.client.0.smithi022.stdout: "rbd" 2024-09-21T09:31:09.526 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.526 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 3, 2024-09-21T09:31:09.526 INFO:tasks.workunit.client.0.smithi022.stdout: "num_pg": 8, 2024-09-21T09:31:09.526 INFO:tasks.workunit.client.0.smithi022.stdout: "num_store_stats": 8, 2024-09-21T09:31:09.526 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 3, 2024-09-21T09:31:09.526 INFO:tasks.workunit.client.0.smithi022.stdout: "poolid": 4, 2024-09-21T09:31:09.526 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:09.526 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 19, 2024-09-21T09:31:09.526 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:09.527 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:09.527 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:09.527 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:09.527 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:09.527 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:09.527 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:09.527 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:09.527 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:09.527 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:09.527 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:09.527 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:09.527 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:09.527 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:09.527 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:09.528 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 6, 2024-09-21T09:31:09.528 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 2, 2024-09-21T09:31:09.528 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:09.528 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 2, 2024-09-21T09:31:09.528 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:09.528 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:09.528 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:09.528 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:09.528 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:09.528 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:09.528 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:09.528 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:09.528 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:09.528 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:09.529 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:09.529 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:09.529 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:09.529 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:09.529 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:09.529 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:09.529 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:09.529 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:09.529 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 2, 2024-09-21T09:31:09.529 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 2 2024-09-21T09:31:09.529 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.529 INFO:tasks.workunit.client.0.smithi022.stdout: "store_stats": { 2024-09-21T09:31:09.529 INFO:tasks.workunit.client.0.smithi022.stdout: "allocated": 12288, 2024-09-21T09:31:09.529 INFO:tasks.workunit.client.0.smithi022.stdout: "available": 0, 2024-09-21T09:31:09.530 INFO:tasks.workunit.client.0.smithi022.stdout: "data_compressed": 0, 2024-09-21T09:31:09.530 INFO:tasks.workunit.client.0.smithi022.stdout: "data_compressed_allocated": 0, 2024-09-21T09:31:09.530 INFO:tasks.workunit.client.0.smithi022.stdout: "data_compressed_original": 0, 2024-09-21T09:31:09.530 INFO:tasks.workunit.client.0.smithi022.stdout: "data_stored": 57, 2024-09-21T09:31:09.530 INFO:tasks.workunit.client.0.smithi022.stdout: "internal_metadata": 0, 2024-09-21T09:31:09.530 INFO:tasks.workunit.client.0.smithi022.stdout: "internally_reserved": 0, 2024-09-21T09:31:09.530 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_allocated": 0, 2024-09-21T09:31:09.530 INFO:tasks.workunit.client.0.smithi022.stdout: "total": 0 2024-09-21T09:31:09.530 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.530 INFO:tasks.workunit.client.0.smithi022.stdout: "up": 24 2024-09-21T09:31:09.530 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.530 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:09.530 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": 192, 2024-09-21T09:31:09.530 INFO:tasks.workunit.client.0.smithi022.stdout: "application": [ 2024-09-21T09:31:09.531 INFO:tasks.workunit.client.0.smithi022.stdout: "cephfs" 2024-09-21T09:31:09.531 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.531 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 0, 2024-09-21T09:31:09.531 INFO:tasks.workunit.client.0.smithi022.stdout: "num_pg": 64, 2024-09-21T09:31:09.531 INFO:tasks.workunit.client.0.smithi022.stdout: "num_store_stats": 8, 2024-09-21T09:31:09.531 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 0, 2024-09-21T09:31:09.531 INFO:tasks.workunit.client.0.smithi022.stdout: "poolid": 3, 2024-09-21T09:31:09.531 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:09.531 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 0, 2024-09-21T09:31:09.531 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:09.531 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:09.531 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:09.531 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:09.531 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:09.532 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:09.532 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:09.532 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:09.532 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:09.532 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:09.532 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:09.532 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:09.532 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:09.532 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:09.532 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:09.532 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 0, 2024-09-21T09:31:09.532 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 0, 2024-09-21T09:31:09.532 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:09.532 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 0, 2024-09-21T09:31:09.532 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:09.533 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:09.533 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:09.533 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:09.533 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:09.533 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:09.533 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:09.533 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:09.533 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:09.533 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:09.533 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:09.533 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:09.533 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:09.533 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:09.533 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:09.534 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:09.534 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:09.534 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:09.534 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 0, 2024-09-21T09:31:09.534 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 0 2024-09-21T09:31:09.534 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.534 INFO:tasks.workunit.client.0.smithi022.stdout: "store_stats": { 2024-09-21T09:31:09.534 INFO:tasks.workunit.client.0.smithi022.stdout: "allocated": 0, 2024-09-21T09:31:09.534 INFO:tasks.workunit.client.0.smithi022.stdout: "available": 0, 2024-09-21T09:31:09.534 INFO:tasks.workunit.client.0.smithi022.stdout: "data_compressed": 0, 2024-09-21T09:31:09.534 INFO:tasks.workunit.client.0.smithi022.stdout: "data_compressed_allocated": 0, 2024-09-21T09:31:09.534 INFO:tasks.workunit.client.0.smithi022.stdout: "data_compressed_original": 0, 2024-09-21T09:31:09.534 INFO:tasks.workunit.client.0.smithi022.stdout: "data_stored": 0, 2024-09-21T09:31:09.534 INFO:tasks.workunit.client.0.smithi022.stdout: "internal_metadata": 0, 2024-09-21T09:31:09.535 INFO:tasks.workunit.client.0.smithi022.stdout: "internally_reserved": 0, 2024-09-21T09:31:09.535 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_allocated": 0, 2024-09-21T09:31:09.535 INFO:tasks.workunit.client.0.smithi022.stdout: "total": 0 2024-09-21T09:31:09.535 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.535 INFO:tasks.workunit.client.0.smithi022.stdout: "up": 192 2024-09-21T09:31:09.535 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.535 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:09.535 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": 96, 2024-09-21T09:31:09.535 INFO:tasks.workunit.client.0.smithi022.stdout: "application": [ 2024-09-21T09:31:09.535 INFO:tasks.workunit.client.0.smithi022.stdout: "cephfs" 2024-09-21T09:31:09.535 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.535 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 45, 2024-09-21T09:31:09.535 INFO:tasks.workunit.client.0.smithi022.stdout: "num_pg": 32, 2024-09-21T09:31:09.535 INFO:tasks.workunit.client.0.smithi022.stdout: "num_store_stats": 8, 2024-09-21T09:31:09.536 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 45, 2024-09-21T09:31:09.536 INFO:tasks.workunit.client.0.smithi022.stdout: "poolid": 2, 2024-09-21T09:31:09.536 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:09.536 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 3410, 2024-09-21T09:31:09.536 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:09.536 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 0, 2024-09-21T09:31:09.536 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:09.536 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:09.536 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:09.536 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:09.536 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:09.536 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:09.536 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:09.536 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:09.536 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:09.537 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:09.537 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:09.537 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:09.537 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:09.537 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 66, 2024-09-21T09:31:09.537 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 22, 2024-09-21T09:31:09.537 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:09.537 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 22, 2024-09-21T09:31:09.537 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:09.537 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:09.537 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:09.537 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:09.537 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:09.537 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 14, 2024-09-21T09:31:09.538 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:09.538 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 0, 2024-09-21T09:31:09.538 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:09.538 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:09.538 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:09.538 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:09.538 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:09.538 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 0, 2024-09-21T09:31:09.538 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 0, 2024-09-21T09:31:09.538 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:09.538 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:09.538 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:09.538 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 51, 2024-09-21T09:31:09.538 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 19 2024-09-21T09:31:09.538 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.539 INFO:tasks.workunit.client.0.smithi022.stdout: "store_stats": { 2024-09-21T09:31:09.539 INFO:tasks.workunit.client.0.smithi022.stdout: "allocated": 98304, 2024-09-21T09:31:09.539 INFO:tasks.workunit.client.0.smithi022.stdout: "available": 0, 2024-09-21T09:31:09.539 INFO:tasks.workunit.client.0.smithi022.stdout: "data_compressed": 0, 2024-09-21T09:31:09.539 INFO:tasks.workunit.client.0.smithi022.stdout: "data_compressed_allocated": 0, 2024-09-21T09:31:09.539 INFO:tasks.workunit.client.0.smithi022.stdout: "data_compressed_original": 0, 2024-09-21T09:31:09.539 INFO:tasks.workunit.client.0.smithi022.stdout: "data_stored": 10230, 2024-09-21T09:31:09.539 INFO:tasks.workunit.client.0.smithi022.stdout: "internal_metadata": 0, 2024-09-21T09:31:09.539 INFO:tasks.workunit.client.0.smithi022.stdout: "internally_reserved": 0, 2024-09-21T09:31:09.539 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_allocated": 0, 2024-09-21T09:31:09.539 INFO:tasks.workunit.client.0.smithi022.stdout: "total": 0 2024-09-21T09:31:09.539 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.539 INFO:tasks.workunit.client.0.smithi022.stdout: "up": 96 2024-09-21T09:31:09.539 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.540 INFO:tasks.workunit.client.0.smithi022.stdout: { 2024-09-21T09:31:09.540 INFO:tasks.workunit.client.0.smithi022.stdout: "acting": 3, 2024-09-21T09:31:09.540 INFO:tasks.workunit.client.0.smithi022.stdout: "application": [ 2024-09-21T09:31:09.540 INFO:tasks.workunit.client.0.smithi022.stdout: "mgr" 2024-09-21T09:31:09.540 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.540 INFO:tasks.workunit.client.0.smithi022.stdout: "log_size": 79, 2024-09-21T09:31:09.540 INFO:tasks.workunit.client.0.smithi022.stdout: "num_pg": 1, 2024-09-21T09:31:09.540 INFO:tasks.workunit.client.0.smithi022.stdout: "num_store_stats": 3, 2024-09-21T09:31:09.540 INFO:tasks.workunit.client.0.smithi022.stdout: "ondisk_log_size": 79, 2024-09-21T09:31:09.540 INFO:tasks.workunit.client.0.smithi022.stdout: "poolid": 1, 2024-09-21T09:31:09.540 INFO:tasks.workunit.client.0.smithi022.stdout: "stat_sum": { 2024-09-21T09:31:09.540 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes": 590368, 2024-09-21T09:31:09.540 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:31:09.540 INFO:tasks.workunit.client.0.smithi022.stdout: "num_bytes_recovered": 590368, 2024-09-21T09:31:09.541 INFO:tasks.workunit.client.0.smithi022.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:31:09.541 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict": 0, 2024-09-21T09:31:09.541 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_kb": 0, 2024-09-21T09:31:09.541 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_full": 0, 2024-09-21T09:31:09.541 INFO:tasks.workunit.client.0.smithi022.stdout: "num_evict_mode_some": 0, 2024-09-21T09:31:09.541 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush": 0, 2024-09-21T09:31:09.541 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_kb": 0, 2024-09-21T09:31:09.541 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_high": 0, 2024-09-21T09:31:09.541 INFO:tasks.workunit.client.0.smithi022.stdout: "num_flush_mode_low": 0, 2024-09-21T09:31:09.541 INFO:tasks.workunit.client.0.smithi022.stdout: "num_keys_recovered": 0, 2024-09-21T09:31:09.541 INFO:tasks.workunit.client.0.smithi022.stdout: "num_large_omap_objects": 0, 2024-09-21T09:31:09.541 INFO:tasks.workunit.client.0.smithi022.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:31:09.541 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_clones": 0, 2024-09-21T09:31:09.541 INFO:tasks.workunit.client.0.smithi022.stdout: "num_object_copies": 6, 2024-09-21T09:31:09.541 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects": 2, 2024-09-21T09:31:09.542 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_degraded": 0, 2024-09-21T09:31:09.542 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_dirty": 2, 2024-09-21T09:31:09.542 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:31:09.542 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_manifest": 0, 2024-09-21T09:31:09.542 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_misplaced": 0, 2024-09-21T09:31:09.542 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing": 0, 2024-09-21T09:31:09.542 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:31:09.542 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_omap": 0, 2024-09-21T09:31:09.542 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_pinned": 0, 2024-09-21T09:31:09.542 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_recovered": 2, 2024-09-21T09:31:09.542 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_repaired": 0, 2024-09-21T09:31:09.542 INFO:tasks.workunit.client.0.smithi022.stdout: "num_objects_unfound": 0, 2024-09-21T09:31:09.542 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_bytes": 0, 2024-09-21T09:31:09.542 INFO:tasks.workunit.client.0.smithi022.stdout: "num_omap_keys": 0, 2024-09-21T09:31:09.542 INFO:tasks.workunit.client.0.smithi022.stdout: "num_promote": 0, 2024-09-21T09:31:09.543 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read": 126, 2024-09-21T09:31:09.543 INFO:tasks.workunit.client.0.smithi022.stdout: "num_read_kb": 231, 2024-09-21T09:31:09.543 INFO:tasks.workunit.client.0.smithi022.stdout: "num_scrub_errors": 0, 2024-09-21T09:31:09.543 INFO:tasks.workunit.client.0.smithi022.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:31:09.543 INFO:tasks.workunit.client.0.smithi022.stdout: "num_whiteouts": 0, 2024-09-21T09:31:09.543 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write": 113, 2024-09-21T09:31:09.543 INFO:tasks.workunit.client.0.smithi022.stdout: "num_write_kb": 1628 2024-09-21T09:31:09.543 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.543 INFO:tasks.workunit.client.0.smithi022.stdout: "store_stats": { 2024-09-21T09:31:09.543 INFO:tasks.workunit.client.0.smithi022.stdout: "allocated": 1781760, 2024-09-21T09:31:09.543 INFO:tasks.workunit.client.0.smithi022.stdout: "available": 0, 2024-09-21T09:31:09.543 INFO:tasks.workunit.client.0.smithi022.stdout: "data_compressed": 0, 2024-09-21T09:31:09.543 INFO:tasks.workunit.client.0.smithi022.stdout: "data_compressed_allocated": 0, 2024-09-21T09:31:09.543 INFO:tasks.workunit.client.0.smithi022.stdout: "data_compressed_original": 0, 2024-09-21T09:31:09.543 INFO:tasks.workunit.client.0.smithi022.stdout: "data_stored": 1771104, 2024-09-21T09:31:09.544 INFO:tasks.workunit.client.0.smithi022.stdout: "internal_metadata": 0, 2024-09-21T09:31:09.544 INFO:tasks.workunit.client.0.smithi022.stdout: "internally_reserved": 0, 2024-09-21T09:31:09.544 INFO:tasks.workunit.client.0.smithi022.stdout: "omap_allocated": 0, 2024-09-21T09:31:09.544 INFO:tasks.workunit.client.0.smithi022.stdout: "total": 0 2024-09-21T09:31:09.544 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.544 INFO:tasks.workunit.client.0.smithi022.stdout: "up": 3 2024-09-21T09:31:09.544 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:09.544 INFO:tasks.workunit.client.0.smithi022.stdout: ], 2024-09-21T09:31:09.544 INFO:tasks.workunit.client.0.smithi022.stdout: "usage": { 2024-09-21T09:31:09.544 INFO:tasks.workunit.client.0.smithi022.stdout: "pg_num": 105, 2024-09-21T09:31:09.544 INFO:tasks.workunit.client.0.smithi022.stdout: "pools": 4, 2024-09-21T09:31:09.544 INFO:tasks.workunit.client.0.smithi022.stdout: "stats_by_class": { 2024-09-21T09:31:09.544 INFO:tasks.workunit.client.0.smithi022.stdout: "ssd": { 2024-09-21T09:31:09.544 INFO:tasks.workunit.client.0.smithi022.stdout: "total_avail_bytes": 765516976128, 2024-09-21T09:31:09.544 INFO:tasks.workunit.client.0.smithi022.stdout: "total_bytes": 767960285184, 2024-09-21T09:31:09.545 INFO:tasks.workunit.client.0.smithi022.stdout: "total_used_bytes": 2443309056, 2024-09-21T09:31:09.545 INFO:tasks.workunit.client.0.smithi022.stdout: "total_used_raw_bytes": 2443309056, 2024-09-21T09:31:09.545 INFO:tasks.workunit.client.0.smithi022.stdout: "total_used_raw_ratio": 0.003181556472554803 2024-09-21T09:31:09.545 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:09.545 INFO:tasks.workunit.client.0.smithi022.stdout: }, 2024-09-21T09:31:09.545 INFO:tasks.workunit.client.0.smithi022.stdout: "total_avail_bytes": 765516976128, 2024-09-21T09:31:09.545 INFO:tasks.workunit.client.0.smithi022.stdout: "total_bytes": 767960285184, 2024-09-21T09:31:09.545 INFO:tasks.workunit.client.0.smithi022.stdout: "total_used_bytes": 2443309056 2024-09-21T09:31:09.545 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:09.545 INFO:tasks.workunit.client.0.smithi022.stdout: } 2024-09-21T09:31:09.545 INFO:tasks.workunit.client.0.smithi022.stdout:} 2024-09-21T09:31:09.545 INFO:tasks.workunit.client.0.smithi022.stdout:OK 2024-09-21T09:31:09.546 INFO:teuthology.orchestra.run:Running command with timeout 3600 2024-09-21T09:31:09.546 DEBUG:teuthology.orchestra.run.smithi022:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2024-09-21T09:31:09.546 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[39525]: from='client.14871 -' entity='client.admin' cmd=[{"prefix": "telemetry show-device", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:31:09.546 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[39525]: from='client.14871 -' entity='client.admin' cmd=[{"prefix": "telemetry show-device", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:31:09.547 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[39525]: pgmap v90: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:31:09.547 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[39525]: from='client.24782 -' entity='client.admin' cmd=[{"prefix": "telemetry show-all", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:31:09.547 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:31:09.547 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-21T09:31:09.547 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:31:09.547 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:09.547 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:09.547 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:31:09.547 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:09.547 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:09.547 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:31:09.547 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:09.547 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:09.548 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[39525]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:31:09.548 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[39525]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:31:09.548 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:31:09.548 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:09.548 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:09.548 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[39525]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:31:09.548 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[39525]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:31:09.548 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:31:09.548 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:09.548 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:09.548 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[39525]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:31:09.548 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[39525]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:31:09.549 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[27763]: from='client.14871 -' entity='client.admin' cmd=[{"prefix": "telemetry show-device", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:31:09.549 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[27763]: from='client.14871 -' entity='client.admin' cmd=[{"prefix": "telemetry show-device", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:31:09.549 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[27763]: pgmap v90: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:31:09.549 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[27763]: from='client.24782 -' entity='client.admin' cmd=[{"prefix": "telemetry show-all", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:31:09.549 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:31:09.549 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-21T09:31:09.549 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:31:09.549 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:09.549 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:09.549 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:31:09.549 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:09.549 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:09.549 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:31:09.550 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:09.550 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:09.550 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[27763]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:31:09.550 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[27763]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:31:09.550 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:31:09.550 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:09.550 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:09.550 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[27763]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:31:09.550 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[27763]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:31:09.550 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:31:09.550 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:09.550 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:09.550 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[27763]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:31:09.550 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[27763]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:31:09.551 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:08 smithi064 ceph-mon[31124]: from='client.24782 -' entity='client.admin' cmd=[{"prefix": "telemetry show-all", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:31:09.551 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:31:09.551 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:09.551 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:09.551 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:31:09.551 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-21T09:31:09.551 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:31:09.551 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:09.552 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:09.552 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:31:09.552 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:09.552 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:09.552 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:31:09.552 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:09.552 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:09.552 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[39525]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:31:09.552 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[39525]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:31:09.552 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:31:09.552 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:09.552 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:09.552 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[39525]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:31:09.552 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[39525]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:31:09.553 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:31:09.553 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:09.553 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:09.553 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[39525]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:31:09.553 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[39525]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:31:09.553 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:31:09.553 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:09.553 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:09.553 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:31:09.553 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:09.553 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:09.553 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:31:09.554 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-21T09:31:09.554 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:31:09.554 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:09.554 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:09.554 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:31:09.554 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:09.554 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:09.554 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:31:09.554 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:09.554 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:09.554 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[27763]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:31:09.554 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[27763]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:31:09.554 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:31:09.554 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:09.555 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:09.555 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[27763]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:31:09.555 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[27763]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:31:09.555 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:31:09.555 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:09.555 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:09.555 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[27763]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:31:09.555 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[27763]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:31:09.555 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:31:09.555 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:09.555 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:07 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:09.555 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:08 smithi022 ceph-mon[39525]: from='client.24782 -' entity='client.admin' cmd=[{"prefix": "telemetry show-all", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:31:09.556 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:08 smithi022 ceph-mon[27763]: from='client.24782 -' entity='client.admin' cmd=[{"prefix": "telemetry show-all", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:31:09.589 INFO:tasks.workunit:Stopping ['test_telemetry_quincy.sh'] on client.0... 2024-09-21T09:31:09.590 DEBUG:teuthology.orchestra.run.smithi022:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2024-09-21T09:31:09.920 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:09 smithi022 ceph-mon[39525]: pgmap v91: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 33 KiB/s rd, 0 B/s wr, 33 op/s 2024-09-21T09:31:09.921 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:09 smithi022 ceph-mon[27763]: pgmap v91: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 33 KiB/s rd, 0 B/s wr, 33 op/s 2024-09-21T09:31:09.922 DEBUG:teuthology.parallel:result is None 2024-09-21T09:31:09.922 DEBUG:teuthology.orchestra.run.smithi022:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2024-09-21T09:31:09.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:09 smithi064 ceph-mon[31124]: pgmap v91: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 33 KiB/s rd, 0 B/s wr, 33 op/s 2024-09-21T09:31:09.948 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2024-09-21T09:31:09.948 DEBUG:teuthology.orchestra.run.smithi022:> rmdir -- /home/ubuntu/cephtest/mnt.0 2024-09-21T09:31:10.017 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2024-09-21T09:31:10.017 INFO:teuthology.run_tasks:Running task print... 2024-09-21T09:31:10.027 INFO:teuthology.task.print:**** done end telemetry quincy... 2024-09-21T09:31:10.027 INFO:teuthology.run_tasks:Running task print... 2024-09-21T09:31:10.035 INFO:teuthology.task.print:**** done start parallel 2024-09-21T09:31:10.035 INFO:teuthology.run_tasks:Running task parallel... 2024-09-21T09:31:10.045 INFO:teuthology.task.parallel:starting parallel... 2024-09-21T09:31:10.045 INFO:teuthology.task.parallel:In parallel, running task full_sequential... 2024-09-21T09:31:10.046 INFO:teuthology.task.full_sequential:In full_sequential, running task print... 2024-09-21T09:31:10.046 INFO:teuthology.task.print:**** done start ec-rados-default.yaml 2024-09-21T09:31:10.046 INFO:teuthology.task.full_sequential:In full_sequential, running task rados... 2024-09-21T09:31:10.051 INFO:tasks.rados:Beginning rados... 2024-09-21T09:31:10.051 INFO:tasks.rados:config is {'clients': ['client.0'], 'ec_pool': True, 'objects': 50, 'op_weights': {'append': 100, 'copy_from': 50, 'delete': 50, 'read': 100, 'rmattr': 25, 'rollback': 50, 'setattr': 25, 'snap_create': 50, 'snap_remove': 50, 'write': 0}, 'ops': 4000, 'write_append_excl': False} 2024-09-21T09:31:10.051 INFO:tasks.rados:overrides is {'admin_socket': {'branch': 'main'}, 'ceph': {'conf': {'global': {'mon election default strategy': 1}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}}, 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(POOL_APP_NOT_ENABLED\\)', 'OSD_DOWN', 'mons down', 'mon down', 'MON_DOWN', 'out of quorum', 'PG_DEGRADED', 'Reduced data availability', 'Degraded data redundancy', 'OSDMAP_FLAGS', 'mons down', 'mon down', 'MON_DOWN', 'out of quorum', 'PG_AVAILABILITY'], 'sha1': '62da54b4b83a96ca9976b7e1d0c8272a564a2208'}, 'ceph-deploy': {'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {}}}, 'install': {'ceph': {'extra_system_packages': ['python3-pytest'], 'flavor': 'default', 'sha1': '62da54b4b83a96ca9976b7e1d0c8272a564a2208'}}, 'selinux': {'allowlist': ['scontext=system_u:system_r:logrotate_t:s0']}, 'workunit': {'branch': 'main', 'sha1': '62da54b4b83a96ca9976b7e1d0c8272a564a2208'}} 2024-09-21T09:31:10.051 INFO:tasks.rados:config is {'clients': ['client.0'], 'ec_pool': True, 'objects': 50, 'op_weights': {'append': 100, 'copy_from': 50, 'delete': 50, 'read': 100, 'rmattr': 25, 'rollback': 50, 'setattr': 25, 'snap_create': 50, 'snap_remove': 50, 'write': 0}, 'ops': 4000, 'write_append_excl': False} 2024-09-21T09:31:10.051 INFO:tasks.rados:joining rados 2024-09-21T09:31:10.051 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-09-21T09:31:10.052 INFO:teuthology.task.sequential:In sequential, running task print... 2024-09-21T09:31:10.052 INFO:teuthology.task.print:**** done start upgrade, wait 2024-09-21T09:31:10.052 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-09-21T09:31:10.052 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi022.front.sepia.ceph.com 2024-09-21T09:31:10.053 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d65e801e-77fa-11ef-baf6-efdc52797490 -e sha1=62da54b4b83a96ca9976b7e1d0c8272a564a2208 -- bash -c 'ceph config set global log_to_journald false --force' 2024-09-21T09:31:10.053 INFO:tasks.rados:clients are ['client.0', 'client.1'] 2024-09-21T09:31:10.053 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid d65e801e-77fa-11ef-baf6-efdc52797490 -- ceph osd erasure-code-profile set teuthologyprofile k=2 m=1 crush-failure-domain=osd 2024-09-21T09:31:10.330 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:31:10.338 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:31:10.941 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:31:10 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[33213]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:31:10] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-21T09:31:11.519 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:31:11 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:31:11] "GET /metrics HTTP/1.1" 200 236430 "" "Prometheus/2.43.0" 2024-09-21T09:31:12.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:11 smithi022 ceph-mon[39525]: pgmap v92: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 33 KiB/s rd, 0 B/s wr, 33 op/s 2024-09-21T09:31:12.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:11 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:31:12.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:11 smithi022 ceph-mon[27763]: pgmap v92: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 33 KiB/s rd, 0 B/s wr, 33 op/s 2024-09-21T09:31:12.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:11 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:31:12.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:11 smithi064 ceph-mon[31124]: pgmap v92: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 33 KiB/s rd, 0 B/s wr, 33 op/s 2024-09-21T09:31:12.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:11 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:31:14.189 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:13 smithi022 ceph-mon[39525]: pgmap v93: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 33 KiB/s rd, 0 B/s wr, 33 op/s 2024-09-21T09:31:14.189 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:13 smithi022 ceph-mon[39525]: from='client.? 172.21.15.22:0/2626941413' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "teuthologyprofile", "profile": ["k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2024-09-21T09:31:14.189 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:13 smithi022 ceph-mon[39525]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "teuthologyprofile", "profile": ["k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2024-09-21T09:31:14.189 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:13 smithi022 ceph-mon[27763]: pgmap v93: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 33 KiB/s rd, 0 B/s wr, 33 op/s 2024-09-21T09:31:14.189 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:13 smithi022 ceph-mon[27763]: from='client.? 172.21.15.22:0/2626941413' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "teuthologyprofile", "profile": ["k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2024-09-21T09:31:14.189 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:13 smithi022 ceph-mon[27763]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "teuthologyprofile", "profile": ["k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2024-09-21T09:31:14.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:13 smithi064 ceph-mon[31124]: pgmap v93: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 33 KiB/s rd, 0 B/s wr, 33 op/s 2024-09-21T09:31:14.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:13 smithi064 ceph-mon[31124]: from='client.? 172.21.15.22:0/2626941413' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "teuthologyprofile", "profile": ["k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2024-09-21T09:31:14.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:13 smithi064 ceph-mon[31124]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "teuthologyprofile", "profile": ["k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2024-09-21T09:31:14.212 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d65e801e-77fa-11ef-baf6-efdc52797490 -e sha1=62da54b4b83a96ca9976b7e1d0c8272a564a2208 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-09-21T09:31:14.825 INFO:tasks.rados:starting run 0 out of 1 2024-09-21T09:31:14.883 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_0 2024-09-21T09:31:14.883 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid d65e801e-77fa-11ef-baf6-efdc52797490 -- ceph osd pool create unique_pool_0 16 16 erasure teuthologyprofile 2024-09-21T09:31:14.905 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:31:15.057 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:14 smithi022 ceph-mon[39525]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "teuthologyprofile", "profile": ["k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2024-09-21T09:31:15.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:14 smithi022 ceph-mon[39525]: osdmap e68: 8 total, 8 up, 8 in 2024-09-21T09:31:15.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:14 smithi022 ceph-mon[27763]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "teuthologyprofile", "profile": ["k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2024-09-21T09:31:15.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:14 smithi022 ceph-mon[27763]: osdmap e68: 8 total, 8 up, 8 in 2024-09-21T09:31:15.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:14 smithi064 ceph-mon[31124]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "teuthologyprofile", "profile": ["k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2024-09-21T09:31:15.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:14 smithi064 ceph-mon[31124]: osdmap e68: 8 total, 8 up, 8 in 2024-09-21T09:31:15.631 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:31:16.009 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:15 smithi022 ceph-mon[39525]: pgmap v95: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 40 KiB/s rd, 0 B/s wr, 39 op/s 2024-09-21T09:31:16.009 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:15 smithi022 ceph-mon[27763]: pgmap v95: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 40 KiB/s rd, 0 B/s wr, 39 op/s 2024-09-21T09:31:16.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:15 smithi064 ceph-mon[31124]: pgmap v95: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 40 KiB/s rd, 0 B/s wr, 39 op/s 2024-09-21T09:31:18.339 INFO:teuthology.orchestra.run.smithi022.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:31:18.670 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:18 smithi022 ceph-mon[39525]: pgmap v96: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 40 KiB/s rd, 0 B/s wr, 39 op/s 2024-09-21T09:31:18.670 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:18 smithi022 ceph-mon[39525]: from='client.? 172.21.15.22:0/2570053552' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16, "pgp_num": 16, "pool_type": "erasure", "erasure_code_profile": "teuthologyprofile"}]: dispatch 2024-09-21T09:31:18.670 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:18 smithi022 ceph-mon[27763]: pgmap v96: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 40 KiB/s rd, 0 B/s wr, 39 op/s 2024-09-21T09:31:18.670 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:18 smithi022 ceph-mon[27763]: from='client.? 172.21.15.22:0/2570053552' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16, "pgp_num": 16, "pool_type": "erasure", "erasure_code_profile": "teuthologyprofile"}]: dispatch 2024-09-21T09:31:18.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:18 smithi064 ceph-mon[31124]: pgmap v96: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 40 KiB/s rd, 0 B/s wr, 39 op/s 2024-09-21T09:31:18.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:18 smithi064 ceph-mon[31124]: from='client.? 172.21.15.22:0/2570053552' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16, "pgp_num": 16, "pool_type": "erasure", "erasure_code_profile": "teuthologyprofile"}]: dispatch 2024-09-21T09:31:18.902 INFO:teuthology.orchestra.run.smithi022.stderr:pool 'unique_pool_0' created 2024-09-21T09:31:19.012 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d65e801e-77fa-11ef-baf6-efdc52797490 -e sha1=62da54b4b83a96ca9976b7e1d0c8272a564a2208 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-09-21T09:31:19.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:19 smithi022 ceph-mon[27763]: from='client.24818 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:31:19.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:19 smithi022 ceph-mon[27763]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:31:19.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:19 smithi022 ceph-mon[27763]: osdmap e69: 8 total, 8 up, 8 in 2024-09-21T09:31:19.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:19 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:31:19.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:19 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:31:19.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:19 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:19.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:19 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:31:19.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:19 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:31:19.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:19 smithi022 ceph-mon[27763]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:31:19.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:19 smithi022 ceph-mon[27763]: pgmap v98: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:31:19.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:19 smithi022 ceph-mon[27763]: from='client.? 172.21.15.22:0/2570053552' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16, "pgp_num": 16, "pool_type": "erasure", "erasure_code_profile": "teuthologyprofile"}]': finished 2024-09-21T09:31:19.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:19 smithi022 ceph-mon[27763]: osdmap e70: 8 total, 8 up, 8 in 2024-09-21T09:31:19.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:19 smithi022 ceph-mon[39525]: from='client.24818 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:31:19.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:19 smithi022 ceph-mon[39525]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:31:19.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:19 smithi022 ceph-mon[39525]: osdmap e69: 8 total, 8 up, 8 in 2024-09-21T09:31:19.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:19 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:31:19.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:19 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:31:19.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:19 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:19.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:19 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:31:19.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:19 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:31:19.560 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:19 smithi022 ceph-mon[39525]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:31:19.560 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:19 smithi022 ceph-mon[39525]: pgmap v98: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:31:19.560 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:19 smithi022 ceph-mon[39525]: from='client.? 172.21.15.22:0/2570053552' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16, "pgp_num": 16, "pool_type": "erasure", "erasure_code_profile": "teuthologyprofile"}]': finished 2024-09-21T09:31:19.560 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:19 smithi022 ceph-mon[39525]: osdmap e70: 8 total, 8 up, 8 in 2024-09-21T09:31:19.587 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid d65e801e-77fa-11ef-baf6-efdc52797490 -- ceph osd pool application enable unique_pool_0 rados --yes-i-really-mean-it || true 2024-09-21T09:31:19.656 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:31:19.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:19 smithi064 ceph-mon[31124]: from='client.24818 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:31:19.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:19 smithi064 ceph-mon[31124]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:31:19.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:19 smithi064 ceph-mon[31124]: osdmap e69: 8 total, 8 up, 8 in 2024-09-21T09:31:19.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:19 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:31:19.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:19 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:31:19.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:19 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:19.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:19 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:31:19.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:19 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:31:19.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:19 smithi064 ceph-mon[31124]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:31:19.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:19 smithi064 ceph-mon[31124]: pgmap v98: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:31:19.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:19 smithi064 ceph-mon[31124]: from='client.? 172.21.15.22:0/2570053552' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16, "pgp_num": 16, "pool_type": "erasure", "erasure_code_profile": "teuthologyprofile"}]': finished 2024-09-21T09:31:19.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:19 smithi064 ceph-mon[31124]: osdmap e70: 8 total, 8 up, 8 in 2024-09-21T09:31:20.514 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:31:20.904 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:20 smithi022 ceph-mon[27763]: osdmap e71: 8 total, 8 up, 8 in 2024-09-21T09:31:20.908 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:31:20 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[33213]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:31:20] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-21T09:31:21.184 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:20 smithi022 ceph-mon[39525]: osdmap e71: 8 total, 8 up, 8 in 2024-09-21T09:31:21.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:20 smithi064 ceph-mon[31124]: osdmap e71: 8 total, 8 up, 8 in 2024-09-21T09:31:21.476 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:31:21 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:31:21] "GET /metrics HTTP/1.1" 200 236470 "" "Prometheus/2.43.0" 2024-09-21T09:31:22.057 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:21 smithi022 ceph-mon[39525]: pgmap v101: 121 pgs: 16 unknown, 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:31:22.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:21 smithi022 ceph-mon[39525]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:31:22.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:21 smithi022 ceph-mon[39525]: osdmap e72: 8 total, 8 up, 8 in 2024-09-21T09:31:22.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:21 smithi022 ceph-mon[27763]: pgmap v101: 121 pgs: 16 unknown, 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:31:22.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:21 smithi022 ceph-mon[27763]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:31:22.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:21 smithi022 ceph-mon[27763]: osdmap e72: 8 total, 8 up, 8 in 2024-09-21T09:31:22.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:21 smithi064 ceph-mon[31124]: pgmap v101: 121 pgs: 16 unknown, 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:31:22.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:21 smithi064 ceph-mon[31124]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:31:22.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:21 smithi064 ceph-mon[31124]: osdmap e72: 8 total, 8 up, 8 in 2024-09-21T09:31:22.453 INFO:teuthology.orchestra.run.smithi022.stdout:true 2024-09-21T09:31:23.053 INFO:teuthology.orchestra.run.smithi022.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T09:31:23.054 INFO:teuthology.orchestra.run.smithi022.stdout:alertmanager.a smithi022 *:9093,9094 running (105s) 86s ago 2m 15.7M - 0.25.0 c8568f914cd2 42a53bb454cd 2024-09-21T09:31:23.054 INFO:teuthology.orchestra.run.smithi022.stdout:grafana.a smithi064 *:3000 running (2m) 88s ago 2m 81.2M - 9.4.7 954c08fa6188 ffde1f806d0b 2024-09-21T09:31:23.054 INFO:teuthology.orchestra.run.smithi022.stdout:mds.a smithi022 running (4m) 86s ago 4m 15.1M - 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 34e4221f3a27 2024-09-21T09:31:23.054 INFO:teuthology.orchestra.run.smithi022.stdout:mds.b smithi064 running (4m) 88s ago 4m 17.4M - 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 0f846b3ec694 2024-09-21T09:31:23.054 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.x smithi064 *:8443 running (7m) 88s ago 7m 399M - 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be c7e735bed227 2024-09-21T09:31:23.054 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.y smithi022 *:9283,8765 running (9m) 86s ago 9m 446M - 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 5bc449c0676f 2024-09-21T09:31:23.054 INFO:teuthology.orchestra.run.smithi022.stdout:mon.a smithi022 running (9m) 86s ago 9m 43.5M 2048M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 1e0a2d74e66e 2024-09-21T09:31:23.054 INFO:teuthology.orchestra.run.smithi022.stdout:mon.b smithi064 running (7m) 88s ago 7m 38.3M 2048M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be ed126533cd64 2024-09-21T09:31:23.054 INFO:teuthology.orchestra.run.smithi022.stdout:mon.c smithi022 running (7m) 86s ago 7m 39.1M 2048M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 79f307e3b64a 2024-09-21T09:31:23.054 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.a smithi022 *:9100 running (2m) 86s ago 2m 8958k - 1.5.0 0da6a335fe13 7cf4c99ebf3a 2024-09-21T09:31:23.054 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.b smithi064 *:9100 running (2m) 88s ago 2m 3414k - 1.5.0 0da6a335fe13 643a1e1aec61 2024-09-21T09:31:23.054 INFO:teuthology.orchestra.run.smithi022.stdout:osd.0 smithi022 running (7m) 86s ago 7m 44.1M 2993M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be fcd812003534 2024-09-21T09:31:23.054 INFO:teuthology.orchestra.run.smithi022.stdout:osd.1 smithi022 running (6m) 86s ago 6m 42.2M 2993M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 740462b5c304 2024-09-21T09:31:23.055 INFO:teuthology.orchestra.run.smithi022.stdout:osd.2 smithi022 running (6m) 86s ago 6m 45.3M 2993M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be fefc7c770c32 2024-09-21T09:31:23.055 INFO:teuthology.orchestra.run.smithi022.stdout:osd.3 smithi022 running (5m) 86s ago 5m 41.5M 2993M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be a56193b67a92 2024-09-21T09:31:23.055 INFO:teuthology.orchestra.run.smithi022.stdout:osd.4 smithi064 running (5m) 88s ago 5m 44.5M 2992M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 2a2937374fc4 2024-09-21T09:31:23.055 INFO:teuthology.orchestra.run.smithi022.stdout:osd.5 smithi064 running (5m) 88s ago 5m 43.1M 2992M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 1900c4af5d28 2024-09-21T09:31:23.055 INFO:teuthology.orchestra.run.smithi022.stdout:osd.6 smithi064 running (4m) 88s ago 4m 40.3M 2992M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 3cc4ec816e5b 2024-09-21T09:31:23.055 INFO:teuthology.orchestra.run.smithi022.stdout:osd.7 smithi064 running (4m) 88s ago 4m 41.5M 2992M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 18ef0be94104 2024-09-21T09:31:23.055 INFO:teuthology.orchestra.run.smithi022.stdout:prometheus.a smithi064 *:9095 running (99s) 88s ago 3m 26.0M - 2.43.0 a07b618ecd1d caee9b54c131 2024-09-21T09:31:23.410 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-09-21T09:31:23.410 INFO:teuthology.orchestra.run.smithi022.stdout: "mon": { 2024-09-21T09:31:23.410 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 3 2024-09-21T09:31:23.410 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-21T09:31:23.410 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr": { 2024-09-21T09:31:23.410 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 2 2024-09-21T09:31:23.410 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-21T09:31:23.410 INFO:teuthology.orchestra.run.smithi022.stdout: "osd": { 2024-09-21T09:31:23.410 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 8 2024-09-21T09:31:23.410 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-21T09:31:23.411 INFO:teuthology.orchestra.run.smithi022.stdout: "mds": { 2024-09-21T09:31:23.411 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 2 2024-09-21T09:31:23.411 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-21T09:31:23.411 INFO:teuthology.orchestra.run.smithi022.stdout: "overall": { 2024-09-21T09:31:23.411 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 15 2024-09-21T09:31:23.411 INFO:teuthology.orchestra.run.smithi022.stdout: } 2024-09-21T09:31:23.411 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-09-21T09:31:23.695 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-09-21T09:31:23.695 INFO:teuthology.orchestra.run.smithi022.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208", 2024-09-21T09:31:23.695 INFO:teuthology.orchestra.run.smithi022.stdout: "in_progress": true, 2024-09-21T09:31:23.695 INFO:teuthology.orchestra.run.smithi022.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T09:31:23.695 INFO:teuthology.orchestra.run.smithi022.stdout: "services_complete": [], 2024-09-21T09:31:23.695 INFO:teuthology.orchestra.run.smithi022.stdout: "progress": "", 2024-09-21T09:31:23.695 INFO:teuthology.orchestra.run.smithi022.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208 image", 2024-09-21T09:31:23.696 INFO:teuthology.orchestra.run.smithi022.stdout: "is_paused": false 2024-09-21T09:31:23.696 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-09-21T09:31:23.916 INFO:teuthology.orchestra.run.smithi022.stderr:enabled application 'rados' on pool 'unique_pool_0' 2024-09-21T09:31:24.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:23 smithi022 ceph-mon[27763]: from='client.14919 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:31:24.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:23 smithi022 ceph-mon[27763]: from='client.14925 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:31:24.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:23 smithi022 ceph-mon[27763]: pgmap v103: 121 pgs: 121 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:31:24.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:23 smithi022 ceph-mon[27763]: from='client.14931 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:31:24.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:23 smithi022 ceph-mon[27763]: from='client.? 172.21.15.22:0/2853426868' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:31:24.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:23 smithi022 ceph-mon[27763]: from='client.? 172.21.15.22:0/243473080' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-09-21T09:31:24.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:23 smithi022 ceph-mon[27763]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-09-21T09:31:24.059 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:23 smithi022 ceph-mon[39525]: from='client.14919 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:31:24.059 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:23 smithi022 ceph-mon[39525]: from='client.14925 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:31:24.059 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:23 smithi022 ceph-mon[39525]: pgmap v103: 121 pgs: 121 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:31:24.059 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:23 smithi022 ceph-mon[39525]: from='client.14931 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:31:24.059 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:23 smithi022 ceph-mon[39525]: from='client.? 172.21.15.22:0/2853426868' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:31:24.059 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:23 smithi022 ceph-mon[39525]: from='client.? 172.21.15.22:0/243473080' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-09-21T09:31:24.059 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:23 smithi022 ceph-mon[39525]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-09-21T09:31:24.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:23 smithi064 ceph-mon[31124]: from='client.14919 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:31:24.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:23 smithi064 ceph-mon[31124]: from='client.14925 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:31:24.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:23 smithi064 ceph-mon[31124]: pgmap v103: 121 pgs: 121 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:31:24.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:23 smithi064 ceph-mon[31124]: from='client.14931 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:31:24.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:23 smithi064 ceph-mon[31124]: from='client.? 172.21.15.22:0/2853426868' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:31:24.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:23 smithi064 ceph-mon[31124]: from='client.? 172.21.15.22:0/243473080' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-09-21T09:31:24.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:23 smithi064 ceph-mon[31124]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-09-21T09:31:25.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:24 smithi064 ceph-mon[31124]: from='client.24854 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:31:25.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:24 smithi064 ceph-mon[31124]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-21T09:31:25.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:24 smithi064 ceph-mon[31124]: osdmap e73: 8 total, 8 up, 8 in 2024-09-21T09:31:25.307 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:24 smithi022 ceph-mon[39525]: from='client.24854 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:31:25.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:24 smithi022 ceph-mon[39525]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-21T09:31:25.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:24 smithi022 ceph-mon[39525]: osdmap e73: 8 total, 8 up, 8 in 2024-09-21T09:31:25.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:24 smithi022 ceph-mon[27763]: from='client.24854 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:31:25.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:24 smithi022 ceph-mon[27763]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-21T09:31:25.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:24 smithi022 ceph-mon[27763]: osdmap e73: 8 total, 8 up, 8 in 2024-09-21T09:31:25.602 DEBUG:teuthology.orchestra.run.smithi022:> CEPH_CLIENT_ID=0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph_test_rados --no-omap --ec-pool --max-ops 4000 --objects 50 --max-in-flight 16 --size 4000000 --min-stride-size 400000 --max-stride-size 800000 --max-seconds 0 --op read 100 --op write 0 --op delete 50 --op snap_create 50 --op snap_remove 50 --op rollback 50 --op setattr 25 --op rmattr 25 --op copy_from 50 --op append 100 --pool unique_pool_0 2024-09-21T09:31:25.629 INFO:tasks.rados.rados.0.smithi022.stdout:adding op weight read -> 100 2024-09-21T09:31:25.630 INFO:tasks.rados.rados.0.smithi022.stdout:adding op weight delete -> 50 2024-09-21T09:31:25.630 INFO:tasks.rados.rados.0.smithi022.stdout:adding op weight snap_create -> 50 2024-09-21T09:31:25.630 INFO:tasks.rados.rados.0.smithi022.stdout:adding op weight snap_remove -> 50 2024-09-21T09:31:25.630 INFO:tasks.rados.rados.0.smithi022.stdout:adding op weight rollback -> 50 2024-09-21T09:31:25.630 INFO:tasks.rados.rados.0.smithi022.stdout:adding op weight setattr -> 25 2024-09-21T09:31:25.630 INFO:tasks.rados.rados.0.smithi022.stdout:adding op weight rmattr -> 25 2024-09-21T09:31:25.630 INFO:tasks.rados.rados.0.smithi022.stdout:adding op weight copy_from -> 50 2024-09-21T09:31:25.630 INFO:tasks.rados.rados.0.smithi022.stdout:adding op weight append -> 100 2024-09-21T09:31:25.630 INFO:tasks.rados.rados.0.smithi022.stdout:ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable) 2024-09-21T09:31:25.630 INFO:tasks.rados.rados.0.smithi022.stdout:Configuration: 2024-09-21T09:31:25.630 INFO:tasks.rados.rados.0.smithi022.stdout: Number of operations: 4000 2024-09-21T09:31:25.630 INFO:tasks.rados.rados.0.smithi022.stdout: Number of objects: 50 2024-09-21T09:31:25.630 INFO:tasks.rados.rados.0.smithi022.stdout: Max in flight operations: 16 2024-09-21T09:31:25.631 INFO:tasks.rados.rados.0.smithi022.stdout: Object size (in bytes): 4000000 2024-09-21T09:31:25.631 INFO:tasks.rados.rados.0.smithi022.stdout: Write stride min: 400000 2024-09-21T09:31:25.631 INFO:tasks.rados.rados.0.smithi022.stdout: Write stride max: 800000 2024-09-21T09:31:25.926 INFO:tasks.rados.rados.0.smithi022.stdout:1: write initial oid 1 2024-09-21T09:31:25.926 INFO:tasks.rados.rados.0.smithi022.stdout:1: seq_num 0 ranges {0=24576} 2024-09-21T09:31:25.926 INFO:tasks.rados.rados.0.smithi022.stdout:1: writing smithi02284960-1 from 0 to 24576 tid 1 2024-09-21T09:31:25.926 INFO:tasks.rados.rados.0.smithi022.stdout:2: write initial oid 2 2024-09-21T09:31:25.926 INFO:tasks.rados.rados.0.smithi022.stdout:2: seq_num 1 ranges {0=24576} 2024-09-21T09:31:25.926 INFO:tasks.rados.rados.0.smithi022.stdout:2: writing smithi02284960-2 from 0 to 24576 tid 1 2024-09-21T09:31:25.926 INFO:tasks.rados.rados.0.smithi022.stdout:3: write initial oid 3 2024-09-21T09:31:25.927 INFO:tasks.rados.rados.0.smithi022.stdout:3: seq_num 2 ranges {0=40960} 2024-09-21T09:31:25.927 INFO:tasks.rados.rados.0.smithi022.stdout:3: writing smithi02284960-3 from 0 to 40960 tid 1 2024-09-21T09:31:25.927 INFO:tasks.rados.rados.0.smithi022.stdout:4: write initial oid 4 2024-09-21T09:31:25.927 INFO:tasks.rados.rados.0.smithi022.stdout:4: seq_num 3 ranges {0=57344} 2024-09-21T09:31:25.927 INFO:tasks.rados.rados.0.smithi022.stdout:4: writing smithi02284960-4 from 0 to 57344 tid 1 2024-09-21T09:31:25.928 INFO:tasks.rados.rados.0.smithi022.stdout:5: write initial oid 5 2024-09-21T09:31:25.928 INFO:tasks.rados.rados.0.smithi022.stdout:5: seq_num 4 ranges {0=73728} 2024-09-21T09:31:25.928 INFO:tasks.rados.rados.0.smithi022.stdout:5: writing smithi02284960-5 from 0 to 73728 tid 1 2024-09-21T09:31:25.928 INFO:tasks.rados.rados.0.smithi022.stdout:6: write initial oid 6 2024-09-21T09:31:25.928 INFO:tasks.rados.rados.0.smithi022.stdout:6: seq_num 5 ranges {0=90112} 2024-09-21T09:31:25.928 INFO:tasks.rados.rados.0.smithi022.stdout:6: writing smithi02284960-6 from 0 to 90112 tid 1 2024-09-21T09:31:25.928 INFO:tasks.rados.rados.0.smithi022.stdout:7: write initial oid 7 2024-09-21T09:31:25.928 INFO:tasks.rados.rados.0.smithi022.stdout:7: seq_num 6 ranges {0=106496} 2024-09-21T09:31:25.929 INFO:tasks.rados.rados.0.smithi022.stdout:7: writing smithi02284960-7 from 0 to 106496 tid 1 2024-09-21T09:31:25.929 INFO:tasks.rados.rados.0.smithi022.stdout:8: write initial oid 8 2024-09-21T09:31:25.930 INFO:tasks.rados.rados.0.smithi022.stdout:8: seq_num 7 ranges {0=122880} 2024-09-21T09:31:25.930 INFO:tasks.rados.rados.0.smithi022.stdout:8: writing smithi02284960-8 from 0 to 122880 tid 1 2024-09-21T09:31:25.930 INFO:tasks.rados.rados.0.smithi022.stdout:1: finishing write tid 1 to smithi02284960-1 2024-09-21T09:31:25.930 INFO:tasks.rados.rados.0.smithi022.stdout:9: write initial oid 9 2024-09-21T09:31:25.930 INFO:tasks.rados.rados.0.smithi022.stdout:9: seq_num 8 ranges {0=139264} 2024-09-21T09:31:25.931 INFO:tasks.rados.rados.0.smithi022.stdout:9: writing smithi02284960-9 from 0 to 139264 tid 1 2024-09-21T09:31:25.931 INFO:tasks.rados.rados.0.smithi022.stdout:10: write initial oid 10 2024-09-21T09:31:25.931 INFO:tasks.rados.rados.0.smithi022.stdout:10: seq_num 9 ranges {0=155648} 2024-09-21T09:31:25.932 INFO:tasks.rados.rados.0.smithi022.stdout:10: writing smithi02284960-10 from 0 to 155648 tid 1 2024-09-21T09:31:25.932 INFO:tasks.rados.rados.0.smithi022.stdout:11: write initial oid 11 2024-09-21T09:31:25.932 INFO:tasks.rados.rados.0.smithi022.stdout:11: seq_num 10 ranges {0=172032} 2024-09-21T09:31:25.934 INFO:tasks.rados.rados.0.smithi022.stdout:11: writing smithi02284960-11 from 0 to 172032 tid 1 2024-09-21T09:31:25.934 INFO:tasks.rados.rados.0.smithi022.stdout:12: write initial oid 12 2024-09-21T09:31:25.934 INFO:tasks.rados.rados.0.smithi022.stdout:12: seq_num 11 ranges {0=188416} 2024-09-21T09:31:25.935 INFO:tasks.rados.rados.0.smithi022.stdout:12: writing smithi02284960-12 from 0 to 188416 tid 1 2024-09-21T09:31:25.935 INFO:tasks.rados.rados.0.smithi022.stdout:13: write initial oid 13 2024-09-21T09:31:25.935 INFO:tasks.rados.rados.0.smithi022.stdout:13: seq_num 12 ranges {0=204800} 2024-09-21T09:31:25.937 INFO:tasks.rados.rados.0.smithi022.stdout:13: writing smithi02284960-13 from 0 to 204800 tid 1 2024-09-21T09:31:25.937 INFO:tasks.rados.rados.0.smithi022.stdout:14: write initial oid 14 2024-09-21T09:31:25.937 INFO:tasks.rados.rados.0.smithi022.stdout:14: seq_num 13 ranges {0=221184} 2024-09-21T09:31:25.939 INFO:tasks.rados.rados.0.smithi022.stdout:14: writing smithi02284960-14 from 0 to 221184 tid 1 2024-09-21T09:31:25.939 INFO:tasks.rados.rados.0.smithi022.stdout:15: write initial oid 15 2024-09-21T09:31:25.939 INFO:tasks.rados.rados.0.smithi022.stdout:15: seq_num 14 ranges {0=237568} 2024-09-21T09:31:25.940 INFO:tasks.rados.rados.0.smithi022.stdout:15: writing smithi02284960-15 from 0 to 237568 tid 1 2024-09-21T09:31:25.940 INFO:tasks.rados.rados.0.smithi022.stdout:16: write initial oid 16 2024-09-21T09:31:25.940 INFO:tasks.rados.rados.0.smithi022.stdout:16: seq_num 15 ranges {0=253952} 2024-09-21T09:31:25.942 INFO:tasks.rados.rados.0.smithi022.stdout:16: writing smithi02284960-16 from 0 to 253952 tid 1 2024-09-21T09:31:25.942 INFO:tasks.rados.rados.0.smithi022.stdout: waiting on 16 2024-09-21T09:31:25.942 INFO:tasks.rados.rados.0.smithi022.stdout:2: finishing write tid 1 to smithi02284960-2 2024-09-21T09:31:25.942 INFO:tasks.rados.rados.0.smithi022.stdout:2: finishing write tid 2 to smithi02284960-2 2024-09-21T09:31:25.942 INFO:tasks.rados.rados.0.smithi022.stdout:3: finishing write tid 1 to smithi02284960-3 2024-09-21T09:31:25.943 INFO:tasks.rados.rados.0.smithi022.stdout:3: finishing write tid 2 to smithi02284960-3 2024-09-21T09:31:25.943 INFO:tasks.rados.rados.0.smithi022.stdout:6: finishing write tid 1 to smithi02284960-6 2024-09-21T09:31:25.943 INFO:tasks.rados.rados.0.smithi022.stdout:6: finishing write tid 2 to smithi02284960-6 2024-09-21T09:31:25.943 INFO:tasks.rados.rados.0.smithi022.stdout:1: finishing write tid 2 to smithi02284960-1 2024-09-21T09:31:25.943 INFO:tasks.rados.rados.0.smithi022.stdout:6: finishing write tid 3 to smithi02284960-6 2024-09-21T09:31:25.943 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 6 v 2 (ObjNum 5 snap 0 seq_num 5) dirty exists 2024-09-21T09:31:25.943 INFO:tasks.rados.rados.0.smithi022.stdout:6: left oid 6 (ObjNum 5 snap 0 seq_num 5) 2024-09-21T09:31:25.943 INFO:tasks.rados.rados.0.smithi022.stdout:2: finishing write tid 3 to smithi02284960-2 2024-09-21T09:31:25.943 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 2 v 2 (ObjNum 1 snap 0 seq_num 1) dirty exists 2024-09-21T09:31:25.943 INFO:tasks.rados.rados.0.smithi022.stdout:2: left oid 2 (ObjNum 1 snap 0 seq_num 1) 2024-09-21T09:31:25.943 INFO:tasks.rados.rados.0.smithi022.stdout:7: finishing write tid 1 to smithi02284960-7 2024-09-21T09:31:25.943 INFO:tasks.rados.rados.0.smithi022.stdout:4: finishing write tid 1 to smithi02284960-4 2024-09-21T09:31:25.943 INFO:tasks.rados.rados.0.smithi022.stdout:1: finishing write tid 3 to smithi02284960-1 2024-09-21T09:31:25.943 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 1 v 2 (ObjNum 0 snap 0 seq_num 0) dirty exists 2024-09-21T09:31:25.943 INFO:tasks.rados.rados.0.smithi022.stdout:1: left oid 1 (ObjNum 0 snap 0 seq_num 0) 2024-09-21T09:31:25.944 INFO:tasks.rados.rados.0.smithi022.stdout:4: finishing write tid 2 to smithi02284960-4 2024-09-21T09:31:25.944 INFO:tasks.rados.rados.0.smithi022.stdout:3: finishing write tid 3 to smithi02284960-3 2024-09-21T09:31:25.944 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 3 v 2 (ObjNum 2 snap 0 seq_num 2) dirty exists 2024-09-21T09:31:25.944 INFO:tasks.rados.rados.0.smithi022.stdout:3: left oid 3 (ObjNum 2 snap 0 seq_num 2) 2024-09-21T09:31:25.944 INFO:tasks.rados.rados.0.smithi022.stdout:7: finishing write tid 2 to smithi02284960-7 2024-09-21T09:31:25.944 INFO:tasks.rados.rados.0.smithi022.stdout:1: done (15 left) 2024-09-21T09:31:25.944 INFO:tasks.rados.rados.0.smithi022.stdout:2: done (14 left) 2024-09-21T09:31:25.944 INFO:tasks.rados.rados.0.smithi022.stdout:3: done (13 left) 2024-09-21T09:31:25.944 INFO:tasks.rados.rados.0.smithi022.stdout:6: done (12 left) 2024-09-21T09:31:25.944 INFO:tasks.rados.rados.0.smithi022.stdout:17: write initial oid 17 2024-09-21T09:31:25.944 INFO:tasks.rados.rados.0.smithi022.stdout:17: seq_num 16 ranges {0=270336} 2024-09-21T09:31:25.945 INFO:tasks.rados.rados.0.smithi022.stdout:17: writing smithi02284960-17 from 0 to 270336 tid 1 2024-09-21T09:31:25.945 INFO:tasks.rados.rados.0.smithi022.stdout:18: write initial oid 18 2024-09-21T09:31:25.945 INFO:tasks.rados.rados.0.smithi022.stdout:18: seq_num 17 ranges {0=286720} 2024-09-21T09:31:25.947 INFO:tasks.rados.rados.0.smithi022.stdout:18: writing smithi02284960-18 from 0 to 286720 tid 1 2024-09-21T09:31:25.947 INFO:tasks.rados.rados.0.smithi022.stdout:5: finishing write tid 1 to smithi02284960-5 2024-09-21T09:31:25.947 INFO:tasks.rados.rados.0.smithi022.stdout:5: finishing write tid 2 to smithi02284960-5 2024-09-21T09:31:25.947 INFO:tasks.rados.rados.0.smithi022.stdout:8: finishing write tid 1 to smithi02284960-8 2024-09-21T09:31:25.947 INFO:tasks.rados.rados.0.smithi022.stdout:8: finishing write tid 2 to smithi02284960-8 2024-09-21T09:31:25.947 INFO:tasks.rados.rados.0.smithi022.stdout:9: finishing write tid 1 to smithi02284960-9 2024-09-21T09:31:25.947 INFO:tasks.rados.rados.0.smithi022.stdout:10: finishing write tid 1 to smithi02284960-10 2024-09-21T09:31:25.948 INFO:tasks.rados.rados.0.smithi022.stdout:11: finishing write tid 1 to smithi02284960-11 2024-09-21T09:31:25.948 INFO:tasks.rados.rados.0.smithi022.stdout:10: finishing write tid 2 to smithi02284960-10 2024-09-21T09:31:25.948 INFO:tasks.rados.rados.0.smithi022.stdout:7: finishing write tid 3 to smithi02284960-7 2024-09-21T09:31:25.948 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 7 v 4 (ObjNum 6 snap 0 seq_num 6) dirty exists 2024-09-21T09:31:25.948 INFO:tasks.rados.rados.0.smithi022.stdout:7: left oid 7 (ObjNum 6 snap 0 seq_num 6) 2024-09-21T09:31:25.948 INFO:tasks.rados.rados.0.smithi022.stdout:11: finishing write tid 2 to smithi02284960-11 2024-09-21T09:31:25.948 INFO:tasks.rados.rados.0.smithi022.stdout:4: finishing write tid 3 to smithi02284960-4 2024-09-21T09:31:25.948 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 4 v 2 (ObjNum 3 snap 0 seq_num 3) dirty exists 2024-09-21T09:31:25.948 INFO:tasks.rados.rados.0.smithi022.stdout:4: left oid 4 (ObjNum 3 snap 0 seq_num 3) 2024-09-21T09:31:25.948 INFO:tasks.rados.rados.0.smithi022.stdout:5: finishing write tid 3 to smithi02284960-5 2024-09-21T09:31:25.948 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 5 v 4 (ObjNum 4 snap 0 seq_num 4) dirty exists 2024-09-21T09:31:25.948 INFO:tasks.rados.rados.0.smithi022.stdout:5: left oid 5 (ObjNum 4 snap 0 seq_num 4) 2024-09-21T09:31:25.948 INFO:tasks.rados.rados.0.smithi022.stdout:8: finishing write tid 3 to smithi02284960-8 2024-09-21T09:31:25.948 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 8 v 6 (ObjNum 7 snap 0 seq_num 7) dirty exists 2024-09-21T09:31:25.949 INFO:tasks.rados.rados.0.smithi022.stdout:8: left oid 8 (ObjNum 7 snap 0 seq_num 7) 2024-09-21T09:31:25.949 INFO:tasks.rados.rados.0.smithi022.stdout:9: finishing write tid 2 to smithi02284960-9 2024-09-21T09:31:25.949 INFO:tasks.rados.rados.0.smithi022.stdout:10: finishing write tid 3 to smithi02284960-10 2024-09-21T09:31:25.949 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 10 v 2 (ObjNum 9 snap 0 seq_num 9) dirty exists 2024-09-21T09:31:25.949 INFO:tasks.rados.rados.0.smithi022.stdout:10: left oid 10 (ObjNum 9 snap 0 seq_num 9) 2024-09-21T09:31:25.949 INFO:tasks.rados.rados.0.smithi022.stdout:11: finishing write tid 3 to smithi02284960-11 2024-09-21T09:31:25.949 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 11 v 4 (ObjNum 10 snap 0 seq_num 10) dirty exists 2024-09-21T09:31:25.949 INFO:tasks.rados.rados.0.smithi022.stdout:11: left oid 11 (ObjNum 10 snap 0 seq_num 10) 2024-09-21T09:31:25.949 INFO:tasks.rados.rados.0.smithi022.stdout:9: finishing write tid 3 to smithi02284960-9 2024-09-21T09:31:25.949 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 9 v 8 (ObjNum 8 snap 0 seq_num 8) dirty exists 2024-09-21T09:31:25.949 INFO:tasks.rados.rados.0.smithi022.stdout:9: left oid 9 (ObjNum 8 snap 0 seq_num 8) 2024-09-21T09:31:25.949 INFO:tasks.rados.rados.0.smithi022.stdout:12: finishing write tid 1 to smithi02284960-12 2024-09-21T09:31:25.949 INFO:tasks.rados.rados.0.smithi022.stdout:12: finishing write tid 2 to smithi02284960-12 2024-09-21T09:31:25.949 INFO:tasks.rados.rados.0.smithi022.stdout:13: finishing write tid 1 to smithi02284960-13 2024-09-21T09:31:25.949 INFO:tasks.rados.rados.0.smithi022.stdout:13: finishing write tid 2 to smithi02284960-13 2024-09-21T09:31:25.950 INFO:tasks.rados.rados.0.smithi022.stdout:12: finishing write tid 3 to smithi02284960-12 2024-09-21T09:31:25.950 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 12 v 2 (ObjNum 11 snap 0 seq_num 11) dirty exists 2024-09-21T09:31:25.950 INFO:tasks.rados.rados.0.smithi022.stdout:12: left oid 12 (ObjNum 11 snap 0 seq_num 11) 2024-09-21T09:31:25.950 INFO:tasks.rados.rados.0.smithi022.stdout:13: finishing write tid 3 to smithi02284960-13 2024-09-21T09:31:25.950 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 13 v 2 (ObjNum 12 snap 0 seq_num 12) dirty exists 2024-09-21T09:31:25.950 INFO:tasks.rados.rados.0.smithi022.stdout:13: left oid 13 (ObjNum 12 snap 0 seq_num 12) 2024-09-21T09:31:25.950 INFO:tasks.rados.rados.0.smithi022.stdout:14: finishing write tid 1 to smithi02284960-14 2024-09-21T09:31:25.950 INFO:tasks.rados.rados.0.smithi022.stdout:15: finishing write tid 1 to smithi02284960-15 2024-09-21T09:31:25.950 INFO:tasks.rados.rados.0.smithi022.stdout:14: finishing write tid 2 to smithi02284960-14 2024-09-21T09:31:25.950 INFO:tasks.rados.rados.0.smithi022.stdout:15: finishing write tid 2 to smithi02284960-15 2024-09-21T09:31:25.950 INFO:tasks.rados.rados.0.smithi022.stdout:14: finishing write tid 3 to smithi02284960-14 2024-09-21T09:31:25.950 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 14 v 2 (ObjNum 13 snap 0 seq_num 13) dirty exists 2024-09-21T09:31:25.950 INFO:tasks.rados.rados.0.smithi022.stdout:14: left oid 14 (ObjNum 13 snap 0 seq_num 13) 2024-09-21T09:31:25.950 INFO:tasks.rados.rados.0.smithi022.stdout:16: finishing write tid 1 to smithi02284960-16 2024-09-21T09:31:25.951 INFO:tasks.rados.rados.0.smithi022.stdout:16: finishing write tid 2 to smithi02284960-16 2024-09-21T09:31:25.951 INFO:tasks.rados.rados.0.smithi022.stdout:15: finishing write tid 3 to smithi02284960-15 2024-09-21T09:31:25.951 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 15 v 4 (ObjNum 14 snap 0 seq_num 14) dirty exists 2024-09-21T09:31:25.951 INFO:tasks.rados.rados.0.smithi022.stdout:15: left oid 15 (ObjNum 14 snap 0 seq_num 14) 2024-09-21T09:31:25.951 INFO:tasks.rados.rados.0.smithi022.stdout:4: done (13 left) 2024-09-21T09:31:25.951 INFO:tasks.rados.rados.0.smithi022.stdout:5: done (12 left) 2024-09-21T09:31:25.951 INFO:tasks.rados.rados.0.smithi022.stdout:7: done (11 left) 2024-09-21T09:31:25.951 INFO:tasks.rados.rados.0.smithi022.stdout:8: done (10 left) 2024-09-21T09:31:25.951 INFO:tasks.rados.rados.0.smithi022.stdout:9: done (9 left) 2024-09-21T09:31:25.951 INFO:tasks.rados.rados.0.smithi022.stdout:10: done (8 left) 2024-09-21T09:31:25.951 INFO:tasks.rados.rados.0.smithi022.stdout:11: done (7 left) 2024-09-21T09:31:25.951 INFO:tasks.rados.rados.0.smithi022.stdout:12: done (6 left) 2024-09-21T09:31:25.951 INFO:tasks.rados.rados.0.smithi022.stdout:13: done (5 left) 2024-09-21T09:31:25.951 INFO:tasks.rados.rados.0.smithi022.stdout:14: done (4 left) 2024-09-21T09:31:25.952 INFO:tasks.rados.rados.0.smithi022.stdout:15: done (3 left) 2024-09-21T09:31:25.952 INFO:tasks.rados.rados.0.smithi022.stdout:19: write initial oid 19 2024-09-21T09:31:25.952 INFO:tasks.rados.rados.0.smithi022.stdout:19: seq_num 18 ranges {0=303104} 2024-09-21T09:31:25.952 INFO:tasks.rados.rados.0.smithi022.stdout:19: writing smithi02284960-19 from 0 to 303104 tid 1 2024-09-21T09:31:25.952 INFO:tasks.rados.rados.0.smithi022.stdout:20: write initial oid 20 2024-09-21T09:31:25.953 INFO:tasks.rados.rados.0.smithi022.stdout:20: seq_num 19 ranges {0=319488} 2024-09-21T09:31:25.953 INFO:tasks.rados.rados.0.smithi022.stdout:20: writing smithi02284960-20 from 0 to 319488 tid 1 2024-09-21T09:31:25.953 INFO:tasks.rados.rados.0.smithi022.stdout:21: write initial oid 21 2024-09-21T09:31:25.953 INFO:tasks.rados.rados.0.smithi022.stdout:21: seq_num 20 ranges {0=344064} 2024-09-21T09:31:25.953 INFO:tasks.rados.rados.0.smithi022.stdout:21: writing smithi02284960-21 from 0 to 344064 tid 1 2024-09-21T09:31:25.953 INFO:tasks.rados.rados.0.smithi022.stdout:22: write initial oid 22 2024-09-21T09:31:25.953 INFO:tasks.rados.rados.0.smithi022.stdout:22: seq_num 21 ranges {0=360448} 2024-09-21T09:31:25.955 INFO:tasks.rados.rados.0.smithi022.stdout:22: writing smithi02284960-22 from 0 to 360448 tid 1 2024-09-21T09:31:25.956 INFO:tasks.rados.rados.0.smithi022.stdout:23: write initial oid 23 2024-09-21T09:31:25.956 INFO:tasks.rados.rados.0.smithi022.stdout:23: seq_num 22 ranges {0=376832} 2024-09-21T09:31:25.958 INFO:tasks.rados.rados.0.smithi022.stdout:23: writing smithi02284960-23 from 0 to 376832 tid 1 2024-09-21T09:31:25.958 INFO:tasks.rados.rados.0.smithi022.stdout:24: write initial oid 24 2024-09-21T09:31:25.958 INFO:tasks.rados.rados.0.smithi022.stdout:24: seq_num 23 ranges {0=393216} 2024-09-21T09:31:25.960 INFO:tasks.rados.rados.0.smithi022.stdout:24: writing smithi02284960-24 from 0 to 393216 tid 1 2024-09-21T09:31:25.961 INFO:tasks.rados.rados.0.smithi022.stdout:25: write initial oid 25 2024-09-21T09:31:25.961 INFO:tasks.rados.rados.0.smithi022.stdout:25: seq_num 24 ranges {0=409600} 2024-09-21T09:31:25.964 INFO:tasks.rados.rados.0.smithi022.stdout:25: writing smithi02284960-25 from 0 to 409600 tid 1 2024-09-21T09:31:25.964 INFO:tasks.rados.rados.0.smithi022.stdout:26: write initial oid 26 2024-09-21T09:31:25.964 INFO:tasks.rados.rados.0.smithi022.stdout:26: seq_num 25 ranges {0=425984} 2024-09-21T09:31:25.967 INFO:tasks.rados.rados.0.smithi022.stdout:26: writing smithi02284960-26 from 0 to 425984 tid 1 2024-09-21T09:31:25.967 INFO:tasks.rados.rados.0.smithi022.stdout:27: write initial oid 27 2024-09-21T09:31:25.967 INFO:tasks.rados.rados.0.smithi022.stdout:27: seq_num 26 ranges {0=442368} 2024-09-21T09:31:25.969 INFO:tasks.rados.rados.0.smithi022.stdout:27: writing smithi02284960-27 from 0 to 442368 tid 1 2024-09-21T09:31:25.969 INFO:tasks.rados.rados.0.smithi022.stdout:28: write initial oid 28 2024-09-21T09:31:25.969 INFO:tasks.rados.rados.0.smithi022.stdout:28: seq_num 27 ranges {0=458752} 2024-09-21T09:31:25.972 INFO:tasks.rados.rados.0.smithi022.stdout:28: writing smithi02284960-28 from 0 to 458752 tid 1 2024-09-21T09:31:25.972 INFO:tasks.rados.rados.0.smithi022.stdout:29: write initial oid 29 2024-09-21T09:31:25.972 INFO:tasks.rados.rados.0.smithi022.stdout:29: seq_num 28 ranges {0=475136} 2024-09-21T09:31:25.976 INFO:tasks.rados.rados.0.smithi022.stdout:29: writing smithi02284960-29 from 0 to 475136 tid 1 2024-09-21T09:31:25.976 INFO:tasks.rados.rados.0.smithi022.stdout:30: write initial oid 30 2024-09-21T09:31:25.976 INFO:tasks.rados.rados.0.smithi022.stdout:16: finishing write tid 3 to smithi02284960-16 2024-09-21T09:31:25.976 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 16 v 2 (ObjNum 15 snap 0 seq_num 15) dirty exists 2024-09-21T09:31:25.976 INFO:tasks.rados.rados.0.smithi022.stdout:16: left oid 16 (ObjNum 15 snap 0 seq_num 15) 2024-09-21T09:31:25.976 INFO:tasks.rados.rados.0.smithi022.stdout:17: finishing write tid 1 to smithi02284960-17 2024-09-21T09:31:25.976 INFO:tasks.rados.rados.0.smithi022.stdout:17: finishing write tid 2 to smithi02284960-17 2024-09-21T09:31:25.977 INFO:tasks.rados.rados.0.smithi022.stdout:18: finishing write tid 1 to smithi02284960-18 2024-09-21T09:31:25.977 INFO:tasks.rados.rados.0.smithi022.stdout:18: finishing write tid 2 to smithi02284960-18 2024-09-21T09:31:25.977 INFO:tasks.rados.rados.0.smithi022.stdout:17: finishing write tid 3 to smithi02284960-17 2024-09-21T09:31:25.977 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 17 v 2 (ObjNum 16 snap 0 seq_num 16) dirty exists 2024-09-21T09:31:25.977 INFO:tasks.rados.rados.0.smithi022.stdout:17: left oid 17 (ObjNum 16 snap 0 seq_num 16) 2024-09-21T09:31:25.977 INFO:tasks.rados.rados.0.smithi022.stdout:19: finishing write tid 1 to smithi02284960-19 2024-09-21T09:31:25.977 INFO:tasks.rados.rados.0.smithi022.stdout:19: finishing write tid 2 to smithi02284960-19 2024-09-21T09:31:25.977 INFO:tasks.rados.rados.0.smithi022.stdout:18: finishing write tid 3 to smithi02284960-18 2024-09-21T09:31:25.977 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 18 v 4 (ObjNum 17 snap 0 seq_num 17) dirty exists 2024-09-21T09:31:25.977 INFO:tasks.rados.rados.0.smithi022.stdout:18: left oid 18 (ObjNum 17 snap 0 seq_num 17) 2024-09-21T09:31:25.977 INFO:tasks.rados.rados.0.smithi022.stdout:19: finishing write tid 3 to smithi02284960-19 2024-09-21T09:31:25.977 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 19 v 6 (ObjNum 18 snap 0 seq_num 18) dirty exists 2024-09-21T09:31:25.977 INFO:tasks.rados.rados.0.smithi022.stdout:19: left oid 19 (ObjNum 18 snap 0 seq_num 18) 2024-09-21T09:31:25.977 INFO:tasks.rados.rados.0.smithi022.stdout:20: finishing write tid 1 to smithi02284960-20 2024-09-21T09:31:25.977 INFO:tasks.rados.rados.0.smithi022.stdout:20: finishing write tid 2 to smithi02284960-20 2024-09-21T09:31:25.978 INFO:tasks.rados.rados.0.smithi022.stdout:20: finishing write tid 3 to smithi02284960-20 2024-09-21T09:31:25.978 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 20 v 6 (ObjNum 19 snap 0 seq_num 19) dirty exists 2024-09-21T09:31:25.978 INFO:tasks.rados.rados.0.smithi022.stdout:20: left oid 20 (ObjNum 19 snap 0 seq_num 19) 2024-09-21T09:31:25.978 INFO:tasks.rados.rados.0.smithi022.stdout:21: finishing write tid 1 to smithi02284960-21 2024-09-21T09:31:25.978 INFO:tasks.rados.rados.0.smithi022.stdout:21: finishing write tid 2 to smithi02284960-21 2024-09-21T09:31:25.978 INFO:tasks.rados.rados.0.smithi022.stdout:21: finishing write tid 3 to smithi02284960-21 2024-09-21T09:31:25.978 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 21 v 4 (ObjNum 20 snap 0 seq_num 20) dirty exists 2024-09-21T09:31:25.978 INFO:tasks.rados.rados.0.smithi022.stdout:21: left oid 21 (ObjNum 20 snap 0 seq_num 20) 2024-09-21T09:31:25.978 INFO:tasks.rados.rados.0.smithi022.stdout:22: finishing write tid 1 to smithi02284960-22 2024-09-21T09:31:25.978 INFO:tasks.rados.rados.0.smithi022.stdout:22: finishing write tid 2 to smithi02284960-22 2024-09-21T09:31:25.978 INFO:tasks.rados.rados.0.smithi022.stdout:22: finishing write tid 3 to smithi02284960-22 2024-09-21T09:31:25.978 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 22 v 4 (ObjNum 21 snap 0 seq_num 21) dirty exists 2024-09-21T09:31:25.978 INFO:tasks.rados.rados.0.smithi022.stdout:22: left oid 22 (ObjNum 21 snap 0 seq_num 21) 2024-09-21T09:31:25.978 INFO:tasks.rados.rados.0.smithi022.stdout:23: finishing write tid 1 to smithi02284960-23 2024-09-21T09:31:25.979 INFO:tasks.rados.rados.0.smithi022.stdout:23: finishing write tid 2 to smithi02284960-23 2024-09-21T09:31:25.979 INFO:tasks.rados.rados.0.smithi022.stdout:23: finishing write tid 3 to smithi02284960-23 2024-09-21T09:31:25.979 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 23 v 2 (ObjNum 22 snap 0 seq_num 22) dirty exists 2024-09-21T09:31:25.979 INFO:tasks.rados.rados.0.smithi022.stdout:23: left oid 23 (ObjNum 22 snap 0 seq_num 22) 2024-09-21T09:31:25.979 INFO:tasks.rados.rados.0.smithi022.stdout:24: finishing write tid 1 to smithi02284960-24 2024-09-21T09:31:25.979 INFO:tasks.rados.rados.0.smithi022.stdout:24: finishing write tid 2 to smithi02284960-24 2024-09-21T09:31:25.979 INFO:tasks.rados.rados.0.smithi022.stdout:24: finishing write tid 3 to smithi02284960-24 2024-09-21T09:31:25.979 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 24 v 8 (ObjNum 23 snap 0 seq_num 23) dirty exists 2024-09-21T09:31:25.979 INFO:tasks.rados.rados.0.smithi022.stdout:24: left oid 24 (ObjNum 23 snap 0 seq_num 23) 2024-09-21T09:31:25.979 INFO:tasks.rados.rados.0.smithi022.stdout:25: finishing write tid 1 to smithi02284960-25 2024-09-21T09:31:25.979 INFO:tasks.rados.rados.0.smithi022.stdout:25: finishing write tid 2 to smithi02284960-25 2024-09-21T09:31:25.979 INFO:tasks.rados.rados.0.smithi022.stdout:26: finishing write tid 1 to smithi02284960-26 2024-09-21T09:31:25.979 INFO:tasks.rados.rados.0.smithi022.stdout:25: finishing write tid 3 to smithi02284960-25 2024-09-21T09:31:25.979 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 25 v 4 (ObjNum 24 snap 0 seq_num 24) dirty exists 2024-09-21T09:31:25.979 INFO:tasks.rados.rados.0.smithi022.stdout:25: left oid 25 (ObjNum 24 snap 0 seq_num 24) 2024-09-21T09:31:25.980 INFO:tasks.rados.rados.0.smithi022.stdout:26: finishing write tid 2 to smithi02284960-26 2024-09-21T09:31:25.980 INFO:tasks.rados.rados.0.smithi022.stdout:26: finishing write tid 3 to smithi02284960-26 2024-09-21T09:31:25.980 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 26 v 6 (ObjNum 25 snap 0 seq_num 25) dirty exists 2024-09-21T09:31:25.980 INFO:tasks.rados.rados.0.smithi022.stdout:26: left oid 26 (ObjNum 25 snap 0 seq_num 25) 2024-09-21T09:31:25.980 INFO:tasks.rados.rados.0.smithi022.stdout:27: finishing write tid 1 to smithi02284960-27 2024-09-21T09:31:25.980 INFO:tasks.rados.rados.0.smithi022.stdout:27: finishing write tid 2 to smithi02284960-27 2024-09-21T09:31:25.980 INFO:tasks.rados.rados.0.smithi022.stdout:28: finishing write tid 1 to smithi02284960-28 2024-09-21T09:31:25.980 INFO:tasks.rados.rados.0.smithi022.stdout:27: finishing write tid 3 to smithi02284960-27 2024-09-21T09:31:25.980 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 27 v 8 (ObjNum 26 snap 0 seq_num 26) dirty exists 2024-09-21T09:31:25.980 INFO:tasks.rados.rados.0.smithi022.stdout:27: left oid 27 (ObjNum 26 snap 0 seq_num 26) 2024-09-21T09:31:25.980 INFO:tasks.rados.rados.0.smithi022.stdout:28: finishing write tid 2 to smithi02284960-28 2024-09-21T09:31:25.980 INFO:tasks.rados.rados.0.smithi022.stdout:30: seq_num 29 ranges {0=491520} 2024-09-21T09:31:25.981 INFO:tasks.rados.rados.0.smithi022.stdout:30: writing smithi02284960-30 from 0 to 491520 tid 1 2024-09-21T09:31:25.981 INFO:tasks.rados.rados.0.smithi022.stdout:16: done (14 left) 2024-09-21T09:31:25.981 INFO:tasks.rados.rados.0.smithi022.stdout:17: done (13 left) 2024-09-21T09:31:25.981 INFO:tasks.rados.rados.0.smithi022.stdout:18: done (12 left) 2024-09-21T09:31:25.982 INFO:tasks.rados.rados.0.smithi022.stdout:19: done (11 left) 2024-09-21T09:31:25.982 INFO:tasks.rados.rados.0.smithi022.stdout:20: done (10 left) 2024-09-21T09:31:25.982 INFO:tasks.rados.rados.0.smithi022.stdout:21: done (9 left) 2024-09-21T09:31:25.982 INFO:tasks.rados.rados.0.smithi022.stdout:22: done (8 left) 2024-09-21T09:31:25.982 INFO:tasks.rados.rados.0.smithi022.stdout:23: done (7 left) 2024-09-21T09:31:25.982 INFO:tasks.rados.rados.0.smithi022.stdout:24: done (6 left) 2024-09-21T09:31:25.982 INFO:tasks.rados.rados.0.smithi022.stdout:25: done (5 left) 2024-09-21T09:31:25.982 INFO:tasks.rados.rados.0.smithi022.stdout:26: done (4 left) 2024-09-21T09:31:25.982 INFO:tasks.rados.rados.0.smithi022.stdout:27: done (3 left) 2024-09-21T09:31:25.982 INFO:tasks.rados.rados.0.smithi022.stdout:31: write initial oid 31 2024-09-21T09:31:25.982 INFO:tasks.rados.rados.0.smithi022.stdout:31: seq_num 30 ranges {0=507904} 2024-09-21T09:31:25.983 INFO:tasks.rados.rados.0.smithi022.stdout:31: writing smithi02284960-31 from 0 to 507904 tid 1 2024-09-21T09:31:25.983 INFO:tasks.rados.rados.0.smithi022.stdout:32: write initial oid 32 2024-09-21T09:31:25.983 INFO:tasks.rados.rados.0.smithi022.stdout:32: seq_num 31 ranges {0=524288} 2024-09-21T09:31:25.986 INFO:tasks.rados.rados.0.smithi022.stdout:32: writing smithi02284960-32 from 0 to 524288 tid 1 2024-09-21T09:31:25.986 INFO:tasks.rados.rados.0.smithi022.stdout:33: write initial oid 33 2024-09-21T09:31:25.986 INFO:tasks.rados.rados.0.smithi022.stdout:33: seq_num 32 ranges {0=540672} 2024-09-21T09:31:25.990 INFO:tasks.rados.rados.0.smithi022.stdout:33: writing smithi02284960-33 from 0 to 540672 tid 1 2024-09-21T09:31:25.990 INFO:tasks.rados.rados.0.smithi022.stdout:28: finishing write tid 3 to smithi02284960-28 2024-09-21T09:31:25.991 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 28 v 2 (ObjNum 27 snap 0 seq_num 27) dirty exists 2024-09-21T09:31:25.991 INFO:tasks.rados.rados.0.smithi022.stdout:28: left oid 28 (ObjNum 27 snap 0 seq_num 27) 2024-09-21T09:31:25.991 INFO:tasks.rados.rados.0.smithi022.stdout:29: finishing write tid 1 to smithi02284960-29 2024-09-21T09:31:25.991 INFO:tasks.rados.rados.0.smithi022.stdout:29: finishing write tid 2 to smithi02284960-29 2024-09-21T09:31:25.991 INFO:tasks.rados.rados.0.smithi022.stdout:29: finishing write tid 3 to smithi02284960-29 2024-09-21T09:31:25.991 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 29 v 4 (ObjNum 28 snap 0 seq_num 28) dirty exists 2024-09-21T09:31:25.991 INFO:tasks.rados.rados.0.smithi022.stdout:29: left oid 29 (ObjNum 28 snap 0 seq_num 28) 2024-09-21T09:31:25.991 INFO:tasks.rados.rados.0.smithi022.stdout:30: finishing write tid 1 to smithi02284960-30 2024-09-21T09:31:25.991 INFO:tasks.rados.rados.0.smithi022.stdout:30: finishing write tid 2 to smithi02284960-30 2024-09-21T09:31:25.991 INFO:tasks.rados.rados.0.smithi022.stdout:30: finishing write tid 3 to smithi02284960-30 2024-09-21T09:31:25.991 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 30 v 4 (ObjNum 29 snap 0 seq_num 29) dirty exists 2024-09-21T09:31:25.991 INFO:tasks.rados.rados.0.smithi022.stdout:30: left oid 30 (ObjNum 29 snap 0 seq_num 29) 2024-09-21T09:31:25.991 INFO:tasks.rados.rados.0.smithi022.stdout:31: finishing write tid 1 to smithi02284960-31 2024-09-21T09:31:25.991 INFO:tasks.rados.rados.0.smithi022.stdout:31: finishing write tid 2 to smithi02284960-31 2024-09-21T09:31:25.991 INFO:tasks.rados.rados.0.smithi022.stdout:31: finishing write tid 3 to smithi02284960-31 2024-09-21T09:31:25.991 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 31 v 10 (ObjNum 30 snap 0 seq_num 30) dirty exists 2024-09-21T09:31:25.992 INFO:tasks.rados.rados.0.smithi022.stdout:31: left oid 31 (ObjNum 30 snap 0 seq_num 30) 2024-09-21T09:31:25.992 INFO:tasks.rados.rados.0.smithi022.stdout:32: finishing write tid 1 to smithi02284960-32 2024-09-21T09:31:25.992 INFO:tasks.rados.rados.0.smithi022.stdout:32: finishing write tid 2 to smithi02284960-32 2024-09-21T09:31:25.992 INFO:tasks.rados.rados.0.smithi022.stdout:28: done (5 left) 2024-09-21T09:31:25.992 INFO:tasks.rados.rados.0.smithi022.stdout:29: done (4 left) 2024-09-21T09:31:25.992 INFO:tasks.rados.rados.0.smithi022.stdout:30: done (3 left) 2024-09-21T09:31:25.992 INFO:tasks.rados.rados.0.smithi022.stdout:31: done (2 left) 2024-09-21T09:31:25.992 INFO:tasks.rados.rados.0.smithi022.stdout:34: write initial oid 34 2024-09-21T09:31:25.992 INFO:tasks.rados.rados.0.smithi022.stdout:34: seq_num 33 ranges {0=557056} 2024-09-21T09:31:25.994 INFO:tasks.rados.rados.0.smithi022.stdout:34: writing smithi02284960-34 from 0 to 557056 tid 1 2024-09-21T09:31:25.994 INFO:tasks.rados.rados.0.smithi022.stdout:35: write initial oid 35 2024-09-21T09:31:25.994 INFO:tasks.rados.rados.0.smithi022.stdout:35: seq_num 34 ranges {0=573440} 2024-09-21T09:31:25.997 INFO:tasks.rados.rados.0.smithi022.stdout:35: writing smithi02284960-35 from 0 to 573440 tid 1 2024-09-21T09:31:25.998 INFO:tasks.rados.rados.0.smithi022.stdout:36: write initial oid 36 2024-09-21T09:31:25.998 INFO:tasks.rados.rados.0.smithi022.stdout:36: seq_num 35 ranges {0=589824} 2024-09-21T09:31:26.001 INFO:tasks.rados.rados.0.smithi022.stdout:36: writing smithi02284960-36 from 0 to 589824 tid 1 2024-09-21T09:31:26.001 INFO:tasks.rados.rados.0.smithi022.stdout:37: write initial oid 37 2024-09-21T09:31:26.001 INFO:tasks.rados.rados.0.smithi022.stdout:37: seq_num 36 ranges {0=606208} 2024-09-21T09:31:26.005 INFO:tasks.rados.rados.0.smithi022.stdout:37: writing smithi02284960-37 from 0 to 606208 tid 1 2024-09-21T09:31:26.005 INFO:tasks.rados.rados.0.smithi022.stdout:38: write initial oid 38 2024-09-21T09:31:26.005 INFO:tasks.rados.rados.0.smithi022.stdout:38: seq_num 37 ranges {0=622592} 2024-09-21T09:31:26.008 INFO:tasks.rados.rados.0.smithi022.stdout:38: writing smithi02284960-38 from 0 to 622592 tid 1 2024-09-21T09:31:26.009 INFO:tasks.rados.rados.0.smithi022.stdout:39: write initial oid 39 2024-09-21T09:31:26.009 INFO:tasks.rados.rados.0.smithi022.stdout:39: seq_num 38 ranges {0=638976} 2024-09-21T09:31:26.012 INFO:tasks.rados.rados.0.smithi022.stdout:39: writing smithi02284960-39 from 0 to 638976 tid 1 2024-09-21T09:31:26.012 INFO:tasks.rados.rados.0.smithi022.stdout:40: write initial oid 40 2024-09-21T09:31:26.012 INFO:tasks.rados.rados.0.smithi022.stdout:40: seq_num 39 ranges {0=663552} 2024-09-21T09:31:26.016 INFO:tasks.rados.rados.0.smithi022.stdout:40: writing smithi02284960-40 from 0 to 663552 tid 1 2024-09-21T09:31:26.016 INFO:tasks.rados.rados.0.smithi022.stdout:41: write initial oid 41 2024-09-21T09:31:26.016 INFO:tasks.rados.rados.0.smithi022.stdout:41: seq_num 40 ranges {0=679936} 2024-09-21T09:31:26.020 INFO:tasks.rados.rados.0.smithi022.stdout:41: writing smithi02284960-41 from 0 to 679936 tid 1 2024-09-21T09:31:26.020 INFO:tasks.rados.rados.0.smithi022.stdout:42: write initial oid 42 2024-09-21T09:31:26.020 INFO:tasks.rados.rados.0.smithi022.stdout:42: seq_num 41 ranges {0=696320} 2024-09-21T09:31:26.024 INFO:tasks.rados.rados.0.smithi022.stdout:42: writing smithi02284960-42 from 0 to 696320 tid 1 2024-09-21T09:31:26.024 INFO:tasks.rados.rados.0.smithi022.stdout:43: write initial oid 43 2024-09-21T09:31:26.024 INFO:tasks.rados.rados.0.smithi022.stdout:43: seq_num 42 ranges {0=712704} 2024-09-21T09:31:26.028 INFO:tasks.rados.rados.0.smithi022.stdout:43: writing smithi02284960-43 from 0 to 712704 tid 1 2024-09-21T09:31:26.028 INFO:tasks.rados.rados.0.smithi022.stdout:44: write initial oid 44 2024-09-21T09:31:26.028 INFO:tasks.rados.rados.0.smithi022.stdout:32: finishing write tid 3 to smithi02284960-32 2024-09-21T09:31:26.029 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 32 v 8 (ObjNum 31 snap 0 seq_num 31) dirty exists 2024-09-21T09:31:26.029 INFO:tasks.rados.rados.0.smithi022.stdout:32: left oid 32 (ObjNum 31 snap 0 seq_num 31) 2024-09-21T09:31:26.029 INFO:tasks.rados.rados.0.smithi022.stdout:44: seq_num 43 ranges {0=729088} 2024-09-21T09:31:26.032 INFO:tasks.rados.rados.0.smithi022.stdout:44: writing smithi02284960-44 from 0 to 729088 tid 1 2024-09-21T09:31:26.033 INFO:tasks.rados.rados.0.smithi022.stdout:32: done (12 left) 2024-09-21T09:31:26.033 INFO:tasks.rados.rados.0.smithi022.stdout:45: write initial oid 45 2024-09-21T09:31:26.033 INFO:tasks.rados.rados.0.smithi022.stdout:45: seq_num 44 ranges {0=745472} 2024-09-21T09:31:26.037 INFO:tasks.rados.rados.0.smithi022.stdout:45: writing smithi02284960-45 from 0 to 745472 tid 1 2024-09-21T09:31:26.037 INFO:tasks.rados.rados.0.smithi022.stdout:46: write initial oid 46 2024-09-21T09:31:26.037 INFO:tasks.rados.rados.0.smithi022.stdout:46: seq_num 45 ranges {0=761856} 2024-09-21T09:31:26.041 INFO:tasks.rados.rados.0.smithi022.stdout:46: writing smithi02284960-46 from 0 to 761856 tid 1 2024-09-21T09:31:26.042 INFO:tasks.rados.rados.0.smithi022.stdout:47: write initial oid 47 2024-09-21T09:31:26.042 INFO:tasks.rados.rados.0.smithi022.stdout:47: seq_num 46 ranges {0=778240} 2024-09-21T09:31:26.046 INFO:tasks.rados.rados.0.smithi022.stdout:47: writing smithi02284960-47 from 0 to 778240 tid 1 2024-09-21T09:31:26.046 INFO:tasks.rados.rados.0.smithi022.stdout:48: write initial oid 48 2024-09-21T09:31:26.047 INFO:tasks.rados.rados.0.smithi022.stdout:48: seq_num 47 ranges {0=794624} 2024-09-21T09:31:26.051 INFO:tasks.rados.rados.0.smithi022.stdout:48: writing smithi02284960-48 from 0 to 794624 tid 1 2024-09-21T09:31:26.052 INFO:tasks.rados.rados.0.smithi022.stdout: waiting on 16 2024-09-21T09:31:26.052 INFO:tasks.rados.rados.0.smithi022.stdout:33: finishing write tid 1 to smithi02284960-33 2024-09-21T09:31:26.052 INFO:tasks.rados.rados.0.smithi022.stdout:33: finishing write tid 2 to smithi02284960-33 2024-09-21T09:31:26.052 INFO:tasks.rados.rados.0.smithi022.stdout:33: finishing write tid 3 to smithi02284960-33 2024-09-21T09:31:26.052 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 33 v 10 (ObjNum 32 snap 0 seq_num 32) dirty exists 2024-09-21T09:31:26.052 INFO:tasks.rados.rados.0.smithi022.stdout:33: left oid 33 (ObjNum 32 snap 0 seq_num 32) 2024-09-21T09:31:26.052 INFO:tasks.rados.rados.0.smithi022.stdout:34: finishing write tid 1 to smithi02284960-34 2024-09-21T09:31:26.052 INFO:tasks.rados.rados.0.smithi022.stdout:33: done (15 left) 2024-09-21T09:31:26.052 INFO:tasks.rados.rados.0.smithi022.stdout:49: write initial oid 49 2024-09-21T09:31:26.052 INFO:tasks.rados.rados.0.smithi022.stdout:49: seq_num 48 ranges {0=409600,409600=401408} 2024-09-21T09:31:26.054 INFO:tasks.rados.rados.0.smithi022.stdout:49: writing smithi02284960-49 from 0 to 409600 tid 1 2024-09-21T09:31:26.057 INFO:tasks.rados.rados.0.smithi022.stdout:49: writing smithi02284960-49 from 409600 to 811008 tid 2 2024-09-21T09:31:26.057 INFO:tasks.rados.rados.0.smithi022.stdout: waiting on 16 2024-09-21T09:31:26.058 INFO:tasks.rados.rados.0.smithi022.stdout:34: finishing write tid 2 to smithi02284960-34 2024-09-21T09:31:26.058 INFO:tasks.rados.rados.0.smithi022.stdout:35: finishing write tid 1 to smithi02284960-35 2024-09-21T09:31:26.058 INFO:tasks.rados.rados.0.smithi022.stdout:35: finishing write tid 2 to smithi02284960-35 2024-09-21T09:31:26.058 INFO:tasks.rados.rados.0.smithi022.stdout:34: finishing write tid 3 to smithi02284960-34 2024-09-21T09:31:26.058 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 34 v 6 (ObjNum 33 snap 0 seq_num 33) dirty exists 2024-09-21T09:31:26.058 INFO:tasks.rados.rados.0.smithi022.stdout:34: left oid 34 (ObjNum 33 snap 0 seq_num 33) 2024-09-21T09:31:26.058 INFO:tasks.rados.rados.0.smithi022.stdout:35: finishing write tid 3 to smithi02284960-35 2024-09-21T09:31:26.058 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 35 v 4 (ObjNum 34 snap 0 seq_num 34) dirty exists 2024-09-21T09:31:26.058 INFO:tasks.rados.rados.0.smithi022.stdout:35: left oid 35 (ObjNum 34 snap 0 seq_num 34) 2024-09-21T09:31:26.058 INFO:tasks.rados.rados.0.smithi022.stdout:34: done (15 left) 2024-09-21T09:31:26.058 INFO:tasks.rados.rados.0.smithi022.stdout:35: done (14 left) 2024-09-21T09:31:26.058 INFO:tasks.rados.rados.0.smithi022.stdout:50: write initial oid 50 2024-09-21T09:31:26.059 INFO:tasks.rados.rados.0.smithi022.stdout:50: seq_num 49 ranges {0=425984,425984=401408} 2024-09-21T09:31:26.059 INFO:tasks.rados.rados.0.smithi022.stdout:50: writing smithi02284960-50 from 0 to 425984 tid 1 2024-09-21T09:31:26.063 INFO:tasks.rados.rados.0.smithi022.stdout:50: writing smithi02284960-50 from 425984 to 827392 tid 2 2024-09-21T09:31:26.063 INFO:tasks.rados.rados.0.smithi022.stdout:51: copy_from oid 2 from oid 11 current snap is 0 2024-09-21T09:31:26.063 INFO:tasks.rados.rados.0.smithi022.stdout:36: finishing write tid 1 to smithi02284960-36 2024-09-21T09:31:26.063 INFO:tasks.rados.rados.0.smithi022.stdout: waiting on 16 2024-09-21T09:31:26.063 INFO:tasks.rados.rados.0.smithi022.stdout:36: finishing write tid 2 to smithi02284960-36 2024-09-21T09:31:26.063 INFO:tasks.rados.rados.0.smithi022.stdout:36: finishing write tid 3 to smithi02284960-36 2024-09-21T09:31:26.064 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 36 v 2 (ObjNum 35 snap 0 seq_num 35) dirty exists 2024-09-21T09:31:26.064 INFO:tasks.rados.rados.0.smithi022.stdout:36: left oid 36 (ObjNum 35 snap 0 seq_num 35) 2024-09-21T09:31:26.064 INFO:tasks.rados.rados.0.smithi022.stdout:37: finishing write tid 1 to smithi02284960-37 2024-09-21T09:31:26.064 INFO:tasks.rados.rados.0.smithi022.stdout:37: finishing write tid 2 to smithi02284960-37 2024-09-21T09:31:26.064 INFO:tasks.rados.rados.0.smithi022.stdout:37: finishing write tid 3 to smithi02284960-37 2024-09-21T09:31:26.064 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 37 v 12 (ObjNum 36 snap 0 seq_num 36) dirty exists 2024-09-21T09:31:26.064 INFO:tasks.rados.rados.0.smithi022.stdout:37: left oid 37 (ObjNum 36 snap 0 seq_num 36) 2024-09-21T09:31:26.064 INFO:tasks.rados.rados.0.smithi022.stdout:38: finishing write tid 1 to smithi02284960-38 2024-09-21T09:31:26.064 INFO:tasks.rados.rados.0.smithi022.stdout:38: finishing write tid 2 to smithi02284960-38 2024-09-21T09:31:26.064 INFO:tasks.rados.rados.0.smithi022.stdout:38: finishing write tid 3 to smithi02284960-38 2024-09-21T09:31:26.064 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 38 v 6 (ObjNum 37 snap 0 seq_num 37) dirty exists 2024-09-21T09:31:26.064 INFO:tasks.rados.rados.0.smithi022.stdout:38: left oid 38 (ObjNum 37 snap 0 seq_num 37) 2024-09-21T09:31:26.064 INFO:tasks.rados.rados.0.smithi022.stdout:39: finishing write tid 1 to smithi02284960-39 2024-09-21T09:31:26.064 INFO:tasks.rados.rados.0.smithi022.stdout:39: finishing write tid 2 to smithi02284960-39 2024-09-21T09:31:26.065 INFO:tasks.rados.rados.0.smithi022.stdout:36: done (15 left) 2024-09-21T09:31:26.065 INFO:tasks.rados.rados.0.smithi022.stdout:37: done (14 left) 2024-09-21T09:31:26.065 INFO:tasks.rados.rados.0.smithi022.stdout:38: done (13 left) 2024-09-21T09:31:26.065 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 15 current snap is 0 2024-09-21T09:31:26.065 INFO:tasks.rados.rados.0.smithi022.stdout:39: finishing write tid 3 to smithi02284960-39 2024-09-21T09:31:26.065 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 39 v 6 (ObjNum 38 snap 0 seq_num 38) dirty exists 2024-09-21T09:31:26.065 INFO:tasks.rados.rados.0.smithi022.stdout:39: left oid 39 (ObjNum 38 snap 0 seq_num 38) 2024-09-21T09:31:26.065 INFO:tasks.rados.rados.0.smithi022.stdout:40: finishing write tid 1 to smithi02284960-40 2024-09-21T09:31:26.065 INFO:tasks.rados.rados.0.smithi022.stdout:40: finishing write tid 2 to smithi02284960-40 2024-09-21T09:31:26.065 INFO:tasks.rados.rados.0.smithi022.stdout:40: finishing write tid 3 to smithi02284960-40 2024-09-21T09:31:26.065 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 40 v 6 (ObjNum 39 snap 0 seq_num 39) dirty exists 2024-09-21T09:31:26.065 INFO:tasks.rados.rados.0.smithi022.stdout:40: left oid 40 (ObjNum 39 snap 0 seq_num 39) 2024-09-21T09:31:26.065 INFO:tasks.rados.rados.0.smithi022.stdout:41: finishing write tid 1 to smithi02284960-41 2024-09-21T09:31:26.065 INFO:tasks.rados.rados.0.smithi022.stdout:52: seq_num 50 ranges {237568=442368,679936=401408} 2024-09-21T09:31:26.066 INFO:tasks.rados.rados.0.smithi022.stdout:52: writing smithi02284960-15 from 237568 to 679936 tid 1 2024-09-21T09:31:26.070 INFO:tasks.rados.rados.0.smithi022.stdout:52: writing smithi02284960-15 from 679936 to 1081344 tid 2 2024-09-21T09:31:26.070 INFO:tasks.rados.rados.0.smithi022.stdout:39: done (13 left) 2024-09-21T09:31:26.070 INFO:tasks.rados.rados.0.smithi022.stdout:40: done (12 left) 2024-09-21T09:31:26.070 INFO:tasks.rados.rados.0.smithi022.stdout:53: rmattr oid 39 current snap is 0 2024-09-21T09:31:26.070 INFO:tasks.rados.rados.0.smithi022.stdout:53: done (12 left) 2024-09-21T09:31:26.070 INFO:tasks.rados.rados.0.smithi022.stdout:54: delete oid 40 current snap is 0 2024-09-21T09:31:26.070 INFO:tasks.rados.rados.0.smithi022.stdout:41: finishing write tid 2 to smithi02284960-41 2024-09-21T09:31:26.070 INFO:tasks.rados.rados.0.smithi022.stdout:41: finishing write tid 3 to smithi02284960-41 2024-09-21T09:31:26.070 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 41 v 8 (ObjNum 40 snap 0 seq_num 40) dirty exists 2024-09-21T09:31:26.070 INFO:tasks.rados.rados.0.smithi022.stdout:41: left oid 41 (ObjNum 40 snap 0 seq_num 40) 2024-09-21T09:31:26.071 INFO:tasks.rados.rados.0.smithi022.stdout:42: finishing write tid 1 to smithi02284960-42 2024-09-21T09:31:26.071 INFO:tasks.rados.rados.0.smithi022.stdout:42: finishing write tid 2 to smithi02284960-42 2024-09-21T09:31:26.071 INFO:tasks.rados.rados.0.smithi022.stdout:42: finishing write tid 3 to smithi02284960-42 2024-09-21T09:31:26.071 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 42 v 10 (ObjNum 41 snap 0 seq_num 41) dirty exists 2024-09-21T09:31:26.071 INFO:tasks.rados.rados.0.smithi022.stdout:42: left oid 42 (ObjNum 41 snap 0 seq_num 41) 2024-09-21T09:31:26.071 INFO:tasks.rados.rados.0.smithi022.stdout:43: finishing write tid 1 to smithi02284960-43 2024-09-21T09:31:26.071 INFO:tasks.rados.rados.0.smithi022.stdout:43: finishing write tid 2 to smithi02284960-43 2024-09-21T09:31:26.071 INFO:tasks.rados.rados.0.smithi022.stdout:43: finishing write tid 3 to smithi02284960-43 2024-09-21T09:31:26.071 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 43 v 10 (ObjNum 42 snap 0 seq_num 42) dirty exists 2024-09-21T09:31:26.071 INFO:tasks.rados.rados.0.smithi022.stdout:43: left oid 43 (ObjNum 42 snap 0 seq_num 42) 2024-09-21T09:31:26.071 INFO:tasks.rados.rados.0.smithi022.stdout:44: finishing write tid 1 to smithi02284960-44 2024-09-21T09:31:26.071 INFO:tasks.rados.rados.0.smithi022.stdout:44: finishing write tid 2 to smithi02284960-44 2024-09-21T09:31:26.071 INFO:tasks.rados.rados.0.smithi022.stdout:44: finishing write tid 3 to smithi02284960-44 2024-09-21T09:31:26.072 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 44 v 6 (ObjNum 43 snap 0 seq_num 43) dirty exists 2024-09-21T09:31:26.072 INFO:tasks.rados.rados.0.smithi022.stdout:44: left oid 44 (ObjNum 43 snap 0 seq_num 43) 2024-09-21T09:31:26.072 INFO:tasks.rados.rados.0.smithi022.stdout:45: finishing write tid 1 to smithi02284960-45 2024-09-21T09:31:26.072 INFO:tasks.rados.rados.0.smithi022.stdout:45: finishing write tid 2 to smithi02284960-45 2024-09-21T09:31:26.072 INFO:tasks.rados.rados.0.smithi022.stdout:45: finishing write tid 3 to smithi02284960-45 2024-09-21T09:31:26.072 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 45 v 4 (ObjNum 44 snap 0 seq_num 44) dirty exists 2024-09-21T09:31:26.072 INFO:tasks.rados.rados.0.smithi022.stdout:45: left oid 45 (ObjNum 44 snap 0 seq_num 44) 2024-09-21T09:31:26.072 INFO:tasks.rados.rados.0.smithi022.stdout:46: finishing write tid 1 to smithi02284960-46 2024-09-21T09:31:26.072 INFO:tasks.rados.rados.0.smithi022.stdout:46: finishing write tid 2 to smithi02284960-46 2024-09-21T09:31:26.072 INFO:tasks.rados.rados.0.smithi022.stdout:46: finishing write tid 3 to smithi02284960-46 2024-09-21T09:31:26.072 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 46 v 8 (ObjNum 45 snap 0 seq_num 45) dirty exists 2024-09-21T09:31:26.072 INFO:tasks.rados.rados.0.smithi022.stdout:46: left oid 46 (ObjNum 45 snap 0 seq_num 45) 2024-09-21T09:31:26.072 INFO:tasks.rados.rados.0.smithi022.stdout:47: finishing write tid 1 to smithi02284960-47 2024-09-21T09:31:26.073 INFO:tasks.rados.rados.0.smithi022.stdout:47: finishing write tid 2 to smithi02284960-47 2024-09-21T09:31:26.073 INFO:tasks.rados.rados.0.smithi022.stdout:47: finishing write tid 3 to smithi02284960-47 2024-09-21T09:31:26.073 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 47 v 10 (ObjNum 46 snap 0 seq_num 46) dirty exists 2024-09-21T09:31:26.073 INFO:tasks.rados.rados.0.smithi022.stdout:47: left oid 47 (ObjNum 46 snap 0 seq_num 46) 2024-09-21T09:31:26.073 INFO:tasks.rados.rados.0.smithi022.stdout:48: finishing write tid 1 to smithi02284960-48 2024-09-21T09:31:26.073 INFO:tasks.rados.rados.0.smithi022.stdout:48: finishing write tid 2 to smithi02284960-48 2024-09-21T09:31:26.073 INFO:tasks.rados.rados.0.smithi022.stdout:48: finishing write tid 3 to smithi02284960-48 2024-09-21T09:31:26.073 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 48 v 12 (ObjNum 47 snap 0 seq_num 47) dirty exists 2024-09-21T09:31:26.073 INFO:tasks.rados.rados.0.smithi022.stdout:48: left oid 48 (ObjNum 47 snap 0 seq_num 47) 2024-09-21T09:31:26.073 INFO:tasks.rados.rados.0.smithi022.stdout:49: finishing write tid 1 to smithi02284960-49 2024-09-21T09:31:26.073 INFO:tasks.rados.rados.0.smithi022.stdout:49: finishing write tid 2 to smithi02284960-49 2024-09-21T09:31:26.073 INFO:tasks.rados.rados.0.smithi022.stdout:49: finishing write tid 3 to smithi02284960-49 2024-09-21T09:31:26.073 INFO:tasks.rados.rados.0.smithi022.stdout:49: finishing write tid 4 to smithi02284960-49 2024-09-21T09:31:26.073 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 49 v 5 (ObjNum 48 snap 0 seq_num 48) dirty exists 2024-09-21T09:31:26.074 INFO:tasks.rados.rados.0.smithi022.stdout:49: left oid 49 (ObjNum 48 snap 0 seq_num 48) 2024-09-21T09:31:26.074 INFO:tasks.rados.rados.0.smithi022.stdout:50: finishing write tid 1 to smithi02284960-50 2024-09-21T09:31:26.074 INFO:tasks.rados.rados.0.smithi022.stdout:50: finishing write tid 2 to smithi02284960-50 2024-09-21T09:31:26.074 INFO:tasks.rados.rados.0.smithi022.stdout:50: finishing write tid 3 to smithi02284960-50 2024-09-21T09:31:26.074 INFO:tasks.rados.rados.0.smithi022.stdout:50: finishing write tid 4 to smithi02284960-50 2024-09-21T09:31:26.074 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 50 v 7 (ObjNum 49 snap 0 seq_num 49) dirty exists 2024-09-21T09:31:26.074 INFO:tasks.rados.rados.0.smithi022.stdout:50: left oid 50 (ObjNum 49 snap 0 seq_num 49) 2024-09-21T09:31:26.074 INFO:tasks.rados.rados.0.smithi022.stdout:51: finishing copy_from racing read to smithi02284960-2 2024-09-21T09:31:26.074 INFO:tasks.rados.rados.0.smithi022.stdout:51: finishing copy_from to smithi02284960-2 2024-09-21T09:31:26.074 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 2 v 4 (ObjNum 10 snap 0 seq_num 10) dirty exists 2024-09-21T09:31:26.074 INFO:tasks.rados.rados.0.smithi022.stdout:52: finishing write tid 1 to smithi02284960-15 2024-09-21T09:31:26.076 INFO:tasks.rados.rados.0.smithi022.stdout:41: done (12 left) 2024-09-21T09:31:26.076 INFO:tasks.rados.rados.0.smithi022.stdout:42: done (11 left) 2024-09-21T09:31:26.076 INFO:tasks.rados.rados.0.smithi022.stdout:43: done (10 left) 2024-09-21T09:31:26.076 INFO:tasks.rados.rados.0.smithi022.stdout:44: done (9 left) 2024-09-21T09:31:26.076 INFO:tasks.rados.rados.0.smithi022.stdout:45: done (8 left) 2024-09-21T09:31:26.076 INFO:tasks.rados.rados.0.smithi022.stdout:46: done (7 left) 2024-09-21T09:31:26.076 INFO:tasks.rados.rados.0.smithi022.stdout:47: done (6 left) 2024-09-21T09:31:26.076 INFO:tasks.rados.rados.0.smithi022.stdout:48: done (5 left) 2024-09-21T09:31:26.076 INFO:tasks.rados.rados.0.smithi022.stdout:49: done (4 left) 2024-09-21T09:31:26.076 INFO:tasks.rados.rados.0.smithi022.stdout:50: done (3 left) 2024-09-21T09:31:26.076 INFO:tasks.rados.rados.0.smithi022.stdout:51: done (2 left) 2024-09-21T09:31:26.076 INFO:tasks.rados.rados.0.smithi022.stdout:54: done (1 left) 2024-09-21T09:31:26.077 INFO:tasks.rados.rados.0.smithi022.stdout:55: read oid 16 snap -1 2024-09-21T09:31:26.077 INFO:tasks.rados.rados.0.smithi022.stdout:55: expect (ObjNum 15 snap 0 seq_num 15) 2024-09-21T09:31:26.077 INFO:tasks.rados.rados.0.smithi022.stdout:56: rollback oid 31 current snap is 0 2024-09-21T09:31:26.077 INFO:tasks.rados.rados.0.smithi022.stdout:56: done (2 left) 2024-09-21T09:31:26.077 INFO:tasks.rados.rados.0.smithi022.stdout:57: read oid 28 snap -1 2024-09-21T09:31:26.077 INFO:tasks.rados.rados.0.smithi022.stdout:57: expect (ObjNum 27 snap 0 seq_num 27) 2024-09-21T09:31:26.077 INFO:tasks.rados.rados.0.smithi022.stdout:58: copy_from oid 29 from oid 47 current snap is 0 2024-09-21T09:31:26.077 INFO:tasks.rados.rados.0.smithi022.stdout:59: rollback oid 45 current snap is 0 2024-09-21T09:31:26.077 INFO:tasks.rados.rados.0.smithi022.stdout:59: done (4 left) 2024-09-21T09:31:26.077 INFO:tasks.rados.rados.0.smithi022.stdout:60: copy_from oid 23 from oid 48 current snap is 0 2024-09-21T09:31:26.077 INFO:tasks.rados.rados.0.smithi022.stdout:61: delete oid 1 current snap is 0 2024-09-21T09:31:26.077 INFO:tasks.rados.rados.0.smithi022.stdout:52: finishing write tid 2 to smithi02284960-15 2024-09-21T09:31:26.078 INFO:tasks.rados.rados.0.smithi022.stdout:52: finishing write tid 3 to smithi02284960-15 2024-09-21T09:31:26.078 INFO:tasks.rados.rados.0.smithi022.stdout:52: finishing write tid 4 to smithi02284960-15 2024-09-21T09:31:26.078 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 15 v 15 (ObjNum 50 snap 0 seq_num 50) dirty exists 2024-09-21T09:31:26.078 INFO:tasks.rados.rados.0.smithi022.stdout:52: left oid 15 (ObjNum 50 snap 0 seq_num 50) 2024-09-21T09:31:26.078 INFO:tasks.rados.rados.0.smithi022.stdout:52: done (5 left) 2024-09-21T09:31:26.078 INFO:tasks.rados.rados.0.smithi022.stdout:61: done (4 left) 2024-09-21T09:31:26.078 INFO:tasks.rados.rados.0.smithi022.stdout:62: read oid 24 snap -1 2024-09-21T09:31:26.078 INFO:tasks.rados.rados.0.smithi022.stdout:62: expect (ObjNum 23 snap 0 seq_num 23) 2024-09-21T09:31:26.078 INFO:tasks.rados.rados.0.smithi022.stdout:63: snap_create 2024-09-21T09:31:26.078 INFO:tasks.rados.rados.0.smithi022.stdout:57: expect (ObjNum 27 snap 0 seq_num 27) 2024-09-21T09:31:26.088 INFO:tasks.rados.rados.0.smithi022.stdout:55: expect (ObjNum 15 snap 0 seq_num 15) 2024-09-21T09:31:26.095 INFO:tasks.rados.rados.0.smithi022.stdout:62: expect (ObjNum 23 snap 0 seq_num 23) 2024-09-21T09:31:26.106 INFO:tasks.rados.rados.0.smithi022.stdout:58: finishing copy_from racing read to smithi02284960-29 2024-09-21T09:31:26.106 INFO:tasks.rados.rados.0.smithi022.stdout:60: finishing copy_from racing read to smithi02284960-23 2024-09-21T09:31:26.106 INFO:tasks.rados.rados.0.smithi022.stdout:60: finishing copy_from to smithi02284960-23 2024-09-21T09:31:26.106 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 23 v 12 (ObjNum 47 snap 0 seq_num 47) dirty exists 2024-09-21T09:31:26.106 INFO:tasks.rados.rados.0.smithi022.stdout:58: finishing copy_from to smithi02284960-29 2024-09-21T09:31:26.106 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 29 v 10 (ObjNum 46 snap 0 seq_num 46) dirty exists 2024-09-21T09:31:26.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:25 smithi064 ceph-mon[31124]: pgmap v105: 121 pgs: 121 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:31:26.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:25 smithi064 ceph-mon[31124]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:31:26.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:25 smithi064 ceph-mon[31124]: Cluster is now healthy 2024-09-21T09:31:26.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:25 smithi064 ceph-mon[31124]: from='client.? 172.21.15.22:0/1887425279' entity='client.0' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "write_fadvise_dontneed", "val": "false"}]: dispatch 2024-09-21T09:31:26.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:25 smithi064 ceph-mon[31124]: from='client.? ' entity='client.0' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "write_fadvise_dontneed", "val": "false"}]: dispatch 2024-09-21T09:31:26.307 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:25 smithi022 ceph-mon[39525]: pgmap v105: 121 pgs: 121 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:31:26.307 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:25 smithi022 ceph-mon[39525]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:31:26.307 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:25 smithi022 ceph-mon[39525]: Cluster is now healthy 2024-09-21T09:31:26.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:25 smithi022 ceph-mon[39525]: from='client.? 172.21.15.22:0/1887425279' entity='client.0' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "write_fadvise_dontneed", "val": "false"}]: dispatch 2024-09-21T09:31:26.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:25 smithi022 ceph-mon[39525]: from='client.? ' entity='client.0' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "write_fadvise_dontneed", "val": "false"}]: dispatch 2024-09-21T09:31:26.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:25 smithi022 ceph-mon[27763]: pgmap v105: 121 pgs: 121 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:31:26.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:25 smithi022 ceph-mon[27763]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:31:26.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:25 smithi022 ceph-mon[27763]: Cluster is now healthy 2024-09-21T09:31:26.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:25 smithi022 ceph-mon[27763]: from='client.? 172.21.15.22:0/1887425279' entity='client.0' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "write_fadvise_dontneed", "val": "false"}]: dispatch 2024-09-21T09:31:26.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:25 smithi022 ceph-mon[27763]: from='client.? ' entity='client.0' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "write_fadvise_dontneed", "val": "false"}]: dispatch 2024-09-21T09:31:26.930 INFO:tasks.rados.rados.0.smithi022.stdout:55: done (5 left) 2024-09-21T09:31:26.930 INFO:tasks.rados.rados.0.smithi022.stdout:57: done (4 left) 2024-09-21T09:31:26.931 INFO:tasks.rados.rados.0.smithi022.stdout:58: done (3 left) 2024-09-21T09:31:26.931 INFO:tasks.rados.rados.0.smithi022.stdout:60: done (2 left) 2024-09-21T09:31:26.931 INFO:tasks.rados.rados.0.smithi022.stdout:62: done (1 left) 2024-09-21T09:31:26.931 INFO:tasks.rados.rados.0.smithi022.stdout:63: done (0 left) 2024-09-21T09:31:26.931 INFO:tasks.rados.rados.0.smithi022.stdout:64: copy_from oid 15 from oid 2 current snap is 1 2024-09-21T09:31:26.931 INFO:tasks.rados.rados.0.smithi022.stdout:65: read oid 30 snap -1 2024-09-21T09:31:26.931 INFO:tasks.rados.rados.0.smithi022.stdout:65: expect (ObjNum 29 snap 0 seq_num 29) 2024-09-21T09:31:26.931 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 23 current snap is 1 2024-09-21T09:31:26.931 INFO:tasks.rados.rados.0.smithi022.stdout:66: seq_num 53 ranges {794624=491520,1286144=401408} 2024-09-21T09:31:26.937 INFO:tasks.rados.rados.0.smithi022.stdout:66: writing smithi02284960-23 from 794624 to 1286144 tid 1 2024-09-21T09:31:26.940 INFO:tasks.rados.rados.0.smithi022.stdout:66: writing smithi02284960-23 from 1286144 to 1687552 tid 2 2024-09-21T09:31:26.940 INFO:tasks.rados.rados.0.smithi022.stdout:67: rollback oid 10 current snap is 1 2024-09-21T09:31:26.940 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 10 to 0 2024-09-21T09:31:26.941 INFO:tasks.rados.rados.0.smithi022.stdout:65: expect (ObjNum 29 snap 0 seq_num 29) 2024-09-21T09:31:26.958 INFO:tasks.rados.rados.0.smithi022.stdout:64: finishing copy_from racing read to smithi02284960-15 2024-09-21T09:31:26.958 INFO:tasks.rados.rados.0.smithi022.stdout:64: finishing copy_from to smithi02284960-15 2024-09-21T09:31:26.958 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 15 v 4 (ObjNum 10 snap 0 seq_num 10) dirty exists 2024-09-21T09:31:26.958 INFO:tasks.rados.rados.0.smithi022.stdout:64: done (3 left) 2024-09-21T09:31:26.958 INFO:tasks.rados.rados.0.smithi022.stdout:65: done (2 left) 2024-09-21T09:31:26.958 INFO:tasks.rados.rados.0.smithi022.stdout:68: setattr oid 20 current snap is 1 2024-09-21T09:31:26.958 INFO:tasks.rados.rados.0.smithi022.stdout:66: finishing write tid 1 to smithi02284960-23 2024-09-21T09:31:26.958 INFO:tasks.rados.rados.0.smithi022.stdout:66: finishing write tid 2 to smithi02284960-23 2024-09-21T09:31:26.958 INFO:tasks.rados.rados.0.smithi022.stdout:66: finishing write tid 3 to smithi02284960-23 2024-09-21T09:31:26.959 INFO:tasks.rados.rados.0.smithi022.stdout:66: finishing write tid 4 to smithi02284960-23 2024-09-21T09:31:26.959 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 23 v 15 (ObjNum 53 snap 1 seq_num 53) dirty exists 2024-09-21T09:31:26.959 INFO:tasks.rados.rados.0.smithi022.stdout:66: left oid 23 (ObjNum 53 snap 1 seq_num 53) 2024-09-21T09:31:26.959 INFO:tasks.rados.rados.0.smithi022.stdout:67: finishing rollback tid 0 to smithi02284960-10 2024-09-21T09:31:26.959 INFO:tasks.rados.rados.0.smithi022.stdout:67: finishing rollback tid 1 to smithi02284960-10 2024-09-21T09:31:26.959 INFO:tasks.rados.rados.0.smithi022.stdout:67: finishing rollback tid 2 to smithi02284960-10 2024-09-21T09:31:26.959 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 10 v 15 (ObjNum 9 snap 0 seq_num 9) dirty exists 2024-09-21T09:31:26.960 INFO:tasks.rados.rados.0.smithi022.stdout:66: done (2 left) 2024-09-21T09:31:26.960 INFO:tasks.rados.rados.0.smithi022.stdout:67: done (1 left) 2024-09-21T09:31:26.960 INFO:tasks.rados.rados.0.smithi022.stdout:69: copy_from oid 45 from oid 34 current snap is 1 2024-09-21T09:31:26.961 INFO:tasks.rados.rados.0.smithi022.stdout:70: read oid 24 snap 0 2024-09-21T09:31:26.961 INFO:tasks.rados.rados.0.smithi022.stdout:70: expect (ObjNum 23 snap 0 seq_num 23) 2024-09-21T09:31:26.961 INFO:tasks.rados.rados.0.smithi022.stdout:71: copy_from oid 36 from oid 28 current snap is 1 2024-09-21T09:31:26.961 INFO:tasks.rados.rados.0.smithi022.stdout:72: rollback oid 49 current snap is 1 2024-09-21T09:31:26.961 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 49 to 0 2024-09-21T09:31:26.961 INFO:tasks.rados.rados.0.smithi022.stdout:73: read oid 4 snap 0 2024-09-21T09:31:26.961 INFO:tasks.rados.rados.0.smithi022.stdout:73: expect (ObjNum 3 snap 0 seq_num 3) 2024-09-21T09:31:26.961 INFO:tasks.rados.rados.0.smithi022.stdout:74: rmattr oid 7 current snap is 1 2024-09-21T09:31:26.961 INFO:tasks.rados.rados.0.smithi022.stdout:74: done (6 left) 2024-09-21T09:31:26.961 INFO:tasks.rados.rados.0.smithi022.stdout:75: copy_from oid 21 from oid 12 current snap is 1 2024-09-21T09:31:26.961 INFO:tasks.rados.rados.0.smithi022.stdout:76: read oid 39 snap -1 2024-09-21T09:31:26.961 INFO:tasks.rados.rados.0.smithi022.stdout:76: expect (ObjNum 38 snap 0 seq_num 38) 2024-09-21T09:31:26.961 INFO:tasks.rados.rados.0.smithi022.stdout:77: copy_from oid 6 from oid 37 current snap is 1 2024-09-21T09:31:26.961 INFO:tasks.rados.rados.0.smithi022.stdout:78: read oid 2 snap -1 2024-09-21T09:31:26.962 INFO:tasks.rados.rados.0.smithi022.stdout:78: expect (ObjNum 10 snap 0 seq_num 10) 2024-09-21T09:31:26.962 INFO:tasks.rados.rados.0.smithi022.stdout:79: setattr oid 41 current snap is 1 2024-09-21T09:31:26.962 INFO:tasks.rados.rados.0.smithi022.stdout:80: snap_create 2024-09-21T09:31:26.962 INFO:tasks.rados.rados.0.smithi022.stdout:72: finishing rollback tid 0 to smithi02284960-49 2024-09-21T09:31:26.962 INFO:tasks.rados.rados.0.smithi022.stdout:78: expect (ObjNum 10 snap 0 seq_num 10) 2024-09-21T09:31:26.969 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 20 v 14 (ObjNum 19 snap 0 seq_num 19) dirty exists 2024-09-21T09:31:26.969 INFO:tasks.rados.rados.0.smithi022.stdout:73: expect (ObjNum 3 snap 0 seq_num 3) 2024-09-21T09:31:26.971 INFO:tasks.rados.rados.0.smithi022.stdout:72: finishing rollback tid 1 to smithi02284960-49 2024-09-21T09:31:26.972 INFO:tasks.rados.rados.0.smithi022.stdout:72: finishing rollback tid 2 to smithi02284960-49 2024-09-21T09:31:26.972 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 49 v 9 (ObjNum 48 snap 0 seq_num 48) dirty exists 2024-09-21T09:31:26.972 INFO:tasks.rados.rados.0.smithi022.stdout:70: expect (ObjNum 23 snap 0 seq_num 23) 2024-09-21T09:31:26.983 INFO:tasks.rados.rados.0.smithi022.stdout:76: expect (ObjNum 38 snap 0 seq_num 38) 2024-09-21T09:31:27.001 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 41 v 17 (ObjNum 40 snap 0 seq_num 40) dirty exists 2024-09-21T09:31:27.001 INFO:tasks.rados.rados.0.smithi022.stdout:71: finishing copy_from racing read to smithi02284960-36 2024-09-21T09:31:27.001 INFO:tasks.rados.rados.0.smithi022.stdout:71: finishing copy_from to smithi02284960-36 2024-09-21T09:31:27.001 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 36 v 2 (ObjNum 27 snap 0 seq_num 27) dirty exists 2024-09-21T09:31:27.001 INFO:tasks.rados.rados.0.smithi022.stdout:75: finishing copy_from racing read to smithi02284960-21 2024-09-21T09:31:27.001 INFO:tasks.rados.rados.0.smithi022.stdout:75: finishing copy_from to smithi02284960-21 2024-09-21T09:31:27.001 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 21 v 2 (ObjNum 11 snap 0 seq_num 11) dirty exists 2024-09-21T09:31:27.001 INFO:tasks.rados.rados.0.smithi022.stdout:77: finishing copy_from racing read to smithi02284960-6 2024-09-21T09:31:27.001 INFO:tasks.rados.rados.0.smithi022.stdout:77: finishing copy_from to smithi02284960-6 2024-09-21T09:31:27.001 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 6 v 12 (ObjNum 36 snap 0 seq_num 36) dirty exists 2024-09-21T09:31:27.001 INFO:tasks.rados.rados.0.smithi022.stdout:69: finishing copy_from racing read to smithi02284960-45 2024-09-21T09:31:27.002 INFO:tasks.rados.rados.0.smithi022.stdout:69: finishing copy_from to smithi02284960-45 2024-09-21T09:31:27.002 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 45 v 6 (ObjNum 33 snap 0 seq_num 33) dirty exists 2024-09-21T09:31:27.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:26 smithi064 ceph-mon[31124]: from='client.? ' entity='client.0' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "write_fadvise_dontneed", "val": "false"}]': finished 2024-09-21T09:31:27.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:26 smithi064 ceph-mon[31124]: osdmap e74: 8 total, 8 up, 8 in 2024-09-21T09:31:27.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:26 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:31:27.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:26 smithi022 ceph-mon[39525]: from='client.? ' entity='client.0' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "write_fadvise_dontneed", "val": "false"}]': finished 2024-09-21T09:31:27.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:26 smithi022 ceph-mon[39525]: osdmap e74: 8 total, 8 up, 8 in 2024-09-21T09:31:27.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:26 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:31:27.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:26 smithi022 ceph-mon[27763]: from='client.? ' entity='client.0' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "write_fadvise_dontneed", "val": "false"}]': finished 2024-09-21T09:31:27.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:26 smithi022 ceph-mon[27763]: osdmap e74: 8 total, 8 up, 8 in 2024-09-21T09:31:27.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:26 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:31:27.937 INFO:tasks.rados.rados.0.smithi022.stdout:68: done (11 left) 2024-09-21T09:31:27.937 INFO:tasks.rados.rados.0.smithi022.stdout:69: done (10 left) 2024-09-21T09:31:27.937 INFO:tasks.rados.rados.0.smithi022.stdout:70: done (9 left) 2024-09-21T09:31:27.937 INFO:tasks.rados.rados.0.smithi022.stdout:71: done (8 left) 2024-09-21T09:31:27.937 INFO:tasks.rados.rados.0.smithi022.stdout:72: done (7 left) 2024-09-21T09:31:27.937 INFO:tasks.rados.rados.0.smithi022.stdout:73: done (6 left) 2024-09-21T09:31:27.938 INFO:tasks.rados.rados.0.smithi022.stdout:75: done (5 left) 2024-09-21T09:31:27.938 INFO:tasks.rados.rados.0.smithi022.stdout:76: done (4 left) 2024-09-21T09:31:27.938 INFO:tasks.rados.rados.0.smithi022.stdout:77: done (3 left) 2024-09-21T09:31:27.938 INFO:tasks.rados.rados.0.smithi022.stdout:78: done (2 left) 2024-09-21T09:31:27.938 INFO:tasks.rados.rados.0.smithi022.stdout:79: done (1 left) 2024-09-21T09:31:27.938 INFO:tasks.rados.rados.0.smithi022.stdout:80: done (0 left) 2024-09-21T09:31:27.938 INFO:tasks.rados.rados.0.smithi022.stdout:81: rollback oid 40 current snap is 2 2024-09-21T09:31:27.938 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 40 to 1 2024-09-21T09:31:27.938 INFO:tasks.rados.rados.0.smithi022.stdout:82: snap_create 2024-09-21T09:31:27.944 INFO:tasks.rados.rados.0.smithi022.stdout:81: finishing rollback tid 1 to smithi02284960-40 2024-09-21T09:31:27.945 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 40 v 0 (ObjNum 289 snap 0 seq_num 1) dirty dne 2024-09-21T09:31:28.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:27 smithi064 ceph-mon[31124]: pgmap v107: 121 pgs: 121 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:31:28.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:27 smithi064 ceph-mon[31124]: osdmap e75: 8 total, 8 up, 8 in 2024-09-21T09:31:28.307 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:27 smithi022 ceph-mon[39525]: pgmap v107: 121 pgs: 121 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:31:28.307 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:27 smithi022 ceph-mon[39525]: osdmap e75: 8 total, 8 up, 8 in 2024-09-21T09:31:28.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:27 smithi022 ceph-mon[27763]: pgmap v107: 121 pgs: 121 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:31:28.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:27 smithi022 ceph-mon[27763]: osdmap e75: 8 total, 8 up, 8 in 2024-09-21T09:31:28.943 INFO:tasks.rados.rados.0.smithi022.stdout:81: done (1 left) 2024-09-21T09:31:28.943 INFO:tasks.rados.rados.0.smithi022.stdout:82: done (0 left) 2024-09-21T09:31:28.943 INFO:tasks.rados.rados.0.smithi022.stdout:83: rollback oid 9 current snap is 3 2024-09-21T09:31:28.943 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 9 to 0 2024-09-21T09:31:28.943 INFO:tasks.rados.rados.0.smithi022.stdout:84: read oid 20 snap 1 2024-09-21T09:31:28.944 INFO:tasks.rados.rados.0.smithi022.stdout:84: expect (ObjNum 19 snap 0 seq_num 19) 2024-09-21T09:31:28.944 INFO:tasks.rados.rados.0.smithi022.stdout:85: read oid 25 snap -1 2024-09-21T09:31:28.944 INFO:tasks.rados.rados.0.smithi022.stdout:85: expect (ObjNum 24 snap 0 seq_num 24) 2024-09-21T09:31:28.944 INFO:tasks.rados.rados.0.smithi022.stdout:86: copy_from oid 34 from oid 28 current snap is 3 2024-09-21T09:31:28.944 INFO:tasks.rados.rados.0.smithi022.stdout:87: setattr oid 48 current snap is 3 2024-09-21T09:31:28.944 INFO:tasks.rados.rados.0.smithi022.stdout:88: snap_remove snap 2 2024-09-21T09:31:29.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:28 smithi064 ceph-mon[31124]: osdmap e76: 8 total, 8 up, 8 in 2024-09-21T09:31:29.307 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:28 smithi022 ceph-mon[39525]: osdmap e76: 8 total, 8 up, 8 in 2024-09-21T09:31:29.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:28 smithi022 ceph-mon[27763]: osdmap e76: 8 total, 8 up, 8 in 2024-09-21T09:31:29.947 INFO:tasks.rados.rados.0.smithi022.stdout:88: done (5 left) 2024-09-21T09:31:29.948 INFO:tasks.rados.rados.0.smithi022.stdout:89: read oid 38 snap 1 2024-09-21T09:31:29.948 INFO:tasks.rados.rados.0.smithi022.stdout:89: expect (ObjNum 37 snap 0 seq_num 37) 2024-09-21T09:31:29.948 INFO:tasks.rados.rados.0.smithi022.stdout:90: read oid 18 snap -1 2024-09-21T09:31:29.948 INFO:tasks.rados.rados.0.smithi022.stdout:90: expect (ObjNum 17 snap 0 seq_num 17) 2024-09-21T09:31:29.948 INFO:tasks.rados.rados.0.smithi022.stdout:91: copy_from oid 2 from oid 29 current snap is 3 2024-09-21T09:31:29.948 INFO:tasks.rados.rados.0.smithi022.stdout:92: rmattr oid 43 current snap is 3 2024-09-21T09:31:29.948 INFO:tasks.rados.rados.0.smithi022.stdout:92: done (8 left) 2024-09-21T09:31:29.948 INFO:tasks.rados.rados.0.smithi022.stdout:93: delete oid 14 current snap is 3 2024-09-21T09:31:29.948 INFO:tasks.rados.rados.0.smithi022.stdout:84: expect (ObjNum 19 snap 0 seq_num 19) 2024-09-21T09:31:29.963 INFO:tasks.rados.rados.0.smithi022.stdout:85: expect (ObjNum 24 snap 0 seq_num 24) 2024-09-21T09:31:29.975 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 48 v 18 (ObjNum 47 snap 0 seq_num 47) dirty exists 2024-09-21T09:31:29.975 INFO:tasks.rados.rados.0.smithi022.stdout:83: finishing rollback tid 0 to smithi02284960-9 2024-09-21T09:31:29.975 INFO:tasks.rados.rados.0.smithi022.stdout:83: finishing rollback tid 1 to smithi02284960-9 2024-09-21T09:31:29.975 INFO:tasks.rados.rados.0.smithi022.stdout:83: finishing rollback tid 2 to smithi02284960-9 2024-09-21T09:31:29.975 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 9 v 12 (ObjNum 8 snap 0 seq_num 8) dirty exists 2024-09-21T09:31:29.975 INFO:tasks.rados.rados.0.smithi022.stdout:86: finishing copy_from racing read to smithi02284960-34 2024-09-21T09:31:29.975 INFO:tasks.rados.rados.0.smithi022.stdout:86: finishing copy_from to smithi02284960-34 2024-09-21T09:31:29.975 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 34 v 2 (ObjNum 27 snap 0 seq_num 27) dirty exists 2024-09-21T09:31:29.976 INFO:tasks.rados.rados.0.smithi022.stdout:83: done (8 left) 2024-09-21T09:31:29.976 INFO:tasks.rados.rados.0.smithi022.stdout:84: done (7 left) 2024-09-21T09:31:29.976 INFO:tasks.rados.rados.0.smithi022.stdout:85: done (6 left) 2024-09-21T09:31:29.976 INFO:tasks.rados.rados.0.smithi022.stdout:86: done (5 left) 2024-09-21T09:31:29.976 INFO:tasks.rados.rados.0.smithi022.stdout:87: done (4 left) 2024-09-21T09:31:29.976 INFO:tasks.rados.rados.0.smithi022.stdout:93: done (3 left) 2024-09-21T09:31:29.976 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 31 current snap is 3 2024-09-21T09:31:29.976 INFO:tasks.rados.rados.0.smithi022.stdout:89: expect (ObjNum 37 snap 0 seq_num 37) 2024-09-21T09:31:29.992 INFO:tasks.rados.rados.0.smithi022.stdout:90: expect (ObjNum 17 snap 0 seq_num 17) 2024-09-21T09:31:30.001 INFO:tasks.rados.rados.0.smithi022.stdout:91: finishing copy_from racing read to smithi02284960-2 2024-09-21T09:31:30.001 INFO:tasks.rados.rados.0.smithi022.stdout:91: finishing copy_from to smithi02284960-2 2024-09-21T09:31:30.002 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 2 v 10 (ObjNum 46 snap 0 seq_num 46) dirty exists 2024-09-21T09:31:30.002 INFO:tasks.rados.rados.0.smithi022.stdout:94: seq_num 55 ranges {507904=524288,1032192=401408} 2024-09-21T09:31:30.007 INFO:tasks.rados.rados.0.smithi022.stdout:94: writing smithi02284960-31 from 507904 to 1032192 tid 1 2024-09-21T09:31:30.010 INFO:tasks.rados.rados.0.smithi022.stdout:94: writing smithi02284960-31 from 1032192 to 1433600 tid 2 2024-09-21T09:31:30.010 INFO:tasks.rados.rados.0.smithi022.stdout:89: done (3 left) 2024-09-21T09:31:30.010 INFO:tasks.rados.rados.0.smithi022.stdout:90: done (2 left) 2024-09-21T09:31:30.010 INFO:tasks.rados.rados.0.smithi022.stdout:91: done (1 left) 2024-09-21T09:31:30.010 INFO:tasks.rados.rados.0.smithi022.stdout:95: read oid 13 snap 0 2024-09-21T09:31:30.010 INFO:tasks.rados.rados.0.smithi022.stdout:95: expect (ObjNum 12 snap 0 seq_num 12) 2024-09-21T09:31:30.010 INFO:tasks.rados.rados.0.smithi022.stdout:96: snap_remove snap 1 2024-09-21T09:31:30.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:29 smithi064 ceph-mon[31124]: pgmap v110: 121 pgs: 121 active+clean; 24 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.7 MiB/s rd, 5.1 MiB/s wr, 61 op/s 2024-09-21T09:31:30.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:29 smithi064 ceph-mon[31124]: osdmap e77: 8 total, 8 up, 8 in 2024-09-21T09:31:30.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:29 smithi022 ceph-mon[39525]: pgmap v110: 121 pgs: 121 active+clean; 24 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.7 MiB/s rd, 5.1 MiB/s wr, 61 op/s 2024-09-21T09:31:30.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:29 smithi022 ceph-mon[39525]: osdmap e77: 8 total, 8 up, 8 in 2024-09-21T09:31:30.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:29 smithi022 ceph-mon[27763]: pgmap v110: 121 pgs: 121 active+clean; 24 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.7 MiB/s rd, 5.1 MiB/s wr, 61 op/s 2024-09-21T09:31:30.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:29 smithi022 ceph-mon[27763]: osdmap e77: 8 total, 8 up, 8 in 2024-09-21T09:31:30.941 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:31:30 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[33213]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:31:30] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-21T09:31:30.954 INFO:tasks.rados.rados.0.smithi022.stdout:96: done (2 left) 2024-09-21T09:31:30.954 INFO:tasks.rados.rados.0.smithi022.stdout:97: copy_from oid 34 from oid 37 current snap is 3 2024-09-21T09:31:30.954 INFO:tasks.rados.rados.0.smithi022.stdout:98: read oid 11 snap 0 2024-09-21T09:31:30.954 INFO:tasks.rados.rados.0.smithi022.stdout:98: expect (ObjNum 10 snap 0 seq_num 10) 2024-09-21T09:31:30.954 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 38 current snap is 3 2024-09-21T09:31:30.954 INFO:tasks.rados.rados.0.smithi022.stdout:99: seq_num 56 ranges {622592=540672,1163264=401408} 2024-09-21T09:31:30.960 INFO:tasks.rados.rados.0.smithi022.stdout:99: writing smithi02284960-38 from 622592 to 1163264 tid 1 2024-09-21T09:31:30.963 INFO:tasks.rados.rados.0.smithi022.stdout:99: writing smithi02284960-38 from 1163264 to 1564672 tid 2 2024-09-21T09:31:30.964 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 47 current snap is 3 2024-09-21T09:31:30.964 INFO:tasks.rados.rados.0.smithi022.stdout:100: seq_num 57 ranges {778240=557056,1335296=401408} 2024-09-21T09:31:30.971 INFO:tasks.rados.rados.0.smithi022.stdout:100: writing smithi02284960-47 from 778240 to 1335296 tid 1 2024-09-21T09:31:30.974 INFO:tasks.rados.rados.0.smithi022.stdout:100: writing smithi02284960-47 from 1335296 to 1736704 tid 2 2024-09-21T09:31:30.974 INFO:tasks.rados.rados.0.smithi022.stdout:101: copy_from oid 42 from oid 2 current snap is 3 2024-09-21T09:31:30.974 INFO:tasks.rados.rados.0.smithi022.stdout:94: finishing write tid 1 to smithi02284960-31 2024-09-21T09:31:30.974 INFO:tasks.rados.rados.0.smithi022.stdout:95: expect (ObjNum 12 snap 0 seq_num 12) 2024-09-21T09:31:30.981 INFO:tasks.rados.rados.0.smithi022.stdout:94: finishing write tid 2 to smithi02284960-31 2024-09-21T09:31:30.981 INFO:tasks.rados.rados.0.smithi022.stdout:94: finishing write tid 3 to smithi02284960-31 2024-09-21T09:31:30.981 INFO:tasks.rados.rados.0.smithi022.stdout:95: done (6 left) 2024-09-21T09:31:30.981 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 21 current snap is 3 2024-09-21T09:31:30.981 INFO:tasks.rados.rados.0.smithi022.stdout:94: finishing write tid 4 to smithi02284960-31 2024-09-21T09:31:30.981 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 31 v 21 (ObjNum 55 snap 3 seq_num 55) dirty exists 2024-09-21T09:31:30.981 INFO:tasks.rados.rados.0.smithi022.stdout:94: left oid 31 (ObjNum 55 snap 3 seq_num 55) 2024-09-21T09:31:30.981 INFO:tasks.rados.rados.0.smithi022.stdout:98: expect (ObjNum 10 snap 0 seq_num 10) 2024-09-21T09:31:30.987 INFO:tasks.rados.rados.0.smithi022.stdout:97: finishing copy_from racing read to smithi02284960-34 2024-09-21T09:31:30.987 INFO:tasks.rados.rados.0.smithi022.stdout:97: finishing copy_from to smithi02284960-34 2024-09-21T09:31:30.987 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 34 v 12 (ObjNum 36 snap 0 seq_num 36) dirty exists 2024-09-21T09:31:30.987 INFO:tasks.rados.rados.0.smithi022.stdout:99: finishing write tid 1 to smithi02284960-38 2024-09-21T09:31:30.987 INFO:tasks.rados.rados.0.smithi022.stdout:102: seq_num 58 ranges {188416=573440,761856=401408} 2024-09-21T09:31:30.991 INFO:tasks.rados.rados.0.smithi022.stdout:102: writing smithi02284960-21 from 188416 to 761856 tid 1 2024-09-21T09:31:30.994 INFO:tasks.rados.rados.0.smithi022.stdout:102: writing smithi02284960-21 from 761856 to 1163264 tid 2 2024-09-21T09:31:30.994 INFO:tasks.rados.rados.0.smithi022.stdout:94: done (6 left) 2024-09-21T09:31:30.994 INFO:tasks.rados.rados.0.smithi022.stdout:97: done (5 left) 2024-09-21T09:31:30.994 INFO:tasks.rados.rados.0.smithi022.stdout:98: done (4 left) 2024-09-21T09:31:30.995 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 37 current snap is 3 2024-09-21T09:31:30.995 INFO:tasks.rados.rados.0.smithi022.stdout:99: finishing write tid 2 to smithi02284960-38 2024-09-21T09:31:30.995 INFO:tasks.rados.rados.0.smithi022.stdout:99: finishing write tid 3 to smithi02284960-38 2024-09-21T09:31:30.995 INFO:tasks.rados.rados.0.smithi022.stdout:99: finishing write tid 4 to smithi02284960-38 2024-09-21T09:31:30.995 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 38 v 18 (ObjNum 56 snap 3 seq_num 56) dirty exists 2024-09-21T09:31:30.995 INFO:tasks.rados.rados.0.smithi022.stdout:99: left oid 38 (ObjNum 56 snap 3 seq_num 56) 2024-09-21T09:31:30.995 INFO:tasks.rados.rados.0.smithi022.stdout:100: finishing write tid 1 to smithi02284960-47 2024-09-21T09:31:30.995 INFO:tasks.rados.rados.0.smithi022.stdout:100: finishing write tid 2 to smithi02284960-47 2024-09-21T09:31:30.995 INFO:tasks.rados.rados.0.smithi022.stdout:100: finishing write tid 3 to smithi02284960-47 2024-09-21T09:31:30.995 INFO:tasks.rados.rados.0.smithi022.stdout:100: finishing write tid 4 to smithi02284960-47 2024-09-21T09:31:30.995 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 47 v 15 (ObjNum 57 snap 3 seq_num 57) dirty exists 2024-09-21T09:31:30.995 INFO:tasks.rados.rados.0.smithi022.stdout:100: left oid 47 (ObjNum 57 snap 3 seq_num 57) 2024-09-21T09:31:30.995 INFO:tasks.rados.rados.0.smithi022.stdout:101: finishing copy_from racing read to smithi02284960-42 2024-09-21T09:31:30.995 INFO:tasks.rados.rados.0.smithi022.stdout:101: finishing copy_from to smithi02284960-42 2024-09-21T09:31:30.996 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 42 v 10 (ObjNum 46 snap 0 seq_num 46) dirty exists 2024-09-21T09:31:30.996 INFO:tasks.rados.rados.0.smithi022.stdout:103: seq_num 59 ranges {606208=598016,1204224=401408} 2024-09-21T09:31:31.000 INFO:tasks.rados.rados.0.smithi022.stdout:103: writing smithi02284960-37 from 606208 to 1204224 tid 1 2024-09-21T09:31:31.003 INFO:tasks.rados.rados.0.smithi022.stdout:103: writing smithi02284960-37 from 1204224 to 1605632 tid 2 2024-09-21T09:31:31.003 INFO:tasks.rados.rados.0.smithi022.stdout:99: done (4 left) 2024-09-21T09:31:31.003 INFO:tasks.rados.rados.0.smithi022.stdout:100: done (3 left) 2024-09-21T09:31:31.003 INFO:tasks.rados.rados.0.smithi022.stdout:101: done (2 left) 2024-09-21T09:31:31.003 INFO:tasks.rados.rados.0.smithi022.stdout:104: rollback oid 9 current snap is 3 2024-09-21T09:31:31.003 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 9 to 0 2024-09-21T09:31:31.004 INFO:tasks.rados.rados.0.smithi022.stdout:102: finishing write tid 1 to smithi02284960-21 2024-09-21T09:31:31.004 INFO:tasks.rados.rados.0.smithi022.stdout:102: finishing write tid 2 to smithi02284960-21 2024-09-21T09:31:31.004 INFO:tasks.rados.rados.0.smithi022.stdout:102: finishing write tid 3 to smithi02284960-21 2024-09-21T09:31:31.004 INFO:tasks.rados.rados.0.smithi022.stdout:102: finishing write tid 4 to smithi02284960-21 2024-09-21T09:31:31.004 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 21 v 23 (ObjNum 58 snap 3 seq_num 58) dirty exists 2024-09-21T09:31:31.004 INFO:tasks.rados.rados.0.smithi022.stdout:102: left oid 21 (ObjNum 58 snap 3 seq_num 58) 2024-09-21T09:31:31.004 INFO:tasks.rados.rados.0.smithi022.stdout:102: done (2 left) 2024-09-21T09:31:31.004 INFO:tasks.rados.rados.0.smithi022.stdout:105: delete oid 33 current snap is 3 2024-09-21T09:31:31.005 INFO:tasks.rados.rados.0.smithi022.stdout:103: finishing write tid 1 to smithi02284960-37 2024-09-21T09:31:31.006 INFO:tasks.rados.rados.0.smithi022.stdout:104: finishing rollback tid 0 to smithi02284960-9 2024-09-21T09:31:31.006 INFO:tasks.rados.rados.0.smithi022.stdout:105: done (2 left) 2024-09-21T09:31:31.006 INFO:tasks.rados.rados.0.smithi022.stdout:106: read oid 44 snap -1 2024-09-21T09:31:31.007 INFO:tasks.rados.rados.0.smithi022.stdout:106: expect (ObjNum 43 snap 0 seq_num 43) 2024-09-21T09:31:31.007 INFO:tasks.rados.rados.0.smithi022.stdout:107: rollback oid 39 current snap is 3 2024-09-21T09:31:31.007 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 39 to 0 2024-09-21T09:31:31.007 INFO:tasks.rados.rados.0.smithi022.stdout:104: finishing rollback tid 1 to smithi02284960-9 2024-09-21T09:31:31.007 INFO:tasks.rados.rados.0.smithi022.stdout:103: finishing write tid 2 to smithi02284960-37 2024-09-21T09:31:31.007 INFO:tasks.rados.rados.0.smithi022.stdout:103: finishing write tid 3 to smithi02284960-37 2024-09-21T09:31:31.007 INFO:tasks.rados.rados.0.smithi022.stdout:108: delete oid 16 current snap is 3 2024-09-21T09:31:31.007 INFO:tasks.rados.rados.0.smithi022.stdout:104: finishing rollback tid 2 to smithi02284960-9 2024-09-21T09:31:31.007 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 9 v 19 (ObjNum 8 snap 0 seq_num 8) dirty exists 2024-09-21T09:31:31.008 INFO:tasks.rados.rados.0.smithi022.stdout:103: finishing write tid 4 to smithi02284960-37 2024-09-21T09:31:31.008 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 37 v 27 (ObjNum 59 snap 3 seq_num 59) dirty exists 2024-09-21T09:31:31.008 INFO:tasks.rados.rados.0.smithi022.stdout:103: left oid 37 (ObjNum 59 snap 3 seq_num 59) 2024-09-21T09:31:31.010 INFO:tasks.rados.rados.0.smithi022.stdout:107: finishing rollback tid 0 to smithi02284960-39 2024-09-21T09:31:31.010 INFO:tasks.rados.rados.0.smithi022.stdout:107: finishing rollback tid 1 to smithi02284960-39 2024-09-21T09:31:31.010 INFO:tasks.rados.rados.0.smithi022.stdout:107: finishing rollback tid 2 to smithi02284960-39 2024-09-21T09:31:31.010 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 39 v 10 (ObjNum 38 snap 0 seq_num 38) dirty exists 2024-09-21T09:31:31.010 INFO:tasks.rados.rados.0.smithi022.stdout:106: expect (ObjNum 43 snap 0 seq_num 43) 2024-09-21T09:31:31.030 INFO:tasks.rados.rados.0.smithi022.stdout:103: done (4 left) 2024-09-21T09:31:31.030 INFO:tasks.rados.rados.0.smithi022.stdout:104: done (3 left) 2024-09-21T09:31:31.030 INFO:tasks.rados.rados.0.smithi022.stdout:106: done (2 left) 2024-09-21T09:31:31.030 INFO:tasks.rados.rados.0.smithi022.stdout:107: done (1 left) 2024-09-21T09:31:31.030 INFO:tasks.rados.rados.0.smithi022.stdout:108: done (0 left) 2024-09-21T09:31:31.030 INFO:tasks.rados.rados.0.smithi022.stdout:109: read oid 33 snap 0 2024-09-21T09:31:31.030 INFO:tasks.rados.rados.0.smithi022.stdout:109: expect (ObjNum 32 snap 0 seq_num 32) 2024-09-21T09:31:31.030 INFO:tasks.rados.rados.0.smithi022.stdout:110: rollback oid 48 current snap is 3 2024-09-21T09:31:31.030 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 48 to 0 2024-09-21T09:31:31.030 INFO:tasks.rados.rados.0.smithi022.stdout:111: delete oid 47 current snap is 3 2024-09-21T09:31:31.031 INFO:tasks.rados.rados.0.smithi022.stdout:111: done (2 left) 2024-09-21T09:31:31.031 INFO:tasks.rados.rados.0.smithi022.stdout:112: setattr oid 31 current snap is 3 2024-09-21T09:31:31.032 INFO:tasks.rados.rados.0.smithi022.stdout:113: snap_create 2024-09-21T09:31:31.032 INFO:tasks.rados.rados.0.smithi022.stdout:109: expect (ObjNum 32 snap 0 seq_num 32) 2024-09-21T09:31:31.048 INFO:tasks.rados.rados.0.smithi022.stdout:110: finishing rollback tid 0 to smithi02284960-48 2024-09-21T09:31:31.048 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 31 v 28 (ObjNum 55 snap 3 seq_num 55) dirty exists 2024-09-21T09:31:31.048 INFO:tasks.rados.rados.0.smithi022.stdout:110: finishing rollback tid 1 to smithi02284960-48 2024-09-21T09:31:31.048 INFO:tasks.rados.rados.0.smithi022.stdout:110: finishing rollback tid 2 to smithi02284960-48 2024-09-21T09:31:31.048 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 48 v 27 (ObjNum 47 snap 0 seq_num 47) dirty exists 2024-09-21T09:31:31.260 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:30 smithi022 ceph-mon[39525]: osdmap e78: 8 total, 8 up, 8 in 2024-09-21T09:31:31.261 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:30 smithi022 ceph-mon[27763]: osdmap e78: 8 total, 8 up, 8 in 2024-09-21T09:31:31.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:30 smithi064 ceph-mon[31124]: osdmap e78: 8 total, 8 up, 8 in 2024-09-21T09:31:31.558 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:31:31 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:31:31] "GET /metrics HTTP/1.1" 200 238835 "" "Prometheus/2.43.0" 2024-09-21T09:31:31.957 INFO:tasks.rados.rados.0.smithi022.stdout:109: done (3 left) 2024-09-21T09:31:31.957 INFO:tasks.rados.rados.0.smithi022.stdout:110: done (2 left) 2024-09-21T09:31:31.957 INFO:tasks.rados.rados.0.smithi022.stdout:112: done (1 left) 2024-09-21T09:31:31.957 INFO:tasks.rados.rados.0.smithi022.stdout:113: done (0 left) 2024-09-21T09:31:31.957 INFO:tasks.rados.rados.0.smithi022.stdout:114: copy_from oid 25 from oid 6 current snap is 4 2024-09-21T09:31:31.957 INFO:tasks.rados.rados.0.smithi022.stdout:115: setattr oid 23 current snap is 4 2024-09-21T09:31:31.958 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 15 current snap is 4 2024-09-21T09:31:31.958 INFO:tasks.rados.rados.0.smithi022.stdout:116: seq_num 63 ranges {172032=663552,835584=401408} 2024-09-21T09:31:31.964 INFO:tasks.rados.rados.0.smithi022.stdout:116: writing smithi02284960-15 from 172032 to 835584 tid 1 2024-09-21T09:31:31.966 INFO:tasks.rados.rados.0.smithi022.stdout:116: writing smithi02284960-15 from 835584 to 1236992 tid 2 2024-09-21T09:31:31.966 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 20 current snap is 4 2024-09-21T09:31:31.966 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 23 v 19 (ObjNum 53 snap 1 seq_num 53) dirty exists 2024-09-21T09:31:31.966 INFO:tasks.rados.rados.0.smithi022.stdout:117: seq_num 64 ranges {319488=679936,999424=401408} 2024-09-21T09:31:31.972 INFO:tasks.rados.rados.0.smithi022.stdout:117: writing smithi02284960-20 from 319488 to 999424 tid 1 2024-09-21T09:31:31.975 INFO:tasks.rados.rados.0.smithi022.stdout:117: writing smithi02284960-20 from 999424 to 1400832 tid 2 2024-09-21T09:31:31.975 INFO:tasks.rados.rados.0.smithi022.stdout:115: done (3 left) 2024-09-21T09:31:31.975 INFO:tasks.rados.rados.0.smithi022.stdout:118: rollback oid 14 current snap is 4 2024-09-21T09:31:31.975 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 14 to 0 2024-09-21T09:31:31.975 INFO:tasks.rados.rados.0.smithi022.stdout:114: finishing copy_from racing read to smithi02284960-25 2024-09-21T09:31:31.976 INFO:tasks.rados.rados.0.smithi022.stdout:114: finishing copy_from to smithi02284960-25 2024-09-21T09:31:31.976 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 25 v 12 (ObjNum 36 snap 0 seq_num 36) dirty exists 2024-09-21T09:31:31.976 INFO:tasks.rados.rados.0.smithi022.stdout:116: finishing write tid 1 to smithi02284960-15 2024-09-21T09:31:31.976 INFO:tasks.rados.rados.0.smithi022.stdout:116: finishing write tid 2 to smithi02284960-15 2024-09-21T09:31:31.976 INFO:tasks.rados.rados.0.smithi022.stdout:116: finishing write tid 3 to smithi02284960-15 2024-09-21T09:31:31.976 INFO:tasks.rados.rados.0.smithi022.stdout:116: finishing write tid 4 to smithi02284960-15 2024-09-21T09:31:31.976 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 15 v 32 (ObjNum 63 snap 4 seq_num 63) dirty exists 2024-09-21T09:31:31.976 INFO:tasks.rados.rados.0.smithi022.stdout:116: left oid 15 (ObjNum 63 snap 4 seq_num 63) 2024-09-21T09:31:31.976 INFO:tasks.rados.rados.0.smithi022.stdout:114: done (3 left) 2024-09-21T09:31:31.976 INFO:tasks.rados.rados.0.smithi022.stdout:116: done (2 left) 2024-09-21T09:31:31.976 INFO:tasks.rados.rados.0.smithi022.stdout:119: read oid 27 snap -1 2024-09-21T09:31:31.976 INFO:tasks.rados.rados.0.smithi022.stdout:119: expect (ObjNum 26 snap 0 seq_num 26) 2024-09-21T09:31:31.976 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 21 current snap is 4 2024-09-21T09:31:31.976 INFO:tasks.rados.rados.0.smithi022.stdout:120: seq_num 65 ranges {1163264=696320,1859584=401408} 2024-09-21T09:31:31.986 INFO:tasks.rados.rados.0.smithi022.stdout:120: writing smithi02284960-21 from 1163264 to 1859584 tid 1 2024-09-21T09:31:31.989 INFO:tasks.rados.rados.0.smithi022.stdout:120: writing smithi02284960-21 from 1859584 to 2260992 tid 2 2024-09-21T09:31:31.989 INFO:tasks.rados.rados.0.smithi022.stdout:121: snap_create 2024-09-21T09:31:31.989 INFO:tasks.rados.rados.0.smithi022.stdout:118: finishing rollback tid 1 to smithi02284960-14 2024-09-21T09:31:31.989 INFO:tasks.rados.rados.0.smithi022.stdout:118: finishing rollback tid 2 to smithi02284960-14 2024-09-21T09:31:31.989 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 14 v 15 (ObjNum 13 snap 0 seq_num 13) dirty exists 2024-09-21T09:31:31.989 INFO:tasks.rados.rados.0.smithi022.stdout:117: finishing write tid 1 to smithi02284960-20 2024-09-21T09:31:31.990 INFO:tasks.rados.rados.0.smithi022.stdout:117: finishing write tid 2 to smithi02284960-20 2024-09-21T09:31:31.990 INFO:tasks.rados.rados.0.smithi022.stdout:117: finishing write tid 3 to smithi02284960-20 2024-09-21T09:31:31.990 INFO:tasks.rados.rados.0.smithi022.stdout:119: expect (ObjNum 26 snap 0 seq_num 26) 2024-09-21T09:31:32.002 INFO:tasks.rados.rados.0.smithi022.stdout:117: finishing write tid 4 to smithi02284960-20 2024-09-21T09:31:32.002 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 20 v 31 (ObjNum 64 snap 4 seq_num 64) dirty exists 2024-09-21T09:31:32.002 INFO:tasks.rados.rados.0.smithi022.stdout:117: left oid 20 (ObjNum 64 snap 4 seq_num 64) 2024-09-21T09:31:32.002 INFO:tasks.rados.rados.0.smithi022.stdout:120: finishing write tid 1 to smithi02284960-21 2024-09-21T09:31:32.002 INFO:tasks.rados.rados.0.smithi022.stdout:120: finishing write tid 2 to smithi02284960-21 2024-09-21T09:31:32.002 INFO:tasks.rados.rados.0.smithi022.stdout:120: finishing write tid 3 to smithi02284960-21 2024-09-21T09:31:32.002 INFO:tasks.rados.rados.0.smithi022.stdout:120: finishing write tid 4 to smithi02284960-21 2024-09-21T09:31:32.002 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 21 v 27 (ObjNum 65 snap 4 seq_num 65) dirty exists 2024-09-21T09:31:32.002 INFO:tasks.rados.rados.0.smithi022.stdout:120: left oid 21 (ObjNum 65 snap 4 seq_num 65) 2024-09-21T09:31:32.307 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:31 smithi022 ceph-mon[39525]: pgmap v113: 121 pgs: 121 active+clean; 24 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.1 MiB/s rd, 6.3 MiB/s wr, 75 op/s 2024-09-21T09:31:32.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:31 smithi022 ceph-mon[39525]: osdmap e79: 8 total, 8 up, 8 in 2024-09-21T09:31:32.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:31 smithi022 ceph-mon[27763]: pgmap v113: 121 pgs: 121 active+clean; 24 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.1 MiB/s rd, 6.3 MiB/s wr, 75 op/s 2024-09-21T09:31:32.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:31 smithi022 ceph-mon[27763]: osdmap e79: 8 total, 8 up, 8 in 2024-09-21T09:31:32.342 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:31 smithi064 ceph-mon[31124]: pgmap v113: 121 pgs: 121 active+clean; 24 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.1 MiB/s rd, 6.3 MiB/s wr, 75 op/s 2024-09-21T09:31:32.343 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:31 smithi064 ceph-mon[31124]: osdmap e79: 8 total, 8 up, 8 in 2024-09-21T09:31:32.959 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:32 smithi022 ceph-mon[27763]: osdmap e80: 8 total, 8 up, 8 in 2024-09-21T09:31:32.962 INFO:tasks.rados.rados.0.smithi022.stdout:117: done (4 left) 2024-09-21T09:31:32.962 INFO:tasks.rados.rados.0.smithi022.stdout:118: done (3 left) 2024-09-21T09:31:32.962 INFO:tasks.rados.rados.0.smithi022.stdout:119: done (2 left) 2024-09-21T09:31:32.963 INFO:tasks.rados.rados.0.smithi022.stdout:120: done (1 left) 2024-09-21T09:31:32.963 INFO:tasks.rados.rados.0.smithi022.stdout:121: done (0 left) 2024-09-21T09:31:32.963 INFO:tasks.rados.rados.0.smithi022.stdout:122: read oid 48 snap 3 2024-09-21T09:31:32.963 INFO:tasks.rados.rados.0.smithi022.stdout:122: expect (ObjNum 47 snap 0 seq_num 47) 2024-09-21T09:31:32.963 INFO:tasks.rados.rados.0.smithi022.stdout:123: read oid 49 snap 4 2024-09-21T09:31:32.963 INFO:tasks.rados.rados.0.smithi022.stdout:123: expect (ObjNum 48 snap 0 seq_num 48) 2024-09-21T09:31:32.963 INFO:tasks.rados.rados.0.smithi022.stdout:124: delete oid 44 current snap is 5 2024-09-21T09:31:32.966 INFO:tasks.rados.rados.0.smithi022.stdout:122: expect (ObjNum 47 snap 0 seq_num 47) 2024-09-21T09:31:32.989 INFO:tasks.rados.rados.0.smithi022.stdout:123: expect (ObjNum 48 snap 0 seq_num 48) 2024-09-21T09:31:33.012 INFO:tasks.rados.rados.0.smithi022.stdout:122: done (2 left) 2024-09-21T09:31:33.012 INFO:tasks.rados.rados.0.smithi022.stdout:123: done (1 left) 2024-09-21T09:31:33.012 INFO:tasks.rados.rados.0.smithi022.stdout:124: done (0 left) 2024-09-21T09:31:33.012 INFO:tasks.rados.rados.0.smithi022.stdout:125: rollback oid 49 current snap is 5 2024-09-21T09:31:33.012 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 49 to 3 2024-09-21T09:31:33.012 INFO:tasks.rados.rados.0.smithi022.stdout:126: read oid 27 snap -1 2024-09-21T09:31:33.013 INFO:tasks.rados.rados.0.smithi022.stdout:126: expect (ObjNum 26 snap 0 seq_num 26) 2024-09-21T09:31:33.013 INFO:tasks.rados.rados.0.smithi022.stdout:127: setattr oid 41 current snap is 5 2024-09-21T09:31:33.013 INFO:tasks.rados.rados.0.smithi022.stdout:128: delete oid 28 current snap is 5 2024-09-21T09:31:33.015 INFO:tasks.rados.rados.0.smithi022.stdout:128: done (3 left) 2024-09-21T09:31:33.015 INFO:tasks.rados.rados.0.smithi022.stdout:129: snap_remove snap 4 2024-09-21T09:31:33.307 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:32 smithi022 ceph-mon[39525]: osdmap e80: 8 total, 8 up, 8 in 2024-09-21T09:31:33.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:32 smithi064 ceph-mon[31124]: osdmap e80: 8 total, 8 up, 8 in 2024-09-21T09:31:33.969 INFO:tasks.rados.rados.0.smithi022.stdout:129: done (3 left) 2024-09-21T09:31:33.969 INFO:tasks.rados.rados.0.smithi022.stdout:130: setattr oid 50 current snap is 5 2024-09-21T09:31:33.969 INFO:tasks.rados.rados.0.smithi022.stdout:125: finishing rollback tid 0 to smithi02284960-49 2024-09-21T09:31:33.969 INFO:tasks.rados.rados.0.smithi022.stdout:125: finishing rollback tid 1 to smithi02284960-49 2024-09-21T09:31:33.969 INFO:tasks.rados.rados.0.smithi022.stdout:126: expect (ObjNum 26 snap 0 seq_num 26) 2024-09-21T09:31:33.984 INFO:tasks.rados.rados.0.smithi022.stdout:125: finishing rollback tid 2 to smithi02284960-49 2024-09-21T09:31:33.984 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 49 v 19 (ObjNum 48 snap 0 seq_num 48) dirty exists 2024-09-21T09:31:33.984 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 41 v 22 (ObjNum 40 snap 0 seq_num 40) dirty exists 2024-09-21T09:31:33.984 INFO:tasks.rados.rados.0.smithi022.stdout:125: done (3 left) 2024-09-21T09:31:33.984 INFO:tasks.rados.rados.0.smithi022.stdout:126: done (2 left) 2024-09-21T09:31:33.985 INFO:tasks.rados.rados.0.smithi022.stdout:127: done (1 left) 2024-09-21T09:31:33.985 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 48 current snap is 5 2024-09-21T09:31:33.985 INFO:tasks.rados.rados.0.smithi022.stdout:131: seq_num 68 ranges {794624=745472,1540096=401408} 2024-09-21T09:31:33.993 INFO:tasks.rados.rados.0.smithi022.stdout:131: writing smithi02284960-48 from 794624 to 1540096 tid 1 2024-09-21T09:31:33.997 INFO:tasks.rados.rados.0.smithi022.stdout:131: writing smithi02284960-48 from 1540096 to 1941504 tid 2 2024-09-21T09:31:33.997 INFO:tasks.rados.rados.0.smithi022.stdout:132: read oid 47 snap 0 2024-09-21T09:31:33.997 INFO:tasks.rados.rados.0.smithi022.stdout:132: expect (ObjNum 46 snap 0 seq_num 46) 2024-09-21T09:31:33.997 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 50 v 13 (ObjNum 49 snap 0 seq_num 49) dirty exists 2024-09-21T09:31:33.997 INFO:tasks.rados.rados.0.smithi022.stdout:130: done (2 left) 2024-09-21T09:31:33.997 INFO:tasks.rados.rados.0.smithi022.stdout:133: rollback oid 33 current snap is 5 2024-09-21T09:31:33.997 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 33 to 0 2024-09-21T09:31:33.997 INFO:tasks.rados.rados.0.smithi022.stdout:134: delete oid 14 current snap is 5 2024-09-21T09:31:33.999 INFO:tasks.rados.rados.0.smithi022.stdout:134: done (3 left) 2024-09-21T09:31:33.999 INFO:tasks.rados.rados.0.smithi022.stdout:135: delete oid 19 current snap is 5 2024-09-21T09:31:34.002 INFO:tasks.rados.rados.0.smithi022.stdout:132: expect (ObjNum 46 snap 0 seq_num 46) 2024-09-21T09:31:34.025 INFO:tasks.rados.rados.0.smithi022.stdout:131: finishing write tid 1 to smithi02284960-48 2024-09-21T09:31:34.026 INFO:tasks.rados.rados.0.smithi022.stdout:131: finishing write tid 2 to smithi02284960-48 2024-09-21T09:31:34.026 INFO:tasks.rados.rados.0.smithi022.stdout:132: done (3 left) 2024-09-21T09:31:34.026 INFO:tasks.rados.rados.0.smithi022.stdout:135: done (2 left) 2024-09-21T09:31:34.026 INFO:tasks.rados.rados.0.smithi022.stdout:136: copy_from oid 12 from oid 8 current snap is 5 2024-09-21T09:31:34.026 INFO:tasks.rados.rados.0.smithi022.stdout:131: finishing write tid 3 to smithi02284960-48 2024-09-21T09:31:34.026 INFO:tasks.rados.rados.0.smithi022.stdout:133: finishing rollback tid 1 to smithi02284960-33 2024-09-21T09:31:34.026 INFO:tasks.rados.rados.0.smithi022.stdout:133: finishing rollback tid 2 to smithi02284960-33 2024-09-21T09:31:34.026 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 33 v 37 (ObjNum 32 snap 0 seq_num 32) dirty exists 2024-09-21T09:31:34.026 INFO:tasks.rados.rados.0.smithi022.stdout:131: finishing write tid 4 to smithi02284960-48 2024-09-21T09:31:34.026 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 48 v 35 (ObjNum 68 snap 5 seq_num 68) dirty exists 2024-09-21T09:31:34.026 INFO:tasks.rados.rados.0.smithi022.stdout:131: left oid 48 (ObjNum 68 snap 5 seq_num 68) 2024-09-21T09:31:34.026 INFO:tasks.rados.rados.0.smithi022.stdout:131: done (2 left) 2024-09-21T09:31:34.026 INFO:tasks.rados.rados.0.smithi022.stdout:133: done (1 left) 2024-09-21T09:31:34.027 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 31 current snap is 5 2024-09-21T09:31:34.027 INFO:tasks.rados.rados.0.smithi022.stdout:137: seq_num 71 ranges {1433600=794624,2228224=401408} 2024-09-21T09:31:34.037 INFO:tasks.rados.rados.0.smithi022.stdout:137: writing smithi02284960-31 from 1433600 to 2228224 tid 1 2024-09-21T09:31:34.040 INFO:tasks.rados.rados.0.smithi022.stdout:137: writing smithi02284960-31 from 2228224 to 2629632 tid 2 2024-09-21T09:31:34.040 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 7 current snap is 5 2024-09-21T09:31:34.040 INFO:tasks.rados.rados.0.smithi022.stdout:136: finishing copy_from racing read to smithi02284960-12 2024-09-21T09:31:34.040 INFO:tasks.rados.rados.0.smithi022.stdout:136: finishing copy_from to smithi02284960-12 2024-09-21T09:31:34.040 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 12 v 6 (ObjNum 7 snap 0 seq_num 7) dirty exists 2024-09-21T09:31:34.040 INFO:tasks.rados.rados.0.smithi022.stdout:138: seq_num 72 ranges {106496=409600,516096=524288,1040384=278528} 2024-09-21T09:31:34.043 INFO:tasks.rados.rados.0.smithi022.stdout:138: writing smithi02284960-7 from 106496 to 516096 tid 1 2024-09-21T09:31:34.046 INFO:tasks.rados.rados.0.smithi022.stdout:138: writing smithi02284960-7 from 516096 to 1040384 tid 2 2024-09-21T09:31:34.049 INFO:tasks.rados.rados.0.smithi022.stdout:138: writing smithi02284960-7 from 1040384 to 1318912 tid 3 2024-09-21T09:31:34.049 INFO:tasks.rados.rados.0.smithi022.stdout:136: done (2 left) 2024-09-21T09:31:34.049 INFO:tasks.rados.rados.0.smithi022.stdout:139: copy_from oid 41 from oid 44 current snap is 5 2024-09-21T09:31:34.049 INFO:tasks.rados.rados.0.smithi022.stdout:137: finishing write tid 1 to smithi02284960-31 2024-09-21T09:31:34.049 INFO:tasks.rados.rados.0.smithi022.stdout:137: finishing write tid 2 to smithi02284960-31 2024-09-21T09:31:34.049 INFO:tasks.rados.rados.0.smithi022.stdout:137: finishing write tid 3 to smithi02284960-31 2024-09-21T09:31:34.049 INFO:tasks.rados.rados.0.smithi022.stdout:137: finishing write tid 4 to smithi02284960-31 2024-09-21T09:31:34.049 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 31 v 40 (ObjNum 71 snap 5 seq_num 71) dirty exists 2024-09-21T09:31:34.049 INFO:tasks.rados.rados.0.smithi022.stdout:137: left oid 31 (ObjNum 71 snap 5 seq_num 71) 2024-09-21T09:31:34.049 INFO:tasks.rados.rados.0.smithi022.stdout:138: finishing write tid 1 to smithi02284960-7 2024-09-21T09:31:34.049 INFO:tasks.rados.rados.0.smithi022.stdout:137: done (2 left) 2024-09-21T09:31:34.050 INFO:tasks.rados.rados.0.smithi022.stdout:140: copy_from oid 28 from oid 10 current snap is 5 2024-09-21T09:31:34.050 INFO:tasks.rados.rados.0.smithi022.stdout:141: snap_create 2024-09-21T09:31:34.050 INFO:tasks.rados.rados.0.smithi022.stdout:138: finishing write tid 2 to smithi02284960-7 2024-09-21T09:31:34.050 INFO:tasks.rados.rados.0.smithi022.stdout:139: finishing copy_from to smithi02284960-41 2024-09-21T09:31:34.050 INFO:tasks.rados.rados.0.smithi022.stdout:139: got expected ENOENT (src dne) 2024-09-21T09:31:34.050 INFO:tasks.rados.rados.0.smithi022.stdout:139: finishing copy_from racing read to smithi02284960-41 2024-09-21T09:31:34.051 INFO:tasks.rados.rados.0.smithi022.stdout:138: finishing write tid 3 to smithi02284960-7 2024-09-21T09:31:34.051 INFO:tasks.rados.rados.0.smithi022.stdout:138: finishing write tid 4 to smithi02284960-7 2024-09-21T09:31:34.052 INFO:tasks.rados.rados.0.smithi022.stdout:138: finishing write tid 5 to smithi02284960-7 2024-09-21T09:31:34.052 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 7 v 42 (ObjNum 72 snap 5 seq_num 72) dirty exists 2024-09-21T09:31:34.052 INFO:tasks.rados.rados.0.smithi022.stdout:138: left oid 7 (ObjNum 72 snap 5 seq_num 72) 2024-09-21T09:31:34.053 INFO:tasks.rados.rados.0.smithi022.stdout:140: finishing copy_from racing read to smithi02284960-28 2024-09-21T09:31:34.053 INFO:tasks.rados.rados.0.smithi022.stdout:140: finishing copy_from to smithi02284960-28 2024-09-21T09:31:34.053 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 28 v 15 (ObjNum 9 snap 0 seq_num 9) dirty exists 2024-09-21T09:31:34.307 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:33 smithi022 ceph-mon[39525]: pgmap v116: 121 pgs: 121 active+clean; 34 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.7 MiB/s wr, 37 op/s 2024-09-21T09:31:34.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:33 smithi022 ceph-mon[39525]: osdmap e81: 8 total, 8 up, 8 in 2024-09-21T09:31:34.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:33 smithi022 ceph-mon[27763]: pgmap v116: 121 pgs: 121 active+clean; 34 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.7 MiB/s wr, 37 op/s 2024-09-21T09:31:34.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:33 smithi022 ceph-mon[27763]: osdmap e81: 8 total, 8 up, 8 in 2024-09-21T09:31:34.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:33 smithi064 ceph-mon[31124]: pgmap v116: 121 pgs: 121 active+clean; 34 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.7 MiB/s wr, 37 op/s 2024-09-21T09:31:34.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:33 smithi064 ceph-mon[31124]: osdmap e81: 8 total, 8 up, 8 in 2024-09-21T09:31:34.973 INFO:tasks.rados.rados.0.smithi022.stdout:138: done (3 left) 2024-09-21T09:31:34.974 INFO:tasks.rados.rados.0.smithi022.stdout:139: done (2 left) 2024-09-21T09:31:34.974 INFO:tasks.rados.rados.0.smithi022.stdout:140: done (1 left) 2024-09-21T09:31:34.974 INFO:tasks.rados.rados.0.smithi022.stdout:141: done (0 left) 2024-09-21T09:31:34.974 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 9 current snap is 6 2024-09-21T09:31:34.974 INFO:tasks.rados.rados.0.smithi022.stdout:142: seq_num 73 ranges {139264=425984,565248=409600,974848=393216} 2024-09-21T09:31:34.976 INFO:tasks.rados.rados.0.smithi022.stdout:142: writing smithi02284960-9 from 139264 to 565248 tid 1 2024-09-21T09:31:34.979 INFO:tasks.rados.rados.0.smithi022.stdout:142: writing smithi02284960-9 from 565248 to 974848 tid 2 2024-09-21T09:31:34.982 INFO:tasks.rados.rados.0.smithi022.stdout:142: writing smithi02284960-9 from 974848 to 1368064 tid 3 2024-09-21T09:31:34.982 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 12 current snap is 6 2024-09-21T09:31:34.982 INFO:tasks.rados.rados.0.smithi022.stdout:142: finishing write tid 1 to smithi02284960-9 2024-09-21T09:31:34.982 INFO:tasks.rados.rados.0.smithi022.stdout:143: seq_num 74 ranges {122880=442368,565248=696320,1261568=106496} 2024-09-21T09:31:34.985 INFO:tasks.rados.rados.0.smithi022.stdout:143: writing smithi02284960-12 from 122880 to 565248 tid 1 2024-09-21T09:31:34.989 INFO:tasks.rados.rados.0.smithi022.stdout:143: writing smithi02284960-12 from 565248 to 1261568 tid 2 2024-09-21T09:31:34.990 INFO:tasks.rados.rados.0.smithi022.stdout:143: writing smithi02284960-12 from 1261568 to 1368064 tid 3 2024-09-21T09:31:34.990 INFO:tasks.rados.rados.0.smithi022.stdout:144: rollback oid 22 current snap is 6 2024-09-21T09:31:34.990 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 22 to 0 2024-09-21T09:31:34.991 INFO:tasks.rados.rados.0.smithi022.stdout:142: finishing write tid 2 to smithi02284960-9 2024-09-21T09:31:34.991 INFO:tasks.rados.rados.0.smithi022.stdout:142: finishing write tid 3 to smithi02284960-9 2024-09-21T09:31:34.991 INFO:tasks.rados.rados.0.smithi022.stdout:142: finishing write tid 4 to smithi02284960-9 2024-09-21T09:31:34.991 INFO:tasks.rados.rados.0.smithi022.stdout:142: finishing write tid 5 to smithi02284960-9 2024-09-21T09:31:34.991 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 9 v 24 (ObjNum 73 snap 6 seq_num 73) dirty exists 2024-09-21T09:31:34.991 INFO:tasks.rados.rados.0.smithi022.stdout:142: left oid 9 (ObjNum 73 snap 6 seq_num 73) 2024-09-21T09:31:34.991 INFO:tasks.rados.rados.0.smithi022.stdout:142: done (2 left) 2024-09-21T09:31:34.991 INFO:tasks.rados.rados.0.smithi022.stdout:145: snap_create 2024-09-21T09:31:34.992 INFO:tasks.rados.rados.0.smithi022.stdout:143: finishing write tid 1 to smithi02284960-12 2024-09-21T09:31:34.993 INFO:tasks.rados.rados.0.smithi022.stdout:144: finishing rollback tid 0 to smithi02284960-22 2024-09-21T09:31:34.993 INFO:tasks.rados.rados.0.smithi022.stdout:144: finishing rollback tid 1 to smithi02284960-22 2024-09-21T09:31:34.994 INFO:tasks.rados.rados.0.smithi022.stdout:144: finishing rollback tid 2 to smithi02284960-22 2024-09-21T09:31:34.994 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 22 v 16 (ObjNum 21 snap 0 seq_num 21) dirty exists 2024-09-21T09:31:34.994 INFO:tasks.rados.rados.0.smithi022.stdout:143: finishing write tid 2 to smithi02284960-12 2024-09-21T09:31:34.994 INFO:tasks.rados.rados.0.smithi022.stdout:143: finishing write tid 3 to smithi02284960-12 2024-09-21T09:31:34.994 INFO:tasks.rados.rados.0.smithi022.stdout:143: finishing write tid 4 to smithi02284960-12 2024-09-21T09:31:34.995 INFO:tasks.rados.rados.0.smithi022.stdout:143: finishing write tid 5 to smithi02284960-12 2024-09-21T09:31:34.995 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 12 v 45 (ObjNum 74 snap 6 seq_num 74) dirty exists 2024-09-21T09:31:34.996 INFO:tasks.rados.rados.0.smithi022.stdout:143: left oid 12 (ObjNum 74 snap 6 seq_num 74) 2024-09-21T09:31:35.307 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:34 smithi022 ceph-mon[39525]: osdmap e82: 8 total, 8 up, 8 in 2024-09-21T09:31:35.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:34 smithi022 ceph-mon[27763]: osdmap e82: 8 total, 8 up, 8 in 2024-09-21T09:31:35.408 INFO:tasks.rados.rados.0.smithi022.stdout:143: done (2 left) 2024-09-21T09:31:35.408 INFO:tasks.rados.rados.0.smithi022.stdout:144: done (1 left) 2024-09-21T09:31:35.408 INFO:tasks.rados.rados.0.smithi022.stdout:145: done (0 left) 2024-09-21T09:31:35.409 INFO:tasks.rados.rados.0.smithi022.stdout:146: read oid 33 snap -1 2024-09-21T09:31:35.409 INFO:tasks.rados.rados.0.smithi022.stdout:146: expect (ObjNum 32 snap 0 seq_num 32) 2024-09-21T09:31:35.409 INFO:tasks.rados.rados.0.smithi022.stdout:147: read oid 45 snap -1 2024-09-21T09:31:35.409 INFO:tasks.rados.rados.0.smithi022.stdout:147: expect (ObjNum 33 snap 0 seq_num 33) 2024-09-21T09:31:35.409 INFO:tasks.rados.rados.0.smithi022.stdout:148: snap_create 2024-09-21T09:31:35.412 INFO:tasks.rados.rados.0.smithi022.stdout:147: expect (ObjNum 33 snap 0 seq_num 33) 2024-09-21T09:31:35.427 INFO:tasks.rados.rados.0.smithi022.stdout:146: expect (ObjNum 32 snap 0 seq_num 32) 2024-09-21T09:31:35.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:34 smithi064 ceph-mon[31124]: osdmap e82: 8 total, 8 up, 8 in 2024-09-21T09:31:36.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:35 smithi022 ceph-mon[39525]: pgmap v119: 121 pgs: 121 active+clean; 34 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.7 MiB/s wr, 37 op/s 2024-09-21T09:31:36.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:35 smithi022 ceph-mon[39525]: osdmap e83: 8 total, 8 up, 8 in 2024-09-21T09:31:36.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:35 smithi022 ceph-mon[39525]: osdmap e84: 8 total, 8 up, 8 in 2024-09-21T09:31:36.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:35 smithi022 ceph-mon[27763]: pgmap v119: 121 pgs: 121 active+clean; 34 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.7 MiB/s wr, 37 op/s 2024-09-21T09:31:36.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:35 smithi022 ceph-mon[27763]: osdmap e83: 8 total, 8 up, 8 in 2024-09-21T09:31:36.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:35 smithi022 ceph-mon[27763]: osdmap e84: 8 total, 8 up, 8 in 2024-09-21T09:31:36.408 INFO:tasks.rados.rados.0.smithi022.stdout:146: done (2 left) 2024-09-21T09:31:36.408 INFO:tasks.rados.rados.0.smithi022.stdout:147: done (1 left) 2024-09-21T09:31:36.408 INFO:tasks.rados.rados.0.smithi022.stdout:148: done (0 left) 2024-09-21T09:31:36.408 INFO:tasks.rados.rados.0.smithi022.stdout:149: copy_from oid 48 from oid 42 current snap is 8 2024-09-21T09:31:36.409 INFO:tasks.rados.rados.0.smithi022.stdout:150: delete oid 29 current snap is 8 2024-09-21T09:31:36.410 INFO:tasks.rados.rados.0.smithi022.stdout:150: done (1 left) 2024-09-21T09:31:36.410 INFO:tasks.rados.rados.0.smithi022.stdout:151: snap_create 2024-09-21T09:31:36.418 INFO:tasks.rados.rados.0.smithi022.stdout:149: finishing copy_from racing read to smithi02284960-48 2024-09-21T09:31:36.419 INFO:tasks.rados.rados.0.smithi022.stdout:149: finishing copy_from to smithi02284960-48 2024-09-21T09:31:36.419 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 48 v 10 (ObjNum 46 snap 0 seq_num 46) dirty exists 2024-09-21T09:31:36.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:35 smithi064 ceph-mon[31124]: pgmap v119: 121 pgs: 121 active+clean; 34 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.7 MiB/s wr, 37 op/s 2024-09-21T09:31:36.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:35 smithi064 ceph-mon[31124]: osdmap e83: 8 total, 8 up, 8 in 2024-09-21T09:31:36.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:35 smithi064 ceph-mon[31124]: osdmap e84: 8 total, 8 up, 8 in 2024-09-21T09:31:37.413 INFO:tasks.rados.rados.0.smithi022.stdout:149: done (1 left) 2024-09-21T09:31:37.413 INFO:tasks.rados.rados.0.smithi022.stdout:151: done (0 left) 2024-09-21T09:31:37.413 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 43 current snap is 9 2024-09-21T09:31:37.414 INFO:tasks.rados.rados.0.smithi022.stdout:152: seq_num 76 ranges {712704=475136,1187840=458752,1646592=344064} 2024-09-21T09:31:37.419 INFO:tasks.rados.rados.0.smithi022.stdout:152: writing smithi02284960-43 from 712704 to 1187840 tid 1 2024-09-21T09:31:37.422 INFO:tasks.rados.rados.0.smithi022.stdout:152: writing smithi02284960-43 from 1187840 to 1646592 tid 2 2024-09-21T09:31:37.424 INFO:tasks.rados.rados.0.smithi022.stdout:152: writing smithi02284960-43 from 1646592 to 1990656 tid 3 2024-09-21T09:31:37.425 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 10 current snap is 9 2024-09-21T09:31:37.425 INFO:tasks.rados.rados.0.smithi022.stdout:153: seq_num 77 ranges {155648=491520,647168=794624,1441792=8192} 2024-09-21T09:31:37.428 INFO:tasks.rados.rados.0.smithi022.stdout:153: writing smithi02284960-10 from 155648 to 647168 tid 1 2024-09-21T09:31:37.434 INFO:tasks.rados.rados.0.smithi022.stdout:153: writing smithi02284960-10 from 647168 to 1441792 tid 2 2024-09-21T09:31:37.434 INFO:tasks.rados.rados.0.smithi022.stdout:153: writing smithi02284960-10 from 1441792 to 1449984 tid 3 2024-09-21T09:31:37.434 INFO:tasks.rados.rados.0.smithi022.stdout:154: snap_create 2024-09-21T09:31:37.434 INFO:tasks.rados.rados.0.smithi022.stdout:152: finishing write tid 1 to smithi02284960-43 2024-09-21T09:31:37.434 INFO:tasks.rados.rados.0.smithi022.stdout:152: finishing write tid 2 to smithi02284960-43 2024-09-21T09:31:37.434 INFO:tasks.rados.rados.0.smithi022.stdout:152: finishing write tid 3 to smithi02284960-43 2024-09-21T09:31:37.434 INFO:tasks.rados.rados.0.smithi022.stdout:152: finishing write tid 4 to smithi02284960-43 2024-09-21T09:31:37.434 INFO:tasks.rados.rados.0.smithi022.stdout:152: finishing write tid 5 to smithi02284960-43 2024-09-21T09:31:37.434 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 43 v 29 (ObjNum 76 snap 9 seq_num 76) dirty exists 2024-09-21T09:31:37.434 INFO:tasks.rados.rados.0.smithi022.stdout:152: left oid 43 (ObjNum 76 snap 9 seq_num 76) 2024-09-21T09:31:37.434 INFO:tasks.rados.rados.0.smithi022.stdout:153: finishing write tid 1 to smithi02284960-10 2024-09-21T09:31:37.436 INFO:tasks.rados.rados.0.smithi022.stdout:153: finishing write tid 2 to smithi02284960-10 2024-09-21T09:31:37.437 INFO:tasks.rados.rados.0.smithi022.stdout:153: finishing write tid 3 to smithi02284960-10 2024-09-21T09:31:37.437 INFO:tasks.rados.rados.0.smithi022.stdout:153: finishing write tid 4 to smithi02284960-10 2024-09-21T09:31:37.438 INFO:tasks.rados.rados.0.smithi022.stdout:153: finishing write tid 5 to smithi02284960-10 2024-09-21T09:31:37.438 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 10 v 34 (ObjNum 77 snap 9 seq_num 77) dirty exists 2024-09-21T09:31:37.438 INFO:tasks.rados.rados.0.smithi022.stdout:153: left oid 10 (ObjNum 77 snap 9 seq_num 77) 2024-09-21T09:31:37.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:37 smithi064 ceph-mon[31124]: osdmap e85: 8 total, 8 up, 8 in 2024-09-21T09:31:37.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:37 smithi064 ceph-mon[31124]: pgmap v123: 121 pgs: 121 active+clean; 34 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:31:37.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:37 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:31:37.807 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:37 smithi022 ceph-mon[39525]: osdmap e85: 8 total, 8 up, 8 in 2024-09-21T09:31:37.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:37 smithi022 ceph-mon[39525]: pgmap v123: 121 pgs: 121 active+clean; 34 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:31:37.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:37 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:31:37.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:37 smithi022 ceph-mon[27763]: osdmap e85: 8 total, 8 up, 8 in 2024-09-21T09:31:37.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:37 smithi022 ceph-mon[27763]: pgmap v123: 121 pgs: 121 active+clean; 34 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:31:37.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:37 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:31:38.418 INFO:tasks.rados.rados.0.smithi022.stdout:152: done (2 left) 2024-09-21T09:31:38.418 INFO:tasks.rados.rados.0.smithi022.stdout:153: done (1 left) 2024-09-21T09:31:38.418 INFO:tasks.rados.rados.0.smithi022.stdout:154: done (0 left) 2024-09-21T09:31:38.418 INFO:tasks.rados.rados.0.smithi022.stdout:155: snap_create 2024-09-21T09:31:38.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:38 smithi064 ceph-mon[31124]: osdmap e86: 8 total, 8 up, 8 in 2024-09-21T09:31:38.807 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:38 smithi022 ceph-mon[39525]: osdmap e86: 8 total, 8 up, 8 in 2024-09-21T09:31:38.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:38 smithi022 ceph-mon[27763]: osdmap e86: 8 total, 8 up, 8 in 2024-09-21T09:31:39.422 INFO:tasks.rados.rados.0.smithi022.stdout:155: done (0 left) 2024-09-21T09:31:39.422 INFO:tasks.rados.rados.0.smithi022.stdout:156: read oid 42 snap -1 2024-09-21T09:31:39.422 INFO:tasks.rados.rados.0.smithi022.stdout:156: expect (ObjNum 46 snap 0 seq_num 46) 2024-09-21T09:31:39.422 INFO:tasks.rados.rados.0.smithi022.stdout:157: rmattr oid 17 current snap is 11 2024-09-21T09:31:39.422 INFO:tasks.rados.rados.0.smithi022.stdout:157: done (1 left) 2024-09-21T09:31:39.422 INFO:tasks.rados.rados.0.smithi022.stdout:158: setattr oid 28 current snap is 11 2024-09-21T09:31:39.424 INFO:tasks.rados.rados.0.smithi022.stdout:159: copy_from oid 33 from oid 16 current snap is 11 2024-09-21T09:31:39.424 INFO:tasks.rados.rados.0.smithi022.stdout:160: setattr oid 32 current snap is 11 2024-09-21T09:31:39.427 INFO:tasks.rados.rados.0.smithi022.stdout:161: snap_remove snap 0 2024-09-21T09:31:39.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:39 smithi064 ceph-mon[31124]: osdmap e87: 8 total, 8 up, 8 in 2024-09-21T09:31:39.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:39 smithi064 ceph-mon[31124]: pgmap v126: 121 pgs: 121 active+clean; 43 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.9 MiB/s rd, 2.4 MiB/s wr, 29 op/s 2024-09-21T09:31:39.807 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:39 smithi022 ceph-mon[39525]: osdmap e87: 8 total, 8 up, 8 in 2024-09-21T09:31:39.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:39 smithi022 ceph-mon[39525]: pgmap v126: 121 pgs: 121 active+clean; 43 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.9 MiB/s rd, 2.4 MiB/s wr, 29 op/s 2024-09-21T09:31:39.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:39 smithi022 ceph-mon[27763]: osdmap e87: 8 total, 8 up, 8 in 2024-09-21T09:31:39.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:39 smithi022 ceph-mon[27763]: pgmap v126: 121 pgs: 121 active+clean; 43 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.9 MiB/s rd, 2.4 MiB/s wr, 29 op/s 2024-09-21T09:31:40.408 INFO:tasks.rados.rados.0.smithi022.stdout:161: done (4 left) 2024-09-21T09:31:40.408 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 25 current snap is 11 2024-09-21T09:31:40.408 INFO:tasks.rados.rados.0.smithi022.stdout:162: seq_num 78 ranges {606208=516096,1122304=679936,1802240=122880} 2024-09-21T09:31:40.416 INFO:tasks.rados.rados.0.smithi022.stdout:162: writing smithi02284960-25 from 606208 to 1122304 tid 1 2024-09-21T09:31:40.422 INFO:tasks.rados.rados.0.smithi022.stdout:162: writing smithi02284960-25 from 1122304 to 1802240 tid 2 2024-09-21T09:31:40.423 INFO:tasks.rados.rados.0.smithi022.stdout:162: writing smithi02284960-25 from 1802240 to 1925120 tid 3 2024-09-21T09:31:40.423 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 35 current snap is 11 2024-09-21T09:31:40.423 INFO:tasks.rados.rados.0.smithi022.stdout:163: seq_num 79 ranges {573440=532480,1105920=565248,1671168=237568} 2024-09-21T09:31:40.428 INFO:tasks.rados.rados.0.smithi022.stdout:163: writing smithi02284960-35 from 573440 to 1105920 tid 1 2024-09-21T09:31:40.432 INFO:tasks.rados.rados.0.smithi022.stdout:163: writing smithi02284960-35 from 1105920 to 1671168 tid 2 2024-09-21T09:31:40.436 INFO:tasks.rados.rados.0.smithi022.stdout:163: writing smithi02284960-35 from 1671168 to 1908736 tid 3 2024-09-21T09:31:40.436 INFO:tasks.rados.rados.0.smithi022.stdout:164: snap_remove snap 10 2024-09-21T09:31:40.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:40 smithi064 ceph-mon[31124]: osdmap e88: 8 total, 8 up, 8 in 2024-09-21T09:31:40.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:40 smithi064 ceph-mon[31124]: osdmap e89: 8 total, 8 up, 8 in 2024-09-21T09:31:40.692 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:31:40 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[33213]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:31:40] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-21T09:31:40.807 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:40 smithi022 ceph-mon[39525]: osdmap e88: 8 total, 8 up, 8 in 2024-09-21T09:31:40.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:40 smithi022 ceph-mon[39525]: osdmap e89: 8 total, 8 up, 8 in 2024-09-21T09:31:40.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:40 smithi022 ceph-mon[27763]: osdmap e88: 8 total, 8 up, 8 in 2024-09-21T09:31:40.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:40 smithi022 ceph-mon[27763]: osdmap e89: 8 total, 8 up, 8 in 2024-09-21T09:31:41.428 INFO:tasks.rados.rados.0.smithi022.stdout:164: done (6 left) 2024-09-21T09:31:41.428 INFO:tasks.rados.rados.0.smithi022.stdout:165: delete oid 31 current snap is 11 2024-09-21T09:31:41.428 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 28 v 16 (ObjNum 9 snap 0 seq_num 9) dirty exists 2024-09-21T09:31:41.428 INFO:tasks.rados.rados.0.smithi022.stdout:156: expect (ObjNum 46 snap 0 seq_num 46) 2024-09-21T09:31:41.452 INFO:tasks.rados.rados.0.smithi022.stdout:159: finishing copy_from to smithi02284960-33 2024-09-21T09:31:41.452 INFO:tasks.rados.rados.0.smithi022.stdout:159: got expected ENOENT (src dne) 2024-09-21T09:31:41.452 INFO:tasks.rados.rados.0.smithi022.stdout:159: finishing copy_from racing read to smithi02284960-33 2024-09-21T09:31:41.453 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 32 v 46 (ObjNum 31 snap 0 seq_num 31) dirty exists 2024-09-21T09:31:41.453 INFO:tasks.rados.rados.0.smithi022.stdout:156: done (6 left) 2024-09-21T09:31:41.453 INFO:tasks.rados.rados.0.smithi022.stdout:158: done (5 left) 2024-09-21T09:31:41.453 INFO:tasks.rados.rados.0.smithi022.stdout:159: done (4 left) 2024-09-21T09:31:41.453 INFO:tasks.rados.rados.0.smithi022.stdout:160: done (3 left) 2024-09-21T09:31:41.453 INFO:tasks.rados.rados.0.smithi022.stdout:165: done (2 left) 2024-09-21T09:31:41.453 INFO:tasks.rados.rados.0.smithi022.stdout:166: snap_create 2024-09-21T09:31:41.453 INFO:tasks.rados.rados.0.smithi022.stdout:162: finishing write tid 1 to smithi02284960-25 2024-09-21T09:31:41.453 INFO:tasks.rados.rados.0.smithi022.stdout:162: finishing write tid 2 to smithi02284960-25 2024-09-21T09:31:41.453 INFO:tasks.rados.rados.0.smithi022.stdout:162: finishing write tid 3 to smithi02284960-25 2024-09-21T09:31:41.453 INFO:tasks.rados.rados.0.smithi022.stdout:162: finishing write tid 4 to smithi02284960-25 2024-09-21T09:31:41.453 INFO:tasks.rados.rados.0.smithi022.stdout:162: finishing write tid 5 to smithi02284960-25 2024-09-21T09:31:41.453 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 25 v 47 (ObjNum 78 snap 11 seq_num 78) dirty exists 2024-09-21T09:31:41.453 INFO:tasks.rados.rados.0.smithi022.stdout:162: left oid 25 (ObjNum 78 snap 11 seq_num 78) 2024-09-21T09:31:41.454 INFO:tasks.rados.rados.0.smithi022.stdout:163: finishing write tid 1 to smithi02284960-35 2024-09-21T09:31:41.454 INFO:tasks.rados.rados.0.smithi022.stdout:163: finishing write tid 2 to smithi02284960-35 2024-09-21T09:31:41.454 INFO:tasks.rados.rados.0.smithi022.stdout:163: finishing write tid 3 to smithi02284960-35 2024-09-21T09:31:41.454 INFO:tasks.rados.rados.0.smithi022.stdout:163: finishing write tid 4 to smithi02284960-35 2024-09-21T09:31:41.454 INFO:tasks.rados.rados.0.smithi022.stdout:163: finishing write tid 5 to smithi02284960-35 2024-09-21T09:31:41.454 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 35 v 20 (ObjNum 79 snap 11 seq_num 79) dirty exists 2024-09-21T09:31:41.454 INFO:tasks.rados.rados.0.smithi022.stdout:163: left oid 35 (ObjNum 79 snap 11 seq_num 79) 2024-09-21T09:31:41.557 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:41 smithi022 ceph-mon[39525]: pgmap v129: 121 pgs: 121 active+clean; 43 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.3 MiB/s wr, 28 op/s 2024-09-21T09:31:41.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:41 smithi022 ceph-mon[27763]: pgmap v129: 121 pgs: 121 active+clean; 43 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.3 MiB/s wr, 28 op/s 2024-09-21T09:31:41.558 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:31:41 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:31:41] "GET /metrics HTTP/1.1" 200 238835 "" "Prometheus/2.43.0" 2024-09-21T09:31:41.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:41 smithi064 ceph-mon[31124]: pgmap v129: 121 pgs: 121 active+clean; 43 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.3 MiB/s wr, 28 op/s 2024-09-21T09:31:42.433 INFO:tasks.rados.rados.0.smithi022.stdout:162: done (2 left) 2024-09-21T09:31:42.433 INFO:tasks.rados.rados.0.smithi022.stdout:163: done (1 left) 2024-09-21T09:31:42.433 INFO:tasks.rados.rados.0.smithi022.stdout:166: done (0 left) 2024-09-21T09:31:42.433 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 31 current snap is 12 2024-09-21T09:31:42.433 INFO:tasks.rados.rados.0.smithi022.stdout:167: seq_num 81 ranges {0=565248,565248=737280,1302528=65536} 2024-09-21T09:31:42.436 INFO:tasks.rados.rados.0.smithi022.stdout:167: writing smithi02284960-31 from 0 to 565248 tid 1 2024-09-21T09:31:42.440 INFO:tasks.rados.rados.0.smithi022.stdout:167: writing smithi02284960-31 from 565248 to 1302528 tid 2 2024-09-21T09:31:42.440 INFO:tasks.rados.rados.0.smithi022.stdout:167: writing smithi02284960-31 from 1302528 to 1368064 tid 3 2024-09-21T09:31:42.441 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 16 current snap is 12 2024-09-21T09:31:42.441 INFO:tasks.rados.rados.0.smithi022.stdout:168: seq_num 82 ranges {0=581632,581632=622592,1204224=180224} 2024-09-21T09:31:42.444 INFO:tasks.rados.rados.0.smithi022.stdout:168: writing smithi02284960-16 from 0 to 581632 tid 1 2024-09-21T09:31:42.449 INFO:tasks.rados.rados.0.smithi022.stdout:168: writing smithi02284960-16 from 581632 to 1204224 tid 2 2024-09-21T09:31:42.451 INFO:tasks.rados.rados.0.smithi022.stdout:168: writing smithi02284960-16 from 1204224 to 1384448 tid 3 2024-09-21T09:31:42.451 INFO:tasks.rados.rados.0.smithi022.stdout:169: copy_from oid 10 from oid 39 current snap is 12 2024-09-21T09:31:42.451 INFO:tasks.rados.rados.0.smithi022.stdout:167: finishing write tid 1 to smithi02284960-31 2024-09-21T09:31:42.451 INFO:tasks.rados.rados.0.smithi022.stdout:167: finishing write tid 2 to smithi02284960-31 2024-09-21T09:31:42.451 INFO:tasks.rados.rados.0.smithi022.stdout:167: finishing write tid 3 to smithi02284960-31 2024-09-21T09:31:42.451 INFO:tasks.rados.rados.0.smithi022.stdout:167: finishing write tid 4 to smithi02284960-31 2024-09-21T09:31:42.451 INFO:tasks.rados.rados.0.smithi022.stdout:167: finishing write tid 5 to smithi02284960-31 2024-09-21T09:31:42.451 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 31 v 61 (ObjNum 81 snap 12 seq_num 81) dirty exists 2024-09-21T09:31:42.451 INFO:tasks.rados.rados.0.smithi022.stdout:167: left oid 31 (ObjNum 81 snap 12 seq_num 81) 2024-09-21T09:31:42.451 INFO:tasks.rados.rados.0.smithi022.stdout:168: finishing write tid 1 to smithi02284960-16 2024-09-21T09:31:42.452 INFO:tasks.rados.rados.0.smithi022.stdout:167: done (2 left) 2024-09-21T09:31:42.452 INFO:tasks.rados.rados.0.smithi022.stdout:170: read oid 25 snap 7 2024-09-21T09:31:42.452 INFO:tasks.rados.rados.0.smithi022.stdout:170: expect (ObjNum 36 snap 0 seq_num 36) 2024-09-21T09:31:42.452 INFO:tasks.rados.rados.0.smithi022.stdout:171: copy_from oid 21 from oid 35 current snap is 12 2024-09-21T09:31:42.452 INFO:tasks.rados.rados.0.smithi022.stdout:172: snap_remove snap 6 2024-09-21T09:31:42.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:42 smithi064 ceph-mon[31124]: osdmap e90: 8 total, 8 up, 8 in 2024-09-21T09:31:42.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:42 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:31:42.807 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:42 smithi022 ceph-mon[39525]: osdmap e90: 8 total, 8 up, 8 in 2024-09-21T09:31:42.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:42 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:31:42.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:42 smithi022 ceph-mon[27763]: osdmap e90: 8 total, 8 up, 8 in 2024-09-21T09:31:42.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:42 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:31:43.439 INFO:tasks.rados.rados.0.smithi022.stdout:172: done (4 left) 2024-09-21T09:31:43.439 INFO:tasks.rados.rados.0.smithi022.stdout:173: delete oid 8 current snap is 12 2024-09-21T09:31:43.440 INFO:tasks.rados.rados.0.smithi022.stdout:168: finishing write tid 2 to smithi02284960-16 2024-09-21T09:31:43.440 INFO:tasks.rados.rados.0.smithi022.stdout:168: finishing write tid 3 to smithi02284960-16 2024-09-21T09:31:43.440 INFO:tasks.rados.rados.0.smithi022.stdout:168: finishing write tid 4 to smithi02284960-16 2024-09-21T09:31:43.440 INFO:tasks.rados.rados.0.smithi022.stdout:168: finishing write tid 5 to smithi02284960-16 2024-09-21T09:31:43.440 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 16 v 12 (ObjNum 82 snap 12 seq_num 82) dirty exists 2024-09-21T09:31:43.440 INFO:tasks.rados.rados.0.smithi022.stdout:168: left oid 16 (ObjNum 82 snap 12 seq_num 82) 2024-09-21T09:31:43.440 INFO:tasks.rados.rados.0.smithi022.stdout:170: expect (ObjNum 36 snap 0 seq_num 36) 2024-09-21T09:31:43.459 INFO:tasks.rados.rados.0.smithi022.stdout:169: finishing copy_from racing read to smithi02284960-10 2024-09-21T09:31:43.459 INFO:tasks.rados.rados.0.smithi022.stdout:169: finishing copy_from to smithi02284960-10 2024-09-21T09:31:43.459 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 10 v 10 (ObjNum 38 snap 0 seq_num 38) dirty exists 2024-09-21T09:31:43.459 INFO:tasks.rados.rados.0.smithi022.stdout:171: finishing copy_from racing read to smithi02284960-21 2024-09-21T09:31:43.459 INFO:tasks.rados.rados.0.smithi022.stdout:171: finishing copy_from to smithi02284960-21 2024-09-21T09:31:43.459 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 21 v 20 (ObjNum 79 snap 11 seq_num 79) dirty exists 2024-09-21T09:31:43.459 INFO:tasks.rados.rados.0.smithi022.stdout:168: done (4 left) 2024-09-21T09:31:43.459 INFO:tasks.rados.rados.0.smithi022.stdout:169: done (3 left) 2024-09-21T09:31:43.459 INFO:tasks.rados.rados.0.smithi022.stdout:170: done (2 left) 2024-09-21T09:31:43.459 INFO:tasks.rados.rados.0.smithi022.stdout:171: done (1 left) 2024-09-21T09:31:43.459 INFO:tasks.rados.rados.0.smithi022.stdout:173: done (0 left) 2024-09-21T09:31:43.459 INFO:tasks.rados.rados.0.smithi022.stdout:174: snap_remove snap 7 2024-09-21T09:31:43.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:43 smithi064 ceph-mon[31124]: osdmap e91: 8 total, 8 up, 8 in 2024-09-21T09:31:43.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:43 smithi064 ceph-mon[31124]: pgmap v132: 121 pgs: 121 active+clean; 45 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1015 KiB/s rd, 1.9 MiB/s wr, 18 op/s 2024-09-21T09:31:43.807 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:43 smithi022 ceph-mon[39525]: osdmap e91: 8 total, 8 up, 8 in 2024-09-21T09:31:43.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:43 smithi022 ceph-mon[39525]: pgmap v132: 121 pgs: 121 active+clean; 45 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1015 KiB/s rd, 1.9 MiB/s wr, 18 op/s 2024-09-21T09:31:43.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:43 smithi022 ceph-mon[27763]: osdmap e91: 8 total, 8 up, 8 in 2024-09-21T09:31:43.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:43 smithi022 ceph-mon[27763]: pgmap v132: 121 pgs: 121 active+clean; 45 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1015 KiB/s rd, 1.9 MiB/s wr, 18 op/s 2024-09-21T09:31:44.445 INFO:tasks.rados.rados.0.smithi022.stdout:174: done (0 left) 2024-09-21T09:31:44.445 INFO:tasks.rados.rados.0.smithi022.stdout:175: snap_create 2024-09-21T09:31:44.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:44 smithi064 ceph-mon[31124]: osdmap e92: 8 total, 8 up, 8 in 2024-09-21T09:31:44.807 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:44 smithi022 ceph-mon[39525]: osdmap e92: 8 total, 8 up, 8 in 2024-09-21T09:31:44.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:44 smithi022 ceph-mon[27763]: osdmap e92: 8 total, 8 up, 8 in 2024-09-21T09:31:45.410 INFO:tasks.rados.rados.0.smithi022.stdout:175: done (0 left) 2024-09-21T09:31:45.410 INFO:tasks.rados.rados.0.smithi022.stdout:176: read oid 46 snap -1 2024-09-21T09:31:45.410 INFO:tasks.rados.rados.0.smithi022.stdout:176: expect (ObjNum 45 snap 0 seq_num 45) 2024-09-21T09:31:45.410 INFO:tasks.rados.rados.0.smithi022.stdout:177: read oid 7 snap 3 2024-09-21T09:31:45.410 INFO:tasks.rados.rados.0.smithi022.stdout:177: expect (ObjNum 6 snap 0 seq_num 6) 2024-09-21T09:31:45.410 INFO:tasks.rados.rados.0.smithi022.stdout:178: snap_create 2024-09-21T09:31:45.414 INFO:tasks.rados.rados.0.smithi022.stdout:177: expect (ObjNum 6 snap 0 seq_num 6) 2024-09-21T09:31:45.417 INFO:tasks.rados.rados.0.smithi022.stdout:176: expect (ObjNum 45 snap 0 seq_num 45) 2024-09-21T09:31:45.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:45 smithi022 ceph-mon[39525]: osdmap e93: 8 total, 8 up, 8 in 2024-09-21T09:31:45.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:45 smithi022 ceph-mon[39525]: pgmap v135: 121 pgs: 121 active+clean; 45 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1015 KiB/s rd, 1.9 MiB/s wr, 18 op/s 2024-09-21T09:31:45.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:45 smithi022 ceph-mon[39525]: osdmap e94: 8 total, 8 up, 8 in 2024-09-21T09:31:45.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:45 smithi022 ceph-mon[27763]: osdmap e93: 8 total, 8 up, 8 in 2024-09-21T09:31:45.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:45 smithi022 ceph-mon[27763]: pgmap v135: 121 pgs: 121 active+clean; 45 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1015 KiB/s rd, 1.9 MiB/s wr, 18 op/s 2024-09-21T09:31:45.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:45 smithi022 ceph-mon[27763]: osdmap e94: 8 total, 8 up, 8 in 2024-09-21T09:31:45.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:45 smithi064 ceph-mon[31124]: osdmap e93: 8 total, 8 up, 8 in 2024-09-21T09:31:45.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:45 smithi064 ceph-mon[31124]: pgmap v135: 121 pgs: 121 active+clean; 45 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1015 KiB/s rd, 1.9 MiB/s wr, 18 op/s 2024-09-21T09:31:45.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:45 smithi064 ceph-mon[31124]: osdmap e94: 8 total, 8 up, 8 in 2024-09-21T09:31:46.411 INFO:tasks.rados.rados.0.smithi022.stdout:176: done (2 left) 2024-09-21T09:31:46.411 INFO:tasks.rados.rados.0.smithi022.stdout:177: done (1 left) 2024-09-21T09:31:46.411 INFO:tasks.rados.rados.0.smithi022.stdout:178: done (0 left) 2024-09-21T09:31:46.411 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 13 current snap is 14 2024-09-21T09:31:46.411 INFO:tasks.rados.rados.0.smithi022.stdout:179: seq_num 84 ranges {204800=614400,819200=434176,1253376=368640} 2024-09-21T09:31:46.415 INFO:tasks.rados.rados.0.smithi022.stdout:179: writing smithi02284960-13 from 204800 to 819200 tid 1 2024-09-21T09:31:46.418 INFO:tasks.rados.rados.0.smithi022.stdout:179: writing smithi02284960-13 from 819200 to 1253376 tid 2 2024-09-21T09:31:46.420 INFO:tasks.rados.rados.0.smithi022.stdout:179: writing smithi02284960-13 from 1253376 to 1622016 tid 3 2024-09-21T09:31:46.420 INFO:tasks.rados.rados.0.smithi022.stdout:180: copy_from oid 15 from oid 5 current snap is 14 2024-09-21T09:31:46.421 INFO:tasks.rados.rados.0.smithi022.stdout:181: rollback oid 19 current snap is 14 2024-09-21T09:31:46.421 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 19 to 8 2024-09-21T09:31:46.421 INFO:tasks.rados.rados.0.smithi022.stdout:182: copy_from oid 21 from oid 24 current snap is 14 2024-09-21T09:31:46.421 INFO:tasks.rados.rados.0.smithi022.stdout:183: copy_from oid 3 from oid 7 current snap is 14 2024-09-21T09:31:46.421 INFO:tasks.rados.rados.0.smithi022.stdout:184: read oid 33 snap -1 2024-09-21T09:31:46.421 INFO:tasks.rados.rados.0.smithi022.stdout:184: expect (ObjNum 32 snap 0 seq_num 32) 2024-09-21T09:31:46.421 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 16 current snap is 14 2024-09-21T09:31:46.421 INFO:tasks.rados.rados.0.smithi022.stdout:185: seq_num 85 ranges {1384448=630784,2015232=720896,2736128=81920} 2024-09-21T09:31:46.432 INFO:tasks.rados.rados.0.smithi022.stdout:185: writing smithi02284960-16 from 1384448 to 2015232 tid 1 2024-09-21T09:31:46.436 INFO:tasks.rados.rados.0.smithi022.stdout:185: writing smithi02284960-16 from 2015232 to 2736128 tid 2 2024-09-21T09:31:46.437 INFO:tasks.rados.rados.0.smithi022.stdout:185: writing smithi02284960-16 from 2736128 to 2818048 tid 3 2024-09-21T09:31:46.437 INFO:tasks.rados.rados.0.smithi022.stdout:186: rmattr oid 34 current snap is 14 2024-09-21T09:31:46.437 INFO:tasks.rados.rados.0.smithi022.stdout:186: done (7 left) 2024-09-21T09:31:46.437 INFO:tasks.rados.rados.0.smithi022.stdout:187: snap_remove snap 3 2024-09-21T09:31:47.417 INFO:tasks.rados.rados.0.smithi022.stdout:187: done (7 left) 2024-09-21T09:31:47.417 INFO:tasks.rados.rados.0.smithi022.stdout:188: read oid 39 snap -1 2024-09-21T09:31:47.417 INFO:tasks.rados.rados.0.smithi022.stdout:188: expect (ObjNum 38 snap 0 seq_num 38) 2024-09-21T09:31:47.417 INFO:tasks.rados.rados.0.smithi022.stdout:189: rmattr oid 4 current snap is 14 2024-09-21T09:31:47.417 INFO:tasks.rados.rados.0.smithi022.stdout:189: done (8 left) 2024-09-21T09:31:47.417 INFO:tasks.rados.rados.0.smithi022.stdout:190: snap_create 2024-09-21T09:31:47.417 INFO:tasks.rados.rados.0.smithi022.stdout:184: expect (ObjNum 32 snap 0 seq_num 32) 2024-09-21T09:31:47.435 INFO:tasks.rados.rados.0.smithi022.stdout:181: finishing rollback tid 1 to smithi02284960-19 2024-09-21T09:31:47.435 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 19 v 34 (ObjNum 65 snap 0 seq_num 3570287296) dirty dne 2024-09-21T09:31:47.435 INFO:tasks.rados.rados.0.smithi022.stdout:179: finishing write tid 1 to smithi02284960-13 2024-09-21T09:31:47.435 INFO:tasks.rados.rados.0.smithi022.stdout:179: finishing write tid 2 to smithi02284960-13 2024-09-21T09:31:47.435 INFO:tasks.rados.rados.0.smithi022.stdout:179: finishing write tid 3 to smithi02284960-13 2024-09-21T09:31:47.435 INFO:tasks.rados.rados.0.smithi022.stdout:179: finishing write tid 4 to smithi02284960-13 2024-09-21T09:31:47.435 INFO:tasks.rados.rados.0.smithi022.stdout:179: finishing write tid 5 to smithi02284960-13 2024-09-21T09:31:47.435 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 13 v 25 (ObjNum 84 snap 14 seq_num 84) dirty exists 2024-09-21T09:31:47.435 INFO:tasks.rados.rados.0.smithi022.stdout:179: left oid 13 (ObjNum 84 snap 14 seq_num 84) 2024-09-21T09:31:47.435 INFO:tasks.rados.rados.0.smithi022.stdout:180: finishing copy_from racing read to smithi02284960-15 2024-09-21T09:31:47.435 INFO:tasks.rados.rados.0.smithi022.stdout:180: finishing copy_from to smithi02284960-15 2024-09-21T09:31:47.435 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 15 v 4 (ObjNum 4 snap 0 seq_num 4) dirty exists 2024-09-21T09:31:47.436 INFO:tasks.rados.rados.0.smithi022.stdout:182: finishing copy_from racing read to smithi02284960-21 2024-09-21T09:31:47.436 INFO:tasks.rados.rados.0.smithi022.stdout:182: finishing copy_from to smithi02284960-21 2024-09-21T09:31:47.436 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 21 v 8 (ObjNum 23 snap 0 seq_num 23) dirty exists 2024-09-21T09:31:47.436 INFO:tasks.rados.rados.0.smithi022.stdout:183: finishing copy_from racing read to smithi02284960-3 2024-09-21T09:31:47.436 INFO:tasks.rados.rados.0.smithi022.stdout:183: finishing copy_from to smithi02284960-3 2024-09-21T09:31:47.436 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 3 v 42 (ObjNum 72 snap 5 seq_num 72) dirty exists 2024-09-21T09:31:47.436 INFO:tasks.rados.rados.0.smithi022.stdout:185: finishing write tid 1 to smithi02284960-16 2024-09-21T09:31:47.436 INFO:tasks.rados.rados.0.smithi022.stdout:185: finishing write tid 2 to smithi02284960-16 2024-09-21T09:31:47.436 INFO:tasks.rados.rados.0.smithi022.stdout:185: finishing write tid 3 to smithi02284960-16 2024-09-21T09:31:47.436 INFO:tasks.rados.rados.0.smithi022.stdout:185: finishing write tid 4 to smithi02284960-16 2024-09-21T09:31:47.436 INFO:tasks.rados.rados.0.smithi022.stdout:185: finishing write tid 5 to smithi02284960-16 2024-09-21T09:31:47.436 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 16 v 17 (ObjNum 85 snap 14 seq_num 85) dirty exists 2024-09-21T09:31:47.436 INFO:tasks.rados.rados.0.smithi022.stdout:185: left oid 16 (ObjNum 85 snap 14 seq_num 85) 2024-09-21T09:31:47.436 INFO:tasks.rados.rados.0.smithi022.stdout:188: expect (ObjNum 38 snap 0 seq_num 38) 2024-09-21T09:31:47.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:47 smithi064 ceph-mon[31124]: osdmap e95: 8 total, 8 up, 8 in 2024-09-21T09:31:47.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:47 smithi064 ceph-mon[31124]: pgmap v138: 121 pgs: 121 active+clean; 45 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:31:47.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:47 smithi022 ceph-mon[39525]: osdmap e95: 8 total, 8 up, 8 in 2024-09-21T09:31:47.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:47 smithi022 ceph-mon[39525]: pgmap v138: 121 pgs: 121 active+clean; 45 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:31:47.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:47 smithi022 ceph-mon[27763]: osdmap e95: 8 total, 8 up, 8 in 2024-09-21T09:31:47.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:47 smithi022 ceph-mon[27763]: pgmap v138: 121 pgs: 121 active+clean; 45 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:31:48.418 INFO:tasks.rados.rados.0.smithi022.stdout:179: done (8 left) 2024-09-21T09:31:48.418 INFO:tasks.rados.rados.0.smithi022.stdout:180: done (7 left) 2024-09-21T09:31:48.418 INFO:tasks.rados.rados.0.smithi022.stdout:181: done (6 left) 2024-09-21T09:31:48.418 INFO:tasks.rados.rados.0.smithi022.stdout:182: done (5 left) 2024-09-21T09:31:48.418 INFO:tasks.rados.rados.0.smithi022.stdout:183: done (4 left) 2024-09-21T09:31:48.418 INFO:tasks.rados.rados.0.smithi022.stdout:184: done (3 left) 2024-09-21T09:31:48.418 INFO:tasks.rados.rados.0.smithi022.stdout:185: done (2 left) 2024-09-21T09:31:48.418 INFO:tasks.rados.rados.0.smithi022.stdout:188: done (1 left) 2024-09-21T09:31:48.418 INFO:tasks.rados.rados.0.smithi022.stdout:190: done (0 left) 2024-09-21T09:31:48.418 INFO:tasks.rados.rados.0.smithi022.stdout:191: copy_from oid 17 from oid 8 current snap is 15 2024-09-21T09:31:48.419 INFO:tasks.rados.rados.0.smithi022.stdout:192: read oid 32 snap -1 2024-09-21T09:31:48.419 INFO:tasks.rados.rados.0.smithi022.stdout:192: expect (ObjNum 31 snap 0 seq_num 31) 2024-09-21T09:31:48.419 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 43 current snap is 15 2024-09-21T09:31:48.419 INFO:tasks.rados.rados.0.smithi022.stdout:193: seq_num 86 ranges {1990656=647168,2637824=606208,3244032=196608} 2024-09-21T09:31:48.432 INFO:tasks.rados.rados.0.smithi022.stdout:193: writing smithi02284960-43 from 1990656 to 2637824 tid 1 2024-09-21T09:31:48.435 INFO:tasks.rados.rados.0.smithi022.stdout:193: writing smithi02284960-43 from 2637824 to 3244032 tid 2 2024-09-21T09:31:48.436 INFO:tasks.rados.rados.0.smithi022.stdout:193: writing smithi02284960-43 from 3244032 to 3440640 tid 3 2024-09-21T09:31:48.437 INFO:tasks.rados.rados.0.smithi022.stdout:194: setattr oid 39 current snap is 15 2024-09-21T09:31:48.437 INFO:tasks.rados.rados.0.smithi022.stdout:191: finishing copy_from to smithi02284960-17 2024-09-21T09:31:48.437 INFO:tasks.rados.rados.0.smithi022.stdout:191: got expected ENOENT (src dne) 2024-09-21T09:31:48.437 INFO:tasks.rados.rados.0.smithi022.stdout:191: finishing copy_from racing read to smithi02284960-17 2024-09-21T09:31:48.437 INFO:tasks.rados.rados.0.smithi022.stdout:192: expect (ObjNum 31 snap 0 seq_num 31) 2024-09-21T09:31:48.456 INFO:tasks.rados.rados.0.smithi022.stdout:193: finishing write tid 1 to smithi02284960-43 2024-09-21T09:31:48.456 INFO:tasks.rados.rados.0.smithi022.stdout:193: finishing write tid 2 to smithi02284960-43 2024-09-21T09:31:48.456 INFO:tasks.rados.rados.0.smithi022.stdout:193: finishing write tid 3 to smithi02284960-43 2024-09-21T09:31:48.456 INFO:tasks.rados.rados.0.smithi022.stdout:193: finishing write tid 4 to smithi02284960-43 2024-09-21T09:31:48.456 INFO:tasks.rados.rados.0.smithi022.stdout:193: finishing write tid 5 to smithi02284960-43 2024-09-21T09:31:48.456 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 43 v 76 (ObjNum 86 snap 15 seq_num 86) dirty exists 2024-09-21T09:31:48.456 INFO:tasks.rados.rados.0.smithi022.stdout:193: left oid 43 (ObjNum 86 snap 15 seq_num 86) 2024-09-21T09:31:48.456 INFO:tasks.rados.rados.0.smithi022.stdout:191: done (3 left) 2024-09-21T09:31:48.456 INFO:tasks.rados.rados.0.smithi022.stdout:192: done (2 left) 2024-09-21T09:31:48.456 INFO:tasks.rados.rados.0.smithi022.stdout:193: done (1 left) 2024-09-21T09:31:48.456 INFO:tasks.rados.rados.0.smithi022.stdout:195: read oid 21 snap 13 2024-09-21T09:31:48.457 INFO:tasks.rados.rados.0.smithi022.stdout:195: expect (ObjNum 79 snap 11 seq_num 79) 2024-09-21T09:31:48.457 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 34 current snap is 15 2024-09-21T09:31:48.457 INFO:tasks.rados.rados.0.smithi022.stdout:196: seq_num 87 ranges {606208=663552,1269760=491520,1761280=311296} 2024-09-21T09:31:48.463 INFO:tasks.rados.rados.0.smithi022.stdout:196: writing smithi02284960-34 from 606208 to 1269760 tid 1 2024-09-21T09:31:48.466 INFO:tasks.rados.rados.0.smithi022.stdout:196: writing smithi02284960-34 from 1269760 to 1761280 tid 2 2024-09-21T09:31:48.468 INFO:tasks.rados.rados.0.smithi022.stdout:196: writing smithi02284960-34 from 1761280 to 2072576 tid 3 2024-09-21T09:31:48.468 INFO:tasks.rados.rados.0.smithi022.stdout:197: snap_create 2024-09-21T09:31:48.468 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 39 v 14 (ObjNum 38 snap 0 seq_num 38) dirty exists 2024-09-21T09:31:48.468 INFO:tasks.rados.rados.0.smithi022.stdout:195: expect (ObjNum 79 snap 11 seq_num 79) 2024-09-21T09:31:48.531 INFO:tasks.rados.rados.0.smithi022.stdout:196: finishing write tid 1 to smithi02284960-34 2024-09-21T09:31:48.531 INFO:tasks.rados.rados.0.smithi022.stdout:196: finishing write tid 2 to smithi02284960-34 2024-09-21T09:31:48.531 INFO:tasks.rados.rados.0.smithi022.stdout:196: finishing write tid 3 to smithi02284960-34 2024-09-21T09:31:48.531 INFO:tasks.rados.rados.0.smithi022.stdout:196: finishing write tid 4 to smithi02284960-34 2024-09-21T09:31:48.531 INFO:tasks.rados.rados.0.smithi022.stdout:196: finishing write tid 5 to smithi02284960-34 2024-09-21T09:31:48.531 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 34 v 48 (ObjNum 87 snap 15 seq_num 87) dirty exists 2024-09-21T09:31:48.531 INFO:tasks.rados.rados.0.smithi022.stdout:196: left oid 34 (ObjNum 87 snap 15 seq_num 87) 2024-09-21T09:31:48.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:48 smithi064 ceph-mon[31124]: osdmap e96: 8 total, 8 up, 8 in 2024-09-21T09:31:48.807 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:48 smithi022 ceph-mon[39525]: osdmap e96: 8 total, 8 up, 8 in 2024-09-21T09:31:48.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:48 smithi022 ceph-mon[27763]: osdmap e96: 8 total, 8 up, 8 in 2024-09-21T09:31:49.429 INFO:tasks.rados.rados.0.smithi022.stdout:194: done (3 left) 2024-09-21T09:31:49.429 INFO:tasks.rados.rados.0.smithi022.stdout:195: done (2 left) 2024-09-21T09:31:49.429 INFO:tasks.rados.rados.0.smithi022.stdout:196: done (1 left) 2024-09-21T09:31:49.429 INFO:tasks.rados.rados.0.smithi022.stdout:197: done (0 left) 2024-09-21T09:31:49.429 INFO:tasks.rados.rados.0.smithi022.stdout:198: snap_create 2024-09-21T09:31:49.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:49 smithi064 ceph-mon[31124]: osdmap e97: 8 total, 8 up, 8 in 2024-09-21T09:31:49.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:49 smithi064 ceph-mon[31124]: pgmap v141: 121 pgs: 121 active+clean; 48 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.5 MiB/s rd, 1.1 MiB/s wr, 8 op/s 2024-09-21T09:31:49.807 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:49 smithi022 ceph-mon[39525]: osdmap e97: 8 total, 8 up, 8 in 2024-09-21T09:31:49.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:49 smithi022 ceph-mon[39525]: pgmap v141: 121 pgs: 121 active+clean; 48 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.5 MiB/s rd, 1.1 MiB/s wr, 8 op/s 2024-09-21T09:31:49.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:49 smithi022 ceph-mon[27763]: osdmap e97: 8 total, 8 up, 8 in 2024-09-21T09:31:49.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:49 smithi022 ceph-mon[27763]: pgmap v141: 121 pgs: 121 active+clean; 48 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.5 MiB/s rd, 1.1 MiB/s wr, 8 op/s 2024-09-21T09:31:50.412 INFO:tasks.rados.rados.0.smithi022.stdout:198: done (0 left) 2024-09-21T09:31:50.412 INFO:tasks.rados.rados.0.smithi022.stdout:199: read oid 45 snap -1 2024-09-21T09:31:50.412 INFO:tasks.rados.rados.0.smithi022.stdout:199: expect (ObjNum 33 snap 0 seq_num 33) 2024-09-21T09:31:50.412 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 31 current snap is 17 2024-09-21T09:31:50.412 INFO:tasks.rados.rados.0.smithi022.stdout:200: seq_num 88 ranges {1368064=679936,2048000=778240,2826240=24576} 2024-09-21T09:31:50.423 INFO:tasks.rados.rados.0.smithi022.stdout:200: writing smithi02284960-31 from 1368064 to 2048000 tid 1 2024-09-21T09:31:50.428 INFO:tasks.rados.rados.0.smithi022.stdout:200: writing smithi02284960-31 from 2048000 to 2826240 tid 2 2024-09-21T09:31:50.428 INFO:tasks.rados.rados.0.smithi022.stdout:200: writing smithi02284960-31 from 2826240 to 2850816 tid 3 2024-09-21T09:31:50.428 INFO:tasks.rados.rados.0.smithi022.stdout:201: read oid 49 snap -1 2024-09-21T09:31:50.428 INFO:tasks.rados.rados.0.smithi022.stdout:201: expect (ObjNum 48 snap 0 seq_num 48) 2024-09-21T09:31:50.428 INFO:tasks.rados.rados.0.smithi022.stdout:202: read oid 38 snap -1 2024-09-21T09:31:50.428 INFO:tasks.rados.rados.0.smithi022.stdout:202: expect (ObjNum 56 snap 3 seq_num 56) 2024-09-21T09:31:50.428 INFO:tasks.rados.rados.0.smithi022.stdout:203: read oid 40 snap -1 2024-09-21T09:31:50.428 INFO:tasks.rados.rados.0.smithi022.stdout:203: expect deleted 2024-09-21T09:31:50.429 INFO:tasks.rados.rados.0.smithi022.stdout:204: snap_create 2024-09-21T09:31:50.429 INFO:tasks.rados.rados.0.smithi022.stdout:199: expect (ObjNum 33 snap 0 seq_num 33) 2024-09-21T09:31:50.447 INFO:tasks.rados.rados.0.smithi022.stdout:200: finishing write tid 1 to smithi02284960-31 2024-09-21T09:31:50.447 INFO:tasks.rados.rados.0.smithi022.stdout:200: finishing write tid 2 to smithi02284960-31 2024-09-21T09:31:50.447 INFO:tasks.rados.rados.0.smithi022.stdout:200: finishing write tid 3 to smithi02284960-31 2024-09-21T09:31:50.447 INFO:tasks.rados.rados.0.smithi022.stdout:200: finishing write tid 4 to smithi02284960-31 2024-09-21T09:31:50.447 INFO:tasks.rados.rados.0.smithi022.stdout:202: expect (ObjNum 56 snap 3 seq_num 56) 2024-09-21T09:31:50.502 INFO:tasks.rados.rados.0.smithi022.stdout:200: finishing write tid 5 to smithi02284960-31 2024-09-21T09:31:50.502 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 31 v 81 (ObjNum 88 snap 17 seq_num 88) dirty exists 2024-09-21T09:31:50.502 INFO:tasks.rados.rados.0.smithi022.stdout:200: left oid 31 (ObjNum 88 snap 17 seq_num 88) 2024-09-21T09:31:50.502 INFO:tasks.rados.rados.0.smithi022.stdout:201: expect (ObjNum 48 snap 0 seq_num 48) 2024-09-21T09:31:50.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:50 smithi064 ceph-mon[31124]: osdmap e98: 8 total, 8 up, 8 in 2024-09-21T09:31:50.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:50 smithi064 ceph-mon[31124]: osdmap e99: 8 total, 8 up, 8 in 2024-09-21T09:31:50.692 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:31:50 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[33213]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:31:50] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-21T09:31:50.709 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:50 smithi022 ceph-mon[39525]: osdmap e98: 8 total, 8 up, 8 in 2024-09-21T09:31:50.709 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:50 smithi022 ceph-mon[39525]: osdmap e99: 8 total, 8 up, 8 in 2024-09-21T09:31:50.709 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:50 smithi022 ceph-mon[27763]: osdmap e98: 8 total, 8 up, 8 in 2024-09-21T09:31:50.709 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:50 smithi022 ceph-mon[27763]: osdmap e99: 8 total, 8 up, 8 in 2024-09-21T09:31:51.417 INFO:tasks.rados.rados.0.smithi022.stdout:199: done (5 left) 2024-09-21T09:31:51.418 INFO:tasks.rados.rados.0.smithi022.stdout:200: done (4 left) 2024-09-21T09:31:51.418 INFO:tasks.rados.rados.0.smithi022.stdout:201: done (3 left) 2024-09-21T09:31:51.418 INFO:tasks.rados.rados.0.smithi022.stdout:202: done (2 left) 2024-09-21T09:31:51.418 INFO:tasks.rados.rados.0.smithi022.stdout:203: done (1 left) 2024-09-21T09:31:51.418 INFO:tasks.rados.rados.0.smithi022.stdout:204: done (0 left) 2024-09-21T09:31:51.418 INFO:tasks.rados.rados.0.smithi022.stdout:205: snap_remove snap 15 2024-09-21T09:31:51.558 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:31:51 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:31:51] "GET /metrics HTTP/1.1" 200 239776 "" "Prometheus/2.43.0" 2024-09-21T09:31:51.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:51 smithi022 ceph-mon[39525]: pgmap v144: 121 pgs: 121 active+clean; 48 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.5 MiB/s rd, 1.1 MiB/s wr, 8 op/s 2024-09-21T09:31:51.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:51 smithi022 ceph-mon[39525]: osdmap e100: 8 total, 8 up, 8 in 2024-09-21T09:31:51.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:51 smithi022 ceph-mon[27763]: pgmap v144: 121 pgs: 121 active+clean; 48 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.5 MiB/s rd, 1.1 MiB/s wr, 8 op/s 2024-09-21T09:31:51.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:51 smithi022 ceph-mon[27763]: osdmap e100: 8 total, 8 up, 8 in 2024-09-21T09:31:51.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:51 smithi064 ceph-mon[31124]: pgmap v144: 121 pgs: 121 active+clean; 48 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.5 MiB/s rd, 1.1 MiB/s wr, 8 op/s 2024-09-21T09:31:51.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:51 smithi064 ceph-mon[31124]: osdmap e100: 8 total, 8 up, 8 in 2024-09-21T09:31:52.421 INFO:tasks.rados.rados.0.smithi022.stdout:205: done (0 left) 2024-09-21T09:31:52.421 INFO:tasks.rados.rados.0.smithi022.stdout:206: read oid 36 snap -1 2024-09-21T09:31:52.421 INFO:tasks.rados.rados.0.smithi022.stdout:206: expect (ObjNum 27 snap 0 seq_num 27) 2024-09-21T09:31:52.421 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 1 current snap is 18 2024-09-21T09:31:52.421 INFO:tasks.rados.rados.0.smithi022.stdout:207: seq_num 89 ranges {0=696320,696320=655360,1351680=147456} 2024-09-21T09:31:52.426 INFO:tasks.rados.rados.0.smithi022.stdout:207: writing smithi02284960-1 from 0 to 696320 tid 1 2024-09-21T09:31:52.431 INFO:tasks.rados.rados.0.smithi022.stdout:207: writing smithi02284960-1 from 696320 to 1351680 tid 2 2024-09-21T09:31:52.433 INFO:tasks.rados.rados.0.smithi022.stdout:207: writing smithi02284960-1 from 1351680 to 1499136 tid 3 2024-09-21T09:31:52.433 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 9 current snap is 18 2024-09-21T09:31:52.433 INFO:tasks.rados.rados.0.smithi022.stdout:208: seq_num 90 ranges {1368064=712704,2080768=540672,2621440=262144} 2024-09-21T09:31:52.446 INFO:tasks.rados.rados.0.smithi022.stdout:208: writing smithi02284960-9 from 1368064 to 2080768 tid 1 2024-09-21T09:31:52.450 INFO:tasks.rados.rados.0.smithi022.stdout:208: writing smithi02284960-9 from 2080768 to 2621440 tid 2 2024-09-21T09:31:52.452 INFO:tasks.rados.rados.0.smithi022.stdout:208: writing smithi02284960-9 from 2621440 to 2883584 tid 3 2024-09-21T09:31:52.452 INFO:tasks.rados.rados.0.smithi022.stdout:209: rmattr oid 11 current snap is 18 2024-09-21T09:31:52.452 INFO:tasks.rados.rados.0.smithi022.stdout:209: done (3 left) 2024-09-21T09:31:52.452 INFO:tasks.rados.rados.0.smithi022.stdout:210: snap_remove snap 13 2024-09-21T09:31:53.425 INFO:tasks.rados.rados.0.smithi022.stdout:210: done (3 left) 2024-09-21T09:31:53.425 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 2 current snap is 18 2024-09-21T09:31:53.425 INFO:tasks.rados.rados.0.smithi022.stdout:211: seq_num 91 ranges {778240=729088,1507328=425984,1933312=376832} 2024-09-21T09:31:53.435 INFO:tasks.rados.rados.0.smithi022.stdout:211: writing smithi02284960-2 from 778240 to 1507328 tid 1 2024-09-21T09:31:53.438 INFO:tasks.rados.rados.0.smithi022.stdout:211: writing smithi02284960-2 from 1507328 to 1933312 tid 2 2024-09-21T09:31:53.441 INFO:tasks.rados.rados.0.smithi022.stdout:211: writing smithi02284960-2 from 1933312 to 2310144 tid 3 2024-09-21T09:31:53.441 INFO:tasks.rados.rados.0.smithi022.stdout:212: copy_from oid 38 from oid 46 current snap is 18 2024-09-21T09:31:53.441 INFO:tasks.rados.rados.0.smithi022.stdout:206: expect (ObjNum 27 snap 0 seq_num 27) 2024-09-21T09:31:53.457 INFO:tasks.rados.rados.0.smithi022.stdout:207: finishing write tid 1 to smithi02284960-1 2024-09-21T09:31:53.457 INFO:tasks.rados.rados.0.smithi022.stdout:207: finishing write tid 2 to smithi02284960-1 2024-09-21T09:31:53.457 INFO:tasks.rados.rados.0.smithi022.stdout:207: finishing write tid 3 to smithi02284960-1 2024-09-21T09:31:53.457 INFO:tasks.rados.rados.0.smithi022.stdout:207: finishing write tid 4 to smithi02284960-1 2024-09-21T09:31:53.457 INFO:tasks.rados.rados.0.smithi022.stdout:207: finishing write tid 5 to smithi02284960-1 2024-09-21T09:31:53.457 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 1 v 30 (ObjNum 89 snap 18 seq_num 89) dirty exists 2024-09-21T09:31:53.458 INFO:tasks.rados.rados.0.smithi022.stdout:207: left oid 1 (ObjNum 89 snap 18 seq_num 89) 2024-09-21T09:31:53.458 INFO:tasks.rados.rados.0.smithi022.stdout:208: finishing write tid 1 to smithi02284960-9 2024-09-21T09:31:53.458 INFO:tasks.rados.rados.0.smithi022.stdout:206: done (4 left) 2024-09-21T09:31:53.458 INFO:tasks.rados.rados.0.smithi022.stdout:207: done (3 left) 2024-09-21T09:31:53.458 INFO:tasks.rados.rados.0.smithi022.stdout:213: snap_remove snap 5 2024-09-21T09:31:53.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:53 smithi022 ceph-mon[39525]: osdmap e101: 8 total, 8 up, 8 in 2024-09-21T09:31:53.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:53 smithi022 ceph-mon[39525]: pgmap v147: 121 pgs: 121 active+clean; 55 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4.2 MiB/s rd, 1.8 MiB/s wr, 17 op/s 2024-09-21T09:31:53.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:53 smithi022 ceph-mon[27763]: osdmap e101: 8 total, 8 up, 8 in 2024-09-21T09:31:53.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:53 smithi022 ceph-mon[27763]: pgmap v147: 121 pgs: 121 active+clean; 55 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4.2 MiB/s rd, 1.8 MiB/s wr, 17 op/s 2024-09-21T09:31:53.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:53 smithi064 ceph-mon[31124]: osdmap e101: 8 total, 8 up, 8 in 2024-09-21T09:31:53.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:53 smithi064 ceph-mon[31124]: pgmap v147: 121 pgs: 121 active+clean; 55 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4.2 MiB/s rd, 1.8 MiB/s wr, 17 op/s 2024-09-21T09:31:54.052 INFO:teuthology.orchestra.run.smithi022.stdout:true 2024-09-21T09:31:54.434 INFO:tasks.rados.rados.0.smithi022.stdout:213: done (3 left) 2024-09-21T09:31:54.435 INFO:tasks.rados.rados.0.smithi022.stdout:214: rmattr oid 7 current snap is 18 2024-09-21T09:31:54.435 INFO:tasks.rados.rados.0.smithi022.stdout:214: done (3 left) 2024-09-21T09:31:54.435 INFO:tasks.rados.rados.0.smithi022.stdout:215: rmattr oid 24 current snap is 18 2024-09-21T09:31:54.435 INFO:tasks.rados.rados.0.smithi022.stdout:215: done (3 left) 2024-09-21T09:31:54.435 INFO:tasks.rados.rados.0.smithi022.stdout:216: rmattr oid 39 current snap is 18 2024-09-21T09:31:54.435 INFO:tasks.rados.rados.0.smithi022.stdout:217: read oid 4 snap -1 2024-09-21T09:31:54.435 INFO:tasks.rados.rados.0.smithi022.stdout:217: expect (ObjNum 3 snap 0 seq_num 3) 2024-09-21T09:31:54.435 INFO:tasks.rados.rados.0.smithi022.stdout:218: read oid 18 snap -1 2024-09-21T09:31:54.435 INFO:tasks.rados.rados.0.smithi022.stdout:218: expect (ObjNum 17 snap 0 seq_num 17) 2024-09-21T09:31:54.435 INFO:tasks.rados.rados.0.smithi022.stdout:219: setattr oid 25 current snap is 18 2024-09-21T09:31:54.436 INFO:tasks.rados.rados.0.smithi022.stdout:208: finishing write tid 2 to smithi02284960-9 2024-09-21T09:31:54.436 INFO:tasks.rados.rados.0.smithi022.stdout:208: finishing write tid 3 to smithi02284960-9 2024-09-21T09:31:54.436 INFO:tasks.rados.rados.0.smithi022.stdout:208: finishing write tid 4 to smithi02284960-9 2024-09-21T09:31:54.436 INFO:tasks.rados.rados.0.smithi022.stdout:208: finishing write tid 5 to smithi02284960-9 2024-09-21T09:31:54.436 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 9 v 39 (ObjNum 90 snap 18 seq_num 90) dirty exists 2024-09-21T09:31:54.436 INFO:tasks.rados.rados.0.smithi022.stdout:208: left oid 9 (ObjNum 90 snap 18 seq_num 90) 2024-09-21T09:31:54.436 INFO:tasks.rados.rados.0.smithi022.stdout:211: finishing write tid 1 to smithi02284960-2 2024-09-21T09:31:54.436 INFO:tasks.rados.rados.0.smithi022.stdout:211: finishing write tid 2 to smithi02284960-2 2024-09-21T09:31:54.436 INFO:tasks.rados.rados.0.smithi022.stdout:211: finishing write tid 3 to smithi02284960-2 2024-09-21T09:31:54.436 INFO:tasks.rados.rados.0.smithi022.stdout:211: finishing write tid 4 to smithi02284960-2 2024-09-21T09:31:54.436 INFO:tasks.rados.rados.0.smithi022.stdout:211: finishing write tid 5 to smithi02284960-2 2024-09-21T09:31:54.436 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 2 v 57 (ObjNum 91 snap 18 seq_num 91) dirty exists 2024-09-21T09:31:54.436 INFO:tasks.rados.rados.0.smithi022.stdout:211: left oid 2 (ObjNum 91 snap 18 seq_num 91) 2024-09-21T09:31:54.437 INFO:tasks.rados.rados.0.smithi022.stdout:212: finishing copy_from racing read to smithi02284960-38 2024-09-21T09:31:54.437 INFO:tasks.rados.rados.0.smithi022.stdout:212: finishing copy_from to smithi02284960-38 2024-09-21T09:31:54.437 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 38 v 8 (ObjNum 45 snap 0 seq_num 45) dirty exists 2024-09-21T09:31:54.441 INFO:tasks.rados.rados.0.smithi022.stdout:208: done (6 left) 2024-09-21T09:31:54.441 INFO:tasks.rados.rados.0.smithi022.stdout:211: done (5 left) 2024-09-21T09:31:54.442 INFO:tasks.rados.rados.0.smithi022.stdout:212: done (4 left) 2024-09-21T09:31:54.442 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 40 current snap is 18 2024-09-21T09:31:54.442 INFO:tasks.rados.rados.0.smithi022.stdout:220: seq_num 92 ranges {0=745472,745472=761856,1507328=40960} 2024-09-21T09:31:54.445 INFO:tasks.rados.rados.0.smithi022.stdout:220: writing smithi02284960-40 from 0 to 745472 tid 1 2024-09-21T09:31:54.450 INFO:tasks.rados.rados.0.smithi022.stdout:220: writing smithi02284960-40 from 745472 to 1507328 tid 2 2024-09-21T09:31:54.452 INFO:tasks.rados.rados.0.smithi022.stdout:220: writing smithi02284960-40 from 1507328 to 1548288 tid 3 2024-09-21T09:31:54.452 INFO:tasks.rados.rados.0.smithi022.stdout:221: copy_from oid 43 from oid 33 current snap is 18 2024-09-21T09:31:54.452 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 39 v 20 (ObjNum 38 snap 0 seq_num 38) dirty exists 2024-09-21T09:31:54.452 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 25 v 84 (ObjNum 78 snap 11 seq_num 78) dirty exists 2024-09-21T09:31:54.453 INFO:tasks.rados.rados.0.smithi022.stdout:220: finishing write tid 1 to smithi02284960-40 2024-09-21T09:31:54.453 INFO:tasks.rados.rados.0.smithi022.stdout:216: done (5 left) 2024-09-21T09:31:54.453 INFO:tasks.rados.rados.0.smithi022.stdout:219: done (4 left) 2024-09-21T09:31:54.453 INFO:tasks.rados.rados.0.smithi022.stdout:222: snap_create 2024-09-21T09:31:54.453 INFO:tasks.rados.rados.0.smithi022.stdout:218: expect (ObjNum 17 snap 0 seq_num 17) 2024-09-21T09:31:54.465 INFO:tasks.rados.rados.0.smithi022.stdout:217: expect (ObjNum 3 snap 0 seq_num 3) 2024-09-21T09:31:54.468 INFO:tasks.rados.rados.0.smithi022.stdout:220: finishing write tid 2 to smithi02284960-40 2024-09-21T09:31:54.468 INFO:tasks.rados.rados.0.smithi022.stdout:220: finishing write tid 3 to smithi02284960-40 2024-09-21T09:31:54.469 INFO:tasks.rados.rados.0.smithi022.stdout:220: finishing write tid 4 to smithi02284960-40 2024-09-21T09:31:54.469 INFO:tasks.rados.rados.0.smithi022.stdout:220: finishing write tid 5 to smithi02284960-40 2024-09-21T09:31:54.469 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 40 v 90 (ObjNum 92 snap 18 seq_num 92) dirty exists 2024-09-21T09:31:54.469 INFO:tasks.rados.rados.0.smithi022.stdout:220: left oid 40 (ObjNum 92 snap 18 seq_num 92) 2024-09-21T09:31:54.469 INFO:tasks.rados.rados.0.smithi022.stdout:221: finishing copy_from racing read to smithi02284960-43 2024-09-21T09:31:54.469 INFO:tasks.rados.rados.0.smithi022.stdout:221: finishing copy_from to smithi02284960-43 2024-09-21T09:31:54.469 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 43 v 37 (ObjNum 32 snap 0 seq_num 32) dirty exists 2024-09-21T09:31:54.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:54 smithi064 ceph-mon[31124]: osdmap e102: 8 total, 8 up, 8 in 2024-09-21T09:31:54.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:54 smithi064 ceph-mon[31124]: from='client.14961 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:31:54.783 INFO:teuthology.orchestra.run.smithi022.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T09:31:54.783 INFO:teuthology.orchestra.run.smithi022.stdout:alertmanager.a smithi022 *:9093,9094 running (2m) 118s ago 3m 15.7M - 0.25.0 c8568f914cd2 42a53bb454cd 2024-09-21T09:31:54.783 INFO:teuthology.orchestra.run.smithi022.stdout:grafana.a smithi064 *:3000 running (2m) 2m ago 2m 81.2M - 9.4.7 954c08fa6188 ffde1f806d0b 2024-09-21T09:31:54.783 INFO:teuthology.orchestra.run.smithi022.stdout:mds.a smithi022 running (4m) 118s ago 4m 15.1M - 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 34e4221f3a27 2024-09-21T09:31:54.783 INFO:teuthology.orchestra.run.smithi022.stdout:mds.b smithi064 running (4m) 2m ago 4m 17.4M - 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 0f846b3ec694 2024-09-21T09:31:54.783 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.x smithi064 *:8443 running (8m) 2m ago 8m 399M - 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be c7e735bed227 2024-09-21T09:31:54.783 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.y smithi022 *:9283,8765 running (9m) 118s ago 9m 446M - 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 5bc449c0676f 2024-09-21T09:31:54.784 INFO:teuthology.orchestra.run.smithi022.stdout:mon.a smithi022 running (9m) 118s ago 9m 43.5M 2048M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 1e0a2d74e66e 2024-09-21T09:31:54.784 INFO:teuthology.orchestra.run.smithi022.stdout:mon.b smithi064 running (8m) 2m ago 8m 38.3M 2048M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be ed126533cd64 2024-09-21T09:31:54.784 INFO:teuthology.orchestra.run.smithi022.stdout:mon.c smithi022 running (8m) 118s ago 8m 39.1M 2048M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 79f307e3b64a 2024-09-21T09:31:54.784 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.a smithi022 *:9100 running (2m) 118s ago 2m 8958k - 1.5.0 0da6a335fe13 7cf4c99ebf3a 2024-09-21T09:31:54.784 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.b smithi064 *:9100 running (2m) 2m ago 2m 3414k - 1.5.0 0da6a335fe13 643a1e1aec61 2024-09-21T09:31:54.784 INFO:teuthology.orchestra.run.smithi022.stdout:osd.0 smithi022 running (7m) 118s ago 7m 44.1M 2993M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be fcd812003534 2024-09-21T09:31:54.784 INFO:teuthology.orchestra.run.smithi022.stdout:osd.1 smithi022 running (7m) 118s ago 7m 42.2M 2993M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 740462b5c304 2024-09-21T09:31:54.784 INFO:teuthology.orchestra.run.smithi022.stdout:osd.2 smithi022 running (6m) 118s ago 6m 45.3M 2993M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be fefc7c770c32 2024-09-21T09:31:54.784 INFO:teuthology.orchestra.run.smithi022.stdout:osd.3 smithi022 running (6m) 118s ago 6m 41.5M 2993M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be a56193b67a92 2024-09-21T09:31:54.784 INFO:teuthology.orchestra.run.smithi022.stdout:osd.4 smithi064 running (6m) 2m ago 6m 44.5M 2992M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 2a2937374fc4 2024-09-21T09:31:54.784 INFO:teuthology.orchestra.run.smithi022.stdout:osd.5 smithi064 running (5m) 2m ago 5m 43.1M 2992M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 1900c4af5d28 2024-09-21T09:31:54.784 INFO:teuthology.orchestra.run.smithi022.stdout:osd.6 smithi064 running (5m) 2m ago 5m 40.3M 2992M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 3cc4ec816e5b 2024-09-21T09:31:54.784 INFO:teuthology.orchestra.run.smithi022.stdout:osd.7 smithi064 running (4m) 2m ago 4m 41.5M 2992M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 18ef0be94104 2024-09-21T09:31:54.784 INFO:teuthology.orchestra.run.smithi022.stdout:prometheus.a smithi064 *:9095 running (2m) 2m ago 3m 26.0M - 2.43.0 a07b618ecd1d caee9b54c131 2024-09-21T09:31:54.785 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:54 smithi022 ceph-mon[39525]: osdmap e102: 8 total, 8 up, 8 in 2024-09-21T09:31:54.785 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:54 smithi022 ceph-mon[39525]: from='client.14961 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:31:54.785 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:54 smithi022 ceph-mon[27763]: osdmap e102: 8 total, 8 up, 8 in 2024-09-21T09:31:54.785 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:54 smithi022 ceph-mon[27763]: from='client.14961 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:31:55.198 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-09-21T09:31:55.198 INFO:teuthology.orchestra.run.smithi022.stdout: "mon": { 2024-09-21T09:31:55.198 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 3 2024-09-21T09:31:55.198 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-21T09:31:55.198 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr": { 2024-09-21T09:31:55.198 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 2 2024-09-21T09:31:55.198 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-21T09:31:55.198 INFO:teuthology.orchestra.run.smithi022.stdout: "osd": { 2024-09-21T09:31:55.198 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 8 2024-09-21T09:31:55.198 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-21T09:31:55.198 INFO:teuthology.orchestra.run.smithi022.stdout: "mds": { 2024-09-21T09:31:55.198 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 2 2024-09-21T09:31:55.199 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-21T09:31:55.199 INFO:teuthology.orchestra.run.smithi022.stdout: "overall": { 2024-09-21T09:31:55.199 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 15 2024-09-21T09:31:55.199 INFO:teuthology.orchestra.run.smithi022.stdout: } 2024-09-21T09:31:55.199 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-09-21T09:31:55.436 INFO:tasks.rados.rados.0.smithi022.stdout:217: done (4 left) 2024-09-21T09:31:55.436 INFO:tasks.rados.rados.0.smithi022.stdout:218: done (3 left) 2024-09-21T09:31:55.436 INFO:tasks.rados.rados.0.smithi022.stdout:220: done (2 left) 2024-09-21T09:31:55.436 INFO:tasks.rados.rados.0.smithi022.stdout:221: done (1 left) 2024-09-21T09:31:55.436 INFO:tasks.rados.rados.0.smithi022.stdout:222: done (0 left) 2024-09-21T09:31:55.436 INFO:tasks.rados.rados.0.smithi022.stdout:223: delete oid 31 current snap is 19 2024-09-21T09:31:55.440 INFO:tasks.rados.rados.0.smithi022.stdout:223: done (0 left) 2024-09-21T09:31:55.440 INFO:tasks.rados.rados.0.smithi022.stdout:224: delete oid 26 current snap is 19 2024-09-21T09:31:55.442 INFO:tasks.rados.rados.0.smithi022.stdout:224: done (0 left) 2024-09-21T09:31:55.442 INFO:tasks.rados.rados.0.smithi022.stdout:225: copy_from oid 20 from oid 42 current snap is 19 2024-09-21T09:31:55.442 INFO:tasks.rados.rados.0.smithi022.stdout:226: delete oid 5 current snap is 19 2024-09-21T09:31:55.444 INFO:tasks.rados.rados.0.smithi022.stdout:226: done (1 left) 2024-09-21T09:31:55.444 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 13 current snap is 19 2024-09-21T09:31:55.444 INFO:tasks.rados.rados.0.smithi022.stdout:227: seq_num 96 ranges {1622016=409600,2031616=696320,2727936=507904} 2024-09-21T09:31:55.457 INFO:tasks.rados.rados.0.smithi022.stdout:227: writing smithi02284960-13 from 1622016 to 2031616 tid 1 2024-09-21T09:31:55.463 INFO:tasks.rados.rados.0.smithi022.stdout:227: writing smithi02284960-13 from 2031616 to 2727936 tid 2 2024-09-21T09:31:55.467 INFO:tasks.rados.rados.0.smithi022.stdout:227: writing smithi02284960-13 from 2727936 to 3235840 tid 3 2024-09-21T09:31:55.467 INFO:tasks.rados.rados.0.smithi022.stdout:225: finishing copy_from racing read to smithi02284960-20 2024-09-21T09:31:55.467 INFO:tasks.rados.rados.0.smithi022.stdout:225: finishing copy_from to smithi02284960-20 2024-09-21T09:31:55.467 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 20 v 10 (ObjNum 46 snap 0 seq_num 46) dirty exists 2024-09-21T09:31:55.467 INFO:tasks.rados.rados.0.smithi022.stdout:227: finishing write tid 1 to smithi02284960-13 2024-09-21T09:31:55.467 INFO:tasks.rados.rados.0.smithi022.stdout:227: finishing write tid 2 to smithi02284960-13 2024-09-21T09:31:55.467 INFO:tasks.rados.rados.0.smithi022.stdout:225: done (1 left) 2024-09-21T09:31:55.468 INFO:tasks.rados.rados.0.smithi022.stdout:228: read oid 34 snap -1 2024-09-21T09:31:55.468 INFO:tasks.rados.rados.0.smithi022.stdout:228: expect (ObjNum 87 snap 15 seq_num 87) 2024-09-21T09:31:55.468 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 42 current snap is 19 2024-09-21T09:31:55.468 INFO:tasks.rados.rados.0.smithi022.stdout:229: seq_num 97 ranges {778240=434176,1212416=581632,1794048=540672,2334720=81920} 2024-09-21T09:31:55.474 INFO:tasks.rados.rados.0.smithi022.stdout:229: writing smithi02284960-42 from 778240 to 1212416 tid 1 2024-09-21T09:31:55.478 INFO:tasks.rados.rados.0.smithi022.stdout:229: writing smithi02284960-42 from 1212416 to 1794048 tid 2 2024-09-21T09:31:55.482 INFO:tasks.rados.rados.0.smithi022.stdout:229: writing smithi02284960-42 from 1794048 to 2334720 tid 3 2024-09-21T09:31:55.483 INFO:tasks.rados.rados.0.smithi022.stdout:229: writing smithi02284960-42 from 2334720 to 2416640 tid 4 2024-09-21T09:31:55.483 INFO:tasks.rados.rados.0.smithi022.stdout:230: delete oid 11 current snap is 19 2024-09-21T09:31:55.483 INFO:tasks.rados.rados.0.smithi022.stdout:227: finishing write tid 3 to smithi02284960-13 2024-09-21T09:31:55.483 INFO:tasks.rados.rados.0.smithi022.stdout:227: finishing write tid 4 to smithi02284960-13 2024-09-21T09:31:55.483 INFO:tasks.rados.rados.0.smithi022.stdout:227: finishing write tid 5 to smithi02284960-13 2024-09-21T09:31:55.483 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 13 v 42 (ObjNum 96 snap 19 seq_num 96) dirty exists 2024-09-21T09:31:55.483 INFO:tasks.rados.rados.0.smithi022.stdout:227: left oid 13 (ObjNum 96 snap 19 seq_num 96) 2024-09-21T09:31:55.484 INFO:tasks.rados.rados.0.smithi022.stdout:229: finishing write tid 1 to smithi02284960-42 2024-09-21T09:31:55.484 INFO:tasks.rados.rados.0.smithi022.stdout:228: expect (ObjNum 87 snap 15 seq_num 87) 2024-09-21T09:31:55.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:55 smithi022 ceph-mon[39525]: from='client.14967 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:31:55.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:55 smithi022 ceph-mon[39525]: osdmap e103: 8 total, 8 up, 8 in 2024-09-21T09:31:55.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:55 smithi022 ceph-mon[39525]: from='client.24878 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:31:55.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:55 smithi022 ceph-mon[39525]: pgmap v150: 121 pgs: 121 active+clean; 55 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4.2 MiB/s rd, 1.8 MiB/s wr, 17 op/s 2024-09-21T09:31:55.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:55 smithi022 ceph-mon[39525]: from='client.? 172.21.15.22:0/2196300634' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:31:55.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:55 smithi022 ceph-mon[27763]: from='client.14967 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:31:55.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:55 smithi022 ceph-mon[27763]: osdmap e103: 8 total, 8 up, 8 in 2024-09-21T09:31:55.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:55 smithi022 ceph-mon[27763]: from='client.24878 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:31:55.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:55 smithi022 ceph-mon[27763]: pgmap v150: 121 pgs: 121 active+clean; 55 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4.2 MiB/s rd, 1.8 MiB/s wr, 17 op/s 2024-09-21T09:31:55.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:55 smithi022 ceph-mon[27763]: from='client.? 172.21.15.22:0/2196300634' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:31:55.573 INFO:tasks.rados.rados.0.smithi022.stdout:229: finishing write tid 2 to smithi02284960-42 2024-09-21T09:31:55.573 INFO:tasks.rados.rados.0.smithi022.stdout:227: done (3 left) 2024-09-21T09:31:55.573 INFO:tasks.rados.rados.0.smithi022.stdout:228: done (2 left) 2024-09-21T09:31:55.574 INFO:tasks.rados.rados.0.smithi022.stdout:230: done (1 left) 2024-09-21T09:31:55.574 INFO:tasks.rados.rados.0.smithi022.stdout:231: read oid 31 snap -1 2024-09-21T09:31:55.574 INFO:tasks.rados.rados.0.smithi022.stdout:231: expect deleted 2024-09-21T09:31:55.574 INFO:tasks.rados.rados.0.smithi022.stdout:232: copy_from oid 27 from oid 4 current snap is 19 2024-09-21T09:31:55.574 INFO:tasks.rados.rados.0.smithi022.stdout:233: copy_from oid 20 from oid 21 current snap is 19 2024-09-21T09:31:55.574 INFO:tasks.rados.rados.0.smithi022.stdout:229: finishing write tid 3 to smithi02284960-42 2024-09-21T09:31:55.574 INFO:tasks.rados.rados.0.smithi022.stdout:229: finishing write tid 4 to smithi02284960-42 2024-09-21T09:31:55.574 INFO:tasks.rados.rados.0.smithi022.stdout:229: finishing write tid 5 to smithi02284960-42 2024-09-21T09:31:55.574 INFO:tasks.rados.rados.0.smithi022.stdout:234: read oid 8 snap 18 2024-09-21T09:31:55.574 INFO:tasks.rados.rados.0.smithi022.stdout:234: expect deleted 2024-09-21T09:31:55.574 INFO:tasks.rados.rados.0.smithi022.stdout:235: delete oid 32 current snap is 19 2024-09-21T09:31:55.574 INFO:tasks.rados.rados.0.smithi022.stdout:229: finishing write tid 6 to smithi02284960-42 2024-09-21T09:31:55.574 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 42 v 48 (ObjNum 97 snap 19 seq_num 97) dirty exists 2024-09-21T09:31:55.575 INFO:tasks.rados.rados.0.smithi022.stdout:229: left oid 42 (ObjNum 97 snap 19 seq_num 97) 2024-09-21T09:31:55.576 INFO:tasks.rados.rados.0.smithi022.stdout:229: done (5 left) 2024-09-21T09:31:55.576 INFO:tasks.rados.rados.0.smithi022.stdout:231: done (4 left) 2024-09-21T09:31:55.576 INFO:tasks.rados.rados.0.smithi022.stdout:234: done (3 left) 2024-09-21T09:31:55.577 INFO:tasks.rados.rados.0.smithi022.stdout:235: done (2 left) 2024-09-21T09:31:55.577 INFO:tasks.rados.rados.0.smithi022.stdout:236: rollback oid 31 current snap is 19 2024-09-21T09:31:55.577 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 31 to 9 2024-09-21T09:31:55.577 INFO:tasks.rados.rados.0.smithi022.stdout:237: delete oid 44 current snap is 19 2024-09-21T09:31:55.577 INFO:tasks.rados.rados.0.smithi022.stdout:232: finishing copy_from racing read to smithi02284960-27 2024-09-21T09:31:55.577 INFO:tasks.rados.rados.0.smithi022.stdout:232: finishing copy_from to smithi02284960-27 2024-09-21T09:31:55.577 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 27 v 2 (ObjNum 3 snap 0 seq_num 3) dirty exists 2024-09-21T09:31:55.579 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-09-21T09:31:55.579 INFO:teuthology.orchestra.run.smithi022.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208", 2024-09-21T09:31:55.579 INFO:teuthology.orchestra.run.smithi022.stdout: "in_progress": true, 2024-09-21T09:31:55.579 INFO:teuthology.orchestra.run.smithi022.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T09:31:55.579 INFO:teuthology.orchestra.run.smithi022.stdout: "services_complete": [], 2024-09-21T09:31:55.579 INFO:teuthology.orchestra.run.smithi022.stdout: "progress": "", 2024-09-21T09:31:55.579 INFO:teuthology.orchestra.run.smithi022.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208 image", 2024-09-21T09:31:55.579 INFO:teuthology.orchestra.run.smithi022.stdout: "is_paused": false 2024-09-21T09:31:55.580 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-09-21T09:31:55.580 INFO:tasks.rados.rados.0.smithi022.stdout:232: done (3 left) 2024-09-21T09:31:55.580 INFO:tasks.rados.rados.0.smithi022.stdout:237: done (2 left) 2024-09-21T09:31:55.580 INFO:tasks.rados.rados.0.smithi022.stdout:238: copy_from oid 41 from oid 12 current snap is 19 2024-09-21T09:31:55.580 INFO:tasks.rados.rados.0.smithi022.stdout:233: finishing copy_from racing read to smithi02284960-20 2024-09-21T09:31:55.580 INFO:tasks.rados.rados.0.smithi022.stdout:236: finishing rollback tid 1 to smithi02284960-31 2024-09-21T09:31:55.580 INFO:tasks.rados.rados.0.smithi022.stdout:239: rollback oid 45 current snap is 19 2024-09-21T09:31:55.581 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 45 to 8 2024-09-21T09:31:55.581 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 16 current snap is 19 2024-09-21T09:31:55.581 INFO:tasks.rados.rados.0.smithi022.stdout:240: seq_num 101 ranges {2818048=499712,3317760=573440,3891200=524288,4415488=106496} 2024-09-21T09:31:55.599 INFO:tasks.rados.rados.0.smithi022.stdout:240: writing smithi02284960-16 from 2818048 to 3317760 tid 1 2024-09-21T09:31:55.603 INFO:tasks.rados.rados.0.smithi022.stdout:240: writing smithi02284960-16 from 3317760 to 3891200 tid 2 2024-09-21T09:31:55.606 INFO:tasks.rados.rados.0.smithi022.stdout:240: writing smithi02284960-16 from 3891200 to 4415488 tid 3 2024-09-21T09:31:55.608 INFO:tasks.rados.rados.0.smithi022.stdout:240: writing smithi02284960-16 from 4415488 to 4521984 tid 4 2024-09-21T09:31:55.608 INFO:tasks.rados.rados.0.smithi022.stdout:241: rollback oid 10 current snap is 19 2024-09-21T09:31:55.608 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 10 to 16 2024-09-21T09:31:55.608 INFO:tasks.rados.rados.0.smithi022.stdout:236: finishing rollback tid 2 to smithi02284960-31 2024-09-21T09:31:55.608 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 31 v 97 (ObjNum 71 snap 5 seq_num 71) dirty exists 2024-09-21T09:31:55.608 INFO:tasks.rados.rados.0.smithi022.stdout:233: finishing copy_from to smithi02284960-20 2024-09-21T09:31:55.609 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 20 v 8 (ObjNum 23 snap 0 seq_num 23) dirty exists 2024-09-21T09:31:55.609 INFO:tasks.rados.rados.0.smithi022.stdout:239: finishing rollback tid 0 to smithi02284960-45 2024-09-21T09:31:55.609 INFO:tasks.rados.rados.0.smithi022.stdout:239: finishing rollback tid 1 to smithi02284960-45 2024-09-21T09:31:55.609 INFO:tasks.rados.rados.0.smithi022.stdout:239: finishing rollback tid 2 to smithi02284960-45 2024-09-21T09:31:55.609 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 45 v 25 (ObjNum 33 snap 0 seq_num 33) dirty exists 2024-09-21T09:31:55.609 INFO:tasks.rados.rados.0.smithi022.stdout:238: finishing copy_from racing read to smithi02284960-41 2024-09-21T09:31:55.609 INFO:tasks.rados.rados.0.smithi022.stdout:238: finishing copy_from to smithi02284960-41 2024-09-21T09:31:55.609 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 41 v 45 (ObjNum 74 snap 6 seq_num 74) dirty exists 2024-09-21T09:31:55.609 INFO:tasks.rados.rados.0.smithi022.stdout:240: finishing write tid 1 to smithi02284960-16 2024-09-21T09:31:55.609 INFO:tasks.rados.rados.0.smithi022.stdout:240: finishing write tid 2 to smithi02284960-16 2024-09-21T09:31:55.609 INFO:tasks.rados.rados.0.smithi022.stdout:233: done (5 left) 2024-09-21T09:31:55.609 INFO:tasks.rados.rados.0.smithi022.stdout:236: done (4 left) 2024-09-21T09:31:55.609 INFO:tasks.rados.rados.0.smithi022.stdout:238: done (3 left) 2024-09-21T09:31:55.609 INFO:tasks.rados.rados.0.smithi022.stdout:239: done (2 left) 2024-09-21T09:31:55.609 INFO:tasks.rados.rados.0.smithi022.stdout:242: snap_remove snap 12 2024-09-21T09:31:55.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:55 smithi064 ceph-mon[31124]: from='client.14967 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:31:55.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:55 smithi064 ceph-mon[31124]: osdmap e103: 8 total, 8 up, 8 in 2024-09-21T09:31:55.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:55 smithi064 ceph-mon[31124]: from='client.24878 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:31:55.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:55 smithi064 ceph-mon[31124]: pgmap v150: 121 pgs: 121 active+clean; 55 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4.2 MiB/s rd, 1.8 MiB/s wr, 17 op/s 2024-09-21T09:31:55.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:55 smithi064 ceph-mon[31124]: from='client.? 172.21.15.22:0/2196300634' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:31:56.440 INFO:tasks.rados.rados.0.smithi022.stdout:242: done (2 left) 2024-09-21T09:31:56.440 INFO:tasks.rados.rados.0.smithi022.stdout:243: snap_remove snap 14 2024-09-21T09:31:56.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:56 smithi064 ceph-mon[31124]: osdmap e104: 8 total, 8 up, 8 in 2024-09-21T09:31:56.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:56 smithi064 ceph-mon[31124]: from='client.14985 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:31:56.807 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:56 smithi022 ceph-mon[39525]: osdmap e104: 8 total, 8 up, 8 in 2024-09-21T09:31:56.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:56 smithi022 ceph-mon[39525]: from='client.14985 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:31:56.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:56 smithi022 ceph-mon[27763]: osdmap e104: 8 total, 8 up, 8 in 2024-09-21T09:31:56.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:56 smithi022 ceph-mon[27763]: from='client.14985 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:31:57.445 INFO:tasks.rados.rados.0.smithi022.stdout:243: done (2 left) 2024-09-21T09:31:57.446 INFO:tasks.rados.rados.0.smithi022.stdout:244: rollback oid 34 current snap is 19 2024-09-21T09:31:57.446 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 34 to 11 2024-09-21T09:31:57.446 INFO:tasks.rados.rados.0.smithi022.stdout:245: copy_from oid 38 from oid 17 current snap is 19 2024-09-21T09:31:57.446 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 32 current snap is 19 2024-09-21T09:31:57.446 INFO:tasks.rados.rados.0.smithi022.stdout:246: seq_num 102 ranges {0=516096,516096=450560,966656=729088,1695744=24576} 2024-09-21T09:31:57.449 INFO:tasks.rados.rados.0.smithi022.stdout:246: writing smithi02284960-32 from 0 to 516096 tid 1 2024-09-21T09:31:57.452 INFO:tasks.rados.rados.0.smithi022.stdout:246: writing smithi02284960-32 from 516096 to 966656 tid 2 2024-09-21T09:31:57.458 INFO:tasks.rados.rados.0.smithi022.stdout:246: writing smithi02284960-32 from 966656 to 1695744 tid 3 2024-09-21T09:31:57.458 INFO:tasks.rados.rados.0.smithi022.stdout:246: writing smithi02284960-32 from 1695744 to 1720320 tid 4 2024-09-21T09:31:57.458 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 11 current snap is 19 2024-09-21T09:31:57.458 INFO:tasks.rados.rados.0.smithi022.stdout:240: finishing write tid 3 to smithi02284960-16 2024-09-21T09:31:57.458 INFO:tasks.rados.rados.0.smithi022.stdout:240: finishing write tid 4 to smithi02284960-16 2024-09-21T09:31:57.458 INFO:tasks.rados.rados.0.smithi022.stdout:241: finishing rollback tid 0 to smithi02284960-10 2024-09-21T09:31:57.458 INFO:tasks.rados.rados.0.smithi022.stdout:241: finishing rollback tid 1 to smithi02284960-10 2024-09-21T09:31:57.459 INFO:tasks.rados.rados.0.smithi022.stdout:240: finishing write tid 5 to smithi02284960-16 2024-09-21T09:31:57.459 INFO:tasks.rados.rados.0.smithi022.stdout:241: finishing rollback tid 2 to smithi02284960-10 2024-09-21T09:31:57.459 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 10 v 98 (ObjNum 38 snap 0 seq_num 38) dirty exists 2024-09-21T09:31:57.459 INFO:tasks.rados.rados.0.smithi022.stdout:240: finishing write tid 6 to smithi02284960-16 2024-09-21T09:31:57.459 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 16 v 25 (ObjNum 101 snap 19 seq_num 101) dirty exists 2024-09-21T09:31:57.459 INFO:tasks.rados.rados.0.smithi022.stdout:240: left oid 16 (ObjNum 101 snap 19 seq_num 101) 2024-09-21T09:31:57.459 INFO:tasks.rados.rados.0.smithi022.stdout:244: finishing rollback tid 0 to smithi02284960-34 2024-09-21T09:31:57.459 INFO:tasks.rados.rados.0.smithi022.stdout:247: seq_num 103 ranges {0=532480,532480=737280,1269760=466944} 2024-09-21T09:31:57.462 INFO:tasks.rados.rados.0.smithi022.stdout:247: writing smithi02284960-11 from 0 to 532480 tid 1 2024-09-21T09:31:57.466 INFO:tasks.rados.rados.0.smithi022.stdout:247: writing smithi02284960-11 from 532480 to 1269760 tid 2 2024-09-21T09:31:57.470 INFO:tasks.rados.rados.0.smithi022.stdout:247: writing smithi02284960-11 from 1269760 to 1736704 tid 3 2024-09-21T09:31:57.470 INFO:tasks.rados.rados.0.smithi022.stdout:240: done (5 left) 2024-09-21T09:31:57.470 INFO:tasks.rados.rados.0.smithi022.stdout:241: done (4 left) 2024-09-21T09:31:57.470 INFO:tasks.rados.rados.0.smithi022.stdout:248: setattr oid 47 current snap is 19 2024-09-21T09:31:57.470 INFO:tasks.rados.rados.0.smithi022.stdout:244: finishing rollback tid 1 to smithi02284960-34 2024-09-21T09:31:57.470 INFO:tasks.rados.rados.0.smithi022.stdout:244: finishing rollback tid 2 to smithi02284960-34 2024-09-21T09:31:57.470 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 34 v 73 (ObjNum 36 snap 0 seq_num 36) dirty exists 2024-09-21T09:31:57.470 INFO:tasks.rados.rados.0.smithi022.stdout:245: finishing copy_from racing read to smithi02284960-38 2024-09-21T09:31:57.470 INFO:tasks.rados.rados.0.smithi022.stdout:245: finishing copy_from to smithi02284960-38 2024-09-21T09:31:57.470 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 38 v 2 (ObjNum 16 snap 0 seq_num 16) dirty exists 2024-09-21T09:31:57.470 INFO:tasks.rados.rados.0.smithi022.stdout:246: finishing write tid 1 to smithi02284960-32 2024-09-21T09:31:57.470 INFO:tasks.rados.rados.0.smithi022.stdout:246: finishing write tid 2 to smithi02284960-32 2024-09-21T09:31:57.470 INFO:tasks.rados.rados.0.smithi022.stdout:246: finishing write tid 3 to smithi02284960-32 2024-09-21T09:31:57.471 INFO:tasks.rados.rados.0.smithi022.stdout:246: finishing write tid 4 to smithi02284960-32 2024-09-21T09:31:57.471 INFO:tasks.rados.rados.0.smithi022.stdout:246: finishing write tid 5 to smithi02284960-32 2024-09-21T09:31:57.471 INFO:tasks.rados.rados.0.smithi022.stdout:246: finishing write tid 6 to smithi02284960-32 2024-09-21T09:31:57.471 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 32 v 98 (ObjNum 102 snap 19 seq_num 102) dirty exists 2024-09-21T09:31:57.471 INFO:tasks.rados.rados.0.smithi022.stdout:246: left oid 32 (ObjNum 102 snap 19 seq_num 102) 2024-09-21T09:31:57.471 INFO:tasks.rados.rados.0.smithi022.stdout:247: finishing write tid 1 to smithi02284960-11 2024-09-21T09:31:57.471 INFO:tasks.rados.rados.0.smithi022.stdout:247: finishing write tid 2 to smithi02284960-11 2024-09-21T09:31:57.471 INFO:tasks.rados.rados.0.smithi022.stdout:244: done (4 left) 2024-09-21T09:31:57.471 INFO:tasks.rados.rados.0.smithi022.stdout:245: done (3 left) 2024-09-21T09:31:57.471 INFO:tasks.rados.rados.0.smithi022.stdout:246: done (2 left) 2024-09-21T09:31:57.471 INFO:tasks.rados.rados.0.smithi022.stdout:249: snap_remove snap 8 2024-09-21T09:31:57.807 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:57 smithi022 ceph-mon[39525]: osdmap e105: 8 total, 8 up, 8 in 2024-09-21T09:31:57.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:57 smithi022 ceph-mon[39525]: pgmap v153: 121 pgs: 121 active+clean; 55 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:31:57.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:57 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:31:57.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:57 smithi022 ceph-mon[27763]: osdmap e105: 8 total, 8 up, 8 in 2024-09-21T09:31:57.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:57 smithi022 ceph-mon[27763]: pgmap v153: 121 pgs: 121 active+clean; 55 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:31:57.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:57 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:31:57.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:57 smithi064 ceph-mon[31124]: osdmap e105: 8 total, 8 up, 8 in 2024-09-21T09:31:57.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:57 smithi064 ceph-mon[31124]: pgmap v153: 121 pgs: 121 active+clean; 55 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:31:57.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:57 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:31:58.448 INFO:tasks.rados.rados.0.smithi022.stdout:249: done (2 left) 2024-09-21T09:31:58.448 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 12 current snap is 19 2024-09-21T09:31:58.448 INFO:tasks.rados.rados.0.smithi022.stdout:250: seq_num 104 ranges {1368064=548864,1916928=622592,2539520=581632} 2024-09-21T09:31:58.459 INFO:tasks.rados.rados.0.smithi022.stdout:250: writing smithi02284960-12 from 1368064 to 1916928 tid 1 2024-09-21T09:31:58.464 INFO:tasks.rados.rados.0.smithi022.stdout:250: writing smithi02284960-12 from 1916928 to 2539520 tid 2 2024-09-21T09:31:58.468 INFO:tasks.rados.rados.0.smithi022.stdout:250: writing smithi02284960-12 from 2539520 to 3121152 tid 3 2024-09-21T09:31:58.468 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 50 current snap is 19 2024-09-21T09:31:58.468 INFO:tasks.rados.rados.0.smithi022.stdout:251: seq_num 105 ranges {827392=565248,1392640=507904,1900544=507904,2408448=188416} 2024-09-21T09:31:58.475 INFO:tasks.rados.rados.0.smithi022.stdout:251: writing smithi02284960-50 from 827392 to 1392640 tid 1 2024-09-21T09:31:58.478 INFO:tasks.rados.rados.0.smithi022.stdout:251: writing smithi02284960-50 from 1392640 to 1900544 tid 2 2024-09-21T09:31:58.482 INFO:tasks.rados.rados.0.smithi022.stdout:251: writing smithi02284960-50 from 1900544 to 2408448 tid 3 2024-09-21T09:31:58.483 INFO:tasks.rados.rados.0.smithi022.stdout:251: writing smithi02284960-50 from 2408448 to 2596864 tid 4 2024-09-21T09:31:58.483 INFO:tasks.rados.rados.0.smithi022.stdout:252: copy_from oid 49 from oid 4 current snap is 19 2024-09-21T09:31:58.483 INFO:tasks.rados.rados.0.smithi022.stdout:247: finishing write tid 3 to smithi02284960-11 2024-09-21T09:31:58.483 INFO:tasks.rados.rados.0.smithi022.stdout:247: finishing write tid 4 to smithi02284960-11 2024-09-21T09:31:58.484 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 47 v 42 (ObjNum 49 snap 0 seq_num 1953066355) dirty exists 2024-09-21T09:31:58.484 INFO:tasks.rados.rados.0.smithi022.stdout:247: finishing write tid 5 to smithi02284960-11 2024-09-21T09:31:58.484 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 11 v 41 (ObjNum 103 snap 19 seq_num 103) dirty exists 2024-09-21T09:31:58.484 INFO:tasks.rados.rados.0.smithi022.stdout:247: left oid 11 (ObjNum 103 snap 19 seq_num 103) 2024-09-21T09:31:58.484 INFO:tasks.rados.rados.0.smithi022.stdout:250: finishing write tid 1 to smithi02284960-12 2024-09-21T09:31:58.484 INFO:tasks.rados.rados.0.smithi022.stdout:250: finishing write tid 2 to smithi02284960-12 2024-09-21T09:31:58.484 INFO:tasks.rados.rados.0.smithi022.stdout:250: finishing write tid 3 to smithi02284960-12 2024-09-21T09:31:58.484 INFO:tasks.rados.rados.0.smithi022.stdout:250: finishing write tid 4 to smithi02284960-12 2024-09-21T09:31:58.484 INFO:tasks.rados.rados.0.smithi022.stdout:247: done (4 left) 2024-09-21T09:31:58.484 INFO:tasks.rados.rados.0.smithi022.stdout:248: done (3 left) 2024-09-21T09:31:58.484 INFO:tasks.rados.rados.0.smithi022.stdout:253: snap_create 2024-09-21T09:31:58.484 INFO:tasks.rados.rados.0.smithi022.stdout:250: finishing write tid 5 to smithi02284960-12 2024-09-21T09:31:58.485 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 12 v 112 (ObjNum 104 snap 19 seq_num 104) dirty exists 2024-09-21T09:31:58.485 INFO:tasks.rados.rados.0.smithi022.stdout:250: left oid 12 (ObjNum 104 snap 19 seq_num 104) 2024-09-21T09:31:58.485 INFO:tasks.rados.rados.0.smithi022.stdout:251: finishing write tid 1 to smithi02284960-50 2024-09-21T09:31:58.485 INFO:tasks.rados.rados.0.smithi022.stdout:251: finishing write tid 2 to smithi02284960-50 2024-09-21T09:31:58.485 INFO:tasks.rados.rados.0.smithi022.stdout:251: finishing write tid 3 to smithi02284960-50 2024-09-21T09:31:58.486 INFO:tasks.rados.rados.0.smithi022.stdout:251: finishing write tid 4 to smithi02284960-50 2024-09-21T09:31:58.486 INFO:tasks.rados.rados.0.smithi022.stdout:251: finishing write tid 5 to smithi02284960-50 2024-09-21T09:31:58.486 INFO:tasks.rados.rados.0.smithi022.stdout:251: finishing write tid 6 to smithi02284960-50 2024-09-21T09:31:58.487 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 50 v 37 (ObjNum 105 snap 19 seq_num 105) dirty exists 2024-09-21T09:31:58.487 INFO:tasks.rados.rados.0.smithi022.stdout:251: left oid 50 (ObjNum 105 snap 19 seq_num 105) 2024-09-21T09:31:58.487 INFO:tasks.rados.rados.0.smithi022.stdout:252: finishing copy_from racing read to smithi02284960-49 2024-09-21T09:31:58.487 INFO:tasks.rados.rados.0.smithi022.stdout:252: finishing copy_from to smithi02284960-49 2024-09-21T09:31:58.487 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 49 v 2 (ObjNum 3 snap 0 seq_num 3) dirty exists 2024-09-21T09:31:58.807 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:58 smithi022 ceph-mon[39525]: osdmap e106: 8 total, 8 up, 8 in 2024-09-21T09:31:58.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:58 smithi022 ceph-mon[27763]: osdmap e106: 8 total, 8 up, 8 in 2024-09-21T09:31:58.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:58 smithi064 ceph-mon[31124]: osdmap e106: 8 total, 8 up, 8 in 2024-09-21T09:31:59.460 INFO:tasks.rados.rados.0.smithi022.stdout:250: done (3 left) 2024-09-21T09:31:59.460 INFO:tasks.rados.rados.0.smithi022.stdout:251: done (2 left) 2024-09-21T09:31:59.460 INFO:tasks.rados.rados.0.smithi022.stdout:252: done (1 left) 2024-09-21T09:31:59.460 INFO:tasks.rados.rados.0.smithi022.stdout:253: done (0 left) 2024-09-21T09:31:59.461 INFO:tasks.rados.rados.0.smithi022.stdout:254: snap_create 2024-09-21T09:31:59.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:59 smithi022 ceph-mon[39525]: osdmap e107: 8 total, 8 up, 8 in 2024-09-21T09:31:59.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:31:59 smithi022 ceph-mon[39525]: pgmap v156: 121 pgs: 121 active+clean; 67 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.7 MiB/s rd, 3.7 MiB/s wr, 30 op/s 2024-09-21T09:31:59.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:59 smithi022 ceph-mon[27763]: osdmap e107: 8 total, 8 up, 8 in 2024-09-21T09:31:59.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:31:59 smithi022 ceph-mon[27763]: pgmap v156: 121 pgs: 121 active+clean; 67 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.7 MiB/s rd, 3.7 MiB/s wr, 30 op/s 2024-09-21T09:31:59.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:59 smithi064 ceph-mon[31124]: osdmap e107: 8 total, 8 up, 8 in 2024-09-21T09:31:59.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:31:59 smithi064 ceph-mon[31124]: pgmap v156: 121 pgs: 121 active+clean; 67 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.7 MiB/s rd, 3.7 MiB/s wr, 30 op/s 2024-09-21T09:32:00.415 INFO:tasks.rados.rados.0.smithi022.stdout:254: done (0 left) 2024-09-21T09:32:00.416 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 17 current snap is 21 2024-09-21T09:32:00.416 INFO:tasks.rados.rados.0.smithi022.stdout:255: seq_num 106 ranges {270336=581632,851968=794624,1646592=409600} 2024-09-21T09:32:00.420 INFO:tasks.rados.rados.0.smithi022.stdout:255: writing smithi02284960-17 from 270336 to 851968 tid 1 2024-09-21T09:32:00.424 INFO:tasks.rados.rados.0.smithi022.stdout:255: writing smithi02284960-17 from 851968 to 1646592 tid 2 2024-09-21T09:32:00.428 INFO:tasks.rados.rados.0.smithi022.stdout:255: writing smithi02284960-17 from 1646592 to 2056192 tid 3 2024-09-21T09:32:00.428 INFO:tasks.rados.rados.0.smithi022.stdout:255: finishing write tid 1 to smithi02284960-17 2024-09-21T09:32:00.428 INFO:tasks.rados.rados.0.smithi022.stdout:256: rollback oid 8 current snap is 21 2024-09-21T09:32:00.428 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 8 to 20 2024-09-21T09:32:00.428 INFO:tasks.rados.rados.0.smithi022.stdout:257: rmattr oid 26 current snap is 21 2024-09-21T09:32:00.428 INFO:tasks.rados.rados.0.smithi022.stdout:257: done (2 left) 2024-09-21T09:32:00.428 INFO:tasks.rados.rados.0.smithi022.stdout:258: delete oid 2 current snap is 21 2024-09-21T09:32:00.428 INFO:tasks.rados.rados.0.smithi022.stdout:255: finishing write tid 2 to smithi02284960-17 2024-09-21T09:32:00.429 INFO:tasks.rados.rados.0.smithi022.stdout:256: finishing rollback tid 1 to smithi02284960-8 2024-09-21T09:32:00.429 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 8 v 28 (ObjNum 129 snap 0 seq_num 3569796000) dirty dne 2024-09-21T09:32:00.430 INFO:tasks.rados.rados.0.smithi022.stdout:255: finishing write tid 3 to smithi02284960-17 2024-09-21T09:32:00.430 INFO:tasks.rados.rados.0.smithi022.stdout:255: finishing write tid 4 to smithi02284960-17 2024-09-21T09:32:00.431 INFO:tasks.rados.rados.0.smithi022.stdout:255: finishing write tid 5 to smithi02284960-17 2024-09-21T09:32:00.431 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 17 v 31 (ObjNum 106 snap 21 seq_num 106) dirty exists 2024-09-21T09:32:00.431 INFO:tasks.rados.rados.0.smithi022.stdout:255: left oid 17 (ObjNum 106 snap 21 seq_num 106) 2024-09-21T09:32:00.432 INFO:tasks.rados.rados.0.smithi022.stdout:255: done (2 left) 2024-09-21T09:32:00.432 INFO:tasks.rados.rados.0.smithi022.stdout:256: done (1 left) 2024-09-21T09:32:00.432 INFO:tasks.rados.rados.0.smithi022.stdout:258: done (0 left) 2024-09-21T09:32:00.432 INFO:tasks.rados.rados.0.smithi022.stdout:259: rmattr oid 38 current snap is 21 2024-09-21T09:32:00.432 INFO:tasks.rados.rados.0.smithi022.stdout:259: done (0 left) 2024-09-21T09:32:00.432 INFO:tasks.rados.rados.0.smithi022.stdout:260: snap_create 2024-09-21T09:32:00.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:00 smithi022 ceph-mon[27763]: osdmap e108: 8 total, 8 up, 8 in 2024-09-21T09:32:00.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:00 smithi022 ceph-mon[27763]: osdmap e109: 8 total, 8 up, 8 in 2024-09-21T09:32:00.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:00 smithi022 ceph-mon[39525]: osdmap e108: 8 total, 8 up, 8 in 2024-09-21T09:32:00.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:00 smithi022 ceph-mon[39525]: osdmap e109: 8 total, 8 up, 8 in 2024-09-21T09:32:00.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:00 smithi064 ceph-mon[31124]: osdmap e108: 8 total, 8 up, 8 in 2024-09-21T09:32:00.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:00 smithi064 ceph-mon[31124]: osdmap e109: 8 total, 8 up, 8 in 2024-09-21T09:32:00.942 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:32:00 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[33213]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:32:00] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-21T09:32:01.416 INFO:tasks.rados.rados.0.smithi022.stdout:260: done (0 left) 2024-09-21T09:32:01.416 INFO:tasks.rados.rados.0.smithi022.stdout:261: copy_from oid 48 from oid 47 current snap is 22 2024-09-21T09:32:01.416 INFO:tasks.rados.rados.0.smithi022.stdout:262: read oid 34 snap -1 2024-09-21T09:32:01.416 INFO:tasks.rados.rados.0.smithi022.stdout:262: expect (ObjNum 36 snap 0 seq_num 36) 2024-09-21T09:32:01.416 INFO:tasks.rados.rados.0.smithi022.stdout:263: snap_create 2024-09-21T09:32:01.416 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:32:01 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:32:01] "GET /metrics HTTP/1.1" 200 239872 "" "Prometheus/2.43.0" 2024-09-21T09:32:01.418 INFO:tasks.rados.rados.0.smithi022.stdout:261: finishing copy_from racing read to smithi02284960-48 2024-09-21T09:32:01.418 INFO:tasks.rados.rados.0.smithi022.stdout:261: finishing copy_from to smithi02284960-48 2024-09-21T09:32:01.419 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 48 v 42 (ObjNum 161 snap 0 seq_num 0) dirty exists 2024-09-21T09:32:01.422 INFO:tasks.rados.rados.0.smithi022.stdout:262: expect (ObjNum 36 snap 0 seq_num 36) 2024-09-21T09:32:01.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:01 smithi022 ceph-mon[39525]: pgmap v159: 121 pgs: 121 active+clean; 67 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.7 MiB/s rd, 3.7 MiB/s wr, 30 op/s 2024-09-21T09:32:01.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:01 smithi022 ceph-mon[39525]: osdmap e110: 8 total, 8 up, 8 in 2024-09-21T09:32:01.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:01 smithi022 ceph-mon[27763]: pgmap v159: 121 pgs: 121 active+clean; 67 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.7 MiB/s rd, 3.7 MiB/s wr, 30 op/s 2024-09-21T09:32:01.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:01 smithi022 ceph-mon[27763]: osdmap e110: 8 total, 8 up, 8 in 2024-09-21T09:32:01.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:01 smithi064 ceph-mon[31124]: pgmap v159: 121 pgs: 121 active+clean; 67 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.7 MiB/s rd, 3.7 MiB/s wr, 30 op/s 2024-09-21T09:32:01.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:01 smithi064 ceph-mon[31124]: osdmap e110: 8 total, 8 up, 8 in 2024-09-21T09:32:02.418 INFO:tasks.rados.rados.0.smithi022.stdout:261: done (2 left) 2024-09-21T09:32:02.418 INFO:tasks.rados.rados.0.smithi022.stdout:262: done (1 left) 2024-09-21T09:32:02.419 INFO:tasks.rados.rados.0.smithi022.stdout:263: done (0 left) 2024-09-21T09:32:02.419 INFO:tasks.rados.rados.0.smithi022.stdout:264: read oid 47 snap -1 2024-09-21T09:32:02.419 INFO:tasks.rados.rados.0.smithi022.stdout:264: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-21T09:32:02.419 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 36 current snap is 23 2024-09-21T09:32:02.419 INFO:tasks.rados.rados.0.smithi022.stdout:265: seq_num 108 ranges {458752=614400,1073152=606208,1679360=598016} 2024-09-21T09:32:02.424 INFO:tasks.rados.rados.0.smithi022.stdout:265: writing smithi02284960-36 from 458752 to 1073152 tid 1 2024-09-21T09:32:02.428 INFO:tasks.rados.rados.0.smithi022.stdout:265: writing smithi02284960-36 from 1073152 to 1679360 tid 2 2024-09-21T09:32:02.432 INFO:tasks.rados.rados.0.smithi022.stdout:265: writing smithi02284960-36 from 1679360 to 2277376 tid 3 2024-09-21T09:32:02.432 INFO:tasks.rados.rados.0.smithi022.stdout:266: rollback oid 18 current snap is 23 2024-09-21T09:32:02.432 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 18 to 21 2024-09-21T09:32:02.432 INFO:tasks.rados.rados.0.smithi022.stdout:264: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-21T09:32:02.435 INFO:tasks.rados.rados.0.smithi022.stdout:265: finishing write tid 1 to smithi02284960-36 2024-09-21T09:32:02.435 INFO:tasks.rados.rados.0.smithi022.stdout:265: finishing write tid 2 to smithi02284960-36 2024-09-21T09:32:02.435 INFO:tasks.rados.rados.0.smithi022.stdout:266: finishing rollback tid 0 to smithi02284960-18 2024-09-21T09:32:02.436 INFO:tasks.rados.rados.0.smithi022.stdout:266: finishing rollback tid 1 to smithi02284960-18 2024-09-21T09:32:02.436 INFO:tasks.rados.rados.0.smithi022.stdout:264: done (2 left) 2024-09-21T09:32:02.436 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 47 current snap is 23 2024-09-21T09:32:02.436 INFO:tasks.rados.rados.0.smithi022.stdout:267: seq_num 109 ranges {0=630784,630784=491520,1122304=483328,1605632=229376} 2024-09-21T09:32:02.439 INFO:tasks.rados.rados.0.smithi022.stdout:267: writing smithi02284960-47 from 0 to 630784 tid 1 2024-09-21T09:32:02.443 INFO:tasks.rados.rados.0.smithi022.stdout:267: writing smithi02284960-47 from 630784 to 1122304 tid 2 2024-09-21T09:32:02.446 INFO:tasks.rados.rados.0.smithi022.stdout:267: writing smithi02284960-47 from 1122304 to 1605632 tid 3 2024-09-21T09:32:02.448 INFO:tasks.rados.rados.0.smithi022.stdout:267: writing smithi02284960-47 from 1605632 to 1835008 tid 4 2024-09-21T09:32:02.448 INFO:tasks.rados.rados.0.smithi022.stdout:268: delete oid 24 current snap is 23 2024-09-21T09:32:02.449 INFO:tasks.rados.rados.0.smithi022.stdout:266: finishing rollback tid 2 to smithi02284960-18 2024-09-21T09:32:02.449 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 18 v 33 (ObjNum 17 snap 0 seq_num 17) dirty exists 2024-09-21T09:32:02.449 INFO:tasks.rados.rados.0.smithi022.stdout:265: finishing write tid 3 to smithi02284960-36 2024-09-21T09:32:02.449 INFO:tasks.rados.rados.0.smithi022.stdout:265: finishing write tid 4 to smithi02284960-36 2024-09-21T09:32:02.449 INFO:tasks.rados.rados.0.smithi022.stdout:265: finishing write tid 5 to smithi02284960-36 2024-09-21T09:32:02.449 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 36 v 42 (ObjNum 108 snap 23 seq_num 108) dirty exists 2024-09-21T09:32:02.449 INFO:tasks.rados.rados.0.smithi022.stdout:265: left oid 36 (ObjNum 108 snap 23 seq_num 108) 2024-09-21T09:32:02.449 INFO:tasks.rados.rados.0.smithi022.stdout:267: finishing write tid 1 to smithi02284960-47 2024-09-21T09:32:02.449 INFO:tasks.rados.rados.0.smithi022.stdout:267: finishing write tid 2 to smithi02284960-47 2024-09-21T09:32:02.450 INFO:tasks.rados.rados.0.smithi022.stdout:267: finishing write tid 3 to smithi02284960-47 2024-09-21T09:32:02.450 INFO:tasks.rados.rados.0.smithi022.stdout:265: done (3 left) 2024-09-21T09:32:02.451 INFO:tasks.rados.rados.0.smithi022.stdout:266: done (2 left) 2024-09-21T09:32:02.451 INFO:tasks.rados.rados.0.smithi022.stdout:268: done (1 left) 2024-09-21T09:32:02.451 INFO:tasks.rados.rados.0.smithi022.stdout:269: read oid 46 snap -1 2024-09-21T09:32:02.451 INFO:tasks.rados.rados.0.smithi022.stdout:269: expect (ObjNum 45 snap 0 seq_num 45) 2024-09-21T09:32:02.451 INFO:tasks.rados.rados.0.smithi022.stdout:270: read oid 4 snap 17 2024-09-21T09:32:02.451 INFO:tasks.rados.rados.0.smithi022.stdout:270: expect (ObjNum 3 snap 0 seq_num 3) 2024-09-21T09:32:02.451 INFO:tasks.rados.rados.0.smithi022.stdout:271: setattr oid 26 current snap is 23 2024-09-21T09:32:02.451 INFO:tasks.rados.rados.0.smithi022.stdout:267: finishing write tid 4 to smithi02284960-47 2024-09-21T09:32:02.451 INFO:tasks.rados.rados.0.smithi022.stdout:267: finishing write tid 5 to smithi02284960-47 2024-09-21T09:32:02.452 INFO:tasks.rados.rados.0.smithi022.stdout:272: rollback oid 14 current snap is 23 2024-09-21T09:32:02.452 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 14 to 16 2024-09-21T09:32:02.452 INFO:tasks.rados.rados.0.smithi022.stdout:273: delete oid 13 current snap is 23 2024-09-21T09:32:02.452 INFO:tasks.rados.rados.0.smithi022.stdout:272: finishing rollback tid 1 to smithi02284960-14 2024-09-21T09:32:02.452 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 14 v 0 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-09-21T09:32:02.452 INFO:tasks.rados.rados.0.smithi022.stdout:270: expect (ObjNum 3 snap 0 seq_num 3) 2024-09-21T09:32:02.454 INFO:tasks.rados.rados.0.smithi022.stdout:267: finishing write tid 6 to smithi02284960-47 2024-09-21T09:32:02.454 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 47 v 50 (ObjNum 109 snap 23 seq_num 109) dirty exists 2024-09-21T09:32:02.455 INFO:tasks.rados.rados.0.smithi022.stdout:267: left oid 47 (ObjNum 109 snap 23 seq_num 109) 2024-09-21T09:32:02.455 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 26 v 75 (ObjNum 0 snap 0 seq_num 304) dirty exists 2024-09-21T09:32:02.455 INFO:tasks.rados.rados.0.smithi022.stdout:269: expect (ObjNum 45 snap 0 seq_num 45) 2024-09-21T09:32:02.475 INFO:tasks.rados.rados.0.smithi022.stdout:267: done (5 left) 2024-09-21T09:32:02.475 INFO:tasks.rados.rados.0.smithi022.stdout:269: done (4 left) 2024-09-21T09:32:02.475 INFO:tasks.rados.rados.0.smithi022.stdout:270: done (3 left) 2024-09-21T09:32:02.475 INFO:tasks.rados.rados.0.smithi022.stdout:271: done (2 left) 2024-09-21T09:32:02.475 INFO:tasks.rados.rados.0.smithi022.stdout:272: done (1 left) 2024-09-21T09:32:02.475 INFO:tasks.rados.rados.0.smithi022.stdout:273: done (0 left) 2024-09-21T09:32:02.475 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 26 current snap is 23 2024-09-21T09:32:02.475 INFO:tasks.rados.rados.0.smithi022.stdout:274: seq_num 112 ranges {0=679936,679936=548864,1228800=655360} 2024-09-21T09:32:02.479 INFO:tasks.rados.rados.0.smithi022.stdout:274: writing smithi02284960-26 from 0 to 679936 tid 1 2024-09-21T09:32:02.482 INFO:tasks.rados.rados.0.smithi022.stdout:274: writing smithi02284960-26 from 679936 to 1228800 tid 2 2024-09-21T09:32:02.485 INFO:tasks.rados.rados.0.smithi022.stdout:274: writing smithi02284960-26 from 1228800 to 1884160 tid 3 2024-09-21T09:32:02.486 INFO:tasks.rados.rados.0.smithi022.stdout:275: setattr oid 47 current snap is 23 2024-09-21T09:32:02.486 INFO:tasks.rados.rados.0.smithi022.stdout:274: finishing write tid 1 to smithi02284960-26 2024-09-21T09:32:02.486 INFO:tasks.rados.rados.0.smithi022.stdout:274: finishing write tid 2 to smithi02284960-26 2024-09-21T09:32:02.487 INFO:tasks.rados.rados.0.smithi022.stdout:276: setattr oid 30 current snap is 23 2024-09-21T09:32:02.488 INFO:tasks.rados.rados.0.smithi022.stdout:274: finishing write tid 3 to smithi02284960-26 2024-09-21T09:32:02.488 INFO:tasks.rados.rados.0.smithi022.stdout:274: finishing write tid 4 to smithi02284960-26 2024-09-21T09:32:02.489 INFO:tasks.rados.rados.0.smithi022.stdout:277: snap_remove snap 16 2024-09-21T09:32:03.426 INFO:tasks.rados.rados.0.smithi022.stdout:277: done (3 left) 2024-09-21T09:32:03.426 INFO:tasks.rados.rados.0.smithi022.stdout:278: read oid 49 snap -1 2024-09-21T09:32:03.426 INFO:tasks.rados.rados.0.smithi022.stdout:278: expect (ObjNum 3 snap 0 seq_num 3) 2024-09-21T09:32:03.426 INFO:tasks.rados.rados.0.smithi022.stdout:279: read oid 42 snap -1 2024-09-21T09:32:03.426 INFO:tasks.rados.rados.0.smithi022.stdout:279: expect (ObjNum 97 snap 19 seq_num 97) 2024-09-21T09:32:03.426 INFO:tasks.rados.rados.0.smithi022.stdout:280: read oid 33 snap -1 2024-09-21T09:32:03.426 INFO:tasks.rados.rados.0.smithi022.stdout:280: expect (ObjNum 32 snap 0 seq_num 32) 2024-09-21T09:32:03.427 INFO:tasks.rados.rados.0.smithi022.stdout:281: snap_remove snap 18 2024-09-21T09:32:03.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:03 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:32:03.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:03 smithi064 ceph-mon[31124]: Upgrade: Target is version 19.3.0-5092-g62da54b4 (unknown) 2024-09-21T09:32:03.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:03 smithi064 ceph-mon[31124]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, digests ['quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47'] 2024-09-21T09:32:03.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:03 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:32:03.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:03 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:32:03.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:03 smithi064 ceph-mon[31124]: Upgrade: Need to upgrade myself (mgr.y) 2024-09-21T09:32:03.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:03 smithi064 ceph-mon[31124]: osdmap e111: 8 total, 8 up, 8 in 2024-09-21T09:32:03.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:03 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:32:03.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:03 smithi022 ceph-mon[27763]: Upgrade: Target is version 19.3.0-5092-g62da54b4 (unknown) 2024-09-21T09:32:03.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:03 smithi022 ceph-mon[27763]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, digests ['quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47'] 2024-09-21T09:32:03.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:03 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:32:03.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:03 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:32:03.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:03 smithi022 ceph-mon[27763]: Upgrade: Need to upgrade myself (mgr.y) 2024-09-21T09:32:03.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:03 smithi022 ceph-mon[27763]: osdmap e111: 8 total, 8 up, 8 in 2024-09-21T09:32:03.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:03 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:32:03.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:03 smithi022 ceph-mon[39525]: Upgrade: Target is version 19.3.0-5092-g62da54b4 (unknown) 2024-09-21T09:32:03.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:03 smithi022 ceph-mon[39525]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, digests ['quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47'] 2024-09-21T09:32:03.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:03 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:32:03.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:03 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:32:03.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:03 smithi022 ceph-mon[39525]: Upgrade: Need to upgrade myself (mgr.y) 2024-09-21T09:32:03.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:03 smithi022 ceph-mon[39525]: osdmap e111: 8 total, 8 up, 8 in 2024-09-21T09:32:04.424 INFO:tasks.rados.rados.0.smithi022.stdout:281: done (6 left) 2024-09-21T09:32:04.424 INFO:tasks.rados.rados.0.smithi022.stdout:282: copy_from oid 15 from oid 40 current snap is 23 2024-09-21T09:32:04.425 INFO:tasks.rados.rados.0.smithi022.stdout:283: read oid 45 snap -1 2024-09-21T09:32:04.425 INFO:tasks.rados.rados.0.smithi022.stdout:283: expect (ObjNum 33 snap 0 seq_num 33) 2024-09-21T09:32:04.425 INFO:tasks.rados.rados.0.smithi022.stdout:284: snap_remove snap 20 2024-09-21T09:32:04.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:04 smithi064 ceph-mon[31124]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 on smithi064 2024-09-21T09:32:04.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:04 smithi064 ceph-mon[31124]: pgmap v162: 121 pgs: 121 active+clean; 77 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.0 MiB/s rd, 2.6 MiB/s wr, 29 op/s 2024-09-21T09:32:04.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:04 smithi064 ceph-mon[31124]: osdmap e112: 8 total, 8 up, 8 in 2024-09-21T09:32:04.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:04 smithi022 ceph-mon[39525]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 on smithi064 2024-09-21T09:32:04.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:04 smithi022 ceph-mon[39525]: pgmap v162: 121 pgs: 121 active+clean; 77 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.0 MiB/s rd, 2.6 MiB/s wr, 29 op/s 2024-09-21T09:32:04.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:04 smithi022 ceph-mon[39525]: osdmap e112: 8 total, 8 up, 8 in 2024-09-21T09:32:04.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:04 smithi022 ceph-mon[27763]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 on smithi064 2024-09-21T09:32:04.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:04 smithi022 ceph-mon[27763]: pgmap v162: 121 pgs: 121 active+clean; 77 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.0 MiB/s rd, 2.6 MiB/s wr, 29 op/s 2024-09-21T09:32:04.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:04 smithi022 ceph-mon[27763]: osdmap e112: 8 total, 8 up, 8 in 2024-09-21T09:32:05.417 INFO:tasks.rados.rados.0.smithi022.stdout:284: done (8 left) 2024-09-21T09:32:05.417 INFO:tasks.rados.rados.0.smithi022.stdout:285: read oid 11 snap -1 2024-09-21T09:32:05.417 INFO:tasks.rados.rados.0.smithi022.stdout:285: expect (ObjNum 103 snap 19 seq_num 103) 2024-09-21T09:32:05.417 INFO:tasks.rados.rados.0.smithi022.stdout:286: delete oid 21 current snap is 23 2024-09-21T09:32:05.417 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 47 v 51 (ObjNum 109 snap 23 seq_num 109) dirty exists 2024-09-21T09:32:05.417 INFO:tasks.rados.rados.0.smithi022.stdout:274: finishing write tid 5 to smithi02284960-26 2024-09-21T09:32:05.417 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 26 v 81 (ObjNum 112 snap 23 seq_num 112) dirty exists 2024-09-21T09:32:05.417 INFO:tasks.rados.rados.0.smithi022.stdout:274: left oid 26 (ObjNum 112 snap 23 seq_num 112) 2024-09-21T09:32:05.418 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 30 v 33 (ObjNum 29 snap 0 seq_num 29) dirty exists 2024-09-21T09:32:05.418 INFO:tasks.rados.rados.0.smithi022.stdout:278: expect (ObjNum 3 snap 0 seq_num 3) 2024-09-21T09:32:05.419 INFO:tasks.rados.rados.0.smithi022.stdout:280: expect (ObjNum 32 snap 0 seq_num 32) 2024-09-21T09:32:05.440 INFO:tasks.rados.rados.0.smithi022.stdout:279: expect (ObjNum 97 snap 19 seq_num 97) 2024-09-21T09:32:05.516 INFO:tasks.rados.rados.0.smithi022.stdout:283: expect (ObjNum 33 snap 0 seq_num 33) 2024-09-21T09:32:05.531 INFO:tasks.rados.rados.0.smithi022.stdout:282: finishing copy_from racing read to smithi02284960-15 2024-09-21T09:32:05.531 INFO:tasks.rados.rados.0.smithi022.stdout:282: finishing copy_from to smithi02284960-15 2024-09-21T09:32:05.531 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 15 v 90 (ObjNum 92 snap 18 seq_num 92) dirty exists 2024-09-21T09:32:05.531 INFO:tasks.rados.rados.0.smithi022.stdout:285: expect (ObjNum 103 snap 19 seq_num 103) 2024-09-21T09:32:05.577 INFO:tasks.rados.rados.0.smithi022.stdout:274: done (9 left) 2024-09-21T09:32:05.577 INFO:tasks.rados.rados.0.smithi022.stdout:275: done (8 left) 2024-09-21T09:32:05.577 INFO:tasks.rados.rados.0.smithi022.stdout:276: done (7 left) 2024-09-21T09:32:05.577 INFO:tasks.rados.rados.0.smithi022.stdout:278: done (6 left) 2024-09-21T09:32:05.577 INFO:tasks.rados.rados.0.smithi022.stdout:279: done (5 left) 2024-09-21T09:32:05.577 INFO:tasks.rados.rados.0.smithi022.stdout:280: done (4 left) 2024-09-21T09:32:05.577 INFO:tasks.rados.rados.0.smithi022.stdout:282: done (3 left) 2024-09-21T09:32:05.577 INFO:tasks.rados.rados.0.smithi022.stdout:283: done (2 left) 2024-09-21T09:32:05.577 INFO:tasks.rados.rados.0.smithi022.stdout:285: done (1 left) 2024-09-21T09:32:05.577 INFO:tasks.rados.rados.0.smithi022.stdout:286: done (0 left) 2024-09-21T09:32:05.577 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 30 current snap is 23 2024-09-21T09:32:05.578 INFO:tasks.rados.rados.0.smithi022.stdout:287: seq_num 114 ranges {491520=712704,1204224=712704,1916928=491520} 2024-09-21T09:32:05.583 INFO:tasks.rados.rados.0.smithi022.stdout:287: writing smithi02284960-30 from 491520 to 1204224 tid 1 2024-09-21T09:32:05.587 INFO:tasks.rados.rados.0.smithi022.stdout:287: writing smithi02284960-30 from 1204224 to 1916928 tid 2 2024-09-21T09:32:05.591 INFO:tasks.rados.rados.0.smithi022.stdout:287: writing smithi02284960-30 from 1916928 to 2408448 tid 3 2024-09-21T09:32:05.591 INFO:tasks.rados.rados.0.smithi022.stdout:288: rmattr oid 17 current snap is 23 2024-09-21T09:32:05.591 INFO:tasks.rados.rados.0.smithi022.stdout:288: done (1 left) 2024-09-21T09:32:05.591 INFO:tasks.rados.rados.0.smithi022.stdout:289: snap_remove snap 9 2024-09-21T09:32:05.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:05 smithi064 ceph-mon[31124]: osdmap e113: 8 total, 8 up, 8 in 2024-09-21T09:32:05.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:05 smithi064 ceph-mon[31124]: pgmap v165: 121 pgs: 121 active+clean; 77 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.0 MiB/s rd, 2.6 MiB/s wr, 29 op/s 2024-09-21T09:32:05.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:05 smithi064 ceph-mon[31124]: osdmap e114: 8 total, 8 up, 8 in 2024-09-21T09:32:05.807 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:05 smithi022 ceph-mon[39525]: osdmap e113: 8 total, 8 up, 8 in 2024-09-21T09:32:05.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:05 smithi022 ceph-mon[39525]: pgmap v165: 121 pgs: 121 active+clean; 77 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.0 MiB/s rd, 2.6 MiB/s wr, 29 op/s 2024-09-21T09:32:05.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:05 smithi022 ceph-mon[39525]: osdmap e114: 8 total, 8 up, 8 in 2024-09-21T09:32:05.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:05 smithi022 ceph-mon[27763]: osdmap e113: 8 total, 8 up, 8 in 2024-09-21T09:32:05.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:05 smithi022 ceph-mon[27763]: pgmap v165: 121 pgs: 121 active+clean; 77 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.0 MiB/s rd, 2.6 MiB/s wr, 29 op/s 2024-09-21T09:32:05.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:05 smithi022 ceph-mon[27763]: osdmap e114: 8 total, 8 up, 8 in 2024-09-21T09:32:06.437 INFO:tasks.rados.rados.0.smithi022.stdout:287: finishing write tid 1 to smithi02284960-30 2024-09-21T09:32:06.437 INFO:tasks.rados.rados.0.smithi022.stdout:287: finishing write tid 2 to smithi02284960-30 2024-09-21T09:32:06.437 INFO:tasks.rados.rados.0.smithi022.stdout:287: finishing write tid 3 to smithi02284960-30 2024-09-21T09:32:06.437 INFO:tasks.rados.rados.0.smithi022.stdout:287: finishing write tid 4 to smithi02284960-30 2024-09-21T09:32:06.437 INFO:tasks.rados.rados.0.smithi022.stdout:287: finishing write tid 5 to smithi02284960-30 2024-09-21T09:32:06.437 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 30 v 51 (ObjNum 114 snap 23 seq_num 114) dirty exists 2024-09-21T09:32:06.437 INFO:tasks.rados.rados.0.smithi022.stdout:287: left oid 30 (ObjNum 114 snap 23 seq_num 114) 2024-09-21T09:32:06.437 INFO:tasks.rados.rados.0.smithi022.stdout:287: done (1 left) 2024-09-21T09:32:06.437 INFO:tasks.rados.rados.0.smithi022.stdout:289: done (0 left) 2024-09-21T09:32:06.437 INFO:tasks.rados.rados.0.smithi022.stdout:290: snap_remove snap 17 2024-09-21T09:32:07.440 INFO:tasks.rados.rados.0.smithi022.stdout:290: done (0 left) 2024-09-21T09:32:07.441 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 14 current snap is 23 2024-09-21T09:32:07.441 INFO:tasks.rados.rados.0.smithi022.stdout:291: seq_num 115 ranges {0=729088,729088=647168,1376256=434176,1810432=122880} 2024-09-21T09:32:07.445 INFO:tasks.rados.rados.0.smithi022.stdout:291: writing smithi02284960-14 from 0 to 729088 tid 1 2024-09-21T09:32:07.450 INFO:tasks.rados.rados.0.smithi022.stdout:291: writing smithi02284960-14 from 729088 to 1376256 tid 2 2024-09-21T09:32:07.453 INFO:tasks.rados.rados.0.smithi022.stdout:291: writing smithi02284960-14 from 1376256 to 1810432 tid 3 2024-09-21T09:32:07.455 INFO:tasks.rados.rados.0.smithi022.stdout:291: writing smithi02284960-14 from 1810432 to 1933312 tid 4 2024-09-21T09:32:07.455 INFO:tasks.rados.rados.0.smithi022.stdout:292: read oid 3 snap -1 2024-09-21T09:32:07.455 INFO:tasks.rados.rados.0.smithi022.stdout:292: expect (ObjNum 72 snap 5 seq_num 72) 2024-09-21T09:32:07.455 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 20 current snap is 23 2024-09-21T09:32:07.455 INFO:tasks.rados.rados.0.smithi022.stdout:291: finishing write tid 1 to smithi02284960-14 2024-09-21T09:32:07.455 INFO:tasks.rados.rados.0.smithi022.stdout:293: seq_num 116 ranges {393216=745472,1138688=532480,1671168=638976,2310144=32768} 2024-09-21T09:32:07.462 INFO:tasks.rados.rados.0.smithi022.stdout:293: writing smithi02284960-20 from 393216 to 1138688 tid 1 2024-09-21T09:32:07.465 INFO:tasks.rados.rados.0.smithi022.stdout:293: writing smithi02284960-20 from 1138688 to 1671168 tid 2 2024-09-21T09:32:07.470 INFO:tasks.rados.rados.0.smithi022.stdout:293: writing smithi02284960-20 from 1671168 to 2310144 tid 3 2024-09-21T09:32:07.470 INFO:tasks.rados.rados.0.smithi022.stdout:293: writing smithi02284960-20 from 2310144 to 2342912 tid 4 2024-09-21T09:32:07.470 INFO:tasks.rados.rados.0.smithi022.stdout:294: snap_create 2024-09-21T09:32:07.470 INFO:tasks.rados.rados.0.smithi022.stdout:291: finishing write tid 2 to smithi02284960-14 2024-09-21T09:32:07.470 INFO:tasks.rados.rados.0.smithi022.stdout:291: finishing write tid 3 to smithi02284960-14 2024-09-21T09:32:07.470 INFO:tasks.rados.rados.0.smithi022.stdout:291: finishing write tid 4 to smithi02284960-14 2024-09-21T09:32:07.470 INFO:tasks.rados.rados.0.smithi022.stdout:291: finishing write tid 5 to smithi02284960-14 2024-09-21T09:32:07.471 INFO:tasks.rados.rados.0.smithi022.stdout:291: finishing write tid 6 to smithi02284960-14 2024-09-21T09:32:07.471 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 14 v 43 (ObjNum 115 snap 23 seq_num 115) dirty exists 2024-09-21T09:32:07.471 INFO:tasks.rados.rados.0.smithi022.stdout:291: left oid 14 (ObjNum 115 snap 23 seq_num 115) 2024-09-21T09:32:07.471 INFO:tasks.rados.rados.0.smithi022.stdout:292: expect (ObjNum 72 snap 5 seq_num 72) 2024-09-21T09:32:07.507 INFO:tasks.rados.rados.0.smithi022.stdout:293: finishing write tid 1 to smithi02284960-20 2024-09-21T09:32:07.507 INFO:tasks.rados.rados.0.smithi022.stdout:293: finishing write tid 2 to smithi02284960-20 2024-09-21T09:32:07.507 INFO:tasks.rados.rados.0.smithi022.stdout:293: finishing write tid 3 to smithi02284960-20 2024-09-21T09:32:07.508 INFO:tasks.rados.rados.0.smithi022.stdout:293: finishing write tid 4 to smithi02284960-20 2024-09-21T09:32:07.508 INFO:tasks.rados.rados.0.smithi022.stdout:293: finishing write tid 5 to smithi02284960-20 2024-09-21T09:32:07.508 INFO:tasks.rados.rados.0.smithi022.stdout:293: finishing write tid 6 to smithi02284960-20 2024-09-21T09:32:07.508 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 20 v 136 (ObjNum 116 snap 23 seq_num 116) dirty exists 2024-09-21T09:32:07.508 INFO:tasks.rados.rados.0.smithi022.stdout:293: left oid 20 (ObjNum 116 snap 23 seq_num 116) 2024-09-21T09:32:07.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:07 smithi064 ceph-mon[31124]: osdmap e115: 8 total, 8 up, 8 in 2024-09-21T09:32:07.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:07 smithi064 ceph-mon[31124]: pgmap v168: 121 pgs: 121 active+clean; 77 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:32:07.807 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:07 smithi022 ceph-mon[39525]: osdmap e115: 8 total, 8 up, 8 in 2024-09-21T09:32:07.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:07 smithi022 ceph-mon[39525]: pgmap v168: 121 pgs: 121 active+clean; 77 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:32:07.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:07 smithi022 ceph-mon[27763]: osdmap e115: 8 total, 8 up, 8 in 2024-09-21T09:32:07.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:07 smithi022 ceph-mon[27763]: pgmap v168: 121 pgs: 121 active+clean; 77 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:32:08.444 INFO:tasks.rados.rados.0.smithi022.stdout:291: done (3 left) 2024-09-21T09:32:08.444 INFO:tasks.rados.rados.0.smithi022.stdout:292: done (2 left) 2024-09-21T09:32:08.444 INFO:tasks.rados.rados.0.smithi022.stdout:293: done (1 left) 2024-09-21T09:32:08.444 INFO:tasks.rados.rados.0.smithi022.stdout:294: done (0 left) 2024-09-21T09:32:08.444 INFO:tasks.rados.rados.0.smithi022.stdout:295: delete oid 41 current snap is 24 2024-09-21T09:32:08.448 INFO:tasks.rados.rados.0.smithi022.stdout:295: done (0 left) 2024-09-21T09:32:08.448 INFO:tasks.rados.rados.0.smithi022.stdout:296: read oid 13 snap -1 2024-09-21T09:32:08.448 INFO:tasks.rados.rados.0.smithi022.stdout:296: expect deleted 2024-09-21T09:32:08.448 INFO:tasks.rados.rados.0.smithi022.stdout:297: setattr oid 26 current snap is 24 2024-09-21T09:32:08.449 INFO:tasks.rados.rados.0.smithi022.stdout:298: copy_from oid 32 from oid 7 current snap is 24 2024-09-21T09:32:08.449 INFO:tasks.rados.rados.0.smithi022.stdout:299: read oid 22 snap -1 2024-09-21T09:32:08.449 INFO:tasks.rados.rados.0.smithi022.stdout:299: expect (ObjNum 21 snap 0 seq_num 21) 2024-09-21T09:32:08.450 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 15 current snap is 24 2024-09-21T09:32:08.450 INFO:tasks.rados.rados.0.smithi022.stdout:300: seq_num 118 ranges {1548288=786432,2334720=704512,3039232=499712} 2024-09-21T09:32:08.463 INFO:tasks.rados.rados.0.smithi022.stdout:300: writing smithi02284960-15 from 1548288 to 2334720 tid 1 2024-09-21T09:32:08.467 INFO:tasks.rados.rados.0.smithi022.stdout:300: writing smithi02284960-15 from 2334720 to 3039232 tid 2 2024-09-21T09:32:08.470 INFO:tasks.rados.rados.0.smithi022.stdout:300: writing smithi02284960-15 from 3039232 to 3538944 tid 3 2024-09-21T09:32:08.470 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 49 current snap is 24 2024-09-21T09:32:08.470 INFO:tasks.rados.rados.0.smithi022.stdout:301: seq_num 119 ranges {57344=802816,860160=589824,1449984=409600,1859584=204800} 2024-09-21T09:32:08.475 INFO:tasks.rados.rados.0.smithi022.stdout:301: writing smithi02284960-49 from 57344 to 860160 tid 1 2024-09-21T09:32:08.478 INFO:tasks.rados.rados.0.smithi022.stdout:301: writing smithi02284960-49 from 860160 to 1449984 tid 2 2024-09-21T09:32:08.480 INFO:tasks.rados.rados.0.smithi022.stdout:301: writing smithi02284960-49 from 1449984 to 1859584 tid 3 2024-09-21T09:32:08.482 INFO:tasks.rados.rados.0.smithi022.stdout:301: writing smithi02284960-49 from 1859584 to 2064384 tid 4 2024-09-21T09:32:08.482 INFO:tasks.rados.rados.0.smithi022.stdout:302: rollback oid 33 current snap is 24 2024-09-21T09:32:08.482 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 33 to 19 2024-09-21T09:32:08.482 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 26 v 117 (ObjNum 112 snap 23 seq_num 112) dirty exists 2024-09-21T09:32:08.482 INFO:tasks.rados.rados.0.smithi022.stdout:299: expect (ObjNum 21 snap 0 seq_num 21) 2024-09-21T09:32:08.492 INFO:tasks.rados.rados.0.smithi022.stdout:298: finishing copy_from racing read to smithi02284960-32 2024-09-21T09:32:08.492 INFO:tasks.rados.rados.0.smithi022.stdout:298: finishing copy_from to smithi02284960-32 2024-09-21T09:32:08.493 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 32 v 42 (ObjNum 72 snap 5 seq_num 72) dirty exists 2024-09-21T09:32:08.493 INFO:tasks.rados.rados.0.smithi022.stdout:300: finishing write tid 1 to smithi02284960-15 2024-09-21T09:32:08.493 INFO:tasks.rados.rados.0.smithi022.stdout:300: finishing write tid 2 to smithi02284960-15 2024-09-21T09:32:08.493 INFO:tasks.rados.rados.0.smithi022.stdout:296: done (6 left) 2024-09-21T09:32:08.493 INFO:tasks.rados.rados.0.smithi022.stdout:297: done (5 left) 2024-09-21T09:32:08.493 INFO:tasks.rados.rados.0.smithi022.stdout:298: done (4 left) 2024-09-21T09:32:08.493 INFO:tasks.rados.rados.0.smithi022.stdout:299: done (3 left) 2024-09-21T09:32:08.493 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 25 current snap is 24 2024-09-21T09:32:08.493 INFO:tasks.rados.rados.0.smithi022.stdout:300: finishing write tid 3 to smithi02284960-15 2024-09-21T09:32:08.493 INFO:tasks.rados.rados.0.smithi022.stdout:303: seq_num 120 ranges {1925120=417792,2342912=475136,2818048=622592,3440640=507904} 2024-09-21T09:32:08.505 INFO:tasks.rados.rados.0.smithi022.stdout:303: writing smithi02284960-25 from 1925120 to 2342912 tid 1 2024-09-21T09:32:08.507 INFO:tasks.rados.rados.0.smithi022.stdout:303: writing smithi02284960-25 from 2342912 to 2818048 tid 2 2024-09-21T09:32:08.511 INFO:tasks.rados.rados.0.smithi022.stdout:303: writing smithi02284960-25 from 2818048 to 3440640 tid 3 2024-09-21T09:32:08.515 INFO:tasks.rados.rados.0.smithi022.stdout:303: writing smithi02284960-25 from 3440640 to 3948544 tid 4 2024-09-21T09:32:08.515 INFO:tasks.rados.rados.0.smithi022.stdout:304: read oid 5 snap -1 2024-09-21T09:32:08.515 INFO:tasks.rados.rados.0.smithi022.stdout:304: expect deleted 2024-09-21T09:32:08.515 INFO:tasks.rados.rados.0.smithi022.stdout:305: rollback oid 28 current snap is 24 2024-09-21T09:32:08.515 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 28 to 19 2024-09-21T09:32:08.515 INFO:tasks.rados.rados.0.smithi022.stdout:300: finishing write tid 4 to smithi02284960-15 2024-09-21T09:32:08.516 INFO:tasks.rados.rados.0.smithi022.stdout:300: finishing write tid 5 to smithi02284960-15 2024-09-21T09:32:08.516 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 15 v 153 (ObjNum 118 snap 24 seq_num 118) dirty exists 2024-09-21T09:32:08.516 INFO:tasks.rados.rados.0.smithi022.stdout:300: left oid 15 (ObjNum 118 snap 24 seq_num 118) 2024-09-21T09:32:08.516 INFO:tasks.rados.rados.0.smithi022.stdout:301: finishing write tid 1 to smithi02284960-49 2024-09-21T09:32:08.516 INFO:tasks.rados.rados.0.smithi022.stdout:301: finishing write tid 2 to smithi02284960-49 2024-09-21T09:32:08.516 INFO:tasks.rados.rados.0.smithi022.stdout:301: finishing write tid 3 to smithi02284960-49 2024-09-21T09:32:08.516 INFO:tasks.rados.rados.0.smithi022.stdout:301: finishing write tid 4 to smithi02284960-49 2024-09-21T09:32:08.516 INFO:tasks.rados.rados.0.smithi022.stdout:301: finishing write tid 5 to smithi02284960-49 2024-09-21T09:32:08.516 INFO:tasks.rados.rados.0.smithi022.stdout:302: finishing rollback tid 0 to smithi02284960-33 2024-09-21T09:32:08.516 INFO:tasks.rados.rados.0.smithi022.stdout:302: finishing rollback tid 1 to smithi02284960-33 2024-09-21T09:32:08.516 INFO:tasks.rados.rados.0.smithi022.stdout:300: done (5 left) 2024-09-21T09:32:08.516 INFO:tasks.rados.rados.0.smithi022.stdout:306: rmattr oid 41 current snap is 24 2024-09-21T09:32:08.516 INFO:tasks.rados.rados.0.smithi022.stdout:306: done (5 left) 2024-09-21T09:32:08.516 INFO:tasks.rados.rados.0.smithi022.stdout:307: copy_from oid 45 from oid 14 current snap is 24 2024-09-21T09:32:08.517 INFO:tasks.rados.rados.0.smithi022.stdout:302: finishing rollback tid 2 to smithi02284960-33 2024-09-21T09:32:08.517 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 33 v 142 (ObjNum 32 snap 0 seq_num 32) dirty exists 2024-09-21T09:32:08.517 INFO:tasks.rados.rados.0.smithi022.stdout:301: finishing write tid 6 to smithi02284960-49 2024-09-21T09:32:08.517 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 49 v 49 (ObjNum 119 snap 24 seq_num 119) dirty exists 2024-09-21T09:32:08.517 INFO:tasks.rados.rados.0.smithi022.stdout:301: left oid 49 (ObjNum 119 snap 24 seq_num 119) 2024-09-21T09:32:08.517 INFO:tasks.rados.rados.0.smithi022.stdout:303: finishing write tid 1 to smithi02284960-25 2024-09-21T09:32:08.517 INFO:tasks.rados.rados.0.smithi022.stdout:303: finishing write tid 2 to smithi02284960-25 2024-09-21T09:32:08.517 INFO:tasks.rados.rados.0.smithi022.stdout:303: finishing write tid 3 to smithi02284960-25 2024-09-21T09:32:08.517 INFO:tasks.rados.rados.0.smithi022.stdout:301: done (5 left) 2024-09-21T09:32:08.517 INFO:tasks.rados.rados.0.smithi022.stdout:302: done (4 left) 2024-09-21T09:32:08.517 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 32 current snap is 24 2024-09-21T09:32:08.517 INFO:tasks.rados.rados.0.smithi022.stdout:308: seq_num 121 ranges {1318912=434176,1753088=753664,2506752=425984,2932736=425984} 2024-09-21T09:32:08.525 INFO:tasks.rados.rados.0.smithi022.stdout:308: writing smithi02284960-32 from 1318912 to 1753088 tid 1 2024-09-21T09:32:08.530 INFO:tasks.rados.rados.0.smithi022.stdout:308: writing smithi02284960-32 from 1753088 to 2506752 tid 2 2024-09-21T09:32:08.533 INFO:tasks.rados.rados.0.smithi022.stdout:308: writing smithi02284960-32 from 2506752 to 2932736 tid 3 2024-09-21T09:32:08.536 INFO:tasks.rados.rados.0.smithi022.stdout:308: writing smithi02284960-32 from 2932736 to 3358720 tid 4 2024-09-21T09:32:08.536 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 11 current snap is 24 2024-09-21T09:32:08.536 INFO:tasks.rados.rados.0.smithi022.stdout:309: seq_num 122 ranges {1736704=450560,2187264=688128,2875392=630784,3506176=286720} 2024-09-21T09:32:08.548 INFO:tasks.rados.rados.0.smithi022.stdout:309: writing smithi02284960-11 from 1736704 to 2187264 tid 1 2024-09-21T09:32:08.552 INFO:tasks.rados.rados.0.smithi022.stdout:309: writing smithi02284960-11 from 2187264 to 2875392 tid 2 2024-09-21T09:32:08.556 INFO:tasks.rados.rados.0.smithi022.stdout:309: writing smithi02284960-11 from 2875392 to 3506176 tid 3 2024-09-21T09:32:08.558 INFO:tasks.rados.rados.0.smithi022.stdout:309: writing smithi02284960-11 from 3506176 to 3792896 tid 4 2024-09-21T09:32:08.558 INFO:tasks.rados.rados.0.smithi022.stdout:310: read oid 46 snap 22 2024-09-21T09:32:08.558 INFO:tasks.rados.rados.0.smithi022.stdout:310: expect (ObjNum 45 snap 0 seq_num 45) 2024-09-21T09:32:08.558 INFO:tasks.rados.rados.0.smithi022.stdout:311: rmattr oid 16 current snap is 24 2024-09-21T09:32:08.558 INFO:tasks.rados.rados.0.smithi022.stdout:311: done (7 left) 2024-09-21T09:32:08.558 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 49 current snap is 24 2024-09-21T09:32:08.558 INFO:tasks.rados.rados.0.smithi022.stdout:312: seq_num 123 ranges {2064384=466944,2531328=573440,3104768=794624,3899392=237568} 2024-09-21T09:32:08.571 INFO:tasks.rados.rados.0.smithi022.stdout:312: writing smithi02284960-49 from 2064384 to 2531328 tid 1 2024-09-21T09:32:08.574 INFO:tasks.rados.rados.0.smithi022.stdout:312: writing smithi02284960-49 from 2531328 to 3104768 tid 2 2024-09-21T09:32:08.579 INFO:tasks.rados.rados.0.smithi022.stdout:312: writing smithi02284960-49 from 3104768 to 3899392 tid 3 2024-09-21T09:32:08.581 INFO:tasks.rados.rados.0.smithi022.stdout:312: writing smithi02284960-49 from 3899392 to 4136960 tid 4 2024-09-21T09:32:08.581 INFO:tasks.rados.rados.0.smithi022.stdout:313: read oid 48 snap -1 2024-09-21T09:32:08.581 INFO:tasks.rados.rados.0.smithi022.stdout:313: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-21T09:32:08.581 INFO:tasks.rados.rados.0.smithi022.stdout:314: rollback oid 3 current snap is 24 2024-09-21T09:32:08.581 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 3 to 19 2024-09-21T09:32:08.581 INFO:tasks.rados.rados.0.smithi022.stdout:303: finishing write tid 4 to smithi02284960-25 2024-09-21T09:32:08.582 INFO:tasks.rados.rados.0.smithi022.stdout:303: finishing write tid 5 to smithi02284960-25 2024-09-21T09:32:08.582 INFO:tasks.rados.rados.0.smithi022.stdout:305: finishing rollback tid 0 to smithi02284960-28 2024-09-21T09:32:08.582 INFO:tasks.rados.rados.0.smithi022.stdout:305: finishing rollback tid 1 to smithi02284960-28 2024-09-21T09:32:08.582 INFO:tasks.rados.rados.0.smithi022.stdout:303: finishing write tid 6 to smithi02284960-25 2024-09-21T09:32:08.582 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 25 v 158 (ObjNum 120 snap 24 seq_num 120) dirty exists 2024-09-21T09:32:08.582 INFO:tasks.rados.rados.0.smithi022.stdout:303: left oid 25 (ObjNum 120 snap 24 seq_num 120) 2024-09-21T09:32:08.582 INFO:tasks.rados.rados.0.smithi022.stdout:305: finishing rollback tid 2 to smithi02284960-28 2024-09-21T09:32:08.582 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 28 v 50 (ObjNum 9 snap 0 seq_num 9) dirty exists 2024-09-21T09:32:08.582 INFO:tasks.rados.rados.0.smithi022.stdout:307: finishing copy_from racing read to smithi02284960-45 2024-09-21T09:32:08.582 INFO:tasks.rados.rados.0.smithi022.stdout:303: done (9 left) 2024-09-21T09:32:08.582 INFO:tasks.rados.rados.0.smithi022.stdout:304: done (8 left) 2024-09-21T09:32:08.582 INFO:tasks.rados.rados.0.smithi022.stdout:305: done (7 left) 2024-09-21T09:32:08.582 INFO:tasks.rados.rados.0.smithi022.stdout:315: snap_remove snap 21 2024-09-21T09:32:08.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:08 smithi064 ceph-mon[31124]: osdmap e116: 8 total, 8 up, 8 in 2024-09-21T09:32:08.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:08 smithi022 ceph-mon[39525]: osdmap e116: 8 total, 8 up, 8 in 2024-09-21T09:32:08.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:08 smithi022 ceph-mon[27763]: osdmap e116: 8 total, 8 up, 8 in 2024-09-21T09:32:09.451 INFO:tasks.rados.rados.0.smithi022.stdout:307: finishing copy_from to smithi02284960-45 2024-09-21T09:32:09.451 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 45 v 43 (ObjNum 115 snap 23 seq_num 115) dirty exists 2024-09-21T09:32:09.451 INFO:tasks.rados.rados.0.smithi022.stdout:308: finishing write tid 1 to smithi02284960-32 2024-09-21T09:32:09.451 INFO:tasks.rados.rados.0.smithi022.stdout:308: finishing write tid 2 to smithi02284960-32 2024-09-21T09:32:09.451 INFO:tasks.rados.rados.0.smithi022.stdout:308: finishing write tid 3 to smithi02284960-32 2024-09-21T09:32:09.451 INFO:tasks.rados.rados.0.smithi022.stdout:308: finishing write tid 4 to smithi02284960-32 2024-09-21T09:32:09.451 INFO:tasks.rados.rados.0.smithi022.stdout:308: finishing write tid 5 to smithi02284960-32 2024-09-21T09:32:09.451 INFO:tasks.rados.rados.0.smithi022.stdout:308: finishing write tid 6 to smithi02284960-32 2024-09-21T09:32:09.451 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 32 v 147 (ObjNum 121 snap 24 seq_num 121) dirty exists 2024-09-21T09:32:09.452 INFO:tasks.rados.rados.0.smithi022.stdout:308: left oid 32 (ObjNum 121 snap 24 seq_num 121) 2024-09-21T09:32:09.452 INFO:tasks.rados.rados.0.smithi022.stdout:309: finishing write tid 1 to smithi02284960-11 2024-09-21T09:32:09.452 INFO:tasks.rados.rados.0.smithi022.stdout:309: finishing write tid 2 to smithi02284960-11 2024-09-21T09:32:09.452 INFO:tasks.rados.rados.0.smithi022.stdout:309: finishing write tid 3 to smithi02284960-11 2024-09-21T09:32:09.452 INFO:tasks.rados.rados.0.smithi022.stdout:309: finishing write tid 4 to smithi02284960-11 2024-09-21T09:32:09.452 INFO:tasks.rados.rados.0.smithi022.stdout:309: finishing write tid 5 to smithi02284960-11 2024-09-21T09:32:09.452 INFO:tasks.rados.rados.0.smithi022.stdout:309: finishing write tid 6 to smithi02284960-11 2024-09-21T09:32:09.452 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 11 v 67 (ObjNum 122 snap 24 seq_num 122) dirty exists 2024-09-21T09:32:09.452 INFO:tasks.rados.rados.0.smithi022.stdout:309: left oid 11 (ObjNum 122 snap 24 seq_num 122) 2024-09-21T09:32:09.452 INFO:tasks.rados.rados.0.smithi022.stdout:310: expect (ObjNum 45 snap 0 seq_num 45) 2024-09-21T09:32:09.476 INFO:tasks.rados.rados.0.smithi022.stdout:312: finishing write tid 1 to smithi02284960-49 2024-09-21T09:32:09.477 INFO:tasks.rados.rados.0.smithi022.stdout:312: finishing write tid 2 to smithi02284960-49 2024-09-21T09:32:09.477 INFO:tasks.rados.rados.0.smithi022.stdout:313: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-21T09:32:09.479 INFO:tasks.rados.rados.0.smithi022.stdout:312: finishing write tid 3 to smithi02284960-49 2024-09-21T09:32:09.479 INFO:tasks.rados.rados.0.smithi022.stdout:312: finishing write tid 4 to smithi02284960-49 2024-09-21T09:32:09.479 INFO:tasks.rados.rados.0.smithi022.stdout:312: finishing write tid 5 to smithi02284960-49 2024-09-21T09:32:09.479 INFO:tasks.rados.rados.0.smithi022.stdout:314: finishing rollback tid 0 to smithi02284960-3 2024-09-21T09:32:09.479 INFO:tasks.rados.rados.0.smithi022.stdout:314: finishing rollback tid 1 to smithi02284960-3 2024-09-21T09:32:09.479 INFO:tasks.rados.rados.0.smithi022.stdout:314: finishing rollback tid 2 to smithi02284960-3 2024-09-21T09:32:09.479 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 3 v 45 (ObjNum 72 snap 5 seq_num 72) dirty exists 2024-09-21T09:32:09.479 INFO:tasks.rados.rados.0.smithi022.stdout:307: done (7 left) 2024-09-21T09:32:09.479 INFO:tasks.rados.rados.0.smithi022.stdout:308: done (6 left) 2024-09-21T09:32:09.480 INFO:tasks.rados.rados.0.smithi022.stdout:309: done (5 left) 2024-09-21T09:32:09.480 INFO:tasks.rados.rados.0.smithi022.stdout:310: done (4 left) 2024-09-21T09:32:09.480 INFO:tasks.rados.rados.0.smithi022.stdout:313: done (3 left) 2024-09-21T09:32:09.480 INFO:tasks.rados.rados.0.smithi022.stdout:314: done (2 left) 2024-09-21T09:32:09.480 INFO:tasks.rados.rados.0.smithi022.stdout:315: done (1 left) 2024-09-21T09:32:09.480 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 17 current snap is 24 2024-09-21T09:32:09.480 INFO:tasks.rados.rados.0.smithi022.stdout:316: seq_num 124 ranges {2056192=483328,2539520=458752,2998272=598016,3596288=548864} 2024-09-21T09:32:09.492 INFO:tasks.rados.rados.0.smithi022.stdout:316: writing smithi02284960-17 from 2056192 to 2539520 tid 1 2024-09-21T09:32:09.495 INFO:tasks.rados.rados.0.smithi022.stdout:316: writing smithi02284960-17 from 2539520 to 2998272 tid 2 2024-09-21T09:32:09.499 INFO:tasks.rados.rados.0.smithi022.stdout:316: writing smithi02284960-17 from 2998272 to 3596288 tid 3 2024-09-21T09:32:09.503 INFO:tasks.rados.rados.0.smithi022.stdout:316: writing smithi02284960-17 from 3596288 to 4145152 tid 4 2024-09-21T09:32:09.503 INFO:tasks.rados.rados.0.smithi022.stdout:317: setattr oid 44 current snap is 24 2024-09-21T09:32:09.503 INFO:tasks.rados.rados.0.smithi022.stdout:312: finishing write tid 6 to smithi02284960-49 2024-09-21T09:32:09.503 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 49 v 54 (ObjNum 123 snap 24 seq_num 123) dirty exists 2024-09-21T09:32:09.504 INFO:tasks.rados.rados.0.smithi022.stdout:312: left oid 49 (ObjNum 123 snap 24 seq_num 123) 2024-09-21T09:32:09.504 INFO:tasks.rados.rados.0.smithi022.stdout:316: finishing write tid 1 to smithi02284960-17 2024-09-21T09:32:09.504 INFO:tasks.rados.rados.0.smithi022.stdout:316: finishing write tid 2 to smithi02284960-17 2024-09-21T09:32:09.504 INFO:tasks.rados.rados.0.smithi022.stdout:316: finishing write tid 3 to smithi02284960-17 2024-09-21T09:32:09.504 INFO:tasks.rados.rados.0.smithi022.stdout:312: done (2 left) 2024-09-21T09:32:09.504 INFO:tasks.rados.rados.0.smithi022.stdout:318: snap_remove snap 11 2024-09-21T09:32:09.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:09 smithi022 ceph-mon[39525]: osdmap e117: 8 total, 8 up, 8 in 2024-09-21T09:32:09.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:09 smithi022 ceph-mon[39525]: pgmap v171: 121 pgs: 121 active+clean; 76 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4.7 MiB/s rd, 1.8 MiB/s wr, 11 op/s 2024-09-21T09:32:09.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:09 smithi022 ceph-mon[27763]: osdmap e117: 8 total, 8 up, 8 in 2024-09-21T09:32:09.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:09 smithi022 ceph-mon[27763]: pgmap v171: 121 pgs: 121 active+clean; 76 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4.7 MiB/s rd, 1.8 MiB/s wr, 11 op/s 2024-09-21T09:32:09.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:09 smithi064 ceph-mon[31124]: osdmap e117: 8 total, 8 up, 8 in 2024-09-21T09:32:09.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:09 smithi064 ceph-mon[31124]: pgmap v171: 121 pgs: 121 active+clean; 76 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4.7 MiB/s rd, 1.8 MiB/s wr, 11 op/s 2024-09-21T09:32:10.418 INFO:tasks.rados.rados.0.smithi022.stdout:316: finishing write tid 4 to smithi02284960-17 2024-09-21T09:32:10.418 INFO:tasks.rados.rados.0.smithi022.stdout:316: finishing write tid 5 to smithi02284960-17 2024-09-21T09:32:10.418 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 44 v 72 (ObjNum 129 snap 0 seq_num 3569498496) dirty exists 2024-09-21T09:32:10.418 INFO:tasks.rados.rados.0.smithi022.stdout:316: finishing write tid 6 to smithi02284960-17 2024-09-21T09:32:10.419 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 17 v 71 (ObjNum 124 snap 24 seq_num 124) dirty exists 2024-09-21T09:32:10.419 INFO:tasks.rados.rados.0.smithi022.stdout:316: left oid 17 (ObjNum 124 snap 24 seq_num 124) 2024-09-21T09:32:10.419 INFO:tasks.rados.rados.0.smithi022.stdout:316: done (2 left) 2024-09-21T09:32:10.419 INFO:tasks.rados.rados.0.smithi022.stdout:317: done (1 left) 2024-09-21T09:32:10.419 INFO:tasks.rados.rados.0.smithi022.stdout:318: done (0 left) 2024-09-21T09:32:10.419 INFO:tasks.rados.rados.0.smithi022.stdout:319: setattr oid 40 current snap is 24 2024-09-21T09:32:10.420 INFO:tasks.rados.rados.0.smithi022.stdout:320: snap_create 2024-09-21T09:32:10.428 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 40 v 164 (ObjNum 92 snap 18 seq_num 92) dirty exists 2024-09-21T09:32:10.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:10 smithi022 ceph-mon[39525]: osdmap e118: 8 total, 8 up, 8 in 2024-09-21T09:32:10.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:10 smithi022 ceph-mon[39525]: osdmap e119: 8 total, 8 up, 8 in 2024-09-21T09:32:10.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:10 smithi022 ceph-mon[27763]: osdmap e118: 8 total, 8 up, 8 in 2024-09-21T09:32:10.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:10 smithi022 ceph-mon[27763]: osdmap e119: 8 total, 8 up, 8 in 2024-09-21T09:32:10.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:10 smithi064 ceph-mon[31124]: osdmap e118: 8 total, 8 up, 8 in 2024-09-21T09:32:10.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:10 smithi064 ceph-mon[31124]: osdmap e119: 8 total, 8 up, 8 in 2024-09-21T09:32:10.941 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:32:10 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[33213]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:32:10] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-21T09:32:11.419 INFO:tasks.rados.rados.0.smithi022.stdout:319: done (1 left) 2024-09-21T09:32:11.420 INFO:tasks.rados.rados.0.smithi022.stdout:320: done (0 left) 2024-09-21T09:32:11.420 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 33 current snap is 25 2024-09-21T09:32:11.420 INFO:tasks.rados.rados.0.smithi022.stdout:321: seq_num 125 ranges {540672=499712,1040384=745472,1785856=409600,2195456=450560} 2024-09-21T09:32:11.425 INFO:tasks.rados.rados.0.smithi022.stdout:321: writing smithi02284960-33 from 540672 to 1040384 tid 1 2024-09-21T09:32:11.430 INFO:tasks.rados.rados.0.smithi022.stdout:321: writing smithi02284960-33 from 1040384 to 1785856 tid 2 2024-09-21T09:32:11.432 INFO:tasks.rados.rados.0.smithi022.stdout:321: writing smithi02284960-33 from 1785856 to 2195456 tid 3 2024-09-21T09:32:11.435 INFO:tasks.rados.rados.0.smithi022.stdout:321: writing smithi02284960-33 from 2195456 to 2646016 tid 4 2024-09-21T09:32:11.435 INFO:tasks.rados.rados.0.smithi022.stdout:322: copy_from oid 18 from oid 13 current snap is 25 2024-09-21T09:32:11.435 INFO:tasks.rados.rados.0.smithi022.stdout:321: finishing write tid 1 to smithi02284960-33 2024-09-21T09:32:11.435 INFO:tasks.rados.rados.0.smithi022.stdout:321: finishing write tid 2 to smithi02284960-33 2024-09-21T09:32:11.436 INFO:tasks.rados.rados.0.smithi022.stdout:321: finishing write tid 3 to smithi02284960-33 2024-09-21T09:32:11.436 INFO:tasks.rados.rados.0.smithi022.stdout:323: snap_create 2024-09-21T09:32:11.438 INFO:tasks.rados.rados.0.smithi022.stdout:321: finishing write tid 4 to smithi02284960-33 2024-09-21T09:32:11.438 INFO:tasks.rados.rados.0.smithi022.stdout:322: finishing copy_from to smithi02284960-18 2024-09-21T09:32:11.438 INFO:tasks.rados.rados.0.smithi022.stdout:322: got expected ENOENT (src dne) 2024-09-21T09:32:11.438 INFO:tasks.rados.rados.0.smithi022.stdout:322: finishing copy_from racing read to smithi02284960-18 2024-09-21T09:32:11.438 INFO:tasks.rados.rados.0.smithi022.stdout:321: finishing write tid 5 to smithi02284960-33 2024-09-21T09:32:11.439 INFO:tasks.rados.rados.0.smithi022.stdout:321: finishing write tid 6 to smithi02284960-33 2024-09-21T09:32:11.439 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 33 v 169 (ObjNum 125 snap 25 seq_num 125) dirty exists 2024-09-21T09:32:11.439 INFO:tasks.rados.rados.0.smithi022.stdout:321: left oid 33 (ObjNum 125 snap 25 seq_num 125) 2024-09-21T09:32:11.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:11 smithi022 ceph-mon[27763]: pgmap v174: 121 pgs: 121 active+clean; 76 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4.7 MiB/s rd, 1.8 MiB/s wr, 11 op/s 2024-09-21T09:32:11.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:11 smithi022 ceph-mon[27763]: osdmap e120: 8 total, 8 up, 8 in 2024-09-21T09:32:11.558 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:32:11 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:32:11] "GET /metrics HTTP/1.1" 200 239872 "" "Prometheus/2.43.0" 2024-09-21T09:32:11.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:11 smithi022 ceph-mon[39525]: pgmap v174: 121 pgs: 121 active+clean; 76 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4.7 MiB/s rd, 1.8 MiB/s wr, 11 op/s 2024-09-21T09:32:11.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:11 smithi022 ceph-mon[39525]: osdmap e120: 8 total, 8 up, 8 in 2024-09-21T09:32:11.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:11 smithi064 ceph-mon[31124]: pgmap v174: 121 pgs: 121 active+clean; 76 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4.7 MiB/s rd, 1.8 MiB/s wr, 11 op/s 2024-09-21T09:32:11.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:11 smithi064 ceph-mon[31124]: osdmap e120: 8 total, 8 up, 8 in 2024-09-21T09:32:12.421 INFO:tasks.rados.rados.0.smithi022.stdout:321: done (2 left) 2024-09-21T09:32:12.421 INFO:tasks.rados.rados.0.smithi022.stdout:322: done (1 left) 2024-09-21T09:32:12.421 INFO:tasks.rados.rados.0.smithi022.stdout:323: done (0 left) 2024-09-21T09:32:12.421 INFO:tasks.rados.rados.0.smithi022.stdout:324: snap_create 2024-09-21T09:32:12.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:12 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:32:12.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:12 smithi022 ceph-mon[39525]: osdmap e121: 8 total, 8 up, 8 in 2024-09-21T09:32:12.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:12 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:32:12.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:12 smithi022 ceph-mon[27763]: osdmap e121: 8 total, 8 up, 8 in 2024-09-21T09:32:12.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:12 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:32:12.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:12 smithi064 ceph-mon[31124]: osdmap e121: 8 total, 8 up, 8 in 2024-09-21T09:32:13.423 INFO:tasks.rados.rados.0.smithi022.stdout:324: done (0 left) 2024-09-21T09:32:13.423 INFO:tasks.rados.rados.0.smithi022.stdout:325: read oid 15 snap -1 2024-09-21T09:32:13.423 INFO:tasks.rados.rados.0.smithi022.stdout:325: expect (ObjNum 118 snap 24 seq_num 118) 2024-09-21T09:32:13.423 INFO:tasks.rados.rados.0.smithi022.stdout:326: read oid 1 snap -1 2024-09-21T09:32:13.423 INFO:tasks.rados.rados.0.smithi022.stdout:326: expect (ObjNum 89 snap 18 seq_num 89) 2024-09-21T09:32:13.423 INFO:tasks.rados.rados.0.smithi022.stdout:327: read oid 10 snap -1 2024-09-21T09:32:13.424 INFO:tasks.rados.rados.0.smithi022.stdout:327: expect (ObjNum 38 snap 0 seq_num 38) 2024-09-21T09:32:13.424 INFO:tasks.rados.rados.0.smithi022.stdout:328: delete oid 49 current snap is 27 2024-09-21T09:32:13.425 INFO:tasks.rados.rados.0.smithi022.stdout:328: done (3 left) 2024-09-21T09:32:13.426 INFO:tasks.rados.rados.0.smithi022.stdout:329: delete oid 27 current snap is 27 2024-09-21T09:32:13.430 INFO:tasks.rados.rados.0.smithi022.stdout:326: expect (ObjNum 89 snap 18 seq_num 89) 2024-09-21T09:32:13.471 INFO:tasks.rados.rados.0.smithi022.stdout:327: expect (ObjNum 38 snap 0 seq_num 38) 2024-09-21T09:32:13.488 INFO:tasks.rados.rados.0.smithi022.stdout:325: expect (ObjNum 118 snap 24 seq_num 118) 2024-09-21T09:32:13.607 INFO:tasks.rados.rados.0.smithi022.stdout:325: done (3 left) 2024-09-21T09:32:13.607 INFO:tasks.rados.rados.0.smithi022.stdout:326: done (2 left) 2024-09-21T09:32:13.607 INFO:tasks.rados.rados.0.smithi022.stdout:327: done (1 left) 2024-09-21T09:32:13.607 INFO:tasks.rados.rados.0.smithi022.stdout:329: done (0 left) 2024-09-21T09:32:13.607 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 25 current snap is 27 2024-09-21T09:32:13.607 INFO:tasks.rados.rados.0.smithi022.stdout:330: seq_num 128 ranges {3948544=548864,4497408=794624,5292032=581632,5873664=229376} 2024-09-21T09:32:13.631 INFO:tasks.rados.rados.0.smithi022.stdout:330: writing smithi02284960-25 from 3948544 to 4497408 tid 1 2024-09-21T09:32:13.635 INFO:tasks.rados.rados.0.smithi022.stdout:330: writing smithi02284960-25 from 4497408 to 5292032 tid 2 2024-09-21T09:32:13.639 INFO:tasks.rados.rados.0.smithi022.stdout:330: writing smithi02284960-25 from 5292032 to 5873664 tid 3 2024-09-21T09:32:13.640 INFO:tasks.rados.rados.0.smithi022.stdout:330: writing smithi02284960-25 from 5873664 to 6103040 tid 4 2024-09-21T09:32:13.640 INFO:tasks.rados.rados.0.smithi022.stdout:331: delete oid 42 current snap is 27 2024-09-21T09:32:13.641 INFO:tasks.rados.rados.0.smithi022.stdout:330: finishing write tid 1 to smithi02284960-25 2024-09-21T09:32:13.641 INFO:tasks.rados.rados.0.smithi022.stdout:330: finishing write tid 2 to smithi02284960-25 2024-09-21T09:32:13.643 INFO:tasks.rados.rados.0.smithi022.stdout:330: finishing write tid 3 to smithi02284960-25 2024-09-21T09:32:13.643 INFO:tasks.rados.rados.0.smithi022.stdout:331: done (1 left) 2024-09-21T09:32:13.643 INFO:tasks.rados.rados.0.smithi022.stdout:332: snap_remove snap 23 2024-09-21T09:32:13.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:13 smithi022 ceph-mon[39525]: pgmap v177: 121 pgs: 121 active+clean; 82 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 838 KiB/s rd, 4.7 MiB/s wr, 32 op/s 2024-09-21T09:32:13.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:13 smithi022 ceph-mon[39525]: osdmap e122: 8 total, 8 up, 8 in 2024-09-21T09:32:13.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:13 smithi022 ceph-mon[27763]: pgmap v177: 121 pgs: 121 active+clean; 82 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 838 KiB/s rd, 4.7 MiB/s wr, 32 op/s 2024-09-21T09:32:13.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:13 smithi022 ceph-mon[27763]: osdmap e122: 8 total, 8 up, 8 in 2024-09-21T09:32:13.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:13 smithi064 ceph-mon[31124]: pgmap v177: 121 pgs: 121 active+clean; 82 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 838 KiB/s rd, 4.7 MiB/s wr, 32 op/s 2024-09-21T09:32:13.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:13 smithi064 ceph-mon[31124]: osdmap e122: 8 total, 8 up, 8 in 2024-09-21T09:32:14.462 INFO:tasks.rados.rados.0.smithi022.stdout:330: finishing write tid 4 to smithi02284960-25 2024-09-21T09:32:14.462 INFO:tasks.rados.rados.0.smithi022.stdout:330: finishing write tid 5 to smithi02284960-25 2024-09-21T09:32:14.462 INFO:tasks.rados.rados.0.smithi022.stdout:330: finishing write tid 6 to smithi02284960-25 2024-09-21T09:32:14.463 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 25 v 178 (ObjNum 128 snap 27 seq_num 128) dirty exists 2024-09-21T09:32:14.463 INFO:tasks.rados.rados.0.smithi022.stdout:330: left oid 25 (ObjNum 128 snap 27 seq_num 128) 2024-09-21T09:32:14.463 INFO:tasks.rados.rados.0.smithi022.stdout:330: done (1 left) 2024-09-21T09:32:14.463 INFO:tasks.rados.rados.0.smithi022.stdout:332: done (0 left) 2024-09-21T09:32:14.463 INFO:tasks.rados.rados.0.smithi022.stdout:333: delete oid 38 current snap is 27 2024-09-21T09:32:14.469 INFO:tasks.rados.rados.0.smithi022.stdout:333: done (0 left) 2024-09-21T09:32:14.470 INFO:tasks.rados.rados.0.smithi022.stdout:334: copy_from oid 36 from oid 46 current snap is 27 2024-09-21T09:32:14.470 INFO:tasks.rados.rados.0.smithi022.stdout:335: rollback oid 15 current snap is 27 2024-09-21T09:32:14.470 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 15 to 22 2024-09-21T09:32:14.470 INFO:tasks.rados.rados.0.smithi022.stdout:336: setattr oid 14 current snap is 27 2024-09-21T09:32:14.472 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 50 current snap is 27 2024-09-21T09:32:14.472 INFO:tasks.rados.rados.0.smithi022.stdout:337: seq_num 131 ranges {2596864=598016,3194880=499712,3694592=753664,4448256=352256} 2024-09-21T09:32:14.490 INFO:tasks.rados.rados.0.smithi022.stdout:337: writing smithi02284960-50 from 2596864 to 3194880 tid 1 2024-09-21T09:32:14.493 INFO:tasks.rados.rados.0.smithi022.stdout:337: writing smithi02284960-50 from 3194880 to 3694592 tid 2 2024-09-21T09:32:14.497 INFO:tasks.rados.rados.0.smithi022.stdout:337: writing smithi02284960-50 from 3694592 to 4448256 tid 3 2024-09-21T09:32:14.500 INFO:tasks.rados.rados.0.smithi022.stdout:337: writing smithi02284960-50 from 4448256 to 4800512 tid 4 2024-09-21T09:32:14.500 INFO:tasks.rados.rados.0.smithi022.stdout:338: read oid 12 snap 25 2024-09-21T09:32:14.500 INFO:tasks.rados.rados.0.smithi022.stdout:338: expect (ObjNum 104 snap 19 seq_num 104) 2024-09-21T09:32:14.500 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 20 current snap is 27 2024-09-21T09:32:14.500 INFO:tasks.rados.rados.0.smithi022.stdout:335: finishing rollback tid 0 to smithi02284960-15 2024-09-21T09:32:14.500 INFO:tasks.rados.rados.0.smithi022.stdout:339: seq_num 132 ranges {2342912=614400,2957312=786432,3743744=565248,4308992=253952} 2024-09-21T09:32:14.516 INFO:tasks.rados.rados.0.smithi022.stdout:339: writing smithi02284960-20 from 2342912 to 2957312 tid 1 2024-09-21T09:32:14.520 INFO:tasks.rados.rados.0.smithi022.stdout:339: writing smithi02284960-20 from 2957312 to 3743744 tid 2 2024-09-21T09:32:14.524 INFO:tasks.rados.rados.0.smithi022.stdout:339: writing smithi02284960-20 from 3743744 to 4308992 tid 3 2024-09-21T09:32:14.526 INFO:tasks.rados.rados.0.smithi022.stdout:339: writing smithi02284960-20 from 4308992 to 4562944 tid 4 2024-09-21T09:32:14.526 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 38 current snap is 27 2024-09-21T09:32:14.526 INFO:tasks.rados.rados.0.smithi022.stdout:335: finishing rollback tid 1 to smithi02284960-15 2024-09-21T09:32:14.526 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 14 v 64 (ObjNum 115 snap 23 seq_num 115) dirty exists 2024-09-21T09:32:14.526 INFO:tasks.rados.rados.0.smithi022.stdout:334: finishing copy_from racing read to smithi02284960-36 2024-09-21T09:32:14.526 INFO:tasks.rados.rados.0.smithi022.stdout:335: finishing rollback tid 2 to smithi02284960-15 2024-09-21T09:32:14.526 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 15 v 169 (ObjNum 4 snap 0 seq_num 4) dirty exists 2024-09-21T09:32:14.526 INFO:tasks.rados.rados.0.smithi022.stdout:334: finishing copy_from to smithi02284960-36 2024-09-21T09:32:14.526 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 36 v 8 (ObjNum 45 snap 0 seq_num 45) dirty exists 2024-09-21T09:32:14.527 INFO:tasks.rados.rados.0.smithi022.stdout:337: finishing write tid 1 to smithi02284960-50 2024-09-21T09:32:14.527 INFO:tasks.rados.rados.0.smithi022.stdout:337: finishing write tid 2 to smithi02284960-50 2024-09-21T09:32:14.527 INFO:tasks.rados.rados.0.smithi022.stdout:340: seq_num 133 ranges {0=630784,630784=663552,1294336=770048,2064384=172032} 2024-09-21T09:32:14.529 INFO:tasks.rados.rados.0.smithi022.stdout:340: writing smithi02284960-38 from 0 to 630784 tid 1 2024-09-21T09:32:14.533 INFO:tasks.rados.rados.0.smithi022.stdout:340: writing smithi02284960-38 from 630784 to 1294336 tid 2 2024-09-21T09:32:14.538 INFO:tasks.rados.rados.0.smithi022.stdout:340: writing smithi02284960-38 from 1294336 to 2064384 tid 3 2024-09-21T09:32:14.540 INFO:tasks.rados.rados.0.smithi022.stdout:340: writing smithi02284960-38 from 2064384 to 2236416 tid 4 2024-09-21T09:32:14.540 INFO:tasks.rados.rados.0.smithi022.stdout:334: done (6 left) 2024-09-21T09:32:14.540 INFO:tasks.rados.rados.0.smithi022.stdout:335: done (5 left) 2024-09-21T09:32:14.540 INFO:tasks.rados.rados.0.smithi022.stdout:336: done (4 left) 2024-09-21T09:32:14.540 INFO:tasks.rados.rados.0.smithi022.stdout:341: rollback oid 9 current snap is 27 2024-09-21T09:32:14.540 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 9 to 26 2024-09-21T09:32:14.540 INFO:tasks.rados.rados.0.smithi022.stdout:337: finishing write tid 3 to smithi02284960-50 2024-09-21T09:32:14.540 INFO:tasks.rados.rados.0.smithi022.stdout:337: finishing write tid 4 to smithi02284960-50 2024-09-21T09:32:14.540 INFO:tasks.rados.rados.0.smithi022.stdout:337: finishing write tid 5 to smithi02284960-50 2024-09-21T09:32:14.540 INFO:tasks.rados.rados.0.smithi022.stdout:337: finishing write tid 6 to smithi02284960-50 2024-09-21T09:32:14.540 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 50 v 90 (ObjNum 131 snap 27 seq_num 131) dirty exists 2024-09-21T09:32:14.540 INFO:tasks.rados.rados.0.smithi022.stdout:337: left oid 50 (ObjNum 131 snap 27 seq_num 131) 2024-09-21T09:32:14.540 INFO:tasks.rados.rados.0.smithi022.stdout:338: expect (ObjNum 104 snap 19 seq_num 104) 2024-09-21T09:32:14.647 INFO:tasks.rados.rados.0.smithi022.stdout:339: finishing write tid 1 to smithi02284960-20 2024-09-21T09:32:14.647 INFO:tasks.rados.rados.0.smithi022.stdout:339: finishing write tid 2 to smithi02284960-20 2024-09-21T09:32:14.647 INFO:tasks.rados.rados.0.smithi022.stdout:339: finishing write tid 3 to smithi02284960-20 2024-09-21T09:32:14.647 INFO:tasks.rados.rados.0.smithi022.stdout:339: finishing write tid 4 to smithi02284960-20 2024-09-21T09:32:14.647 INFO:tasks.rados.rados.0.smithi022.stdout:339: finishing write tid 5 to smithi02284960-20 2024-09-21T09:32:14.647 INFO:tasks.rados.rados.0.smithi022.stdout:339: finishing write tid 6 to smithi02284960-20 2024-09-21T09:32:14.648 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 20 v 179 (ObjNum 132 snap 27 seq_num 132) dirty exists 2024-09-21T09:32:14.648 INFO:tasks.rados.rados.0.smithi022.stdout:339: left oid 20 (ObjNum 132 snap 27 seq_num 132) 2024-09-21T09:32:14.648 INFO:tasks.rados.rados.0.smithi022.stdout:340: finishing write tid 1 to smithi02284960-38 2024-09-21T09:32:14.648 INFO:tasks.rados.rados.0.smithi022.stdout:337: done (4 left) 2024-09-21T09:32:14.648 INFO:tasks.rados.rados.0.smithi022.stdout:338: done (3 left) 2024-09-21T09:32:14.648 INFO:tasks.rados.rados.0.smithi022.stdout:339: done (2 left) 2024-09-21T09:32:14.648 INFO:tasks.rados.rados.0.smithi022.stdout:342: snap_remove snap 24 2024-09-21T09:32:15.420 INFO:tasks.rados.rados.0.smithi022.stdout:342: done (2 left) 2024-09-21T09:32:15.420 INFO:tasks.rados.rados.0.smithi022.stdout:343: read oid 18 snap 25 2024-09-21T09:32:15.420 INFO:tasks.rados.rados.0.smithi022.stdout:343: expect (ObjNum 17 snap 0 seq_num 17) 2024-09-21T09:32:15.420 INFO:tasks.rados.rados.0.smithi022.stdout:344: snap_create 2024-09-21T09:32:15.420 INFO:tasks.rados.rados.0.smithi022.stdout:340: finishing write tid 2 to smithi02284960-38 2024-09-21T09:32:15.420 INFO:tasks.rados.rados.0.smithi022.stdout:340: finishing write tid 3 to smithi02284960-38 2024-09-21T09:32:15.421 INFO:tasks.rados.rados.0.smithi022.stdout:340: finishing write tid 4 to smithi02284960-38 2024-09-21T09:32:15.421 INFO:tasks.rados.rados.0.smithi022.stdout:340: finishing write tid 5 to smithi02284960-38 2024-09-21T09:32:15.421 INFO:tasks.rados.rados.0.smithi022.stdout:341: finishing rollback tid 0 to smithi02284960-9 2024-09-21T09:32:15.421 INFO:tasks.rados.rados.0.smithi022.stdout:341: finishing rollback tid 1 to smithi02284960-9 2024-09-21T09:32:15.421 INFO:tasks.rados.rados.0.smithi022.stdout:341: finishing rollback tid 2 to smithi02284960-9 2024-09-21T09:32:15.421 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 9 v 88 (ObjNum 90 snap 18 seq_num 90) dirty exists 2024-09-21T09:32:15.421 INFO:tasks.rados.rados.0.smithi022.stdout:340: finishing write tid 6 to smithi02284960-38 2024-09-21T09:32:15.421 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 38 v 65 (ObjNum 133 snap 27 seq_num 133) dirty exists 2024-09-21T09:32:15.421 INFO:tasks.rados.rados.0.smithi022.stdout:340: left oid 38 (ObjNum 133 snap 27 seq_num 133) 2024-09-21T09:32:15.425 INFO:tasks.rados.rados.0.smithi022.stdout:343: expect (ObjNum 17 snap 0 seq_num 17) 2024-09-21T09:32:15.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:15 smithi022 ceph-mon[39525]: osdmap e123: 8 total, 8 up, 8 in 2024-09-21T09:32:15.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:15 smithi022 ceph-mon[39525]: pgmap v180: 121 pgs: 121 active+clean; 82 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 838 KiB/s rd, 4.7 MiB/s wr, 32 op/s 2024-09-21T09:32:15.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:15 smithi022 ceph-mon[39525]: osdmap e124: 8 total, 8 up, 8 in 2024-09-21T09:32:15.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:15 smithi022 ceph-mon[27763]: osdmap e123: 8 total, 8 up, 8 in 2024-09-21T09:32:15.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:15 smithi022 ceph-mon[27763]: pgmap v180: 121 pgs: 121 active+clean; 82 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 838 KiB/s rd, 4.7 MiB/s wr, 32 op/s 2024-09-21T09:32:15.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:15 smithi022 ceph-mon[27763]: osdmap e124: 8 total, 8 up, 8 in 2024-09-21T09:32:15.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:15 smithi064 ceph-mon[31124]: osdmap e123: 8 total, 8 up, 8 in 2024-09-21T09:32:15.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:15 smithi064 ceph-mon[31124]: pgmap v180: 121 pgs: 121 active+clean; 82 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 838 KiB/s rd, 4.7 MiB/s wr, 32 op/s 2024-09-21T09:32:15.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:15 smithi064 ceph-mon[31124]: osdmap e124: 8 total, 8 up, 8 in 2024-09-21T09:32:16.421 INFO:tasks.rados.rados.0.smithi022.stdout:340: done (3 left) 2024-09-21T09:32:16.422 INFO:tasks.rados.rados.0.smithi022.stdout:341: done (2 left) 2024-09-21T09:32:16.422 INFO:tasks.rados.rados.0.smithi022.stdout:343: done (1 left) 2024-09-21T09:32:16.422 INFO:tasks.rados.rados.0.smithi022.stdout:344: done (0 left) 2024-09-21T09:32:16.422 INFO:tasks.rados.rados.0.smithi022.stdout:345: read oid 23 snap -1 2024-09-21T09:32:16.422 INFO:tasks.rados.rados.0.smithi022.stdout:345: expect (ObjNum 53 snap 1 seq_num 53) 2024-09-21T09:32:16.422 INFO:tasks.rados.rados.0.smithi022.stdout:346: delete oid 34 current snap is 28 2024-09-21T09:32:16.426 INFO:tasks.rados.rados.0.smithi022.stdout:346: done (1 left) 2024-09-21T09:32:16.426 INFO:tasks.rados.rados.0.smithi022.stdout:347: delete oid 10 current snap is 28 2024-09-21T09:32:16.428 INFO:tasks.rados.rados.0.smithi022.stdout:347: done (1 left) 2024-09-21T09:32:16.428 INFO:tasks.rados.rados.0.smithi022.stdout:348: read oid 37 snap -1 2024-09-21T09:32:16.428 INFO:tasks.rados.rados.0.smithi022.stdout:348: expect (ObjNum 59 snap 3 seq_num 59) 2024-09-21T09:32:16.428 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 48 current snap is 28 2024-09-21T09:32:16.428 INFO:tasks.rados.rados.0.smithi022.stdout:349: seq_num 136 ranges {0=688128,688128=720896,1409024=540672,1949696=344064} 2024-09-21T09:32:16.432 INFO:tasks.rados.rados.0.smithi022.stdout:349: writing smithi02284960-48 from 0 to 688128 tid 1 2024-09-21T09:32:16.436 INFO:tasks.rados.rados.0.smithi022.stdout:349: writing smithi02284960-48 from 688128 to 1409024 tid 2 2024-09-21T09:32:16.440 INFO:tasks.rados.rados.0.smithi022.stdout:349: writing smithi02284960-48 from 1409024 to 1949696 tid 3 2024-09-21T09:32:16.442 INFO:tasks.rados.rados.0.smithi022.stdout:349: writing smithi02284960-48 from 1949696 to 2293760 tid 4 2024-09-21T09:32:16.442 INFO:tasks.rados.rados.0.smithi022.stdout:350: snap_remove snap 19 2024-09-21T09:32:17.426 INFO:tasks.rados.rados.0.smithi022.stdout:345: expect (ObjNum 53 snap 1 seq_num 53) 2024-09-21T09:32:17.488 INFO:tasks.rados.rados.0.smithi022.stdout:348: expect (ObjNum 59 snap 3 seq_num 59) 2024-09-21T09:32:17.541 INFO:tasks.rados.rados.0.smithi022.stdout:349: finishing write tid 1 to smithi02284960-48 2024-09-21T09:32:17.541 INFO:tasks.rados.rados.0.smithi022.stdout:349: finishing write tid 2 to smithi02284960-48 2024-09-21T09:32:17.541 INFO:tasks.rados.rados.0.smithi022.stdout:349: finishing write tid 3 to smithi02284960-48 2024-09-21T09:32:17.541 INFO:tasks.rados.rados.0.smithi022.stdout:349: finishing write tid 4 to smithi02284960-48 2024-09-21T09:32:17.541 INFO:tasks.rados.rados.0.smithi022.stdout:349: finishing write tid 5 to smithi02284960-48 2024-09-21T09:32:17.541 INFO:tasks.rados.rados.0.smithi022.stdout:345: done (3 left) 2024-09-21T09:32:17.541 INFO:tasks.rados.rados.0.smithi022.stdout:348: done (2 left) 2024-09-21T09:32:17.541 INFO:tasks.rados.rados.0.smithi022.stdout:350: done (1 left) 2024-09-21T09:32:17.542 INFO:tasks.rados.rados.0.smithi022.stdout:351: read oid 31 snap -1 2024-09-21T09:32:17.542 INFO:tasks.rados.rados.0.smithi022.stdout:351: expect (ObjNum 71 snap 5 seq_num 71) 2024-09-21T09:32:17.542 INFO:tasks.rados.rados.0.smithi022.stdout:352: snap_create 2024-09-21T09:32:17.542 INFO:tasks.rados.rados.0.smithi022.stdout:349: finishing write tid 6 to smithi02284960-48 2024-09-21T09:32:17.542 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 48 v 189 (ObjNum 136 snap 28 seq_num 136) dirty exists 2024-09-21T09:32:17.542 INFO:tasks.rados.rados.0.smithi022.stdout:349: left oid 48 (ObjNum 136 snap 28 seq_num 136) 2024-09-21T09:32:17.558 INFO:tasks.rados.rados.0.smithi022.stdout:351: expect (ObjNum 71 snap 5 seq_num 71) 2024-09-21T09:32:17.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:17 smithi064 ceph-mon[31124]: osdmap e125: 8 total, 8 up, 8 in 2024-09-21T09:32:17.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:17 smithi064 ceph-mon[31124]: pgmap v183: 121 pgs: 121 active+clean; 82 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:32:17.807 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:17 smithi022 ceph-mon[27763]: osdmap e125: 8 total, 8 up, 8 in 2024-09-21T09:32:17.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:17 smithi022 ceph-mon[27763]: pgmap v183: 121 pgs: 121 active+clean; 82 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:32:17.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:17 smithi022 ceph-mon[39525]: osdmap e125: 8 total, 8 up, 8 in 2024-09-21T09:32:17.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:17 smithi022 ceph-mon[39525]: pgmap v183: 121 pgs: 121 active+clean; 82 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:32:18.427 INFO:tasks.rados.rados.0.smithi022.stdout:349: done (2 left) 2024-09-21T09:32:18.427 INFO:tasks.rados.rados.0.smithi022.stdout:351: done (1 left) 2024-09-21T09:32:18.427 INFO:tasks.rados.rados.0.smithi022.stdout:352: done (0 left) 2024-09-21T09:32:18.427 INFO:tasks.rados.rados.0.smithi022.stdout:353: snap_create 2024-09-21T09:32:18.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:18 smithi064 ceph-mon[31124]: osdmap e126: 8 total, 8 up, 8 in 2024-09-21T09:32:18.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:18 smithi022 ceph-mon[27763]: osdmap e126: 8 total, 8 up, 8 in 2024-09-21T09:32:18.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:18 smithi022 ceph-mon[39525]: osdmap e126: 8 total, 8 up, 8 in 2024-09-21T09:32:19.437 INFO:tasks.rados.rados.0.smithi022.stdout:353: done (0 left) 2024-09-21T09:32:19.437 INFO:tasks.rados.rados.0.smithi022.stdout:354: rollback oid 14 current snap is 30 2024-09-21T09:32:19.437 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 14 to 28 2024-09-21T09:32:19.437 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 9 current snap is 30 2024-09-21T09:32:19.437 INFO:tasks.rados.rados.0.smithi022.stdout:355: seq_num 137 ranges {2883584=704512,3588096=655360,4243456=753664,4997120=196608} 2024-09-21T09:32:19.457 INFO:tasks.rados.rados.0.smithi022.stdout:355: writing smithi02284960-9 from 2883584 to 3588096 tid 1 2024-09-21T09:32:19.461 INFO:tasks.rados.rados.0.smithi022.stdout:355: writing smithi02284960-9 from 3588096 to 4243456 tid 2 2024-09-21T09:32:19.465 INFO:tasks.rados.rados.0.smithi022.stdout:355: writing smithi02284960-9 from 4243456 to 4997120 tid 3 2024-09-21T09:32:19.467 INFO:tasks.rados.rados.0.smithi022.stdout:355: writing smithi02284960-9 from 4997120 to 5193728 tid 4 2024-09-21T09:32:19.467 INFO:tasks.rados.rados.0.smithi022.stdout:356: read oid 29 snap -1 2024-09-21T09:32:19.467 INFO:tasks.rados.rados.0.smithi022.stdout:356: expect deleted 2024-09-21T09:32:19.467 INFO:tasks.rados.rados.0.smithi022.stdout:357: rollback oid 35 current snap is 30 2024-09-21T09:32:19.467 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 35 to 25 2024-09-21T09:32:19.467 INFO:tasks.rados.rados.0.smithi022.stdout:354: finishing rollback tid 0 to smithi02284960-14 2024-09-21T09:32:19.467 INFO:tasks.rados.rados.0.smithi022.stdout:354: finishing rollback tid 1 to smithi02284960-14 2024-09-21T09:32:19.467 INFO:tasks.rados.rados.0.smithi022.stdout:354: finishing rollback tid 2 to smithi02284960-14 2024-09-21T09:32:19.467 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 14 v 74 (ObjNum 115 snap 23 seq_num 115) dirty exists 2024-09-21T09:32:19.467 INFO:tasks.rados.rados.0.smithi022.stdout:355: finishing write tid 1 to smithi02284960-9 2024-09-21T09:32:19.467 INFO:tasks.rados.rados.0.smithi022.stdout:355: finishing write tid 2 to smithi02284960-9 2024-09-21T09:32:19.467 INFO:tasks.rados.rados.0.smithi022.stdout:354: done (3 left) 2024-09-21T09:32:19.467 INFO:tasks.rados.rados.0.smithi022.stdout:358: snap_create 2024-09-21T09:32:19.470 INFO:tasks.rados.rados.0.smithi022.stdout:355: finishing write tid 3 to smithi02284960-9 2024-09-21T09:32:19.471 INFO:tasks.rados.rados.0.smithi022.stdout:355: finishing write tid 4 to smithi02284960-9 2024-09-21T09:32:19.471 INFO:tasks.rados.rados.0.smithi022.stdout:357: finishing rollback tid 0 to smithi02284960-35 2024-09-21T09:32:19.471 INFO:tasks.rados.rados.0.smithi022.stdout:357: finishing rollback tid 1 to smithi02284960-35 2024-09-21T09:32:19.471 INFO:tasks.rados.rados.0.smithi022.stdout:357: finishing rollback tid 2 to smithi02284960-35 2024-09-21T09:32:19.471 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 35 v 62 (ObjNum 79 snap 11 seq_num 79) dirty exists 2024-09-21T09:32:19.472 INFO:tasks.rados.rados.0.smithi022.stdout:355: finishing write tid 5 to smithi02284960-9 2024-09-21T09:32:19.472 INFO:tasks.rados.rados.0.smithi022.stdout:355: finishing write tid 6 to smithi02284960-9 2024-09-21T09:32:19.472 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 9 v 98 (ObjNum 137 snap 30 seq_num 137) dirty exists 2024-09-21T09:32:19.472 INFO:tasks.rados.rados.0.smithi022.stdout:355: left oid 9 (ObjNum 137 snap 30 seq_num 137) 2024-09-21T09:32:19.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:19 smithi064 ceph-mon[31124]: osdmap e127: 8 total, 8 up, 8 in 2024-09-21T09:32:19.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:19 smithi064 ceph-mon[31124]: pgmap v186: 121 pgs: 121 active+clean; 90 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 2.8 MiB/s wr, 24 op/s 2024-09-21T09:32:19.807 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:19 smithi022 ceph-mon[39525]: osdmap e127: 8 total, 8 up, 8 in 2024-09-21T09:32:19.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:19 smithi022 ceph-mon[39525]: pgmap v186: 121 pgs: 121 active+clean; 90 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 2.8 MiB/s wr, 24 op/s 2024-09-21T09:32:19.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:19 smithi022 ceph-mon[27763]: osdmap e127: 8 total, 8 up, 8 in 2024-09-21T09:32:19.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:19 smithi022 ceph-mon[27763]: pgmap v186: 121 pgs: 121 active+clean; 90 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 2.8 MiB/s wr, 24 op/s 2024-09-21T09:32:20.421 INFO:tasks.rados.rados.0.smithi022.stdout:355: done (3 left) 2024-09-21T09:32:20.421 INFO:tasks.rados.rados.0.smithi022.stdout:356: done (2 left) 2024-09-21T09:32:20.421 INFO:tasks.rados.rados.0.smithi022.stdout:357: done (1 left) 2024-09-21T09:32:20.421 INFO:tasks.rados.rados.0.smithi022.stdout:358: done (0 left) 2024-09-21T09:32:20.421 INFO:tasks.rados.rados.0.smithi022.stdout:359: rollback oid 50 current snap is 31 2024-09-21T09:32:20.421 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 50 to 29 2024-09-21T09:32:20.421 INFO:tasks.rados.rados.0.smithi022.stdout:360: rmattr oid 38 current snap is 31 2024-09-21T09:32:20.422 INFO:tasks.rados.rados.0.smithi022.stdout:360: done (1 left) 2024-09-21T09:32:20.422 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 21 current snap is 31 2024-09-21T09:32:20.422 INFO:tasks.rados.rados.0.smithi022.stdout:361: seq_num 138 ranges {0=720896,720896=540672,1261568=557056,1818624=507904} 2024-09-21T09:32:20.425 INFO:tasks.rados.rados.0.smithi022.stdout:361: writing smithi02284960-21 from 0 to 720896 tid 1 2024-09-21T09:32:20.429 INFO:tasks.rados.rados.0.smithi022.stdout:361: writing smithi02284960-21 from 720896 to 1261568 tid 2 2024-09-21T09:32:20.432 INFO:tasks.rados.rados.0.smithi022.stdout:361: writing smithi02284960-21 from 1261568 to 1818624 tid 3 2024-09-21T09:32:20.436 INFO:tasks.rados.rados.0.smithi022.stdout:361: writing smithi02284960-21 from 1818624 to 2326528 tid 4 2024-09-21T09:32:20.436 INFO:tasks.rados.rados.0.smithi022.stdout:362: copy_from oid 37 from oid 14 current snap is 31 2024-09-21T09:32:20.436 INFO:tasks.rados.rados.0.smithi022.stdout:359: finishing rollback tid 0 to smithi02284960-50 2024-09-21T09:32:20.436 INFO:tasks.rados.rados.0.smithi022.stdout:359: finishing rollback tid 1 to smithi02284960-50 2024-09-21T09:32:20.436 INFO:tasks.rados.rados.0.smithi022.stdout:359: finishing rollback tid 2 to smithi02284960-50 2024-09-21T09:32:20.436 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 50 v 104 (ObjNum 131 snap 27 seq_num 131) dirty exists 2024-09-21T09:32:20.436 INFO:tasks.rados.rados.0.smithi022.stdout:361: finishing write tid 1 to smithi02284960-21 2024-09-21T09:32:20.436 INFO:tasks.rados.rados.0.smithi022.stdout:361: finishing write tid 2 to smithi02284960-21 2024-09-21T09:32:20.436 INFO:tasks.rados.rados.0.smithi022.stdout:359: done (2 left) 2024-09-21T09:32:20.436 INFO:tasks.rados.rados.0.smithi022.stdout:363: rmattr oid 44 current snap is 31 2024-09-21T09:32:20.437 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 12 current snap is 31 2024-09-21T09:32:20.437 INFO:tasks.rados.rados.0.smithi022.stdout:364: seq_num 139 ranges {3121152=737280,3858432=425984,4284416=761856,5046272=417792} 2024-09-21T09:32:20.457 INFO:tasks.rados.rados.0.smithi022.stdout:364: writing smithi02284960-12 from 3121152 to 3858432 tid 1 2024-09-21T09:32:20.460 INFO:tasks.rados.rados.0.smithi022.stdout:364: writing smithi02284960-12 from 3858432 to 4284416 tid 2 2024-09-21T09:32:20.464 INFO:tasks.rados.rados.0.smithi022.stdout:364: writing smithi02284960-12 from 4284416 to 5046272 tid 3 2024-09-21T09:32:20.467 INFO:tasks.rados.rados.0.smithi022.stdout:364: writing smithi02284960-12 from 5046272 to 5464064 tid 4 2024-09-21T09:32:20.467 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 38 current snap is 31 2024-09-21T09:32:20.467 INFO:tasks.rados.rados.0.smithi022.stdout:361: finishing write tid 3 to smithi02284960-21 2024-09-21T09:32:20.467 INFO:tasks.rados.rados.0.smithi022.stdout:365: seq_num 140 ranges {2236416=753664,2990080=704512,3694592=524288,4218880=376832} 2024-09-21T09:32:20.483 INFO:tasks.rados.rados.0.smithi022.stdout:365: writing smithi02284960-38 from 2236416 to 2990080 tid 1 2024-09-21T09:32:20.487 INFO:tasks.rados.rados.0.smithi022.stdout:365: writing smithi02284960-38 from 2990080 to 3694592 tid 2 2024-09-21T09:32:20.490 INFO:tasks.rados.rados.0.smithi022.stdout:365: writing smithi02284960-38 from 3694592 to 4218880 tid 3 2024-09-21T09:32:20.493 INFO:tasks.rados.rados.0.smithi022.stdout:365: writing smithi02284960-38 from 4218880 to 4595712 tid 4 2024-09-21T09:32:20.493 INFO:tasks.rados.rados.0.smithi022.stdout:366: copy_from oid 45 from oid 8 current snap is 31 2024-09-21T09:32:20.493 INFO:tasks.rados.rados.0.smithi022.stdout:361: finishing write tid 4 to smithi02284960-21 2024-09-21T09:32:20.493 INFO:tasks.rados.rados.0.smithi022.stdout:361: finishing write tid 5 to smithi02284960-21 2024-09-21T09:32:20.493 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 44 v 82 (ObjNum 289 snap 0 seq_num 1) dirty exists 2024-09-21T09:32:20.493 INFO:tasks.rados.rados.0.smithi022.stdout:361: finishing write tid 6 to smithi02284960-21 2024-09-21T09:32:20.494 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 21 v 130 (ObjNum 138 snap 31 seq_num 138) dirty exists 2024-09-21T09:32:20.494 INFO:tasks.rados.rados.0.smithi022.stdout:361: left oid 21 (ObjNum 138 snap 31 seq_num 138) 2024-09-21T09:32:20.494 INFO:tasks.rados.rados.0.smithi022.stdout:362: finishing copy_from racing read to smithi02284960-37 2024-09-21T09:32:20.494 INFO:tasks.rados.rados.0.smithi022.stdout:361: done (5 left) 2024-09-21T09:32:20.494 INFO:tasks.rados.rados.0.smithi022.stdout:363: done (4 left) 2024-09-21T09:32:20.494 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 27 current snap is 31 2024-09-21T09:32:20.494 INFO:tasks.rados.rados.0.smithi022.stdout:362: finishing copy_from to smithi02284960-37 2024-09-21T09:32:20.494 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 37 v 74 (ObjNum 115 snap 23 seq_num 115) dirty exists 2024-09-21T09:32:20.494 INFO:tasks.rados.rados.0.smithi022.stdout:364: finishing write tid 1 to smithi02284960-12 2024-09-21T09:32:20.494 INFO:tasks.rados.rados.0.smithi022.stdout:364: finishing write tid 2 to smithi02284960-12 2024-09-21T09:32:20.494 INFO:tasks.rados.rados.0.smithi022.stdout:367: seq_num 141 ranges {0=770048,770048=589824,1359872=729088,2088960=286720} 2024-09-21T09:32:20.497 INFO:tasks.rados.rados.0.smithi022.stdout:367: writing smithi02284960-27 from 0 to 770048 tid 1 2024-09-21T09:32:20.501 INFO:tasks.rados.rados.0.smithi022.stdout:367: writing smithi02284960-27 from 770048 to 1359872 tid 2 2024-09-21T09:32:20.505 INFO:tasks.rados.rados.0.smithi022.stdout:367: writing smithi02284960-27 from 1359872 to 2088960 tid 3 2024-09-21T09:32:20.507 INFO:tasks.rados.rados.0.smithi022.stdout:367: writing smithi02284960-27 from 2088960 to 2375680 tid 4 2024-09-21T09:32:20.507 INFO:tasks.rados.rados.0.smithi022.stdout:362: done (4 left) 2024-09-21T09:32:20.507 INFO:tasks.rados.rados.0.smithi022.stdout:368: snap_remove snap 26 2024-09-21T09:32:20.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:20 smithi064 ceph-mon[31124]: osdmap e128: 8 total, 8 up, 8 in 2024-09-21T09:32:20.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:20 smithi064 ceph-mon[31124]: osdmap e129: 8 total, 8 up, 8 in 2024-09-21T09:32:20.691 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:32:20 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[33213]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:32:20] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-21T09:32:20.709 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:20 smithi022 ceph-mon[39525]: osdmap e128: 8 total, 8 up, 8 in 2024-09-21T09:32:20.709 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:20 smithi022 ceph-mon[39525]: osdmap e129: 8 total, 8 up, 8 in 2024-09-21T09:32:20.709 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:20 smithi022 ceph-mon[27763]: osdmap e128: 8 total, 8 up, 8 in 2024-09-21T09:32:20.709 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:20 smithi022 ceph-mon[27763]: osdmap e129: 8 total, 8 up, 8 in 2024-09-21T09:32:21.447 INFO:tasks.rados.rados.0.smithi022.stdout:368: done (4 left) 2024-09-21T09:32:21.447 INFO:tasks.rados.rados.0.smithi022.stdout:369: copy_from oid 41 from oid 11 current snap is 31 2024-09-21T09:32:21.447 INFO:tasks.rados.rados.0.smithi022.stdout:364: finishing write tid 3 to smithi02284960-12 2024-09-21T09:32:21.447 INFO:tasks.rados.rados.0.smithi022.stdout:364: finishing write tid 4 to smithi02284960-12 2024-09-21T09:32:21.447 INFO:tasks.rados.rados.0.smithi022.stdout:364: finishing write tid 5 to smithi02284960-12 2024-09-21T09:32:21.447 INFO:tasks.rados.rados.0.smithi022.stdout:364: finishing write tid 6 to smithi02284960-12 2024-09-21T09:32:21.448 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 12 v 183 (ObjNum 139 snap 31 seq_num 139) dirty exists 2024-09-21T09:32:21.448 INFO:tasks.rados.rados.0.smithi022.stdout:364: left oid 12 (ObjNum 139 snap 31 seq_num 139) 2024-09-21T09:32:21.448 INFO:tasks.rados.rados.0.smithi022.stdout:365: finishing write tid 1 to smithi02284960-38 2024-09-21T09:32:21.448 INFO:tasks.rados.rados.0.smithi022.stdout:365: finishing write tid 2 to smithi02284960-38 2024-09-21T09:32:21.448 INFO:tasks.rados.rados.0.smithi022.stdout:365: finishing write tid 3 to smithi02284960-38 2024-09-21T09:32:21.448 INFO:tasks.rados.rados.0.smithi022.stdout:366: finishing copy_from to smithi02284960-45 2024-09-21T09:32:21.448 INFO:tasks.rados.rados.0.smithi022.stdout:366: got expected ENOENT (src dne) 2024-09-21T09:32:21.448 INFO:tasks.rados.rados.0.smithi022.stdout:366: finishing copy_from racing read to smithi02284960-45 2024-09-21T09:32:21.448 INFO:tasks.rados.rados.0.smithi022.stdout:365: finishing write tid 4 to smithi02284960-38 2024-09-21T09:32:21.448 INFO:tasks.rados.rados.0.smithi022.stdout:365: finishing write tid 5 to smithi02284960-38 2024-09-21T09:32:21.448 INFO:tasks.rados.rados.0.smithi022.stdout:365: finishing write tid 6 to smithi02284960-38 2024-09-21T09:32:21.448 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 38 v 75 (ObjNum 140 snap 31 seq_num 140) dirty exists 2024-09-21T09:32:21.448 INFO:tasks.rados.rados.0.smithi022.stdout:365: left oid 38 (ObjNum 140 snap 31 seq_num 140) 2024-09-21T09:32:21.448 INFO:tasks.rados.rados.0.smithi022.stdout:367: finishing write tid 1 to smithi02284960-27 2024-09-21T09:32:21.449 INFO:tasks.rados.rados.0.smithi022.stdout:367: finishing write tid 2 to smithi02284960-27 2024-09-21T09:32:21.449 INFO:tasks.rados.rados.0.smithi022.stdout:367: finishing write tid 3 to smithi02284960-27 2024-09-21T09:32:21.449 INFO:tasks.rados.rados.0.smithi022.stdout:367: finishing write tid 4 to smithi02284960-27 2024-09-21T09:32:21.449 INFO:tasks.rados.rados.0.smithi022.stdout:367: finishing write tid 5 to smithi02284960-27 2024-09-21T09:32:21.449 INFO:tasks.rados.rados.0.smithi022.stdout:367: finishing write tid 6 to smithi02284960-27 2024-09-21T09:32:21.449 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 27 v 152 (ObjNum 141 snap 31 seq_num 141) dirty exists 2024-09-21T09:32:21.449 INFO:tasks.rados.rados.0.smithi022.stdout:367: left oid 27 (ObjNum 141 snap 31 seq_num 141) 2024-09-21T09:32:21.449 INFO:tasks.rados.rados.0.smithi022.stdout:364: done (4 left) 2024-09-21T09:32:21.449 INFO:tasks.rados.rados.0.smithi022.stdout:365: done (3 left) 2024-09-21T09:32:21.449 INFO:tasks.rados.rados.0.smithi022.stdout:366: done (2 left) 2024-09-21T09:32:21.449 INFO:tasks.rados.rados.0.smithi022.stdout:367: done (1 left) 2024-09-21T09:32:21.449 INFO:tasks.rados.rados.0.smithi022.stdout:370: copy_from oid 19 from oid 27 current snap is 31 2024-09-21T09:32:21.449 INFO:tasks.rados.rados.0.smithi022.stdout:371: snap_create 2024-09-21T09:32:21.464 INFO:tasks.rados.rados.0.smithi022.stdout:370: finishing copy_from racing read to smithi02284960-19 2024-09-21T09:32:21.464 INFO:tasks.rados.rados.0.smithi022.stdout:370: finishing copy_from to smithi02284960-19 2024-09-21T09:32:21.464 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 19 v 152 (ObjNum 141 snap 31 seq_num 141) dirty exists 2024-09-21T09:32:21.471 INFO:tasks.rados.rados.0.smithi022.stdout:369: finishing copy_from racing read to smithi02284960-41 2024-09-21T09:32:21.474 INFO:tasks.rados.rados.0.smithi022.stdout:369: finishing copy_from to smithi02284960-41 2024-09-21T09:32:21.474 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 41 v 67 (ObjNum 122 snap 24 seq_num 122) dirty exists 2024-09-21T09:32:21.557 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:21 smithi022 ceph-mon[27763]: pgmap v189: 121 pgs: 121 active+clean; 90 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 2.8 MiB/s wr, 24 op/s 2024-09-21T09:32:21.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:21 smithi022 ceph-mon[39525]: pgmap v189: 121 pgs: 121 active+clean; 90 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 2.8 MiB/s wr, 24 op/s 2024-09-21T09:32:21.558 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:32:21 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:32:21] "GET /metrics HTTP/1.1" 200 239985 "" "Prometheus/2.43.0" 2024-09-21T09:32:21.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:21 smithi064 ceph-mon[31124]: pgmap v189: 121 pgs: 121 active+clean; 90 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 2.8 MiB/s wr, 24 op/s 2024-09-21T09:32:22.444 INFO:tasks.rados.rados.0.smithi022.stdout:369: done (2 left) 2024-09-21T09:32:22.444 INFO:tasks.rados.rados.0.smithi022.stdout:370: done (1 left) 2024-09-21T09:32:22.444 INFO:tasks.rados.rados.0.smithi022.stdout:371: done (0 left) 2024-09-21T09:32:22.444 INFO:tasks.rados.rados.0.smithi022.stdout:372: read oid 15 snap -1 2024-09-21T09:32:22.444 INFO:tasks.rados.rados.0.smithi022.stdout:372: expect (ObjNum 4 snap 0 seq_num 4) 2024-09-21T09:32:22.444 INFO:tasks.rados.rados.0.smithi022.stdout:373: copy_from oid 3 from oid 47 current snap is 32 2024-09-21T09:32:22.444 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 38 current snap is 32 2024-09-21T09:32:22.444 INFO:tasks.rados.rados.0.smithi022.stdout:374: seq_num 142 ranges {4595712=786432,5382144=475136,5857280=540672,6397952=589824} 2024-09-21T09:32:22.473 INFO:tasks.rados.rados.0.smithi022.stdout:374: writing smithi02284960-38 from 4595712 to 5382144 tid 1 2024-09-21T09:32:22.476 INFO:tasks.rados.rados.0.smithi022.stdout:374: writing smithi02284960-38 from 5382144 to 5857280 tid 2 2024-09-21T09:32:22.479 INFO:tasks.rados.rados.0.smithi022.stdout:374: writing smithi02284960-38 from 5857280 to 6397952 tid 3 2024-09-21T09:32:22.483 INFO:tasks.rados.rados.0.smithi022.stdout:374: writing smithi02284960-38 from 6397952 to 6987776 tid 4 2024-09-21T09:32:22.483 INFO:tasks.rados.rados.0.smithi022.stdout:375: read oid 39 snap -1 2024-09-21T09:32:22.483 INFO:tasks.rados.rados.0.smithi022.stdout:375: expect (ObjNum 38 snap 0 seq_num 38) 2024-09-21T09:32:22.483 INFO:tasks.rados.rados.0.smithi022.stdout:376: copy_from oid 5 from oid 25 current snap is 32 2024-09-21T09:32:22.483 INFO:tasks.rados.rados.0.smithi022.stdout:372: expect (ObjNum 4 snap 0 seq_num 4) 2024-09-21T09:32:22.487 INFO:tasks.rados.rados.0.smithi022.stdout:373: finishing copy_from racing read to smithi02284960-3 2024-09-21T09:32:22.487 INFO:tasks.rados.rados.0.smithi022.stdout:373: finishing copy_from to smithi02284960-3 2024-09-21T09:32:22.488 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 3 v 51 (ObjNum 109 snap 23 seq_num 109) dirty exists 2024-09-21T09:32:22.488 INFO:tasks.rados.rados.0.smithi022.stdout:374: finishing write tid 1 to smithi02284960-38 2024-09-21T09:32:22.488 INFO:tasks.rados.rados.0.smithi022.stdout:374: finishing write tid 2 to smithi02284960-38 2024-09-21T09:32:22.488 INFO:tasks.rados.rados.0.smithi022.stdout:374: finishing write tid 3 to smithi02284960-38 2024-09-21T09:32:22.488 INFO:tasks.rados.rados.0.smithi022.stdout:372: done (4 left) 2024-09-21T09:32:22.488 INFO:tasks.rados.rados.0.smithi022.stdout:373: done (3 left) 2024-09-21T09:32:22.488 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 32 current snap is 32 2024-09-21T09:32:22.488 INFO:tasks.rados.rados.0.smithi022.stdout:377: seq_num 143 ranges {3358720=802816,4161536=761856,4923392=745472,5668864=98304} 2024-09-21T09:32:22.510 INFO:tasks.rados.rados.0.smithi022.stdout:377: writing smithi02284960-32 from 3358720 to 4161536 tid 1 2024-09-21T09:32:22.515 INFO:tasks.rados.rados.0.smithi022.stdout:377: writing smithi02284960-32 from 4161536 to 4923392 tid 2 2024-09-21T09:32:22.520 INFO:tasks.rados.rados.0.smithi022.stdout:377: writing smithi02284960-32 from 4923392 to 5668864 tid 3 2024-09-21T09:32:22.521 INFO:tasks.rados.rados.0.smithi022.stdout:377: writing smithi02284960-32 from 5668864 to 5767168 tid 4 2024-09-21T09:32:22.521 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 10 current snap is 32 2024-09-21T09:32:22.521 INFO:tasks.rados.rados.0.smithi022.stdout:374: finishing write tid 4 to smithi02284960-38 2024-09-21T09:32:22.521 INFO:tasks.rados.rados.0.smithi022.stdout:375: expect (ObjNum 38 snap 0 seq_num 38) 2024-09-21T09:32:22.541 INFO:tasks.rados.rados.0.smithi022.stdout:374: finishing write tid 5 to smithi02284960-38 2024-09-21T09:32:22.541 INFO:tasks.rados.rados.0.smithi022.stdout:374: finishing write tid 6 to smithi02284960-38 2024-09-21T09:32:22.541 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 38 v 83 (ObjNum 142 snap 32 seq_num 142) dirty exists 2024-09-21T09:32:22.541 INFO:tasks.rados.rados.0.smithi022.stdout:374: left oid 38 (ObjNum 142 snap 32 seq_num 142) 2024-09-21T09:32:22.541 INFO:tasks.rados.rados.0.smithi022.stdout:377: finishing write tid 1 to smithi02284960-32 2024-09-21T09:32:22.541 INFO:tasks.rados.rados.0.smithi022.stdout:376: finishing copy_from racing read to smithi02284960-5 2024-09-21T09:32:22.541 INFO:tasks.rados.rados.0.smithi022.stdout:376: finishing copy_from to smithi02284960-5 2024-09-21T09:32:22.541 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 5 v 178 (ObjNum 128 snap 27 seq_num 128) dirty exists 2024-09-21T09:32:22.541 INFO:tasks.rados.rados.0.smithi022.stdout:377: finishing write tid 2 to smithi02284960-32 2024-09-21T09:32:22.541 INFO:tasks.rados.rados.0.smithi022.stdout:377: finishing write tid 3 to smithi02284960-32 2024-09-21T09:32:22.542 INFO:tasks.rados.rados.0.smithi022.stdout:377: finishing write tid 4 to smithi02284960-32 2024-09-21T09:32:22.542 INFO:tasks.rados.rados.0.smithi022.stdout:377: finishing write tid 5 to smithi02284960-32 2024-09-21T09:32:22.542 INFO:tasks.rados.rados.0.smithi022.stdout:377: finishing write tid 6 to smithi02284960-32 2024-09-21T09:32:22.542 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 32 v 207 (ObjNum 143 snap 32 seq_num 143) dirty exists 2024-09-21T09:32:22.542 INFO:tasks.rados.rados.0.smithi022.stdout:377: left oid 32 (ObjNum 143 snap 32 seq_num 143) 2024-09-21T09:32:22.542 INFO:tasks.rados.rados.0.smithi022.stdout:378: seq_num 144 ranges {0=16384} 2024-09-21T09:32:22.542 INFO:tasks.rados.rados.0.smithi022.stdout:378: writing smithi02284960-10 from 0 to 16384 tid 1 2024-09-21T09:32:22.542 INFO:tasks.rados.rados.0.smithi022.stdout:374: done (4 left) 2024-09-21T09:32:22.542 INFO:tasks.rados.rados.0.smithi022.stdout:375: done (3 left) 2024-09-21T09:32:22.542 INFO:tasks.rados.rados.0.smithi022.stdout:376: done (2 left) 2024-09-21T09:32:22.542 INFO:tasks.rados.rados.0.smithi022.stdout:377: done (1 left) 2024-09-21T09:32:22.542 INFO:tasks.rados.rados.0.smithi022.stdout:379: snap_create 2024-09-21T09:32:22.543 INFO:tasks.rados.rados.0.smithi022.stdout:378: finishing write tid 1 to smithi02284960-10 2024-09-21T09:32:22.543 INFO:tasks.rados.rados.0.smithi022.stdout:378: finishing write tid 2 to smithi02284960-10 2024-09-21T09:32:22.546 INFO:tasks.rados.rados.0.smithi022.stdout:378: finishing write tid 3 to smithi02284960-10 2024-09-21T09:32:22.546 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 10 v 203 (ObjNum 144 snap 32 seq_num 144) dirty exists 2024-09-21T09:32:22.546 INFO:tasks.rados.rados.0.smithi022.stdout:378: left oid 10 (ObjNum 144 snap 32 seq_num 144) 2024-09-21T09:32:22.807 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:22 smithi022 ceph-mon[27763]: osdmap e130: 8 total, 8 up, 8 in 2024-09-21T09:32:22.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:22 smithi022 ceph-mon[39525]: osdmap e130: 8 total, 8 up, 8 in 2024-09-21T09:32:22.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:22 smithi064 ceph-mon[31124]: osdmap e130: 8 total, 8 up, 8 in 2024-09-21T09:32:23.453 INFO:tasks.rados.rados.0.smithi022.stdout:378: done (1 left) 2024-09-21T09:32:23.453 INFO:tasks.rados.rados.0.smithi022.stdout:379: done (0 left) 2024-09-21T09:32:23.453 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 5 current snap is 33 2024-09-21T09:32:23.453 INFO:tasks.rados.rados.0.smithi022.stdout:380: seq_num 145 ranges {6103040=32768} 2024-09-21T09:32:23.487 INFO:tasks.rados.rados.0.smithi022.stdout:380: writing smithi02284960-5 from 6103040 to 6135808 tid 1 2024-09-21T09:32:23.487 INFO:tasks.rados.rados.0.smithi022.stdout:381: rollback oid 31 current snap is 33 2024-09-21T09:32:23.487 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 31 to 31 2024-09-21T09:32:23.487 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 49 current snap is 33 2024-09-21T09:32:23.487 INFO:tasks.rados.rados.0.smithi022.stdout:382: seq_num 146 ranges {0=49152} 2024-09-21T09:32:23.487 INFO:tasks.rados.rados.0.smithi022.stdout:382: writing smithi02284960-49 from 0 to 49152 tid 1 2024-09-21T09:32:23.487 INFO:tasks.rados.rados.0.smithi022.stdout:383: rmattr oid 41 current snap is 33 2024-09-21T09:32:23.487 INFO:tasks.rados.rados.0.smithi022.stdout:383: done (3 left) 2024-09-21T09:32:23.487 INFO:tasks.rados.rados.0.smithi022.stdout:384: read oid 7 snap -1 2024-09-21T09:32:23.488 INFO:tasks.rados.rados.0.smithi022.stdout:384: expect (ObjNum 72 snap 5 seq_num 72) 2024-09-21T09:32:23.488 INFO:tasks.rados.rados.0.smithi022.stdout:385: snap_create 2024-09-21T09:32:23.489 INFO:tasks.rados.rados.0.smithi022.stdout:382: finishing write tid 1 to smithi02284960-49 2024-09-21T09:32:23.489 INFO:tasks.rados.rados.0.smithi022.stdout:382: finishing write tid 2 to smithi02284960-49 2024-09-21T09:32:23.492 INFO:tasks.rados.rados.0.smithi022.stdout:382: finishing write tid 3 to smithi02284960-49 2024-09-21T09:32:23.492 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 49 v 80 (ObjNum 146 snap 33 seq_num 146) dirty exists 2024-09-21T09:32:23.492 INFO:tasks.rados.rados.0.smithi022.stdout:382: left oid 49 (ObjNum 146 snap 33 seq_num 146) 2024-09-21T09:32:23.492 INFO:tasks.rados.rados.0.smithi022.stdout:381: finishing rollback tid 0 to smithi02284960-31 2024-09-21T09:32:23.493 INFO:tasks.rados.rados.0.smithi022.stdout:381: finishing rollback tid 1 to smithi02284960-31 2024-09-21T09:32:23.493 INFO:tasks.rados.rados.0.smithi022.stdout:384: expect (ObjNum 72 snap 5 seq_num 72) 2024-09-21T09:32:23.530 INFO:tasks.rados.rados.0.smithi022.stdout:381: finishing rollback tid 2 to smithi02284960-31 2024-09-21T09:32:23.530 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 31 v 194 (ObjNum 71 snap 5 seq_num 71) dirty exists 2024-09-21T09:32:23.530 INFO:tasks.rados.rados.0.smithi022.stdout:380: finishing write tid 1 to smithi02284960-5 2024-09-21T09:32:23.530 INFO:tasks.rados.rados.0.smithi022.stdout:380: finishing write tid 2 to smithi02284960-5 2024-09-21T09:32:23.530 INFO:tasks.rados.rados.0.smithi022.stdout:380: finishing write tid 3 to smithi02284960-5 2024-09-21T09:32:23.530 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 5 v 180 (ObjNum 145 snap 33 seq_num 145) dirty exists 2024-09-21T09:32:23.531 INFO:tasks.rados.rados.0.smithi022.stdout:380: left oid 5 (ObjNum 145 snap 33 seq_num 145) 2024-09-21T09:32:23.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:23 smithi022 ceph-mon[39525]: osdmap e131: 8 total, 8 up, 8 in 2024-09-21T09:32:23.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:23 smithi022 ceph-mon[39525]: pgmap v192: 121 pgs: 121 active+clean; 121 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 536 KiB/s rd, 7.8 MiB/s wr, 18 op/s 2024-09-21T09:32:23.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:23 smithi022 ceph-mon[27763]: osdmap e131: 8 total, 8 up, 8 in 2024-09-21T09:32:23.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:23 smithi022 ceph-mon[27763]: pgmap v192: 121 pgs: 121 active+clean; 121 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 536 KiB/s rd, 7.8 MiB/s wr, 18 op/s 2024-09-21T09:32:23.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:23 smithi064 ceph-mon[31124]: osdmap e131: 8 total, 8 up, 8 in 2024-09-21T09:32:23.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:23 smithi064 ceph-mon[31124]: pgmap v192: 121 pgs: 121 active+clean; 121 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 536 KiB/s rd, 7.8 MiB/s wr, 18 op/s 2024-09-21T09:32:24.457 INFO:tasks.rados.rados.0.smithi022.stdout:380: done (4 left) 2024-09-21T09:32:24.457 INFO:tasks.rados.rados.0.smithi022.stdout:381: done (3 left) 2024-09-21T09:32:24.457 INFO:tasks.rados.rados.0.smithi022.stdout:382: done (2 left) 2024-09-21T09:32:24.457 INFO:tasks.rados.rados.0.smithi022.stdout:384: done (1 left) 2024-09-21T09:32:24.457 INFO:tasks.rados.rados.0.smithi022.stdout:385: done (0 left) 2024-09-21T09:32:24.457 INFO:tasks.rados.rados.0.smithi022.stdout:386: delete oid 49 current snap is 34 2024-09-21T09:32:24.459 INFO:tasks.rados.rados.0.smithi022.stdout:386: done (0 left) 2024-09-21T09:32:24.459 INFO:tasks.rados.rados.0.smithi022.stdout:387: rollback oid 18 current snap is 34 2024-09-21T09:32:24.459 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 18 to 32 2024-09-21T09:32:24.459 INFO:tasks.rados.rados.0.smithi022.stdout:388: snap_create 2024-09-21T09:32:24.463 INFO:tasks.rados.rados.0.smithi022.stdout:387: finishing rollback tid 0 to smithi02284960-18 2024-09-21T09:32:24.463 INFO:tasks.rados.rados.0.smithi022.stdout:387: finishing rollback tid 1 to smithi02284960-18 2024-09-21T09:32:24.463 INFO:tasks.rados.rados.0.smithi022.stdout:387: finishing rollback tid 2 to smithi02284960-18 2024-09-21T09:32:24.463 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 18 v 59 (ObjNum 17 snap 0 seq_num 17) dirty exists 2024-09-21T09:32:24.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:24 smithi022 ceph-mon[27763]: osdmap e132: 8 total, 8 up, 8 in 2024-09-21T09:32:24.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:24 smithi022 ceph-mon[39525]: osdmap e132: 8 total, 8 up, 8 in 2024-09-21T09:32:24.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:24 smithi064 ceph-mon[31124]: osdmap e132: 8 total, 8 up, 8 in 2024-09-21T09:32:25.423 INFO:tasks.rados.rados.0.smithi022.stdout:387: done (1 left) 2024-09-21T09:32:25.423 INFO:tasks.rados.rados.0.smithi022.stdout:388: done (0 left) 2024-09-21T09:32:25.423 INFO:tasks.rados.rados.0.smithi022.stdout:389: rollback oid 41 current snap is 35 2024-09-21T09:32:25.423 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 41 to 33 2024-09-21T09:32:25.423 INFO:tasks.rados.rados.0.smithi022.stdout:390: rollback oid 18 current snap is 35 2024-09-21T09:32:25.423 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 18 to 28 2024-09-21T09:32:25.424 INFO:tasks.rados.rados.0.smithi022.stdout:391: rmattr oid 23 current snap is 35 2024-09-21T09:32:25.424 INFO:tasks.rados.rados.0.smithi022.stdout:392: copy_from oid 13 from oid 37 current snap is 35 2024-09-21T09:32:25.424 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 36 current snap is 35 2024-09-21T09:32:25.424 INFO:tasks.rados.rados.0.smithi022.stdout:393: seq_num 148 ranges {761856=81920} 2024-09-21T09:32:25.428 INFO:tasks.rados.rados.0.smithi022.stdout:393: writing smithi02284960-36 from 761856 to 843776 tid 1 2024-09-21T09:32:25.428 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 31 current snap is 35 2024-09-21T09:32:25.428 INFO:tasks.rados.rados.0.smithi022.stdout:394: seq_num 149 ranges {2629632=98304} 2024-09-21T09:32:25.444 INFO:tasks.rados.rados.0.smithi022.stdout:394: writing smithi02284960-31 from 2629632 to 2727936 tid 1 2024-09-21T09:32:25.444 INFO:tasks.rados.rados.0.smithi022.stdout:395: setattr oid 39 current snap is 35 2024-09-21T09:32:25.444 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 23 v 85 (ObjNum 53 snap 1 seq_num 53) dirty exists 2024-09-21T09:32:25.444 INFO:tasks.rados.rados.0.smithi022.stdout:389: finishing rollback tid 0 to smithi02284960-41 2024-09-21T09:32:25.444 INFO:tasks.rados.rados.0.smithi022.stdout:389: finishing rollback tid 1 to smithi02284960-41 2024-09-21T09:32:25.444 INFO:tasks.rados.rados.0.smithi022.stdout:389: finishing rollback tid 2 to smithi02284960-41 2024-09-21T09:32:25.444 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 41 v 207 (ObjNum 122 snap 24 seq_num 122) dirty exists 2024-09-21T09:32:25.444 INFO:tasks.rados.rados.0.smithi022.stdout:390: finishing rollback tid 0 to smithi02284960-18 2024-09-21T09:32:25.444 INFO:tasks.rados.rados.0.smithi022.stdout:390: finishing rollback tid 1 to smithi02284960-18 2024-09-21T09:32:25.444 INFO:tasks.rados.rados.0.smithi022.stdout:393: finishing write tid 1 to smithi02284960-36 2024-09-21T09:32:25.444 INFO:tasks.rados.rados.0.smithi022.stdout:390: finishing rollback tid 2 to smithi02284960-18 2024-09-21T09:32:25.444 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 18 v 63 (ObjNum 17 snap 0 seq_num 17) dirty exists 2024-09-21T09:32:25.445 INFO:tasks.rados.rados.0.smithi022.stdout:393: finishing write tid 2 to smithi02284960-36 2024-09-21T09:32:25.445 INFO:tasks.rados.rados.0.smithi022.stdout:393: finishing write tid 3 to smithi02284960-36 2024-09-21T09:32:25.445 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 36 v 111 (ObjNum 148 snap 35 seq_num 148) dirty exists 2024-09-21T09:32:25.445 INFO:tasks.rados.rados.0.smithi022.stdout:393: left oid 36 (ObjNum 148 snap 35 seq_num 148) 2024-09-21T09:32:25.445 INFO:tasks.rados.rados.0.smithi022.stdout:392: finishing copy_from racing read to smithi02284960-13 2024-09-21T09:32:25.445 INFO:tasks.rados.rados.0.smithi022.stdout:392: finishing copy_from to smithi02284960-13 2024-09-21T09:32:25.445 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 13 v 74 (ObjNum 115 snap 23 seq_num 115) dirty exists 2024-09-21T09:32:25.446 INFO:tasks.rados.rados.0.smithi022.stdout:394: finishing write tid 1 to smithi02284960-31 2024-09-21T09:32:25.446 INFO:tasks.rados.rados.0.smithi022.stdout:394: finishing write tid 2 to smithi02284960-31 2024-09-21T09:32:25.446 INFO:tasks.rados.rados.0.smithi022.stdout:389: done (6 left) 2024-09-21T09:32:25.446 INFO:tasks.rados.rados.0.smithi022.stdout:390: done (5 left) 2024-09-21T09:32:25.446 INFO:tasks.rados.rados.0.smithi022.stdout:391: done (4 left) 2024-09-21T09:32:25.446 INFO:tasks.rados.rados.0.smithi022.stdout:392: done (3 left) 2024-09-21T09:32:25.446 INFO:tasks.rados.rados.0.smithi022.stdout:393: done (2 left) 2024-09-21T09:32:25.447 INFO:tasks.rados.rados.0.smithi022.stdout:396: snap_create 2024-09-21T09:32:25.447 INFO:tasks.rados.rados.0.smithi022.stdout:394: finishing write tid 3 to smithi02284960-31 2024-09-21T09:32:25.447 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 31 v 197 (ObjNum 149 snap 35 seq_num 149) dirty exists 2024-09-21T09:32:25.447 INFO:tasks.rados.rados.0.smithi022.stdout:394: left oid 31 (ObjNum 149 snap 35 seq_num 149) 2024-09-21T09:32:25.449 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 39 v 85 (ObjNum 38 snap 0 seq_num 38) dirty exists 2024-09-21T09:32:25.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:25 smithi022 ceph-mon[39525]: osdmap e133: 8 total, 8 up, 8 in 2024-09-21T09:32:25.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:25 smithi022 ceph-mon[39525]: pgmap v195: 121 pgs: 121 active+clean; 121 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 536 KiB/s rd, 7.8 MiB/s wr, 18 op/s 2024-09-21T09:32:25.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:25 smithi022 ceph-mon[39525]: osdmap e134: 8 total, 8 up, 8 in 2024-09-21T09:32:25.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:25 smithi022 ceph-mon[27763]: osdmap e133: 8 total, 8 up, 8 in 2024-09-21T09:32:25.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:25 smithi022 ceph-mon[27763]: pgmap v195: 121 pgs: 121 active+clean; 121 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 536 KiB/s rd, 7.8 MiB/s wr, 18 op/s 2024-09-21T09:32:25.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:25 smithi022 ceph-mon[27763]: osdmap e134: 8 total, 8 up, 8 in 2024-09-21T09:32:25.880 INFO:teuthology.orchestra.run.smithi022.stdout:true 2024-09-21T09:32:25.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:25 smithi064 ceph-mon[31124]: osdmap e133: 8 total, 8 up, 8 in 2024-09-21T09:32:25.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:25 smithi064 ceph-mon[31124]: pgmap v195: 121 pgs: 121 active+clean; 121 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 536 KiB/s rd, 7.8 MiB/s wr, 18 op/s 2024-09-21T09:32:25.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:25 smithi064 ceph-mon[31124]: osdmap e134: 8 total, 8 up, 8 in 2024-09-21T09:32:26.408 INFO:teuthology.orchestra.run.smithi022.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T09:32:26.408 INFO:teuthology.orchestra.run.smithi022.stdout:alertmanager.a smithi022 *:9093,9094 running (2m) 2m ago 3m 15.7M - 0.25.0 c8568f914cd2 42a53bb454cd 2024-09-21T09:32:26.408 INFO:teuthology.orchestra.run.smithi022.stdout:grafana.a smithi064 *:3000 running (3m) 2m ago 3m 81.2M - 9.4.7 954c08fa6188 ffde1f806d0b 2024-09-21T09:32:26.408 INFO:teuthology.orchestra.run.smithi022.stdout:mds.a smithi022 running (5m) 2m ago 5m 15.1M - 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 34e4221f3a27 2024-09-21T09:32:26.408 INFO:teuthology.orchestra.run.smithi022.stdout:mds.b smithi064 running (5m) 2m ago 5m 17.4M - 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 0f846b3ec694 2024-09-21T09:32:26.408 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.x smithi064 *:8443 running (8m) 2m ago 8m 399M - 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be c7e735bed227 2024-09-21T09:32:26.409 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.y smithi022 *:9283,8765 running (10m) 2m ago 10m 446M - 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 5bc449c0676f 2024-09-21T09:32:26.409 INFO:teuthology.orchestra.run.smithi022.stdout:mon.a smithi022 running (10m) 2m ago 10m 43.5M 2048M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 1e0a2d74e66e 2024-09-21T09:32:26.409 INFO:teuthology.orchestra.run.smithi022.stdout:mon.b smithi064 running (8m) 2m ago 8m 38.3M 2048M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be ed126533cd64 2024-09-21T09:32:26.409 INFO:teuthology.orchestra.run.smithi022.stdout:mon.c smithi022 running (8m) 2m ago 8m 39.1M 2048M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 79f307e3b64a 2024-09-21T09:32:26.409 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.a smithi022 *:9100 running (3m) 2m ago 3m 8958k - 1.5.0 0da6a335fe13 7cf4c99ebf3a 2024-09-21T09:32:26.409 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.b smithi064 *:9100 running (3m) 2m ago 3m 3414k - 1.5.0 0da6a335fe13 643a1e1aec61 2024-09-21T09:32:26.409 INFO:teuthology.orchestra.run.smithi022.stdout:osd.0 smithi022 running (8m) 2m ago 8m 44.1M 2993M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be fcd812003534 2024-09-21T09:32:26.409 INFO:teuthology.orchestra.run.smithi022.stdout:osd.1 smithi022 running (7m) 2m ago 7m 42.2M 2993M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 740462b5c304 2024-09-21T09:32:26.409 INFO:teuthology.orchestra.run.smithi022.stdout:osd.2 smithi022 running (7m) 2m ago 7m 45.3M 2993M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be fefc7c770c32 2024-09-21T09:32:26.409 INFO:teuthology.orchestra.run.smithi022.stdout:osd.3 smithi022 running (7m) 2m ago 6m 41.5M 2993M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be a56193b67a92 2024-09-21T09:32:26.409 INFO:teuthology.orchestra.run.smithi022.stdout:osd.4 smithi064 running (6m) 2m ago 6m 44.5M 2992M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 2a2937374fc4 2024-09-21T09:32:26.409 INFO:teuthology.orchestra.run.smithi022.stdout:osd.5 smithi064 running (6m) 2m ago 6m 43.1M 2992M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 1900c4af5d28 2024-09-21T09:32:26.409 INFO:teuthology.orchestra.run.smithi022.stdout:osd.6 smithi064 running (5m) 2m ago 5m 40.3M 2992M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 3cc4ec816e5b 2024-09-21T09:32:26.409 INFO:teuthology.orchestra.run.smithi022.stdout:osd.7 smithi064 running (5m) 2m ago 5m 41.5M 2992M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 18ef0be94104 2024-09-21T09:32:26.410 INFO:teuthology.orchestra.run.smithi022.stdout:prometheus.a smithi064 *:9095 running (2m) 2m ago 4m 26.0M - 2.43.0 a07b618ecd1d caee9b54c131 2024-09-21T09:32:26.423 INFO:tasks.rados.rados.0.smithi022.stdout:394: done (2 left) 2024-09-21T09:32:26.423 INFO:tasks.rados.rados.0.smithi022.stdout:395: done (1 left) 2024-09-21T09:32:26.423 INFO:tasks.rados.rados.0.smithi022.stdout:396: done (0 left) 2024-09-21T09:32:26.423 INFO:tasks.rados.rados.0.smithi022.stdout:397: copy_from oid 34 from oid 46 current snap is 36 2024-09-21T09:32:26.423 INFO:tasks.rados.rados.0.smithi022.stdout:398: snap_remove snap 35 2024-09-21T09:32:26.729 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-09-21T09:32:26.729 INFO:teuthology.orchestra.run.smithi022.stdout: "mon": { 2024-09-21T09:32:26.729 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 3 2024-09-21T09:32:26.730 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-21T09:32:26.730 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr": { 2024-09-21T09:32:26.730 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 2 2024-09-21T09:32:26.730 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-21T09:32:26.730 INFO:teuthology.orchestra.run.smithi022.stdout: "osd": { 2024-09-21T09:32:26.730 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 8 2024-09-21T09:32:26.730 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-21T09:32:26.730 INFO:teuthology.orchestra.run.smithi022.stdout: "mds": { 2024-09-21T09:32:26.730 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 2 2024-09-21T09:32:26.730 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-21T09:32:26.730 INFO:teuthology.orchestra.run.smithi022.stdout: "overall": { 2024-09-21T09:32:26.730 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 15 2024-09-21T09:32:26.730 INFO:teuthology.orchestra.run.smithi022.stdout: } 2024-09-21T09:32:26.731 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-09-21T09:32:26.731 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:26 smithi022 ceph-mon[39525]: from='client.14991 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:32:26.731 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:26 smithi022 ceph-mon[39525]: from='client.14997 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:32:26.731 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:26 smithi022 ceph-mon[39525]: osdmap e135: 8 total, 8 up, 8 in 2024-09-21T09:32:26.731 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:26 smithi022 ceph-mon[27763]: from='client.14991 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:32:26.731 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:26 smithi022 ceph-mon[27763]: from='client.14997 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:32:26.731 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:26 smithi022 ceph-mon[27763]: osdmap e135: 8 total, 8 up, 8 in 2024-09-21T09:32:26.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:26 smithi064 ceph-mon[31124]: from='client.14991 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:32:26.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:26 smithi064 ceph-mon[31124]: from='client.14997 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:32:26.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:26 smithi064 ceph-mon[31124]: osdmap e135: 8 total, 8 up, 8 in 2024-09-21T09:32:27.002 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-09-21T09:32:27.002 INFO:teuthology.orchestra.run.smithi022.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47", 2024-09-21T09:32:27.002 INFO:teuthology.orchestra.run.smithi022.stdout: "in_progress": true, 2024-09-21T09:32:27.002 INFO:teuthology.orchestra.run.smithi022.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T09:32:27.002 INFO:teuthology.orchestra.run.smithi022.stdout: "services_complete": [], 2024-09-21T09:32:27.002 INFO:teuthology.orchestra.run.smithi022.stdout: "progress": "0/20 daemons upgraded", 2024-09-21T09:32:27.002 INFO:teuthology.orchestra.run.smithi022.stdout: "message": "Pulling quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 image on host smithi064", 2024-09-21T09:32:27.002 INFO:teuthology.orchestra.run.smithi022.stdout: "is_paused": false 2024-09-21T09:32:27.002 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-09-21T09:32:27.426 INFO:tasks.rados.rados.0.smithi022.stdout:398: done (1 left) 2024-09-21T09:32:27.426 INFO:tasks.rados.rados.0.smithi022.stdout:399: rollback oid 45 current snap is 36 2024-09-21T09:32:27.426 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 45 to 28 2024-09-21T09:32:27.426 INFO:tasks.rados.rados.0.smithi022.stdout:397: finishing copy_from racing read to smithi02284960-34 2024-09-21T09:32:27.427 INFO:tasks.rados.rados.0.smithi022.stdout:397: finishing copy_from to smithi02284960-34 2024-09-21T09:32:27.427 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 34 v 8 (ObjNum 45 snap 0 seq_num 45) dirty exists 2024-09-21T09:32:27.427 INFO:tasks.rados.rados.0.smithi022.stdout:397: done (1 left) 2024-09-21T09:32:27.427 INFO:tasks.rados.rados.0.smithi022.stdout:400: read oid 28 snap -1 2024-09-21T09:32:27.427 INFO:tasks.rados.rados.0.smithi022.stdout:400: expect (ObjNum 9 snap 0 seq_num 9) 2024-09-21T09:32:27.427 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 21 current snap is 36 2024-09-21T09:32:27.427 INFO:tasks.rados.rados.0.smithi022.stdout:401: seq_num 150 ranges {2326528=114688} 2024-09-21T09:32:27.439 INFO:tasks.rados.rados.0.smithi022.stdout:401: writing smithi02284960-21 from 2326528 to 2441216 tid 1 2024-09-21T09:32:27.440 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 39 current snap is 36 2024-09-21T09:32:27.440 INFO:tasks.rados.rados.0.smithi022.stdout:399: finishing rollback tid 0 to smithi02284960-45 2024-09-21T09:32:27.440 INFO:tasks.rados.rados.0.smithi022.stdout:399: finishing rollback tid 1 to smithi02284960-45 2024-09-21T09:32:27.440 INFO:tasks.rados.rados.0.smithi022.stdout:399: finishing rollback tid 2 to smithi02284960-45 2024-09-21T09:32:27.440 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 45 v 115 (ObjNum 115 snap 23 seq_num 115) dirty exists 2024-09-21T09:32:27.440 INFO:tasks.rados.rados.0.smithi022.stdout:400: expect (ObjNum 9 snap 0 seq_num 9) 2024-09-21T09:32:27.447 INFO:tasks.rados.rados.0.smithi022.stdout:401: finishing write tid 1 to smithi02284960-21 2024-09-21T09:32:27.447 INFO:tasks.rados.rados.0.smithi022.stdout:401: finishing write tid 2 to smithi02284960-21 2024-09-21T09:32:27.447 INFO:tasks.rados.rados.0.smithi022.stdout:402: seq_num 151 ranges {638976=131072} 2024-09-21T09:32:27.452 INFO:tasks.rados.rados.0.smithi022.stdout:402: writing smithi02284960-39 from 638976 to 770048 tid 1 2024-09-21T09:32:27.452 INFO:tasks.rados.rados.0.smithi022.stdout:399: done (3 left) 2024-09-21T09:32:27.452 INFO:tasks.rados.rados.0.smithi022.stdout:400: done (2 left) 2024-09-21T09:32:27.452 INFO:tasks.rados.rados.0.smithi022.stdout:403: delete oid 37 current snap is 36 2024-09-21T09:32:27.452 INFO:tasks.rados.rados.0.smithi022.stdout:401: finishing write tid 3 to smithi02284960-21 2024-09-21T09:32:27.452 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 21 v 136 (ObjNum 150 snap 36 seq_num 150) dirty exists 2024-09-21T09:32:27.452 INFO:tasks.rados.rados.0.smithi022.stdout:401: left oid 21 (ObjNum 150 snap 36 seq_num 150) 2024-09-21T09:32:27.454 INFO:tasks.rados.rados.0.smithi022.stdout:402: finishing write tid 1 to smithi02284960-39 2024-09-21T09:32:27.455 INFO:tasks.rados.rados.0.smithi022.stdout:402: finishing write tid 2 to smithi02284960-39 2024-09-21T09:32:27.456 INFO:tasks.rados.rados.0.smithi022.stdout:402: finishing write tid 3 to smithi02284960-39 2024-09-21T09:32:27.456 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 39 v 87 (ObjNum 151 snap 36 seq_num 151) dirty exists 2024-09-21T09:32:27.456 INFO:tasks.rados.rados.0.smithi022.stdout:402: left oid 39 (ObjNum 151 snap 36 seq_num 151) 2024-09-21T09:32:27.458 INFO:tasks.rados.rados.0.smithi022.stdout:401: done (2 left) 2024-09-21T09:32:27.458 INFO:tasks.rados.rados.0.smithi022.stdout:402: done (1 left) 2024-09-21T09:32:27.458 INFO:tasks.rados.rados.0.smithi022.stdout:403: done (0 left) 2024-09-21T09:32:27.458 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 26 current snap is 36 2024-09-21T09:32:27.459 INFO:tasks.rados.rados.0.smithi022.stdout:404: seq_num 153 ranges {1884160=163840} 2024-09-21T09:32:27.470 INFO:tasks.rados.rados.0.smithi022.stdout:404: writing smithi02284960-26 from 1884160 to 2048000 tid 1 2024-09-21T09:32:27.470 INFO:tasks.rados.rados.0.smithi022.stdout:405: delete oid 6 current snap is 36 2024-09-21T09:32:27.472 INFO:tasks.rados.rados.0.smithi022.stdout:405: done (1 left) 2024-09-21T09:32:27.472 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 37 current snap is 36 2024-09-21T09:32:27.472 INFO:tasks.rados.rados.0.smithi022.stdout:406: seq_num 155 ranges {0=204800} 2024-09-21T09:32:27.473 INFO:tasks.rados.rados.0.smithi022.stdout:406: writing smithi02284960-37 from 0 to 204800 tid 1 2024-09-21T09:32:27.473 INFO:tasks.rados.rados.0.smithi022.stdout:407: copy_from oid 21 from oid 20 current snap is 36 2024-09-21T09:32:27.474 INFO:tasks.rados.rados.0.smithi022.stdout:408: setattr oid 34 current snap is 36 2024-09-21T09:32:27.474 INFO:tasks.rados.rados.0.smithi022.stdout:404: finishing write tid 1 to smithi02284960-26 2024-09-21T09:32:27.474 INFO:tasks.rados.rados.0.smithi022.stdout:404: finishing write tid 2 to smithi02284960-26 2024-09-21T09:32:27.475 INFO:tasks.rados.rados.0.smithi022.stdout:409: read oid 24 snap -1 2024-09-21T09:32:27.476 INFO:tasks.rados.rados.0.smithi022.stdout:409: expect deleted 2024-09-21T09:32:27.476 INFO:tasks.rados.rados.0.smithi022.stdout:410: snap_create 2024-09-21T09:32:27.476 INFO:tasks.rados.rados.0.smithi022.stdout:404: finishing write tid 3 to smithi02284960-26 2024-09-21T09:32:27.476 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 26 v 160 (ObjNum 153 snap 36 seq_num 153) dirty exists 2024-09-21T09:32:27.476 INFO:tasks.rados.rados.0.smithi022.stdout:404: left oid 26 (ObjNum 153 snap 36 seq_num 153) 2024-09-21T09:32:27.477 INFO:tasks.rados.rados.0.smithi022.stdout:406: finishing write tid 1 to smithi02284960-37 2024-09-21T09:32:27.477 INFO:tasks.rados.rados.0.smithi022.stdout:406: finishing write tid 2 to smithi02284960-37 2024-09-21T09:32:27.478 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 34 v 137 (ObjNum 45 snap 0 seq_num 45) dirty exists 2024-09-21T09:32:27.479 INFO:tasks.rados.rados.0.smithi022.stdout:406: finishing write tid 3 to smithi02284960-37 2024-09-21T09:32:27.479 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 37 v 201 (ObjNum 155 snap 36 seq_num 155) dirty exists 2024-09-21T09:32:27.479 INFO:tasks.rados.rados.0.smithi022.stdout:406: left oid 37 (ObjNum 155 snap 36 seq_num 155) 2024-09-21T09:32:27.493 INFO:tasks.rados.rados.0.smithi022.stdout:407: finishing copy_from racing read to smithi02284960-21 2024-09-21T09:32:27.498 INFO:tasks.rados.rados.0.smithi022.stdout:407: finishing copy_from to smithi02284960-21 2024-09-21T09:32:27.498 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 21 v 179 (ObjNum 132 snap 27 seq_num 132) dirty exists 2024-09-21T09:32:27.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:27 smithi022 ceph-mon[27763]: from='client.24908 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:32:27.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:27 smithi022 ceph-mon[27763]: from='client.? 172.21.15.22:0/1120577957' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:32:27.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:27 smithi022 ceph-mon[27763]: pgmap v198: 121 pgs: 121 active+clean; 121 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:32:27.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:27 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:32:27.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:27 smithi022 ceph-mon[27763]: from='client.15015 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:32:27.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:27 smithi022 ceph-mon[27763]: osdmap e136: 8 total, 8 up, 8 in 2024-09-21T09:32:27.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:27 smithi022 ceph-mon[39525]: from='client.24908 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:32:27.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:27 smithi022 ceph-mon[39525]: from='client.? 172.21.15.22:0/1120577957' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:32:27.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:27 smithi022 ceph-mon[39525]: pgmap v198: 121 pgs: 121 active+clean; 121 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:32:27.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:27 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:32:27.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:27 smithi022 ceph-mon[39525]: from='client.15015 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:32:27.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:27 smithi022 ceph-mon[39525]: osdmap e136: 8 total, 8 up, 8 in 2024-09-21T09:32:27.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:27 smithi064 ceph-mon[31124]: from='client.24908 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:32:27.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:27 smithi064 ceph-mon[31124]: from='client.? 172.21.15.22:0/1120577957' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:32:27.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:27 smithi064 ceph-mon[31124]: pgmap v198: 121 pgs: 121 active+clean; 121 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:32:27.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:27 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:32:27.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:27 smithi064 ceph-mon[31124]: from='client.15015 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:32:27.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:27 smithi064 ceph-mon[31124]: osdmap e136: 8 total, 8 up, 8 in 2024-09-21T09:32:28.470 INFO:tasks.rados.rados.0.smithi022.stdout:404: done (5 left) 2024-09-21T09:32:28.470 INFO:tasks.rados.rados.0.smithi022.stdout:406: done (4 left) 2024-09-21T09:32:28.470 INFO:tasks.rados.rados.0.smithi022.stdout:407: done (3 left) 2024-09-21T09:32:28.470 INFO:tasks.rados.rados.0.smithi022.stdout:408: done (2 left) 2024-09-21T09:32:28.470 INFO:tasks.rados.rados.0.smithi022.stdout:409: done (1 left) 2024-09-21T09:32:28.470 INFO:tasks.rados.rados.0.smithi022.stdout:410: done (0 left) 2024-09-21T09:32:28.470 INFO:tasks.rados.rados.0.smithi022.stdout:411: read oid 31 snap -1 2024-09-21T09:32:28.470 INFO:tasks.rados.rados.0.smithi022.stdout:411: expect (ObjNum 149 snap 35 seq_num 149) 2024-09-21T09:32:28.471 INFO:tasks.rados.rados.0.smithi022.stdout:412: copy_from oid 13 from oid 33 current snap is 37 2024-09-21T09:32:28.471 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 27 current snap is 37 2024-09-21T09:32:28.471 INFO:tasks.rados.rados.0.smithi022.stdout:413: seq_num 156 ranges {2375680=221184} 2024-09-21T09:32:28.485 INFO:tasks.rados.rados.0.smithi022.stdout:413: writing smithi02284960-27 from 2375680 to 2596864 tid 1 2024-09-21T09:32:28.485 INFO:tasks.rados.rados.0.smithi022.stdout:414: copy_from oid 5 from oid 22 current snap is 37 2024-09-21T09:32:28.485 INFO:tasks.rados.rados.0.smithi022.stdout:415: snap_create 2024-09-21T09:32:28.486 INFO:tasks.rados.rados.0.smithi022.stdout:411: expect (ObjNum 149 snap 35 seq_num 149) 2024-09-21T09:32:28.635 INFO:tasks.rados.rados.0.smithi022.stdout:412: finishing copy_from racing read to smithi02284960-13 2024-09-21T09:32:28.635 INFO:tasks.rados.rados.0.smithi022.stdout:412: finishing copy_from to smithi02284960-13 2024-09-21T09:32:28.635 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 13 v 169 (ObjNum 125 snap 25 seq_num 125) dirty exists 2024-09-21T09:32:28.635 INFO:tasks.rados.rados.0.smithi022.stdout:413: finishing write tid 1 to smithi02284960-27 2024-09-21T09:32:28.635 INFO:tasks.rados.rados.0.smithi022.stdout:413: finishing write tid 2 to smithi02284960-27 2024-09-21T09:32:28.635 INFO:tasks.rados.rados.0.smithi022.stdout:413: finishing write tid 3 to smithi02284960-27 2024-09-21T09:32:28.635 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 27 v 171 (ObjNum 156 snap 37 seq_num 156) dirty exists 2024-09-21T09:32:28.635 INFO:tasks.rados.rados.0.smithi022.stdout:413: left oid 27 (ObjNum 156 snap 37 seq_num 156) 2024-09-21T09:32:28.636 INFO:tasks.rados.rados.0.smithi022.stdout:414: finishing copy_from racing read to smithi02284960-5 2024-09-21T09:32:28.636 INFO:tasks.rados.rados.0.smithi022.stdout:414: finishing copy_from to smithi02284960-5 2024-09-21T09:32:28.636 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 5 v 16 (ObjNum 21 snap 0 seq_num 21) dirty exists 2024-09-21T09:32:29.475 INFO:tasks.rados.rados.0.smithi022.stdout:411: done (4 left) 2024-09-21T09:32:29.475 INFO:tasks.rados.rados.0.smithi022.stdout:412: done (3 left) 2024-09-21T09:32:29.475 INFO:tasks.rados.rados.0.smithi022.stdout:413: done (2 left) 2024-09-21T09:32:29.475 INFO:tasks.rados.rados.0.smithi022.stdout:414: done (1 left) 2024-09-21T09:32:29.475 INFO:tasks.rados.rados.0.smithi022.stdout:415: done (0 left) 2024-09-21T09:32:29.475 INFO:tasks.rados.rados.0.smithi022.stdout:416: read oid 36 snap -1 2024-09-21T09:32:29.475 INFO:tasks.rados.rados.0.smithi022.stdout:416: expect (ObjNum 148 snap 35 seq_num 148) 2024-09-21T09:32:29.475 INFO:tasks.rados.rados.0.smithi022.stdout:417: delete oid 21 current snap is 38 2024-09-21T09:32:29.478 INFO:tasks.rados.rados.0.smithi022.stdout:416: expect (ObjNum 148 snap 35 seq_num 148) 2024-09-21T09:32:29.514 INFO:tasks.rados.rados.0.smithi022.stdout:416: done (1 left) 2024-09-21T09:32:29.514 INFO:tasks.rados.rados.0.smithi022.stdout:417: done (0 left) 2024-09-21T09:32:29.514 INFO:tasks.rados.rados.0.smithi022.stdout:418: rmattr oid 22 current snap is 38 2024-09-21T09:32:29.514 INFO:tasks.rados.rados.0.smithi022.stdout:418: done (0 left) 2024-09-21T09:32:29.515 INFO:tasks.rados.rados.0.smithi022.stdout:419: rmattr oid 42 current snap is 38 2024-09-21T09:32:29.515 INFO:tasks.rados.rados.0.smithi022.stdout:419: done (0 left) 2024-09-21T09:32:29.515 INFO:tasks.rados.rados.0.smithi022.stdout:420: rmattr oid 23 current snap is 38 2024-09-21T09:32:29.515 INFO:tasks.rados.rados.0.smithi022.stdout:421: copy_from oid 15 from oid 19 current snap is 38 2024-09-21T09:32:29.515 INFO:tasks.rados.rados.0.smithi022.stdout:422: delete oid 30 current snap is 38 2024-09-21T09:32:29.517 INFO:tasks.rados.rados.0.smithi022.stdout:422: done (2 left) 2024-09-21T09:32:29.517 INFO:tasks.rados.rados.0.smithi022.stdout:423: rollback oid 49 current snap is 38 2024-09-21T09:32:29.517 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 49 to 37 2024-09-21T09:32:29.517 INFO:tasks.rados.rados.0.smithi022.stdout:424: read oid 45 snap -1 2024-09-21T09:32:29.517 INFO:tasks.rados.rados.0.smithi022.stdout:424: expect (ObjNum 115 snap 23 seq_num 115) 2024-09-21T09:32:29.517 INFO:tasks.rados.rados.0.smithi022.stdout:425: snap_remove snap 33 2024-09-21T09:32:29.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:29 smithi022 ceph-mon[39525]: osdmap e137: 8 total, 8 up, 8 in 2024-09-21T09:32:29.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:29 smithi022 ceph-mon[39525]: pgmap v201: 121 pgs: 121 active+clean; 129 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s rd, 1.9 MiB/s wr, 24 op/s 2024-09-21T09:32:29.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:29 smithi022 ceph-mon[27763]: osdmap e137: 8 total, 8 up, 8 in 2024-09-21T09:32:29.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:29 smithi022 ceph-mon[27763]: pgmap v201: 121 pgs: 121 active+clean; 129 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s rd, 1.9 MiB/s wr, 24 op/s 2024-09-21T09:32:29.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:29 smithi064 ceph-mon[31124]: osdmap e137: 8 total, 8 up, 8 in 2024-09-21T09:32:29.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:29 smithi064 ceph-mon[31124]: pgmap v201: 121 pgs: 121 active+clean; 129 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s rd, 1.9 MiB/s wr, 24 op/s 2024-09-21T09:32:30.426 INFO:tasks.rados.rados.0.smithi022.stdout:425: done (4 left) 2024-09-21T09:32:30.426 INFO:tasks.rados.rados.0.smithi022.stdout:426: delete oid 5 current snap is 38 2024-09-21T09:32:30.426 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 23 v 87 (ObjNum 53 snap 1 seq_num 53) dirty exists 2024-09-21T09:32:30.426 INFO:tasks.rados.rados.0.smithi022.stdout:423: finishing rollback tid 1 to smithi02284960-49 2024-09-21T09:32:30.426 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 49 v 82 (ObjNum 0 snap 0 seq_num 304) dirty dne 2024-09-21T09:32:30.426 INFO:tasks.rados.rados.0.smithi022.stdout:424: expect (ObjNum 115 snap 23 seq_num 115) 2024-09-21T09:32:30.482 INFO:tasks.rados.rados.0.smithi022.stdout:421: finishing copy_from racing read to smithi02284960-15 2024-09-21T09:32:30.482 INFO:tasks.rados.rados.0.smithi022.stdout:421: finishing copy_from to smithi02284960-15 2024-09-21T09:32:30.482 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 15 v 152 (ObjNum 141 snap 31 seq_num 141) dirty exists 2024-09-21T09:32:30.482 INFO:tasks.rados.rados.0.smithi022.stdout:420: done (4 left) 2024-09-21T09:32:30.482 INFO:tasks.rados.rados.0.smithi022.stdout:421: done (3 left) 2024-09-21T09:32:30.482 INFO:tasks.rados.rados.0.smithi022.stdout:423: done (2 left) 2024-09-21T09:32:30.482 INFO:tasks.rados.rados.0.smithi022.stdout:424: done (1 left) 2024-09-21T09:32:30.482 INFO:tasks.rados.rados.0.smithi022.stdout:426: done (0 left) 2024-09-21T09:32:30.482 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 12 current snap is 38 2024-09-21T09:32:30.482 INFO:tasks.rados.rados.0.smithi022.stdout:427: seq_num 160 ranges {5464064=286720} 2024-09-21T09:32:30.513 INFO:tasks.rados.rados.0.smithi022.stdout:427: writing smithi02284960-12 from 5464064 to 5750784 tid 1 2024-09-21T09:32:30.513 INFO:tasks.rados.rados.0.smithi022.stdout:428: snap_remove snap 36 2024-09-21T09:32:30.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:30 smithi022 ceph-mon[27763]: osdmap e138: 8 total, 8 up, 8 in 2024-09-21T09:32:30.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:30 smithi022 ceph-mon[27763]: osdmap e139: 8 total, 8 up, 8 in 2024-09-21T09:32:30.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:30 smithi022 ceph-mon[39525]: osdmap e138: 8 total, 8 up, 8 in 2024-09-21T09:32:30.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:30 smithi022 ceph-mon[39525]: osdmap e139: 8 total, 8 up, 8 in 2024-09-21T09:32:30.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:30 smithi064 ceph-mon[31124]: osdmap e138: 8 total, 8 up, 8 in 2024-09-21T09:32:30.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:30 smithi064 ceph-mon[31124]: osdmap e139: 8 total, 8 up, 8 in 2024-09-21T09:32:30.941 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:32:30 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[33213]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:32:30] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-21T09:32:31.481 INFO:tasks.rados.rados.0.smithi022.stdout:428: done (1 left) 2024-09-21T09:32:31.481 INFO:tasks.rados.rados.0.smithi022.stdout:429: snap_remove snap 29 2024-09-21T09:32:31.481 INFO:tasks.rados.rados.0.smithi022.stdout:427: finishing write tid 1 to smithi02284960-12 2024-09-21T09:32:31.481 INFO:tasks.rados.rados.0.smithi022.stdout:427: finishing write tid 2 to smithi02284960-12 2024-09-21T09:32:31.481 INFO:tasks.rados.rados.0.smithi022.stdout:427: finishing write tid 3 to smithi02284960-12 2024-09-21T09:32:31.481 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 12 v 206 (ObjNum 160 snap 38 seq_num 160) dirty exists 2024-09-21T09:32:31.481 INFO:tasks.rados.rados.0.smithi022.stdout:427: left oid 12 (ObjNum 160 snap 38 seq_num 160) 2024-09-21T09:32:31.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:31 smithi022 ceph-mon[39525]: pgmap v204: 121 pgs: 121 active+clean; 129 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s rd, 1.9 MiB/s wr, 24 op/s 2024-09-21T09:32:31.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:31 smithi022 ceph-mon[27763]: pgmap v204: 121 pgs: 121 active+clean; 129 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s rd, 1.9 MiB/s wr, 24 op/s 2024-09-21T09:32:31.558 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:32:31 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:32:31] "GET /metrics HTTP/1.1" 200 240054 "" "Prometheus/2.43.0" 2024-09-21T09:32:31.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:31 smithi064 ceph-mon[31124]: pgmap v204: 121 pgs: 121 active+clean; 129 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s rd, 1.9 MiB/s wr, 24 op/s 2024-09-21T09:32:32.486 INFO:tasks.rados.rados.0.smithi022.stdout:427: done (1 left) 2024-09-21T09:32:32.486 INFO:tasks.rados.rados.0.smithi022.stdout:429: done (0 left) 2024-09-21T09:32:32.486 INFO:tasks.rados.rados.0.smithi022.stdout:430: snap_remove snap 28 2024-09-21T09:32:32.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:32 smithi064 ceph-mon[31124]: osdmap e140: 8 total, 8 up, 8 in 2024-09-21T09:32:32.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:32 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:32:32.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:32 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T09:32:32.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:32 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T09:32:32.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:32 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T09:32:32.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:32 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:32:32.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:32 smithi022 ceph-mon[27763]: osdmap e140: 8 total, 8 up, 8 in 2024-09-21T09:32:32.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:32 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:32:32.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:32 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T09:32:32.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:32 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T09:32:32.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:32 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T09:32:32.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:32 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:32:32.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:32 smithi022 ceph-mon[39525]: osdmap e140: 8 total, 8 up, 8 in 2024-09-21T09:32:32.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:32 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:32:32.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:32 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T09:32:32.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:32 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T09:32:32.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:32 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T09:32:32.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:32 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:32:33.492 INFO:tasks.rados.rados.0.smithi022.stdout:430: done (0 left) 2024-09-21T09:32:33.492 INFO:tasks.rados.rados.0.smithi022.stdout:431: snap_remove snap 30 2024-09-21T09:32:33.786 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:33 smithi064 ceph-mon[31124]: Upgrade: Updating mgr.x 2024-09-21T09:32:33.787 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:33 smithi064 ceph-mon[31124]: Deploying daemon mgr.x on smithi064 2024-09-21T09:32:33.787 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:33 smithi064 ceph-mon[31124]: osdmap e141: 8 total, 8 up, 8 in 2024-09-21T09:32:33.787 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:33 smithi064 ceph-mon[31124]: pgmap v207: 121 pgs: 121 active+clean; 133 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.9 MiB/s rd, 1.4 MiB/s wr, 8 op/s 2024-09-21T09:32:33.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:33 smithi022 ceph-mon[39525]: Upgrade: Updating mgr.x 2024-09-21T09:32:33.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:33 smithi022 ceph-mon[39525]: Deploying daemon mgr.x on smithi064 2024-09-21T09:32:33.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:33 smithi022 ceph-mon[39525]: osdmap e141: 8 total, 8 up, 8 in 2024-09-21T09:32:33.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:33 smithi022 ceph-mon[39525]: pgmap v207: 121 pgs: 121 active+clean; 133 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.9 MiB/s rd, 1.4 MiB/s wr, 8 op/s 2024-09-21T09:32:33.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:33 smithi022 ceph-mon[27763]: Upgrade: Updating mgr.x 2024-09-21T09:32:33.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:33 smithi022 ceph-mon[27763]: Deploying daemon mgr.x on smithi064 2024-09-21T09:32:33.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:33 smithi022 ceph-mon[27763]: osdmap e141: 8 total, 8 up, 8 in 2024-09-21T09:32:33.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:33 smithi022 ceph-mon[27763]: pgmap v207: 121 pgs: 121 active+clean; 133 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.9 MiB/s rd, 1.4 MiB/s wr, 8 op/s 2024-09-21T09:32:34.191 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:32:33 smithi064 systemd[1]: Stopping Ceph mgr.x for d65e801e-77fa-11ef-baf6-efdc52797490... 2024-09-21T09:32:34.492 INFO:tasks.rados.rados.0.smithi022.stdout:431: done (0 left) 2024-09-21T09:32:34.493 INFO:tasks.rados.rados.0.smithi022.stdout:432: setattr oid 41 current snap is 38 2024-09-21T09:32:34.496 INFO:tasks.rados.rados.0.smithi022.stdout:433: read oid 26 snap -1 2024-09-21T09:32:34.496 INFO:tasks.rados.rados.0.smithi022.stdout:433: expect (ObjNum 153 snap 36 seq_num 153) 2024-09-21T09:32:34.496 INFO:tasks.rados.rados.0.smithi022.stdout:434: read oid 36 snap -1 2024-09-21T09:32:34.496 INFO:tasks.rados.rados.0.smithi022.stdout:434: expect (ObjNum 148 snap 35 seq_num 148) 2024-09-21T09:32:34.496 INFO:tasks.rados.rados.0.smithi022.stdout:435: snap_remove snap 37 2024-09-21T09:32:34.532 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:34 smithi064 ceph-mon[31124]: osdmap e142: 8 total, 8 up, 8 in 2024-09-21T09:32:34.532 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:32:34 smithi064 podman[67325]: 2024-09-21 09:32:34.261480194 +0000 UTC m=+0.116301958 container died c7e735bed227093eb6967a1f874f3ca6f0ce3147269d5c2af46a72c02ac3dcf0 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x, org.label-schema.vendor=CentOS, RELEASE=quincy-fbd1a6a, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T09:32:34.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:34 smithi022 ceph-mon[39525]: osdmap e142: 8 total, 8 up, 8 in 2024-09-21T09:32:34.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:34 smithi022 ceph-mon[27763]: osdmap e142: 8 total, 8 up, 8 in 2024-09-21T09:32:34.941 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:32:34 smithi064 podman[67325]: 2024-09-21 09:32:34.596426087 +0000 UTC m=+0.451247858 container cleanup c7e735bed227093eb6967a1f874f3ca6f0ce3147269d5c2af46a72c02ac3dcf0 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, RELEASE=quincy-fbd1a6a, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True) 2024-09-21T09:32:34.941 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:32:34 smithi064 bash[67325]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x 2024-09-21T09:32:35.379 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:32:35 smithi064 podman[67342]: 2024-09-21 09:32:35.131259957 +0000 UTC m=+0.859849647 container remove c7e735bed227093eb6967a1f874f3ca6f0ce3147269d5c2af46a72c02ac3dcf0 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x, RELEASE=quincy-fbd1a6a, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0) 2024-09-21T09:32:35.379 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:32:35 smithi064 systemd[1]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-09-21T09:32:35.427 INFO:tasks.rados.rados.0.smithi022.stdout:435: done (3 left) 2024-09-21T09:32:35.427 INFO:tasks.rados.rados.0.smithi022.stdout:436: read oid 29 snap -1 2024-09-21T09:32:35.427 INFO:tasks.rados.rados.0.smithi022.stdout:436: expect deleted 2024-09-21T09:32:35.427 INFO:tasks.rados.rados.0.smithi022.stdout:437: read oid 38 snap 22 2024-09-21T09:32:35.427 INFO:tasks.rados.rados.0.smithi022.stdout:437: expect (ObjNum 16 snap 0 seq_num 16) 2024-09-21T09:32:35.427 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 41 v 211 (ObjNum 122 snap 24 seq_num 122) dirty exists 2024-09-21T09:32:35.427 INFO:tasks.rados.rados.0.smithi022.stdout:434: expect (ObjNum 148 snap 35 seq_num 148) 2024-09-21T09:32:35.466 INFO:tasks.rados.rados.0.smithi022.stdout:433: expect (ObjNum 153 snap 36 seq_num 153) 2024-09-21T09:32:35.556 INFO:tasks.rados.rados.0.smithi022.stdout:437: expect (ObjNum 16 snap 0 seq_num 16) 2024-09-21T09:32:35.563 INFO:tasks.rados.rados.0.smithi022.stdout:432: done (4 left) 2024-09-21T09:32:35.563 INFO:tasks.rados.rados.0.smithi022.stdout:433: done (3 left) 2024-09-21T09:32:35.563 INFO:tasks.rados.rados.0.smithi022.stdout:434: done (2 left) 2024-09-21T09:32:35.563 INFO:tasks.rados.rados.0.smithi022.stdout:436: done (1 left) 2024-09-21T09:32:35.563 INFO:tasks.rados.rados.0.smithi022.stdout:437: done (0 left) 2024-09-21T09:32:35.563 INFO:tasks.rados.rados.0.smithi022.stdout:438: snap_create 2024-09-21T09:32:35.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:35 smithi064 ceph-mon[31124]: osdmap e143: 8 total, 8 up, 8 in 2024-09-21T09:32:35.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:35 smithi064 ceph-mon[31124]: pgmap v210: 121 pgs: 121 active+clean; 133 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.9 MiB/s rd, 1.4 MiB/s wr, 8 op/s 2024-09-21T09:32:35.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:35 smithi064 ceph-mon[31124]: osdmap e144: 8 total, 8 up, 8 in 2024-09-21T09:32:35.692 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:32:35 smithi064 systemd[1]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490@mgr.x.service: Failed with result 'exit-code'. 2024-09-21T09:32:35.692 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:32:35 smithi064 systemd[1]: Stopped Ceph mgr.x for d65e801e-77fa-11ef-baf6-efdc52797490. 2024-09-21T09:32:35.692 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:32:35 smithi064 systemd[1]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490@mgr.x.service: Consumed 10.661s CPU time. 2024-09-21T09:32:35.692 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:32:35 smithi064 systemd[1]: Starting Ceph mgr.x for d65e801e-77fa-11ef-baf6-efdc52797490... 2024-09-21T09:32:35.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:35 smithi022 ceph-mon[39525]: osdmap e143: 8 total, 8 up, 8 in 2024-09-21T09:32:35.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:35 smithi022 ceph-mon[39525]: pgmap v210: 121 pgs: 121 active+clean; 133 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.9 MiB/s rd, 1.4 MiB/s wr, 8 op/s 2024-09-21T09:32:35.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:35 smithi022 ceph-mon[39525]: osdmap e144: 8 total, 8 up, 8 in 2024-09-21T09:32:35.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:35 smithi022 ceph-mon[27763]: osdmap e143: 8 total, 8 up, 8 in 2024-09-21T09:32:35.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:35 smithi022 ceph-mon[27763]: pgmap v210: 121 pgs: 121 active+clean; 133 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.9 MiB/s rd, 1.4 MiB/s wr, 8 op/s 2024-09-21T09:32:35.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:35 smithi022 ceph-mon[27763]: osdmap e144: 8 total, 8 up, 8 in 2024-09-21T09:32:36.069 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:32:35 smithi064 podman[67487]: 2024-09-21 09:32:35.720279622 +0000 UTC m=+0.016739052 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:32:36.069 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:32:35 smithi064 podman[67487]: 2024-09-21 09:32:35.849859291 +0000 UTC m=+0.146318705 container create c102c1678bcc148d502e16cf7be5992efdfd0cd8bae917048620aa9c3b28cbb8 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x, RELEASE=main-62da54b, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2) 2024-09-21T09:32:36.441 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:32:36 smithi064 podman[67487]: 2024-09-21 09:32:36.068589871 +0000 UTC m=+0.365049298 container init c102c1678bcc148d502e16cf7be5992efdfd0cd8bae917048620aa9c3b28cbb8 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=main-62da54b, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=) 2024-09-21T09:32:36.441 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:32:36 smithi064 podman[67487]: 2024-09-21 09:32:36.074011808 +0000 UTC m=+0.370471222 container start c102c1678bcc148d502e16cf7be5992efdfd0cd8bae917048620aa9c3b28cbb8 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.37.2, RELEASE=main-62da54b) 2024-09-21T09:32:36.442 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:32:36 smithi064 ceph-mgr[67526]: -- 172.21.15.64:0/212234115 <== mon.1 v2:172.21.15.64:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x564d69ebd0e0 con 0x564d69e9b000 2024-09-21T09:32:36.442 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:32:36 smithi064 bash[67487]: c102c1678bcc148d502e16cf7be5992efdfd0cd8bae917048620aa9c3b28cbb8 2024-09-21T09:32:36.442 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:32:36 smithi064 systemd[1]: Started Ceph mgr.x for d65e801e-77fa-11ef-baf6-efdc52797490. 2024-09-21T09:32:36.505 INFO:tasks.rados.rados.0.smithi022.stdout:438: done (0 left) 2024-09-21T09:32:36.505 INFO:tasks.rados.rados.0.smithi022.stdout:439: delete oid 4 current snap is 39 2024-09-21T09:32:36.508 INFO:tasks.rados.rados.0.smithi022.stdout:439: done (0 left) 2024-09-21T09:32:36.508 INFO:tasks.rados.rados.0.smithi022.stdout:440: snap_create 2024-09-21T09:32:37.510 INFO:tasks.rados.rados.0.smithi022.stdout:440: done (0 left) 2024-09-21T09:32:37.510 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 28 current snap is 40 2024-09-21T09:32:37.510 INFO:tasks.rados.rados.0.smithi022.stdout:441: seq_num 162 ranges {155648=319488} 2024-09-21T09:32:37.512 INFO:tasks.rados.rados.0.smithi022.stdout:441: writing smithi02284960-28 from 155648 to 475136 tid 1 2024-09-21T09:32:37.512 INFO:tasks.rados.rados.0.smithi022.stdout:442: rollback oid 34 current snap is 40 2024-09-21T09:32:37.512 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 34 to 31 2024-09-21T09:32:37.512 INFO:tasks.rados.rados.0.smithi022.stdout:443: delete oid 8 current snap is 40 2024-09-21T09:32:37.517 INFO:tasks.rados.rados.0.smithi022.stdout:441: finishing write tid 1 to smithi02284960-28 2024-09-21T09:32:37.517 INFO:tasks.rados.rados.0.smithi022.stdout:443: done (2 left) 2024-09-21T09:32:37.517 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 47 current snap is 40 2024-09-21T09:32:37.518 INFO:tasks.rados.rados.0.smithi022.stdout:444: seq_num 164 ranges {1835008=352256} 2024-09-21T09:32:37.530 INFO:tasks.rados.rados.0.smithi022.stdout:444: writing smithi02284960-47 from 1835008 to 2187264 tid 1 2024-09-21T09:32:37.530 INFO:tasks.rados.rados.0.smithi022.stdout:445: snap_create 2024-09-21T09:32:37.530 INFO:tasks.rados.rados.0.smithi022.stdout:441: finishing write tid 2 to smithi02284960-28 2024-09-21T09:32:37.530 INFO:tasks.rados.rados.0.smithi022.stdout:442: finishing rollback tid 0 to smithi02284960-34 2024-09-21T09:32:37.530 INFO:tasks.rados.rados.0.smithi022.stdout:442: finishing rollback tid 1 to smithi02284960-34 2024-09-21T09:32:37.530 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 34 v 182 (ObjNum 129 snap 0 seq_num 3569991888) dirty dne 2024-09-21T09:32:37.530 INFO:tasks.rados.rados.0.smithi022.stdout:441: finishing write tid 3 to smithi02284960-28 2024-09-21T09:32:37.530 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 28 v 71 (ObjNum 162 snap 40 seq_num 162) dirty exists 2024-09-21T09:32:37.530 INFO:tasks.rados.rados.0.smithi022.stdout:441: left oid 28 (ObjNum 162 snap 40 seq_num 162) 2024-09-21T09:32:37.533 INFO:tasks.rados.rados.0.smithi022.stdout:444: finishing write tid 1 to smithi02284960-47 2024-09-21T09:32:37.534 INFO:tasks.rados.rados.0.smithi022.stdout:444: finishing write tid 2 to smithi02284960-47 2024-09-21T09:32:37.535 INFO:tasks.rados.rados.0.smithi022.stdout:444: finishing write tid 3 to smithi02284960-47 2024-09-21T09:32:37.535 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 47 v 93 (ObjNum 164 snap 40 seq_num 164) dirty exists 2024-09-21T09:32:37.535 INFO:tasks.rados.rados.0.smithi022.stdout:444: left oid 47 (ObjNum 164 snap 40 seq_num 164) 2024-09-21T09:32:37.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:37 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:32:37.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:37 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:32:37.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:37 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:32:37.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:37 smithi022 ceph-mon[39525]: osdmap e145: 8 total, 8 up, 8 in 2024-09-21T09:32:37.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:37 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:32:37.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:37 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:32:37.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:37 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:32:37.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:37 smithi022 ceph-mon[27763]: osdmap e145: 8 total, 8 up, 8 in 2024-09-21T09:32:37.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:37 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:32:37.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:37 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:32:37.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:37 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:32:37.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:37 smithi064 ceph-mon[31124]: osdmap e145: 8 total, 8 up, 8 in 2024-09-21T09:32:38.512 INFO:tasks.rados.rados.0.smithi022.stdout:441: done (3 left) 2024-09-21T09:32:38.512 INFO:tasks.rados.rados.0.smithi022.stdout:442: done (2 left) 2024-09-21T09:32:38.512 INFO:tasks.rados.rados.0.smithi022.stdout:444: done (1 left) 2024-09-21T09:32:38.512 INFO:tasks.rados.rados.0.smithi022.stdout:445: done (0 left) 2024-09-21T09:32:38.512 INFO:tasks.rados.rados.0.smithi022.stdout:446: read oid 45 snap -1 2024-09-21T09:32:38.512 INFO:tasks.rados.rados.0.smithi022.stdout:446: expect (ObjNum 115 snap 23 seq_num 115) 2024-09-21T09:32:38.512 INFO:tasks.rados.rados.0.smithi022.stdout:447: read oid 29 snap 27 2024-09-21T09:32:38.512 INFO:tasks.rados.rados.0.smithi022.stdout:447: expect deleted 2024-09-21T09:32:38.512 INFO:tasks.rados.rados.0.smithi022.stdout:448: setattr oid 2 current snap is 41 2024-09-21T09:32:38.514 INFO:tasks.rados.rados.0.smithi022.stdout:449: rmattr oid 12 current snap is 41 2024-09-21T09:32:38.514 INFO:tasks.rados.rados.0.smithi022.stdout:449: done (3 left) 2024-09-21T09:32:38.514 INFO:tasks.rados.rados.0.smithi022.stdout:450: snap_remove snap 40 2024-09-21T09:32:38.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:38 smithi022 ceph-mon[39525]: pgmap v213: 121 pgs: 121 active+clean; 133 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:32:38.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:38 smithi022 ceph-mon[39525]: osdmap e146: 8 total, 8 up, 8 in 2024-09-21T09:32:38.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:38 smithi022 ceph-mon[27763]: pgmap v213: 121 pgs: 121 active+clean; 133 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:32:38.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:38 smithi022 ceph-mon[27763]: osdmap e146: 8 total, 8 up, 8 in 2024-09-21T09:32:38.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:38 smithi064 ceph-mon[31124]: pgmap v213: 121 pgs: 121 active+clean; 133 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:32:38.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:38 smithi064 ceph-mon[31124]: osdmap e146: 8 total, 8 up, 8 in 2024-09-21T09:32:39.518 INFO:tasks.rados.rados.0.smithi022.stdout:450: done (3 left) 2024-09-21T09:32:39.518 INFO:tasks.rados.rados.0.smithi022.stdout:451: read oid 38 snap 27 2024-09-21T09:32:39.518 INFO:tasks.rados.rados.0.smithi022.stdout:451: expect (ObjNum 133 snap 27 seq_num 133) 2024-09-21T09:32:39.518 INFO:tasks.rados.rados.0.smithi022.stdout:452: delete oid 17 current snap is 41 2024-09-21T09:32:39.518 INFO:tasks.rados.rados.0.smithi022.stdout:446: expect (ObjNum 115 snap 23 seq_num 115) 2024-09-21T09:32:39.572 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 2 v 183 (ObjNum 129 snap 0 seq_num 2955199934) dirty exists 2024-09-21T09:32:39.573 INFO:tasks.rados.rados.0.smithi022.stdout:451: expect (ObjNum 133 snap 27 seq_num 133) 2024-09-21T09:32:39.631 INFO:tasks.rados.rados.0.smithi022.stdout:446: done (4 left) 2024-09-21T09:32:39.632 INFO:tasks.rados.rados.0.smithi022.stdout:447: done (3 left) 2024-09-21T09:32:39.632 INFO:tasks.rados.rados.0.smithi022.stdout:448: done (2 left) 2024-09-21T09:32:39.632 INFO:tasks.rados.rados.0.smithi022.stdout:451: done (1 left) 2024-09-21T09:32:39.632 INFO:tasks.rados.rados.0.smithi022.stdout:452: done (0 left) 2024-09-21T09:32:39.632 INFO:tasks.rados.rados.0.smithi022.stdout:453: copy_from oid 11 from oid 50 current snap is 41 2024-09-21T09:32:39.632 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 16 current snap is 41 2024-09-21T09:32:39.632 INFO:tasks.rados.rados.0.smithi022.stdout:454: seq_num 166 ranges {4521984=385024} 2024-09-21T09:32:39.660 INFO:tasks.rados.rados.0.smithi022.stdout:454: writing smithi02284960-16 from 4521984 to 4907008 tid 1 2024-09-21T09:32:39.660 INFO:tasks.rados.rados.0.smithi022.stdout:455: rmattr oid 2 current snap is 41 2024-09-21T09:32:39.660 INFO:tasks.rados.rados.0.smithi022.stdout:453: finishing copy_from racing read to smithi02284960-11 2024-09-21T09:32:39.660 INFO:tasks.rados.rados.0.smithi022.stdout:453: finishing copy_from to smithi02284960-11 2024-09-21T09:32:39.660 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 11 v 104 (ObjNum 131 snap 27 seq_num 131) dirty exists 2024-09-21T09:32:39.660 INFO:tasks.rados.rados.0.smithi022.stdout:453: done (2 left) 2024-09-21T09:32:39.660 INFO:tasks.rados.rados.0.smithi022.stdout:456: delete oid 23 current snap is 41 2024-09-21T09:32:39.661 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 2 v 184 (ObjNum 129 snap 0 seq_num 3569977824) dirty exists 2024-09-21T09:32:39.662 INFO:tasks.rados.rados.0.smithi022.stdout:455: done (2 left) 2024-09-21T09:32:39.662 INFO:tasks.rados.rados.0.smithi022.stdout:456: done (1 left) 2024-09-21T09:32:39.662 INFO:tasks.rados.rados.0.smithi022.stdout:457: read oid 3 snap -1 2024-09-21T09:32:39.662 INFO:tasks.rados.rados.0.smithi022.stdout:457: expect (ObjNum 109 snap 23 seq_num 109) 2024-09-21T09:32:39.662 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 29 current snap is 41 2024-09-21T09:32:39.662 INFO:tasks.rados.rados.0.smithi022.stdout:458: seq_num 168 ranges {0=417792} 2024-09-21T09:32:39.665 INFO:tasks.rados.rados.0.smithi022.stdout:458: writing smithi02284960-29 from 0 to 417792 tid 1 2024-09-21T09:32:39.665 INFO:tasks.rados.rados.0.smithi022.stdout:459: copy_from oid 33 from oid 36 current snap is 41 2024-09-21T09:32:39.665 INFO:tasks.rados.rados.0.smithi022.stdout:454: finishing write tid 1 to smithi02284960-16 2024-09-21T09:32:39.665 INFO:tasks.rados.rados.0.smithi022.stdout:454: finishing write tid 2 to smithi02284960-16 2024-09-21T09:32:39.665 INFO:tasks.rados.rados.0.smithi022.stdout:454: finishing write tid 3 to smithi02284960-16 2024-09-21T09:32:39.665 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 16 v 74 (ObjNum 166 snap 41 seq_num 166) dirty exists 2024-09-21T09:32:39.665 INFO:tasks.rados.rados.0.smithi022.stdout:454: left oid 16 (ObjNum 166 snap 41 seq_num 166) 2024-09-21T09:32:39.665 INFO:tasks.rados.rados.0.smithi022.stdout:454: done (3 left) 2024-09-21T09:32:39.666 INFO:tasks.rados.rados.0.smithi022.stdout:460: read oid 34 snap -1 2024-09-21T09:32:39.666 INFO:tasks.rados.rados.0.smithi022.stdout:460: expect deleted 2024-09-21T09:32:39.666 INFO:tasks.rados.rados.0.smithi022.stdout:461: rollback oid 44 current snap is 41 2024-09-21T09:32:39.666 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 44 to 27 2024-09-21T09:32:39.666 INFO:tasks.rados.rados.0.smithi022.stdout:462: snap_create 2024-09-21T09:32:39.668 INFO:tasks.rados.rados.0.smithi022.stdout:461: finishing rollback tid 0 to smithi02284960-44 2024-09-21T09:32:39.668 INFO:tasks.rados.rados.0.smithi022.stdout:458: finishing write tid 1 to smithi02284960-29 2024-09-21T09:32:39.668 INFO:tasks.rados.rados.0.smithi022.stdout:461: finishing rollback tid 1 to smithi02284960-44 2024-09-21T09:32:39.668 INFO:tasks.rados.rados.0.smithi022.stdout:458: finishing write tid 2 to smithi02284960-29 2024-09-21T09:32:39.668 INFO:tasks.rados.rados.0.smithi022.stdout:461: finishing rollback tid 2 to smithi02284960-44 2024-09-21T09:32:39.669 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 44 v 107 (ObjNum 161 snap 0 seq_num 1) dirty exists 2024-09-21T09:32:39.669 INFO:tasks.rados.rados.0.smithi022.stdout:458: finishing write tid 3 to smithi02284960-29 2024-09-21T09:32:39.669 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 29 v 109 (ObjNum 168 snap 41 seq_num 168) dirty exists 2024-09-21T09:32:39.670 INFO:tasks.rados.rados.0.smithi022.stdout:458: left oid 29 (ObjNum 168 snap 41 seq_num 168) 2024-09-21T09:32:39.670 INFO:tasks.rados.rados.0.smithi022.stdout:459: finishing copy_from racing read to smithi02284960-33 2024-09-21T09:32:39.670 INFO:tasks.rados.rados.0.smithi022.stdout:459: finishing copy_from to smithi02284960-33 2024-09-21T09:32:39.670 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 33 v 111 (ObjNum 148 snap 35 seq_num 148) dirty exists 2024-09-21T09:32:39.672 INFO:tasks.rados.rados.0.smithi022.stdout:457: expect (ObjNum 109 snap 23 seq_num 109) 2024-09-21T09:32:39.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:39 smithi022 ceph-mon[39525]: osdmap e147: 8 total, 8 up, 8 in 2024-09-21T09:32:39.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:39 smithi022 ceph-mon[39525]: pgmap v216: 121 pgs: 121 active+clean; 129 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 2.3 MiB/s rd, 164 KiB/s wr, 10 op/s 2024-09-21T09:32:39.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:39 smithi022 ceph-mon[27763]: osdmap e147: 8 total, 8 up, 8 in 2024-09-21T09:32:39.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:39 smithi022 ceph-mon[27763]: pgmap v216: 121 pgs: 121 active+clean; 129 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 2.3 MiB/s rd, 164 KiB/s wr, 10 op/s 2024-09-21T09:32:39.923 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:39 smithi064 ceph-mon[31124]: osdmap e147: 8 total, 8 up, 8 in 2024-09-21T09:32:39.923 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:39 smithi064 ceph-mon[31124]: pgmap v216: 121 pgs: 121 active+clean; 129 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 2.3 MiB/s rd, 164 KiB/s wr, 10 op/s 2024-09-21T09:32:40.428 INFO:tasks.rados.rados.0.smithi022.stdout:457: done (5 left) 2024-09-21T09:32:40.428 INFO:tasks.rados.rados.0.smithi022.stdout:458: done (4 left) 2024-09-21T09:32:40.428 INFO:tasks.rados.rados.0.smithi022.stdout:459: done (3 left) 2024-09-21T09:32:40.428 INFO:tasks.rados.rados.0.smithi022.stdout:460: done (2 left) 2024-09-21T09:32:40.428 INFO:tasks.rados.rados.0.smithi022.stdout:461: done (1 left) 2024-09-21T09:32:40.428 INFO:tasks.rados.rados.0.smithi022.stdout:462: done (0 left) 2024-09-21T09:32:40.428 INFO:tasks.rados.rados.0.smithi022.stdout:463: read oid 14 snap -1 2024-09-21T09:32:40.428 INFO:tasks.rados.rados.0.smithi022.stdout:463: expect (ObjNum 115 snap 23 seq_num 115) 2024-09-21T09:32:40.428 INFO:tasks.rados.rados.0.smithi022.stdout:464: rollback oid 34 current snap is 42 2024-09-21T09:32:40.428 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 34 to 27 2024-09-21T09:32:40.428 INFO:tasks.rados.rados.0.smithi022.stdout:465: delete oid 25 current snap is 42 2024-09-21T09:32:40.434 INFO:tasks.rados.rados.0.smithi022.stdout:465: done (2 left) 2024-09-21T09:32:40.434 INFO:tasks.rados.rados.0.smithi022.stdout:466: read oid 26 snap -1 2024-09-21T09:32:40.434 INFO:tasks.rados.rados.0.smithi022.stdout:466: expect (ObjNum 153 snap 36 seq_num 153) 2024-09-21T09:32:40.434 INFO:tasks.rados.rados.0.smithi022.stdout:467: rmattr oid 2 current snap is 42 2024-09-21T09:32:40.434 INFO:tasks.rados.rados.0.smithi022.stdout:468: setattr oid 32 current snap is 42 2024-09-21T09:32:40.434 INFO:tasks.rados.rados.0.smithi022.stdout:464: finishing rollback tid 1 to smithi02284960-34 2024-09-21T09:32:40.435 INFO:tasks.rados.rados.0.smithi022.stdout:464: finishing rollback tid 2 to smithi02284960-34 2024-09-21T09:32:40.435 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 34 v 186 (ObjNum 36 snap 0 seq_num 36) dirty exists 2024-09-21T09:32:40.435 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 2 v 187 (ObjNum 161 snap 0 seq_num 1) dirty exists 2024-09-21T09:32:40.435 INFO:tasks.rados.rados.0.smithi022.stdout:464: done (4 left) 2024-09-21T09:32:40.436 INFO:tasks.rados.rados.0.smithi022.stdout:467: done (3 left) 2024-09-21T09:32:40.436 INFO:tasks.rados.rados.0.smithi022.stdout:469: snap_create 2024-09-21T09:32:40.440 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 32 v 227 (ObjNum 143 snap 32 seq_num 143) dirty exists 2024-09-21T09:32:40.450 INFO:tasks.rados.rados.0.smithi022.stdout:466: expect (ObjNum 153 snap 36 seq_num 153) 2024-09-21T09:32:40.538 INFO:tasks.rados.rados.0.smithi022.stdout:463: expect (ObjNum 115 snap 23 seq_num 115) 2024-09-21T09:32:40.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:40 smithi022 ceph-mon[39525]: osdmap e148: 8 total, 8 up, 8 in 2024-09-21T09:32:40.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:40 smithi022 ceph-mon[39525]: osdmap e149: 8 total, 8 up, 8 in 2024-09-21T09:32:40.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:40 smithi022 ceph-mon[27763]: osdmap e148: 8 total, 8 up, 8 in 2024-09-21T09:32:40.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:40 smithi022 ceph-mon[27763]: osdmap e149: 8 total, 8 up, 8 in 2024-09-21T09:32:40.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:40 smithi064 ceph-mon[31124]: osdmap e148: 8 total, 8 up, 8 in 2024-09-21T09:32:40.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:40 smithi064 ceph-mon[31124]: osdmap e149: 8 total, 8 up, 8 in 2024-09-21T09:32:41.428 INFO:tasks.rados.rados.0.smithi022.stdout:463: done (3 left) 2024-09-21T09:32:41.428 INFO:tasks.rados.rados.0.smithi022.stdout:466: done (2 left) 2024-09-21T09:32:41.429 INFO:tasks.rados.rados.0.smithi022.stdout:468: done (1 left) 2024-09-21T09:32:41.429 INFO:tasks.rados.rados.0.smithi022.stdout:469: done (0 left) 2024-09-21T09:32:41.429 INFO:tasks.rados.rados.0.smithi022.stdout:470: setattr oid 39 current snap is 43 2024-09-21T09:32:41.430 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 22 current snap is 43 2024-09-21T09:32:41.430 INFO:tasks.rados.rados.0.smithi022.stdout:471: seq_num 170 ranges {360448=450560} 2024-09-21T09:32:41.435 INFO:tasks.rados.rados.0.smithi022.stdout:471: writing smithi02284960-22 from 360448 to 811008 tid 1 2024-09-21T09:32:41.435 INFO:tasks.rados.rados.0.smithi022.stdout:472: snap_remove snap 25 2024-09-21T09:32:41.518 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:32:41 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:32:41] "GET /metrics HTTP/1.1" 200 240054 "" "Prometheus/2.43.0" 2024-09-21T09:32:41.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:41 smithi022 ceph-mon[39525]: pgmap v219: 121 pgs: 121 active+clean; 129 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 2.3 MiB/s rd, 164 KiB/s wr, 10 op/s 2024-09-21T09:32:41.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:41 smithi022 ceph-mon[39525]: osdmap e150: 8 total, 8 up, 8 in 2024-09-21T09:32:41.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:41 smithi022 ceph-mon[27763]: pgmap v219: 121 pgs: 121 active+clean; 129 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 2.3 MiB/s rd, 164 KiB/s wr, 10 op/s 2024-09-21T09:32:41.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:41 smithi022 ceph-mon[27763]: osdmap e150: 8 total, 8 up, 8 in 2024-09-21T09:32:41.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:41 smithi064 ceph-mon[31124]: pgmap v219: 121 pgs: 121 active+clean; 129 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 2.3 MiB/s rd, 164 KiB/s wr, 10 op/s 2024-09-21T09:32:41.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:41 smithi064 ceph-mon[31124]: osdmap e150: 8 total, 8 up, 8 in 2024-09-21T09:32:41.942 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:32:41 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[67503]: [21/Sep/2024:09:32:41] ENGINE Bus STARTING 2024-09-21T09:32:41.942 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:32:41 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[67503]: CherryPy Checker: 2024-09-21T09:32:41.942 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:32:41 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[67503]: The Application mounted at '' has an empty config. 2024-09-21T09:32:41.942 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:32:41 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[67503]: 2024-09-21T09:32:41.942 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:32:41 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[67503]: [21/Sep/2024:09:32:41] ENGINE Serving on http://:::9283 2024-09-21T09:32:41.942 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:32:41 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[67503]: [21/Sep/2024:09:32:41] ENGINE Bus STARTED 2024-09-21T09:32:42.432 INFO:tasks.rados.rados.0.smithi022.stdout:472: done (2 left) 2024-09-21T09:32:42.433 INFO:tasks.rados.rados.0.smithi022.stdout:473: rmattr oid 7 current snap is 43 2024-09-21T09:32:42.433 INFO:tasks.rados.rados.0.smithi022.stdout:473: done (2 left) 2024-09-21T09:32:42.433 INFO:tasks.rados.rados.0.smithi022.stdout:474: copy_from oid 19 from oid 37 current snap is 43 2024-09-21T09:32:42.433 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 39 v 113 (ObjNum 151 snap 36 seq_num 151) dirty exists 2024-09-21T09:32:42.433 INFO:tasks.rados.rados.0.smithi022.stdout:471: finishing write tid 1 to smithi02284960-22 2024-09-21T09:32:42.433 INFO:tasks.rados.rados.0.smithi022.stdout:471: finishing write tid 2 to smithi02284960-22 2024-09-21T09:32:42.433 INFO:tasks.rados.rados.0.smithi022.stdout:471: finishing write tid 3 to smithi02284960-22 2024-09-21T09:32:42.433 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 22 v 182 (ObjNum 170 snap 43 seq_num 170) dirty exists 2024-09-21T09:32:42.433 INFO:tasks.rados.rados.0.smithi022.stdout:471: left oid 22 (ObjNum 170 snap 43 seq_num 170) 2024-09-21T09:32:42.434 INFO:tasks.rados.rados.0.smithi022.stdout:470: done (2 left) 2024-09-21T09:32:42.434 INFO:tasks.rados.rados.0.smithi022.stdout:471: done (1 left) 2024-09-21T09:32:42.434 INFO:tasks.rados.rados.0.smithi022.stdout:475: copy_from oid 48 from oid 31 current snap is 43 2024-09-21T09:32:42.434 INFO:tasks.rados.rados.0.smithi022.stdout:476: delete oid 16 current snap is 43 2024-09-21T09:32:42.445 INFO:tasks.rados.rados.0.smithi022.stdout:476: done (2 left) 2024-09-21T09:32:42.445 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 21 current snap is 43 2024-09-21T09:32:42.445 INFO:tasks.rados.rados.0.smithi022.stdout:477: seq_num 172 ranges {0=483328} 2024-09-21T09:32:42.449 INFO:tasks.rados.rados.0.smithi022.stdout:477: writing smithi02284960-21 from 0 to 483328 tid 1 2024-09-21T09:32:42.449 INFO:tasks.rados.rados.0.smithi022.stdout:478: delete oid 39 current snap is 43 2024-09-21T09:32:42.451 INFO:tasks.rados.rados.0.smithi022.stdout:474: finishing copy_from racing read to smithi02284960-19 2024-09-21T09:32:42.452 INFO:tasks.rados.rados.0.smithi022.stdout:478: done (3 left) 2024-09-21T09:32:42.452 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 3 current snap is 43 2024-09-21T09:32:42.452 INFO:tasks.rados.rados.0.smithi022.stdout:479: seq_num 174 ranges {1835008=516096} 2024-09-21T09:32:42.466 INFO:tasks.rados.rados.0.smithi022.stdout:479: writing smithi02284960-3 from 1835008 to 2351104 tid 1 2024-09-21T09:32:42.466 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 12 current snap is 43 2024-09-21T09:32:42.466 INFO:tasks.rados.rados.0.smithi022.stdout:474: finishing copy_from to smithi02284960-19 2024-09-21T09:32:42.466 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 19 v 201 (ObjNum 155 snap 36 seq_num 155) dirty exists 2024-09-21T09:32:42.466 INFO:tasks.rados.rados.0.smithi022.stdout:480: seq_num 175 ranges {5750784=540672} 2024-09-21T09:32:42.500 INFO:tasks.rados.rados.0.smithi022.stdout:480: writing smithi02284960-12 from 5750784 to 6291456 tid 1 2024-09-21T09:32:42.500 INFO:tasks.rados.rados.0.smithi022.stdout:474: done (4 left) 2024-09-21T09:32:42.500 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 16 current snap is 43 2024-09-21T09:32:42.500 INFO:tasks.rados.rados.0.smithi022.stdout:481: seq_num 176 ranges {0=557056} 2024-09-21T09:32:42.504 INFO:tasks.rados.rados.0.smithi022.stdout:481: writing smithi02284960-16 from 0 to 557056 tid 1 2024-09-21T09:32:42.504 INFO:tasks.rados.rados.0.smithi022.stdout:482: read oid 26 snap -1 2024-09-21T09:32:42.504 INFO:tasks.rados.rados.0.smithi022.stdout:482: expect (ObjNum 153 snap 36 seq_num 153) 2024-09-21T09:32:42.504 INFO:tasks.rados.rados.0.smithi022.stdout:483: read oid 34 snap -1 2024-09-21T09:32:42.505 INFO:tasks.rados.rados.0.smithi022.stdout:483: expect (ObjNum 36 snap 0 seq_num 36) 2024-09-21T09:32:42.505 INFO:tasks.rados.rados.0.smithi022.stdout:484: rollback oid 7 current snap is 43 2024-09-21T09:32:42.505 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 7 to 41 2024-09-21T09:32:42.505 INFO:tasks.rados.rados.0.smithi022.stdout:477: finishing write tid 1 to smithi02284960-21 2024-09-21T09:32:42.505 INFO:tasks.rados.rados.0.smithi022.stdout:477: finishing write tid 2 to smithi02284960-21 2024-09-21T09:32:42.505 INFO:tasks.rados.rados.0.smithi022.stdout:477: finishing write tid 3 to smithi02284960-21 2024-09-21T09:32:42.505 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 21 v 189 (ObjNum 172 snap 43 seq_num 172) dirty exists 2024-09-21T09:32:42.505 INFO:tasks.rados.rados.0.smithi022.stdout:477: left oid 21 (ObjNum 172 snap 43 seq_num 172) 2024-09-21T09:32:42.505 INFO:tasks.rados.rados.0.smithi022.stdout:475: finishing copy_from racing read to smithi02284960-48 2024-09-21T09:32:42.505 INFO:tasks.rados.rados.0.smithi022.stdout:475: finishing copy_from to smithi02284960-48 2024-09-21T09:32:42.505 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 48 v 197 (ObjNum 149 snap 35 seq_num 149) dirty exists 2024-09-21T09:32:42.505 INFO:tasks.rados.rados.0.smithi022.stdout:479: finishing write tid 1 to smithi02284960-3 2024-09-21T09:32:42.505 INFO:tasks.rados.rados.0.smithi022.stdout:475: done (7 left) 2024-09-21T09:32:42.505 INFO:tasks.rados.rados.0.smithi022.stdout:477: done (6 left) 2024-09-21T09:32:42.506 INFO:tasks.rados.rados.0.smithi022.stdout:485: snap_create 2024-09-21T09:32:42.506 INFO:tasks.rados.rados.0.smithi022.stdout:479: finishing write tid 2 to smithi02284960-3 2024-09-21T09:32:42.506 INFO:tasks.rados.rados.0.smithi022.stdout:479: finishing write tid 3 to smithi02284960-3 2024-09-21T09:32:42.506 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 3 v 53 (ObjNum 174 snap 43 seq_num 174) dirty exists 2024-09-21T09:32:42.506 INFO:tasks.rados.rados.0.smithi022.stdout:479: left oid 3 (ObjNum 174 snap 43 seq_num 174) 2024-09-21T09:32:42.506 INFO:tasks.rados.rados.0.smithi022.stdout:480: finishing write tid 1 to smithi02284960-12 2024-09-21T09:32:42.506 INFO:tasks.rados.rados.0.smithi022.stdout:480: finishing write tid 2 to smithi02284960-12 2024-09-21T09:32:42.507 INFO:tasks.rados.rados.0.smithi022.stdout:480: finishing write tid 3 to smithi02284960-12 2024-09-21T09:32:42.507 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 12 v 257 (ObjNum 175 snap 43 seq_num 175) dirty exists 2024-09-21T09:32:42.507 INFO:tasks.rados.rados.0.smithi022.stdout:480: left oid 12 (ObjNum 175 snap 43 seq_num 175) 2024-09-21T09:32:42.507 INFO:tasks.rados.rados.0.smithi022.stdout:484: finishing rollback tid 0 to smithi02284960-7 2024-09-21T09:32:42.507 INFO:tasks.rados.rados.0.smithi022.stdout:484: finishing rollback tid 1 to smithi02284960-7 2024-09-21T09:32:42.507 INFO:tasks.rados.rados.0.smithi022.stdout:481: finishing write tid 1 to smithi02284960-16 2024-09-21T09:32:42.507 INFO:tasks.rados.rados.0.smithi022.stdout:484: finishing rollback tid 2 to smithi02284960-7 2024-09-21T09:32:42.507 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 7 v 243 (ObjNum 72 snap 5 seq_num 72) dirty exists 2024-09-21T09:32:42.507 INFO:tasks.rados.rados.0.smithi022.stdout:481: finishing write tid 2 to smithi02284960-16 2024-09-21T09:32:42.509 INFO:tasks.rados.rados.0.smithi022.stdout:483: expect (ObjNum 36 snap 0 seq_num 36) 2024-09-21T09:32:42.527 INFO:tasks.rados.rados.0.smithi022.stdout:481: finishing write tid 3 to smithi02284960-16 2024-09-21T09:32:42.527 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 16 v 82 (ObjNum 176 snap 43 seq_num 176) dirty exists 2024-09-21T09:32:42.527 INFO:tasks.rados.rados.0.smithi022.stdout:481: left oid 16 (ObjNum 176 snap 43 seq_num 176) 2024-09-21T09:32:42.527 INFO:tasks.rados.rados.0.smithi022.stdout:482: expect (ObjNum 153 snap 36 seq_num 153) 2024-09-21T09:32:42.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:42 smithi022 ceph-mon[39525]: Standby manager daemon x restarted 2024-09-21T09:32:42.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:42 smithi022 ceph-mon[39525]: Standby manager daemon x started 2024-09-21T09:32:42.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:42 smithi022 ceph-mon[39525]: from='mgr.? 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-21T09:32:42.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:42 smithi022 ceph-mon[39525]: from='mgr.? 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:32:42.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:42 smithi022 ceph-mon[39525]: from='mgr.? 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-21T09:32:42.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:42 smithi022 ceph-mon[39525]: from='mgr.? 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:32:42.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:42 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:32:42.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:42 smithi022 ceph-mon[39525]: osdmap e151: 8 total, 8 up, 8 in 2024-09-21T09:32:42.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:42 smithi022 ceph-mon[27763]: Standby manager daemon x restarted 2024-09-21T09:32:42.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:42 smithi022 ceph-mon[27763]: Standby manager daemon x started 2024-09-21T09:32:42.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:42 smithi022 ceph-mon[27763]: from='mgr.? 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-21T09:32:42.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:42 smithi022 ceph-mon[27763]: from='mgr.? 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:32:42.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:42 smithi022 ceph-mon[27763]: from='mgr.? 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-21T09:32:42.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:42 smithi022 ceph-mon[27763]: from='mgr.? 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:32:42.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:42 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:32:42.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:42 smithi022 ceph-mon[27763]: osdmap e151: 8 total, 8 up, 8 in 2024-09-21T09:32:42.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:42 smithi064 ceph-mon[31124]: Standby manager daemon x restarted 2024-09-21T09:32:42.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:42 smithi064 ceph-mon[31124]: Standby manager daemon x started 2024-09-21T09:32:42.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:42 smithi064 ceph-mon[31124]: from='mgr.? 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-21T09:32:42.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:42 smithi064 ceph-mon[31124]: from='mgr.? 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:32:42.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:42 smithi064 ceph-mon[31124]: from='mgr.? 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-21T09:32:42.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:42 smithi064 ceph-mon[31124]: from='mgr.? 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:32:42.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:42 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:32:42.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:42 smithi064 ceph-mon[31124]: osdmap e151: 8 total, 8 up, 8 in 2024-09-21T09:32:43.433 INFO:tasks.rados.rados.0.smithi022.stdout:479: done (6 left) 2024-09-21T09:32:43.433 INFO:tasks.rados.rados.0.smithi022.stdout:480: done (5 left) 2024-09-21T09:32:43.434 INFO:tasks.rados.rados.0.smithi022.stdout:481: done (4 left) 2024-09-21T09:32:43.434 INFO:tasks.rados.rados.0.smithi022.stdout:482: done (3 left) 2024-09-21T09:32:43.434 INFO:tasks.rados.rados.0.smithi022.stdout:483: done (2 left) 2024-09-21T09:32:43.434 INFO:tasks.rados.rados.0.smithi022.stdout:484: done (1 left) 2024-09-21T09:32:43.434 INFO:tasks.rados.rados.0.smithi022.stdout:485: done (0 left) 2024-09-21T09:32:43.434 INFO:tasks.rados.rados.0.smithi022.stdout:486: rollback oid 11 current snap is 44 2024-09-21T09:32:43.434 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 11 to 22 2024-09-21T09:32:43.434 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 17 current snap is 44 2024-09-21T09:32:43.434 INFO:tasks.rados.rados.0.smithi022.stdout:487: seq_num 177 ranges {0=573440} 2024-09-21T09:32:43.437 INFO:tasks.rados.rados.0.smithi022.stdout:487: writing smithi02284960-17 from 0 to 573440 tid 1 2024-09-21T09:32:43.437 INFO:tasks.rados.rados.0.smithi022.stdout:488: rollback oid 47 current snap is 44 2024-09-21T09:32:43.437 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 47 to 22 2024-09-21T09:32:43.437 INFO:tasks.rados.rados.0.smithi022.stdout:489: snap_remove snap 39 2024-09-21T09:32:43.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:43 smithi064 ceph-mon[31124]: mgrmap e23: y(active, since 4m), standbys: x 2024-09-21T09:32:43.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:43 smithi064 ceph-mon[31124]: pgmap v222: 121 pgs: 121 active+clean; 131 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.1 MiB/s rd, 2.8 MiB/s wr, 34 op/s 2024-09-21T09:32:43.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:43 smithi064 ceph-mon[31124]: osdmap e152: 8 total, 8 up, 8 in 2024-09-21T09:32:43.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:43 smithi022 ceph-mon[39525]: mgrmap e23: y(active, since 4m), standbys: x 2024-09-21T09:32:43.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:43 smithi022 ceph-mon[39525]: pgmap v222: 121 pgs: 121 active+clean; 131 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.1 MiB/s rd, 2.8 MiB/s wr, 34 op/s 2024-09-21T09:32:43.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:43 smithi022 ceph-mon[39525]: osdmap e152: 8 total, 8 up, 8 in 2024-09-21T09:32:43.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:43 smithi022 ceph-mon[27763]: mgrmap e23: y(active, since 4m), standbys: x 2024-09-21T09:32:43.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:43 smithi022 ceph-mon[27763]: pgmap v222: 121 pgs: 121 active+clean; 131 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.1 MiB/s rd, 2.8 MiB/s wr, 34 op/s 2024-09-21T09:32:43.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:43 smithi022 ceph-mon[27763]: osdmap e152: 8 total, 8 up, 8 in 2024-09-21T09:32:44.436 INFO:tasks.rados.rados.0.smithi022.stdout:489: done (3 left) 2024-09-21T09:32:44.436 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 18 current snap is 44 2024-09-21T09:32:44.436 INFO:tasks.rados.rados.0.smithi022.stdout:490: seq_num 178 ranges {286720=589824} 2024-09-21T09:32:44.442 INFO:tasks.rados.rados.0.smithi022.stdout:490: writing smithi02284960-18 from 286720 to 876544 tid 1 2024-09-21T09:32:44.442 INFO:tasks.rados.rados.0.smithi022.stdout:491: delete oid 2 current snap is 44 2024-09-21T09:32:44.442 INFO:tasks.rados.rados.0.smithi022.stdout:486: finishing rollback tid 0 to smithi02284960-11 2024-09-21T09:32:44.442 INFO:tasks.rados.rados.0.smithi022.stdout:487: finishing write tid 1 to smithi02284960-17 2024-09-21T09:32:44.442 INFO:tasks.rados.rados.0.smithi022.stdout:486: finishing rollback tid 1 to smithi02284960-11 2024-09-21T09:32:44.442 INFO:tasks.rados.rados.0.smithi022.stdout:487: finishing write tid 2 to smithi02284960-17 2024-09-21T09:32:44.442 INFO:tasks.rados.rados.0.smithi022.stdout:486: finishing rollback tid 2 to smithi02284960-11 2024-09-21T09:32:44.443 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 11 v 110 (ObjNum 103 snap 19 seq_num 103) dirty exists 2024-09-21T09:32:44.443 INFO:tasks.rados.rados.0.smithi022.stdout:488: finishing rollback tid 0 to smithi02284960-47 2024-09-21T09:32:44.443 INFO:tasks.rados.rados.0.smithi022.stdout:488: finishing rollback tid 1 to smithi02284960-47 2024-09-21T09:32:44.443 INFO:tasks.rados.rados.0.smithi022.stdout:488: finishing rollback tid 2 to smithi02284960-47 2024-09-21T09:32:44.443 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 47 v 113 (ObjNum 97 snap 0 seq_num 3569725984) dirty exists 2024-09-21T09:32:44.443 INFO:tasks.rados.rados.0.smithi022.stdout:487: finishing write tid 3 to smithi02284960-17 2024-09-21T09:32:44.443 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 17 v 122 (ObjNum 177 snap 44 seq_num 177) dirty exists 2024-09-21T09:32:44.443 INFO:tasks.rados.rados.0.smithi022.stdout:487: left oid 17 (ObjNum 177 snap 44 seq_num 177) 2024-09-21T09:32:44.448 INFO:tasks.rados.rados.0.smithi022.stdout:490: finishing write tid 1 to smithi02284960-18 2024-09-21T09:32:44.450 INFO:tasks.rados.rados.0.smithi022.stdout:490: finishing write tid 2 to smithi02284960-18 2024-09-21T09:32:44.451 INFO:tasks.rados.rados.0.smithi022.stdout:486: done (4 left) 2024-09-21T09:32:44.451 INFO:tasks.rados.rados.0.smithi022.stdout:487: done (3 left) 2024-09-21T09:32:44.451 INFO:tasks.rados.rados.0.smithi022.stdout:488: done (2 left) 2024-09-21T09:32:44.451 INFO:tasks.rados.rados.0.smithi022.stdout:491: done (1 left) 2024-09-21T09:32:44.451 INFO:tasks.rados.rados.0.smithi022.stdout:492: snap_create 2024-09-21T09:32:44.451 INFO:tasks.rados.rados.0.smithi022.stdout:490: finishing write tid 3 to smithi02284960-18 2024-09-21T09:32:44.451 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 18 v 87 (ObjNum 178 snap 44 seq_num 178) dirty exists 2024-09-21T09:32:44.451 INFO:tasks.rados.rados.0.smithi022.stdout:490: left oid 18 (ObjNum 178 snap 44 seq_num 178) 2024-09-21T09:32:45.429 INFO:tasks.rados.rados.0.smithi022.stdout:490: done (1 left) 2024-09-21T09:32:45.429 INFO:tasks.rados.rados.0.smithi022.stdout:492: done (0 left) 2024-09-21T09:32:45.429 INFO:tasks.rados.rados.0.smithi022.stdout:493: snap_create 2024-09-21T09:32:45.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:45 smithi064 ceph-mon[31124]: osdmap e153: 8 total, 8 up, 8 in 2024-09-21T09:32:45.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:45 smithi064 ceph-mon[31124]: pgmap v225: 121 pgs: 121 active+clean; 131 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.1 MiB/s rd, 2.8 MiB/s wr, 34 op/s 2024-09-21T09:32:45.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:45 smithi064 ceph-mon[31124]: osdmap e154: 8 total, 8 up, 8 in 2024-09-21T09:32:45.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:45 smithi022 ceph-mon[39525]: osdmap e153: 8 total, 8 up, 8 in 2024-09-21T09:32:45.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:45 smithi022 ceph-mon[39525]: pgmap v225: 121 pgs: 121 active+clean; 131 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.1 MiB/s rd, 2.8 MiB/s wr, 34 op/s 2024-09-21T09:32:45.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:45 smithi022 ceph-mon[39525]: osdmap e154: 8 total, 8 up, 8 in 2024-09-21T09:32:45.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:45 smithi022 ceph-mon[27763]: osdmap e153: 8 total, 8 up, 8 in 2024-09-21T09:32:45.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:45 smithi022 ceph-mon[27763]: pgmap v225: 121 pgs: 121 active+clean; 131 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.1 MiB/s rd, 2.8 MiB/s wr, 34 op/s 2024-09-21T09:32:45.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:45 smithi022 ceph-mon[27763]: osdmap e154: 8 total, 8 up, 8 in 2024-09-21T09:32:46.430 INFO:tasks.rados.rados.0.smithi022.stdout:493: done (0 left) 2024-09-21T09:32:46.430 INFO:tasks.rados.rados.0.smithi022.stdout:494: snap_remove snap 27 2024-09-21T09:32:47.433 INFO:tasks.rados.rados.0.smithi022.stdout:494: done (0 left) 2024-09-21T09:32:47.433 INFO:tasks.rados.rados.0.smithi022.stdout:495: read oid 21 snap -1 2024-09-21T09:32:47.433 INFO:tasks.rados.rados.0.smithi022.stdout:495: expect (ObjNum 172 snap 43 seq_num 172) 2024-09-21T09:32:47.433 INFO:tasks.rados.rados.0.smithi022.stdout:496: snap_create 2024-09-21T09:32:47.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:47 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:32:47.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:47 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:32:47.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:47 smithi064 ceph-mon[31124]: osdmap e155: 8 total, 8 up, 8 in 2024-09-21T09:32:47.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:47 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:32:47.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:47 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:32:47.445 INFO:tasks.rados.rados.0.smithi022.stdout:495: expect (ObjNum 172 snap 43 seq_num 172) 2024-09-21T09:32:47.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:47 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:32:47.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:47 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:32:47.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:47 smithi022 ceph-mon[39525]: osdmap e155: 8 total, 8 up, 8 in 2024-09-21T09:32:47.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:47 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:32:47.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:47 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:32:47.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:47 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:32:47.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:47 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:32:47.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:47 smithi022 ceph-mon[27763]: osdmap e155: 8 total, 8 up, 8 in 2024-09-21T09:32:47.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:47 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:32:47.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:47 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:32:48.434 INFO:tasks.rados.rados.0.smithi022.stdout:495: done (1 left) 2024-09-21T09:32:48.434 INFO:tasks.rados.rados.0.smithi022.stdout:496: done (0 left) 2024-09-21T09:32:48.434 INFO:tasks.rados.rados.0.smithi022.stdout:497: copy_from oid 24 from oid 25 current snap is 47 2024-09-21T09:32:48.434 INFO:tasks.rados.rados.0.smithi022.stdout:498: snap_remove snap 45 2024-09-21T09:32:48.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:48 smithi022 ceph-mon[27763]: pgmap v228: 121 pgs: 121 active+clean; 131 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:32:48.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:48 smithi022 ceph-mon[27763]: osdmap e156: 8 total, 8 up, 8 in 2024-09-21T09:32:48.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:48 smithi022 ceph-mon[39525]: pgmap v228: 121 pgs: 121 active+clean; 131 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:32:48.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:48 smithi022 ceph-mon[39525]: osdmap e156: 8 total, 8 up, 8 in 2024-09-21T09:32:48.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:48 smithi064 ceph-mon[31124]: pgmap v228: 121 pgs: 121 active+clean; 131 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:32:48.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:48 smithi064 ceph-mon[31124]: osdmap e156: 8 total, 8 up, 8 in 2024-09-21T09:32:49.440 INFO:tasks.rados.rados.0.smithi022.stdout:498: done (1 left) 2024-09-21T09:32:49.441 INFO:tasks.rados.rados.0.smithi022.stdout:499: rollback oid 16 current snap is 47 2024-09-21T09:32:49.441 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 16 to 41 2024-09-21T09:32:49.441 INFO:tasks.rados.rados.0.smithi022.stdout:497: finishing copy_from to smithi02284960-24 2024-09-21T09:32:49.441 INFO:tasks.rados.rados.0.smithi022.stdout:497: got expected ENOENT (src dne) 2024-09-21T09:32:49.441 INFO:tasks.rados.rados.0.smithi022.stdout:497: finishing copy_from racing read to smithi02284960-24 2024-09-21T09:32:49.441 INFO:tasks.rados.rados.0.smithi022.stdout:497: done (1 left) 2024-09-21T09:32:49.441 INFO:tasks.rados.rados.0.smithi022.stdout:500: read oid 47 snap 32 2024-09-21T09:32:49.441 INFO:tasks.rados.rados.0.smithi022.stdout:500: expect (ObjNum 109 snap 23 seq_num 109) 2024-09-21T09:32:49.441 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 10 current snap is 47 2024-09-21T09:32:49.441 INFO:tasks.rados.rados.0.smithi022.stdout:501: seq_num 180 ranges {16384=622592} 2024-09-21T09:32:49.445 INFO:tasks.rados.rados.0.smithi022.stdout:501: writing smithi02284960-10 from 16384 to 638976 tid 1 2024-09-21T09:32:49.445 INFO:tasks.rados.rados.0.smithi022.stdout:502: copy_from oid 8 from oid 22 current snap is 47 2024-09-21T09:32:49.445 INFO:tasks.rados.rados.0.smithi022.stdout:499: finishing rollback tid 0 to smithi02284960-16 2024-09-21T09:32:49.445 INFO:tasks.rados.rados.0.smithi022.stdout:503: delete oid 3 current snap is 47 2024-09-21T09:32:49.447 INFO:tasks.rados.rados.0.smithi022.stdout:499: finishing rollback tid 1 to smithi02284960-16 2024-09-21T09:32:49.448 INFO:tasks.rados.rados.0.smithi022.stdout:499: finishing rollback tid 2 to smithi02284960-16 2024-09-21T09:32:49.448 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 16 v 95 (ObjNum 166 snap 41 seq_num 166) dirty exists 2024-09-21T09:32:49.449 INFO:tasks.rados.rados.0.smithi022.stdout:500: expect (ObjNum 109 snap 23 seq_num 109) 2024-09-21T09:32:49.503 INFO:tasks.rados.rados.0.smithi022.stdout:501: finishing write tid 1 to smithi02284960-10 2024-09-21T09:32:49.504 INFO:tasks.rados.rados.0.smithi022.stdout:501: finishing write tid 2 to smithi02284960-10 2024-09-21T09:32:49.504 INFO:tasks.rados.rados.0.smithi022.stdout:502: finishing copy_from racing read to smithi02284960-8 2024-09-21T09:32:49.504 INFO:tasks.rados.rados.0.smithi022.stdout:502: finishing copy_from to smithi02284960-8 2024-09-21T09:32:49.504 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 8 v 182 (ObjNum 170 snap 43 seq_num 170) dirty exists 2024-09-21T09:32:49.504 INFO:tasks.rados.rados.0.smithi022.stdout:499: done (4 left) 2024-09-21T09:32:49.504 INFO:tasks.rados.rados.0.smithi022.stdout:500: done (3 left) 2024-09-21T09:32:49.504 INFO:tasks.rados.rados.0.smithi022.stdout:502: done (2 left) 2024-09-21T09:32:49.504 INFO:tasks.rados.rados.0.smithi022.stdout:503: done (1 left) 2024-09-21T09:32:49.504 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 7 current snap is 47 2024-09-21T09:32:49.504 INFO:tasks.rados.rados.0.smithi022.stdout:501: finishing write tid 3 to smithi02284960-10 2024-09-21T09:32:49.504 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 10 v 228 (ObjNum 180 snap 47 seq_num 180) dirty exists 2024-09-21T09:32:49.504 INFO:tasks.rados.rados.0.smithi022.stdout:501: left oid 10 (ObjNum 180 snap 47 seq_num 180) 2024-09-21T09:32:49.504 INFO:tasks.rados.rados.0.smithi022.stdout:504: seq_num 182 ranges {1318912=655360} 2024-09-21T09:32:49.514 INFO:tasks.rados.rados.0.smithi022.stdout:504: writing smithi02284960-7 from 1318912 to 1974272 tid 1 2024-09-21T09:32:49.514 INFO:tasks.rados.rados.0.smithi022.stdout:501: done (1 left) 2024-09-21T09:32:49.514 INFO:tasks.rados.rados.0.smithi022.stdout:505: rollback oid 8 current snap is 47 2024-09-21T09:32:49.514 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 8 to 34 2024-09-21T09:32:49.515 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 42 current snap is 47 2024-09-21T09:32:49.515 INFO:tasks.rados.rados.0.smithi022.stdout:506: seq_num 183 ranges {0=671744} 2024-09-21T09:32:49.519 INFO:tasks.rados.rados.0.smithi022.stdout:506: writing smithi02284960-42 from 0 to 671744 tid 1 2024-09-21T09:32:49.519 INFO:tasks.rados.rados.0.smithi022.stdout:507: delete oid 48 current snap is 47 2024-09-21T09:32:49.519 INFO:tasks.rados.rados.0.smithi022.stdout:505: finishing rollback tid 0 to smithi02284960-8 2024-09-21T09:32:49.519 INFO:tasks.rados.rados.0.smithi022.stdout:505: finishing rollback tid 1 to smithi02284960-8 2024-09-21T09:32:49.519 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 8 v 184 (ObjNum 1265 snap 0 seq_num 3436565768) dirty dne 2024-09-21T09:32:49.519 INFO:tasks.rados.rados.0.smithi022.stdout:504: finishing write tid 1 to smithi02284960-7 2024-09-21T09:32:49.519 INFO:tasks.rados.rados.0.smithi022.stdout:504: finishing write tid 2 to smithi02284960-7 2024-09-21T09:32:49.519 INFO:tasks.rados.rados.0.smithi022.stdout:504: finishing write tid 3 to smithi02284960-7 2024-09-21T09:32:49.520 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 7 v 264 (ObjNum 182 snap 47 seq_num 182) dirty exists 2024-09-21T09:32:49.520 INFO:tasks.rados.rados.0.smithi022.stdout:504: left oid 7 (ObjNum 182 snap 47 seq_num 182) 2024-09-21T09:32:49.521 INFO:tasks.rados.rados.0.smithi022.stdout:504: done (3 left) 2024-09-21T09:32:49.521 INFO:tasks.rados.rados.0.smithi022.stdout:505: done (2 left) 2024-09-21T09:32:49.521 INFO:tasks.rados.rados.0.smithi022.stdout:507: done (1 left) 2024-09-21T09:32:49.521 INFO:tasks.rados.rados.0.smithi022.stdout:508: delete oid 10 current snap is 47 2024-09-21T09:32:49.522 INFO:tasks.rados.rados.0.smithi022.stdout:506: finishing write tid 1 to smithi02284960-42 2024-09-21T09:32:49.522 INFO:tasks.rados.rados.0.smithi022.stdout:506: finishing write tid 2 to smithi02284960-42 2024-09-21T09:32:49.523 INFO:tasks.rados.rados.0.smithi022.stdout:508: done (1 left) 2024-09-21T09:32:49.523 INFO:tasks.rados.rados.0.smithi022.stdout:509: rmattr oid 29 current snap is 47 2024-09-21T09:32:49.523 INFO:tasks.rados.rados.0.smithi022.stdout:509: done (1 left) 2024-09-21T09:32:49.523 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 37 current snap is 47 2024-09-21T09:32:49.523 INFO:tasks.rados.rados.0.smithi022.stdout:510: seq_num 186 ranges {204800=720896} 2024-09-21T09:32:49.528 INFO:tasks.rados.rados.0.smithi022.stdout:510: writing smithi02284960-37 from 204800 to 925696 tid 1 2024-09-21T09:32:49.528 INFO:tasks.rados.rados.0.smithi022.stdout:511: read oid 29 snap -1 2024-09-21T09:32:49.528 INFO:tasks.rados.rados.0.smithi022.stdout:511: expect (ObjNum 168 snap 41 seq_num 168) 2024-09-21T09:32:49.528 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 12 current snap is 47 2024-09-21T09:32:49.528 INFO:tasks.rados.rados.0.smithi022.stdout:506: finishing write tid 3 to smithi02284960-42 2024-09-21T09:32:49.528 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 42 v 198 (ObjNum 183 snap 47 seq_num 183) dirty exists 2024-09-21T09:32:49.528 INFO:tasks.rados.rados.0.smithi022.stdout:506: left oid 42 (ObjNum 183 snap 47 seq_num 183) 2024-09-21T09:32:49.528 INFO:tasks.rados.rados.0.smithi022.stdout:512: seq_num 187 ranges {6291456=737280} 2024-09-21T09:32:49.567 INFO:tasks.rados.rados.0.smithi022.stdout:512: writing smithi02284960-12 from 6291456 to 7028736 tid 1 2024-09-21T09:32:49.567 INFO:tasks.rados.rados.0.smithi022.stdout:506: done (3 left) 2024-09-21T09:32:49.567 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 36 current snap is 47 2024-09-21T09:32:49.567 INFO:tasks.rados.rados.0.smithi022.stdout:513: seq_num 188 ranges {843776=753664} 2024-09-21T09:32:49.575 INFO:tasks.rados.rados.0.smithi022.stdout:513: writing smithi02284960-36 from 843776 to 1597440 tid 1 2024-09-21T09:32:49.575 INFO:tasks.rados.rados.0.smithi022.stdout:514: snap_remove snap 44 2024-09-21T09:32:49.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:49 smithi064 ceph-mon[31124]: osdmap e157: 8 total, 8 up, 8 in 2024-09-21T09:32:49.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:49 smithi064 ceph-mon[31124]: pgmap v231: 121 pgs: 121 active+clean; 134 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 355 KiB/s rd, 284 KiB/s wr, 3 op/s 2024-09-21T09:32:49.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:49 smithi022 ceph-mon[39525]: osdmap e157: 8 total, 8 up, 8 in 2024-09-21T09:32:49.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:49 smithi022 ceph-mon[39525]: pgmap v231: 121 pgs: 121 active+clean; 134 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 355 KiB/s rd, 284 KiB/s wr, 3 op/s 2024-09-21T09:32:49.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:49 smithi022 ceph-mon[27763]: osdmap e157: 8 total, 8 up, 8 in 2024-09-21T09:32:49.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:49 smithi022 ceph-mon[27763]: pgmap v231: 121 pgs: 121 active+clean; 134 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 355 KiB/s rd, 284 KiB/s wr, 3 op/s 2024-09-21T09:32:50.431 INFO:tasks.rados.rados.0.smithi022.stdout:514: done (4 left) 2024-09-21T09:32:50.431 INFO:tasks.rados.rados.0.smithi022.stdout:515: snap_create 2024-09-21T09:32:50.431 INFO:tasks.rados.rados.0.smithi022.stdout:511: expect (ObjNum 168 snap 41 seq_num 168) 2024-09-21T09:32:50.445 INFO:tasks.rados.rados.0.smithi022.stdout:510: finishing write tid 1 to smithi02284960-37 2024-09-21T09:32:50.446 INFO:tasks.rados.rados.0.smithi022.stdout:510: finishing write tid 2 to smithi02284960-37 2024-09-21T09:32:50.446 INFO:tasks.rados.rados.0.smithi022.stdout:510: finishing write tid 3 to smithi02284960-37 2024-09-21T09:32:50.446 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 37 v 272 (ObjNum 186 snap 47 seq_num 186) dirty exists 2024-09-21T09:32:50.446 INFO:tasks.rados.rados.0.smithi022.stdout:510: left oid 37 (ObjNum 186 snap 47 seq_num 186) 2024-09-21T09:32:50.446 INFO:tasks.rados.rados.0.smithi022.stdout:512: finishing write tid 1 to smithi02284960-12 2024-09-21T09:32:50.446 INFO:tasks.rados.rados.0.smithi022.stdout:512: finishing write tid 2 to smithi02284960-12 2024-09-21T09:32:50.446 INFO:tasks.rados.rados.0.smithi022.stdout:512: finishing write tid 3 to smithi02284960-12 2024-09-21T09:32:50.446 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 12 v 275 (ObjNum 187 snap 47 seq_num 187) dirty exists 2024-09-21T09:32:50.446 INFO:tasks.rados.rados.0.smithi022.stdout:512: left oid 12 (ObjNum 187 snap 47 seq_num 187) 2024-09-21T09:32:50.446 INFO:tasks.rados.rados.0.smithi022.stdout:513: finishing write tid 1 to smithi02284960-36 2024-09-21T09:32:50.446 INFO:tasks.rados.rados.0.smithi022.stdout:513: finishing write tid 2 to smithi02284960-36 2024-09-21T09:32:50.446 INFO:tasks.rados.rados.0.smithi022.stdout:513: finishing write tid 3 to smithi02284960-36 2024-09-21T09:32:50.446 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 36 v 152 (ObjNum 188 snap 47 seq_num 188) dirty exists 2024-09-21T09:32:50.447 INFO:tasks.rados.rados.0.smithi022.stdout:513: left oid 36 (ObjNum 188 snap 47 seq_num 188) 2024-09-21T09:32:50.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:50 smithi064 ceph-mon[31124]: osdmap e158: 8 total, 8 up, 8 in 2024-09-21T09:32:50.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:50 smithi064 ceph-mon[31124]: osdmap e159: 8 total, 8 up, 8 in 2024-09-21T09:32:50.692 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:32:50 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[67503]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:32:50] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-21T09:32:50.709 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:50 smithi022 ceph-mon[39525]: osdmap e158: 8 total, 8 up, 8 in 2024-09-21T09:32:50.709 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:50 smithi022 ceph-mon[39525]: osdmap e159: 8 total, 8 up, 8 in 2024-09-21T09:32:50.709 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:50 smithi022 ceph-mon[27763]: osdmap e158: 8 total, 8 up, 8 in 2024-09-21T09:32:50.709 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:50 smithi022 ceph-mon[27763]: osdmap e159: 8 total, 8 up, 8 in 2024-09-21T09:32:51.432 INFO:tasks.rados.rados.0.smithi022.stdout:510: done (4 left) 2024-09-21T09:32:51.432 INFO:tasks.rados.rados.0.smithi022.stdout:511: done (3 left) 2024-09-21T09:32:51.432 INFO:tasks.rados.rados.0.smithi022.stdout:512: done (2 left) 2024-09-21T09:32:51.432 INFO:tasks.rados.rados.0.smithi022.stdout:513: done (1 left) 2024-09-21T09:32:51.432 INFO:tasks.rados.rados.0.smithi022.stdout:515: done (0 left) 2024-09-21T09:32:51.432 INFO:tasks.rados.rados.0.smithi022.stdout:516: snap_remove snap 22 2024-09-21T09:32:51.558 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:32:51 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:32:51] "GET /metrics HTTP/1.1" 200 240116 "" "Prometheus/2.43.0" 2024-09-21T09:32:51.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:51 smithi022 ceph-mon[39525]: pgmap v234: 121 pgs: 121 active+clean; 134 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 355 KiB/s rd, 284 KiB/s wr, 3 op/s 2024-09-21T09:32:51.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:51 smithi022 ceph-mon[39525]: osdmap e160: 8 total, 8 up, 8 in 2024-09-21T09:32:51.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:51 smithi022 ceph-mon[27763]: pgmap v234: 121 pgs: 121 active+clean; 134 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 355 KiB/s rd, 284 KiB/s wr, 3 op/s 2024-09-21T09:32:51.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:51 smithi022 ceph-mon[27763]: osdmap e160: 8 total, 8 up, 8 in 2024-09-21T09:32:51.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:51 smithi064 ceph-mon[31124]: pgmap v234: 121 pgs: 121 active+clean; 134 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 355 KiB/s rd, 284 KiB/s wr, 3 op/s 2024-09-21T09:32:51.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:51 smithi064 ceph-mon[31124]: osdmap e160: 8 total, 8 up, 8 in 2024-09-21T09:32:52.436 INFO:tasks.rados.rados.0.smithi022.stdout:516: done (0 left) 2024-09-21T09:32:52.436 INFO:tasks.rados.rados.0.smithi022.stdout:517: read oid 16 snap 42 2024-09-21T09:32:52.436 INFO:tasks.rados.rados.0.smithi022.stdout:517: expect (ObjNum 166 snap 41 seq_num 166) 2024-09-21T09:32:52.437 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 2 current snap is 48 2024-09-21T09:32:52.437 INFO:tasks.rados.rados.0.smithi022.stdout:518: seq_num 189 ranges {0=770048} 2024-09-21T09:32:52.442 INFO:tasks.rados.rados.0.smithi022.stdout:518: writing smithi02284960-2 from 0 to 770048 tid 1 2024-09-21T09:32:52.442 INFO:tasks.rados.rados.0.smithi022.stdout:519: setattr oid 49 current snap is 48 2024-09-21T09:32:52.444 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 45 current snap is 48 2024-09-21T09:32:52.444 INFO:tasks.rados.rados.0.smithi022.stdout:520: seq_num 190 ranges {1933312=786432} 2024-09-21T09:32:52.461 INFO:tasks.rados.rados.0.smithi022.stdout:520: writing smithi02284960-45 from 1933312 to 2719744 tid 1 2024-09-21T09:32:52.462 INFO:tasks.rados.rados.0.smithi022.stdout:521: read oid 35 snap -1 2024-09-21T09:32:52.462 INFO:tasks.rados.rados.0.smithi022.stdout:521: expect (ObjNum 79 snap 11 seq_num 79) 2024-09-21T09:32:52.462 INFO:tasks.rados.rados.0.smithi022.stdout:522: read oid 14 snap -1 2024-09-21T09:32:52.462 INFO:tasks.rados.rados.0.smithi022.stdout:522: expect (ObjNum 115 snap 23 seq_num 115) 2024-09-21T09:32:52.462 INFO:tasks.rados.rados.0.smithi022.stdout:523: delete oid 39 current snap is 48 2024-09-21T09:32:52.462 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 49 v 98 (ObjNum 833 snap 0 seq_num 3569769008) dirty exists 2024-09-21T09:32:52.462 INFO:tasks.rados.rados.0.smithi022.stdout:518: finishing write tid 1 to smithi02284960-2 2024-09-21T09:32:52.462 INFO:tasks.rados.rados.0.smithi022.stdout:518: finishing write tid 2 to smithi02284960-2 2024-09-21T09:32:52.462 INFO:tasks.rados.rados.0.smithi022.stdout:518: finishing write tid 3 to smithi02284960-2 2024-09-21T09:32:52.462 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 2 v 192 (ObjNum 189 snap 48 seq_num 189) dirty exists 2024-09-21T09:32:52.462 INFO:tasks.rados.rados.0.smithi022.stdout:518: left oid 2 (ObjNum 189 snap 48 seq_num 189) 2024-09-21T09:32:52.466 INFO:tasks.rados.rados.0.smithi022.stdout:518: done (6 left) 2024-09-21T09:32:52.466 INFO:tasks.rados.rados.0.smithi022.stdout:519: done (5 left) 2024-09-21T09:32:52.466 INFO:tasks.rados.rados.0.smithi022.stdout:523: done (4 left) 2024-09-21T09:32:52.466 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 20 current snap is 48 2024-09-21T09:32:52.466 INFO:tasks.rados.rados.0.smithi022.stdout:524: seq_num 192 ranges {4562944=417792,4980736=401408} 2024-09-21T09:32:52.493 INFO:tasks.rados.rados.0.smithi022.stdout:524: writing smithi02284960-20 from 4562944 to 4980736 tid 1 2024-09-21T09:32:52.496 INFO:tasks.rados.rados.0.smithi022.stdout:524: writing smithi02284960-20 from 4980736 to 5382144 tid 2 2024-09-21T09:32:52.496 INFO:tasks.rados.rados.0.smithi022.stdout:525: snap_create 2024-09-21T09:32:52.496 INFO:tasks.rados.rados.0.smithi022.stdout:520: finishing write tid 1 to smithi02284960-45 2024-09-21T09:32:52.496 INFO:tasks.rados.rados.0.smithi022.stdout:520: finishing write tid 2 to smithi02284960-45 2024-09-21T09:32:52.496 INFO:tasks.rados.rados.0.smithi022.stdout:520: finishing write tid 3 to smithi02284960-45 2024-09-21T09:32:52.496 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 45 v 163 (ObjNum 190 snap 48 seq_num 190) dirty exists 2024-09-21T09:32:52.496 INFO:tasks.rados.rados.0.smithi022.stdout:520: left oid 45 (ObjNum 190 snap 48 seq_num 190) 2024-09-21T09:32:52.496 INFO:tasks.rados.rados.0.smithi022.stdout:517: expect (ObjNum 166 snap 41 seq_num 166) 2024-09-21T09:32:52.769 INFO:tasks.rados.rados.0.smithi022.stdout:521: expect (ObjNum 79 snap 11 seq_num 79) 2024-09-21T09:32:52.828 INFO:tasks.rados.rados.0.smithi022.stdout:522: expect (ObjNum 115 snap 23 seq_num 115) 2024-09-21T09:32:52.882 INFO:tasks.rados.rados.0.smithi022.stdout:524: finishing write tid 1 to smithi02284960-20 2024-09-21T09:32:52.882 INFO:tasks.rados.rados.0.smithi022.stdout:524: finishing write tid 2 to smithi02284960-20 2024-09-21T09:32:52.882 INFO:tasks.rados.rados.0.smithi022.stdout:524: finishing write tid 3 to smithi02284960-20 2024-09-21T09:32:52.883 INFO:tasks.rados.rados.0.smithi022.stdout:524: finishing write tid 4 to smithi02284960-20 2024-09-21T09:32:52.883 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 20 v 286 (ObjNum 192 snap 48 seq_num 192) dirty exists 2024-09-21T09:32:52.883 INFO:tasks.rados.rados.0.smithi022.stdout:524: left oid 20 (ObjNum 192 snap 48 seq_num 192) 2024-09-21T09:32:53.439 INFO:tasks.rados.rados.0.smithi022.stdout:517: done (5 left) 2024-09-21T09:32:53.440 INFO:tasks.rados.rados.0.smithi022.stdout:520: done (4 left) 2024-09-21T09:32:53.440 INFO:tasks.rados.rados.0.smithi022.stdout:521: done (3 left) 2024-09-21T09:32:53.440 INFO:tasks.rados.rados.0.smithi022.stdout:522: done (2 left) 2024-09-21T09:32:53.440 INFO:tasks.rados.rados.0.smithi022.stdout:524: done (1 left) 2024-09-21T09:32:53.440 INFO:tasks.rados.rados.0.smithi022.stdout:525: done (0 left) 2024-09-21T09:32:53.440 INFO:tasks.rados.rados.0.smithi022.stdout:526: setattr oid 5 current snap is 49 2024-09-21T09:32:53.440 INFO:tasks.rados.rados.0.smithi022.stdout:527: rollback oid 50 current snap is 49 2024-09-21T09:32:53.440 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 50 to 41 2024-09-21T09:32:53.441 INFO:tasks.rados.rados.0.smithi022.stdout:528: setattr oid 47 current snap is 49 2024-09-21T09:32:53.441 INFO:tasks.rados.rados.0.smithi022.stdout:529: rollback oid 10 current snap is 49 2024-09-21T09:32:53.441 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 10 to 46 2024-09-21T09:32:53.441 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 11 current snap is 49 2024-09-21T09:32:53.441 INFO:tasks.rados.rados.0.smithi022.stdout:530: seq_num 193 ranges {1736704=434176,2170880=401408} 2024-09-21T09:32:53.453 INFO:tasks.rados.rados.0.smithi022.stdout:530: writing smithi02284960-11 from 1736704 to 2170880 tid 1 2024-09-21T09:32:53.456 INFO:tasks.rados.rados.0.smithi022.stdout:530: writing smithi02284960-11 from 2170880 to 2572288 tid 2 2024-09-21T09:32:53.456 INFO:tasks.rados.rados.0.smithi022.stdout:531: snap_remove snap 34 2024-09-21T09:32:53.500 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:53 smithi064 ceph-mon[31124]: osdmap e161: 8 total, 8 up, 8 in 2024-09-21T09:32:53.500 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:53 smithi064 ceph-mon[31124]: pgmap v237: 121 pgs: 121 active+clean; 131 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.9 MiB/s rd, 1.8 MiB/s wr, 19 op/s 2024-09-21T09:32:53.807 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:53 smithi022 ceph-mon[39525]: osdmap e161: 8 total, 8 up, 8 in 2024-09-21T09:32:53.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:53 smithi022 ceph-mon[39525]: pgmap v237: 121 pgs: 121 active+clean; 131 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.9 MiB/s rd, 1.8 MiB/s wr, 19 op/s 2024-09-21T09:32:53.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:53 smithi022 ceph-mon[27763]: osdmap e161: 8 total, 8 up, 8 in 2024-09-21T09:32:53.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:53 smithi022 ceph-mon[27763]: pgmap v237: 121 pgs: 121 active+clean; 131 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.9 MiB/s rd, 1.8 MiB/s wr, 19 op/s 2024-09-21T09:32:54.445 INFO:tasks.rados.rados.0.smithi022.stdout:531: done (5 left) 2024-09-21T09:32:54.446 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 23 current snap is 49 2024-09-21T09:32:54.446 INFO:tasks.rados.rados.0.smithi022.stdout:532: seq_num 194 ranges {0=458752,458752=401408} 2024-09-21T09:32:54.449 INFO:tasks.rados.rados.0.smithi022.stdout:532: writing smithi02284960-23 from 0 to 458752 tid 1 2024-09-21T09:32:54.454 INFO:tasks.rados.rados.0.smithi022.stdout:532: writing smithi02284960-23 from 458752 to 860160 tid 2 2024-09-21T09:32:54.454 INFO:tasks.rados.rados.0.smithi022.stdout:533: rollback oid 20 current snap is 49 2024-09-21T09:32:54.454 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 20 to 46 2024-09-21T09:32:54.454 INFO:tasks.rados.rados.0.smithi022.stdout:534: rmattr oid 26 current snap is 49 2024-09-21T09:32:54.454 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 5 v 185 (ObjNum 161 snap 0 seq_num 1) dirty exists 2024-09-21T09:32:54.454 INFO:tasks.rados.rados.0.smithi022.stdout:527: finishing rollback tid 0 to smithi02284960-50 2024-09-21T09:32:54.454 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 47 v 131 (ObjNum 65 snap 0 seq_num 1634100835) dirty exists 2024-09-21T09:32:54.454 INFO:tasks.rados.rados.0.smithi022.stdout:529: finishing rollback tid 1 to smithi02284960-10 2024-09-21T09:32:54.454 INFO:tasks.rados.rados.0.smithi022.stdout:527: finishing rollback tid 1 to smithi02284960-50 2024-09-21T09:32:54.454 INFO:tasks.rados.rados.0.smithi022.stdout:529: finishing rollback tid 2 to smithi02284960-10 2024-09-21T09:32:54.455 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 10 v 241 (ObjNum 144 snap 32 seq_num 144) dirty exists 2024-09-21T09:32:54.455 INFO:tasks.rados.rados.0.smithi022.stdout:527: finishing rollback tid 2 to smithi02284960-50 2024-09-21T09:32:54.455 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 50 v 167 (ObjNum 131 snap 27 seq_num 131) dirty exists 2024-09-21T09:32:54.455 INFO:tasks.rados.rados.0.smithi022.stdout:530: finishing write tid 1 to smithi02284960-11 2024-09-21T09:32:54.455 INFO:tasks.rados.rados.0.smithi022.stdout:530: finishing write tid 2 to smithi02284960-11 2024-09-21T09:32:54.455 INFO:tasks.rados.rados.0.smithi022.stdout:530: finishing write tid 3 to smithi02284960-11 2024-09-21T09:32:54.455 INFO:tasks.rados.rados.0.smithi022.stdout:530: finishing write tid 4 to smithi02284960-11 2024-09-21T09:32:54.455 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 11 v 135 (ObjNum 193 snap 49 seq_num 193) dirty exists 2024-09-21T09:32:54.455 INFO:tasks.rados.rados.0.smithi022.stdout:530: left oid 11 (ObjNum 193 snap 49 seq_num 193) 2024-09-21T09:32:54.455 INFO:tasks.rados.rados.0.smithi022.stdout:526: done (7 left) 2024-09-21T09:32:54.455 INFO:tasks.rados.rados.0.smithi022.stdout:527: done (6 left) 2024-09-21T09:32:54.455 INFO:tasks.rados.rados.0.smithi022.stdout:528: done (5 left) 2024-09-21T09:32:54.455 INFO:tasks.rados.rados.0.smithi022.stdout:529: done (4 left) 2024-09-21T09:32:54.455 INFO:tasks.rados.rados.0.smithi022.stdout:530: done (3 left) 2024-09-21T09:32:54.456 INFO:tasks.rados.rados.0.smithi022.stdout:535: read oid 18 snap -1 2024-09-21T09:32:54.456 INFO:tasks.rados.rados.0.smithi022.stdout:535: expect (ObjNum 178 snap 44 seq_num 178) 2024-09-21T09:32:54.456 INFO:tasks.rados.rados.0.smithi022.stdout:536: snap_create 2024-09-21T09:32:54.460 INFO:tasks.rados.rados.0.smithi022.stdout:532: finishing write tid 1 to smithi02284960-23 2024-09-21T09:32:54.462 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 26 v 212 (ObjNum 153 snap 36 seq_num 153) dirty exists 2024-09-21T09:32:54.463 INFO:tasks.rados.rados.0.smithi022.stdout:532: finishing write tid 2 to smithi02284960-23 2024-09-21T09:32:54.463 INFO:tasks.rados.rados.0.smithi022.stdout:532: finishing write tid 3 to smithi02284960-23 2024-09-21T09:32:54.465 INFO:tasks.rados.rados.0.smithi022.stdout:535: expect (ObjNum 178 snap 44 seq_num 178) 2024-09-21T09:32:54.494 INFO:tasks.rados.rados.0.smithi022.stdout:532: finishing write tid 4 to smithi02284960-23 2024-09-21T09:32:54.494 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 23 v 128 (ObjNum 194 snap 49 seq_num 194) dirty exists 2024-09-21T09:32:54.494 INFO:tasks.rados.rados.0.smithi022.stdout:532: left oid 23 (ObjNum 194 snap 49 seq_num 194) 2024-09-21T09:32:54.494 INFO:tasks.rados.rados.0.smithi022.stdout:533: finishing rollback tid 0 to smithi02284960-20 2024-09-21T09:32:54.495 INFO:tasks.rados.rados.0.smithi022.stdout:533: finishing rollback tid 1 to smithi02284960-20 2024-09-21T09:32:54.495 INFO:tasks.rados.rados.0.smithi022.stdout:533: finishing rollback tid 2 to smithi02284960-20 2024-09-21T09:32:54.495 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 20 v 298 (ObjNum 132 snap 27 seq_num 132) dirty exists 2024-09-21T09:32:54.559 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:54 smithi064 ceph-mon[31124]: osdmap e162: 8 total, 8 up, 8 in 2024-09-21T09:32:54.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:54 smithi022 ceph-mon[39525]: osdmap e162: 8 total, 8 up, 8 in 2024-09-21T09:32:54.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:54 smithi022 ceph-mon[27763]: osdmap e162: 8 total, 8 up, 8 in 2024-09-21T09:32:55.012 INFO:tasks.rados.rados.0.smithi022.stdout:532: done (4 left) 2024-09-21T09:32:55.012 INFO:tasks.rados.rados.0.smithi022.stdout:533: done (3 left) 2024-09-21T09:32:55.012 INFO:tasks.rados.rados.0.smithi022.stdout:534: done (2 left) 2024-09-21T09:32:55.013 INFO:tasks.rados.rados.0.smithi022.stdout:535: done (1 left) 2024-09-21T09:32:55.013 INFO:tasks.rados.rados.0.smithi022.stdout:536: done (0 left) 2024-09-21T09:32:55.013 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 3 current snap is 50 2024-09-21T09:32:55.013 INFO:tasks.rados.rados.0.smithi022.stdout:537: seq_num 195 ranges {0=475136,475136=401408} 2024-09-21T09:32:55.015 INFO:tasks.rados.rados.0.smithi022.stdout:537: writing smithi02284960-3 from 0 to 475136 tid 1 2024-09-21T09:32:55.017 INFO:tasks.rados.rados.0.smithi022.stdout:537: writing smithi02284960-3 from 475136 to 876544 tid 2 2024-09-21T09:32:55.017 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 49 current snap is 50 2024-09-21T09:32:55.018 INFO:tasks.rados.rados.0.smithi022.stdout:538: seq_num 196 ranges {0=491520,491520=401408} 2024-09-21T09:32:55.019 INFO:tasks.rados.rados.0.smithi022.stdout:538: writing smithi02284960-49 from 0 to 491520 tid 1 2024-09-21T09:32:55.023 INFO:tasks.rados.rados.0.smithi022.stdout:538: writing smithi02284960-49 from 491520 to 892928 tid 2 2024-09-21T09:32:55.023 INFO:tasks.rados.rados.0.smithi022.stdout:539: read oid 32 snap 47 2024-09-21T09:32:55.023 INFO:tasks.rados.rados.0.smithi022.stdout:539: expect (ObjNum 143 snap 32 seq_num 143) 2024-09-21T09:32:55.023 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 50 current snap is 50 2024-09-21T09:32:55.023 INFO:tasks.rados.rados.0.smithi022.stdout:537: finishing write tid 1 to smithi02284960-3 2024-09-21T09:32:55.023 INFO:tasks.rados.rados.0.smithi022.stdout:537: finishing write tid 2 to smithi02284960-3 2024-09-21T09:32:55.024 INFO:tasks.rados.rados.0.smithi022.stdout:537: finishing write tid 3 to smithi02284960-3 2024-09-21T09:32:55.024 INFO:tasks.rados.rados.0.smithi022.stdout:540: seq_num 197 ranges {4800512=507904,5308416=401408} 2024-09-21T09:32:55.052 INFO:tasks.rados.rados.0.smithi022.stdout:540: writing smithi02284960-50 from 4800512 to 5308416 tid 1 2024-09-21T09:32:55.055 INFO:tasks.rados.rados.0.smithi022.stdout:540: writing smithi02284960-50 from 5308416 to 5709824 tid 2 2024-09-21T09:32:55.055 INFO:tasks.rados.rados.0.smithi022.stdout:541: delete oid 37 current snap is 50 2024-09-21T09:32:55.055 INFO:tasks.rados.rados.0.smithi022.stdout:537: finishing write tid 4 to smithi02284960-3 2024-09-21T09:32:55.055 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 3 v 57 (ObjNum 195 snap 50 seq_num 195) dirty exists 2024-09-21T09:32:55.055 INFO:tasks.rados.rados.0.smithi022.stdout:537: left oid 3 (ObjNum 195 snap 50 seq_num 195) 2024-09-21T09:32:55.055 INFO:tasks.rados.rados.0.smithi022.stdout:538: finishing write tid 1 to smithi02284960-49 2024-09-21T09:32:55.055 INFO:tasks.rados.rados.0.smithi022.stdout:538: finishing write tid 2 to smithi02284960-49 2024-09-21T09:32:55.055 INFO:tasks.rados.rados.0.smithi022.stdout:538: finishing write tid 3 to smithi02284960-49 2024-09-21T09:32:55.056 INFO:tasks.rados.rados.0.smithi022.stdout:538: finishing write tid 4 to smithi02284960-49 2024-09-21T09:32:55.056 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 49 v 102 (ObjNum 196 snap 50 seq_num 196) dirty exists 2024-09-21T09:32:55.056 INFO:tasks.rados.rados.0.smithi022.stdout:538: left oid 49 (ObjNum 196 snap 50 seq_num 196) 2024-09-21T09:32:55.056 INFO:tasks.rados.rados.0.smithi022.stdout:539: expect (ObjNum 143 snap 32 seq_num 143) 2024-09-21T09:32:55.191 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:32:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[67503]: [21/Sep/2024:09:32:55] ENGINE Bus STOPPING 2024-09-21T09:32:55.301 INFO:tasks.rados.rados.0.smithi022.stdout:540: finishing write tid 1 to smithi02284960-50 2024-09-21T09:32:55.301 INFO:tasks.rados.rados.0.smithi022.stdout:540: finishing write tid 2 to smithi02284960-50 2024-09-21T09:32:55.301 INFO:tasks.rados.rados.0.smithi022.stdout:537: done (4 left) 2024-09-21T09:32:55.301 INFO:tasks.rados.rados.0.smithi022.stdout:538: done (3 left) 2024-09-21T09:32:55.302 INFO:tasks.rados.rados.0.smithi022.stdout:539: done (2 left) 2024-09-21T09:32:55.302 INFO:tasks.rados.rados.0.smithi022.stdout:541: done (1 left) 2024-09-21T09:32:55.302 INFO:tasks.rados.rados.0.smithi022.stdout:542: rollback oid 42 current snap is 50 2024-09-21T09:32:55.302 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 42 to 48 2024-09-21T09:32:55.302 INFO:tasks.rados.rados.0.smithi022.stdout:540: finishing write tid 3 to smithi02284960-50 2024-09-21T09:32:55.302 INFO:tasks.rados.rados.0.smithi022.stdout:540: finishing write tid 4 to smithi02284960-50 2024-09-21T09:32:55.302 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 50 v 177 (ObjNum 197 snap 50 seq_num 197) dirty exists 2024-09-21T09:32:55.303 INFO:tasks.rados.rados.0.smithi022.stdout:540: left oid 50 (ObjNum 197 snap 50 seq_num 197) 2024-09-21T09:32:55.303 INFO:tasks.rados.rados.0.smithi022.stdout:540: done (1 left) 2024-09-21T09:32:55.303 INFO:tasks.rados.rados.0.smithi022.stdout:543: read oid 9 snap -1 2024-09-21T09:32:55.303 INFO:tasks.rados.rados.0.smithi022.stdout:543: expect (ObjNum 137 snap 30 seq_num 137) 2024-09-21T09:32:55.303 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 8 current snap is 50 2024-09-21T09:32:55.303 INFO:tasks.rados.rados.0.smithi022.stdout:544: seq_num 199 ranges {0=540672,540672=401408} 2024-09-21T09:32:55.304 INFO:tasks.rados.rados.0.smithi022.stdout:544: writing smithi02284960-8 from 0 to 540672 tid 1 2024-09-21T09:32:55.307 INFO:tasks.rados.rados.0.smithi022.stdout:544: writing smithi02284960-8 from 540672 to 942080 tid 2 2024-09-21T09:32:55.307 INFO:tasks.rados.rados.0.smithi022.stdout:545: snap_create 2024-09-21T09:32:55.307 INFO:tasks.rados.rados.0.smithi022.stdout:542: finishing rollback tid 0 to smithi02284960-42 2024-09-21T09:32:55.307 INFO:tasks.rados.rados.0.smithi022.stdout:542: finishing rollback tid 1 to smithi02284960-42 2024-09-21T09:32:55.307 INFO:tasks.rados.rados.0.smithi022.stdout:542: finishing rollback tid 2 to smithi02284960-42 2024-09-21T09:32:55.307 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 42 v 218 (ObjNum 183 snap 47 seq_num 183) dirty exists 2024-09-21T09:32:55.308 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:32:55 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: ignoring --setuser ceph since I am not root 2024-09-21T09:32:55.308 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:32:55 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: ignoring --setgroup ceph since I am not root 2024-09-21T09:32:55.308 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:32:55 smithi022 ceph-mgr[28035]: -- 172.21.15.22:0/1643877162 <== mon.0 v2:172.21.15.22:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5573d39bc340 con 0x5573d39c8800 2024-09-21T09:32:55.309 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:32:55 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:32:55.168+0000 7f8531dbdec0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-21T09:32:55.309 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:32:55 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:32:55.230+0000 7f8531dbdec0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-21T09:32:55.309 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:32:55 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:32:55.305+0000 7f8531dbdec0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-21T09:32:55.319 INFO:tasks.rados.rados.0.smithi022.stdout:543: expect (ObjNum 137 snap 30 seq_num 137) 2024-09-21T09:32:55.530 INFO:tasks.rados.rados.0.smithi022.stdout:544: finishing write tid 1 to smithi02284960-8 2024-09-21T09:32:55.530 INFO:tasks.rados.rados.0.smithi022.stdout:544: finishing write tid 2 to smithi02284960-8 2024-09-21T09:32:55.530 INFO:tasks.rados.rados.0.smithi022.stdout:544: finishing write tid 3 to smithi02284960-8 2024-09-21T09:32:55.531 INFO:tasks.rados.rados.0.smithi022.stdout:544: finishing write tid 4 to smithi02284960-8 2024-09-21T09:32:55.531 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 8 v 188 (ObjNum 199 snap 50 seq_num 199) dirty exists 2024-09-21T09:32:55.531 INFO:tasks.rados.rados.0.smithi022.stdout:544: left oid 8 (ObjNum 199 snap 50 seq_num 199) 2024-09-21T09:32:55.531 INFO:tasks.rados.rados.0.smithi022.stdout:542: done (3 left) 2024-09-21T09:32:55.531 INFO:tasks.rados.rados.0.smithi022.stdout:543: done (2 left) 2024-09-21T09:32:55.531 INFO:tasks.rados.rados.0.smithi022.stdout:544: done (1 left) 2024-09-21T09:32:55.531 INFO:tasks.rados.rados.0.smithi022.stdout:545: done (0 left) 2024-09-21T09:32:55.532 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 23 current snap is 51 2024-09-21T09:32:55.532 INFO:tasks.rados.rados.0.smithi022.stdout:546: seq_num 200 ranges {860160=557056,1417216=401408} 2024-09-21T09:32:55.537 INFO:tasks.rados.rados.0.smithi022.stdout:546: writing smithi02284960-23 from 860160 to 1417216 tid 1 2024-09-21T09:32:55.540 INFO:tasks.rados.rados.0.smithi022.stdout:546: writing smithi02284960-23 from 1417216 to 1818624 tid 2 2024-09-21T09:32:55.540 INFO:tasks.rados.rados.0.smithi022.stdout:547: copy_from oid 26 from oid 22 current snap is 51 2024-09-21T09:32:55.540 INFO:tasks.rados.rados.0.smithi022.stdout:548: snap_create 2024-09-21T09:32:55.543 INFO:tasks.rados.rados.0.smithi022.stdout:546: finishing write tid 1 to smithi02284960-23 2024-09-21T09:32:55.544 INFO:tasks.rados.rados.0.smithi022.stdout:546: finishing write tid 2 to smithi02284960-23 2024-09-21T09:32:55.544 INFO:tasks.rados.rados.0.smithi022.stdout:546: finishing write tid 3 to smithi02284960-23 2024-09-21T09:32:55.546 INFO:tasks.rados.rados.0.smithi022.stdout:546: finishing write tid 4 to smithi02284960-23 2024-09-21T09:32:55.546 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 23 v 132 (ObjNum 200 snap 51 seq_num 200) dirty exists 2024-09-21T09:32:55.546 INFO:tasks.rados.rados.0.smithi022.stdout:546: left oid 23 (ObjNum 200 snap 51 seq_num 200) 2024-09-21T09:32:55.549 INFO:tasks.rados.rados.0.smithi022.stdout:547: finishing copy_from racing read to smithi02284960-26 2024-09-21T09:32:55.550 INFO:tasks.rados.rados.0.smithi022.stdout:547: finishing copy_from to smithi02284960-26 2024-09-21T09:32:55.550 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 26 v 182 (ObjNum 170 snap 43 seq_num 170) dirty exists 2024-09-21T09:32:55.558 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:32:55 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:32:55.447+0000 7f8531dbdec0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-21T09:32:55.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:55 smithi022 ceph-mon[39525]: osdmap e163: 8 total, 8 up, 8 in 2024-09-21T09:32:55.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:55 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:32:55.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:55 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:32:55.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:55 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:32:55.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:55 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:32:55.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:55 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:32:55.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:55 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:32:55.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:55 smithi022 ceph-mon[39525]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-09-21T09:32:55.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:55 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-09-21T09:32:55.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:55 smithi022 ceph-mon[39525]: Activating manager daemon x 2024-09-21T09:32:55.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:55 smithi022 ceph-mon[39525]: osdmap e164: 8 total, 8 up, 8 in 2024-09-21T09:32:55.560 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:55 smithi022 ceph-mon[39525]: from='mgr.14541 ' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-09-21T09:32:55.560 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:55 smithi022 ceph-mon[39525]: mgrmap e24: x(active, starting, since 0.00732148s) 2024-09-21T09:32:55.560 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:55 smithi022 ceph-mon[39525]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:32:55.560 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:55 smithi022 ceph-mon[39525]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:32:55.560 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:55 smithi022 ceph-mon[39525]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:32:55.560 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:55 smithi022 ceph-mon[39525]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-09-21T09:32:55.560 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:55 smithi022 ceph-mon[39525]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-09-21T09:32:55.560 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:55 smithi022 ceph-mon[39525]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-21T09:32:55.560 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:55 smithi022 ceph-mon[39525]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:32:55.560 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:55 smithi022 ceph-mon[39525]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:32:55.561 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:55 smithi022 ceph-mon[39525]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:32:55.561 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:55 smithi022 ceph-mon[39525]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:32:55.561 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:55 smithi022 ceph-mon[39525]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:32:55.561 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:55 smithi022 ceph-mon[39525]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:32:55.561 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:55 smithi022 ceph-mon[39525]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:32:55.561 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:55 smithi022 ceph-mon[39525]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:32:55.561 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:55 smithi022 ceph-mon[39525]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T09:32:55.561 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:55 smithi022 ceph-mon[39525]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T09:32:55.561 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:55 smithi022 ceph-mon[39525]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T09:32:55.561 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:55 smithi022 ceph-mon[39525]: osdmap e165: 8 total, 8 up, 8 in 2024-09-21T09:32:55.561 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:55 smithi022 ceph-mon[27763]: osdmap e163: 8 total, 8 up, 8 in 2024-09-21T09:32:55.561 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:55 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:32:55.562 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:55 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:32:55.562 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:55 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:32:55.562 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:55 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:32:55.562 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:55 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:32:55.562 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:55 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:32:55.562 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:55 smithi022 ceph-mon[27763]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-09-21T09:32:55.562 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:55 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-09-21T09:32:55.562 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:55 smithi022 ceph-mon[27763]: Activating manager daemon x 2024-09-21T09:32:55.562 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:55 smithi022 ceph-mon[27763]: osdmap e164: 8 total, 8 up, 8 in 2024-09-21T09:32:55.562 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:55 smithi022 ceph-mon[27763]: from='mgr.14541 ' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-09-21T09:32:55.563 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:55 smithi022 ceph-mon[27763]: mgrmap e24: x(active, starting, since 0.00732148s) 2024-09-21T09:32:55.563 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:55 smithi022 ceph-mon[27763]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:32:55.563 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:55 smithi022 ceph-mon[27763]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:32:55.563 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:55 smithi022 ceph-mon[27763]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:32:55.563 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:55 smithi022 ceph-mon[27763]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-09-21T09:32:55.563 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:55 smithi022 ceph-mon[27763]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-09-21T09:32:55.563 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:55 smithi022 ceph-mon[27763]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-21T09:32:55.563 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:55 smithi022 ceph-mon[27763]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:32:55.563 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:55 smithi022 ceph-mon[27763]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:32:55.563 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:55 smithi022 ceph-mon[27763]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:32:55.563 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:55 smithi022 ceph-mon[27763]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:32:55.563 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:55 smithi022 ceph-mon[27763]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:32:55.564 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:55 smithi022 ceph-mon[27763]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:32:55.564 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:55 smithi022 ceph-mon[27763]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:32:55.564 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:55 smithi022 ceph-mon[27763]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:32:55.564 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:55 smithi022 ceph-mon[27763]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T09:32:55.564 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:55 smithi022 ceph-mon[27763]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T09:32:55.564 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:55 smithi022 ceph-mon[27763]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T09:32:55.564 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:55 smithi022 ceph-mon[27763]: osdmap e165: 8 total, 8 up, 8 in 2024-09-21T09:32:55.866 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:32:55 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:32:55.594+0000 7f8531dbdec0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-21T09:32:55.941 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:32:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[67503]: [21/Sep/2024:09:32:55] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-21T09:32:55.941 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:32:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[67503]: [21/Sep/2024:09:32:55] ENGINE Bus STOPPED 2024-09-21T09:32:55.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:55 smithi064 ceph-mon[31124]: osdmap e163: 8 total, 8 up, 8 in 2024-09-21T09:32:55.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:55 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:32:55.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:55 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:32:55.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:55 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:32:55.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:55 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:32:55.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:55 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' 2024-09-21T09:32:55.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:55 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:32:55.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:55 smithi064 ceph-mon[31124]: from='mgr.14541 172.21.15.22:0/3481847205' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-09-21T09:32:55.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:55 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-09-21T09:32:55.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:55 smithi064 ceph-mon[31124]: Activating manager daemon x 2024-09-21T09:32:55.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:55 smithi064 ceph-mon[31124]: osdmap e164: 8 total, 8 up, 8 in 2024-09-21T09:32:55.943 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:55 smithi064 ceph-mon[31124]: from='mgr.14541 ' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-09-21T09:32:55.943 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:55 smithi064 ceph-mon[31124]: mgrmap e24: x(active, starting, since 0.00732148s) 2024-09-21T09:32:55.943 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:55 smithi064 ceph-mon[31124]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:32:55.943 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:55 smithi064 ceph-mon[31124]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:32:55.943 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:55 smithi064 ceph-mon[31124]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:32:55.943 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:55 smithi064 ceph-mon[31124]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-09-21T09:32:55.943 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:55 smithi064 ceph-mon[31124]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-09-21T09:32:55.943 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:55 smithi064 ceph-mon[31124]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-21T09:32:55.943 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:55 smithi064 ceph-mon[31124]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:32:55.943 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:55 smithi064 ceph-mon[31124]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:32:55.943 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:55 smithi064 ceph-mon[31124]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:32:55.943 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:55 smithi064 ceph-mon[31124]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:32:55.943 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:55 smithi064 ceph-mon[31124]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:32:55.943 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:55 smithi064 ceph-mon[31124]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:32:55.944 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:55 smithi064 ceph-mon[31124]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:32:55.944 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:55 smithi064 ceph-mon[31124]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:32:55.944 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:55 smithi064 ceph-mon[31124]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T09:32:55.944 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:55 smithi064 ceph-mon[31124]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T09:32:55.944 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:55 smithi064 ceph-mon[31124]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T09:32:55.944 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:55 smithi064 ceph-mon[31124]: osdmap e165: 8 total, 8 up, 8 in 2024-09-21T09:32:56.307 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:32:55 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:32:55.864+0000 7f8531dbdec0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-21T09:32:56.308 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:32:55 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:32:55.928+0000 7f8531dbdec0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-21T09:32:56.308 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:32:56 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:32:56.097+0000 7f8531dbdec0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-21T09:32:56.457 INFO:tasks.rados.rados.0.smithi022.stdout:546: done (2 left) 2024-09-21T09:32:56.457 INFO:tasks.rados.rados.0.smithi022.stdout:547: done (1 left) 2024-09-21T09:32:56.457 INFO:tasks.rados.rados.0.smithi022.stdout:548: done (0 left) 2024-09-21T09:32:56.457 INFO:tasks.rados.rados.0.smithi022.stdout:549: snap_create 2024-09-21T09:32:56.716 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:56 smithi022 ceph-mon[39525]: Manager daemon x is now available 2024-09-21T09:32:56.717 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:56 smithi022 ceph-mon[27763]: Manager daemon x is now available 2024-09-21T09:32:56.799 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:56 smithi064 ceph-mon[31124]: Manager daemon x is now available 2024-09-21T09:32:56.799 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:32:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[67503]: [21/Sep/2024:09:32:56] ENGINE Bus STARTING 2024-09-21T09:32:56.799 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:32:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[67503]: 2024-09-21T09:32:56.574+0000 7f1ef62cd640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:32:56.799 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:32:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[67503]: 2024-09-21T09:32:56.574+0000 7f1ef62cd640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:32:56.799 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:32:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[67503]: 2024-09-21T09:32:56.574+0000 7f1ef62cd640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:32:56.799 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:32:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[67503]: 2024-09-21T09:32:56.574+0000 7f1ef62cd640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:32:56.799 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:32:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[67503]: 2024-09-21T09:32:56.574+0000 7f1ef62cd640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:32:56.800 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:32:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[67503]: 2024-09-21T09:32:56.574+0000 7f1ef42c9640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:32:56.800 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:32:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[67503]: 2024-09-21T09:32:56.574+0000 7f1ef42c9640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:32:56.800 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:32:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[67503]: 2024-09-21T09:32:56.574+0000 7f1ef42c9640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:32:56.800 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:32:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[67503]: 2024-09-21T09:32:56.574+0000 7f1ef42c9640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:32:56.800 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:32:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[67503]: 2024-09-21T09:32:56.574+0000 7f1ef42c9640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:32:56.800 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:32:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[67503]: [21/Sep/2024:09:32:56] ENGINE Serving on http://:::9283 2024-09-21T09:32:56.800 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:32:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[67503]: [21/Sep/2024:09:32:56] ENGINE Bus STARTED 2024-09-21T09:32:57.307 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:32:57 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:32:56.999+0000 7f8531dbdec0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-21T09:32:57.308 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:32:57 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:32:57.070+0000 7f8531dbdec0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-21T09:32:57.325 INFO:teuthology.orchestra.run.smithi022.stdout:true 2024-09-21T09:32:57.459 INFO:tasks.rados.rados.0.smithi022.stdout:549: done (0 left) 2024-09-21T09:32:57.460 INFO:tasks.rados.rados.0.smithi022.stdout:550: delete oid 4 current snap is 53 2024-09-21T09:32:57.463 INFO:tasks.rados.rados.0.smithi022.stdout:550: done (0 left) 2024-09-21T09:32:57.463 INFO:tasks.rados.rados.0.smithi022.stdout:551: delete oid 15 current snap is 53 2024-09-21T09:32:57.467 INFO:tasks.rados.rados.0.smithi022.stdout:551: done (0 left) 2024-09-21T09:32:57.467 INFO:tasks.rados.rados.0.smithi022.stdout:552: snap_create 2024-09-21T09:32:57.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:57 smithi022 ceph-mon[39525]: osdmap e166: 8 total, 8 up, 8 in 2024-09-21T09:32:57.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:57 smithi022 ceph-mon[39525]: mgrmap e25: x(active, since 1.45254s) 2024-09-21T09:32:57.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:57 smithi022 ceph-mon[39525]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:32:57.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:57 smithi022 ceph-mon[39525]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:32:57.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:57 smithi022 ceph-mon[39525]: Checking for cert/key for grafana.a 2024-09-21T09:32:57.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:57 smithi022 ceph-mon[39525]: Migrating grafana.a cert to cert store 2024-09-21T09:32:57.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:57 smithi022 ceph-mon[39525]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:32:57.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:57 smithi022 ceph-mon[39525]: Migrating grafana.a key to cert store 2024-09-21T09:32:57.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:57 smithi022 ceph-mon[39525]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:32:57.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:57 smithi022 ceph-mon[39525]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:32:57.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:57 smithi022 ceph-mon[39525]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:32:57.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:57 smithi022 ceph-mon[39525]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:32:57.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:57 smithi022 ceph-mon[39525]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T09:32:57.810 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:57 smithi022 ceph-mon[39525]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T09:32:57.810 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:57 smithi022 ceph-mon[39525]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-21T09:32:57.810 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:57 smithi022 ceph-mon[39525]: from='mgr.24775 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-21T09:32:57.810 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:57 smithi022 ceph-mon[39525]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-21T09:32:57.810 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:57 smithi022 ceph-mon[39525]: from='mgr.24775 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-21T09:32:57.810 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:57 smithi022 ceph-mon[39525]: Deploying cephadm binary to smithi022 2024-09-21T09:32:57.810 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:57 smithi022 ceph-mon[39525]: pgmap v6: 121 pgs: 121 active+clean; 138 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:32:57.810 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:57 smithi022 ceph-mon[27763]: osdmap e166: 8 total, 8 up, 8 in 2024-09-21T09:32:57.810 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:57 smithi022 ceph-mon[27763]: mgrmap e25: x(active, since 1.45254s) 2024-09-21T09:32:57.810 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:57 smithi022 ceph-mon[27763]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:32:57.810 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:57 smithi022 ceph-mon[27763]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:32:57.811 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:57 smithi022 ceph-mon[27763]: Checking for cert/key for grafana.a 2024-09-21T09:32:57.811 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:57 smithi022 ceph-mon[27763]: Migrating grafana.a cert to cert store 2024-09-21T09:32:57.811 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:57 smithi022 ceph-mon[27763]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:32:57.811 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:57 smithi022 ceph-mon[27763]: Migrating grafana.a key to cert store 2024-09-21T09:32:57.811 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:57 smithi022 ceph-mon[27763]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:32:57.811 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:57 smithi022 ceph-mon[27763]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:32:57.811 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:57 smithi022 ceph-mon[27763]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:32:57.811 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:57 smithi022 ceph-mon[27763]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:32:57.811 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:57 smithi022 ceph-mon[27763]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T09:32:57.811 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:57 smithi022 ceph-mon[27763]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T09:32:57.811 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:57 smithi022 ceph-mon[27763]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-21T09:32:57.811 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:57 smithi022 ceph-mon[27763]: from='mgr.24775 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-21T09:32:57.811 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:57 smithi022 ceph-mon[27763]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-21T09:32:57.812 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:57 smithi022 ceph-mon[27763]: from='mgr.24775 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-21T09:32:57.812 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:57 smithi022 ceph-mon[27763]: Deploying cephadm binary to smithi022 2024-09-21T09:32:57.812 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:57 smithi022 ceph-mon[27763]: pgmap v6: 121 pgs: 121 active+clean; 138 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:32:57.926 INFO:teuthology.orchestra.run.smithi022.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T09:32:57.926 INFO:teuthology.orchestra.run.smithi022.stdout:alertmanager.a smithi022 *:9093,9094 running (3m) 3m ago 4m 15.7M - 0.25.0 c8568f914cd2 42a53bb454cd 2024-09-21T09:32:57.926 INFO:teuthology.orchestra.run.smithi022.stdout:grafana.a smithi064 *:3000 running (4m) 11s ago 4m 89.1M - 9.4.7 954c08fa6188 ffde1f806d0b 2024-09-21T09:32:57.926 INFO:teuthology.orchestra.run.smithi022.stdout:mds.a smithi022 running (5m) 3m ago 5m 15.1M - 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 34e4221f3a27 2024-09-21T09:32:57.926 INFO:teuthology.orchestra.run.smithi022.stdout:mds.b smithi064 running (5m) 11s ago 5m 18.9M - 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 0f846b3ec694 2024-09-21T09:32:57.926 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.x smithi064 *:8443,9283 running (22s) 11s ago 9m 72.3M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 c102c1678bcc 2024-09-21T09:32:57.926 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.y smithi022 *:9283,8765 running (10m) 3m ago 10m 446M - 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 5bc449c0676f 2024-09-21T09:32:57.926 INFO:teuthology.orchestra.run.smithi022.stdout:mon.a smithi022 running (10m) 3m ago 10m 43.5M 2048M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 1e0a2d74e66e 2024-09-21T09:32:57.927 INFO:teuthology.orchestra.run.smithi022.stdout:mon.b smithi064 running (9m) 11s ago 9m 41.5M 2048M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be ed126533cd64 2024-09-21T09:32:57.927 INFO:teuthology.orchestra.run.smithi022.stdout:mon.c smithi022 running (9m) 3m ago 9m 39.1M 2048M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 79f307e3b64a 2024-09-21T09:32:57.927 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.a smithi022 *:9100 running (3m) 3m ago 3m 8958k - 1.5.0 0da6a335fe13 7cf4c99ebf3a 2024-09-21T09:32:57.927 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.b smithi064 *:9100 running (3m) 11s ago 3m 15.0M - 1.5.0 0da6a335fe13 643a1e1aec61 2024-09-21T09:32:57.927 INFO:teuthology.orchestra.run.smithi022.stdout:osd.0 smithi022 running (8m) 3m ago 8m 44.1M 2993M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be fcd812003534 2024-09-21T09:32:57.927 INFO:teuthology.orchestra.run.smithi022.stdout:osd.1 smithi022 running (8m) 3m ago 8m 42.2M 2993M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 740462b5c304 2024-09-21T09:32:57.927 INFO:teuthology.orchestra.run.smithi022.stdout:osd.2 smithi022 running (7m) 3m ago 7m 45.3M 2993M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be fefc7c770c32 2024-09-21T09:32:57.927 INFO:teuthology.orchestra.run.smithi022.stdout:osd.3 smithi022 running (7m) 3m ago 7m 41.5M 2993M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be a56193b67a92 2024-09-21T09:32:57.927 INFO:teuthology.orchestra.run.smithi022.stdout:osd.4 smithi064 running (7m) 11s ago 7m 71.6M 2992M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 2a2937374fc4 2024-09-21T09:32:57.927 INFO:teuthology.orchestra.run.smithi022.stdout:osd.5 smithi064 running (6m) 11s ago 6m 83.7M 2992M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 1900c4af5d28 2024-09-21T09:32:57.927 INFO:teuthology.orchestra.run.smithi022.stdout:osd.6 smithi064 running (6m) 11s ago 6m 79.1M 2992M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 3cc4ec816e5b 2024-09-21T09:32:57.927 INFO:teuthology.orchestra.run.smithi022.stdout:osd.7 smithi064 running (5m) 11s ago 5m 82.2M 2992M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 18ef0be94104 2024-09-21T09:32:57.927 INFO:teuthology.orchestra.run.smithi022.stdout:prometheus.a smithi064 *:9095 running (3m) 11s ago 4m 52.3M - 2.43.0 a07b618ecd1d caee9b54c131 2024-09-21T09:32:57.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:57 smithi064 ceph-mon[31124]: osdmap e166: 8 total, 8 up, 8 in 2024-09-21T09:32:57.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:57 smithi064 ceph-mon[31124]: mgrmap e25: x(active, since 1.45254s) 2024-09-21T09:32:57.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:57 smithi064 ceph-mon[31124]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:32:57.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:57 smithi064 ceph-mon[31124]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:32:57.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:57 smithi064 ceph-mon[31124]: Checking for cert/key for grafana.a 2024-09-21T09:32:57.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:57 smithi064 ceph-mon[31124]: Migrating grafana.a cert to cert store 2024-09-21T09:32:57.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:57 smithi064 ceph-mon[31124]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:32:57.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:57 smithi064 ceph-mon[31124]: Migrating grafana.a key to cert store 2024-09-21T09:32:57.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:57 smithi064 ceph-mon[31124]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:32:57.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:57 smithi064 ceph-mon[31124]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:32:57.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:57 smithi064 ceph-mon[31124]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:32:57.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:57 smithi064 ceph-mon[31124]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:32:57.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:57 smithi064 ceph-mon[31124]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T09:32:57.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:57 smithi064 ceph-mon[31124]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T09:32:57.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:57 smithi064 ceph-mon[31124]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-21T09:32:57.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:57 smithi064 ceph-mon[31124]: from='mgr.24775 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-21T09:32:57.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:57 smithi064 ceph-mon[31124]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-21T09:32:57.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:57 smithi064 ceph-mon[31124]: from='mgr.24775 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-21T09:32:57.943 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:57 smithi064 ceph-mon[31124]: Deploying cephadm binary to smithi022 2024-09-21T09:32:57.943 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:57 smithi064 ceph-mon[31124]: pgmap v6: 121 pgs: 121 active+clean; 138 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:32:58.093 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:32:57 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:32:57.836+0000 7f8531dbdec0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-21T09:32:58.093 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:32:57 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:32:57.957+0000 7f8531dbdec0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-21T09:32:58.093 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:32:58 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:32:58.091+0000 7f8531dbdec0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-21T09:32:58.288 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-09-21T09:32:58.288 INFO:teuthology.orchestra.run.smithi022.stdout: "mon": { 2024-09-21T09:32:58.288 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 3 2024-09-21T09:32:58.288 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-21T09:32:58.288 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr": { 2024-09-21T09:32:58.288 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 1 2024-09-21T09:32:58.288 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-21T09:32:58.289 INFO:teuthology.orchestra.run.smithi022.stdout: "osd": { 2024-09-21T09:32:58.289 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 8 2024-09-21T09:32:58.289 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-21T09:32:58.289 INFO:teuthology.orchestra.run.smithi022.stdout: "mds": { 2024-09-21T09:32:58.289 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 2 2024-09-21T09:32:58.289 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-21T09:32:58.289 INFO:teuthology.orchestra.run.smithi022.stdout: "overall": { 2024-09-21T09:32:58.289 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 13, 2024-09-21T09:32:58.289 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 1 2024-09-21T09:32:58.289 INFO:teuthology.orchestra.run.smithi022.stdout: } 2024-09-21T09:32:58.289 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-09-21T09:32:58.360 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:32:58 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:32:58.153+0000 7f8531dbdec0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-21T09:32:58.360 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:32:58 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:32:58.285+0000 7f8531dbdec0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-21T09:32:58.360 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:32:58 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:32:58.358+0000 7f8531dbdec0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-21T09:32:58.442 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:32:58 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[67503]: 2024-09-21T09:32:58.009+0000 7f1eda819640 -1 log_channel(cephadm) log [ERR] : [21/Sep/2024:09:32:58] ENGINE Error in HTTPServer.serve 2024-09-21T09:32:58.442 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:32:58 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[67503]: Traceback (most recent call last): 2024-09-21T09:32:58.442 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:32:58 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[67503]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T09:32:58.442 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:32:58 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[67503]: self._connections.run(self.expiration_interval) 2024-09-21T09:32:58.442 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:32:58 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[67503]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T09:32:58.442 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:32:58 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[67503]: self._run(expiration_interval) 2024-09-21T09:32:58.442 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:32:58 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[67503]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T09:32:58.442 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:32:58 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[67503]: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T09:32:58.443 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:32:58 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[67503]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T09:32:58.443 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:32:58 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[67503]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T09:32:58.443 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:32:58 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[67503]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T09:32:58.443 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:32:58 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[67503]: s = self.context.wrap_socket( 2024-09-21T09:32:58.443 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:32:58 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[67503]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T09:32:58.443 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:32:58 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[67503]: return self.sslsocket_class._create( 2024-09-21T09:32:58.443 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:32:58 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[67503]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T09:32:58.443 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:32:58 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[67503]: self.do_handshake() 2024-09-21T09:32:58.443 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:32:58 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[67503]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T09:32:58.443 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:32:58 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[67503]: self._sslobj.do_handshake() 2024-09-21T09:32:58.443 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:32:58 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[67503]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T09:32:58.443 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:32:58 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[67503]: 2024-09-21T09:32:58.466 INFO:tasks.rados.rados.0.smithi022.stdout:552: done (0 left) 2024-09-21T09:32:58.467 INFO:tasks.rados.rados.0.smithi022.stdout:553: setattr oid 19 current snap is 54 2024-09-21T09:32:58.468 INFO:tasks.rados.rados.0.smithi022.stdout:554: rmattr oid 42 current snap is 54 2024-09-21T09:32:58.468 INFO:tasks.rados.rados.0.smithi022.stdout:554: done (1 left) 2024-09-21T09:32:58.468 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 39 current snap is 54 2024-09-21T09:32:58.468 INFO:tasks.rados.rados.0.smithi022.stdout:555: seq_num 203 ranges {0=606208,606208=401408} 2024-09-21T09:32:58.471 INFO:tasks.rados.rados.0.smithi022.stdout:555: writing smithi02284960-39 from 0 to 606208 tid 1 2024-09-21T09:32:58.476 INFO:tasks.rados.rados.0.smithi022.stdout:555: writing smithi02284960-39 from 606208 to 1007616 tid 2 2024-09-21T09:32:58.476 INFO:tasks.rados.rados.0.smithi022.stdout:556: read oid 47 snap -1 2024-09-21T09:32:58.476 INFO:tasks.rados.rados.0.smithi022.stdout:556: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-21T09:32:58.476 INFO:tasks.rados.rados.0.smithi022.stdout:557: read oid 17 snap 49 2024-09-21T09:32:58.476 INFO:tasks.rados.rados.0.smithi022.stdout:557: expect (ObjNum 177 snap 44 seq_num 177) 2024-09-21T09:32:58.476 INFO:tasks.rados.rados.0.smithi022.stdout:558: rollback oid 45 current snap is 54 2024-09-21T09:32:58.477 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 45 to 42 2024-09-21T09:32:58.477 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 19 v 305 (ObjNum 155 snap 36 seq_num 155) dirty exists 2024-09-21T09:32:58.477 INFO:tasks.rados.rados.0.smithi022.stdout:553: done (4 left) 2024-09-21T09:32:58.477 INFO:tasks.rados.rados.0.smithi022.stdout:559: snap_create 2024-09-21T09:32:58.477 INFO:tasks.rados.rados.0.smithi022.stdout:556: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-21T09:32:58.481 INFO:tasks.rados.rados.0.smithi022.stdout:555: finishing write tid 1 to smithi02284960-39 2024-09-21T09:32:58.481 INFO:tasks.rados.rados.0.smithi022.stdout:555: finishing write tid 2 to smithi02284960-39 2024-09-21T09:32:58.481 INFO:tasks.rados.rados.0.smithi022.stdout:558: finishing rollback tid 0 to smithi02284960-45 2024-09-21T09:32:58.481 INFO:tasks.rados.rados.0.smithi022.stdout:555: finishing write tid 3 to smithi02284960-39 2024-09-21T09:32:58.481 INFO:tasks.rados.rados.0.smithi022.stdout:558: finishing rollback tid 1 to smithi02284960-45 2024-09-21T09:32:58.481 INFO:tasks.rados.rados.0.smithi022.stdout:558: finishing rollback tid 2 to smithi02284960-45 2024-09-21T09:32:58.482 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 45 v 181 (ObjNum 115 snap 23 seq_num 115) dirty exists 2024-09-21T09:32:58.482 INFO:tasks.rados.rados.0.smithi022.stdout:555: finishing write tid 4 to smithi02284960-39 2024-09-21T09:32:58.482 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 39 v 148 (ObjNum 203 snap 54 seq_num 203) dirty exists 2024-09-21T09:32:58.482 INFO:tasks.rados.rados.0.smithi022.stdout:555: left oid 39 (ObjNum 203 snap 54 seq_num 203) 2024-09-21T09:32:58.482 INFO:tasks.rados.rados.0.smithi022.stdout:557: expect (ObjNum 177 snap 44 seq_num 177) 2024-09-21T09:32:58.597 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-09-21T09:32:58.597 INFO:teuthology.orchestra.run.smithi022.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47", 2024-09-21T09:32:58.597 INFO:teuthology.orchestra.run.smithi022.stdout: "in_progress": true, 2024-09-21T09:32:58.597 INFO:teuthology.orchestra.run.smithi022.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T09:32:58.597 INFO:teuthology.orchestra.run.smithi022.stdout: "services_complete": [], 2024-09-21T09:32:58.597 INFO:teuthology.orchestra.run.smithi022.stdout: "progress": "1/20 daemons upgraded", 2024-09-21T09:32:58.597 INFO:teuthology.orchestra.run.smithi022.stdout: "message": "", 2024-09-21T09:32:58.597 INFO:teuthology.orchestra.run.smithi022.stdout: "is_paused": false 2024-09-21T09:32:58.597 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-09-21T09:32:58.655 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:32:58 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:32:58.421+0000 7f8531dbdec0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-21T09:32:58.655 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:32:58 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:32:58.582+0000 7f8531dbdec0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-21T09:32:58.655 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:32:58 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:32:58.653+0000 7f8531dbdec0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-21T09:32:58.655 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:58 smithi022 ceph-mon[39525]: from='client.15063 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:32:58.655 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:58 smithi022 ceph-mon[39525]: osdmap e167: 8 total, 8 up, 8 in 2024-09-21T09:32:58.655 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:58 smithi022 ceph-mon[39525]: from='client.24968 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:32:58.655 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:58 smithi022 ceph-mon[39525]: [21/Sep/2024:09:32:57] ENGINE Bus STARTING 2024-09-21T09:32:58.656 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:58 smithi022 ceph-mon[39525]: Deploying cephadm binary to smithi064 2024-09-21T09:32:58.656 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:58 smithi022 ceph-mon[39525]: [21/Sep/2024:09:32:57] ENGINE Serving on http://172.21.15.64:8765 2024-09-21T09:32:58.656 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:58 smithi022 ceph-mon[39525]: from='client.15075 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:32:58.656 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:58 smithi022 ceph-mon[39525]: [21/Sep/2024:09:32:57] ENGINE Serving on https://172.21.15.64:7150 2024-09-21T09:32:58.656 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:58 smithi022 ceph-mon[39525]: [21/Sep/2024:09:32:57] ENGINE Bus STARTED 2024-09-21T09:32:58.656 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:58 smithi022 ceph-mon[39525]: [21/Sep/2024:09:32:58] ENGINE Error in HTTPServer.serve 2024-09-21T09:32:58.656 INFO:journalctl@ceph.mon.c.smithi022.stdout: Traceback (most recent call last): 2024-09-21T09:32:58.656 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T09:32:58.656 INFO:journalctl@ceph.mon.c.smithi022.stdout: self._connections.run(self.expiration_interval) 2024-09-21T09:32:58.656 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T09:32:58.656 INFO:journalctl@ceph.mon.c.smithi022.stdout: self._run(expiration_interval) 2024-09-21T09:32:58.656 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T09:32:58.656 INFO:journalctl@ceph.mon.c.smithi022.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T09:32:58.656 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T09:32:58.657 INFO:journalctl@ceph.mon.c.smithi022.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T09:32:58.657 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T09:32:58.657 INFO:journalctl@ceph.mon.c.smithi022.stdout: s = self.context.wrap_socket( 2024-09-21T09:32:58.657 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T09:32:58.657 INFO:journalctl@ceph.mon.c.smithi022.stdout: return self.sslsocket_class._create( 2024-09-21T09:32:58.657 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T09:32:58.657 INFO:journalctl@ceph.mon.c.smithi022.stdout: self.do_handshake() 2024-09-21T09:32:58.657 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T09:32:58.657 INFO:journalctl@ceph.mon.c.smithi022.stdout: self._sslobj.do_handshake() 2024-09-21T09:32:58.657 INFO:journalctl@ceph.mon.c.smithi022.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T09:32:58.657 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:58 smithi022 ceph-mon[39525]: from='client.? 172.21.15.22:0/3100594640' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:32:58.657 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:58 smithi022 ceph-mon[27763]: from='client.15063 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:32:58.657 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:58 smithi022 ceph-mon[27763]: osdmap e167: 8 total, 8 up, 8 in 2024-09-21T09:32:58.658 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:58 smithi022 ceph-mon[27763]: from='client.24968 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:32:58.658 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:58 smithi022 ceph-mon[27763]: [21/Sep/2024:09:32:57] ENGINE Bus STARTING 2024-09-21T09:32:58.658 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:58 smithi022 ceph-mon[27763]: Deploying cephadm binary to smithi064 2024-09-21T09:32:58.658 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:58 smithi022 ceph-mon[27763]: [21/Sep/2024:09:32:57] ENGINE Serving on http://172.21.15.64:8765 2024-09-21T09:32:58.658 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:58 smithi022 ceph-mon[27763]: from='client.15075 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:32:58.658 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:58 smithi022 ceph-mon[27763]: [21/Sep/2024:09:32:57] ENGINE Serving on https://172.21.15.64:7150 2024-09-21T09:32:58.658 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:58 smithi022 ceph-mon[27763]: [21/Sep/2024:09:32:57] ENGINE Bus STARTED 2024-09-21T09:32:58.658 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:58 smithi022 ceph-mon[27763]: [21/Sep/2024:09:32:58] ENGINE Error in HTTPServer.serve 2024-09-21T09:32:58.658 INFO:journalctl@ceph.mon.a.smithi022.stdout: Traceback (most recent call last): 2024-09-21T09:32:58.658 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T09:32:58.658 INFO:journalctl@ceph.mon.a.smithi022.stdout: self._connections.run(self.expiration_interval) 2024-09-21T09:32:58.658 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T09:32:58.658 INFO:journalctl@ceph.mon.a.smithi022.stdout: self._run(expiration_interval) 2024-09-21T09:32:58.659 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T09:32:58.659 INFO:journalctl@ceph.mon.a.smithi022.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T09:32:58.659 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T09:32:58.659 INFO:journalctl@ceph.mon.a.smithi022.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T09:32:58.659 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T09:32:58.659 INFO:journalctl@ceph.mon.a.smithi022.stdout: s = self.context.wrap_socket( 2024-09-21T09:32:58.659 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T09:32:58.659 INFO:journalctl@ceph.mon.a.smithi022.stdout: return self.sslsocket_class._create( 2024-09-21T09:32:58.659 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T09:32:58.659 INFO:journalctl@ceph.mon.a.smithi022.stdout: self.do_handshake() 2024-09-21T09:32:58.659 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T09:32:58.659 INFO:journalctl@ceph.mon.a.smithi022.stdout: self._sslobj.do_handshake() 2024-09-21T09:32:58.659 INFO:journalctl@ceph.mon.a.smithi022.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T09:32:58.659 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:58 smithi022 ceph-mon[27763]: from='client.? 172.21.15.22:0/3100594640' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:32:58.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:58 smithi064 ceph-mon[31124]: from='client.15063 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:32:58.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:58 smithi064 ceph-mon[31124]: osdmap e167: 8 total, 8 up, 8 in 2024-09-21T09:32:58.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:58 smithi064 ceph-mon[31124]: from='client.24968 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:32:58.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:58 smithi064 ceph-mon[31124]: [21/Sep/2024:09:32:57] ENGINE Bus STARTING 2024-09-21T09:32:58.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:58 smithi064 ceph-mon[31124]: Deploying cephadm binary to smithi064 2024-09-21T09:32:58.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:58 smithi064 ceph-mon[31124]: [21/Sep/2024:09:32:57] ENGINE Serving on http://172.21.15.64:8765 2024-09-21T09:32:58.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:58 smithi064 ceph-mon[31124]: from='client.15075 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:32:58.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:58 smithi064 ceph-mon[31124]: [21/Sep/2024:09:32:57] ENGINE Serving on https://172.21.15.64:7150 2024-09-21T09:32:58.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:58 smithi064 ceph-mon[31124]: [21/Sep/2024:09:32:57] ENGINE Bus STARTED 2024-09-21T09:32:58.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:58 smithi064 ceph-mon[31124]: [21/Sep/2024:09:32:58] ENGINE Error in HTTPServer.serve 2024-09-21T09:32:58.942 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-09-21T09:32:58.942 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T09:32:58.942 INFO:journalctl@ceph.mon.b.smithi064.stdout: self._connections.run(self.expiration_interval) 2024-09-21T09:32:58.942 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T09:32:58.943 INFO:journalctl@ceph.mon.b.smithi064.stdout: self._run(expiration_interval) 2024-09-21T09:32:58.943 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T09:32:58.943 INFO:journalctl@ceph.mon.b.smithi064.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T09:32:58.943 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T09:32:58.943 INFO:journalctl@ceph.mon.b.smithi064.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T09:32:58.943 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T09:32:58.943 INFO:journalctl@ceph.mon.b.smithi064.stdout: s = self.context.wrap_socket( 2024-09-21T09:32:58.943 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T09:32:58.943 INFO:journalctl@ceph.mon.b.smithi064.stdout: return self.sslsocket_class._create( 2024-09-21T09:32:58.943 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T09:32:58.943 INFO:journalctl@ceph.mon.b.smithi064.stdout: self.do_handshake() 2024-09-21T09:32:58.944 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T09:32:58.944 INFO:journalctl@ceph.mon.b.smithi064.stdout: self._sslobj.do_handshake() 2024-09-21T09:32:58.944 INFO:journalctl@ceph.mon.b.smithi064.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T09:32:58.944 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:58 smithi064 ceph-mon[31124]: from='client.? 172.21.15.22:0/3100594640' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:32:58.953 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:32:58 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:32:58.723+0000 7f8531dbdec0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-21T09:32:58.953 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:32:58 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:32:58.809+0000 7f8531dbdec0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-21T09:32:58.953 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:32:58 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:32:58.883+0000 7f8531dbdec0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-21T09:32:58.953 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:32:58 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:32:58.951+0000 7f8531dbdec0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-21T09:32:59.470 INFO:tasks.rados.rados.0.smithi022.stdout:555: done (4 left) 2024-09-21T09:32:59.471 INFO:tasks.rados.rados.0.smithi022.stdout:556: done (3 left) 2024-09-21T09:32:59.471 INFO:tasks.rados.rados.0.smithi022.stdout:557: done (2 left) 2024-09-21T09:32:59.471 INFO:tasks.rados.rados.0.smithi022.stdout:558: done (1 left) 2024-09-21T09:32:59.471 INFO:tasks.rados.rados.0.smithi022.stdout:559: done (0 left) 2024-09-21T09:32:59.471 INFO:tasks.rados.rados.0.smithi022.stdout:560: rollback oid 42 current snap is 55 2024-09-21T09:32:59.472 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 42 to 46 2024-09-21T09:32:59.472 INFO:tasks.rados.rados.0.smithi022.stdout:561: copy_from oid 3 from oid 1 current snap is 55 2024-09-21T09:32:59.472 INFO:tasks.rados.rados.0.smithi022.stdout:562: snap_remove snap 31 2024-09-21T09:32:59.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:59 smithi022 ceph-mon[39525]: osdmap e168: 8 total, 8 up, 8 in 2024-09-21T09:32:59.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:59 smithi022 ceph-mon[39525]: mgrmap e26: x(active, since 3s) 2024-09-21T09:32:59.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:59 smithi022 ceph-mon[39525]: from='client.15087 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:32:59.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:59 smithi022 ceph-mon[39525]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:32:59.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:59 smithi022 ceph-mon[39525]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:32:59.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:32:59 smithi022 ceph-mon[39525]: pgmap v9: 121 pgs: 121 active+clean; 138 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:32:59.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:59 smithi022 ceph-mon[27763]: osdmap e168: 8 total, 8 up, 8 in 2024-09-21T09:32:59.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:59 smithi022 ceph-mon[27763]: mgrmap e26: x(active, since 3s) 2024-09-21T09:32:59.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:59 smithi022 ceph-mon[27763]: from='client.15087 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:32:59.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:59 smithi022 ceph-mon[27763]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:32:59.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:59 smithi022 ceph-mon[27763]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:32:59.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:32:59 smithi022 ceph-mon[27763]: pgmap v9: 121 pgs: 121 active+clean; 138 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:32:59.933 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:59 smithi064 ceph-mon[31124]: osdmap e168: 8 total, 8 up, 8 in 2024-09-21T09:32:59.933 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:59 smithi064 ceph-mon[31124]: mgrmap e26: x(active, since 3s) 2024-09-21T09:32:59.933 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:59 smithi064 ceph-mon[31124]: from='client.15087 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:32:59.934 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:59 smithi064 ceph-mon[31124]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:32:59.934 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:59 smithi064 ceph-mon[31124]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:32:59.934 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:32:59 smithi064 ceph-mon[31124]: pgmap v9: 121 pgs: 121 active+clean; 138 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:33:00.058 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:32:59 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:32:59.761+0000 7f8531dbdec0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-21T09:33:00.058 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:32:59 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: [21/Sep/2024:09:32:59] ENGINE Bus STARTING 2024-09-21T09:33:00.059 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:32:59 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: CherryPy Checker: 2024-09-21T09:33:00.059 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:32:59 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: The Application mounted at '' has an empty config. 2024-09-21T09:33:00.059 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:32:59 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: 2024-09-21T09:33:00.059 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:32:59 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: [21/Sep/2024:09:32:59] ENGINE Serving on http://:::9283 2024-09-21T09:33:00.059 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:32:59 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: [21/Sep/2024:09:32:59] ENGINE Bus STARTED 2024-09-21T09:33:00.436 INFO:tasks.rados.rados.0.smithi022.stdout:562: done (2 left) 2024-09-21T09:33:00.436 INFO:tasks.rados.rados.0.smithi022.stdout:563: snap_create 2024-09-21T09:33:00.436 INFO:tasks.rados.rados.0.smithi022.stdout:560: finishing rollback tid 0 to smithi02284960-42 2024-09-21T09:33:00.436 INFO:tasks.rados.rados.0.smithi022.stdout:560: finishing rollback tid 1 to smithi02284960-42 2024-09-21T09:33:00.436 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 42 v 223 (ObjNum 177 snap 0 seq_num 1) dirty dne 2024-09-21T09:33:00.436 INFO:tasks.rados.rados.0.smithi022.stdout:561: finishing copy_from racing read to smithi02284960-3 2024-09-21T09:33:00.437 INFO:tasks.rados.rados.0.smithi022.stdout:561: finishing copy_from to smithi02284960-3 2024-09-21T09:33:00.437 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 3 v 30 (ObjNum 89 snap 18 seq_num 89) dirty exists 2024-09-21T09:33:00.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:00 smithi022 ceph-mon[27763]: osdmap e169: 8 total, 8 up, 8 in 2024-09-21T09:33:00.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:00 smithi022 ceph-mon[27763]: Standby manager daemon y started 2024-09-21T09:33:00.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:00 smithi022 ceph-mon[27763]: from='mgr.? 172.21.15.22:0/541570477' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-21T09:33:00.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:00 smithi022 ceph-mon[27763]: from='mgr.? 172.21.15.22:0/541570477' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:33:00.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:00 smithi022 ceph-mon[27763]: from='mgr.? 172.21.15.22:0/541570477' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-21T09:33:00.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:00 smithi022 ceph-mon[27763]: from='mgr.? 172.21.15.22:0/541570477' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:33:00.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:00 smithi022 ceph-mon[27763]: osdmap e170: 8 total, 8 up, 8 in 2024-09-21T09:33:00.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:00 smithi022 ceph-mon[39525]: osdmap e169: 8 total, 8 up, 8 in 2024-09-21T09:33:00.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:00 smithi022 ceph-mon[39525]: Standby manager daemon y started 2024-09-21T09:33:00.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:00 smithi022 ceph-mon[39525]: from='mgr.? 172.21.15.22:0/541570477' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-21T09:33:00.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:00 smithi022 ceph-mon[39525]: from='mgr.? 172.21.15.22:0/541570477' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:33:00.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:00 smithi022 ceph-mon[39525]: from='mgr.? 172.21.15.22:0/541570477' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-21T09:33:00.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:00 smithi022 ceph-mon[39525]: from='mgr.? 172.21.15.22:0/541570477' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:33:00.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:00 smithi022 ceph-mon[39525]: osdmap e170: 8 total, 8 up, 8 in 2024-09-21T09:33:00.928 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:00 smithi064 ceph-mon[31124]: osdmap e169: 8 total, 8 up, 8 in 2024-09-21T09:33:00.929 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:00 smithi064 ceph-mon[31124]: Standby manager daemon y started 2024-09-21T09:33:00.929 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:00 smithi064 ceph-mon[31124]: from='mgr.? 172.21.15.22:0/541570477' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-21T09:33:00.929 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:00 smithi064 ceph-mon[31124]: from='mgr.? 172.21.15.22:0/541570477' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:33:00.929 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:00 smithi064 ceph-mon[31124]: from='mgr.? 172.21.15.22:0/541570477' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-21T09:33:00.929 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:00 smithi064 ceph-mon[31124]: from='mgr.? 172.21.15.22:0/541570477' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:33:00.929 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:00 smithi064 ceph-mon[31124]: osdmap e170: 8 total, 8 up, 8 in 2024-09-21T09:33:00.929 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:33:00 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[67503]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:33:00] "GET /metrics HTTP/1.1" 200 35848 "" "Prometheus/2.43.0" 2024-09-21T09:33:01.441 INFO:tasks.rados.rados.0.smithi022.stdout:560: done (2 left) 2024-09-21T09:33:01.442 INFO:tasks.rados.rados.0.smithi022.stdout:561: done (1 left) 2024-09-21T09:33:01.442 INFO:tasks.rados.rados.0.smithi022.stdout:563: done (0 left) 2024-09-21T09:33:01.442 INFO:tasks.rados.rados.0.smithi022.stdout:564: read oid 46 snap -1 2024-09-21T09:33:01.442 INFO:tasks.rados.rados.0.smithi022.stdout:564: expect (ObjNum 45 snap 0 seq_num 45) 2024-09-21T09:33:01.442 INFO:tasks.rados.rados.0.smithi022.stdout:565: rollback oid 42 current snap is 56 2024-09-21T09:33:01.442 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 42 to 38 2024-09-21T09:33:01.442 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 26 current snap is 56 2024-09-21T09:33:01.442 INFO:tasks.rados.rados.0.smithi022.stdout:566: seq_num 204 ranges {811008=622592,1433600=401408} 2024-09-21T09:33:01.449 INFO:tasks.rados.rados.0.smithi022.stdout:566: writing smithi02284960-26 from 811008 to 1433600 tid 1 2024-09-21T09:33:01.451 INFO:tasks.rados.rados.0.smithi022.stdout:566: writing smithi02284960-26 from 1433600 to 1835008 tid 2 2024-09-21T09:33:01.452 INFO:tasks.rados.rados.0.smithi022.stdout:567: copy_from oid 41 from oid 40 current snap is 56 2024-09-21T09:33:01.452 INFO:tasks.rados.rados.0.smithi022.stdout:565: finishing rollback tid 1 to smithi02284960-42 2024-09-21T09:33:01.452 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 42 v 223 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-09-21T09:33:01.452 INFO:tasks.rados.rados.0.smithi022.stdout:565: done (3 left) 2024-09-21T09:33:01.452 INFO:tasks.rados.rados.0.smithi022.stdout:568: rollback oid 39 current snap is 56 2024-09-21T09:33:01.452 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 39 to 42 2024-09-21T09:33:01.452 INFO:tasks.rados.rados.0.smithi022.stdout:564: expect (ObjNum 45 snap 0 seq_num 45) 2024-09-21T09:33:01.475 INFO:tasks.rados.rados.0.smithi022.stdout:568: finishing rollback tid 0 to smithi02284960-39 2024-09-21T09:33:01.475 INFO:tasks.rados.rados.0.smithi022.stdout:566: finishing write tid 1 to smithi02284960-26 2024-09-21T09:33:01.475 INFO:tasks.rados.rados.0.smithi022.stdout:568: finishing rollback tid 1 to smithi02284960-39 2024-09-21T09:33:01.475 INFO:tasks.rados.rados.0.smithi022.stdout:566: finishing write tid 2 to smithi02284960-26 2024-09-21T09:33:01.475 INFO:tasks.rados.rados.0.smithi022.stdout:566: finishing write tid 3 to smithi02284960-26 2024-09-21T09:33:01.475 INFO:tasks.rados.rados.0.smithi022.stdout:568: finishing rollback tid 2 to smithi02284960-39 2024-09-21T09:33:01.475 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 39 v 158 (ObjNum 151 snap 36 seq_num 151) dirty exists 2024-09-21T09:33:01.476 INFO:tasks.rados.rados.0.smithi022.stdout:566: finishing write tid 4 to smithi02284960-26 2024-09-21T09:33:01.476 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 26 v 234 (ObjNum 204 snap 56 seq_num 204) dirty exists 2024-09-21T09:33:01.476 INFO:tasks.rados.rados.0.smithi022.stdout:566: left oid 26 (ObjNum 204 snap 56 seq_num 204) 2024-09-21T09:33:01.476 INFO:tasks.rados.rados.0.smithi022.stdout:564: done (3 left) 2024-09-21T09:33:01.476 INFO:tasks.rados.rados.0.smithi022.stdout:566: done (2 left) 2024-09-21T09:33:01.476 INFO:tasks.rados.rados.0.smithi022.stdout:568: done (1 left) 2024-09-21T09:33:01.476 INFO:tasks.rados.rados.0.smithi022.stdout:569: copy_from oid 1 from oid 19 current snap is 56 2024-09-21T09:33:01.476 INFO:tasks.rados.rados.0.smithi022.stdout:567: finishing copy_from racing read to smithi02284960-41 2024-09-21T09:33:01.476 INFO:tasks.rados.rados.0.smithi022.stdout:567: finishing copy_from to smithi02284960-41 2024-09-21T09:33:01.476 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 41 v 164 (ObjNum 92 snap 18 seq_num 92) dirty exists 2024-09-21T09:33:01.476 INFO:tasks.rados.rados.0.smithi022.stdout:567: done (1 left) 2024-09-21T09:33:01.476 INFO:tasks.rados.rados.0.smithi022.stdout:570: rollback oid 47 current snap is 56 2024-09-21T09:33:01.476 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 47 to 48 2024-09-21T09:33:01.476 INFO:tasks.rados.rados.0.smithi022.stdout:571: delete oid 50 current snap is 56 2024-09-21T09:33:01.477 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:33:01 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:33:01] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-21T09:33:01.477 INFO:tasks.rados.rados.0.smithi022.stdout:571: done (2 left) 2024-09-21T09:33:01.478 INFO:tasks.rados.rados.0.smithi022.stdout:572: delete oid 15 current snap is 56 2024-09-21T09:33:01.480 INFO:tasks.rados.rados.0.smithi022.stdout:569: finishing copy_from racing read to smithi02284960-1 2024-09-21T09:33:01.480 INFO:tasks.rados.rados.0.smithi022.stdout:570: finishing rollback tid 0 to smithi02284960-47 2024-09-21T09:33:01.481 INFO:tasks.rados.rados.0.smithi022.stdout:570: finishing rollback tid 1 to smithi02284960-47 2024-09-21T09:33:01.481 INFO:tasks.rados.rados.0.smithi022.stdout:572: done (2 left) 2024-09-21T09:33:01.481 INFO:tasks.rados.rados.0.smithi022.stdout:573: read oid 9 snap -1 2024-09-21T09:33:01.481 INFO:tasks.rados.rados.0.smithi022.stdout:573: expect (ObjNum 137 snap 30 seq_num 137) 2024-09-21T09:33:01.481 INFO:tasks.rados.rados.0.smithi022.stdout:574: rollback oid 49 current snap is 56 2024-09-21T09:33:01.481 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 49 to 50 2024-09-21T09:33:01.481 INFO:tasks.rados.rados.0.smithi022.stdout:575: delete oid 45 current snap is 56 2024-09-21T09:33:01.481 INFO:tasks.rados.rados.0.smithi022.stdout:570: finishing rollback tid 2 to smithi02284960-47 2024-09-21T09:33:01.481 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 47 v 151 (ObjNum 161 snap 0 seq_num 1) dirty exists 2024-09-21T09:33:01.481 INFO:tasks.rados.rados.0.smithi022.stdout:569: finishing copy_from to smithi02284960-1 2024-09-21T09:33:01.481 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 1 v 305 (ObjNum 155 snap 36 seq_num 155) dirty exists 2024-09-21T09:33:01.483 INFO:tasks.rados.rados.0.smithi022.stdout:569: done (4 left) 2024-09-21T09:33:01.483 INFO:tasks.rados.rados.0.smithi022.stdout:570: done (3 left) 2024-09-21T09:33:01.483 INFO:tasks.rados.rados.0.smithi022.stdout:575: done (2 left) 2024-09-21T09:33:01.483 INFO:tasks.rados.rados.0.smithi022.stdout:576: read oid 10 snap -1 2024-09-21T09:33:01.483 INFO:tasks.rados.rados.0.smithi022.stdout:576: expect (ObjNum 144 snap 32 seq_num 144) 2024-09-21T09:33:01.483 INFO:tasks.rados.rados.0.smithi022.stdout:577: snap_create 2024-09-21T09:33:01.484 INFO:tasks.rados.rados.0.smithi022.stdout:574: finishing rollback tid 0 to smithi02284960-49 2024-09-21T09:33:01.484 INFO:tasks.rados.rados.0.smithi022.stdout:574: finishing rollback tid 1 to smithi02284960-49 2024-09-21T09:33:01.484 INFO:tasks.rados.rados.0.smithi022.stdout:574: finishing rollback tid 2 to smithi02284960-49 2024-09-21T09:33:01.484 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 49 v 106 (ObjNum 196 snap 50 seq_num 196) dirty exists 2024-09-21T09:33:01.485 INFO:tasks.rados.rados.0.smithi022.stdout:576: expect (ObjNum 144 snap 32 seq_num 144) 2024-09-21T09:33:01.495 INFO:tasks.rados.rados.0.smithi022.stdout:573: expect (ObjNum 137 snap 30 seq_num 137) 2024-09-21T09:33:01.732 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:01 smithi064 ceph-mon[31124]: mgrmap e27: x(active, since 5s), standbys: y 2024-09-21T09:33:01.733 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:01 smithi064 ceph-mon[31124]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-21T09:33:01.733 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:01 smithi064 ceph-mon[31124]: pgmap v12: 121 pgs: 121 active+clean; 138 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:33:01.733 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:01 smithi064 ceph-mon[31124]: osdmap e171: 8 total, 8 up, 8 in 2024-09-21T09:33:01.807 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:01 smithi022 ceph-mon[39525]: mgrmap e27: x(active, since 5s), standbys: y 2024-09-21T09:33:01.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:01 smithi022 ceph-mon[39525]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-21T09:33:01.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:01 smithi022 ceph-mon[39525]: pgmap v12: 121 pgs: 121 active+clean; 138 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:33:01.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:01 smithi022 ceph-mon[39525]: osdmap e171: 8 total, 8 up, 8 in 2024-09-21T09:33:01.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:01 smithi022 ceph-mon[27763]: mgrmap e27: x(active, since 5s), standbys: y 2024-09-21T09:33:01.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:01 smithi022 ceph-mon[27763]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-21T09:33:01.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:01 smithi022 ceph-mon[27763]: pgmap v12: 121 pgs: 121 active+clean; 138 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:33:01.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:01 smithi022 ceph-mon[27763]: osdmap e171: 8 total, 8 up, 8 in 2024-09-21T09:33:02.482 INFO:tasks.rados.rados.0.smithi022.stdout:573: done (3 left) 2024-09-21T09:33:02.482 INFO:tasks.rados.rados.0.smithi022.stdout:574: done (2 left) 2024-09-21T09:33:02.482 INFO:tasks.rados.rados.0.smithi022.stdout:576: done (1 left) 2024-09-21T09:33:02.482 INFO:tasks.rados.rados.0.smithi022.stdout:577: done (0 left) 2024-09-21T09:33:02.482 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 39 current snap is 57 2024-09-21T09:33:02.482 INFO:tasks.rados.rados.0.smithi022.stdout:578: seq_num 208 ranges {770048=688128,1458176=401408} 2024-09-21T09:33:02.489 INFO:tasks.rados.rados.0.smithi022.stdout:578: writing smithi02284960-39 from 770048 to 1458176 tid 1 2024-09-21T09:33:02.492 INFO:tasks.rados.rados.0.smithi022.stdout:578: writing smithi02284960-39 from 1458176 to 1859584 tid 2 2024-09-21T09:33:02.492 INFO:tasks.rados.rados.0.smithi022.stdout:579: read oid 48 snap -1 2024-09-21T09:33:02.492 INFO:tasks.rados.rados.0.smithi022.stdout:579: expect deleted 2024-09-21T09:33:02.492 INFO:tasks.rados.rados.0.smithi022.stdout:580: rollback oid 18 current snap is 57 2024-09-21T09:33:02.492 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 18 to 55 2024-09-21T09:33:02.492 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 44 current snap is 57 2024-09-21T09:33:02.492 INFO:tasks.rados.rados.0.smithi022.stdout:581: seq_num 209 ranges {0=704512,704512=401408} 2024-09-21T09:33:02.497 INFO:tasks.rados.rados.0.smithi022.stdout:581: writing smithi02284960-44 from 0 to 704512 tid 1 2024-09-21T09:33:02.500 INFO:tasks.rados.rados.0.smithi022.stdout:581: writing smithi02284960-44 from 704512 to 1105920 tid 2 2024-09-21T09:33:02.500 INFO:tasks.rados.rados.0.smithi022.stdout:582: snap_create 2024-09-21T09:33:02.500 INFO:tasks.rados.rados.0.smithi022.stdout:580: finishing rollback tid 0 to smithi02284960-18 2024-09-21T09:33:02.500 INFO:tasks.rados.rados.0.smithi022.stdout:580: finishing rollback tid 1 to smithi02284960-18 2024-09-21T09:33:02.500 INFO:tasks.rados.rados.0.smithi022.stdout:580: finishing rollback tid 2 to smithi02284960-18 2024-09-21T09:33:02.500 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 18 v 113 (ObjNum 178 snap 44 seq_num 178) dirty exists 2024-09-21T09:33:02.500 INFO:tasks.rados.rados.0.smithi022.stdout:578: finishing write tid 1 to smithi02284960-39 2024-09-21T09:33:02.501 INFO:tasks.rados.rados.0.smithi022.stdout:578: finishing write tid 2 to smithi02284960-39 2024-09-21T09:33:02.501 INFO:tasks.rados.rados.0.smithi022.stdout:578: finishing write tid 3 to smithi02284960-39 2024-09-21T09:33:02.501 INFO:tasks.rados.rados.0.smithi022.stdout:578: finishing write tid 4 to smithi02284960-39 2024-09-21T09:33:02.501 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 39 v 162 (ObjNum 208 snap 57 seq_num 208) dirty exists 2024-09-21T09:33:02.501 INFO:tasks.rados.rados.0.smithi022.stdout:578: left oid 39 (ObjNum 208 snap 57 seq_num 208) 2024-09-21T09:33:02.503 INFO:tasks.rados.rados.0.smithi022.stdout:581: finishing write tid 1 to smithi02284960-44 2024-09-21T09:33:02.503 INFO:tasks.rados.rados.0.smithi022.stdout:581: finishing write tid 2 to smithi02284960-44 2024-09-21T09:33:02.503 INFO:tasks.rados.rados.0.smithi022.stdout:581: finishing write tid 3 to smithi02284960-44 2024-09-21T09:33:02.504 INFO:tasks.rados.rados.0.smithi022.stdout:581: finishing write tid 4 to smithi02284960-44 2024-09-21T09:33:02.504 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 44 v 308 (ObjNum 209 snap 57 seq_num 209) dirty exists 2024-09-21T09:33:02.504 INFO:tasks.rados.rados.0.smithi022.stdout:581: left oid 44 (ObjNum 209 snap 57 seq_num 209) 2024-09-21T09:33:02.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:02 smithi064 ceph-mon[31124]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:02.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:02 smithi064 ceph-mon[31124]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:02.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:02 smithi064 ceph-mon[31124]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:02.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:02 smithi064 ceph-mon[31124]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:02.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:02 smithi064 ceph-mon[31124]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:33:02.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:02 smithi064 ceph-mon[31124]: from='mgr.24775 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:33:02.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:02 smithi064 ceph-mon[31124]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:33:02.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:02 smithi064 ceph-mon[31124]: from='mgr.24775 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:33:02.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:02 smithi064 ceph-mon[31124]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:33:02.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:02 smithi064 ceph-mon[31124]: from='mgr.24775 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:33:02.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:02 smithi064 ceph-mon[31124]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:33:02.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:02 smithi064 ceph-mon[31124]: from='mgr.24775 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:33:02.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:02 smithi064 ceph-mon[31124]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:02.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:02 smithi064 ceph-mon[31124]: osdmap e172: 8 total, 8 up, 8 in 2024-09-21T09:33:03.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:02 smithi022 ceph-mon[27763]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:03.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:02 smithi022 ceph-mon[27763]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:03.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:02 smithi022 ceph-mon[27763]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:03.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:02 smithi022 ceph-mon[27763]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:03.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:02 smithi022 ceph-mon[27763]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:33:03.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:02 smithi022 ceph-mon[27763]: from='mgr.24775 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:33:03.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:02 smithi022 ceph-mon[27763]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:33:03.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:02 smithi022 ceph-mon[27763]: from='mgr.24775 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:33:03.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:02 smithi022 ceph-mon[27763]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:33:03.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:02 smithi022 ceph-mon[27763]: from='mgr.24775 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:33:03.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:02 smithi022 ceph-mon[27763]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:33:03.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:02 smithi022 ceph-mon[27763]: from='mgr.24775 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:33:03.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:02 smithi022 ceph-mon[27763]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:03.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:02 smithi022 ceph-mon[27763]: osdmap e172: 8 total, 8 up, 8 in 2024-09-21T09:33:03.059 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:02 smithi022 ceph-mon[39525]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:03.059 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:02 smithi022 ceph-mon[39525]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:03.059 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:02 smithi022 ceph-mon[39525]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:03.059 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:02 smithi022 ceph-mon[39525]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:03.059 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:02 smithi022 ceph-mon[39525]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:33:03.060 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:02 smithi022 ceph-mon[39525]: from='mgr.24775 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:33:03.060 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:02 smithi022 ceph-mon[39525]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:33:03.060 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:02 smithi022 ceph-mon[39525]: from='mgr.24775 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:33:03.060 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:02 smithi022 ceph-mon[39525]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:33:03.060 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:02 smithi022 ceph-mon[39525]: from='mgr.24775 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:33:03.060 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:02 smithi022 ceph-mon[39525]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:33:03.060 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:02 smithi022 ceph-mon[39525]: from='mgr.24775 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:33:03.060 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:02 smithi022 ceph-mon[39525]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:03.060 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:02 smithi022 ceph-mon[39525]: osdmap e172: 8 total, 8 up, 8 in 2024-09-21T09:33:03.484 INFO:tasks.rados.rados.0.smithi022.stdout:578: done (4 left) 2024-09-21T09:33:03.484 INFO:tasks.rados.rados.0.smithi022.stdout:579: done (3 left) 2024-09-21T09:33:03.484 INFO:tasks.rados.rados.0.smithi022.stdout:580: done (2 left) 2024-09-21T09:33:03.484 INFO:tasks.rados.rados.0.smithi022.stdout:581: done (1 left) 2024-09-21T09:33:03.484 INFO:tasks.rados.rados.0.smithi022.stdout:582: done (0 left) 2024-09-21T09:33:03.484 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 18 current snap is 58 2024-09-21T09:33:03.484 INFO:tasks.rados.rados.0.smithi022.stdout:583: seq_num 210 ranges {876544=720896,1597440=401408} 2024-09-21T09:33:03.492 INFO:tasks.rados.rados.0.smithi022.stdout:583: writing smithi02284960-18 from 876544 to 1597440 tid 1 2024-09-21T09:33:03.495 INFO:tasks.rados.rados.0.smithi022.stdout:583: writing smithi02284960-18 from 1597440 to 1998848 tid 2 2024-09-21T09:33:03.495 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 39 current snap is 58 2024-09-21T09:33:03.495 INFO:tasks.rados.rados.0.smithi022.stdout:584: seq_num 211 ranges {1859584=737280,2596864=401408} 2024-09-21T09:33:03.509 INFO:tasks.rados.rados.0.smithi022.stdout:584: writing smithi02284960-39 from 1859584 to 2596864 tid 1 2024-09-21T09:33:03.512 INFO:tasks.rados.rados.0.smithi022.stdout:584: writing smithi02284960-39 from 2596864 to 2998272 tid 2 2024-09-21T09:33:03.513 INFO:tasks.rados.rados.0.smithi022.stdout:585: setattr oid 49 current snap is 58 2024-09-21T09:33:03.513 INFO:tasks.rados.rados.0.smithi022.stdout:583: finishing write tid 1 to smithi02284960-18 2024-09-21T09:33:03.513 INFO:tasks.rados.rados.0.smithi022.stdout:583: finishing write tid 2 to smithi02284960-18 2024-09-21T09:33:03.513 INFO:tasks.rados.rados.0.smithi022.stdout:583: finishing write tid 3 to smithi02284960-18 2024-09-21T09:33:03.513 INFO:tasks.rados.rados.0.smithi022.stdout:583: finishing write tid 4 to smithi02284960-18 2024-09-21T09:33:03.513 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 18 v 117 (ObjNum 210 snap 58 seq_num 210) dirty exists 2024-09-21T09:33:03.513 INFO:tasks.rados.rados.0.smithi022.stdout:583: left oid 18 (ObjNum 210 snap 58 seq_num 210) 2024-09-21T09:33:03.515 INFO:tasks.rados.rados.0.smithi022.stdout:583: done (2 left) 2024-09-21T09:33:03.515 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 8 current snap is 58 2024-09-21T09:33:03.515 INFO:tasks.rados.rados.0.smithi022.stdout:586: seq_num 212 ranges {942080=753664,1695744=401408} 2024-09-21T09:33:03.524 INFO:tasks.rados.rados.0.smithi022.stdout:586: writing smithi02284960-8 from 942080 to 1695744 tid 1 2024-09-21T09:33:03.527 INFO:tasks.rados.rados.0.smithi022.stdout:586: writing smithi02284960-8 from 1695744 to 2097152 tid 2 2024-09-21T09:33:03.528 INFO:tasks.rados.rados.0.smithi022.stdout:587: rollback oid 23 current snap is 58 2024-09-21T09:33:03.528 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 23 to 54 2024-09-21T09:33:03.528 INFO:tasks.rados.rados.0.smithi022.stdout:584: finishing write tid 1 to smithi02284960-39 2024-09-21T09:33:03.528 INFO:tasks.rados.rados.0.smithi022.stdout:584: finishing write tid 2 to smithi02284960-39 2024-09-21T09:33:03.528 INFO:tasks.rados.rados.0.smithi022.stdout:584: finishing write tid 3 to smithi02284960-39 2024-09-21T09:33:03.528 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 49 v 108 (ObjNum 196 snap 50 seq_num 196) dirty exists 2024-09-21T09:33:03.528 INFO:tasks.rados.rados.0.smithi022.stdout:584: finishing write tid 4 to smithi02284960-39 2024-09-21T09:33:03.528 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 39 v 166 (ObjNum 211 snap 58 seq_num 211) dirty exists 2024-09-21T09:33:03.528 INFO:tasks.rados.rados.0.smithi022.stdout:584: left oid 39 (ObjNum 211 snap 58 seq_num 211) 2024-09-21T09:33:03.528 INFO:tasks.rados.rados.0.smithi022.stdout:584: done (3 left) 2024-09-21T09:33:03.528 INFO:tasks.rados.rados.0.smithi022.stdout:585: done (2 left) 2024-09-21T09:33:03.528 INFO:tasks.rados.rados.0.smithi022.stdout:588: copy_from oid 46 from oid 12 current snap is 58 2024-09-21T09:33:03.528 INFO:tasks.rados.rados.0.smithi022.stdout:589: snap_remove snap 38 2024-09-21T09:33:04.091 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:03 smithi064 ceph-mon[31124]: Adjusting osd_memory_target on smithi022 to 2481M 2024-09-21T09:33:04.091 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:03 smithi064 ceph-mon[31124]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:04.092 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:03 smithi064 ceph-mon[31124]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:04.092 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:03 smithi064 ceph-mon[31124]: pgmap v15: 121 pgs: 121 active+clean; 142 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 1.0 MiB/s rd, 1.2 MiB/s wr, 31 op/s 2024-09-21T09:33:04.092 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:03 smithi064 ceph-mon[31124]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:04.092 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:03 smithi064 ceph-mon[31124]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:04.092 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:03 smithi064 ceph-mon[31124]: osdmap e173: 8 total, 8 up, 8 in 2024-09-21T09:33:04.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:03 smithi022 ceph-mon[39525]: Adjusting osd_memory_target on smithi022 to 2481M 2024-09-21T09:33:04.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:03 smithi022 ceph-mon[39525]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:04.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:03 smithi022 ceph-mon[39525]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:04.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:03 smithi022 ceph-mon[39525]: pgmap v15: 121 pgs: 121 active+clean; 142 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 1.0 MiB/s rd, 1.2 MiB/s wr, 31 op/s 2024-09-21T09:33:04.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:03 smithi022 ceph-mon[39525]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:04.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:03 smithi022 ceph-mon[39525]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:04.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:03 smithi022 ceph-mon[39525]: osdmap e173: 8 total, 8 up, 8 in 2024-09-21T09:33:04.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:03 smithi022 ceph-mon[27763]: Adjusting osd_memory_target on smithi022 to 2481M 2024-09-21T09:33:04.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:03 smithi022 ceph-mon[27763]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:04.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:03 smithi022 ceph-mon[27763]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:04.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:03 smithi022 ceph-mon[27763]: pgmap v15: 121 pgs: 121 active+clean; 142 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 1.0 MiB/s rd, 1.2 MiB/s wr, 31 op/s 2024-09-21T09:33:04.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:03 smithi022 ceph-mon[27763]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:04.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:03 smithi022 ceph-mon[27763]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:04.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:03 smithi022 ceph-mon[27763]: osdmap e173: 8 total, 8 up, 8 in 2024-09-21T09:33:04.486 INFO:tasks.rados.rados.0.smithi022.stdout:589: done (3 left) 2024-09-21T09:33:04.486 INFO:tasks.rados.rados.0.smithi022.stdout:590: setattr oid 35 current snap is 58 2024-09-21T09:33:04.486 INFO:tasks.rados.rados.0.smithi022.stdout:586: finishing write tid 1 to smithi02284960-8 2024-09-21T09:33:04.486 INFO:tasks.rados.rados.0.smithi022.stdout:586: finishing write tid 2 to smithi02284960-8 2024-09-21T09:33:04.486 INFO:tasks.rados.rados.0.smithi022.stdout:586: finishing write tid 3 to smithi02284960-8 2024-09-21T09:33:04.486 INFO:tasks.rados.rados.0.smithi022.stdout:587: finishing rollback tid 0 to smithi02284960-23 2024-09-21T09:33:04.486 INFO:tasks.rados.rados.0.smithi022.stdout:587: finishing rollback tid 1 to smithi02284960-23 2024-09-21T09:33:04.487 INFO:tasks.rados.rados.0.smithi022.stdout:587: finishing rollback tid 2 to smithi02284960-23 2024-09-21T09:33:04.487 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 23 v 140 (ObjNum 200 snap 51 seq_num 200) dirty exists 2024-09-21T09:33:04.487 INFO:tasks.rados.rados.0.smithi022.stdout:586: finishing write tid 4 to smithi02284960-8 2024-09-21T09:33:04.487 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 8 v 191 (ObjNum 212 snap 58 seq_num 212) dirty exists 2024-09-21T09:33:04.487 INFO:tasks.rados.rados.0.smithi022.stdout:586: left oid 8 (ObjNum 212 snap 58 seq_num 212) 2024-09-21T09:33:04.487 INFO:tasks.rados.rados.0.smithi022.stdout:588: finishing copy_from racing read to smithi02284960-46 2024-09-21T09:33:04.487 INFO:tasks.rados.rados.0.smithi022.stdout:588: finishing copy_from to smithi02284960-46 2024-09-21T09:33:04.487 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 46 v 275 (ObjNum 187 snap 47 seq_num 187) dirty exists 2024-09-21T09:33:04.489 INFO:tasks.rados.rados.0.smithi022.stdout:586: done (3 left) 2024-09-21T09:33:04.489 INFO:tasks.rados.rados.0.smithi022.stdout:587: done (2 left) 2024-09-21T09:33:04.489 INFO:tasks.rados.rados.0.smithi022.stdout:588: done (1 left) 2024-09-21T09:33:04.489 INFO:tasks.rados.rados.0.smithi022.stdout:591: read oid 21 snap 57 2024-09-21T09:33:04.489 INFO:tasks.rados.rados.0.smithi022.stdout:591: expect (ObjNum 172 snap 43 seq_num 172) 2024-09-21T09:33:04.489 INFO:tasks.rados.rados.0.smithi022.stdout:592: copy_from oid 39 from oid 12 current snap is 58 2024-09-21T09:33:04.489 INFO:tasks.rados.rados.0.smithi022.stdout:593: delete oid 4 current snap is 58 2024-09-21T09:33:04.493 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 35 v 93 (ObjNum 79 snap 11 seq_num 79) dirty exists 2024-09-21T09:33:04.498 INFO:tasks.rados.rados.0.smithi022.stdout:590: done (3 left) 2024-09-21T09:33:04.498 INFO:tasks.rados.rados.0.smithi022.stdout:593: done (2 left) 2024-09-21T09:33:04.498 INFO:tasks.rados.rados.0.smithi022.stdout:594: snap_create 2024-09-21T09:33:04.501 INFO:tasks.rados.rados.0.smithi022.stdout:591: expect (ObjNum 172 snap 43 seq_num 172) 2024-09-21T09:33:04.528 INFO:tasks.rados.rados.0.smithi022.stdout:592: finishing copy_from racing read to smithi02284960-39 2024-09-21T09:33:04.533 INFO:tasks.rados.rados.0.smithi022.stdout:592: finishing copy_from to smithi02284960-39 2024-09-21T09:33:04.533 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 39 v 275 (ObjNum 187 snap 47 seq_num 187) dirty exists 2024-09-21T09:33:05.438 INFO:tasks.rados.rados.0.smithi022.stdout:591: done (2 left) 2024-09-21T09:33:05.438 INFO:tasks.rados.rados.0.smithi022.stdout:592: done (1 left) 2024-09-21T09:33:05.438 INFO:tasks.rados.rados.0.smithi022.stdout:594: done (0 left) 2024-09-21T09:33:05.438 INFO:tasks.rados.rados.0.smithi022.stdout:595: read oid 45 snap -1 2024-09-21T09:33:05.438 INFO:tasks.rados.rados.0.smithi022.stdout:595: expect deleted 2024-09-21T09:33:05.438 INFO:tasks.rados.rados.0.smithi022.stdout:596: copy_from oid 49 from oid 4 current snap is 59 2024-09-21T09:33:05.438 INFO:tasks.rados.rados.0.smithi022.stdout:597: snap_remove snap 55 2024-09-21T09:33:05.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:05 smithi022 ceph-mon[39525]: osdmap e174: 8 total, 8 up, 8 in 2024-09-21T09:33:05.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:05 smithi022 ceph-mon[39525]: pgmap v18: 121 pgs: 121 active+clean; 142 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 1.0 MiB/s rd, 1.2 MiB/s wr, 31 op/s 2024-09-21T09:33:05.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:05 smithi022 ceph-mon[39525]: osdmap e175: 8 total, 8 up, 8 in 2024-09-21T09:33:05.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:05 smithi022 ceph-mon[27763]: osdmap e174: 8 total, 8 up, 8 in 2024-09-21T09:33:05.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:05 smithi022 ceph-mon[27763]: pgmap v18: 121 pgs: 121 active+clean; 142 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 1.0 MiB/s rd, 1.2 MiB/s wr, 31 op/s 2024-09-21T09:33:05.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:05 smithi022 ceph-mon[27763]: osdmap e175: 8 total, 8 up, 8 in 2024-09-21T09:33:05.938 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:05 smithi064 ceph-mon[31124]: osdmap e174: 8 total, 8 up, 8 in 2024-09-21T09:33:05.939 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:05 smithi064 ceph-mon[31124]: pgmap v18: 121 pgs: 121 active+clean; 142 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 1.0 MiB/s rd, 1.2 MiB/s wr, 31 op/s 2024-09-21T09:33:05.939 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:05 smithi064 ceph-mon[31124]: osdmap e175: 8 total, 8 up, 8 in 2024-09-21T09:33:06.440 INFO:tasks.rados.rados.0.smithi022.stdout:597: done (2 left) 2024-09-21T09:33:06.440 INFO:tasks.rados.rados.0.smithi022.stdout:598: rollback oid 14 current snap is 59 2024-09-21T09:33:06.440 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 14 to 48 2024-09-21T09:33:06.440 INFO:tasks.rados.rados.0.smithi022.stdout:599: snap_create 2024-09-21T09:33:06.440 INFO:tasks.rados.rados.0.smithi022.stdout:596: finishing copy_from to smithi02284960-49 2024-09-21T09:33:06.440 INFO:tasks.rados.rados.0.smithi022.stdout:596: got expected ENOENT (src dne) 2024-09-21T09:33:06.440 INFO:tasks.rados.rados.0.smithi022.stdout:596: finishing copy_from racing read to smithi02284960-49 2024-09-21T09:33:06.443 INFO:tasks.rados.rados.0.smithi022.stdout:598: finishing rollback tid 0 to smithi02284960-14 2024-09-21T09:33:06.444 INFO:tasks.rados.rados.0.smithi022.stdout:598: finishing rollback tid 1 to smithi02284960-14 2024-09-21T09:33:06.444 INFO:tasks.rados.rados.0.smithi022.stdout:598: finishing rollback tid 2 to smithi02284960-14 2024-09-21T09:33:06.444 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 14 v 114 (ObjNum 115 snap 23 seq_num 115) dirty exists 2024-09-21T09:33:07.440 INFO:tasks.rados.rados.0.smithi022.stdout:595: done (3 left) 2024-09-21T09:33:07.440 INFO:tasks.rados.rados.0.smithi022.stdout:596: done (2 left) 2024-09-21T09:33:07.440 INFO:tasks.rados.rados.0.smithi022.stdout:598: done (1 left) 2024-09-21T09:33:07.440 INFO:tasks.rados.rados.0.smithi022.stdout:599: done (0 left) 2024-09-21T09:33:07.440 INFO:tasks.rados.rados.0.smithi022.stdout:600: copy_from oid 16 from oid 10 current snap is 60 2024-09-21T09:33:07.440 INFO:tasks.rados.rados.0.smithi022.stdout:601: snap_remove snap 57 2024-09-21T09:33:07.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:07 smithi064 ceph-mon[31124]: osdmap e176: 8 total, 8 up, 8 in 2024-09-21T09:33:07.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:07 smithi064 ceph-mon[31124]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:07.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:07 smithi064 ceph-mon[31124]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:07.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:07 smithi064 ceph-mon[31124]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:33:07.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:07 smithi064 ceph-mon[31124]: from='mgr.24775 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:33:07.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:07 smithi064 ceph-mon[31124]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:33:07.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:07 smithi064 ceph-mon[31124]: from='mgr.24775 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:33:07.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:07 smithi064 ceph-mon[31124]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:33:07.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:07 smithi064 ceph-mon[31124]: from='mgr.24775 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:33:07.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:07 smithi064 ceph-mon[31124]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:33:07.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:07 smithi064 ceph-mon[31124]: from='mgr.24775 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:33:07.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:07 smithi064 ceph-mon[31124]: Adjusting osd_memory_target on smithi064 to 2480M 2024-09-21T09:33:07.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:07 smithi064 ceph-mon[31124]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:07.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:07 smithi064 ceph-mon[31124]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:33:07.693 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:07 smithi064 ceph-mon[31124]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:33:07.693 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:07 smithi064 ceph-mon[31124]: Updating smithi022:/etc/ceph/ceph.conf 2024-09-21T09:33:07.693 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:07 smithi064 ceph-mon[31124]: Updating smithi064:/etc/ceph/ceph.conf 2024-09-21T09:33:07.693 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:07 smithi064 ceph-mon[31124]: Updating smithi064:/var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:33:07.693 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:07 smithi064 ceph-mon[31124]: Updating smithi022:/var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:33:07.693 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:07 smithi064 ceph-mon[31124]: Updating smithi064:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:33:07.693 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:07 smithi064 ceph-mon[31124]: pgmap v21: 121 pgs: 121 active+clean; 160 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.1 MiB/s rd, 4.7 MiB/s wr, 22 op/s 2024-09-21T09:33:07.693 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:07 smithi064 ceph-mon[31124]: Updating smithi022:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:33:07.693 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:07 smithi064 ceph-mon[31124]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:07.693 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:07 smithi064 ceph-mon[31124]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:07.693 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:07 smithi064 ceph-mon[31124]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:07.693 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:07 smithi064 ceph-mon[31124]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:07.694 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:07 smithi064 ceph-mon[31124]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:07.694 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:07 smithi064 ceph-mon[31124]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:07.694 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:07 smithi064 ceph-mon[31124]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:07.694 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:07 smithi064 ceph-mon[31124]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:07.694 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:07 smithi064 ceph-mon[31124]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:07.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:07 smithi022 ceph-mon[39525]: osdmap e176: 8 total, 8 up, 8 in 2024-09-21T09:33:07.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:07 smithi022 ceph-mon[39525]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:07.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:07 smithi022 ceph-mon[39525]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:07.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:07 smithi022 ceph-mon[39525]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:33:07.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:07 smithi022 ceph-mon[39525]: from='mgr.24775 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:33:07.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:07 smithi022 ceph-mon[39525]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:33:07.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:07 smithi022 ceph-mon[39525]: from='mgr.24775 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:33:07.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:07 smithi022 ceph-mon[39525]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:33:07.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:07 smithi022 ceph-mon[39525]: from='mgr.24775 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:33:07.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:07 smithi022 ceph-mon[39525]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:33:07.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:07 smithi022 ceph-mon[39525]: from='mgr.24775 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:33:07.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:07 smithi022 ceph-mon[39525]: Adjusting osd_memory_target on smithi064 to 2480M 2024-09-21T09:33:07.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:07 smithi022 ceph-mon[39525]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:07.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:07 smithi022 ceph-mon[39525]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:33:07.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:07 smithi022 ceph-mon[39525]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:33:07.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:07 smithi022 ceph-mon[39525]: Updating smithi022:/etc/ceph/ceph.conf 2024-09-21T09:33:07.810 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:07 smithi022 ceph-mon[39525]: Updating smithi064:/etc/ceph/ceph.conf 2024-09-21T09:33:07.810 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:07 smithi022 ceph-mon[39525]: Updating smithi064:/var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:33:07.810 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:07 smithi022 ceph-mon[39525]: Updating smithi022:/var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:33:07.810 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:07 smithi022 ceph-mon[39525]: Updating smithi064:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:33:07.810 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:07 smithi022 ceph-mon[39525]: pgmap v21: 121 pgs: 121 active+clean; 160 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.1 MiB/s rd, 4.7 MiB/s wr, 22 op/s 2024-09-21T09:33:07.810 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:07 smithi022 ceph-mon[39525]: Updating smithi022:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:33:07.810 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:07 smithi022 ceph-mon[39525]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:07.810 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:07 smithi022 ceph-mon[39525]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:07.810 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:07 smithi022 ceph-mon[39525]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:07.810 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:07 smithi022 ceph-mon[39525]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:07.810 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:07 smithi022 ceph-mon[39525]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:07.810 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:07 smithi022 ceph-mon[39525]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:07.811 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:07 smithi022 ceph-mon[39525]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:07.811 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:07 smithi022 ceph-mon[39525]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:07.811 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:07 smithi022 ceph-mon[39525]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:07.811 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:07 smithi022 ceph-mon[27763]: osdmap e176: 8 total, 8 up, 8 in 2024-09-21T09:33:07.811 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:07 smithi022 ceph-mon[27763]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:07.811 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:07 smithi022 ceph-mon[27763]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:07.811 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:07 smithi022 ceph-mon[27763]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:33:07.811 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:07 smithi022 ceph-mon[27763]: from='mgr.24775 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:33:07.811 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:07 smithi022 ceph-mon[27763]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:33:07.811 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:07 smithi022 ceph-mon[27763]: from='mgr.24775 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:33:07.811 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:07 smithi022 ceph-mon[27763]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:33:07.812 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:07 smithi022 ceph-mon[27763]: from='mgr.24775 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:33:07.812 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:07 smithi022 ceph-mon[27763]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:33:07.812 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:07 smithi022 ceph-mon[27763]: from='mgr.24775 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:33:07.812 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:07 smithi022 ceph-mon[27763]: Adjusting osd_memory_target on smithi064 to 2480M 2024-09-21T09:33:07.812 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:07 smithi022 ceph-mon[27763]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:07.812 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:07 smithi022 ceph-mon[27763]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:33:07.812 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:07 smithi022 ceph-mon[27763]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:33:07.812 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:07 smithi022 ceph-mon[27763]: Updating smithi022:/etc/ceph/ceph.conf 2024-09-21T09:33:07.812 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:07 smithi022 ceph-mon[27763]: Updating smithi064:/etc/ceph/ceph.conf 2024-09-21T09:33:07.812 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:07 smithi022 ceph-mon[27763]: Updating smithi064:/var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:33:07.812 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:07 smithi022 ceph-mon[27763]: Updating smithi022:/var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:33:07.812 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:07 smithi022 ceph-mon[27763]: Updating smithi064:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:33:07.812 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:07 smithi022 ceph-mon[27763]: pgmap v21: 121 pgs: 121 active+clean; 160 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.1 MiB/s rd, 4.7 MiB/s wr, 22 op/s 2024-09-21T09:33:07.812 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:07 smithi022 ceph-mon[27763]: Updating smithi022:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:33:07.812 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:07 smithi022 ceph-mon[27763]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:07.813 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:07 smithi022 ceph-mon[27763]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:07.813 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:07 smithi022 ceph-mon[27763]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:07.813 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:07 smithi022 ceph-mon[27763]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:07.813 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:07 smithi022 ceph-mon[27763]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:07.813 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:07 smithi022 ceph-mon[27763]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:07.813 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:07 smithi022 ceph-mon[27763]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:07.813 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:07 smithi022 ceph-mon[27763]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:07.813 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:07 smithi022 ceph-mon[27763]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:08.450 INFO:tasks.rados.rados.0.smithi022.stdout:601: done (1 left) 2024-09-21T09:33:08.450 INFO:tasks.rados.rados.0.smithi022.stdout:602: read oid 35 snap -1 2024-09-21T09:33:08.450 INFO:tasks.rados.rados.0.smithi022.stdout:602: expect (ObjNum 79 snap 11 seq_num 79) 2024-09-21T09:33:08.450 INFO:tasks.rados.rados.0.smithi022.stdout:603: rollback oid 40 current snap is 60 2024-09-21T09:33:08.450 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 40 to 43 2024-09-21T09:33:08.450 INFO:tasks.rados.rados.0.smithi022.stdout:604: read oid 43 snap -1 2024-09-21T09:33:08.451 INFO:tasks.rados.rados.0.smithi022.stdout:604: expect (ObjNum 32 snap 0 seq_num 32) 2024-09-21T09:33:08.451 INFO:tasks.rados.rados.0.smithi022.stdout:605: rollback oid 39 current snap is 60 2024-09-21T09:33:08.451 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 39 to 42 2024-09-21T09:33:08.451 INFO:tasks.rados.rados.0.smithi022.stdout:606: snap_create 2024-09-21T09:33:08.451 INFO:tasks.rados.rados.0.smithi022.stdout:600: finishing copy_from racing read to smithi02284960-16 2024-09-21T09:33:08.451 INFO:tasks.rados.rados.0.smithi022.stdout:600: finishing copy_from to smithi02284960-16 2024-09-21T09:33:08.451 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 16 v 241 (ObjNum 144 snap 32 seq_num 144) dirty exists 2024-09-21T09:33:08.458 INFO:tasks.rados.rados.0.smithi022.stdout:605: finishing rollback tid 0 to smithi02284960-39 2024-09-21T09:33:08.460 INFO:tasks.rados.rados.0.smithi022.stdout:605: finishing rollback tid 1 to smithi02284960-39 2024-09-21T09:33:08.461 INFO:tasks.rados.rados.0.smithi022.stdout:605: finishing rollback tid 2 to smithi02284960-39 2024-09-21T09:33:08.461 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 39 v 278 (ObjNum 151 snap 36 seq_num 151) dirty exists 2024-09-21T09:33:08.461 INFO:tasks.rados.rados.0.smithi022.stdout:603: finishing rollback tid 0 to smithi02284960-40 2024-09-21T09:33:08.462 INFO:tasks.rados.rados.0.smithi022.stdout:603: finishing rollback tid 1 to smithi02284960-40 2024-09-21T09:33:08.462 INFO:tasks.rados.rados.0.smithi022.stdout:602: expect (ObjNum 79 snap 11 seq_num 79) 2024-09-21T09:33:08.526 INFO:tasks.rados.rados.0.smithi022.stdout:603: finishing rollback tid 2 to smithi02284960-40 2024-09-21T09:33:08.526 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 40 v 265 (ObjNum 92 snap 18 seq_num 92) dirty exists 2024-09-21T09:33:08.527 INFO:tasks.rados.rados.0.smithi022.stdout:604: expect (ObjNum 32 snap 0 seq_num 32) 2024-09-21T09:33:08.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:08 smithi022 ceph-mon[39525]: Updating smithi064:/var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:33:08.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:08 smithi022 ceph-mon[39525]: Updating smithi022:/var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:33:08.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:08 smithi022 ceph-mon[39525]: Reconfiguring alertmanager.a (dependencies changed)... 2024-09-21T09:33:08.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:08 smithi022 ceph-mon[39525]: Deploying daemon alertmanager.a on smithi022 2024-09-21T09:33:08.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:08 smithi022 ceph-mon[39525]: osdmap e177: 8 total, 8 up, 8 in 2024-09-21T09:33:08.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:08 smithi022 ceph-mon[27763]: Updating smithi064:/var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:33:08.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:08 smithi022 ceph-mon[27763]: Updating smithi022:/var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:33:08.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:08 smithi022 ceph-mon[27763]: Reconfiguring alertmanager.a (dependencies changed)... 2024-09-21T09:33:08.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:08 smithi022 ceph-mon[27763]: Deploying daemon alertmanager.a on smithi022 2024-09-21T09:33:08.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:08 smithi022 ceph-mon[27763]: osdmap e177: 8 total, 8 up, 8 in 2024-09-21T09:33:08.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:08 smithi064 ceph-mon[31124]: Updating smithi064:/var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:33:08.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:08 smithi064 ceph-mon[31124]: Updating smithi022:/var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:33:08.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:08 smithi064 ceph-mon[31124]: Reconfiguring alertmanager.a (dependencies changed)... 2024-09-21T09:33:08.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:08 smithi064 ceph-mon[31124]: Deploying daemon alertmanager.a on smithi022 2024-09-21T09:33:08.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:08 smithi064 ceph-mon[31124]: osdmap e177: 8 total, 8 up, 8 in 2024-09-21T09:33:09.455 INFO:tasks.rados.rados.0.smithi022.stdout:600: done (5 left) 2024-09-21T09:33:09.455 INFO:tasks.rados.rados.0.smithi022.stdout:602: done (4 left) 2024-09-21T09:33:09.455 INFO:tasks.rados.rados.0.smithi022.stdout:603: done (3 left) 2024-09-21T09:33:09.455 INFO:tasks.rados.rados.0.smithi022.stdout:604: done (2 left) 2024-09-21T09:33:09.455 INFO:tasks.rados.rados.0.smithi022.stdout:605: done (1 left) 2024-09-21T09:33:09.455 INFO:tasks.rados.rados.0.smithi022.stdout:606: done (0 left) 2024-09-21T09:33:09.455 INFO:tasks.rados.rados.0.smithi022.stdout:607: rollback oid 44 current snap is 61 2024-09-21T09:33:09.455 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 44 to 51 2024-09-21T09:33:09.455 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 30 current snap is 61 2024-09-21T09:33:09.456 INFO:tasks.rados.rados.0.smithi022.stdout:608: seq_num 214 ranges {0=794624,794624=401408} 2024-09-21T09:33:09.459 INFO:tasks.rados.rados.0.smithi022.stdout:608: writing smithi02284960-30 from 0 to 794624 tid 1 2024-09-21T09:33:09.462 INFO:tasks.rados.rados.0.smithi022.stdout:608: writing smithi02284960-30 from 794624 to 1196032 tid 2 2024-09-21T09:33:09.462 INFO:tasks.rados.rados.0.smithi022.stdout:609: rollback oid 40 current snap is 61 2024-09-21T09:33:09.462 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 40 to 49 2024-09-21T09:33:09.462 INFO:tasks.rados.rados.0.smithi022.stdout:607: finishing rollback tid 0 to smithi02284960-44 2024-09-21T09:33:09.463 INFO:tasks.rados.rados.0.smithi022.stdout:607: finishing rollback tid 1 to smithi02284960-44 2024-09-21T09:33:09.463 INFO:tasks.rados.rados.0.smithi022.stdout:610: snap_remove snap 48 2024-09-21T09:33:09.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:09 smithi022 ceph-mon[39525]: osdmap e178: 8 total, 8 up, 8 in 2024-09-21T09:33:09.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:09 smithi022 ceph-mon[39525]: pgmap v24: 121 pgs: 121 active+clean; 160 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.1 MiB/s rd, 4.7 MiB/s wr, 22 op/s 2024-09-21T09:33:09.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:09 smithi022 ceph-mon[27763]: osdmap e178: 8 total, 8 up, 8 in 2024-09-21T09:33:09.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:09 smithi022 ceph-mon[27763]: pgmap v24: 121 pgs: 121 active+clean; 160 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.1 MiB/s rd, 4.7 MiB/s wr, 22 op/s 2024-09-21T09:33:09.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:09 smithi064 ceph-mon[31124]: osdmap e178: 8 total, 8 up, 8 in 2024-09-21T09:33:09.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:09 smithi064 ceph-mon[31124]: pgmap v24: 121 pgs: 121 active+clean; 160 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.1 MiB/s rd, 4.7 MiB/s wr, 22 op/s 2024-09-21T09:33:10.439 INFO:tasks.rados.rados.0.smithi022.stdout:610: done (3 left) 2024-09-21T09:33:10.439 INFO:tasks.rados.rados.0.smithi022.stdout:611: snap_create 2024-09-21T09:33:10.439 INFO:tasks.rados.rados.0.smithi022.stdout:607: finishing rollback tid 2 to smithi02284960-44 2024-09-21T09:33:10.439 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 44 v 311 (ObjNum 33 snap 0 seq_num 3569914752) dirty exists 2024-09-21T09:33:10.439 INFO:tasks.rados.rados.0.smithi022.stdout:608: finishing write tid 1 to smithi02284960-30 2024-09-21T09:33:10.439 INFO:tasks.rados.rados.0.smithi022.stdout:608: finishing write tid 2 to smithi02284960-30 2024-09-21T09:33:10.439 INFO:tasks.rados.rados.0.smithi022.stdout:608: finishing write tid 3 to smithi02284960-30 2024-09-21T09:33:10.439 INFO:tasks.rados.rados.0.smithi022.stdout:608: finishing write tid 4 to smithi02284960-30 2024-09-21T09:33:10.439 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 30 v 281 (ObjNum 214 snap 61 seq_num 214) dirty exists 2024-09-21T09:33:10.439 INFO:tasks.rados.rados.0.smithi022.stdout:608: left oid 30 (ObjNum 214 snap 61 seq_num 214) 2024-09-21T09:33:10.439 INFO:tasks.rados.rados.0.smithi022.stdout:609: finishing rollback tid 0 to smithi02284960-40 2024-09-21T09:33:10.439 INFO:tasks.rados.rados.0.smithi022.stdout:609: finishing rollback tid 1 to smithi02284960-40 2024-09-21T09:33:10.439 INFO:tasks.rados.rados.0.smithi022.stdout:609: finishing rollback tid 2 to smithi02284960-40 2024-09-21T09:33:10.439 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 40 v 269 (ObjNum 92 snap 18 seq_num 92) dirty exists 2024-09-21T09:33:10.807 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:10 smithi022 ceph-mon[39525]: osdmap e179: 8 total, 8 up, 8 in 2024-09-21T09:33:10.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:10 smithi022 ceph-mon[39525]: osdmap e180: 8 total, 8 up, 8 in 2024-09-21T09:33:10.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:10 smithi022 ceph-mon[27763]: osdmap e179: 8 total, 8 up, 8 in 2024-09-21T09:33:10.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:10 smithi022 ceph-mon[27763]: osdmap e180: 8 total, 8 up, 8 in 2024-09-21T09:33:10.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:10 smithi064 ceph-mon[31124]: osdmap e179: 8 total, 8 up, 8 in 2024-09-21T09:33:10.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:10 smithi064 ceph-mon[31124]: osdmap e180: 8 total, 8 up, 8 in 2024-09-21T09:33:10.942 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:33:10 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[67503]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:33:10] "GET /metrics HTTP/1.1" 200 35848 "" "Prometheus/2.43.0" 2024-09-21T09:33:11.439 INFO:tasks.rados.rados.0.smithi022.stdout:607: done (3 left) 2024-09-21T09:33:11.439 INFO:tasks.rados.rados.0.smithi022.stdout:608: done (2 left) 2024-09-21T09:33:11.440 INFO:tasks.rados.rados.0.smithi022.stdout:609: done (1 left) 2024-09-21T09:33:11.440 INFO:tasks.rados.rados.0.smithi022.stdout:611: done (0 left) 2024-09-21T09:33:11.440 INFO:tasks.rados.rados.0.smithi022.stdout:612: rmattr oid 40 current snap is 62 2024-09-21T09:33:11.440 INFO:tasks.rados.rados.0.smithi022.stdout:613: read oid 48 snap 43 2024-09-21T09:33:11.440 INFO:tasks.rados.rados.0.smithi022.stdout:613: expect (ObjNum 149 snap 35 seq_num 149) 2024-09-21T09:33:11.440 INFO:tasks.rados.rados.0.smithi022.stdout:614: copy_from oid 43 from oid 12 current snap is 62 2024-09-21T09:33:11.440 INFO:tasks.rados.rados.0.smithi022.stdout:615: read oid 20 snap -1 2024-09-21T09:33:11.440 INFO:tasks.rados.rados.0.smithi022.stdout:615: expect (ObjNum 132 snap 27 seq_num 132) 2024-09-21T09:33:11.440 INFO:tasks.rados.rados.0.smithi022.stdout:616: rollback oid 50 current snap is 62 2024-09-21T09:33:11.440 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 50 to 56 2024-09-21T09:33:11.440 INFO:tasks.rados.rados.0.smithi022.stdout:617: delete oid 2 current snap is 62 2024-09-21T09:33:11.444 INFO:tasks.rados.rados.0.smithi022.stdout:616: finishing rollback tid 1 to smithi02284960-50 2024-09-21T09:33:11.444 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 50 v 189 (ObjNum 129 snap 0 seq_num 3569728552) dirty dne 2024-09-21T09:33:11.447 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 40 v 275 (ObjNum 92 snap 18 seq_num 92) dirty exists 2024-09-21T09:33:11.448 INFO:tasks.rados.rados.0.smithi022.stdout:612: done (5 left) 2024-09-21T09:33:11.448 INFO:tasks.rados.rados.0.smithi022.stdout:616: done (4 left) 2024-09-21T09:33:11.448 INFO:tasks.rados.rados.0.smithi022.stdout:617: done (3 left) 2024-09-21T09:33:11.448 INFO:tasks.rados.rados.0.smithi022.stdout:618: read oid 6 snap -1 2024-09-21T09:33:11.448 INFO:tasks.rados.rados.0.smithi022.stdout:618: expect deleted 2024-09-21T09:33:11.448 INFO:tasks.rados.rados.0.smithi022.stdout:619: setattr oid 37 current snap is 62 2024-09-21T09:33:11.450 INFO:tasks.rados.rados.0.smithi022.stdout:620: rmattr oid 35 current snap is 62 2024-09-21T09:33:11.450 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 40 current snap is 62 2024-09-21T09:33:11.450 INFO:tasks.rados.rados.0.smithi022.stdout:621: seq_num 216 ranges {1548288=425984,1974272=409600,2383872=393216} 2024-09-21T09:33:11.461 INFO:tasks.rados.rados.0.smithi022.stdout:621: writing smithi02284960-40 from 1548288 to 1974272 tid 1 2024-09-21T09:33:11.464 INFO:tasks.rados.rados.0.smithi022.stdout:621: writing smithi02284960-40 from 1974272 to 2383872 tid 2 2024-09-21T09:33:11.466 INFO:tasks.rados.rados.0.smithi022.stdout:621: writing smithi02284960-40 from 2383872 to 2777088 tid 3 2024-09-21T09:33:11.467 INFO:tasks.rados.rados.0.smithi022.stdout:622: snap_create 2024-09-21T09:33:11.467 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 35 v 101 (ObjNum 79 snap 11 seq_num 79) dirty exists 2024-09-21T09:33:11.467 INFO:tasks.rados.rados.0.smithi022.stdout:613: expect (ObjNum 149 snap 35 seq_num 149) 2024-09-21T09:33:11.557 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:33:11 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:33:11] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-21T09:33:11.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:11 smithi022 ceph-mon[39525]: pgmap v27: 121 pgs: 121 active+clean; 160 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:33:11.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:11 smithi022 ceph-mon[39525]: osdmap e181: 8 total, 8 up, 8 in 2024-09-21T09:33:11.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:11 smithi022 ceph-mon[27763]: pgmap v27: 121 pgs: 121 active+clean; 160 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:33:11.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:11 smithi022 ceph-mon[27763]: osdmap e181: 8 total, 8 up, 8 in 2024-09-21T09:33:11.616 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 37 v 331 (ObjNum 161 snap 0 seq_num 0) dirty exists 2024-09-21T09:33:11.617 INFO:tasks.rados.rados.0.smithi022.stdout:615: expect (ObjNum 132 snap 27 seq_num 132) 2024-09-21T09:33:11.783 INFO:tasks.rados.rados.0.smithi022.stdout:614: finishing copy_from racing read to smithi02284960-43 2024-09-21T09:33:11.783 INFO:tasks.rados.rados.0.smithi022.stdout:614: finishing copy_from to smithi02284960-43 2024-09-21T09:33:11.783 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 43 v 275 (ObjNum 187 snap 47 seq_num 187) dirty exists 2024-09-21T09:33:11.783 INFO:tasks.rados.rados.0.smithi022.stdout:621: finishing write tid 1 to smithi02284960-40 2024-09-21T09:33:11.783 INFO:tasks.rados.rados.0.smithi022.stdout:621: finishing write tid 2 to smithi02284960-40 2024-09-21T09:33:11.783 INFO:tasks.rados.rados.0.smithi022.stdout:621: finishing write tid 3 to smithi02284960-40 2024-09-21T09:33:11.783 INFO:tasks.rados.rados.0.smithi022.stdout:621: finishing write tid 4 to smithi02284960-40 2024-09-21T09:33:11.783 INFO:tasks.rados.rados.0.smithi022.stdout:621: finishing write tid 5 to smithi02284960-40 2024-09-21T09:33:11.783 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 40 v 281 (ObjNum 216 snap 62 seq_num 216) dirty exists 2024-09-21T09:33:11.783 INFO:tasks.rados.rados.0.smithi022.stdout:621: left oid 40 (ObjNum 216 snap 62 seq_num 216) 2024-09-21T09:33:11.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:11 smithi064 ceph-mon[31124]: pgmap v27: 121 pgs: 121 active+clean; 160 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:33:11.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:11 smithi064 ceph-mon[31124]: osdmap e181: 8 total, 8 up, 8 in 2024-09-21T09:33:12.470 INFO:tasks.rados.rados.0.smithi022.stdout:613: done (7 left) 2024-09-21T09:33:12.470 INFO:tasks.rados.rados.0.smithi022.stdout:614: done (6 left) 2024-09-21T09:33:12.470 INFO:tasks.rados.rados.0.smithi022.stdout:615: done (5 left) 2024-09-21T09:33:12.470 INFO:tasks.rados.rados.0.smithi022.stdout:618: done (4 left) 2024-09-21T09:33:12.470 INFO:tasks.rados.rados.0.smithi022.stdout:619: done (3 left) 2024-09-21T09:33:12.470 INFO:tasks.rados.rados.0.smithi022.stdout:620: done (2 left) 2024-09-21T09:33:12.470 INFO:tasks.rados.rados.0.smithi022.stdout:621: done (1 left) 2024-09-21T09:33:12.470 INFO:tasks.rados.rados.0.smithi022.stdout:622: done (0 left) 2024-09-21T09:33:12.471 INFO:tasks.rados.rados.0.smithi022.stdout:623: snap_remove snap 49 2024-09-21T09:33:12.794 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:12 smithi022 ceph-mon[39525]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:33:12.794 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:12 smithi022 ceph-mon[27763]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:33:12.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:12 smithi064 ceph-mon[31124]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:33:13.476 INFO:tasks.rados.rados.0.smithi022.stdout:623: done (0 left) 2024-09-21T09:33:13.476 INFO:tasks.rados.rados.0.smithi022.stdout:624: snap_remove snap 47 2024-09-21T09:33:13.557 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:13 smithi022 ceph-mon[39525]: osdmap e182: 8 total, 8 up, 8 in 2024-09-21T09:33:13.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:13 smithi022 ceph-mon[39525]: pgmap v30: 121 pgs: 121 active+clean; 166 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.0 MiB/s rd, 2.0 MiB/s wr, 17 op/s 2024-09-21T09:33:13.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:13 smithi022 ceph-mon[27763]: osdmap e182: 8 total, 8 up, 8 in 2024-09-21T09:33:13.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:13 smithi022 ceph-mon[27763]: pgmap v30: 121 pgs: 121 active+clean; 166 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.0 MiB/s rd, 2.0 MiB/s wr, 17 op/s 2024-09-21T09:33:13.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:13 smithi064 ceph-mon[31124]: osdmap e182: 8 total, 8 up, 8 in 2024-09-21T09:33:13.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:13 smithi064 ceph-mon[31124]: pgmap v30: 121 pgs: 121 active+clean; 166 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.0 MiB/s rd, 2.0 MiB/s wr, 17 op/s 2024-09-21T09:33:14.477 INFO:tasks.rados.rados.0.smithi022.stdout:624: done (0 left) 2024-09-21T09:33:14.478 INFO:tasks.rados.rados.0.smithi022.stdout:625: rollback oid 12 current snap is 63 2024-09-21T09:33:14.478 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 12 to 58 2024-09-21T09:33:14.478 INFO:tasks.rados.rados.0.smithi022.stdout:626: read oid 21 snap -1 2024-09-21T09:33:14.478 INFO:tasks.rados.rados.0.smithi022.stdout:626: expect (ObjNum 172 snap 43 seq_num 172) 2024-09-21T09:33:14.478 INFO:tasks.rados.rados.0.smithi022.stdout:627: snap_remove snap 51 2024-09-21T09:33:14.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:14 smithi022 ceph-mon[39525]: osdmap e183: 8 total, 8 up, 8 in 2024-09-21T09:33:14.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:14 smithi022 ceph-mon[27763]: osdmap e183: 8 total, 8 up, 8 in 2024-09-21T09:33:14.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:14 smithi064 ceph-mon[31124]: osdmap e183: 8 total, 8 up, 8 in 2024-09-21T09:33:15.440 INFO:tasks.rados.rados.0.smithi022.stdout:627: done (2 left) 2024-09-21T09:33:15.440 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 44 current snap is 63 2024-09-21T09:33:15.440 INFO:tasks.rados.rados.0.smithi022.stdout:628: seq_num 217 ranges {0=442368,442368=696320,1138688=106496} 2024-09-21T09:33:15.442 INFO:tasks.rados.rados.0.smithi022.stdout:628: writing smithi02284960-44 from 0 to 442368 tid 1 2024-09-21T09:33:15.447 INFO:tasks.rados.rados.0.smithi022.stdout:628: writing smithi02284960-44 from 442368 to 1138688 tid 2 2024-09-21T09:33:15.448 INFO:tasks.rados.rados.0.smithi022.stdout:628: writing smithi02284960-44 from 1138688 to 1245184 tid 3 2024-09-21T09:33:15.450 INFO:tasks.rados.rados.0.smithi022.stdout:626: expect (ObjNum 172 snap 43 seq_num 172) 2024-09-21T09:33:15.466 INFO:tasks.rados.rados.0.smithi022.stdout:625: finishing rollback tid 0 to smithi02284960-12 2024-09-21T09:33:15.466 INFO:tasks.rados.rados.0.smithi022.stdout:625: finishing rollback tid 1 to smithi02284960-12 2024-09-21T09:33:15.466 INFO:tasks.rados.rados.0.smithi022.stdout:625: finishing rollback tid 2 to smithi02284960-12 2024-09-21T09:33:15.466 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 12 v 341 (ObjNum 187 snap 47 seq_num 187) dirty exists 2024-09-21T09:33:15.466 INFO:tasks.rados.rados.0.smithi022.stdout:625: done (2 left) 2024-09-21T09:33:15.466 INFO:tasks.rados.rados.0.smithi022.stdout:626: done (1 left) 2024-09-21T09:33:15.466 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 37 current snap is 63 2024-09-21T09:33:15.466 INFO:tasks.rados.rados.0.smithi022.stdout:628: finishing write tid 1 to smithi02284960-44 2024-09-21T09:33:15.467 INFO:tasks.rados.rados.0.smithi022.stdout:628: finishing write tid 2 to smithi02284960-44 2024-09-21T09:33:15.467 INFO:tasks.rados.rados.0.smithi022.stdout:628: finishing write tid 3 to smithi02284960-44 2024-09-21T09:33:15.467 INFO:tasks.rados.rados.0.smithi022.stdout:628: finishing write tid 4 to smithi02284960-44 2024-09-21T09:33:15.467 INFO:tasks.rados.rados.0.smithi022.stdout:628: finishing write tid 5 to smithi02284960-44 2024-09-21T09:33:15.467 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 44 v 315 (ObjNum 217 snap 63 seq_num 217) dirty exists 2024-09-21T09:33:15.467 INFO:tasks.rados.rados.0.smithi022.stdout:628: left oid 44 (ObjNum 217 snap 63 seq_num 217) 2024-09-21T09:33:15.467 INFO:tasks.rados.rados.0.smithi022.stdout:629: seq_num 218 ranges {0=458752,458752=581632,1040384=221184} 2024-09-21T09:33:15.468 INFO:tasks.rados.rados.0.smithi022.stdout:629: writing smithi02284960-37 from 0 to 458752 tid 1 2024-09-21T09:33:15.472 INFO:tasks.rados.rados.0.smithi022.stdout:629: writing smithi02284960-37 from 458752 to 1040384 tid 2 2024-09-21T09:33:15.473 INFO:tasks.rados.rados.0.smithi022.stdout:629: writing smithi02284960-37 from 1040384 to 1261568 tid 3 2024-09-21T09:33:15.473 INFO:tasks.rados.rados.0.smithi022.stdout:628: done (1 left) 2024-09-21T09:33:15.474 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 45 current snap is 63 2024-09-21T09:33:15.474 INFO:tasks.rados.rados.0.smithi022.stdout:629: finishing write tid 1 to smithi02284960-37 2024-09-21T09:33:15.474 INFO:tasks.rados.rados.0.smithi022.stdout:630: seq_num 219 ranges {0=475136,475136=466944,942080=335872} 2024-09-21T09:33:15.476 INFO:tasks.rados.rados.0.smithi022.stdout:630: writing smithi02284960-45 from 0 to 475136 tid 1 2024-09-21T09:33:15.479 INFO:tasks.rados.rados.0.smithi022.stdout:630: writing smithi02284960-45 from 475136 to 942080 tid 2 2024-09-21T09:33:15.481 INFO:tasks.rados.rados.0.smithi022.stdout:630: writing smithi02284960-45 from 942080 to 1277952 tid 3 2024-09-21T09:33:15.481 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 19 current snap is 63 2024-09-21T09:33:15.482 INFO:tasks.rados.rados.0.smithi022.stdout:631: seq_num 220 ranges {204800=491520,696320=753664,1449984=49152} 2024-09-21T09:33:15.485 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:15 smithi022 ceph-mon[39525]: osdmap e184: 8 total, 8 up, 8 in 2024-09-21T09:33:15.485 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:15 smithi022 ceph-mon[39525]: pgmap v33: 121 pgs: 121 active+clean; 166 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.0 MiB/s rd, 2.0 MiB/s wr, 17 op/s 2024-09-21T09:33:15.486 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:15 smithi022 ceph-mon[39525]: osdmap e185: 8 total, 8 up, 8 in 2024-09-21T09:33:15.486 INFO:tasks.rados.rados.0.smithi022.stdout:631: writing smithi02284960-19 from 204800 to 696320 tid 1 2024-09-21T09:33:15.490 INFO:tasks.rados.rados.0.smithi022.stdout:631: writing smithi02284960-19 from 696320 to 1449984 tid 2 2024-09-21T09:33:15.491 INFO:tasks.rados.rados.0.smithi022.stdout:631: writing smithi02284960-19 from 1449984 to 1499136 tid 3 2024-09-21T09:33:15.491 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 14 current snap is 63 2024-09-21T09:33:15.491 INFO:tasks.rados.rados.0.smithi022.stdout:629: finishing write tid 2 to smithi02284960-37 2024-09-21T09:33:15.491 INFO:tasks.rados.rados.0.smithi022.stdout:629: finishing write tid 3 to smithi02284960-37 2024-09-21T09:33:15.491 INFO:tasks.rados.rados.0.smithi022.stdout:629: finishing write tid 4 to smithi02284960-37 2024-09-21T09:33:15.491 INFO:tasks.rados.rados.0.smithi022.stdout:630: finishing write tid 1 to smithi02284960-45 2024-09-21T09:33:15.491 INFO:tasks.rados.rados.0.smithi022.stdout:632: seq_num 221 ranges {1933312=507904,2441216=688128,3129344=114688} 2024-09-21T09:33:15.504 INFO:tasks.rados.rados.0.smithi022.stdout:632: writing smithi02284960-14 from 1933312 to 2441216 tid 1 2024-09-21T09:33:15.508 INFO:tasks.rados.rados.0.smithi022.stdout:632: writing smithi02284960-14 from 2441216 to 3129344 tid 2 2024-09-21T09:33:15.509 INFO:tasks.rados.rados.0.smithi022.stdout:632: writing smithi02284960-14 from 3129344 to 3244032 tid 3 2024-09-21T09:33:15.509 INFO:tasks.rados.rados.0.smithi022.stdout:633: setattr oid 29 current snap is 63 2024-09-21T09:33:15.509 INFO:tasks.rados.rados.0.smithi022.stdout:629: finishing write tid 5 to smithi02284960-37 2024-09-21T09:33:15.509 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 37 v 358 (ObjNum 218 snap 63 seq_num 218) dirty exists 2024-09-21T09:33:15.509 INFO:tasks.rados.rados.0.smithi022.stdout:629: left oid 37 (ObjNum 218 snap 63 seq_num 218) 2024-09-21T09:33:15.509 INFO:tasks.rados.rados.0.smithi022.stdout:630: finishing write tid 2 to smithi02284960-45 2024-09-21T09:33:15.509 INFO:tasks.rados.rados.0.smithi022.stdout:630: finishing write tid 3 to smithi02284960-45 2024-09-21T09:33:15.510 INFO:tasks.rados.rados.0.smithi022.stdout:630: finishing write tid 4 to smithi02284960-45 2024-09-21T09:33:15.510 INFO:tasks.rados.rados.0.smithi022.stdout:630: finishing write tid 5 to smithi02284960-45 2024-09-21T09:33:15.510 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 45 v 222 (ObjNum 219 snap 63 seq_num 219) dirty exists 2024-09-21T09:33:15.510 INFO:tasks.rados.rados.0.smithi022.stdout:630: left oid 45 (ObjNum 219 snap 63 seq_num 219) 2024-09-21T09:33:15.510 INFO:tasks.rados.rados.0.smithi022.stdout:631: finishing write tid 1 to smithi02284960-19 2024-09-21T09:33:15.510 INFO:tasks.rados.rados.0.smithi022.stdout:631: finishing write tid 2 to smithi02284960-19 2024-09-21T09:33:15.510 INFO:tasks.rados.rados.0.smithi022.stdout:631: finishing write tid 3 to smithi02284960-19 2024-09-21T09:33:15.510 INFO:tasks.rados.rados.0.smithi022.stdout:631: finishing write tid 4 to smithi02284960-19 2024-09-21T09:33:15.510 INFO:tasks.rados.rados.0.smithi022.stdout:631: finishing write tid 5 to smithi02284960-19 2024-09-21T09:33:15.510 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 19 v 363 (ObjNum 220 snap 63 seq_num 220) dirty exists 2024-09-21T09:33:15.510 INFO:tasks.rados.rados.0.smithi022.stdout:631: left oid 19 (ObjNum 220 snap 63 seq_num 220) 2024-09-21T09:33:15.510 INFO:tasks.rados.rados.0.smithi022.stdout:632: finishing write tid 1 to smithi02284960-14 2024-09-21T09:33:15.511 INFO:tasks.rados.rados.0.smithi022.stdout:629: done (4 left) 2024-09-21T09:33:15.511 INFO:tasks.rados.rados.0.smithi022.stdout:630: done (3 left) 2024-09-21T09:33:15.511 INFO:tasks.rados.rados.0.smithi022.stdout:631: done (2 left) 2024-09-21T09:33:15.511 INFO:tasks.rados.rados.0.smithi022.stdout:634: delete oid 7 current snap is 63 2024-09-21T09:33:15.512 INFO:tasks.rados.rados.0.smithi022.stdout:632: finishing write tid 2 to smithi02284960-14 2024-09-21T09:33:15.512 INFO:tasks.rados.rados.0.smithi022.stdout:632: finishing write tid 3 to smithi02284960-14 2024-09-21T09:33:15.512 INFO:tasks.rados.rados.0.smithi022.stdout:632: finishing write tid 4 to smithi02284960-14 2024-09-21T09:33:15.513 INFO:tasks.rados.rados.0.smithi022.stdout:634: done (2 left) 2024-09-21T09:33:15.513 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 25 current snap is 63 2024-09-21T09:33:15.513 INFO:tasks.rados.rados.0.smithi022.stdout:635: seq_num 223 ranges {0=540672,540672=450560,991232=352256} 2024-09-21T09:33:15.516 INFO:tasks.rados.rados.0.smithi022.stdout:635: writing smithi02284960-25 from 0 to 540672 tid 1 2024-09-21T09:33:15.519 INFO:tasks.rados.rados.0.smithi022.stdout:635: writing smithi02284960-25 from 540672 to 991232 tid 2 2024-09-21T09:33:15.522 INFO:tasks.rados.rados.0.smithi022.stdout:635: writing smithi02284960-25 from 991232 to 1343488 tid 3 2024-09-21T09:33:15.522 INFO:tasks.rados.rados.0.smithi022.stdout:636: copy_from oid 45 from oid 6 current snap is 63 2024-09-21T09:33:15.522 INFO:tasks.rados.rados.0.smithi022.stdout:632: finishing write tid 5 to smithi02284960-14 2024-09-21T09:33:15.522 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 14 v 137 (ObjNum 221 snap 63 seq_num 221) dirty exists 2024-09-21T09:33:15.522 INFO:tasks.rados.rados.0.smithi022.stdout:632: left oid 14 (ObjNum 221 snap 63 seq_num 221) 2024-09-21T09:33:15.523 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 29 v 152 (ObjNum 168 snap 41 seq_num 168) dirty exists 2024-09-21T09:33:15.523 INFO:tasks.rados.rados.0.smithi022.stdout:635: finishing write tid 1 to smithi02284960-25 2024-09-21T09:33:15.523 INFO:tasks.rados.rados.0.smithi022.stdout:632: done (3 left) 2024-09-21T09:33:15.523 INFO:tasks.rados.rados.0.smithi022.stdout:633: done (2 left) 2024-09-21T09:33:15.523 INFO:tasks.rados.rados.0.smithi022.stdout:637: rollback oid 9 current snap is 63 2024-09-21T09:33:15.523 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 9 to 61 2024-09-21T09:33:15.523 INFO:tasks.rados.rados.0.smithi022.stdout:638: copy_from oid 32 from oid 18 current snap is 63 2024-09-21T09:33:15.523 INFO:tasks.rados.rados.0.smithi022.stdout:639: copy_from oid 17 from oid 7 current snap is 63 2024-09-21T09:33:15.523 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 16 current snap is 63 2024-09-21T09:33:15.523 INFO:tasks.rados.rados.0.smithi022.stdout:635: finishing write tid 2 to smithi02284960-25 2024-09-21T09:33:15.523 INFO:tasks.rados.rados.0.smithi022.stdout:640: seq_num 224 ranges {16384=557056,573440=737280,1310720=65536} 2024-09-21T09:33:15.525 INFO:tasks.rados.rados.0.smithi022.stdout:640: writing smithi02284960-16 from 16384 to 573440 tid 1 2024-09-21T09:33:15.530 INFO:tasks.rados.rados.0.smithi022.stdout:640: writing smithi02284960-16 from 573440 to 1310720 tid 2 2024-09-21T09:33:15.531 INFO:tasks.rados.rados.0.smithi022.stdout:640: writing smithi02284960-16 from 1310720 to 1376256 tid 3 2024-09-21T09:33:15.531 INFO:tasks.rados.rados.0.smithi022.stdout:641: snap_remove snap 46 2024-09-21T09:33:15.797 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 21 09:33:15 smithi022 systemd[1]: Stopping Ceph alertmanager.a for d65e801e-77fa-11ef-baf6-efdc52797490... 2024-09-21T09:33:15.797 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:15 smithi022 ceph-mon[27763]: osdmap e184: 8 total, 8 up, 8 in 2024-09-21T09:33:15.797 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:15 smithi022 ceph-mon[27763]: pgmap v33: 121 pgs: 121 active+clean; 166 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.0 MiB/s rd, 2.0 MiB/s wr, 17 op/s 2024-09-21T09:33:15.797 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:15 smithi022 ceph-mon[27763]: osdmap e185: 8 total, 8 up, 8 in 2024-09-21T09:33:15.896 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:15 smithi064 ceph-mon[31124]: osdmap e184: 8 total, 8 up, 8 in 2024-09-21T09:33:15.896 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:15 smithi064 ceph-mon[31124]: pgmap v33: 121 pgs: 121 active+clean; 166 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.0 MiB/s rd, 2.0 MiB/s wr, 17 op/s 2024-09-21T09:33:15.896 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:15 smithi064 ceph-mon[31124]: osdmap e185: 8 total, 8 up, 8 in 2024-09-21T09:33:16.058 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 21 09:33:15 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-alertmanager-a[79862]: ts=2024-09-21T09:33:15.795Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-09-21T09:33:16.058 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 21 09:33:15 smithi022 podman[87207]: 2024-09-21 09:33:15.846318037 +0000 UTC m=+0.181708007 container died 42a53bb454cd0933ba1b7686cfc05459135bf705c792ef1bc31c2906f3b7f9f1 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-21T09:33:16.453 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 21 09:33:16 smithi022 podman[87207]: 2024-09-21 09:33:16.22612821 +0000 UTC m=+0.561518176 container cleanup 42a53bb454cd0933ba1b7686cfc05459135bf705c792ef1bc31c2906f3b7f9f1 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-21T09:33:16.454 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 21 09:33:16 smithi022 bash[87207]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490-alertmanager-a 2024-09-21T09:33:16.490 INFO:tasks.rados.rados.0.smithi022.stdout:641: done (6 left) 2024-09-21T09:33:16.490 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 27 current snap is 63 2024-09-21T09:33:16.490 INFO:tasks.rados.rados.0.smithi022.stdout:642: seq_num 225 ranges {2596864=573440,3170304=622592,3792896=180224} 2024-09-21T09:33:16.507 INFO:tasks.rados.rados.0.smithi022.stdout:642: writing smithi02284960-27 from 2596864 to 3170304 tid 1 2024-09-21T09:33:16.510 INFO:tasks.rados.rados.0.smithi022.stdout:642: writing smithi02284960-27 from 3170304 to 3792896 tid 2 2024-09-21T09:33:16.512 INFO:tasks.rados.rados.0.smithi022.stdout:642: writing smithi02284960-27 from 3792896 to 3973120 tid 3 2024-09-21T09:33:16.512 INFO:tasks.rados.rados.0.smithi022.stdout:643: read oid 34 snap -1 2024-09-21T09:33:16.512 INFO:tasks.rados.rados.0.smithi022.stdout:643: expect (ObjNum 36 snap 0 seq_num 36) 2024-09-21T09:33:16.512 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 3 current snap is 63 2024-09-21T09:33:16.512 INFO:tasks.rados.rados.0.smithi022.stdout:644: seq_num 226 ranges {1499136=589824,2088960=507904,2596864=294912} 2024-09-21T09:33:16.524 INFO:tasks.rados.rados.0.smithi022.stdout:644: writing smithi02284960-3 from 1499136 to 2088960 tid 1 2024-09-21T09:33:16.527 INFO:tasks.rados.rados.0.smithi022.stdout:644: writing smithi02284960-3 from 2088960 to 2596864 tid 2 2024-09-21T09:33:16.530 INFO:tasks.rados.rados.0.smithi022.stdout:644: writing smithi02284960-3 from 2596864 to 2891776 tid 3 2024-09-21T09:33:16.530 INFO:tasks.rados.rados.0.smithi022.stdout:645: snap_create 2024-09-21T09:33:16.530 INFO:tasks.rados.rados.0.smithi022.stdout:636: finishing copy_from to smithi02284960-45 2024-09-21T09:33:16.530 INFO:tasks.rados.rados.0.smithi022.stdout:636: got expected ENOENT (src dne) 2024-09-21T09:33:16.530 INFO:tasks.rados.rados.0.smithi022.stdout:636: finishing copy_from racing read to smithi02284960-45 2024-09-21T09:33:16.530 INFO:tasks.rados.rados.0.smithi022.stdout:639: finishing copy_from to smithi02284960-17 2024-09-21T09:33:16.531 INFO:tasks.rados.rados.0.smithi022.stdout:639: got expected ENOENT (src dne) 2024-09-21T09:33:16.531 INFO:tasks.rados.rados.0.smithi022.stdout:639: finishing copy_from racing read to smithi02284960-17 2024-09-21T09:33:16.531 INFO:tasks.rados.rados.0.smithi022.stdout:635: finishing write tid 3 to smithi02284960-25 2024-09-21T09:33:16.531 INFO:tasks.rados.rados.0.smithi022.stdout:635: finishing write tid 4 to smithi02284960-25 2024-09-21T09:33:16.531 INFO:tasks.rados.rados.0.smithi022.stdout:635: finishing write tid 5 to smithi02284960-25 2024-09-21T09:33:16.531 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 25 v 303 (ObjNum 223 snap 63 seq_num 223) dirty exists 2024-09-21T09:33:16.531 INFO:tasks.rados.rados.0.smithi022.stdout:635: left oid 25 (ObjNum 223 snap 63 seq_num 223) 2024-09-21T09:33:16.531 INFO:tasks.rados.rados.0.smithi022.stdout:637: finishing rollback tid 0 to smithi02284960-9 2024-09-21T09:33:16.531 INFO:tasks.rados.rados.0.smithi022.stdout:637: finishing rollback tid 1 to smithi02284960-9 2024-09-21T09:33:16.531 INFO:tasks.rados.rados.0.smithi022.stdout:637: finishing rollback tid 2 to smithi02284960-9 2024-09-21T09:33:16.531 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 9 v 194 (ObjNum 137 snap 30 seq_num 137) dirty exists 2024-09-21T09:33:16.531 INFO:tasks.rados.rados.0.smithi022.stdout:640: finishing write tid 1 to smithi02284960-16 2024-09-21T09:33:16.531 INFO:tasks.rados.rados.0.smithi022.stdout:638: finishing copy_from racing read to smithi02284960-32 2024-09-21T09:33:16.532 INFO:tasks.rados.rados.0.smithi022.stdout:638: finishing copy_from to smithi02284960-32 2024-09-21T09:33:16.532 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 32 v 117 (ObjNum 210 snap 58 seq_num 210) dirty exists 2024-09-21T09:33:16.532 INFO:tasks.rados.rados.0.smithi022.stdout:640: finishing write tid 2 to smithi02284960-16 2024-09-21T09:33:16.532 INFO:tasks.rados.rados.0.smithi022.stdout:640: finishing write tid 3 to smithi02284960-16 2024-09-21T09:33:16.532 INFO:tasks.rados.rados.0.smithi022.stdout:640: finishing write tid 4 to smithi02284960-16 2024-09-21T09:33:16.532 INFO:tasks.rados.rados.0.smithi022.stdout:640: finishing write tid 5 to smithi02284960-16 2024-09-21T09:33:16.532 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 16 v 245 (ObjNum 224 snap 63 seq_num 224) dirty exists 2024-09-21T09:33:16.532 INFO:tasks.rados.rados.0.smithi022.stdout:640: left oid 16 (ObjNum 224 snap 63 seq_num 224) 2024-09-21T09:33:16.532 INFO:tasks.rados.rados.0.smithi022.stdout:642: finishing write tid 1 to smithi02284960-27 2024-09-21T09:33:16.532 INFO:tasks.rados.rados.0.smithi022.stdout:642: finishing write tid 2 to smithi02284960-27 2024-09-21T09:33:16.532 INFO:tasks.rados.rados.0.smithi022.stdout:642: finishing write tid 3 to smithi02284960-27 2024-09-21T09:33:16.532 INFO:tasks.rados.rados.0.smithi022.stdout:642: finishing write tid 4 to smithi02284960-27 2024-09-21T09:33:16.532 INFO:tasks.rados.rados.0.smithi022.stdout:643: expect (ObjNum 36 snap 0 seq_num 36) 2024-09-21T09:33:16.546 INFO:tasks.rados.rados.0.smithi022.stdout:642: finishing write tid 5 to smithi02284960-27 2024-09-21T09:33:16.546 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 27 v 263 (ObjNum 225 snap 63 seq_num 225) dirty exists 2024-09-21T09:33:16.546 INFO:tasks.rados.rados.0.smithi022.stdout:642: left oid 27 (ObjNum 225 snap 63 seq_num 225) 2024-09-21T09:33:16.546 INFO:tasks.rados.rados.0.smithi022.stdout:644: finishing write tid 1 to smithi02284960-3 2024-09-21T09:33:16.546 INFO:tasks.rados.rados.0.smithi022.stdout:644: finishing write tid 2 to smithi02284960-3 2024-09-21T09:33:16.546 INFO:tasks.rados.rados.0.smithi022.stdout:644: finishing write tid 3 to smithi02284960-3 2024-09-21T09:33:16.546 INFO:tasks.rados.rados.0.smithi022.stdout:644: finishing write tid 4 to smithi02284960-3 2024-09-21T09:33:16.546 INFO:tasks.rados.rados.0.smithi022.stdout:644: finishing write tid 5 to smithi02284960-3 2024-09-21T09:33:16.547 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 3 v 71 (ObjNum 226 snap 63 seq_num 226) dirty exists 2024-09-21T09:33:16.547 INFO:tasks.rados.rados.0.smithi022.stdout:644: left oid 3 (ObjNum 226 snap 63 seq_num 226) 2024-09-21T09:33:16.808 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 21 09:33:16 smithi022 podman[87224]: 2024-09-21 09:33:16.452707631 +0000 UTC m=+0.604450090 container remove 42a53bb454cd0933ba1b7686cfc05459135bf705c792ef1bc31c2906f3b7f9f1 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-21T09:33:16.808 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 21 09:33:16 smithi022 podman[87224]: 2024-09-21 09:33:16.536969674 +0000 UTC m=+0.688712089 volume remove 98a53bc2ba3e24c430d2f42d9a6e529c58e5969051c3f11d6ca2c5977e640a1f 2024-09-21T09:33:16.808 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 21 09:33:16 smithi022 systemd[1]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490@alertmanager.a.service: Deactivated successfully. 2024-09-21T09:33:16.808 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 21 09:33:16 smithi022 systemd[1]: Stopped Ceph alertmanager.a for d65e801e-77fa-11ef-baf6-efdc52797490. 2024-09-21T09:33:16.808 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 21 09:33:16 smithi022 systemd[1]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490@alertmanager.a.service: Consumed 1.114s CPU time. 2024-09-21T09:33:17.181 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 21 09:33:16 smithi022 systemd[1]: Starting Ceph alertmanager.a for d65e801e-77fa-11ef-baf6-efdc52797490... 2024-09-21T09:33:17.181 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 21 09:33:17 smithi022 podman[87358]: 2024-09-21 09:33:16.989312505 +0000 UTC m=+0.016528128 image pull 11f11916f8cdfb17c1da74b20deb2075d778357b4d60474d82453303f8d461f5 quay.io/prometheus/alertmanager:v0.27.0 2024-09-21T09:33:17.181 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 21 09:33:17 smithi022 podman[87358]: 2024-09-21 09:33:17.105125972 +0000 UTC m=+0.132341597 volume create f6536fdbfc955265bcebe950983bc750c0fae9a39245c8ef86833e82215289ab 2024-09-21T09:33:17.182 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 21 09:33:17 smithi022 podman[87358]: 2024-09-21 09:33:17.180467222 +0000 UTC m=+0.207682845 container create caced80ba5448751abdbbc168dbbbe80bfdfc40282230f296db2806bfee957b0 (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-21T09:33:17.494 INFO:tasks.rados.rados.0.smithi022.stdout:635: done (9 left) 2024-09-21T09:33:17.494 INFO:tasks.rados.rados.0.smithi022.stdout:636: done (8 left) 2024-09-21T09:33:17.494 INFO:tasks.rados.rados.0.smithi022.stdout:637: done (7 left) 2024-09-21T09:33:17.494 INFO:tasks.rados.rados.0.smithi022.stdout:638: done (6 left) 2024-09-21T09:33:17.494 INFO:tasks.rados.rados.0.smithi022.stdout:639: done (5 left) 2024-09-21T09:33:17.494 INFO:tasks.rados.rados.0.smithi022.stdout:640: done (4 left) 2024-09-21T09:33:17.494 INFO:tasks.rados.rados.0.smithi022.stdout:642: done (3 left) 2024-09-21T09:33:17.494 INFO:tasks.rados.rados.0.smithi022.stdout:643: done (2 left) 2024-09-21T09:33:17.495 INFO:tasks.rados.rados.0.smithi022.stdout:644: done (1 left) 2024-09-21T09:33:17.495 INFO:tasks.rados.rados.0.smithi022.stdout:645: done (0 left) 2024-09-21T09:33:17.495 INFO:tasks.rados.rados.0.smithi022.stdout:646: delete oid 2 current snap is 64 2024-09-21T09:33:17.551 INFO:tasks.rados.rados.0.smithi022.stdout:646: done (0 left) 2024-09-21T09:33:17.551 INFO:tasks.rados.rados.0.smithi022.stdout:647: delete oid 5 current snap is 64 2024-09-21T09:33:17.554 INFO:tasks.rados.rados.0.smithi022.stdout:647: done (0 left) 2024-09-21T09:33:17.554 INFO:tasks.rados.rados.0.smithi022.stdout:648: read oid 43 snap -1 2024-09-21T09:33:17.554 INFO:tasks.rados.rados.0.smithi022.stdout:648: expect (ObjNum 187 snap 47 seq_num 187) 2024-09-21T09:33:17.554 INFO:tasks.rados.rados.0.smithi022.stdout:649: read oid 28 snap 60 2024-09-21T09:33:17.554 INFO:tasks.rados.rados.0.smithi022.stdout:649: expect (ObjNum 162 snap 40 seq_num 162) 2024-09-21T09:33:17.554 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 11 current snap is 64 2024-09-21T09:33:17.554 INFO:tasks.rados.rados.0.smithi022.stdout:650: seq_num 229 ranges {2572288=638976,3211264=606208,3817472=196608} 2024-09-21T09:33:17.572 INFO:tasks.rados.rados.0.smithi022.stdout:650: writing smithi02284960-11 from 2572288 to 3211264 tid 1 2024-09-21T09:33:17.575 INFO:tasks.rados.rados.0.smithi022.stdout:650: writing smithi02284960-11 from 3211264 to 3817472 tid 2 2024-09-21T09:33:17.577 INFO:tasks.rados.rados.0.smithi022.stdout:650: writing smithi02284960-11 from 3817472 to 4014080 tid 3 2024-09-21T09:33:17.577 INFO:tasks.rados.rados.0.smithi022.stdout:651: read oid 39 snap -1 2024-09-21T09:33:17.577 INFO:tasks.rados.rados.0.smithi022.stdout:651: expect (ObjNum 151 snap 36 seq_num 151) 2024-09-21T09:33:17.577 INFO:tasks.rados.rados.0.smithi022.stdout:652: snap_create 2024-09-21T09:33:17.577 INFO:tasks.rados.rados.0.smithi022.stdout:649: expect (ObjNum 162 snap 40 seq_num 162) 2024-09-21T09:33:17.597 INFO:tasks.rados.rados.0.smithi022.stdout:650: finishing write tid 1 to smithi02284960-11 2024-09-21T09:33:17.597 INFO:tasks.rados.rados.0.smithi022.stdout:650: finishing write tid 2 to smithi02284960-11 2024-09-21T09:33:17.597 INFO:tasks.rados.rados.0.smithi022.stdout:650: finishing write tid 3 to smithi02284960-11 2024-09-21T09:33:17.597 INFO:tasks.rados.rados.0.smithi022.stdout:650: finishing write tid 4 to smithi02284960-11 2024-09-21T09:33:17.597 INFO:tasks.rados.rados.0.smithi022.stdout:650: finishing write tid 5 to smithi02284960-11 2024-09-21T09:33:17.597 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 11 v 319 (ObjNum 229 snap 64 seq_num 229) dirty exists 2024-09-21T09:33:17.597 INFO:tasks.rados.rados.0.smithi022.stdout:650: left oid 11 (ObjNum 229 snap 64 seq_num 229) 2024-09-21T09:33:17.597 INFO:tasks.rados.rados.0.smithi022.stdout:651: expect (ObjNum 151 snap 36 seq_num 151) 2024-09-21T09:33:17.633 INFO:tasks.rados.rados.0.smithi022.stdout:648: expect (ObjNum 187 snap 47 seq_num 187) 2024-09-21T09:33:17.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:17 smithi022 ceph-mon[39525]: osdmap e186: 8 total, 8 up, 8 in 2024-09-21T09:33:17.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:17 smithi022 ceph-mon[39525]: pgmap v36: 121 pgs: 121 active+clean; 177 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 356 KiB/s rd, 2.9 MiB/s wr, 18 op/s 2024-09-21T09:33:17.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:17 smithi022 ceph-mon[27763]: osdmap e186: 8 total, 8 up, 8 in 2024-09-21T09:33:17.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:17 smithi022 ceph-mon[27763]: pgmap v36: 121 pgs: 121 active+clean; 177 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 356 KiB/s rd, 2.9 MiB/s wr, 18 op/s 2024-09-21T09:33:17.809 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 21 09:33:17 smithi022 podman[87358]: 2024-09-21 09:33:17.457942578 +0000 UTC m=+0.485158201 container init caced80ba5448751abdbbc168dbbbe80bfdfc40282230f296db2806bfee957b0 (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-21T09:33:17.809 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 21 09:33:17 smithi022 podman[87358]: 2024-09-21 09:33:17.463029151 +0000 UTC m=+0.490244775 container start caced80ba5448751abdbbc168dbbbe80bfdfc40282230f296db2806bfee957b0 (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-21T09:33:17.809 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 21 09:33:17 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-alertmanager-a[87373]: ts=2024-09-21T09:33:17.474Z caller=main.go:181 level=info msg="Starting Alertmanager" version="(version=0.27.0, branch=HEAD, revision=0aa3c2aad14cff039931923ab16b26b7481783b5)" 2024-09-21T09:33:17.809 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 21 09:33:17 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-alertmanager-a[87373]: ts=2024-09-21T09:33:17.474Z caller=main.go:182 level=info build_context="(go=go1.21.7, platform=linux/amd64, user=root@22cd11f671e9, date=20240228-11:51:20, tags=netgo)" 2024-09-21T09:33:17.809 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 21 09:33:17 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-alertmanager-a[87373]: ts=2024-09-21T09:33:17.475Z caller=cluster.go:186 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.22 port=9094 2024-09-21T09:33:17.809 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 21 09:33:17 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-alertmanager-a[87373]: ts=2024-09-21T09:33:17.476Z caller=cluster.go:683 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-09-21T09:33:17.809 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 21 09:33:17 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-alertmanager-a[87373]: ts=2024-09-21T09:33:17.501Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-21T09:33:17.809 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 21 09:33:17 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-alertmanager-a[87373]: ts=2024-09-21T09:33:17.501Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-21T09:33:17.809 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 21 09:33:17 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-alertmanager-a[87373]: ts=2024-09-21T09:33:17.503Z caller=tls_config.go:313 level=info msg="Listening on" address=[::]:9093 2024-09-21T09:33:17.809 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 21 09:33:17 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-alertmanager-a[87373]: ts=2024-09-21T09:33:17.503Z caller=tls_config.go:316 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-09-21T09:33:17.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:17 smithi064 ceph-mon[31124]: osdmap e186: 8 total, 8 up, 8 in 2024-09-21T09:33:17.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:17 smithi064 ceph-mon[31124]: pgmap v36: 121 pgs: 121 active+clean; 177 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 356 KiB/s rd, 2.9 MiB/s wr, 18 op/s 2024-09-21T09:33:18.308 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 21 09:33:17 smithi022 bash[87358]: caced80ba5448751abdbbc168dbbbe80bfdfc40282230f296db2806bfee957b0 2024-09-21T09:33:18.308 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 21 09:33:17 smithi022 systemd[1]: Started Ceph alertmanager.a for d65e801e-77fa-11ef-baf6-efdc52797490. 2024-09-21T09:33:18.501 INFO:tasks.rados.rados.0.smithi022.stdout:648: done (4 left) 2024-09-21T09:33:18.501 INFO:tasks.rados.rados.0.smithi022.stdout:649: done (3 left) 2024-09-21T09:33:18.502 INFO:tasks.rados.rados.0.smithi022.stdout:650: done (2 left) 2024-09-21T09:33:18.502 INFO:tasks.rados.rados.0.smithi022.stdout:651: done (1 left) 2024-09-21T09:33:18.502 INFO:tasks.rados.rados.0.smithi022.stdout:652: done (0 left) 2024-09-21T09:33:18.502 INFO:tasks.rados.rados.0.smithi022.stdout:653: rollback oid 47 current snap is 65 2024-09-21T09:33:18.502 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 47 to 63 2024-09-21T09:33:18.502 INFO:tasks.rados.rados.0.smithi022.stdout:654: read oid 10 snap -1 2024-09-21T09:33:18.502 INFO:tasks.rados.rados.0.smithi022.stdout:654: expect (ObjNum 144 snap 32 seq_num 144) 2024-09-21T09:33:18.502 INFO:tasks.rados.rados.0.smithi022.stdout:655: read oid 42 snap 42 2024-09-21T09:33:18.502 INFO:tasks.rados.rados.0.smithi022.stdout:655: expect deleted 2024-09-21T09:33:18.502 INFO:tasks.rados.rados.0.smithi022.stdout:656: snap_remove snap 58 2024-09-21T09:33:18.808 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:33:18 smithi022 systemd[1]: Stopping Ceph node-exporter.a for d65e801e-77fa-11ef-baf6-efdc52797490... 2024-09-21T09:33:18.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:18 smithi022 ceph-mon[39525]: osdmap e187: 8 total, 8 up, 8 in 2024-09-21T09:33:18.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:18 smithi022 ceph-mon[39525]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:18.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:18 smithi022 ceph-mon[39525]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:18.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:18 smithi022 ceph-mon[39525]: Reconfiguring node-exporter.a (dependencies changed)... 2024-09-21T09:33:18.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:18 smithi022 ceph-mon[39525]: Deploying daemon node-exporter.a on smithi022 2024-09-21T09:33:18.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:18 smithi022 ceph-mon[27763]: osdmap e187: 8 total, 8 up, 8 in 2024-09-21T09:33:18.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:18 smithi022 ceph-mon[27763]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:18.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:18 smithi022 ceph-mon[27763]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:18.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:18 smithi022 ceph-mon[27763]: Reconfiguring node-exporter.a (dependencies changed)... 2024-09-21T09:33:18.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:18 smithi022 ceph-mon[27763]: Deploying daemon node-exporter.a on smithi022 2024-09-21T09:33:18.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:18 smithi064 ceph-mon[31124]: osdmap e187: 8 total, 8 up, 8 in 2024-09-21T09:33:18.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:18 smithi064 ceph-mon[31124]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:18.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:18 smithi064 ceph-mon[31124]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:18.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:18 smithi064 ceph-mon[31124]: Reconfiguring node-exporter.a (dependencies changed)... 2024-09-21T09:33:18.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:18 smithi064 ceph-mon[31124]: Deploying daemon node-exporter.a on smithi022 2024-09-21T09:33:19.149 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:33:18 smithi022 podman[87523]: 2024-09-21 09:33:18.856925104 +0000 UTC m=+0.149301680 container died 7cf4c99ebf3aed86db0ef63d6e9b50874848ebdc9f47803f29d61f6b3cecc372 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-21T09:33:19.478 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:33:19 smithi022 podman[87523]: 2024-09-21 09:33:19.338501706 +0000 UTC m=+0.630878280 container cleanup 7cf4c99ebf3aed86db0ef63d6e9b50874848ebdc9f47803f29d61f6b3cecc372 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-21T09:33:19.478 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:33:19 smithi022 bash[87523]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a 2024-09-21T09:33:19.504 INFO:tasks.rados.rados.0.smithi022.stdout:656: done (3 left) 2024-09-21T09:33:19.504 INFO:tasks.rados.rados.0.smithi022.stdout:657: rmattr oid 46 current snap is 65 2024-09-21T09:33:19.504 INFO:tasks.rados.rados.0.smithi022.stdout:657: done (3 left) 2024-09-21T09:33:19.504 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 40 current snap is 65 2024-09-21T09:33:19.504 INFO:tasks.rados.rados.0.smithi022.stdout:658: seq_num 230 ranges {2777088=655360,3432448=491520,3923968=311296} 2024-09-21T09:33:19.523 INFO:tasks.rados.rados.0.smithi022.stdout:658: writing smithi02284960-40 from 2777088 to 3432448 tid 1 2024-09-21T09:33:19.526 INFO:tasks.rados.rados.0.smithi022.stdout:658: writing smithi02284960-40 from 3432448 to 3923968 tid 2 2024-09-21T09:33:19.528 INFO:tasks.rados.rados.0.smithi022.stdout:658: writing smithi02284960-40 from 3923968 to 4235264 tid 3 2024-09-21T09:33:19.528 INFO:tasks.rados.rados.0.smithi022.stdout:659: read oid 36 snap -1 2024-09-21T09:33:19.528 INFO:tasks.rados.rados.0.smithi022.stdout:659: expect (ObjNum 188 snap 47 seq_num 188) 2024-09-21T09:33:19.528 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 16 current snap is 65 2024-09-21T09:33:19.528 INFO:tasks.rados.rados.0.smithi022.stdout:660: seq_num 231 ranges {1376256=671744,2048000=778240,2826240=24576} 2024-09-21T09:33:19.539 INFO:tasks.rados.rados.0.smithi022.stdout:660: writing smithi02284960-16 from 1376256 to 2048000 tid 1 2024-09-21T09:33:19.545 INFO:tasks.rados.rados.0.smithi022.stdout:660: writing smithi02284960-16 from 2048000 to 2826240 tid 2 2024-09-21T09:33:19.545 INFO:tasks.rados.rados.0.smithi022.stdout:660: writing smithi02284960-16 from 2826240 to 2850816 tid 3 2024-09-21T09:33:19.545 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 4 current snap is 65 2024-09-21T09:33:19.545 INFO:tasks.rados.rados.0.smithi022.stdout:653: finishing rollback tid 0 to smithi02284960-47 2024-09-21T09:33:19.545 INFO:tasks.rados.rados.0.smithi022.stdout:653: finishing rollback tid 1 to smithi02284960-47 2024-09-21T09:33:19.545 INFO:tasks.rados.rados.0.smithi022.stdout:653: finishing rollback tid 2 to smithi02284960-47 2024-09-21T09:33:19.545 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 47 v 322 (ObjNum 161 snap 0 seq_num 1) dirty exists 2024-09-21T09:33:19.545 INFO:tasks.rados.rados.0.smithi022.stdout:661: seq_num 232 ranges {0=688128,688128=663552,1351680=139264} 2024-09-21T09:33:19.548 INFO:tasks.rados.rados.0.smithi022.stdout:661: writing smithi02284960-4 from 0 to 688128 tid 1 2024-09-21T09:33:19.552 INFO:tasks.rados.rados.0.smithi022.stdout:661: writing smithi02284960-4 from 688128 to 1351680 tid 2 2024-09-21T09:33:19.553 INFO:tasks.rados.rados.0.smithi022.stdout:661: writing smithi02284960-4 from 1351680 to 1490944 tid 3 2024-09-21T09:33:19.553 INFO:tasks.rados.rados.0.smithi022.stdout:653: done (6 left) 2024-09-21T09:33:19.553 INFO:tasks.rados.rados.0.smithi022.stdout:655: done (5 left) 2024-09-21T09:33:19.554 INFO:tasks.rados.rados.0.smithi022.stdout:662: read oid 42 snap 53 2024-09-21T09:33:19.554 INFO:tasks.rados.rados.0.smithi022.stdout:662: expect (ObjNum 183 snap 47 seq_num 183) 2024-09-21T09:33:19.554 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 31 current snap is 65 2024-09-21T09:33:19.554 INFO:tasks.rados.rados.0.smithi022.stdout:654: expect (ObjNum 144 snap 32 seq_num 144) 2024-09-21T09:33:19.554 INFO:tasks.rados.rados.0.smithi022.stdout:658: finishing write tid 1 to smithi02284960-40 2024-09-21T09:33:19.554 INFO:tasks.rados.rados.0.smithi022.stdout:658: finishing write tid 2 to smithi02284960-40 2024-09-21T09:33:19.555 INFO:tasks.rados.rados.0.smithi022.stdout:658: finishing write tid 3 to smithi02284960-40 2024-09-21T09:33:19.555 INFO:tasks.rados.rados.0.smithi022.stdout:658: finishing write tid 4 to smithi02284960-40 2024-09-21T09:33:19.555 INFO:tasks.rados.rados.0.smithi022.stdout:658: finishing write tid 5 to smithi02284960-40 2024-09-21T09:33:19.555 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 40 v 320 (ObjNum 230 snap 65 seq_num 230) dirty exists 2024-09-21T09:33:19.555 INFO:tasks.rados.rados.0.smithi022.stdout:658: left oid 40 (ObjNum 230 snap 65 seq_num 230) 2024-09-21T09:33:19.555 INFO:tasks.rados.rados.0.smithi022.stdout:659: expect (ObjNum 188 snap 47 seq_num 188) 2024-09-21T09:33:19.624 INFO:tasks.rados.rados.0.smithi022.stdout:660: finishing write tid 1 to smithi02284960-16 2024-09-21T09:33:19.624 INFO:tasks.rados.rados.0.smithi022.stdout:660: finishing write tid 2 to smithi02284960-16 2024-09-21T09:33:19.624 INFO:tasks.rados.rados.0.smithi022.stdout:660: finishing write tid 3 to smithi02284960-16 2024-09-21T09:33:19.624 INFO:tasks.rados.rados.0.smithi022.stdout:660: finishing write tid 4 to smithi02284960-16 2024-09-21T09:33:19.624 INFO:tasks.rados.rados.0.smithi022.stdout:663: seq_num 233 ranges {2727936=712704,3440640=548864,3989504=253952} 2024-09-21T09:33:19.642 INFO:tasks.rados.rados.0.smithi022.stdout:663: writing smithi02284960-31 from 2727936 to 3440640 tid 1 2024-09-21T09:33:19.646 INFO:tasks.rados.rados.0.smithi022.stdout:663: writing smithi02284960-31 from 3440640 to 3989504 tid 2 2024-09-21T09:33:19.649 INFO:tasks.rados.rados.0.smithi022.stdout:663: writing smithi02284960-31 from 3989504 to 4243456 tid 3 2024-09-21T09:33:19.649 INFO:tasks.rados.rados.0.smithi022.stdout:654: done (6 left) 2024-09-21T09:33:19.649 INFO:tasks.rados.rados.0.smithi022.stdout:658: done (5 left) 2024-09-21T09:33:19.649 INFO:tasks.rados.rados.0.smithi022.stdout:659: done (4 left) 2024-09-21T09:33:19.649 INFO:tasks.rados.rados.0.smithi022.stdout:664: rmattr oid 30 current snap is 65 2024-09-21T09:33:19.649 INFO:tasks.rados.rados.0.smithi022.stdout:664: done (4 left) 2024-09-21T09:33:19.649 INFO:tasks.rados.rados.0.smithi022.stdout:665: rmattr oid 48 current snap is 65 2024-09-21T09:33:19.649 INFO:tasks.rados.rados.0.smithi022.stdout:665: done (4 left) 2024-09-21T09:33:19.649 INFO:tasks.rados.rados.0.smithi022.stdout:666: rmattr oid 48 current snap is 65 2024-09-21T09:33:19.649 INFO:tasks.rados.rados.0.smithi022.stdout:666: done (4 left) 2024-09-21T09:33:19.649 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 33 current snap is 65 2024-09-21T09:33:19.649 INFO:tasks.rados.rados.0.smithi022.stdout:660: finishing write tid 5 to smithi02284960-16 2024-09-21T09:33:19.649 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 16 v 249 (ObjNum 231 snap 65 seq_num 231) dirty exists 2024-09-21T09:33:19.650 INFO:tasks.rados.rados.0.smithi022.stdout:660: left oid 16 (ObjNum 231 snap 65 seq_num 231) 2024-09-21T09:33:19.650 INFO:tasks.rados.rados.0.smithi022.stdout:661: finishing write tid 1 to smithi02284960-4 2024-09-21T09:33:19.650 INFO:tasks.rados.rados.0.smithi022.stdout:667: seq_num 234 ranges {843776=729088,1572864=434176,2007040=368640} 2024-09-21T09:33:19.657 INFO:tasks.rados.rados.0.smithi022.stdout:667: writing smithi02284960-33 from 843776 to 1572864 tid 1 2024-09-21T09:33:19.660 INFO:tasks.rados.rados.0.smithi022.stdout:667: writing smithi02284960-33 from 1572864 to 2007040 tid 2 2024-09-21T09:33:19.662 INFO:tasks.rados.rados.0.smithi022.stdout:667: writing smithi02284960-33 from 2007040 to 2375680 tid 3 2024-09-21T09:33:19.662 INFO:tasks.rados.rados.0.smithi022.stdout:660: done (4 left) 2024-09-21T09:33:19.662 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 19 current snap is 65 2024-09-21T09:33:19.663 INFO:tasks.rados.rados.0.smithi022.stdout:661: finishing write tid 2 to smithi02284960-4 2024-09-21T09:33:19.663 INFO:tasks.rados.rados.0.smithi022.stdout:661: finishing write tid 3 to smithi02284960-4 2024-09-21T09:33:19.663 INFO:tasks.rados.rados.0.smithi022.stdout:661: finishing write tid 4 to smithi02284960-4 2024-09-21T09:33:19.663 INFO:tasks.rados.rados.0.smithi022.stdout:661: finishing write tid 5 to smithi02284960-4 2024-09-21T09:33:19.663 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 4 v 199 (ObjNum 232 snap 65 seq_num 232) dirty exists 2024-09-21T09:33:19.663 INFO:tasks.rados.rados.0.smithi022.stdout:661: left oid 4 (ObjNum 232 snap 65 seq_num 232) 2024-09-21T09:33:19.663 INFO:tasks.rados.rados.0.smithi022.stdout:662: expect (ObjNum 183 snap 47 seq_num 183) 2024-09-21T09:33:19.688 INFO:tasks.rados.rados.0.smithi022.stdout:663: finishing write tid 1 to smithi02284960-31 2024-09-21T09:33:19.688 INFO:tasks.rados.rados.0.smithi022.stdout:663: finishing write tid 2 to smithi02284960-31 2024-09-21T09:33:19.688 INFO:tasks.rados.rados.0.smithi022.stdout:663: finishing write tid 3 to smithi02284960-31 2024-09-21T09:33:19.689 INFO:tasks.rados.rados.0.smithi022.stdout:663: finishing write tid 4 to smithi02284960-31 2024-09-21T09:33:19.689 INFO:tasks.rados.rados.0.smithi022.stdout:663: finishing write tid 5 to smithi02284960-31 2024-09-21T09:33:19.689 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 31 v 380 (ObjNum 233 snap 65 seq_num 233) dirty exists 2024-09-21T09:33:19.689 INFO:tasks.rados.rados.0.smithi022.stdout:663: left oid 31 (ObjNum 233 snap 65 seq_num 233) 2024-09-21T09:33:19.689 INFO:tasks.rados.rados.0.smithi022.stdout:667: finishing write tid 1 to smithi02284960-33 2024-09-21T09:33:19.689 INFO:tasks.rados.rados.0.smithi022.stdout:667: finishing write tid 2 to smithi02284960-33 2024-09-21T09:33:19.689 INFO:tasks.rados.rados.0.smithi022.stdout:667: finishing write tid 3 to smithi02284960-33 2024-09-21T09:33:19.689 INFO:tasks.rados.rados.0.smithi022.stdout:668: seq_num 235 ranges {1499136=745472,2244608=712704,2957312=90112} 2024-09-21T09:33:19.700 INFO:tasks.rados.rados.0.smithi022.stdout:668: writing smithi02284960-19 from 1499136 to 2244608 tid 1 2024-09-21T09:33:19.704 INFO:tasks.rados.rados.0.smithi022.stdout:668: writing smithi02284960-19 from 2244608 to 2957312 tid 2 2024-09-21T09:33:19.705 INFO:tasks.rados.rados.0.smithi022.stdout:668: writing smithi02284960-19 from 2957312 to 3047424 tid 3 2024-09-21T09:33:19.705 INFO:tasks.rados.rados.0.smithi022.stdout:661: done (4 left) 2024-09-21T09:33:19.705 INFO:tasks.rados.rados.0.smithi022.stdout:662: done (3 left) 2024-09-21T09:33:19.705 INFO:tasks.rados.rados.0.smithi022.stdout:663: done (2 left) 2024-09-21T09:33:19.705 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 24 current snap is 65 2024-09-21T09:33:19.706 INFO:tasks.rados.rados.0.smithi022.stdout:667: finishing write tid 4 to smithi02284960-33 2024-09-21T09:33:19.706 INFO:tasks.rados.rados.0.smithi022.stdout:667: finishing write tid 5 to smithi02284960-33 2024-09-21T09:33:19.706 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 33 v 349 (ObjNum 234 snap 65 seq_num 234) dirty exists 2024-09-21T09:33:19.706 INFO:tasks.rados.rados.0.smithi022.stdout:667: left oid 33 (ObjNum 234 snap 65 seq_num 234) 2024-09-21T09:33:19.706 INFO:tasks.rados.rados.0.smithi022.stdout:669: seq_num 236 ranges {0=761856,761856=647168,1409024=155648} 2024-09-21T09:33:19.709 INFO:tasks.rados.rados.0.smithi022.stdout:669: writing smithi02284960-24 from 0 to 761856 tid 1 2024-09-21T09:33:19.713 INFO:tasks.rados.rados.0.smithi022.stdout:669: writing smithi02284960-24 from 761856 to 1409024 tid 2 2024-09-21T09:33:19.714 INFO:tasks.rados.rados.0.smithi022.stdout:669: writing smithi02284960-24 from 1409024 to 1564672 tid 3 2024-09-21T09:33:19.714 INFO:tasks.rados.rados.0.smithi022.stdout:667: done (2 left) 2024-09-21T09:33:19.714 INFO:tasks.rados.rados.0.smithi022.stdout:670: read oid 21 snap -1 2024-09-21T09:33:19.714 INFO:tasks.rados.rados.0.smithi022.stdout:670: expect (ObjNum 172 snap 43 seq_num 172) 2024-09-21T09:33:19.715 INFO:tasks.rados.rados.0.smithi022.stdout:671: read oid 22 snap -1 2024-09-21T09:33:19.715 INFO:tasks.rados.rados.0.smithi022.stdout:671: expect (ObjNum 170 snap 43 seq_num 170) 2024-09-21T09:33:19.715 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 43 current snap is 65 2024-09-21T09:33:19.715 INFO:tasks.rados.rados.0.smithi022.stdout:668: finishing write tid 1 to smithi02284960-19 2024-09-21T09:33:19.715 INFO:tasks.rados.rados.0.smithi022.stdout:672: seq_num 237 ranges {7028736=778240,7806976=532480,8339456=270336} 2024-09-21T09:33:19.757 INFO:tasks.rados.rados.0.smithi022.stdout:672: writing smithi02284960-43 from 7028736 to 7806976 tid 1 2024-09-21T09:33:19.759 INFO:tasks.rados.rados.0.smithi022.stdout:672: writing smithi02284960-43 from 7806976 to 8339456 tid 2 2024-09-21T09:33:19.761 INFO:tasks.rados.rados.0.smithi022.stdout:672: writing smithi02284960-43 from 8339456 to 8609792 tid 3 2024-09-21T09:33:19.761 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 16 current snap is 65 2024-09-21T09:33:19.762 INFO:tasks.rados.rados.0.smithi022.stdout:668: finishing write tid 2 to smithi02284960-19 2024-09-21T09:33:19.762 INFO:tasks.rados.rados.0.smithi022.stdout:668: finishing write tid 3 to smithi02284960-19 2024-09-21T09:33:19.762 INFO:tasks.rados.rados.0.smithi022.stdout:668: finishing write tid 4 to smithi02284960-19 2024-09-21T09:33:19.762 INFO:tasks.rados.rados.0.smithi022.stdout:668: finishing write tid 5 to smithi02284960-19 2024-09-21T09:33:19.762 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 19 v 385 (ObjNum 235 snap 65 seq_num 235) dirty exists 2024-09-21T09:33:19.762 INFO:tasks.rados.rados.0.smithi022.stdout:668: left oid 19 (ObjNum 235 snap 65 seq_num 235) 2024-09-21T09:33:19.762 INFO:tasks.rados.rados.0.smithi022.stdout:669: finishing write tid 1 to smithi02284960-24 2024-09-21T09:33:19.762 INFO:tasks.rados.rados.0.smithi022.stdout:669: finishing write tid 2 to smithi02284960-24 2024-09-21T09:33:19.762 INFO:tasks.rados.rados.0.smithi022.stdout:673: seq_num 238 ranges {2850816=794624,3645440=417792,4063232=385024} 2024-09-21T09:33:19.771 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:19 smithi022 ceph-mon[27763]: osdmap e188: 8 total, 8 up, 8 in 2024-09-21T09:33:19.771 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:19 smithi022 ceph-mon[27763]: pgmap v39: 121 pgs: 121 active+clean; 177 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 356 KiB/s rd, 2.9 MiB/s wr, 18 op/s 2024-09-21T09:33:19.771 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:19 smithi022 ceph-mon[39525]: osdmap e188: 8 total, 8 up, 8 in 2024-09-21T09:33:19.771 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:19 smithi022 ceph-mon[39525]: pgmap v39: 121 pgs: 121 active+clean; 177 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 356 KiB/s rd, 2.9 MiB/s wr, 18 op/s 2024-09-21T09:33:19.772 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:33:19 smithi022 podman[87542]: 2024-09-21 09:33:19.615096052 +0000 UTC m=+0.758371384 container remove 7cf4c99ebf3aed86db0ef63d6e9b50874848ebdc9f47803f29d61f6b3cecc372 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-21T09:33:19.772 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:33:19 smithi022 systemd[1]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-09-21T09:33:19.772 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 21 09:33:19 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-alertmanager-a[87373]: ts=2024-09-21T09:33:19.477Z caller=cluster.go:708 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000628302s 2024-09-21T09:33:19.781 INFO:tasks.rados.rados.0.smithi022.stdout:673: writing smithi02284960-16 from 2850816 to 3645440 tid 1 2024-09-21T09:33:19.783 INFO:tasks.rados.rados.0.smithi022.stdout:673: writing smithi02284960-16 from 3645440 to 4063232 tid 2 2024-09-21T09:33:19.786 INFO:tasks.rados.rados.0.smithi022.stdout:673: writing smithi02284960-16 from 4063232 to 4448256 tid 3 2024-09-21T09:33:19.786 INFO:tasks.rados.rados.0.smithi022.stdout:668: done (5 left) 2024-09-21T09:33:19.786 INFO:tasks.rados.rados.0.smithi022.stdout:674: snap_remove snap 60 2024-09-21T09:33:19.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:19 smithi064 ceph-mon[31124]: osdmap e188: 8 total, 8 up, 8 in 2024-09-21T09:33:19.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:19 smithi064 ceph-mon[31124]: pgmap v39: 121 pgs: 121 active+clean; 177 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 356 KiB/s rd, 2.9 MiB/s wr, 18 op/s 2024-09-21T09:33:20.058 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:33:19 smithi022 systemd[1]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490@node-exporter.a.service: Failed with result 'exit-code'. 2024-09-21T09:33:20.058 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:33:19 smithi022 systemd[1]: Stopped Ceph node-exporter.a for d65e801e-77fa-11ef-baf6-efdc52797490. 2024-09-21T09:33:20.058 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:33:19 smithi022 systemd[1]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490@node-exporter.a.service: Consumed 2.336s CPU time. 2024-09-21T09:33:20.058 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:33:19 smithi022 systemd[1]: Starting Ceph node-exporter.a for d65e801e-77fa-11ef-baf6-efdc52797490... 2024-09-21T09:33:20.441 INFO:tasks.rados.rados.0.smithi022.stdout:669: finishing write tid 3 to smithi02284960-24 2024-09-21T09:33:20.441 INFO:tasks.rados.rados.0.smithi022.stdout:669: finishing write tid 4 to smithi02284960-24 2024-09-21T09:33:20.441 INFO:tasks.rados.rados.0.smithi022.stdout:670: expect (ObjNum 172 snap 43 seq_num 172) 2024-09-21T09:33:20.460 INFO:tasks.rados.rados.0.smithi022.stdout:671: expect (ObjNum 170 snap 43 seq_num 170) 2024-09-21T09:33:20.488 INFO:tasks.rados.rados.0.smithi022.stdout:669: finishing write tid 5 to smithi02284960-24 2024-09-21T09:33:20.488 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 24 v 389 (ObjNum 236 snap 65 seq_num 236) dirty exists 2024-09-21T09:33:20.488 INFO:tasks.rados.rados.0.smithi022.stdout:669: left oid 24 (ObjNum 236 snap 65 seq_num 236) 2024-09-21T09:33:20.488 INFO:tasks.rados.rados.0.smithi022.stdout:669: done (5 left) 2024-09-21T09:33:20.488 INFO:tasks.rados.rados.0.smithi022.stdout:670: done (4 left) 2024-09-21T09:33:20.488 INFO:tasks.rados.rados.0.smithi022.stdout:671: done (3 left) 2024-09-21T09:33:20.488 INFO:tasks.rados.rados.0.smithi022.stdout:674: done (2 left) 2024-09-21T09:33:20.488 INFO:tasks.rados.rados.0.smithi022.stdout:675: snap_remove snap 43 2024-09-21T09:33:20.513 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:33:20 smithi022 bash[87672]: Trying to pull quay.io/prometheus/node-exporter:v1.7.0... 2024-09-21T09:33:20.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:20 smithi022 ceph-mon[39525]: osdmap e189: 8 total, 8 up, 8 in 2024-09-21T09:33:20.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:20 smithi022 ceph-mon[39525]: osdmap e190: 8 total, 8 up, 8 in 2024-09-21T09:33:20.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:20 smithi022 ceph-mon[27763]: osdmap e189: 8 total, 8 up, 8 in 2024-09-21T09:33:20.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:20 smithi022 ceph-mon[27763]: osdmap e190: 8 total, 8 up, 8 in 2024-09-21T09:33:20.808 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:33:20 smithi022 bash[87672]: Getting image source signatures 2024-09-21T09:33:20.808 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:33:20 smithi022 bash[87672]: Copying blob sha256:324153f2810a9927fcce320af9e4e291e0b6e805cbdd1f338386c756b9defa24 2024-09-21T09:33:20.808 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:33:20 smithi022 bash[87672]: Copying blob sha256:2abcce694348cd2c949c0e98a7400ebdfd8341021bcf6b541bc72033ce982510 2024-09-21T09:33:20.809 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:33:20 smithi022 bash[87672]: Copying blob sha256:455fd88e5221bc1e278ef2d059cd70e4df99a24e5af050ede621534276f6cf9a 2024-09-21T09:33:20.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:20 smithi064 ceph-mon[31124]: osdmap e189: 8 total, 8 up, 8 in 2024-09-21T09:33:20.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:20 smithi064 ceph-mon[31124]: osdmap e190: 8 total, 8 up, 8 in 2024-09-21T09:33:20.942 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:33:20 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[67503]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:33:20] "GET /metrics HTTP/1.1" 200 36133 "" "Prometheus/2.43.0" 2024-09-21T09:33:21.330 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:33:21 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:33:21] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-21T09:33:21.445 INFO:tasks.rados.rados.0.smithi022.stdout:675: done (2 left) 2024-09-21T09:33:21.445 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 1 current snap is 65 2024-09-21T09:33:21.445 INFO:tasks.rados.rados.0.smithi022.stdout:676: seq_num 239 ranges {204800=409600,614400=704512,1318912=499712} 2024-09-21T09:33:21.448 INFO:tasks.rados.rados.0.smithi022.stdout:676: writing smithi02284960-1 from 204800 to 614400 tid 1 2024-09-21T09:33:21.454 INFO:tasks.rados.rados.0.smithi022.stdout:676: writing smithi02284960-1 from 614400 to 1318912 tid 2 2024-09-21T09:33:21.459 INFO:tasks.rados.rados.0.smithi022.stdout:676: writing smithi02284960-1 from 1318912 to 1818624 tid 3 2024-09-21T09:33:21.459 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 39 current snap is 65 2024-09-21T09:33:21.459 INFO:tasks.rados.rados.0.smithi022.stdout:672: finishing write tid 1 to smithi02284960-43 2024-09-21T09:33:21.459 INFO:tasks.rados.rados.0.smithi022.stdout:672: finishing write tid 2 to smithi02284960-43 2024-09-21T09:33:21.459 INFO:tasks.rados.rados.0.smithi022.stdout:672: finishing write tid 3 to smithi02284960-43 2024-09-21T09:33:21.459 INFO:tasks.rados.rados.0.smithi022.stdout:672: finishing write tid 4 to smithi02284960-43 2024-09-21T09:33:21.459 INFO:tasks.rados.rados.0.smithi022.stdout:672: finishing write tid 5 to smithi02284960-43 2024-09-21T09:33:21.459 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 43 v 325 (ObjNum 237 snap 65 seq_num 237) dirty exists 2024-09-21T09:33:21.459 INFO:tasks.rados.rados.0.smithi022.stdout:672: left oid 43 (ObjNum 237 snap 65 seq_num 237) 2024-09-21T09:33:21.459 INFO:tasks.rados.rados.0.smithi022.stdout:673: finishing write tid 1 to smithi02284960-16 2024-09-21T09:33:21.459 INFO:tasks.rados.rados.0.smithi022.stdout:677: seq_num 240 ranges {770048=425984,1196032=589824,1785856=483328,2269184=131072} 2024-09-21T09:33:21.466 INFO:tasks.rados.rados.0.smithi022.stdout:677: writing smithi02284960-39 from 770048 to 1196032 tid 1 2024-09-21T09:33:21.471 INFO:tasks.rados.rados.0.smithi022.stdout:677: writing smithi02284960-39 from 1196032 to 1785856 tid 2 2024-09-21T09:33:21.474 INFO:tasks.rados.rados.0.smithi022.stdout:677: writing smithi02284960-39 from 1785856 to 2269184 tid 3 2024-09-21T09:33:21.476 INFO:tasks.rados.rados.0.smithi022.stdout:677: writing smithi02284960-39 from 2269184 to 2400256 tid 4 2024-09-21T09:33:21.476 INFO:tasks.rados.rados.0.smithi022.stdout:672: done (3 left) 2024-09-21T09:33:21.476 INFO:tasks.rados.rados.0.smithi022.stdout:678: snap_create 2024-09-21T09:33:21.476 INFO:tasks.rados.rados.0.smithi022.stdout:673: finishing write tid 2 to smithi02284960-16 2024-09-21T09:33:21.476 INFO:tasks.rados.rados.0.smithi022.stdout:673: finishing write tid 3 to smithi02284960-16 2024-09-21T09:33:21.476 INFO:tasks.rados.rados.0.smithi022.stdout:673: finishing write tid 4 to smithi02284960-16 2024-09-21T09:33:21.476 INFO:tasks.rados.rados.0.smithi022.stdout:673: finishing write tid 5 to smithi02284960-16 2024-09-21T09:33:21.476 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 16 v 253 (ObjNum 238 snap 65 seq_num 238) dirty exists 2024-09-21T09:33:21.476 INFO:tasks.rados.rados.0.smithi022.stdout:673: left oid 16 (ObjNum 238 snap 65 seq_num 238) 2024-09-21T09:33:21.476 INFO:tasks.rados.rados.0.smithi022.stdout:676: finishing write tid 1 to smithi02284960-1 2024-09-21T09:33:21.477 INFO:tasks.rados.rados.0.smithi022.stdout:676: finishing write tid 2 to smithi02284960-1 2024-09-21T09:33:21.477 INFO:tasks.rados.rados.0.smithi022.stdout:676: finishing write tid 3 to smithi02284960-1 2024-09-21T09:33:21.477 INFO:tasks.rados.rados.0.smithi022.stdout:676: finishing write tid 4 to smithi02284960-1 2024-09-21T09:33:21.477 INFO:tasks.rados.rados.0.smithi022.stdout:676: finishing write tid 5 to smithi02284960-1 2024-09-21T09:33:21.477 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 1 v 326 (ObjNum 239 snap 65 seq_num 239) dirty exists 2024-09-21T09:33:21.477 INFO:tasks.rados.rados.0.smithi022.stdout:676: left oid 1 (ObjNum 239 snap 65 seq_num 239) 2024-09-21T09:33:21.477 INFO:tasks.rados.rados.0.smithi022.stdout:677: finishing write tid 1 to smithi02284960-39 2024-09-21T09:33:21.477 INFO:tasks.rados.rados.0.smithi022.stdout:677: finishing write tid 2 to smithi02284960-39 2024-09-21T09:33:21.478 INFO:tasks.rados.rados.0.smithi022.stdout:677: finishing write tid 3 to smithi02284960-39 2024-09-21T09:33:21.479 INFO:tasks.rados.rados.0.smithi022.stdout:677: finishing write tid 4 to smithi02284960-39 2024-09-21T09:33:21.479 INFO:tasks.rados.rados.0.smithi022.stdout:677: finishing write tid 5 to smithi02284960-39 2024-09-21T09:33:21.480 INFO:tasks.rados.rados.0.smithi022.stdout:677: finishing write tid 6 to smithi02284960-39 2024-09-21T09:33:21.480 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 39 v 286 (ObjNum 240 snap 65 seq_num 240) dirty exists 2024-09-21T09:33:21.480 INFO:tasks.rados.rados.0.smithi022.stdout:677: left oid 39 (ObjNum 240 snap 65 seq_num 240) 2024-09-21T09:33:21.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:21 smithi022 ceph-mon[39525]: pgmap v42: 121 pgs: 121 active+clean; 177 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:33:21.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:21 smithi022 ceph-mon[39525]: osdmap e191: 8 total, 8 up, 8 in 2024-09-21T09:33:21.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:21 smithi022 ceph-mon[27763]: pgmap v42: 121 pgs: 121 active+clean; 177 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:33:21.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:21 smithi022 ceph-mon[27763]: osdmap e191: 8 total, 8 up, 8 in 2024-09-21T09:33:21.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:21 smithi064 ceph-mon[31124]: pgmap v42: 121 pgs: 121 active+clean; 177 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:33:21.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:21 smithi064 ceph-mon[31124]: osdmap e191: 8 total, 8 up, 8 in 2024-09-21T09:33:22.386 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:33:22 smithi022 bash[87672]: Copying config sha256:72c9c208898624938c9e4183d6686ea4a5fd3f912bc29bc3f00147924c521a3e 2024-09-21T09:33:22.386 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:33:22 smithi022 bash[87672]: Writing manifest to image destination 2024-09-21T09:33:22.443 INFO:tasks.rados.rados.0.smithi022.stdout:673: done (3 left) 2024-09-21T09:33:22.443 INFO:tasks.rados.rados.0.smithi022.stdout:676: done (2 left) 2024-09-21T09:33:22.443 INFO:tasks.rados.rados.0.smithi022.stdout:677: done (1 left) 2024-09-21T09:33:22.444 INFO:tasks.rados.rados.0.smithi022.stdout:678: done (0 left) 2024-09-21T09:33:22.444 INFO:tasks.rados.rados.0.smithi022.stdout:679: rollback oid 13 current snap is 66 2024-09-21T09:33:22.444 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 13 to 53 2024-09-21T09:33:22.444 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 35 current snap is 66 2024-09-21T09:33:22.444 INFO:tasks.rados.rados.0.smithi022.stdout:680: seq_num 241 ranges {1908736=442368,2351104=466944,2818048=688128,3506176=49152} 2024-09-21T09:33:22.456 INFO:tasks.rados.rados.0.smithi022.stdout:680: writing smithi02284960-35 from 1908736 to 2351104 tid 1 2024-09-21T09:33:22.458 INFO:tasks.rados.rados.0.smithi022.stdout:680: writing smithi02284960-35 from 2351104 to 2818048 tid 2 2024-09-21T09:33:22.463 INFO:tasks.rados.rados.0.smithi022.stdout:680: writing smithi02284960-35 from 2818048 to 3506176 tid 3 2024-09-21T09:33:22.463 INFO:tasks.rados.rados.0.smithi022.stdout:680: writing smithi02284960-35 from 3506176 to 3555328 tid 4 2024-09-21T09:33:22.463 INFO:tasks.rados.rados.0.smithi022.stdout:681: read oid 1 snap -1 2024-09-21T09:33:22.464 INFO:tasks.rados.rados.0.smithi022.stdout:681: expect (ObjNum 239 snap 65 seq_num 239) 2024-09-21T09:33:22.464 INFO:tasks.rados.rados.0.smithi022.stdout:682: rollback oid 26 current snap is 66 2024-09-21T09:33:22.464 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 26 to 62 2024-09-21T09:33:22.464 INFO:tasks.rados.rados.0.smithi022.stdout:679: finishing rollback tid 0 to smithi02284960-13 2024-09-21T09:33:22.464 INFO:tasks.rados.rados.0.smithi022.stdout:679: finishing rollback tid 1 to smithi02284960-13 2024-09-21T09:33:22.464 INFO:tasks.rados.rados.0.smithi022.stdout:679: finishing rollback tid 2 to smithi02284960-13 2024-09-21T09:33:22.464 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 13 v 275 (ObjNum 125 snap 25 seq_num 125) dirty exists 2024-09-21T09:33:22.464 INFO:tasks.rados.rados.0.smithi022.stdout:680: finishing write tid 1 to smithi02284960-35 2024-09-21T09:33:22.464 INFO:tasks.rados.rados.0.smithi022.stdout:680: finishing write tid 2 to smithi02284960-35 2024-09-21T09:33:22.465 INFO:tasks.rados.rados.0.smithi022.stdout:679: done (3 left) 2024-09-21T09:33:22.465 INFO:tasks.rados.rados.0.smithi022.stdout:683: rollback oid 7 current snap is 66 2024-09-21T09:33:22.465 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 7 to 54 2024-09-21T09:33:22.465 INFO:tasks.rados.rados.0.smithi022.stdout:684: setattr oid 45 current snap is 66 2024-09-21T09:33:22.465 INFO:tasks.rados.rados.0.smithi022.stdout:685: delete oid 17 current snap is 66 2024-09-21T09:33:22.468 INFO:tasks.rados.rados.0.smithi022.stdout:680: finishing write tid 3 to smithi02284960-35 2024-09-21T09:33:22.468 INFO:tasks.rados.rados.0.smithi022.stdout:682: finishing rollback tid 0 to smithi02284960-26 2024-09-21T09:33:22.468 INFO:tasks.rados.rados.0.smithi022.stdout:683: finishing rollback tid 1 to smithi02284960-7 2024-09-21T09:33:22.469 INFO:tasks.rados.rados.0.smithi022.stdout:680: finishing write tid 4 to smithi02284960-35 2024-09-21T09:33:22.469 INFO:tasks.rados.rados.0.smithi022.stdout:683: finishing rollback tid 2 to smithi02284960-7 2024-09-21T09:33:22.470 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 7 v 367 (ObjNum 182 snap 47 seq_num 182) dirty exists 2024-09-21T09:33:22.470 INFO:tasks.rados.rados.0.smithi022.stdout:682: finishing rollback tid 1 to smithi02284960-26 2024-09-21T09:33:22.470 INFO:tasks.rados.rados.0.smithi022.stdout:682: finishing rollback tid 2 to smithi02284960-26 2024-09-21T09:33:22.470 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 26 v 279 (ObjNum 204 snap 56 seq_num 204) dirty exists 2024-09-21T09:33:22.470 INFO:tasks.rados.rados.0.smithi022.stdout:680: finishing write tid 5 to smithi02284960-35 2024-09-21T09:33:22.470 INFO:tasks.rados.rados.0.smithi022.stdout:682: done (5 left) 2024-09-21T09:33:22.470 INFO:tasks.rados.rados.0.smithi022.stdout:683: done (4 left) 2024-09-21T09:33:22.470 INFO:tasks.rados.rados.0.smithi022.stdout:685: done (3 left) 2024-09-21T09:33:22.470 INFO:tasks.rados.rados.0.smithi022.stdout:686: delete oid 13 current snap is 66 2024-09-21T09:33:22.471 INFO:tasks.rados.rados.0.smithi022.stdout:680: finishing write tid 6 to smithi02284960-35 2024-09-21T09:33:22.471 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 35 v 121 (ObjNum 241 snap 66 seq_num 241) dirty exists 2024-09-21T09:33:22.471 INFO:tasks.rados.rados.0.smithi022.stdout:680: left oid 35 (ObjNum 241 snap 66 seq_num 241) 2024-09-21T09:33:22.471 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 45 v 236 (ObjNum 219 snap 63 seq_num 219) dirty exists 2024-09-21T09:33:22.473 INFO:tasks.rados.rados.0.smithi022.stdout:680: done (3 left) 2024-09-21T09:33:22.473 INFO:tasks.rados.rados.0.smithi022.stdout:684: done (2 left) 2024-09-21T09:33:22.473 INFO:tasks.rados.rados.0.smithi022.stdout:686: done (1 left) 2024-09-21T09:33:22.473 INFO:tasks.rados.rados.0.smithi022.stdout:687: rmattr oid 33 current snap is 66 2024-09-21T09:33:22.473 INFO:tasks.rados.rados.0.smithi022.stdout:687: done (1 left) 2024-09-21T09:33:22.473 INFO:tasks.rados.rados.0.smithi022.stdout:688: read oid 28 snap -1 2024-09-21T09:33:22.473 INFO:tasks.rados.rados.0.smithi022.stdout:688: expect (ObjNum 162 snap 40 seq_num 162) 2024-09-21T09:33:22.473 INFO:tasks.rados.rados.0.smithi022.stdout:689: delete oid 29 current snap is 66 2024-09-21T09:33:22.475 INFO:tasks.rados.rados.0.smithi022.stdout:681: expect (ObjNum 239 snap 65 seq_num 239) 2024-09-21T09:33:22.529 INFO:tasks.rados.rados.0.smithi022.stdout:688: expect (ObjNum 162 snap 40 seq_num 162) 2024-09-21T09:33:22.548 INFO:tasks.rados.rados.0.smithi022.stdout:681: done (2 left) 2024-09-21T09:33:22.548 INFO:tasks.rados.rados.0.smithi022.stdout:688: done (1 left) 2024-09-21T09:33:22.548 INFO:tasks.rados.rados.0.smithi022.stdout:689: done (0 left) 2024-09-21T09:33:22.548 INFO:tasks.rados.rados.0.smithi022.stdout:690: rollback oid 9 current snap is 66 2024-09-21T09:33:22.548 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 9 to 32 2024-09-21T09:33:22.548 INFO:tasks.rados.rados.0.smithi022.stdout:691: read oid 6 snap -1 2024-09-21T09:33:22.548 INFO:tasks.rados.rados.0.smithi022.stdout:691: expect deleted 2024-09-21T09:33:22.548 INFO:tasks.rados.rados.0.smithi022.stdout:692: read oid 24 snap 53 2024-09-21T09:33:22.548 INFO:tasks.rados.rados.0.smithi022.stdout:692: expect deleted 2024-09-21T09:33:22.548 INFO:tasks.rados.rados.0.smithi022.stdout:693: snap_create 2024-09-21T09:33:22.549 INFO:tasks.rados.rados.0.smithi022.stdout:690: finishing rollback tid 0 to smithi02284960-9 2024-09-21T09:33:22.550 INFO:tasks.rados.rados.0.smithi022.stdout:690: finishing rollback tid 1 to smithi02284960-9 2024-09-21T09:33:22.550 INFO:tasks.rados.rados.0.smithi022.stdout:690: finishing rollback tid 2 to smithi02284960-9 2024-09-21T09:33:22.550 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 9 v 202 (ObjNum 137 snap 30 seq_num 137) dirty exists 2024-09-21T09:33:22.641 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:33:22 smithi022 podman[87672]: 2024-09-21 09:33:22.533468459 +0000 UTC m=+2.445920964 image pull 72c9c208898624938c9e4183d6686ea4a5fd3f912bc29bc3f00147924c521a3e quay.io/prometheus/node-exporter:v1.7.0 2024-09-21T09:33:22.641 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:33:22 smithi022 podman[87672]: 2024-09-21 09:33:22.640169144 +0000 UTC m=+2.552621693 container create 593c6158f524345fa890f79e64eccc692672235f972c563806a4c9320d33195e (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-21T09:33:22.960 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:33:22 smithi022 podman[87672]: 2024-09-21 09:33:22.875991514 +0000 UTC m=+2.788444019 container init 593c6158f524345fa890f79e64eccc692672235f972c563806a4c9320d33195e (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-21T09:33:22.960 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:33:22 smithi022 podman[87672]: 2024-09-21 09:33:22.881626775 +0000 UTC m=+2.794079280 container start 593c6158f524345fa890f79e64eccc692672235f972c563806a4c9320d33195e (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-21T09:33:22.961 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:33:22 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a[87756]: ts=2024-09-21T09:33:22.884Z caller=node_exporter.go:192 level=info msg="Starting node_exporter" version="(version=1.7.0, branch=HEAD, revision=7333465abf9efba81876303bb57e6fadb946041b)" 2024-09-21T09:33:22.961 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:33:22 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a[87756]: ts=2024-09-21T09:33:22.884Z caller=node_exporter.go:193 level=info msg="Build context" build_context="(go=go1.21.4, platform=linux/amd64, user=root@35918982f6d8, date=20231112-23:53:35, tags=netgo osusergo static_build)" 2024-09-21T09:33:22.961 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:33:22 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a[87756]: ts=2024-09-21T09:33:22.885Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-09-21T09:33:22.961 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:33:22 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a[87756]: ts=2024-09-21T09:33:22.885Z caller=diskstats_linux.go:265 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-09-21T09:33:22.961 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:33:22 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a[87756]: ts=2024-09-21T09:33:22.885Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-09-21T09:33:22.961 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:33:22 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a[87756]: ts=2024-09-21T09:33:22.885Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-09-21T09:33:22.961 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:33:22 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a[87756]: ts=2024-09-21T09:33:22.885Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-09-21T09:33:22.961 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:33:22 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a[87756]: ts=2024-09-21T09:33:22.885Z caller=node_exporter.go:117 level=info collector=arp 2024-09-21T09:33:22.961 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:33:22 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a[87756]: ts=2024-09-21T09:33:22.885Z caller=node_exporter.go:117 level=info collector=bcache 2024-09-21T09:33:22.961 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:33:22 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a[87756]: ts=2024-09-21T09:33:22.885Z caller=node_exporter.go:117 level=info collector=bonding 2024-09-21T09:33:22.961 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:33:22 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a[87756]: ts=2024-09-21T09:33:22.885Z caller=node_exporter.go:117 level=info collector=btrfs 2024-09-21T09:33:22.961 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:33:22 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a[87756]: ts=2024-09-21T09:33:22.885Z caller=node_exporter.go:117 level=info collector=conntrack 2024-09-21T09:33:22.961 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:33:22 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a[87756]: ts=2024-09-21T09:33:22.885Z caller=node_exporter.go:117 level=info collector=cpu 2024-09-21T09:33:22.961 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:33:22 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a[87756]: ts=2024-09-21T09:33:22.885Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-09-21T09:33:22.962 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:33:22 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a[87756]: ts=2024-09-21T09:33:22.885Z caller=node_exporter.go:117 level=info collector=diskstats 2024-09-21T09:33:22.962 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:33:22 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a[87756]: ts=2024-09-21T09:33:22.885Z caller=node_exporter.go:117 level=info collector=dmi 2024-09-21T09:33:22.962 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:33:22 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a[87756]: ts=2024-09-21T09:33:22.885Z caller=node_exporter.go:117 level=info collector=edac 2024-09-21T09:33:22.962 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:33:22 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a[87756]: ts=2024-09-21T09:33:22.885Z caller=node_exporter.go:117 level=info collector=entropy 2024-09-21T09:33:22.962 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:33:22 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a[87756]: ts=2024-09-21T09:33:22.885Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-09-21T09:33:22.962 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:33:22 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a[87756]: ts=2024-09-21T09:33:22.885Z caller=node_exporter.go:117 level=info collector=filefd 2024-09-21T09:33:22.962 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:33:22 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a[87756]: ts=2024-09-21T09:33:22.885Z caller=node_exporter.go:117 level=info collector=filesystem 2024-09-21T09:33:22.962 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:33:22 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a[87756]: ts=2024-09-21T09:33:22.885Z caller=node_exporter.go:117 level=info collector=hwmon 2024-09-21T09:33:22.963 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:33:22 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a[87756]: ts=2024-09-21T09:33:22.885Z caller=node_exporter.go:117 level=info collector=infiniband 2024-09-21T09:33:22.963 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:33:22 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a[87756]: ts=2024-09-21T09:33:22.885Z caller=node_exporter.go:117 level=info collector=ipvs 2024-09-21T09:33:22.963 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:33:22 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a[87756]: ts=2024-09-21T09:33:22.885Z caller=node_exporter.go:117 level=info collector=loadavg 2024-09-21T09:33:22.963 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:33:22 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a[87756]: ts=2024-09-21T09:33:22.885Z caller=node_exporter.go:117 level=info collector=mdadm 2024-09-21T09:33:22.963 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:33:22 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a[87756]: ts=2024-09-21T09:33:22.885Z caller=node_exporter.go:117 level=info collector=meminfo 2024-09-21T09:33:22.963 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:33:22 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a[87756]: ts=2024-09-21T09:33:22.885Z caller=node_exporter.go:117 level=info collector=netclass 2024-09-21T09:33:22.963 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:33:22 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a[87756]: ts=2024-09-21T09:33:22.885Z caller=node_exporter.go:117 level=info collector=netdev 2024-09-21T09:33:22.963 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:33:22 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a[87756]: ts=2024-09-21T09:33:22.885Z caller=node_exporter.go:117 level=info collector=netstat 2024-09-21T09:33:22.963 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:33:22 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a[87756]: ts=2024-09-21T09:33:22.885Z caller=node_exporter.go:117 level=info collector=nfs 2024-09-21T09:33:22.963 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:33:22 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a[87756]: ts=2024-09-21T09:33:22.885Z caller=node_exporter.go:117 level=info collector=nfsd 2024-09-21T09:33:22.963 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:33:22 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a[87756]: ts=2024-09-21T09:33:22.885Z caller=node_exporter.go:117 level=info collector=nvme 2024-09-21T09:33:22.963 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:33:22 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a[87756]: ts=2024-09-21T09:33:22.885Z caller=node_exporter.go:117 level=info collector=os 2024-09-21T09:33:22.963 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:33:22 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a[87756]: ts=2024-09-21T09:33:22.885Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-09-21T09:33:22.963 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:33:22 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a[87756]: ts=2024-09-21T09:33:22.885Z caller=node_exporter.go:117 level=info collector=pressure 2024-09-21T09:33:22.963 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:33:22 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a[87756]: ts=2024-09-21T09:33:22.885Z caller=node_exporter.go:117 level=info collector=rapl 2024-09-21T09:33:22.964 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:33:22 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a[87756]: ts=2024-09-21T09:33:22.885Z caller=node_exporter.go:117 level=info collector=schedstat 2024-09-21T09:33:22.964 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:33:22 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a[87756]: ts=2024-09-21T09:33:22.885Z caller=node_exporter.go:117 level=info collector=selinux 2024-09-21T09:33:22.964 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:33:22 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a[87756]: ts=2024-09-21T09:33:22.885Z caller=node_exporter.go:117 level=info collector=sockstat 2024-09-21T09:33:22.964 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:33:22 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a[87756]: ts=2024-09-21T09:33:22.885Z caller=node_exporter.go:117 level=info collector=softnet 2024-09-21T09:33:22.964 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:33:22 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a[87756]: ts=2024-09-21T09:33:22.885Z caller=node_exporter.go:117 level=info collector=stat 2024-09-21T09:33:22.964 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:33:22 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a[87756]: ts=2024-09-21T09:33:22.885Z caller=node_exporter.go:117 level=info collector=tapestats 2024-09-21T09:33:22.964 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:33:22 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a[87756]: ts=2024-09-21T09:33:22.885Z caller=node_exporter.go:117 level=info collector=textfile 2024-09-21T09:33:22.964 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:33:22 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a[87756]: ts=2024-09-21T09:33:22.885Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-09-21T09:33:22.964 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:33:22 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a[87756]: ts=2024-09-21T09:33:22.885Z caller=node_exporter.go:117 level=info collector=time 2024-09-21T09:33:22.964 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:33:22 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a[87756]: ts=2024-09-21T09:33:22.885Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-09-21T09:33:22.964 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:33:22 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a[87756]: ts=2024-09-21T09:33:22.885Z caller=node_exporter.go:117 level=info collector=uname 2024-09-21T09:33:22.964 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:33:22 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a[87756]: ts=2024-09-21T09:33:22.885Z caller=node_exporter.go:117 level=info collector=vmstat 2024-09-21T09:33:22.964 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:33:22 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a[87756]: ts=2024-09-21T09:33:22.885Z caller=node_exporter.go:117 level=info collector=xfs 2024-09-21T09:33:22.964 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:33:22 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a[87756]: ts=2024-09-21T09:33:22.885Z caller=node_exporter.go:117 level=info collector=zfs 2024-09-21T09:33:22.964 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:33:22 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a[87756]: ts=2024-09-21T09:33:22.886Z caller=tls_config.go:274 level=info msg="Listening on" address=[::]:9100 2024-09-21T09:33:22.964 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:33:22 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a[87756]: ts=2024-09-21T09:33:22.886Z caller=tls_config.go:277 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-09-21T09:33:22.965 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:33:22 smithi022 bash[87672]: 593c6158f524345fa890f79e64eccc692672235f972c563806a4c9320d33195e 2024-09-21T09:33:23.308 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:33:22 smithi022 systemd[1]: Started Ceph node-exporter.a for d65e801e-77fa-11ef-baf6-efdc52797490. 2024-09-21T09:33:23.451 INFO:tasks.rados.rados.0.smithi022.stdout:690: done (3 left) 2024-09-21T09:33:23.451 INFO:tasks.rados.rados.0.smithi022.stdout:691: done (2 left) 2024-09-21T09:33:23.452 INFO:tasks.rados.rados.0.smithi022.stdout:692: done (1 left) 2024-09-21T09:33:23.452 INFO:tasks.rados.rados.0.smithi022.stdout:693: done (0 left) 2024-09-21T09:33:23.452 INFO:tasks.rados.rados.0.smithi022.stdout:694: snap_remove snap 52 2024-09-21T09:33:23.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:23 smithi064 ceph-mon[31124]: osdmap e192: 8 total, 8 up, 8 in 2024-09-21T09:33:23.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:23 smithi064 ceph-mon[31124]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:23.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:23 smithi064 ceph-mon[31124]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:23.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:23 smithi064 ceph-mon[31124]: pgmap v45: 121 pgs: 4 active+clean+snaptrim, 117 active+clean; 195 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 9.0 MiB/s rd, 4.5 MiB/s wr, 25 op/s 2024-09-21T09:33:23.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:23 smithi064 ceph-mon[31124]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-21T09:33:23.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:23 smithi022 ceph-mon[39525]: osdmap e192: 8 total, 8 up, 8 in 2024-09-21T09:33:23.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:23 smithi022 ceph-mon[39525]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:23.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:23 smithi022 ceph-mon[39525]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:23.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:23 smithi022 ceph-mon[39525]: pgmap v45: 121 pgs: 4 active+clean+snaptrim, 117 active+clean; 195 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 9.0 MiB/s rd, 4.5 MiB/s wr, 25 op/s 2024-09-21T09:33:23.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:23 smithi022 ceph-mon[39525]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-21T09:33:23.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:23 smithi022 ceph-mon[27763]: osdmap e192: 8 total, 8 up, 8 in 2024-09-21T09:33:23.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:23 smithi022 ceph-mon[27763]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:23.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:23 smithi022 ceph-mon[27763]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:23.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:23 smithi022 ceph-mon[27763]: pgmap v45: 121 pgs: 4 active+clean+snaptrim, 117 active+clean; 195 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 9.0 MiB/s rd, 4.5 MiB/s wr, 25 op/s 2024-09-21T09:33:23.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:23 smithi022 ceph-mon[27763]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-21T09:33:24.455 INFO:tasks.rados.rados.0.smithi022.stdout:694: done (0 left) 2024-09-21T09:33:24.455 INFO:tasks.rados.rados.0.smithi022.stdout:695: snap_remove snap 50 2024-09-21T09:33:24.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:24 smithi022 ceph-mon[39525]: Deploying daemon prometheus.a on smithi064 2024-09-21T09:33:24.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:24 smithi022 ceph-mon[39525]: osdmap e193: 8 total, 8 up, 8 in 2024-09-21T09:33:24.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:24 smithi022 ceph-mon[27763]: Deploying daemon prometheus.a on smithi064 2024-09-21T09:33:24.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:24 smithi022 ceph-mon[27763]: osdmap e193: 8 total, 8 up, 8 in 2024-09-21T09:33:24.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:24 smithi064 ceph-mon[31124]: Deploying daemon prometheus.a on smithi064 2024-09-21T09:33:24.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:24 smithi064 ceph-mon[31124]: osdmap e193: 8 total, 8 up, 8 in 2024-09-21T09:33:25.443 INFO:tasks.rados.rados.0.smithi022.stdout:695: done (0 left) 2024-09-21T09:33:25.443 INFO:tasks.rados.rados.0.smithi022.stdout:696: copy_from oid 48 from oid 5 current snap is 67 2024-09-21T09:33:25.444 INFO:tasks.rados.rados.0.smithi022.stdout:697: setattr oid 20 current snap is 67 2024-09-21T09:33:25.445 INFO:tasks.rados.rados.0.smithi022.stdout:698: read oid 3 snap -1 2024-09-21T09:33:25.445 INFO:tasks.rados.rados.0.smithi022.stdout:698: expect (ObjNum 226 snap 63 seq_num 226) 2024-09-21T09:33:25.445 INFO:tasks.rados.rados.0.smithi022.stdout:699: copy_from oid 49 from oid 17 current snap is 67 2024-09-21T09:33:25.446 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 35 current snap is 67 2024-09-21T09:33:25.446 INFO:tasks.rados.rados.0.smithi022.stdout:700: seq_num 245 ranges {3555328=507904,4063232=458752,4521984=671744,5193728=73728} 2024-09-21T09:33:25.471 INFO:tasks.rados.rados.0.smithi022.stdout:700: writing smithi02284960-35 from 3555328 to 4063232 tid 1 2024-09-21T09:33:25.473 INFO:tasks.rados.rados.0.smithi022.stdout:700: writing smithi02284960-35 from 4063232 to 4521984 tid 2 2024-09-21T09:33:25.478 INFO:tasks.rados.rados.0.smithi022.stdout:700: writing smithi02284960-35 from 4521984 to 5193728 tid 3 2024-09-21T09:33:25.479 INFO:tasks.rados.rados.0.smithi022.stdout:700: writing smithi02284960-35 from 5193728 to 5267456 tid 4 2024-09-21T09:33:25.479 INFO:tasks.rados.rados.0.smithi022.stdout:701: copy_from oid 31 from oid 40 current snap is 67 2024-09-21T09:33:25.479 INFO:tasks.rados.rados.0.smithi022.stdout:696: finishing copy_from to smithi02284960-48 2024-09-21T09:33:25.479 INFO:tasks.rados.rados.0.smithi022.stdout:696: got expected ENOENT (src dne) 2024-09-21T09:33:25.480 INFO:tasks.rados.rados.0.smithi022.stdout:696: finishing copy_from racing read to smithi02284960-48 2024-09-21T09:33:25.480 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 20 v 379 (ObjNum 132 snap 27 seq_num 132) dirty exists 2024-09-21T09:33:25.480 INFO:tasks.rados.rados.0.smithi022.stdout:696: done (5 left) 2024-09-21T09:33:25.480 INFO:tasks.rados.rados.0.smithi022.stdout:697: done (4 left) 2024-09-21T09:33:25.480 INFO:tasks.rados.rados.0.smithi022.stdout:702: delete oid 27 current snap is 67 2024-09-21T09:33:25.481 INFO:tasks.rados.rados.0.smithi022.stdout:699: finishing copy_from to smithi02284960-49 2024-09-21T09:33:25.481 INFO:tasks.rados.rados.0.smithi022.stdout:699: got expected ENOENT (src dne) 2024-09-21T09:33:25.481 INFO:tasks.rados.rados.0.smithi022.stdout:699: finishing copy_from racing read to smithi02284960-49 2024-09-21T09:33:25.481 INFO:tasks.rados.rados.0.smithi022.stdout:698: expect (ObjNum 226 snap 63 seq_num 226) 2024-09-21T09:33:25.581 INFO:tasks.rados.rados.0.smithi022.stdout:700: finishing write tid 1 to smithi02284960-35 2024-09-21T09:33:25.581 INFO:tasks.rados.rados.0.smithi022.stdout:700: finishing write tid 2 to smithi02284960-35 2024-09-21T09:33:25.581 INFO:tasks.rados.rados.0.smithi022.stdout:700: finishing write tid 3 to smithi02284960-35 2024-09-21T09:33:25.581 INFO:tasks.rados.rados.0.smithi022.stdout:700: finishing write tid 4 to smithi02284960-35 2024-09-21T09:33:25.581 INFO:tasks.rados.rados.0.smithi022.stdout:700: finishing write tid 5 to smithi02284960-35 2024-09-21T09:33:25.582 INFO:tasks.rados.rados.0.smithi022.stdout:698: done (4 left) 2024-09-21T09:33:25.582 INFO:tasks.rados.rados.0.smithi022.stdout:699: done (3 left) 2024-09-21T09:33:25.582 INFO:tasks.rados.rados.0.smithi022.stdout:702: done (2 left) 2024-09-21T09:33:25.582 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 41 current snap is 67 2024-09-21T09:33:25.582 INFO:tasks.rados.rados.0.smithi022.stdout:700: finishing write tid 6 to smithi02284960-35 2024-09-21T09:33:25.582 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 35 v 131 (ObjNum 245 snap 67 seq_num 245) dirty exists 2024-09-21T09:33:25.582 INFO:tasks.rados.rados.0.smithi022.stdout:700: left oid 35 (ObjNum 245 snap 67 seq_num 245) 2024-09-21T09:33:25.582 INFO:tasks.rados.rados.0.smithi022.stdout:701: finishing copy_from racing read to smithi02284960-31 2024-09-21T09:33:25.582 INFO:tasks.rados.rados.0.smithi022.stdout:701: finishing copy_from to smithi02284960-31 2024-09-21T09:33:25.582 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 31 v 320 (ObjNum 230 snap 65 seq_num 230) dirty exists 2024-09-21T09:33:25.582 INFO:tasks.rados.rados.0.smithi022.stdout:703: seq_num 247 ranges {1548288=540672,2088960=622592,2711552=581632} 2024-09-21T09:33:25.592 INFO:tasks.rados.rados.0.smithi022.stdout:703: writing smithi02284960-41 from 1548288 to 2088960 tid 1 2024-09-21T09:33:25.595 INFO:tasks.rados.rados.0.smithi022.stdout:703: writing smithi02284960-41 from 2088960 to 2711552 tid 2 2024-09-21T09:33:25.599 INFO:tasks.rados.rados.0.smithi022.stdout:703: writing smithi02284960-41 from 2711552 to 3293184 tid 3 2024-09-21T09:33:25.599 INFO:tasks.rados.rados.0.smithi022.stdout:700: done (2 left) 2024-09-21T09:33:25.599 INFO:tasks.rados.rados.0.smithi022.stdout:701: done (1 left) 2024-09-21T09:33:25.599 INFO:tasks.rados.rados.0.smithi022.stdout:704: snap_create 2024-09-21T09:33:25.599 INFO:tasks.rados.rados.0.smithi022.stdout:703: finishing write tid 1 to smithi02284960-41 2024-09-21T09:33:25.599 INFO:tasks.rados.rados.0.smithi022.stdout:703: finishing write tid 2 to smithi02284960-41 2024-09-21T09:33:25.601 INFO:tasks.rados.rados.0.smithi022.stdout:703: finishing write tid 3 to smithi02284960-41 2024-09-21T09:33:25.601 INFO:tasks.rados.rados.0.smithi022.stdout:703: finishing write tid 4 to smithi02284960-41 2024-09-21T09:33:25.602 INFO:tasks.rados.rados.0.smithi022.stdout:703: finishing write tid 5 to smithi02284960-41 2024-09-21T09:33:25.602 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 41 v 354 (ObjNum 247 snap 67 seq_num 247) dirty exists 2024-09-21T09:33:25.602 INFO:tasks.rados.rados.0.smithi022.stdout:703: left oid 41 (ObjNum 247 snap 67 seq_num 247) 2024-09-21T09:33:25.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:25 smithi022 ceph-mon[39525]: osdmap e194: 8 total, 8 up, 8 in 2024-09-21T09:33:25.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:25 smithi022 ceph-mon[39525]: pgmap v48: 121 pgs: 4 active+clean+snaptrim, 117 active+clean; 195 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 9.0 MiB/s rd, 4.5 MiB/s wr, 25 op/s 2024-09-21T09:33:25.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:25 smithi022 ceph-mon[39525]: osdmap e195: 8 total, 8 up, 8 in 2024-09-21T09:33:25.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:25 smithi022 ceph-mon[27763]: osdmap e194: 8 total, 8 up, 8 in 2024-09-21T09:33:25.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:25 smithi022 ceph-mon[27763]: pgmap v48: 121 pgs: 4 active+clean+snaptrim, 117 active+clean; 195 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 9.0 MiB/s rd, 4.5 MiB/s wr, 25 op/s 2024-09-21T09:33:25.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:25 smithi022 ceph-mon[27763]: osdmap e195: 8 total, 8 up, 8 in 2024-09-21T09:33:25.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:25 smithi064 ceph-mon[31124]: osdmap e194: 8 total, 8 up, 8 in 2024-09-21T09:33:25.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:25 smithi064 ceph-mon[31124]: pgmap v48: 121 pgs: 4 active+clean+snaptrim, 117 active+clean; 195 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 9.0 MiB/s rd, 4.5 MiB/s wr, 25 op/s 2024-09-21T09:33:25.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:25 smithi064 ceph-mon[31124]: osdmap e195: 8 total, 8 up, 8 in 2024-09-21T09:33:26.474 INFO:tasks.rados.rados.0.smithi022.stdout:703: done (1 left) 2024-09-21T09:33:26.474 INFO:tasks.rados.rados.0.smithi022.stdout:704: done (0 left) 2024-09-21T09:33:26.474 INFO:tasks.rados.rados.0.smithi022.stdout:705: delete oid 35 current snap is 68 2024-09-21T09:33:26.479 INFO:tasks.rados.rados.0.smithi022.stdout:705: done (0 left) 2024-09-21T09:33:26.479 INFO:tasks.rados.rados.0.smithi022.stdout:706: snap_create 2024-09-21T09:33:27.479 INFO:tasks.rados.rados.0.smithi022.stdout:706: done (0 left) 2024-09-21T09:33:27.479 INFO:tasks.rados.rados.0.smithi022.stdout:707: snap_create 2024-09-21T09:33:27.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:27 smithi022 ceph-mon[39525]: osdmap e196: 8 total, 8 up, 8 in 2024-09-21T09:33:27.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:27 smithi022 ceph-mon[39525]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:33:27.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:27 smithi022 ceph-mon[39525]: pgmap v51: 121 pgs: 121 active+clean; 205 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.7 MiB/s rd, 2.7 MiB/s wr, 23 op/s 2024-09-21T09:33:27.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:27 smithi022 ceph-mon[27763]: osdmap e196: 8 total, 8 up, 8 in 2024-09-21T09:33:27.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:27 smithi022 ceph-mon[27763]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:33:27.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:27 smithi022 ceph-mon[27763]: pgmap v51: 121 pgs: 121 active+clean; 205 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.7 MiB/s rd, 2.7 MiB/s wr, 23 op/s 2024-09-21T09:33:27.809 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 21 09:33:27 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-alertmanager-a[87373]: ts=2024-09-21T09:33:27.478Z caller=cluster.go:700 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.002189809s 2024-09-21T09:33:27.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:27 smithi064 ceph-mon[31124]: osdmap e196: 8 total, 8 up, 8 in 2024-09-21T09:33:27.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:27 smithi064 ceph-mon[31124]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:33:27.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:27 smithi064 ceph-mon[31124]: pgmap v51: 121 pgs: 121 active+clean; 205 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.7 MiB/s rd, 2.7 MiB/s wr, 23 op/s 2024-09-21T09:33:28.488 INFO:tasks.rados.rados.0.smithi022.stdout:707: done (0 left) 2024-09-21T09:33:28.488 INFO:tasks.rados.rados.0.smithi022.stdout:708: read oid 22 snap -1 2024-09-21T09:33:28.488 INFO:tasks.rados.rados.0.smithi022.stdout:708: expect (ObjNum 170 snap 43 seq_num 170) 2024-09-21T09:33:28.488 INFO:tasks.rados.rados.0.smithi022.stdout:709: snap_remove snap 32 2024-09-21T09:33:28.807 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:28 smithi022 ceph-mon[39525]: osdmap e197: 8 total, 8 up, 8 in 2024-09-21T09:33:28.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:28 smithi022 ceph-mon[27763]: osdmap e197: 8 total, 8 up, 8 in 2024-09-21T09:33:28.921 INFO:teuthology.orchestra.run.smithi022.stdout:true 2024-09-21T09:33:28.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:28 smithi064 ceph-mon[31124]: osdmap e197: 8 total, 8 up, 8 in 2024-09-21T09:33:29.492 INFO:tasks.rados.rados.0.smithi022.stdout:709: done (1 left) 2024-09-21T09:33:29.492 INFO:tasks.rados.rados.0.smithi022.stdout:710: read oid 48 snap -1 2024-09-21T09:33:29.492 INFO:tasks.rados.rados.0.smithi022.stdout:710: expect deleted 2024-09-21T09:33:29.492 INFO:tasks.rados.rados.0.smithi022.stdout:711: rmattr oid 49 current snap is 70 2024-09-21T09:33:29.492 INFO:tasks.rados.rados.0.smithi022.stdout:708: expect (ObjNum 170 snap 43 seq_num 170) 2024-09-21T09:33:29.506 INFO:teuthology.orchestra.run.smithi022.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T09:33:29.506 INFO:teuthology.orchestra.run.smithi022.stdout:alertmanager.a smithi022 *:9093,9094 starting - - - - 2024-09-21T09:33:29.506 INFO:teuthology.orchestra.run.smithi022.stdout:grafana.a smithi064 *:3000 running (4m) 26s ago 4m 89.2M - 9.4.7 954c08fa6188 ffde1f806d0b 2024-09-21T09:33:29.506 INFO:teuthology.orchestra.run.smithi022.stdout:mds.a smithi022 running (6m) 27s ago 6m 16.0M - 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 34e4221f3a27 2024-09-21T09:33:29.506 INFO:teuthology.orchestra.run.smithi022.stdout:mds.b smithi064 running (6m) 26s ago 6m 19.4M - 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 0f846b3ec694 2024-09-21T09:33:29.507 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.x smithi064 *:8443,9283 running (53s) 26s ago 9m 500M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 c102c1678bcc 2024-09-21T09:33:29.507 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.y smithi022 *:9283,8765 running (11m) 27s ago 11m 330M - 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 5bc449c0676f 2024-09-21T09:33:29.507 INFO:teuthology.orchestra.run.smithi022.stdout:mon.a smithi022 running (11m) 27s ago 11m 53.7M 2048M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 1e0a2d74e66e 2024-09-21T09:33:29.507 INFO:teuthology.orchestra.run.smithi022.stdout:mon.b smithi064 running (10m) 26s ago 10m 45.1M 2048M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be ed126533cd64 2024-09-21T09:33:29.507 INFO:teuthology.orchestra.run.smithi022.stdout:mon.c smithi022 running (9m) 27s ago 9m 46.8M 2048M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 79f307e3b64a 2024-09-21T09:33:29.507 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.a smithi022 *:9100 starting - - - - 2024-09-21T09:33:29.508 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.b smithi064 *:9100 running (4m) 26s ago 4m 14.7M - 1.5.0 0da6a335fe13 643a1e1aec61 2024-09-21T09:33:29.508 INFO:teuthology.orchestra.run.smithi022.stdout:osd.0 smithi022 running (9m) 27s ago 9m 113M 2481M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be fcd812003534 2024-09-21T09:33:29.508 INFO:teuthology.orchestra.run.smithi022.stdout:osd.1 smithi022 running (8m) 27s ago 8m 93.3M 2481M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 740462b5c304 2024-09-21T09:33:29.508 INFO:teuthology.orchestra.run.smithi022.stdout:osd.2 smithi022 running (8m) 27s ago 8m 108M 2481M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be fefc7c770c32 2024-09-21T09:33:29.508 INFO:teuthology.orchestra.run.smithi022.stdout:osd.3 smithi022 running (8m) 27s ago 8m 62.4M 2481M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be a56193b67a92 2024-09-21T09:33:29.508 INFO:teuthology.orchestra.run.smithi022.stdout:osd.4 smithi064 running (7m) 26s ago 7m 83.8M 2480M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 2a2937374fc4 2024-09-21T09:33:29.508 INFO:teuthology.orchestra.run.smithi022.stdout:osd.5 smithi064 running (7m) 26s ago 7m 95.5M 2480M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 1900c4af5d28 2024-09-21T09:33:29.508 INFO:teuthology.orchestra.run.smithi022.stdout:osd.6 smithi064 running (6m) 26s ago 6m 84.9M 2480M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 3cc4ec816e5b 2024-09-21T09:33:29.509 INFO:teuthology.orchestra.run.smithi022.stdout:osd.7 smithi064 running (6m) 26s ago 6m 88.0M 2480M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 18ef0be94104 2024-09-21T09:33:29.509 INFO:teuthology.orchestra.run.smithi022.stdout:prometheus.a smithi064 *:9095 running (3m) 26s ago 5m 58.4M - 2.43.0 a07b618ecd1d caee9b54c131 2024-09-21T09:33:29.532 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 49 v 157 (ObjNum 196 snap 50 seq_num 196) dirty exists 2024-09-21T09:33:29.532 INFO:tasks.rados.rados.0.smithi022.stdout:708: done (2 left) 2024-09-21T09:33:29.532 INFO:tasks.rados.rados.0.smithi022.stdout:710: done (1 left) 2024-09-21T09:33:29.532 INFO:tasks.rados.rados.0.smithi022.stdout:711: done (0 left) 2024-09-21T09:33:29.532 INFO:tasks.rados.rados.0.smithi022.stdout:712: delete oid 16 current snap is 70 2024-09-21T09:33:29.533 INFO:tasks.rados.rados.0.smithi022.stdout:712: done (0 left) 2024-09-21T09:33:29.533 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 4 current snap is 70 2024-09-21T09:33:29.533 INFO:tasks.rados.rados.0.smithi022.stdout:713: seq_num 250 ranges {1490944=589824,2080768=679936,2760704=409600,3170304=114688} 2024-09-21T09:33:29.545 INFO:tasks.rados.rados.0.smithi022.stdout:713: writing smithi02284960-4 from 1490944 to 2080768 tid 1 2024-09-21T09:33:29.549 INFO:tasks.rados.rados.0.smithi022.stdout:713: writing smithi02284960-4 from 2080768 to 2760704 tid 2 2024-09-21T09:33:29.551 INFO:tasks.rados.rados.0.smithi022.stdout:713: writing smithi02284960-4 from 2760704 to 3170304 tid 3 2024-09-21T09:33:29.552 INFO:tasks.rados.rados.0.smithi022.stdout:713: writing smithi02284960-4 from 3170304 to 3284992 tid 4 2024-09-21T09:33:29.552 INFO:tasks.rados.rados.0.smithi022.stdout:714: read oid 11 snap -1 2024-09-21T09:33:29.552 INFO:tasks.rados.rados.0.smithi022.stdout:714: expect (ObjNum 229 snap 64 seq_num 229) 2024-09-21T09:33:29.552 INFO:tasks.rados.rados.0.smithi022.stdout:713: finishing write tid 1 to smithi02284960-4 2024-09-21T09:33:29.552 INFO:tasks.rados.rados.0.smithi022.stdout:713: finishing write tid 2 to smithi02284960-4 2024-09-21T09:33:29.552 INFO:tasks.rados.rados.0.smithi022.stdout:715: read oid 42 snap 63 2024-09-21T09:33:29.553 INFO:tasks.rados.rados.0.smithi022.stdout:715: expect deleted 2024-09-21T09:33:29.553 INFO:tasks.rados.rados.0.smithi022.stdout:716: setattr oid 18 current snap is 70 2024-09-21T09:33:29.553 INFO:tasks.rados.rados.0.smithi022.stdout:715: done (3 left) 2024-09-21T09:33:29.553 INFO:tasks.rados.rados.0.smithi022.stdout:717: snap_remove snap 61 2024-09-21T09:33:29.807 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:29 smithi022 ceph-mon[39525]: osdmap e198: 8 total, 8 up, 8 in 2024-09-21T09:33:29.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:29 smithi022 ceph-mon[39525]: from='client.15099 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:33:29.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:29 smithi022 ceph-mon[39525]: pgmap v54: 121 pgs: 121 active+clean; 205 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.7 MiB/s rd, 2.7 MiB/s wr, 23 op/s 2024-09-21T09:33:29.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:29 smithi022 ceph-mon[27763]: osdmap e198: 8 total, 8 up, 8 in 2024-09-21T09:33:29.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:29 smithi022 ceph-mon[27763]: from='client.15099 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:33:29.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:29 smithi022 ceph-mon[27763]: pgmap v54: 121 pgs: 121 active+clean; 205 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.7 MiB/s rd, 2.7 MiB/s wr, 23 op/s 2024-09-21T09:33:29.872 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-09-21T09:33:29.872 INFO:teuthology.orchestra.run.smithi022.stdout: "mon": { 2024-09-21T09:33:29.872 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 3 2024-09-21T09:33:29.872 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-21T09:33:29.872 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr": { 2024-09-21T09:33:29.872 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 1, 2024-09-21T09:33:29.872 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 1 2024-09-21T09:33:29.872 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-21T09:33:29.872 INFO:teuthology.orchestra.run.smithi022.stdout: "osd": { 2024-09-21T09:33:29.872 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 8 2024-09-21T09:33:29.872 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-21T09:33:29.873 INFO:teuthology.orchestra.run.smithi022.stdout: "mds": { 2024-09-21T09:33:29.873 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 2 2024-09-21T09:33:29.873 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-21T09:33:29.873 INFO:teuthology.orchestra.run.smithi022.stdout: "overall": { 2024-09-21T09:33:29.873 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 14, 2024-09-21T09:33:29.873 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 1 2024-09-21T09:33:29.873 INFO:teuthology.orchestra.run.smithi022.stdout: } 2024-09-21T09:33:29.873 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-09-21T09:33:29.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:29 smithi064 ceph-mon[31124]: osdmap e198: 8 total, 8 up, 8 in 2024-09-21T09:33:29.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:29 smithi064 ceph-mon[31124]: from='client.15099 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:33:29.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:29 smithi064 ceph-mon[31124]: pgmap v54: 121 pgs: 121 active+clean; 205 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.7 MiB/s rd, 2.7 MiB/s wr, 23 op/s 2024-09-21T09:33:30.165 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-09-21T09:33:30.165 INFO:teuthology.orchestra.run.smithi022.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47", 2024-09-21T09:33:30.165 INFO:teuthology.orchestra.run.smithi022.stdout: "in_progress": true, 2024-09-21T09:33:30.165 INFO:teuthology.orchestra.run.smithi022.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T09:33:30.165 INFO:teuthology.orchestra.run.smithi022.stdout: "services_complete": [], 2024-09-21T09:33:30.166 INFO:teuthology.orchestra.run.smithi022.stdout: "progress": "1/20 daemons upgraded", 2024-09-21T09:33:30.166 INFO:teuthology.orchestra.run.smithi022.stdout: "message": "", 2024-09-21T09:33:30.166 INFO:teuthology.orchestra.run.smithi022.stdout: "is_paused": false 2024-09-21T09:33:30.166 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-09-21T09:33:30.447 INFO:tasks.rados.rados.0.smithi022.stdout:713: finishing write tid 3 to smithi02284960-4 2024-09-21T09:33:30.447 INFO:tasks.rados.rados.0.smithi022.stdout:713: finishing write tid 4 to smithi02284960-4 2024-09-21T09:33:30.447 INFO:tasks.rados.rados.0.smithi022.stdout:713: finishing write tid 5 to smithi02284960-4 2024-09-21T09:33:30.447 INFO:tasks.rados.rados.0.smithi022.stdout:713: finishing write tid 6 to smithi02284960-4 2024-09-21T09:33:30.447 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 4 v 210 (ObjNum 250 snap 70 seq_num 250) dirty exists 2024-09-21T09:33:30.448 INFO:tasks.rados.rados.0.smithi022.stdout:713: left oid 4 (ObjNum 250 snap 70 seq_num 250) 2024-09-21T09:33:30.448 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 18 v 255 (ObjNum 210 snap 58 seq_num 210) dirty exists 2024-09-21T09:33:30.448 INFO:tasks.rados.rados.0.smithi022.stdout:714: expect (ObjNum 229 snap 64 seq_num 229) 2024-09-21T09:33:30.628 INFO:tasks.rados.rados.0.smithi022.stdout:713: done (3 left) 2024-09-21T09:33:30.628 INFO:tasks.rados.rados.0.smithi022.stdout:714: done (2 left) 2024-09-21T09:33:30.628 INFO:tasks.rados.rados.0.smithi022.stdout:716: done (1 left) 2024-09-21T09:33:30.629 INFO:tasks.rados.rados.0.smithi022.stdout:717: done (0 left) 2024-09-21T09:33:30.629 INFO:tasks.rados.rados.0.smithi022.stdout:718: delete oid 43 current snap is 70 2024-09-21T09:33:30.630 INFO:tasks.rados.rados.0.smithi022.stdout:718: done (0 left) 2024-09-21T09:33:30.630 INFO:tasks.rados.rados.0.smithi022.stdout:719: snap_create 2024-09-21T09:33:30.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:30 smithi064 ceph-mon[31124]: from='client.25004 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:33:30.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:30 smithi064 ceph-mon[31124]: osdmap e199: 8 total, 8 up, 8 in 2024-09-21T09:33:30.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:30 smithi064 ceph-mon[31124]: from='client.15111 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:33:30.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:30 smithi064 ceph-mon[31124]: from='client.? 172.21.15.22:0/647194302' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:33:30.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:30 smithi064 ceph-mon[31124]: osdmap e200: 8 total, 8 up, 8 in 2024-09-21T09:33:30.693 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:33:30 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[67503]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:33:30] "GET /metrics HTTP/1.1" 200 36129 "" "Prometheus/2.43.0" 2024-09-21T09:33:30.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:30 smithi022 ceph-mon[39525]: from='client.25004 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:33:30.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:30 smithi022 ceph-mon[39525]: osdmap e199: 8 total, 8 up, 8 in 2024-09-21T09:33:30.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:30 smithi022 ceph-mon[39525]: from='client.15111 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:33:30.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:30 smithi022 ceph-mon[39525]: from='client.? 172.21.15.22:0/647194302' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:33:30.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:30 smithi022 ceph-mon[39525]: osdmap e200: 8 total, 8 up, 8 in 2024-09-21T09:33:30.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:30 smithi022 ceph-mon[27763]: from='client.25004 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:33:30.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:30 smithi022 ceph-mon[27763]: osdmap e199: 8 total, 8 up, 8 in 2024-09-21T09:33:30.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:30 smithi022 ceph-mon[27763]: from='client.15111 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:33:30.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:30 smithi022 ceph-mon[27763]: from='client.? 172.21.15.22:0/647194302' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:33:30.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:30 smithi022 ceph-mon[27763]: osdmap e200: 8 total, 8 up, 8 in 2024-09-21T09:33:31.498 INFO:tasks.rados.rados.0.smithi022.stdout:719: done (0 left) 2024-09-21T09:33:31.498 INFO:tasks.rados.rados.0.smithi022.stdout:720: setattr oid 43 current snap is 71 2024-09-21T09:33:31.500 INFO:tasks.rados.rados.0.smithi022.stdout:721: snap_remove snap 63 2024-09-21T09:33:31.558 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:33:31 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[28012]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:33:31] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-21T09:33:31.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:31 smithi022 ceph-mon[39525]: from='client.15123 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:33:31.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:31 smithi022 ceph-mon[39525]: pgmap v57: 121 pgs: 121 active+clean; 205 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:33:31.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:31 smithi022 ceph-mon[27763]: from='client.15123 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:33:31.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:31 smithi022 ceph-mon[27763]: pgmap v57: 121 pgs: 121 active+clean; 205 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:33:31.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:31 smithi064 ceph-mon[31124]: from='client.15123 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:33:31.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:31 smithi064 ceph-mon[31124]: pgmap v57: 121 pgs: 121 active+clean; 205 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:33:32.500 INFO:tasks.rados.rados.0.smithi022.stdout:721: done (1 left) 2024-09-21T09:33:32.500 INFO:tasks.rados.rados.0.smithi022.stdout:722: read oid 21 snap -1 2024-09-21T09:33:32.500 INFO:tasks.rados.rados.0.smithi022.stdout:722: expect (ObjNum 172 snap 43 seq_num 172) 2024-09-21T09:33:32.500 INFO:tasks.rados.rados.0.smithi022.stdout:723: snap_remove snap 65 2024-09-21T09:33:32.807 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:32 smithi022 ceph-mon[39525]: osdmap e201: 8 total, 8 up, 8 in 2024-09-21T09:33:32.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:32 smithi022 ceph-mon[27763]: osdmap e201: 8 total, 8 up, 8 in 2024-09-21T09:33:32.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:32 smithi064 ceph-mon[31124]: osdmap e201: 8 total, 8 up, 8 in 2024-09-21T09:33:33.514 INFO:tasks.rados.rados.0.smithi022.stdout:723: done (2 left) 2024-09-21T09:33:33.514 INFO:tasks.rados.rados.0.smithi022.stdout:724: copy_from oid 46 from oid 26 current snap is 71 2024-09-21T09:33:33.514 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 43 v 373 (ObjNum 289 snap 0 seq_num 1) dirty exists 2024-09-21T09:33:33.514 INFO:tasks.rados.rados.0.smithi022.stdout:722: expect (ObjNum 172 snap 43 seq_num 172) 2024-09-21T09:33:33.532 INFO:tasks.rados.rados.0.smithi022.stdout:720: done (2 left) 2024-09-21T09:33:33.532 INFO:tasks.rados.rados.0.smithi022.stdout:722: done (1 left) 2024-09-21T09:33:33.532 INFO:tasks.rados.rados.0.smithi022.stdout:725: copy_from oid 40 from oid 7 current snap is 71 2024-09-21T09:33:33.532 INFO:tasks.rados.rados.0.smithi022.stdout:726: snap_remove snap 68 2024-09-21T09:33:33.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:33 smithi064 ceph-mon[31124]: osdmap e202: 8 total, 8 up, 8 in 2024-09-21T09:33:33.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:33 smithi064 ceph-mon[31124]: pgmap v60: 121 pgs: 121 active+clean; 182 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.5 MiB/s rd, 438 KiB/s wr, 12 op/s 2024-09-21T09:33:33.807 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:33 smithi022 ceph-mon[39525]: osdmap e202: 8 total, 8 up, 8 in 2024-09-21T09:33:33.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:33 smithi022 ceph-mon[39525]: pgmap v60: 121 pgs: 121 active+clean; 182 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.5 MiB/s rd, 438 KiB/s wr, 12 op/s 2024-09-21T09:33:33.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:33 smithi022 ceph-mon[27763]: osdmap e202: 8 total, 8 up, 8 in 2024-09-21T09:33:33.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:33 smithi022 ceph-mon[27763]: pgmap v60: 121 pgs: 121 active+clean; 182 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.5 MiB/s rd, 438 KiB/s wr, 12 op/s 2024-09-21T09:33:33.959 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:33:33 smithi064 systemd[1]: Stopping Ceph prometheus.a for d65e801e-77fa-11ef-baf6-efdc52797490... 2024-09-21T09:33:34.281 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:33:33 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[65177]: ts=2024-09-21T09:33:33.957Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-09-21T09:33:34.281 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:33:33 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[65177]: ts=2024-09-21T09:33:33.958Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-09-21T09:33:34.281 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:33:33 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[65177]: ts=2024-09-21T09:33:33.958Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-09-21T09:33:34.281 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:33:33 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[65177]: ts=2024-09-21T09:33:33.958Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-09-21T09:33:34.281 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:33:33 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[65177]: ts=2024-09-21T09:33:33.958Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-09-21T09:33:34.281 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:33:33 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[65177]: ts=2024-09-21T09:33:33.958Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-09-21T09:33:34.282 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:33:33 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[65177]: ts=2024-09-21T09:33:33.958Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-09-21T09:33:34.282 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:33:33 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[65177]: ts=2024-09-21T09:33:33.958Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-09-21T09:33:34.282 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:33:33 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[65177]: ts=2024-09-21T09:33:33.958Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-09-21T09:33:34.282 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:33:33 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[65177]: ts=2024-09-21T09:33:33.958Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-09-21T09:33:34.282 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:33:33 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[65177]: ts=2024-09-21T09:33:33.958Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-09-21T09:33:34.282 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:33:33 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[65177]: ts=2024-09-21T09:33:33.958Z caller=main.go:1147 level=info msg="See you next time!" 2024-09-21T09:33:34.282 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:33:33 smithi064 podman[75248]: 2024-09-21 09:33:33.968302791 +0000 UTC m=+0.118831541 container died caee9b54c1319453eb55a86c6515efffd6936b629fd2bfe37633a259dfd353a8 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:33:34.282 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:33:34 smithi064 podman[75248]: 2024-09-21 09:33:34.27992943 +0000 UTC m=+0.430458169 container cleanup caee9b54c1319453eb55a86c6515efffd6936b629fd2bfe37633a259dfd353a8 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:33:34.282 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:33:34 smithi064 bash[75248]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a 2024-09-21T09:33:34.518 INFO:tasks.rados.rados.0.smithi022.stdout:726: done (2 left) 2024-09-21T09:33:34.518 INFO:tasks.rados.rados.0.smithi022.stdout:727: delete oid 22 current snap is 71 2024-09-21T09:33:34.518 INFO:tasks.rados.rados.0.smithi022.stdout:724: finishing copy_from racing read to smithi02284960-46 2024-09-21T09:33:34.518 INFO:tasks.rados.rados.0.smithi022.stdout:724: finishing copy_from to smithi02284960-46 2024-09-21T09:33:34.518 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 46 v 279 (ObjNum 204 snap 56 seq_num 204) dirty exists 2024-09-21T09:33:34.518 INFO:tasks.rados.rados.0.smithi022.stdout:725: finishing copy_from racing read to smithi02284960-40 2024-09-21T09:33:34.518 INFO:tasks.rados.rados.0.smithi022.stdout:725: finishing copy_from to smithi02284960-40 2024-09-21T09:33:34.518 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 40 v 367 (ObjNum 182 snap 47 seq_num 182) dirty exists 2024-09-21T09:33:34.522 INFO:tasks.rados.rados.0.smithi022.stdout:724: done (2 left) 2024-09-21T09:33:34.522 INFO:tasks.rados.rados.0.smithi022.stdout:725: done (1 left) 2024-09-21T09:33:34.522 INFO:tasks.rados.rados.0.smithi022.stdout:727: done (0 left) 2024-09-21T09:33:34.522 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 24 current snap is 71 2024-09-21T09:33:34.522 INFO:tasks.rados.rados.0.smithi022.stdout:728: seq_num 253 ranges {1564672=647168,2211840=786432,2998272=417792} 2024-09-21T09:33:34.534 INFO:tasks.rados.rados.0.smithi022.stdout:728: writing smithi02284960-24 from 1564672 to 2211840 tid 1 2024-09-21T09:33:34.538 INFO:tasks.rados.rados.0.smithi022.stdout:728: writing smithi02284960-24 from 2211840 to 2998272 tid 2 2024-09-21T09:33:34.540 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:34 smithi064 ceph-mon[31124]: osdmap e203: 8 total, 8 up, 8 in 2024-09-21T09:33:34.541 INFO:tasks.rados.rados.0.smithi022.stdout:728: writing smithi02284960-24 from 2998272 to 3416064 tid 3 2024-09-21T09:33:34.542 INFO:tasks.rados.rados.0.smithi022.stdout:729: copy_from oid 49 from oid 22 current snap is 71 2024-09-21T09:33:34.542 INFO:tasks.rados.rados.0.smithi022.stdout:728: finishing write tid 1 to smithi02284960-24 2024-09-21T09:33:34.542 INFO:tasks.rados.rados.0.smithi022.stdout:730: delete oid 35 current snap is 71 2024-09-21T09:33:34.543 INFO:tasks.rados.rados.0.smithi022.stdout:730: done (2 left) 2024-09-21T09:33:34.543 INFO:tasks.rados.rados.0.smithi022.stdout:731: read oid 11 snap -1 2024-09-21T09:33:34.544 INFO:tasks.rados.rados.0.smithi022.stdout:731: expect (ObjNum 229 snap 64 seq_num 229) 2024-09-21T09:33:34.544 INFO:tasks.rados.rados.0.smithi022.stdout:732: snap_remove snap 42 2024-09-21T09:33:34.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:34 smithi022 ceph-mon[39525]: osdmap e203: 8 total, 8 up, 8 in 2024-09-21T09:33:34.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:34 smithi022 ceph-mon[27763]: osdmap e203: 8 total, 8 up, 8 in 2024-09-21T09:33:34.941 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:33:34 smithi064 podman[75264]: 2024-09-21 09:33:34.539876619 +0000 UTC m=+0.573776675 container remove caee9b54c1319453eb55a86c6515efffd6936b629fd2bfe37633a259dfd353a8 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:33:34.941 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:33:34 smithi064 systemd[1]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490@prometheus.a.service: Deactivated successfully. 2024-09-21T09:33:34.942 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:33:34 smithi064 systemd[1]: Stopped Ceph prometheus.a for d65e801e-77fa-11ef-baf6-efdc52797490. 2024-09-21T09:33:34.942 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:33:34 smithi064 systemd[1]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490@prometheus.a.service: Consumed 1.816s CPU time. 2024-09-21T09:33:35.284 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:33:34 smithi064 systemd[1]: Starting Ceph prometheus.a for d65e801e-77fa-11ef-baf6-efdc52797490... 2024-09-21T09:33:35.284 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:33:35 smithi064 podman[75398]: 2024-09-21 09:33:35.077352675 +0000 UTC m=+0.017175571 image pull 1d3b7f56885b6dd623f1785be963aa9c195f86bc256ea454e8d02a7980b79c53 quay.io/prometheus/prometheus:v2.51.0 2024-09-21T09:33:35.447 INFO:tasks.rados.rados.0.smithi022.stdout:732: done (3 left) 2024-09-21T09:33:35.447 INFO:tasks.rados.rados.0.smithi022.stdout:733: snap_remove snap 41 2024-09-21T09:33:35.447 INFO:tasks.rados.rados.0.smithi022.stdout:728: finishing write tid 2 to smithi02284960-24 2024-09-21T09:33:35.447 INFO:tasks.rados.rados.0.smithi022.stdout:728: finishing write tid 3 to smithi02284960-24 2024-09-21T09:33:35.447 INFO:tasks.rados.rados.0.smithi022.stdout:729: finishing copy_from to smithi02284960-49 2024-09-21T09:33:35.447 INFO:tasks.rados.rados.0.smithi022.stdout:729: got expected ENOENT (src dne) 2024-09-21T09:33:35.447 INFO:tasks.rados.rados.0.smithi022.stdout:729: finishing copy_from racing read to smithi02284960-49 2024-09-21T09:33:35.447 INFO:tasks.rados.rados.0.smithi022.stdout:728: finishing write tid 4 to smithi02284960-24 2024-09-21T09:33:35.447 INFO:tasks.rados.rados.0.smithi022.stdout:728: finishing write tid 5 to smithi02284960-24 2024-09-21T09:33:35.447 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 24 v 450 (ObjNum 253 snap 71 seq_num 253) dirty exists 2024-09-21T09:33:35.447 INFO:tasks.rados.rados.0.smithi022.stdout:728: left oid 24 (ObjNum 253 snap 71 seq_num 253) 2024-09-21T09:33:35.448 INFO:tasks.rados.rados.0.smithi022.stdout:731: expect (ObjNum 229 snap 64 seq_num 229) 2024-09-21T09:33:35.691 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:33:35 smithi064 podman[75398]: 2024-09-21 09:33:35.283484922 +0000 UTC m=+0.223307803 container create e96ed2646ce4124317b10f6f75452885c3affcab89f3bb4cddb9ebc9d46a9ec0 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:33:35.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:35 smithi064 ceph-mon[31124]: osdmap e204: 8 total, 8 up, 8 in 2024-09-21T09:33:35.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:35 smithi064 ceph-mon[31124]: pgmap v63: 121 pgs: 121 active+clean; 182 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.5 MiB/s rd, 438 KiB/s wr, 12 op/s 2024-09-21T09:33:35.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:35 smithi064 ceph-mon[31124]: osdmap e205: 8 total, 8 up, 8 in 2024-09-21T09:33:35.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:35 smithi022 ceph-mon[39525]: osdmap e204: 8 total, 8 up, 8 in 2024-09-21T09:33:35.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:35 smithi022 ceph-mon[39525]: pgmap v63: 121 pgs: 121 active+clean; 182 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.5 MiB/s rd, 438 KiB/s wr, 12 op/s 2024-09-21T09:33:35.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:35 smithi022 ceph-mon[39525]: osdmap e205: 8 total, 8 up, 8 in 2024-09-21T09:33:35.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:35 smithi022 ceph-mon[27763]: osdmap e204: 8 total, 8 up, 8 in 2024-09-21T09:33:35.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:35 smithi022 ceph-mon[27763]: pgmap v63: 121 pgs: 121 active+clean; 182 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.5 MiB/s rd, 438 KiB/s wr, 12 op/s 2024-09-21T09:33:35.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:35 smithi022 ceph-mon[27763]: osdmap e205: 8 total, 8 up, 8 in 2024-09-21T09:33:36.524 INFO:tasks.rados.rados.0.smithi022.stdout:728: done (3 left) 2024-09-21T09:33:36.524 INFO:tasks.rados.rados.0.smithi022.stdout:729: done (2 left) 2024-09-21T09:33:36.524 INFO:tasks.rados.rados.0.smithi022.stdout:731: done (1 left) 2024-09-21T09:33:36.525 INFO:tasks.rados.rados.0.smithi022.stdout:733: done (0 left) 2024-09-21T09:33:36.525 INFO:tasks.rados.rados.0.smithi022.stdout:734: setattr oid 48 current snap is 71 2024-09-21T09:33:36.529 INFO:tasks.rados.rados.0.smithi022.stdout:735: copy_from oid 14 from oid 10 current snap is 71 2024-09-21T09:33:36.529 INFO:tasks.rados.rados.0.smithi022.stdout:736: rollback oid 18 current snap is 71 2024-09-21T09:33:36.529 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 18 to 59 2024-09-21T09:33:36.529 INFO:tasks.rados.rados.0.smithi022.stdout:737: read oid 35 snap -1 2024-09-21T09:33:36.529 INFO:tasks.rados.rados.0.smithi022.stdout:737: expect deleted 2024-09-21T09:33:36.529 INFO:tasks.rados.rados.0.smithi022.stdout:738: snap_remove snap 70 2024-09-21T09:33:37.442 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:33:36 smithi064 podman[75398]: 2024-09-21 09:33:36.987628215 +0000 UTC m=+1.927451119 container init e96ed2646ce4124317b10f6f75452885c3affcab89f3bb4cddb9ebc9d46a9ec0 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:33:37.442 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:33:36 smithi064 podman[75398]: 2024-09-21 09:33:36.993334084 +0000 UTC m=+1.933156956 container start e96ed2646ce4124317b10f6f75452885c3affcab89f3bb4cddb9ebc9d46a9ec0 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:33:37.443 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:33:37 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[75413]: ts=2024-09-21T09:33:37.013Z caller=main.go:617 level=info msg="Starting Prometheus Server" mode=server version="(version=2.51.0, branch=HEAD, revision=c05c15512acb675e3f6cd662a6727854e93fc024)" 2024-09-21T09:33:37.443 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:33:37 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[75413]: ts=2024-09-21T09:33:37.013Z caller=main.go:622 level=info build_context="(go=go1.22.1, platform=linux/amd64, user=root@b5723e458358, date=20240319-10:54:45, tags=netgo,builtinassets,stringlabels)" 2024-09-21T09:33:37.443 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:33:37 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[75413]: ts=2024-09-21T09:33:37.013Z caller=main.go:623 level=info host_details="(Linux 5.14.0-508.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024 x86_64 smithi064 (none))" 2024-09-21T09:33:37.443 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:33:37 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[75413]: ts=2024-09-21T09:33:37.013Z caller=main.go:624 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-09-21T09:33:37.443 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:33:37 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[75413]: ts=2024-09-21T09:33:37.013Z caller=main.go:625 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-21T09:33:37.443 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:33:37 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[75413]: ts=2024-09-21T09:33:37.015Z caller=web.go:568 level=info component=web msg="Start listening for connections" address=:9095 2024-09-21T09:33:37.443 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:33:37 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[75413]: ts=2024-09-21T09:33:37.016Z caller=main.go:1129 level=info msg="Starting TSDB ..." 2024-09-21T09:33:37.443 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:33:37 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[75413]: ts=2024-09-21T09:33:37.017Z caller=tls_config.go:313 level=info component=web msg="Listening on" address=[::]:9095 2024-09-21T09:33:37.443 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:33:37 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[75413]: ts=2024-09-21T09:33:37.017Z caller=tls_config.go:316 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-09-21T09:33:37.444 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:33:37 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[75413]: ts=2024-09-21T09:33:37.020Z caller=head.go:616 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-09-21T09:33:37.444 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:33:37 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[75413]: ts=2024-09-21T09:33:37.020Z caller=head.go:698 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.539µs 2024-09-21T09:33:37.444 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:33:37 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[75413]: ts=2024-09-21T09:33:37.020Z caller=head.go:706 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-09-21T09:33:37.444 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:33:37 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[75413]: ts=2024-09-21T09:33:37.020Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=2 2024-09-21T09:33:37.444 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:33:37 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[75413]: ts=2024-09-21T09:33:37.034Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=2 2024-09-21T09:33:37.444 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:33:37 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[75413]: ts=2024-09-21T09:33:37.034Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=2 2024-09-21T09:33:37.444 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:33:37 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[75413]: ts=2024-09-21T09:33:37.034Z caller=head.go:815 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=44.075µs wal_replay_duration=14.41026ms wbl_replay_duration=140ns total_replay_duration=14.482146ms 2024-09-21T09:33:37.444 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:33:37 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[75413]: ts=2024-09-21T09:33:37.037Z caller=main.go:1150 level=info fs_type=XFS_SUPER_MAGIC 2024-09-21T09:33:37.444 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:33:37 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[75413]: ts=2024-09-21T09:33:37.037Z caller=main.go:1153 level=info msg="TSDB started" 2024-09-21T09:33:37.444 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:33:37 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[75413]: ts=2024-09-21T09:33:37.037Z caller=main.go:1335 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-21T09:33:37.444 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:33:37 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[75413]: ts=2024-09-21T09:33:37.055Z caller=main.go:1372 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=18.065102ms db_storage=928ns remote_storage=1.081µs web_handler=331ns query_engine=708ns scrape=464.244µs scrape_sd=105.408µs notify=9.709µs notify_sd=22.949µs rules=17.091128ms tracing=6.023µs 2024-09-21T09:33:37.444 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:33:37 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[75413]: ts=2024-09-21T09:33:37.055Z caller=main.go:1114 level=info msg="Server is ready to receive web requests." 2024-09-21T09:33:37.445 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:33:37 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[75413]: ts=2024-09-21T09:33:37.055Z caller=manager.go:163 level=info component="rule manager" msg="Starting rule manager..." 2024-09-21T09:33:37.445 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:33:37 smithi064 bash[75398]: e96ed2646ce4124317b10f6f75452885c3affcab89f3bb4cddb9ebc9d46a9ec0 2024-09-21T09:33:37.445 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:33:37 smithi064 systemd[1]: Started Ceph prometheus.a for d65e801e-77fa-11ef-baf6-efdc52797490. 2024-09-21T09:33:37.530 INFO:tasks.rados.rados.0.smithi022.stdout:738: done (4 left) 2024-09-21T09:33:37.530 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 19 current snap is 71 2024-09-21T09:33:37.530 INFO:tasks.rados.rados.0.smithi022.stdout:739: seq_num 255 ranges {3047424=679936,3727360=548864,4276224=598016,4874240=57344} 2024-09-21T09:33:37.550 INFO:tasks.rados.rados.0.smithi022.stdout:739: writing smithi02284960-19 from 3047424 to 3727360 tid 1 2024-09-21T09:33:37.553 INFO:tasks.rados.rados.0.smithi022.stdout:739: writing smithi02284960-19 from 3727360 to 4276224 tid 2 2024-09-21T09:33:37.557 INFO:tasks.rados.rados.0.smithi022.stdout:739: writing smithi02284960-19 from 4276224 to 4874240 tid 3 2024-09-21T09:33:37.557 INFO:tasks.rados.rados.0.smithi022.stdout:739: writing smithi02284960-19 from 4874240 to 4931584 tid 4 2024-09-21T09:33:37.558 INFO:tasks.rados.rados.0.smithi022.stdout:740: read oid 40 snap -1 2024-09-21T09:33:37.558 INFO:tasks.rados.rados.0.smithi022.stdout:740: expect (ObjNum 182 snap 47 seq_num 182) 2024-09-21T09:33:37.558 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 50 current snap is 71 2024-09-21T09:33:37.558 INFO:tasks.rados.rados.0.smithi022.stdout:741: seq_num 256 ranges {0=696320,696320=434176,1130496=770048} 2024-09-21T09:33:37.563 INFO:tasks.rados.rados.0.smithi022.stdout:741: writing smithi02284960-50 from 0 to 696320 tid 1 2024-09-21T09:33:37.565 INFO:tasks.rados.rados.0.smithi022.stdout:741: writing smithi02284960-50 from 696320 to 1130496 tid 2 2024-09-21T09:33:37.571 INFO:tasks.rados.rados.0.smithi022.stdout:741: writing smithi02284960-50 from 1130496 to 1900544 tid 3 2024-09-21T09:33:37.571 INFO:tasks.rados.rados.0.smithi022.stdout:742: read oid 28 snap -1 2024-09-21T09:33:37.571 INFO:tasks.rados.rados.0.smithi022.stdout:742: expect (ObjNum 162 snap 40 seq_num 162) 2024-09-21T09:33:37.571 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 34 current snap is 71 2024-09-21T09:33:37.571 INFO:tasks.rados.rados.0.smithi022.stdout:743: seq_num 257 ranges {606208=712704,1318912=720896,2039808=483328} 2024-09-21T09:33:37.578 INFO:tasks.rados.rados.0.smithi022.stdout:743: writing smithi02284960-34 from 606208 to 1318912 tid 1 2024-09-21T09:33:37.583 INFO:tasks.rados.rados.0.smithi022.stdout:743: writing smithi02284960-34 from 1318912 to 2039808 tid 2 2024-09-21T09:33:37.587 INFO:tasks.rados.rados.0.smithi022.stdout:743: writing smithi02284960-34 from 2039808 to 2523136 tid 3 2024-09-21T09:33:37.587 INFO:tasks.rados.rados.0.smithi022.stdout:744: read oid 45 snap 54 2024-09-21T09:33:37.587 INFO:tasks.rados.rados.0.smithi022.stdout:744: expect (ObjNum 115 snap 23 seq_num 115) 2024-09-21T09:33:37.587 INFO:tasks.rados.rados.0.smithi022.stdout:745: read oid 21 snap -1 2024-09-21T09:33:37.587 INFO:tasks.rados.rados.0.smithi022.stdout:745: expect (ObjNum 172 snap 43 seq_num 172) 2024-09-21T09:33:37.588 INFO:tasks.rados.rados.0.smithi022.stdout:746: delete oid 20 current snap is 71 2024-09-21T09:33:37.588 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 48 v 418 (ObjNum 289 snap 0 seq_num 1) dirty exists 2024-09-21T09:33:37.588 INFO:tasks.rados.rados.0.smithi022.stdout:736: finishing rollback tid 0 to smithi02284960-18 2024-09-21T09:33:37.588 INFO:tasks.rados.rados.0.smithi022.stdout:736: finishing rollback tid 1 to smithi02284960-18 2024-09-21T09:33:37.588 INFO:tasks.rados.rados.0.smithi022.stdout:736: finishing rollback tid 2 to smithi02284960-18 2024-09-21T09:33:37.588 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 18 v 258 (ObjNum 210 snap 58 seq_num 210) dirty exists 2024-09-21T09:33:37.588 INFO:tasks.rados.rados.0.smithi022.stdout:735: finishing copy_from racing read to smithi02284960-14 2024-09-21T09:33:37.588 INFO:tasks.rados.rados.0.smithi022.stdout:735: finishing copy_from to smithi02284960-14 2024-09-21T09:33:37.588 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 14 v 241 (ObjNum 144 snap 32 seq_num 144) dirty exists 2024-09-21T09:33:37.588 INFO:tasks.rados.rados.0.smithi022.stdout:739: finishing write tid 1 to smithi02284960-19 2024-09-21T09:33:37.588 INFO:tasks.rados.rados.0.smithi022.stdout:739: finishing write tid 2 to smithi02284960-19 2024-09-21T09:33:37.588 INFO:tasks.rados.rados.0.smithi022.stdout:739: finishing write tid 3 to smithi02284960-19 2024-09-21T09:33:37.588 INFO:tasks.rados.rados.0.smithi022.stdout:739: finishing write tid 4 to smithi02284960-19 2024-09-21T09:33:37.588 INFO:tasks.rados.rados.0.smithi022.stdout:739: finishing write tid 5 to smithi02284960-19 2024-09-21T09:33:37.589 INFO:tasks.rados.rados.0.smithi022.stdout:739: finishing write tid 6 to smithi02284960-19 2024-09-21T09:33:37.589 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 19 v 478 (ObjNum 255 snap 71 seq_num 255) dirty exists 2024-09-21T09:33:37.589 INFO:tasks.rados.rados.0.smithi022.stdout:739: left oid 19 (ObjNum 255 snap 71 seq_num 255) 2024-09-21T09:33:37.589 INFO:tasks.rados.rados.0.smithi022.stdout:741: finishing write tid 1 to smithi02284960-50 2024-09-21T09:33:37.589 INFO:tasks.rados.rados.0.smithi022.stdout:741: finishing write tid 2 to smithi02284960-50 2024-09-21T09:33:37.589 INFO:tasks.rados.rados.0.smithi022.stdout:740: expect (ObjNum 182 snap 47 seq_num 182) 2024-09-21T09:33:37.664 INFO:tasks.rados.rados.0.smithi022.stdout:741: finishing write tid 3 to smithi02284960-50 2024-09-21T09:33:37.664 INFO:tasks.rados.rados.0.smithi022.stdout:741: finishing write tid 4 to smithi02284960-50 2024-09-21T09:33:37.664 INFO:tasks.rados.rados.0.smithi022.stdout:742: expect (ObjNum 162 snap 40 seq_num 162) 2024-09-21T09:33:37.683 INFO:tasks.rados.rados.0.smithi022.stdout:741: finishing write tid 5 to smithi02284960-50 2024-09-21T09:33:37.683 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 50 v 270 (ObjNum 256 snap 71 seq_num 256) dirty exists 2024-09-21T09:33:37.683 INFO:tasks.rados.rados.0.smithi022.stdout:741: left oid 50 (ObjNum 256 snap 71 seq_num 256) 2024-09-21T09:33:37.683 INFO:tasks.rados.rados.0.smithi022.stdout:743: finishing write tid 1 to smithi02284960-34 2024-09-21T09:33:37.683 INFO:tasks.rados.rados.0.smithi022.stdout:734: done (11 left) 2024-09-21T09:33:37.683 INFO:tasks.rados.rados.0.smithi022.stdout:735: done (10 left) 2024-09-21T09:33:37.683 INFO:tasks.rados.rados.0.smithi022.stdout:736: done (9 left) 2024-09-21T09:33:37.683 INFO:tasks.rados.rados.0.smithi022.stdout:737: done (8 left) 2024-09-21T09:33:37.683 INFO:tasks.rados.rados.0.smithi022.stdout:739: done (7 left) 2024-09-21T09:33:37.684 INFO:tasks.rados.rados.0.smithi022.stdout:740: done (6 left) 2024-09-21T09:33:37.684 INFO:tasks.rados.rados.0.smithi022.stdout:741: done (5 left) 2024-09-21T09:33:37.684 INFO:tasks.rados.rados.0.smithi022.stdout:742: done (4 left) 2024-09-21T09:33:37.684 INFO:tasks.rados.rados.0.smithi022.stdout:746: done (3 left) 2024-09-21T09:33:37.684 INFO:tasks.rados.rados.0.smithi022.stdout:747: read oid 37 snap -1 2024-09-21T09:33:37.684 INFO:tasks.rados.rados.0.smithi022.stdout:747: expect (ObjNum 218 snap 63 seq_num 218) 2024-09-21T09:33:37.684 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 13 current snap is 71 2024-09-21T09:33:37.684 INFO:tasks.rados.rados.0.smithi022.stdout:743: finishing write tid 2 to smithi02284960-34 2024-09-21T09:33:37.684 INFO:tasks.rados.rados.0.smithi022.stdout:748: seq_num 259 ranges {0=745472,745472=540672,1286144=581632,1867776=81920} 2024-09-21T09:33:37.686 INFO:tasks.rados.rados.0.smithi022.stdout:748: writing smithi02284960-13 from 0 to 745472 tid 1 2024-09-21T09:33:37.689 INFO:tasks.rados.rados.0.smithi022.stdout:748: writing smithi02284960-13 from 745472 to 1286144 tid 2 2024-09-21T09:33:37.692 INFO:tasks.rados.rados.0.smithi022.stdout:748: writing smithi02284960-13 from 1286144 to 1867776 tid 3 2024-09-21T09:33:37.693 INFO:tasks.rados.rados.0.smithi022.stdout:748: writing smithi02284960-13 from 1867776 to 1949696 tid 4 2024-09-21T09:33:37.693 INFO:tasks.rados.rados.0.smithi022.stdout:749: delete oid 5 current snap is 71 2024-09-21T09:33:37.693 INFO:tasks.rados.rados.0.smithi022.stdout:743: finishing write tid 3 to smithi02284960-34 2024-09-21T09:33:37.693 INFO:tasks.rados.rados.0.smithi022.stdout:743: finishing write tid 4 to smithi02284960-34 2024-09-21T09:33:37.693 INFO:tasks.rados.rados.0.smithi022.stdout:745: expect (ObjNum 172 snap 43 seq_num 172) 2024-09-21T09:33:37.707 INFO:tasks.rados.rados.0.smithi022.stdout:743: finishing write tid 5 to smithi02284960-34 2024-09-21T09:33:37.707 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 34 v 207 (ObjNum 257 snap 71 seq_num 257) dirty exists 2024-09-21T09:33:37.707 INFO:tasks.rados.rados.0.smithi022.stdout:743: left oid 34 (ObjNum 257 snap 71 seq_num 257) 2024-09-21T09:33:37.707 INFO:tasks.rados.rados.0.smithi022.stdout:744: expect (ObjNum 115 snap 23 seq_num 115) 2024-09-21T09:33:37.757 INFO:tasks.rados.rados.0.smithi022.stdout:747: expect (ObjNum 218 snap 63 seq_num 218) 2024-09-21T09:33:37.789 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:37 smithi064 ceph-mon[31124]: osdmap e206: 8 total, 8 up, 8 in 2024-09-21T09:33:37.789 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:37 smithi064 ceph-mon[31124]: pgmap v66: 121 pgs: 121 active+clean; 187 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.2 MiB/s rd, 1.3 MiB/s wr, 9 op/s 2024-09-21T09:33:37.789 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:37 smithi064 ceph-mon[31124]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:37.790 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:37 smithi064 ceph-mon[31124]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:37.793 INFO:tasks.rados.rados.0.smithi022.stdout:748: finishing write tid 1 to smithi02284960-13 2024-09-21T09:33:37.793 INFO:tasks.rados.rados.0.smithi022.stdout:748: finishing write tid 2 to smithi02284960-13 2024-09-21T09:33:37.793 INFO:tasks.rados.rados.0.smithi022.stdout:748: finishing write tid 3 to smithi02284960-13 2024-09-21T09:33:37.793 INFO:tasks.rados.rados.0.smithi022.stdout:748: finishing write tid 4 to smithi02284960-13 2024-09-21T09:33:37.794 INFO:tasks.rados.rados.0.smithi022.stdout:748: finishing write tid 5 to smithi02284960-13 2024-09-21T09:33:37.794 INFO:tasks.rados.rados.0.smithi022.stdout:748: finishing write tid 6 to smithi02284960-13 2024-09-21T09:33:37.794 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 13 v 347 (ObjNum 259 snap 71 seq_num 259) dirty exists 2024-09-21T09:33:37.794 INFO:tasks.rados.rados.0.smithi022.stdout:748: left oid 13 (ObjNum 259 snap 71 seq_num 259) 2024-09-21T09:33:37.794 INFO:tasks.rados.rados.0.smithi022.stdout:743: done (5 left) 2024-09-21T09:33:37.794 INFO:tasks.rados.rados.0.smithi022.stdout:744: done (4 left) 2024-09-21T09:33:37.794 INFO:tasks.rados.rados.0.smithi022.stdout:745: done (3 left) 2024-09-21T09:33:37.794 INFO:tasks.rados.rados.0.smithi022.stdout:747: done (2 left) 2024-09-21T09:33:37.794 INFO:tasks.rados.rados.0.smithi022.stdout:748: done (1 left) 2024-09-21T09:33:37.794 INFO:tasks.rados.rados.0.smithi022.stdout:749: done (0 left) 2024-09-21T09:33:37.794 INFO:tasks.rados.rados.0.smithi022.stdout:750: snap_create 2024-09-21T09:33:37.807 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:37 smithi022 ceph-mon[39525]: osdmap e206: 8 total, 8 up, 8 in 2024-09-21T09:33:37.807 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:37 smithi022 ceph-mon[39525]: pgmap v66: 121 pgs: 121 active+clean; 187 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.2 MiB/s rd, 1.3 MiB/s wr, 9 op/s 2024-09-21T09:33:37.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:37 smithi022 ceph-mon[39525]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:37.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:37 smithi022 ceph-mon[39525]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:37.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:37 smithi022 ceph-mon[27763]: osdmap e206: 8 total, 8 up, 8 in 2024-09-21T09:33:37.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:37 smithi022 ceph-mon[27763]: pgmap v66: 121 pgs: 121 active+clean; 187 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.2 MiB/s rd, 1.3 MiB/s wr, 9 op/s 2024-09-21T09:33:37.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:37 smithi022 ceph-mon[27763]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:37.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:37 smithi022 ceph-mon[27763]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:38.531 INFO:tasks.rados.rados.0.smithi022.stdout:750: done (0 left) 2024-09-21T09:33:38.531 INFO:tasks.rados.rados.0.smithi022.stdout:751: rmattr oid 13 current snap is 72 2024-09-21T09:33:38.531 INFO:tasks.rados.rados.0.smithi022.stdout:751: done (0 left) 2024-09-21T09:33:38.531 INFO:tasks.rados.rados.0.smithi022.stdout:752: snap_create 2024-09-21T09:33:38.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:38 smithi022 ceph-mon[39525]: Reconfiguring grafana.a (dependencies changed)... 2024-09-21T09:33:38.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:38 smithi022 ceph-mon[39525]: Reconfiguring daemon grafana.a on smithi064 2024-09-21T09:33:38.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:38 smithi022 ceph-mon[39525]: osdmap e207: 8 total, 8 up, 8 in 2024-09-21T09:33:38.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:38 smithi022 ceph-mon[27763]: Reconfiguring grafana.a (dependencies changed)... 2024-09-21T09:33:38.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:38 smithi022 ceph-mon[27763]: Reconfiguring daemon grafana.a on smithi064 2024-09-21T09:33:38.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:38 smithi022 ceph-mon[27763]: osdmap e207: 8 total, 8 up, 8 in 2024-09-21T09:33:38.811 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:38 smithi064 ceph-mon[31124]: Reconfiguring grafana.a (dependencies changed)... 2024-09-21T09:33:38.811 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:38 smithi064 ceph-mon[31124]: Reconfiguring daemon grafana.a on smithi064 2024-09-21T09:33:38.811 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:38 smithi064 ceph-mon[31124]: osdmap e207: 8 total, 8 up, 8 in 2024-09-21T09:33:39.542 INFO:tasks.rados.rados.0.smithi022.stdout:752: done (0 left) 2024-09-21T09:33:39.542 INFO:tasks.rados.rados.0.smithi022.stdout:753: copy_from oid 43 from oid 39 current snap is 73 2024-09-21T09:33:39.542 INFO:tasks.rados.rados.0.smithi022.stdout:754: delete oid 49 current snap is 73 2024-09-21T09:33:39.547 INFO:tasks.rados.rados.0.smithi022.stdout:754: done (1 left) 2024-09-21T09:33:39.547 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 23 current snap is 73 2024-09-21T09:33:39.548 INFO:tasks.rados.rados.0.smithi022.stdout:755: seq_num 262 ranges {1818624=794624,2613248=589824,3203072=614400} 2024-09-21T09:33:39.562 INFO:tasks.rados.rados.0.smithi022.stdout:755: writing smithi02284960-23 from 1818624 to 2613248 tid 1 2024-09-21T09:33:39.565 INFO:tasks.rados.rados.0.smithi022.stdout:755: writing smithi02284960-23 from 2613248 to 3203072 tid 2 2024-09-21T09:33:39.570 INFO:tasks.rados.rados.0.smithi022.stdout:755: writing smithi02284960-23 from 3203072 to 3817472 tid 3 2024-09-21T09:33:39.570 INFO:tasks.rados.rados.0.smithi022.stdout:756: rollback oid 3 current snap is 73 2024-09-21T09:33:39.570 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 3 to 53 2024-09-21T09:33:39.570 INFO:tasks.rados.rados.0.smithi022.stdout:753: finishing copy_from racing read to smithi02284960-43 2024-09-21T09:33:39.570 INFO:tasks.rados.rados.0.smithi022.stdout:753: finishing copy_from to smithi02284960-43 2024-09-21T09:33:39.570 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 43 v 286 (ObjNum 240 snap 65 seq_num 240) dirty exists 2024-09-21T09:33:39.570 INFO:tasks.rados.rados.0.smithi022.stdout:755: finishing write tid 1 to smithi02284960-23 2024-09-21T09:33:39.570 INFO:tasks.rados.rados.0.smithi022.stdout:755: finishing write tid 2 to smithi02284960-23 2024-09-21T09:33:39.570 INFO:tasks.rados.rados.0.smithi022.stdout:753: done (2 left) 2024-09-21T09:33:39.571 INFO:tasks.rados.rados.0.smithi022.stdout:757: snap_create 2024-09-21T09:33:39.572 INFO:tasks.rados.rados.0.smithi022.stdout:755: finishing write tid 3 to smithi02284960-23 2024-09-21T09:33:39.572 INFO:tasks.rados.rados.0.smithi022.stdout:755: finishing write tid 4 to smithi02284960-23 2024-09-21T09:33:39.573 INFO:tasks.rados.rados.0.smithi022.stdout:756: finishing rollback tid 0 to smithi02284960-3 2024-09-21T09:33:39.573 INFO:tasks.rados.rados.0.smithi022.stdout:756: finishing rollback tid 1 to smithi02284960-3 2024-09-21T09:33:39.573 INFO:tasks.rados.rados.0.smithi022.stdout:756: finishing rollback tid 2 to smithi02284960-3 2024-09-21T09:33:39.573 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 3 v 93 (ObjNum 195 snap 50 seq_num 195) dirty exists 2024-09-21T09:33:39.573 INFO:tasks.rados.rados.0.smithi022.stdout:755: finishing write tid 5 to smithi02284960-23 2024-09-21T09:33:39.574 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 23 v 187 (ObjNum 262 snap 73 seq_num 262) dirty exists 2024-09-21T09:33:39.574 INFO:tasks.rados.rados.0.smithi022.stdout:755: left oid 23 (ObjNum 262 snap 73 seq_num 262) 2024-09-21T09:33:39.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:39 smithi022 ceph-mon[27763]: osdmap e208: 8 total, 8 up, 8 in 2024-09-21T09:33:39.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:39 smithi022 ceph-mon[27763]: pgmap v69: 121 pgs: 121 active+clean; 187 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.2 MiB/s rd, 1.3 MiB/s wr, 9 op/s 2024-09-21T09:33:39.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:39 smithi022 ceph-mon[39525]: osdmap e208: 8 total, 8 up, 8 in 2024-09-21T09:33:39.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:39 smithi022 ceph-mon[39525]: pgmap v69: 121 pgs: 121 active+clean; 187 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.2 MiB/s rd, 1.3 MiB/s wr, 9 op/s 2024-09-21T09:33:39.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:39 smithi064 ceph-mon[31124]: osdmap e208: 8 total, 8 up, 8 in 2024-09-21T09:33:39.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:39 smithi064 ceph-mon[31124]: pgmap v69: 121 pgs: 121 active+clean; 187 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.2 MiB/s rd, 1.3 MiB/s wr, 9 op/s 2024-09-21T09:33:40.270 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:33:40 smithi064 systemd[1]: Stopping Ceph grafana.a for d65e801e-77fa-11ef-baf6-efdc52797490... 2024-09-21T09:33:40.448 INFO:tasks.rados.rados.0.smithi022.stdout:755: done (2 left) 2024-09-21T09:33:40.448 INFO:tasks.rados.rados.0.smithi022.stdout:756: done (1 left) 2024-09-21T09:33:40.448 INFO:tasks.rados.rados.0.smithi022.stdout:757: done (0 left) 2024-09-21T09:33:40.448 INFO:tasks.rados.rados.0.smithi022.stdout:758: read oid 42 snap -1 2024-09-21T09:33:40.448 INFO:tasks.rados.rados.0.smithi022.stdout:758: expect deleted 2024-09-21T09:33:40.448 INFO:tasks.rados.rados.0.smithi022.stdout:759: read oid 24 snap -1 2024-09-21T09:33:40.448 INFO:tasks.rados.rados.0.smithi022.stdout:759: expect (ObjNum 253 snap 71 seq_num 253) 2024-09-21T09:33:40.448 INFO:tasks.rados.rados.0.smithi022.stdout:760: snap_create 2024-09-21T09:33:40.465 INFO:tasks.rados.rados.0.smithi022.stdout:759: expect (ObjNum 253 snap 71 seq_num 253) 2024-09-21T09:33:40.538 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:33:40 smithi064 podman[75671]: 2024-09-21 09:33:40.269176758 +0000 UTC m=+0.106645329 container died ffde1f806d0b42f9721d265e4c699a0d9778f0f82cb7ea98bcee27db7f6ec187 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a, build-date=2022-04-29T11:48:11.055897, name=ubi8, summary=Grafana Container configured for Ceph mgr/dashboard integration, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, version=8.5, io.openshift.tags=base rhel8, release=239.1651231664, com.redhat.component=ubi8-container, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, distribution-scope=public, vcs-type=git, vendor=Red Hat, Inc., com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, description=Ceph Grafana Container, io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.expose-services=, io.buildah.version=1.32.0, maintainer=Paul Cuzner , architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-09-21T09:33:40.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:40 smithi022 ceph-mon[27763]: osdmap e209: 8 total, 8 up, 8 in 2024-09-21T09:33:40.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:40 smithi022 ceph-mon[27763]: osdmap e210: 8 total, 8 up, 8 in 2024-09-21T09:33:40.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:40 smithi022 ceph-mon[39525]: osdmap e209: 8 total, 8 up, 8 in 2024-09-21T09:33:40.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:40 smithi022 ceph-mon[39525]: osdmap e210: 8 total, 8 up, 8 in 2024-09-21T09:33:40.901 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:40 smithi064 ceph-mon[31124]: osdmap e209: 8 total, 8 up, 8 in 2024-09-21T09:33:40.901 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:40 smithi064 ceph-mon[31124]: osdmap e210: 8 total, 8 up, 8 in 2024-09-21T09:33:40.901 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:33:40 smithi064 podman[75671]: 2024-09-21 09:33:40.657063139 +0000 UTC m=+0.494531706 container cleanup ffde1f806d0b42f9721d265e4c699a0d9778f0f82cb7ea98bcee27db7f6ec187 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a, com.redhat.component=ubi8-container, io.k8s.display-name=Red Hat Universal Base Image 8, architecture=x86_64, io.openshift.expose-services=, io.buildah.version=1.32.0, version=8.5, distribution-scope=public, io.openshift.tags=base rhel8, maintainer=Paul Cuzner , build-date=2022-04-29T11:48:11.055897, summary=Grafana Container configured for Ceph mgr/dashboard integration, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=239.1651231664, vcs-type=git, vendor=Red Hat, Inc., com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, description=Ceph Grafana Container, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, name=ubi8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56) 2024-09-21T09:33:40.901 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:33:40 smithi064 bash[75671]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a 2024-09-21T09:33:41.191 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:33:40 smithi064 podman[75688]: 2024-09-21 09:33:40.900256206 +0000 UTC m=+0.633006522 container remove ffde1f806d0b42f9721d265e4c699a0d9778f0f82cb7ea98bcee27db7f6ec187 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.display-name=Red Hat Universal Base Image 8, distribution-scope=public, io.openshift.expose-services=, io.buildah.version=1.32.0, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=ubi8, io.openshift.tags=base rhel8, release=239.1651231664, vendor=Red Hat, Inc., version=8.5, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=ubi8-container, description=Ceph Grafana Container, maintainer=Paul Cuzner , build-date=2022-04-29T11:48:11.055897, vcs-type=git, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664) 2024-09-21T09:33:41.191 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:33:40 smithi064 systemd[1]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490@grafana.a.service: Main process exited, code=exited, status=143/n/a 2024-09-21T09:33:41.191 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:33:41 smithi064 systemd[1]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490@grafana.a.service: Failed with result 'exit-code'. 2024-09-21T09:33:41.191 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:33:41 smithi064 systemd[1]: Stopped Ceph grafana.a for d65e801e-77fa-11ef-baf6-efdc52797490. 2024-09-21T09:33:41.192 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:33:41 smithi064 systemd[1]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490@grafana.a.service: Consumed 5.616s CPU time. 2024-09-21T09:33:41.192 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:33:41 smithi064 systemd[1]: Starting Ceph grafana.a for d65e801e-77fa-11ef-baf6-efdc52797490... 2024-09-21T09:33:41.449 INFO:tasks.rados.rados.0.smithi022.stdout:758: done (2 left) 2024-09-21T09:33:41.449 INFO:tasks.rados.rados.0.smithi022.stdout:759: done (1 left) 2024-09-21T09:33:41.449 INFO:tasks.rados.rados.0.smithi022.stdout:760: done (0 left) 2024-09-21T09:33:41.449 INFO:tasks.rados.rados.0.smithi022.stdout:761: delete oid 10 current snap is 75 2024-09-21T09:33:41.452 INFO:tasks.rados.rados.0.smithi022.stdout:761: done (0 left) 2024-09-21T09:33:41.452 INFO:tasks.rados.rados.0.smithi022.stdout:762: snap_create 2024-09-21T09:33:41.543 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:33:41 smithi064 podman[75786]: 2024-09-21 09:33:41.155610028 +0000 UTC m=+0.015406813 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-09-21T09:33:41.543 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:33:41 smithi064 podman[75786]: 2024-09-21 09:33:41.268187428 +0000 UTC m=+0.127984205 container create 527bb657d70d6d420e657297c4d3f77a9c60385cf52e4028a1c7dc70b92aecf7 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.openshift.expose-services=, architecture=x86_64, io.openshift.tags=base rhel8, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.component=ubi8-container, description=Ceph Grafana Container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=ubi8, version=8.5, io.k8s.display-name=Red Hat Universal Base Image 8, distribution-scope=public, io.buildah.version=1.32.0, build-date=2022-04-29T11:48:11.055897, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, maintainer=Paul Cuzner , com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vendor=Red Hat, Inc., release=239.1651231664) 2024-09-21T09:33:41.543 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:33:41 smithi064 podman[75786]: 2024-09-21 09:33:41.487488723 +0000 UTC m=+0.347285494 container init 527bb657d70d6d420e657297c4d3f77a9c60385cf52e4028a1c7dc70b92aecf7 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a, architecture=x86_64, maintainer=Paul Cuzner , vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, distribution-scope=public, io.openshift.expose-services=, io.k8s.display-name=Red Hat Universal Base Image 8, vcs-type=git, release=239.1651231664, io.openshift.tags=base rhel8, io.buildah.version=1.32.0, com.redhat.component=ubi8-container, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=ubi8, version=8.5, build-date=2022-04-29T11:48:11.055897, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, description=Ceph Grafana Container) 2024-09-21T09:33:41.543 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:33:41 smithi064 podman[75786]: 2024-09-21 09:33:41.492363698 +0000 UTC m=+0.352160469 container start 527bb657d70d6d420e657297c4d3f77a9c60385cf52e4028a1c7dc70b92aecf7 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a, build-date=2022-04-29T11:48:11.055897, vendor=Red Hat, Inc., distribution-scope=public, vcs-type=git, io.buildah.version=1.32.0, io.openshift.expose-services=, version=8.5, com.redhat.component=ubi8-container, release=239.1651231664, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, description=Ceph Grafana Container, name=ubi8, io.k8s.display-name=Red Hat Universal Base Image 8, maintainer=Paul Cuzner , com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base rhel8, architecture=x86_64, summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI) 2024-09-21T09:33:41.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:41 smithi022 ceph-mon[39525]: pgmap v72: 121 pgs: 121 active+clean; 187 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:33:41.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:41 smithi022 ceph-mon[39525]: osdmap e211: 8 total, 8 up, 8 in 2024-09-21T09:33:41.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:41 smithi022 ceph-mon[39525]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:33:41.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:41 smithi022 ceph-mon[27763]: pgmap v72: 121 pgs: 121 active+clean; 187 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:33:41.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:41 smithi022 ceph-mon[27763]: osdmap e211: 8 total, 8 up, 8 in 2024-09-21T09:33:41.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:41 smithi022 ceph-mon[27763]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:33:41.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:41 smithi064 ceph-mon[31124]: pgmap v72: 121 pgs: 121 active+clean; 187 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:33:41.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:41 smithi064 ceph-mon[31124]: osdmap e211: 8 total, 8 up, 8 in 2024-09-21T09:33:41.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:41 smithi064 ceph-mon[31124]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:33:41.942 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:33:41 smithi064 bash[75786]: 527bb657d70d6d420e657297c4d3f77a9c60385cf52e4028a1c7dc70b92aecf7 2024-09-21T09:33:41.942 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:33:41 smithi064 systemd[1]: Started Ceph grafana.a for d65e801e-77fa-11ef-baf6-efdc52797490. 2024-09-21T09:33:41.942 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:33:41 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[75801]: logger=settings t=2024-09-21T09:33:41.654893434Z level=info msg="Starting Grafana" version=9.4.7 commit=4add91f03d branch=HEAD compiled=2023-03-16T23:56:52Z 2024-09-21T09:33:41.942 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:33:41 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[75801]: logger=settings t=2024-09-21T09:33:41.655077736Z level=warn msg="\"sentry\" frontend logging provider is deprecated and will be removed in the next major version. Use \"grafana\" provider instead." 2024-09-21T09:33:41.942 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:33:41 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[75801]: logger=settings t=2024-09-21T09:33:41.655104884Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2024-09-21T09:33:41.942 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:33:41 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[75801]: logger=settings t=2024-09-21T09:33:41.655119956Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2024-09-21T09:33:41.942 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:33:41 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[75801]: logger=settings t=2024-09-21T09:33:41.655131805Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2024-09-21T09:33:41.942 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:33:41 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[75801]: logger=settings t=2024-09-21T09:33:41.655141037Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2024-09-21T09:33:41.942 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:33:41 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[75801]: logger=settings t=2024-09-21T09:33:41.655147458Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-09-21T09:33:41.942 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:33:41 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[75801]: logger=settings t=2024-09-21T09:33:41.655154089Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-09-21T09:33:41.943 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:33:41 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[75801]: logger=settings t=2024-09-21T09:33:41.655161834Z level=info msg="Path Home" path=/usr/share/grafana 2024-09-21T09:33:41.943 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:33:41 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[75801]: logger=settings t=2024-09-21T09:33:41.655168555Z level=info msg="Path Data" path=/var/lib/grafana 2024-09-21T09:33:41.943 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:33:41 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[75801]: logger=settings t=2024-09-21T09:33:41.655174707Z level=info msg="Path Logs" path=/var/log/grafana 2024-09-21T09:33:41.943 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:33:41 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[75801]: logger=settings t=2024-09-21T09:33:41.655181784Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2024-09-21T09:33:41.943 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:33:41 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[75801]: logger=settings t=2024-09-21T09:33:41.655188724Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2024-09-21T09:33:41.943 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:33:41 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[75801]: logger=settings t=2024-09-21T09:33:41.655199046Z level=info msg="App mode production" 2024-09-21T09:33:41.943 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:33:41 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[75801]: logger=sqlstore t=2024-09-21T09:33:41.655259928Z level=info msg="Connecting to DB" dbtype=sqlite3 2024-09-21T09:33:41.943 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:33:41 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[75801]: logger=sqlstore t=2024-09-21T09:33:41.655278435Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-09-21T09:33:41.943 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:33:41 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[75801]: logger=migrator t=2024-09-21T09:33:41.669803681Z level=info msg="Starting DB migrations" 2024-09-21T09:33:41.943 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:33:41 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[75801]: logger=migrator t=2024-09-21T09:33:41.674883809Z level=info msg="migrations completed" performed=0 skipped=481 duration=538.608µs 2024-09-21T09:33:41.943 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:33:41 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[75801]: logger=sqlstore t=2024-09-21T09:33:41.675269085Z level=info msg="Created default organization" 2024-09-21T09:33:41.944 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:33:41 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[75801]: logger=plugin.loader t=2024-09-21T09:33:41.734949781Z level=info msg="Plugin registered" pluginID=input 2024-09-21T09:33:41.944 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:33:41 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[75801]: logger=plugin.loader t=2024-09-21T09:33:41.764316301Z level=info msg="Plugin registered" pluginID=grafana-piechart-panel 2024-09-21T09:33:41.944 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:33:41 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[75801]: logger=plugin.loader t=2024-09-21T09:33:41.764349541Z level=info msg="Plugin registered" pluginID=vonage-status-panel 2024-09-21T09:33:41.944 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:33:41 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[75801]: logger=secrets t=2024-09-21T09:33:41.764385401Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2024-09-21T09:33:41.944 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:33:41 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[75801]: logger=query_data t=2024-09-21T09:33:41.767652368Z level=info msg="Query Service initialization" 2024-09-21T09:33:41.944 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:33:41 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[75801]: logger=live.push_http t=2024-09-21T09:33:41.770222244Z level=info msg="Live Push Gateway initialization" 2024-09-21T09:33:42.436 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:33:42 smithi064 systemd[1]: Stopping Ceph node-exporter.b for d65e801e-77fa-11ef-baf6-efdc52797490... 2024-09-21T09:33:42.451 INFO:tasks.rados.rados.0.smithi022.stdout:762: done (0 left) 2024-09-21T09:33:42.451 INFO:tasks.rados.rados.0.smithi022.stdout:763: read oid 17 snap -1 2024-09-21T09:33:42.451 INFO:tasks.rados.rados.0.smithi022.stdout:763: expect deleted 2024-09-21T09:33:42.451 INFO:tasks.rados.rados.0.smithi022.stdout:764: rollback oid 6 current snap is 76 2024-09-21T09:33:42.451 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 6 to 67 2024-09-21T09:33:42.451 INFO:tasks.rados.rados.0.smithi022.stdout:765: read oid 27 snap 75 2024-09-21T09:33:42.451 INFO:tasks.rados.rados.0.smithi022.stdout:765: expect deleted 2024-09-21T09:33:42.451 INFO:tasks.rados.rados.0.smithi022.stdout:766: copy_from oid 18 from oid 22 current snap is 76 2024-09-21T09:33:42.451 INFO:tasks.rados.rados.0.smithi022.stdout:767: rmattr oid 14 current snap is 76 2024-09-21T09:33:42.451 INFO:tasks.rados.rados.0.smithi022.stdout:767: done (4 left) 2024-09-21T09:33:42.451 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 11 current snap is 76 2024-09-21T09:33:42.452 INFO:tasks.rados.rados.0.smithi022.stdout:768: seq_num 264 ranges {4014080=425984,4440064=761856,5201920=770048,5971968=73728} 2024-09-21T09:33:42.475 INFO:tasks.rados.rados.0.smithi022.stdout:768: writing smithi02284960-11 from 4014080 to 4440064 tid 1 2024-09-21T09:33:42.479 INFO:tasks.rados.rados.0.smithi022.stdout:768: writing smithi02284960-11 from 4440064 to 5201920 tid 2 2024-09-21T09:33:42.483 INFO:tasks.rados.rados.0.smithi022.stdout:768: writing smithi02284960-11 from 5201920 to 5971968 tid 3 2024-09-21T09:33:42.484 INFO:tasks.rados.rados.0.smithi022.stdout:768: writing smithi02284960-11 from 5971968 to 6045696 tid 4 2024-09-21T09:33:42.484 INFO:tasks.rados.rados.0.smithi022.stdout:769: setattr oid 25 current snap is 76 2024-09-21T09:33:42.484 INFO:tasks.rados.rados.0.smithi022.stdout:764: finishing rollback tid 1 to smithi02284960-6 2024-09-21T09:33:42.484 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 6 v 0 (ObjNum 129 snap 0 seq_num 2955814956) dirty dne 2024-09-21T09:33:42.484 INFO:tasks.rados.rados.0.smithi022.stdout:766: finishing copy_from to smithi02284960-18 2024-09-21T09:33:42.484 INFO:tasks.rados.rados.0.smithi022.stdout:766: got expected ENOENT (src dne) 2024-09-21T09:33:42.484 INFO:tasks.rados.rados.0.smithi022.stdout:766: finishing copy_from racing read to smithi02284960-18 2024-09-21T09:33:42.485 INFO:tasks.rados.rados.0.smithi022.stdout:768: finishing write tid 1 to smithi02284960-11 2024-09-21T09:33:42.485 INFO:tasks.rados.rados.0.smithi022.stdout:768: finishing write tid 2 to smithi02284960-11 2024-09-21T09:33:42.485 INFO:tasks.rados.rados.0.smithi022.stdout:763: done (5 left) 2024-09-21T09:33:42.485 INFO:tasks.rados.rados.0.smithi022.stdout:764: done (4 left) 2024-09-21T09:33:42.486 INFO:tasks.rados.rados.0.smithi022.stdout:765: done (3 left) 2024-09-21T09:33:42.486 INFO:tasks.rados.rados.0.smithi022.stdout:766: done (2 left) 2024-09-21T09:33:42.486 INFO:tasks.rados.rados.0.smithi022.stdout:770: read oid 3 snap -1 2024-09-21T09:33:42.486 INFO:tasks.rados.rados.0.smithi022.stdout:770: expect (ObjNum 195 snap 50 seq_num 195) 2024-09-21T09:33:42.486 INFO:tasks.rados.rados.0.smithi022.stdout:771: delete oid 33 current snap is 76 2024-09-21T09:33:42.488 INFO:tasks.rados.rados.0.smithi022.stdout:771: done (3 left) 2024-09-21T09:33:42.488 INFO:tasks.rados.rados.0.smithi022.stdout:772: delete oid 15 current snap is 76 2024-09-21T09:33:42.491 INFO:tasks.rados.rados.0.smithi022.stdout:768: finishing write tid 3 to smithi02284960-11 2024-09-21T09:33:42.492 INFO:tasks.rados.rados.0.smithi022.stdout:768: finishing write tid 4 to smithi02284960-11 2024-09-21T09:33:42.493 INFO:tasks.rados.rados.0.smithi022.stdout:768: finishing write tid 5 to smithi02284960-11 2024-09-21T09:33:42.493 INFO:tasks.rados.rados.0.smithi022.stdout:772: done (3 left) 2024-09-21T09:33:42.493 INFO:tasks.rados.rados.0.smithi022.stdout:773: delete oid 27 current snap is 76 2024-09-21T09:33:42.495 INFO:tasks.rados.rados.0.smithi022.stdout:768: finishing write tid 6 to smithi02284960-11 2024-09-21T09:33:42.495 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 11 v 346 (ObjNum 264 snap 76 seq_num 264) dirty exists 2024-09-21T09:33:42.495 INFO:tasks.rados.rados.0.smithi022.stdout:768: left oid 11 (ObjNum 264 snap 76 seq_num 264) 2024-09-21T09:33:42.495 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 25 v 415 (ObjNum 223 snap 63 seq_num 223) dirty exists 2024-09-21T09:33:42.495 INFO:tasks.rados.rados.0.smithi022.stdout:768: done (3 left) 2024-09-21T09:33:42.495 INFO:tasks.rados.rados.0.smithi022.stdout:769: done (2 left) 2024-09-21T09:33:42.496 INFO:tasks.rados.rados.0.smithi022.stdout:773: done (1 left) 2024-09-21T09:33:42.496 INFO:tasks.rados.rados.0.smithi022.stdout:774: snap_create 2024-09-21T09:33:42.499 INFO:tasks.rados.rados.0.smithi022.stdout:770: expect (ObjNum 195 snap 50 seq_num 195) 2024-09-21T09:33:42.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:42 smithi064 ceph-mon[31124]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:42.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:42 smithi064 ceph-mon[31124]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:42.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:42 smithi064 ceph-mon[31124]: Reconfiguring node-exporter.b (dependencies changed)... 2024-09-21T09:33:42.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:42 smithi064 ceph-mon[31124]: Deploying daemon node-exporter.b on smithi064 2024-09-21T09:33:42.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:42 smithi064 ceph-mon[31124]: osdmap e212: 8 total, 8 up, 8 in 2024-09-21T09:33:42.692 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:33:42 smithi064 podman[75947]: 2024-09-21 09:33:42.434882881 +0000 UTC m=+0.119057729 container died 643a1e1aec614f5d11706830fdcc0b7d62c379d649e64324299fa4475b85f6db (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-21T09:33:42.692 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:33:42 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[75801]: logger=infra.usagestats.collector t=2024-09-21T09:33:42.574658904Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2024-09-21T09:33:42.692 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:33:42 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[75801]: logger=provisioning.datasources t=2024-09-21T09:33:42.662423859Z level=info msg="deleted correlations based on configuration" ds_name=Dashboard1 2024-09-21T09:33:42.692 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:33:42 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[75801]: logger=provisioning.datasources t=2024-09-21T09:33:42.662453338Z level=info msg="deleted datasource based on configuration" name=Dashboard1 2024-09-21T09:33:42.692 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:33:42 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[75801]: logger=provisioning.datasources t=2024-09-21T09:33:42.672877371Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2024-09-21T09:33:43.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:42 smithi022 ceph-mon[39525]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:43.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:42 smithi022 ceph-mon[39525]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:43.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:42 smithi022 ceph-mon[39525]: Reconfiguring node-exporter.b (dependencies changed)... 2024-09-21T09:33:43.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:42 smithi022 ceph-mon[39525]: Deploying daemon node-exporter.b on smithi064 2024-09-21T09:33:43.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:42 smithi022 ceph-mon[39525]: osdmap e212: 8 total, 8 up, 8 in 2024-09-21T09:33:43.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:42 smithi022 ceph-mon[27763]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:43.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:42 smithi022 ceph-mon[27763]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:43.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:42 smithi022 ceph-mon[27763]: Reconfiguring node-exporter.b (dependencies changed)... 2024-09-21T09:33:43.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:42 smithi022 ceph-mon[27763]: Deploying daemon node-exporter.b on smithi064 2024-09-21T09:33:43.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:42 smithi022 ceph-mon[27763]: osdmap e212: 8 total, 8 up, 8 in 2024-09-21T09:33:43.109 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:33:42 smithi064 podman[75947]: 2024-09-21 09:33:42.857217667 +0000 UTC m=+0.541392515 container cleanup 643a1e1aec614f5d11706830fdcc0b7d62c379d649e64324299fa4475b85f6db (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-21T09:33:43.110 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:33:42 smithi064 bash[75947]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b 2024-09-21T09:33:43.110 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:33:42 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[75801]: logger=provisioning.alerting t=2024-09-21T09:33:42.754940857Z level=info msg="starting to provision alerting" 2024-09-21T09:33:43.110 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:33:42 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[75801]: logger=provisioning.alerting t=2024-09-21T09:33:42.754962545Z level=info msg="finished to provision alerting" 2024-09-21T09:33:43.110 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:33:42 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[75801]: logger=grafanaStorageLogger t=2024-09-21T09:33:42.755040922Z level=info msg="storage starting" 2024-09-21T09:33:43.110 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:33:42 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[75801]: logger=ngalert.state.manager t=2024-09-21T09:33:42.755103227Z level=info msg="Warming state cache for startup" 2024-09-21T09:33:43.110 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:33:42 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[75801]: logger=http.server t=2024-09-21T09:33:42.757787525Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2024-09-21T09:33:43.110 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:33:42 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[75801]: logger=ngalert.state.manager t=2024-09-21T09:33:42.765344699Z level=info msg="State cache has been initialized" states=0 duration=10.240362ms 2024-09-21T09:33:43.110 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:33:42 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[75801]: logger=ngalert.multiorg.alertmanager t=2024-09-21T09:33:42.765404976Z level=info msg="starting MultiOrg Alertmanager" 2024-09-21T09:33:43.110 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:33:42 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[75801]: logger=ticker t=2024-09-21T09:33:42.765416442Z level=info msg=starting first_tick=2024-09-21T09:33:50Z 2024-09-21T09:33:43.441 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:33:43 smithi064 podman[75965]: 2024-09-21 09:33:43.108736946 +0000 UTC m=+0.674347897 container remove 643a1e1aec614f5d11706830fdcc0b7d62c379d649e64324299fa4475b85f6db (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-21T09:33:43.441 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:33:43 smithi064 systemd[1]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-09-21T09:33:43.442 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:33:43 smithi064 systemd[1]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490@node-exporter.b.service: Failed with result 'exit-code'. 2024-09-21T09:33:43.442 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:33:43 smithi064 systemd[1]: Stopped Ceph node-exporter.b for d65e801e-77fa-11ef-baf6-efdc52797490. 2024-09-21T09:33:43.442 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:33:43 smithi064 systemd[1]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490@node-exporter.b.service: Consumed 2.354s CPU time. 2024-09-21T09:33:43.451 INFO:tasks.rados.rados.0.smithi022.stdout:770: done (1 left) 2024-09-21T09:33:43.452 INFO:tasks.rados.rados.0.smithi022.stdout:774: done (0 left) 2024-09-21T09:33:43.452 INFO:tasks.rados.rados.0.smithi022.stdout:775: snap_create 2024-09-21T09:33:43.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:43 smithi064 ceph-mon[31124]: pgmap v75: 121 pgs: 121 active+clean; 171 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.9 MiB/s rd, 2.9 MiB/s wr, 21 op/s 2024-09-21T09:33:43.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:43 smithi064 ceph-mon[31124]: osdmap e213: 8 total, 8 up, 8 in 2024-09-21T09:33:43.942 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:33:43 smithi064 systemd[1]: Starting Ceph node-exporter.b for d65e801e-77fa-11ef-baf6-efdc52797490... 2024-09-21T09:33:43.942 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:33:43 smithi064 bash[76097]: Trying to pull quay.io/prometheus/node-exporter:v1.7.0... 2024-09-21T09:33:44.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:43 smithi022 ceph-mon[39525]: pgmap v75: 121 pgs: 121 active+clean; 171 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.9 MiB/s rd, 2.9 MiB/s wr, 21 op/s 2024-09-21T09:33:44.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:43 smithi022 ceph-mon[39525]: osdmap e213: 8 total, 8 up, 8 in 2024-09-21T09:33:44.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:43 smithi022 ceph-mon[27763]: pgmap v75: 121 pgs: 121 active+clean; 171 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.9 MiB/s rd, 2.9 MiB/s wr, 21 op/s 2024-09-21T09:33:44.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:43 smithi022 ceph-mon[27763]: osdmap e213: 8 total, 8 up, 8 in 2024-09-21T09:33:44.441 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:33:44 smithi064 bash[76097]: Getting image source signatures 2024-09-21T09:33:44.442 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:33:44 smithi064 bash[76097]: Copying blob sha256:2abcce694348cd2c949c0e98a7400ebdfd8341021bcf6b541bc72033ce982510 2024-09-21T09:33:44.442 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:33:44 smithi064 bash[76097]: Copying blob sha256:455fd88e5221bc1e278ef2d059cd70e4df99a24e5af050ede621534276f6cf9a 2024-09-21T09:33:44.442 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:33:44 smithi064 bash[76097]: Copying blob sha256:324153f2810a9927fcce320af9e4e291e0b6e805cbdd1f338386c756b9defa24 2024-09-21T09:33:44.453 INFO:tasks.rados.rados.0.smithi022.stdout:775: done (0 left) 2024-09-21T09:33:44.453 INFO:tasks.rados.rados.0.smithi022.stdout:776: copy_from oid 48 from oid 1 current snap is 78 2024-09-21T09:33:44.453 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 41 current snap is 78 2024-09-21T09:33:44.453 INFO:tasks.rados.rados.0.smithi022.stdout:777: seq_num 268 ranges {3293184=491520,3784704=745472,4530176=745472,5275648=114688} 2024-09-21T09:33:44.474 INFO:tasks.rados.rados.0.smithi022.stdout:777: writing smithi02284960-41 from 3293184 to 3784704 tid 1 2024-09-21T09:33:44.478 INFO:tasks.rados.rados.0.smithi022.stdout:777: writing smithi02284960-41 from 3784704 to 4530176 tid 2 2024-09-21T09:33:44.483 INFO:tasks.rados.rados.0.smithi022.stdout:777: writing smithi02284960-41 from 4530176 to 5275648 tid 3 2024-09-21T09:33:44.483 INFO:tasks.rados.rados.0.smithi022.stdout:777: writing smithi02284960-41 from 5275648 to 5390336 tid 4 2024-09-21T09:33:44.484 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 25 current snap is 78 2024-09-21T09:33:44.484 INFO:tasks.rados.rados.0.smithi022.stdout:776: finishing copy_from racing read to smithi02284960-48 2024-09-21T09:33:44.484 INFO:tasks.rados.rados.0.smithi022.stdout:778: seq_num 269 ranges {1343488=507904,1851392=630784,2482176=557056,3039232=417792} 2024-09-21T09:33:44.493 INFO:tasks.rados.rados.0.smithi022.stdout:778: writing smithi02284960-25 from 1343488 to 1851392 tid 1 2024-09-21T09:33:44.497 INFO:tasks.rados.rados.0.smithi022.stdout:778: writing smithi02284960-25 from 1851392 to 2482176 tid 2 2024-09-21T09:33:44.500 INFO:tasks.rados.rados.0.smithi022.stdout:778: writing smithi02284960-25 from 2482176 to 3039232 tid 3 2024-09-21T09:33:44.503 INFO:tasks.rados.rados.0.smithi022.stdout:778: writing smithi02284960-25 from 3039232 to 3457024 tid 4 2024-09-21T09:33:44.503 INFO:tasks.rados.rados.0.smithi022.stdout:779: rmattr oid 32 current snap is 78 2024-09-21T09:33:44.503 INFO:tasks.rados.rados.0.smithi022.stdout:779: done (3 left) 2024-09-21T09:33:44.504 INFO:tasks.rados.rados.0.smithi022.stdout:780: read oid 36 snap -1 2024-09-21T09:33:44.504 INFO:tasks.rados.rados.0.smithi022.stdout:780: expect (ObjNum 188 snap 47 seq_num 188) 2024-09-21T09:33:44.504 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 28 current snap is 78 2024-09-21T09:33:44.504 INFO:tasks.rados.rados.0.smithi022.stdout:776: finishing copy_from to smithi02284960-48 2024-09-21T09:33:44.504 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 48 v 326 (ObjNum 239 snap 65 seq_num 239) dirty exists 2024-09-21T09:33:44.504 INFO:tasks.rados.rados.0.smithi022.stdout:777: finishing write tid 1 to smithi02284960-41 2024-09-21T09:33:44.504 INFO:tasks.rados.rados.0.smithi022.stdout:777: finishing write tid 2 to smithi02284960-41 2024-09-21T09:33:44.504 INFO:tasks.rados.rados.0.smithi022.stdout:781: seq_num 270 ranges {475136=524288,999424=516096,1515520=761856,2277376=327680} 2024-09-21T09:33:44.509 INFO:tasks.rados.rados.0.smithi022.stdout:781: writing smithi02284960-28 from 475136 to 999424 tid 1 2024-09-21T09:33:44.512 INFO:tasks.rados.rados.0.smithi022.stdout:781: writing smithi02284960-28 from 999424 to 1515520 tid 2 2024-09-21T09:33:44.517 INFO:tasks.rados.rados.0.smithi022.stdout:781: writing smithi02284960-28 from 1515520 to 2277376 tid 3 2024-09-21T09:33:44.519 INFO:tasks.rados.rados.0.smithi022.stdout:781: writing smithi02284960-28 from 2277376 to 2605056 tid 4 2024-09-21T09:33:44.520 INFO:tasks.rados.rados.0.smithi022.stdout:776: done (4 left) 2024-09-21T09:33:44.520 INFO:tasks.rados.rados.0.smithi022.stdout:782: copy_from oid 26 from oid 37 current snap is 78 2024-09-21T09:33:44.520 INFO:tasks.rados.rados.0.smithi022.stdout:777: finishing write tid 3 to smithi02284960-41 2024-09-21T09:33:44.520 INFO:tasks.rados.rados.0.smithi022.stdout:777: finishing write tid 4 to smithi02284960-41 2024-09-21T09:33:44.520 INFO:tasks.rados.rados.0.smithi022.stdout:777: finishing write tid 5 to smithi02284960-41 2024-09-21T09:33:44.520 INFO:tasks.rados.rados.0.smithi022.stdout:777: finishing write tid 6 to smithi02284960-41 2024-09-21T09:33:44.520 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 41 v 421 (ObjNum 268 snap 78 seq_num 268) dirty exists 2024-09-21T09:33:44.520 INFO:tasks.rados.rados.0.smithi022.stdout:777: left oid 41 (ObjNum 268 snap 78 seq_num 268) 2024-09-21T09:33:44.520 INFO:tasks.rados.rados.0.smithi022.stdout:778: finishing write tid 1 to smithi02284960-25 2024-09-21T09:33:44.520 INFO:tasks.rados.rados.0.smithi022.stdout:777: done (4 left) 2024-09-21T09:33:44.520 INFO:tasks.rados.rados.0.smithi022.stdout:783: snap_create 2024-09-21T09:33:44.520 INFO:tasks.rados.rados.0.smithi022.stdout:778: finishing write tid 2 to smithi02284960-25 2024-09-21T09:33:44.520 INFO:tasks.rados.rados.0.smithi022.stdout:778: finishing write tid 3 to smithi02284960-25 2024-09-21T09:33:44.520 INFO:tasks.rados.rados.0.smithi022.stdout:778: finishing write tid 4 to smithi02284960-25 2024-09-21T09:33:44.520 INFO:tasks.rados.rados.0.smithi022.stdout:778: finishing write tid 5 to smithi02284960-25 2024-09-21T09:33:44.520 INFO:tasks.rados.rados.0.smithi022.stdout:778: finishing write tid 6 to smithi02284960-25 2024-09-21T09:33:44.521 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 25 v 427 (ObjNum 269 snap 78 seq_num 269) dirty exists 2024-09-21T09:33:44.521 INFO:tasks.rados.rados.0.smithi022.stdout:778: left oid 25 (ObjNum 269 snap 78 seq_num 269) 2024-09-21T09:33:44.521 INFO:tasks.rados.rados.0.smithi022.stdout:780: expect (ObjNum 188 snap 47 seq_num 188) 2024-09-21T09:33:44.588 INFO:tasks.rados.rados.0.smithi022.stdout:781: finishing write tid 1 to smithi02284960-28 2024-09-21T09:33:44.589 INFO:tasks.rados.rados.0.smithi022.stdout:781: finishing write tid 2 to smithi02284960-28 2024-09-21T09:33:44.589 INFO:tasks.rados.rados.0.smithi022.stdout:781: finishing write tid 3 to smithi02284960-28 2024-09-21T09:33:44.589 INFO:tasks.rados.rados.0.smithi022.stdout:781: finishing write tid 4 to smithi02284960-28 2024-09-21T09:33:44.589 INFO:tasks.rados.rados.0.smithi022.stdout:781: finishing write tid 5 to smithi02284960-28 2024-09-21T09:33:44.589 INFO:tasks.rados.rados.0.smithi022.stdout:781: finishing write tid 6 to smithi02284960-28 2024-09-21T09:33:44.589 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 28 v 154 (ObjNum 270 snap 78 seq_num 270) dirty exists 2024-09-21T09:33:44.589 INFO:tasks.rados.rados.0.smithi022.stdout:781: left oid 28 (ObjNum 270 snap 78 seq_num 270) 2024-09-21T09:33:44.589 INFO:tasks.rados.rados.0.smithi022.stdout:782: finishing copy_from racing read to smithi02284960-26 2024-09-21T09:33:44.589 INFO:tasks.rados.rados.0.smithi022.stdout:782: finishing copy_from to smithi02284960-26 2024-09-21T09:33:44.589 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 26 v 358 (ObjNum 218 snap 63 seq_num 218) dirty exists 2024-09-21T09:33:45.449 INFO:tasks.rados.rados.0.smithi022.stdout:778: done (4 left) 2024-09-21T09:33:45.449 INFO:tasks.rados.rados.0.smithi022.stdout:780: done (3 left) 2024-09-21T09:33:45.450 INFO:tasks.rados.rados.0.smithi022.stdout:781: done (2 left) 2024-09-21T09:33:45.450 INFO:tasks.rados.rados.0.smithi022.stdout:782: done (1 left) 2024-09-21T09:33:45.450 INFO:tasks.rados.rados.0.smithi022.stdout:783: done (0 left) 2024-09-21T09:33:45.450 INFO:tasks.rados.rados.0.smithi022.stdout:784: rollback oid 46 current snap is 79 2024-09-21T09:33:45.450 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 46 to 75 2024-09-21T09:33:45.450 INFO:tasks.rados.rados.0.smithi022.stdout:785: copy_from oid 35 from oid 11 current snap is 79 2024-09-21T09:33:45.450 INFO:tasks.rados.rados.0.smithi022.stdout:786: read oid 30 snap -1 2024-09-21T09:33:45.450 INFO:tasks.rados.rados.0.smithi022.stdout:786: expect (ObjNum 214 snap 61 seq_num 214) 2024-09-21T09:33:45.450 INFO:tasks.rados.rados.0.smithi022.stdout:787: copy_from oid 6 from oid 38 current snap is 79 2024-09-21T09:33:45.450 INFO:tasks.rados.rados.0.smithi022.stdout:788: delete oid 33 current snap is 79 2024-09-21T09:33:45.455 INFO:tasks.rados.rados.0.smithi022.stdout:788: done (4 left) 2024-09-21T09:33:45.455 INFO:tasks.rados.rados.0.smithi022.stdout:789: read oid 32 snap -1 2024-09-21T09:33:45.455 INFO:tasks.rados.rados.0.smithi022.stdout:789: expect (ObjNum 210 snap 58 seq_num 210) 2024-09-21T09:33:45.455 INFO:tasks.rados.rados.0.smithi022.stdout:790: read oid 23 snap 56 2024-09-21T09:33:45.455 INFO:tasks.rados.rados.0.smithi022.stdout:790: expect (ObjNum 200 snap 51 seq_num 200) 2024-09-21T09:33:45.455 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 31 current snap is 79 2024-09-21T09:33:45.455 INFO:tasks.rados.rados.0.smithi022.stdout:791: seq_num 272 ranges {4235264=565248,4800512=688128,5488640=729088,6217728=188416} 2024-09-21T09:33:45.483 INFO:tasks.rados.rados.0.smithi022.stdout:791: writing smithi02284960-31 from 4235264 to 4800512 tid 1 2024-09-21T09:33:45.487 INFO:tasks.rados.rados.0.smithi022.stdout:791: writing smithi02284960-31 from 4800512 to 5488640 tid 2 2024-09-21T09:33:45.492 INFO:tasks.rados.rados.0.smithi022.stdout:791: writing smithi02284960-31 from 5488640 to 6217728 tid 3 2024-09-21T09:33:45.493 INFO:tasks.rados.rados.0.smithi022.stdout:791: writing smithi02284960-31 from 6217728 to 6406144 tid 4 2024-09-21T09:33:45.493 INFO:tasks.rados.rados.0.smithi022.stdout:792: rmattr oid 44 current snap is 79 2024-09-21T09:33:45.493 INFO:tasks.rados.rados.0.smithi022.stdout:786: expect (ObjNum 214 snap 61 seq_num 214) 2024-09-21T09:33:45.525 INFO:tasks.rados.rados.0.smithi022.stdout:784: finishing rollback tid 0 to smithi02284960-46 2024-09-21T09:33:45.526 INFO:tasks.rados.rados.0.smithi022.stdout:784: finishing rollback tid 1 to smithi02284960-46 2024-09-21T09:33:45.526 INFO:tasks.rados.rados.0.smithi022.stdout:784: finishing rollback tid 2 to smithi02284960-46 2024-09-21T09:33:45.526 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 46 v 350 (ObjNum 204 snap 56 seq_num 204) dirty exists 2024-09-21T09:33:45.526 INFO:tasks.rados.rados.0.smithi022.stdout:789: expect (ObjNum 210 snap 58 seq_num 210) 2024-09-21T09:33:45.597 INFO:tasks.rados.rados.0.smithi022.stdout:790: expect (ObjNum 200 snap 51 seq_num 200) 2024-09-21T09:33:45.659 INFO:tasks.rados.rados.0.smithi022.stdout:785: finishing copy_from racing read to smithi02284960-35 2024-09-21T09:33:45.659 INFO:tasks.rados.rados.0.smithi022.stdout:791: finishing write tid 1 to smithi02284960-31 2024-09-21T09:33:45.659 INFO:tasks.rados.rados.0.smithi022.stdout:791: finishing write tid 2 to smithi02284960-31 2024-09-21T09:33:45.659 INFO:tasks.rados.rados.0.smithi022.stdout:785: finishing copy_from to smithi02284960-35 2024-09-21T09:33:45.660 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 35 v 346 (ObjNum 264 snap 76 seq_num 264) dirty exists 2024-09-21T09:33:45.660 INFO:tasks.rados.rados.0.smithi022.stdout:787: finishing copy_from racing read to smithi02284960-6 2024-09-21T09:33:45.660 INFO:tasks.rados.rados.0.smithi022.stdout:784: done (7 left) 2024-09-21T09:33:45.660 INFO:tasks.rados.rados.0.smithi022.stdout:785: done (6 left) 2024-09-21T09:33:45.660 INFO:tasks.rados.rados.0.smithi022.stdout:786: done (5 left) 2024-09-21T09:33:45.660 INFO:tasks.rados.rados.0.smithi022.stdout:789: done (4 left) 2024-09-21T09:33:45.660 INFO:tasks.rados.rados.0.smithi022.stdout:790: done (3 left) 2024-09-21T09:33:45.660 INFO:tasks.rados.rados.0.smithi022.stdout:793: snap_remove snap 54 2024-09-21T09:33:45.779 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:45 smithi064 ceph-mon[31124]: osdmap e214: 8 total, 8 up, 8 in 2024-09-21T09:33:45.779 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:45 smithi064 ceph-mon[31124]: pgmap v78: 121 pgs: 121 active+clean; 171 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.9 MiB/s rd, 2.9 MiB/s wr, 21 op/s 2024-09-21T09:33:45.779 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:45 smithi064 ceph-mon[31124]: osdmap e215: 8 total, 8 up, 8 in 2024-09-21T09:33:45.807 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:45 smithi022 ceph-mon[39525]: osdmap e214: 8 total, 8 up, 8 in 2024-09-21T09:33:45.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:45 smithi022 ceph-mon[39525]: pgmap v78: 121 pgs: 121 active+clean; 171 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.9 MiB/s rd, 2.9 MiB/s wr, 21 op/s 2024-09-21T09:33:45.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:45 smithi022 ceph-mon[39525]: osdmap e215: 8 total, 8 up, 8 in 2024-09-21T09:33:45.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:45 smithi022 ceph-mon[27763]: osdmap e214: 8 total, 8 up, 8 in 2024-09-21T09:33:45.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:45 smithi022 ceph-mon[27763]: pgmap v78: 121 pgs: 121 active+clean; 171 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.9 MiB/s rd, 2.9 MiB/s wr, 21 op/s 2024-09-21T09:33:45.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:45 smithi022 ceph-mon[27763]: osdmap e215: 8 total, 8 up, 8 in 2024-09-21T09:33:46.191 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:33:45 smithi064 bash[76097]: Copying config sha256:72c9c208898624938c9e4183d6686ea4a5fd3f912bc29bc3f00147924c521a3e 2024-09-21T09:33:46.192 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:33:45 smithi064 bash[76097]: Writing manifest to image destination 2024-09-21T09:33:46.464 INFO:tasks.rados.rados.0.smithi022.stdout:793: done (3 left) 2024-09-21T09:33:46.465 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 5 current snap is 79 2024-09-21T09:33:46.465 INFO:tasks.rados.rados.0.smithi022.stdout:794: seq_num 273 ranges {0=581632,581632=565248,1146880=532480,1679360=507904} 2024-09-21T09:33:46.468 INFO:tasks.rados.rados.0.smithi022.stdout:794: writing smithi02284960-5 from 0 to 581632 tid 1 2024-09-21T09:33:46.472 INFO:tasks.rados.rados.0.smithi022.stdout:794: writing smithi02284960-5 from 581632 to 1146880 tid 2 2024-09-21T09:33:46.477 INFO:tasks.rados.rados.0.smithi022.stdout:794: writing smithi02284960-5 from 1146880 to 1679360 tid 3 2024-09-21T09:33:46.480 INFO:tasks.rados.rados.0.smithi022.stdout:794: writing smithi02284960-5 from 1679360 to 2187264 tid 4 2024-09-21T09:33:46.482 INFO:tasks.rados.rados.0.smithi022.stdout:795: rollback oid 41 current snap is 79 2024-09-21T09:33:46.482 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 41 to 69 2024-09-21T09:33:46.482 INFO:tasks.rados.rados.0.smithi022.stdout:787: finishing copy_from to smithi02284960-6 2024-09-21T09:33:46.482 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 6 v 83 (ObjNum 142 snap 32 seq_num 142) dirty exists 2024-09-21T09:33:46.482 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 44 v 352 (ObjNum 217 snap 63 seq_num 217) dirty exists 2024-09-21T09:33:46.482 INFO:tasks.rados.rados.0.smithi022.stdout:791: finishing write tid 3 to smithi02284960-31 2024-09-21T09:33:46.482 INFO:tasks.rados.rados.0.smithi022.stdout:791: finishing write tid 4 to smithi02284960-31 2024-09-21T09:33:46.482 INFO:tasks.rados.rados.0.smithi022.stdout:791: finishing write tid 5 to smithi02284960-31 2024-09-21T09:33:46.483 INFO:tasks.rados.rados.0.smithi022.stdout:791: finishing write tid 6 to smithi02284960-31 2024-09-21T09:33:46.483 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 31 v 485 (ObjNum 272 snap 79 seq_num 272) dirty exists 2024-09-21T09:33:46.483 INFO:tasks.rados.rados.0.smithi022.stdout:791: left oid 31 (ObjNum 272 snap 79 seq_num 272) 2024-09-21T09:33:46.483 INFO:tasks.rados.rados.0.smithi022.stdout:787: done (4 left) 2024-09-21T09:33:46.483 INFO:tasks.rados.rados.0.smithi022.stdout:791: done (3 left) 2024-09-21T09:33:46.483 INFO:tasks.rados.rados.0.smithi022.stdout:792: done (2 left) 2024-09-21T09:33:46.483 INFO:tasks.rados.rados.0.smithi022.stdout:796: snap_create 2024-09-21T09:33:46.483 INFO:tasks.rados.rados.0.smithi022.stdout:794: finishing write tid 1 to smithi02284960-5 2024-09-21T09:33:46.484 INFO:tasks.rados.rados.0.smithi022.stdout:794: finishing write tid 2 to smithi02284960-5 2024-09-21T09:33:46.484 INFO:tasks.rados.rados.0.smithi022.stdout:794: finishing write tid 3 to smithi02284960-5 2024-09-21T09:33:46.484 INFO:tasks.rados.rados.0.smithi022.stdout:794: finishing write tid 4 to smithi02284960-5 2024-09-21T09:33:46.485 INFO:tasks.rados.rados.0.smithi022.stdout:795: finishing rollback tid 0 to smithi02284960-41 2024-09-21T09:33:46.485 INFO:tasks.rados.rados.0.smithi022.stdout:794: finishing write tid 5 to smithi02284960-5 2024-09-21T09:33:46.486 INFO:tasks.rados.rados.0.smithi022.stdout:795: finishing rollback tid 1 to smithi02284960-41 2024-09-21T09:33:46.487 INFO:tasks.rados.rados.0.smithi022.stdout:795: finishing rollback tid 2 to smithi02284960-41 2024-09-21T09:33:46.487 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 41 v 439 (ObjNum 247 snap 67 seq_num 247) dirty exists 2024-09-21T09:33:46.487 INFO:tasks.rados.rados.0.smithi022.stdout:794: finishing write tid 6 to smithi02284960-5 2024-09-21T09:33:46.487 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 5 v 240 (ObjNum 273 snap 79 seq_num 273) dirty exists 2024-09-21T09:33:46.488 INFO:tasks.rados.rados.0.smithi022.stdout:794: left oid 5 (ObjNum 273 snap 79 seq_num 273) 2024-09-21T09:33:46.554 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:33:46 smithi064 podman[76097]: 2024-09-21 09:33:46.209884471 +0000 UTC m=+2.534106078 image pull 72c9c208898624938c9e4183d6686ea4a5fd3f912bc29bc3f00147924c521a3e quay.io/prometheus/node-exporter:v1.7.0 2024-09-21T09:33:46.554 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:33:46 smithi064 podman[76097]: 2024-09-21 09:33:46.317007477 +0000 UTC m=+2.641229070 container create a533420b0139f835486bd93cc492d96ac0333027426f3c475d08bf722a1f6e61 (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-21T09:33:46.554 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:33:46 smithi064 podman[76097]: 2024-09-21 09:33:46.553203441 +0000 UTC m=+2.877425045 container init a533420b0139f835486bd93cc492d96ac0333027426f3c475d08bf722a1f6e61 (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-21T09:33:46.942 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:33:46 smithi064 podman[76097]: 2024-09-21 09:33:46.558936644 +0000 UTC m=+2.883158237 container start a533420b0139f835486bd93cc492d96ac0333027426f3c475d08bf722a1f6e61 (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-21T09:33:46.942 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:33:46 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b[76171]: ts=2024-09-21T09:33:46.562Z caller=node_exporter.go:192 level=info msg="Starting node_exporter" version="(version=1.7.0, branch=HEAD, revision=7333465abf9efba81876303bb57e6fadb946041b)" 2024-09-21T09:33:46.942 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:33:46 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b[76171]: ts=2024-09-21T09:33:46.562Z caller=node_exporter.go:193 level=info msg="Build context" build_context="(go=go1.21.4, platform=linux/amd64, user=root@35918982f6d8, date=20231112-23:53:35, tags=netgo osusergo static_build)" 2024-09-21T09:33:46.942 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:33:46 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b[76171]: ts=2024-09-21T09:33:46.563Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-09-21T09:33:46.942 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:33:46 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b[76171]: ts=2024-09-21T09:33:46.563Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-09-21T09:33:46.942 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:33:46 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b[76171]: ts=2024-09-21T09:33:46.563Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-09-21T09:33:46.942 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:33:46 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b[76171]: ts=2024-09-21T09:33:46.563Z caller=diskstats_linux.go:265 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-09-21T09:33:46.942 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:33:46 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b[76171]: ts=2024-09-21T09:33:46.563Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-09-21T09:33:46.942 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:33:46 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b[76171]: ts=2024-09-21T09:33:46.563Z caller=node_exporter.go:117 level=info collector=arp 2024-09-21T09:33:46.942 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:33:46 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b[76171]: ts=2024-09-21T09:33:46.563Z caller=node_exporter.go:117 level=info collector=bcache 2024-09-21T09:33:46.943 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:33:46 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b[76171]: ts=2024-09-21T09:33:46.563Z caller=node_exporter.go:117 level=info collector=bonding 2024-09-21T09:33:46.943 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:33:46 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b[76171]: ts=2024-09-21T09:33:46.563Z caller=node_exporter.go:117 level=info collector=btrfs 2024-09-21T09:33:46.943 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:33:46 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b[76171]: ts=2024-09-21T09:33:46.563Z caller=node_exporter.go:117 level=info collector=conntrack 2024-09-21T09:33:46.943 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:33:46 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b[76171]: ts=2024-09-21T09:33:46.563Z caller=node_exporter.go:117 level=info collector=cpu 2024-09-21T09:33:46.943 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:33:46 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b[76171]: ts=2024-09-21T09:33:46.563Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-09-21T09:33:46.943 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:33:46 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b[76171]: ts=2024-09-21T09:33:46.563Z caller=node_exporter.go:117 level=info collector=diskstats 2024-09-21T09:33:46.943 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:33:46 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b[76171]: ts=2024-09-21T09:33:46.563Z caller=node_exporter.go:117 level=info collector=dmi 2024-09-21T09:33:46.943 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:33:46 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b[76171]: ts=2024-09-21T09:33:46.563Z caller=node_exporter.go:117 level=info collector=edac 2024-09-21T09:33:46.944 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:33:46 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b[76171]: ts=2024-09-21T09:33:46.563Z caller=node_exporter.go:117 level=info collector=entropy 2024-09-21T09:33:46.944 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:33:46 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b[76171]: ts=2024-09-21T09:33:46.563Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-09-21T09:33:46.944 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:33:46 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b[76171]: ts=2024-09-21T09:33:46.563Z caller=node_exporter.go:117 level=info collector=filefd 2024-09-21T09:33:46.944 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:33:46 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b[76171]: ts=2024-09-21T09:33:46.563Z caller=node_exporter.go:117 level=info collector=filesystem 2024-09-21T09:33:46.944 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:33:46 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b[76171]: ts=2024-09-21T09:33:46.563Z caller=node_exporter.go:117 level=info collector=hwmon 2024-09-21T09:33:46.944 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:33:46 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b[76171]: ts=2024-09-21T09:33:46.563Z caller=node_exporter.go:117 level=info collector=infiniband 2024-09-21T09:33:46.944 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:33:46 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b[76171]: ts=2024-09-21T09:33:46.563Z caller=node_exporter.go:117 level=info collector=ipvs 2024-09-21T09:33:46.944 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:33:46 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b[76171]: ts=2024-09-21T09:33:46.563Z caller=node_exporter.go:117 level=info collector=loadavg 2024-09-21T09:33:46.944 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:33:46 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b[76171]: ts=2024-09-21T09:33:46.563Z caller=node_exporter.go:117 level=info collector=mdadm 2024-09-21T09:33:46.944 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:33:46 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b[76171]: ts=2024-09-21T09:33:46.563Z caller=node_exporter.go:117 level=info collector=meminfo 2024-09-21T09:33:46.944 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:33:46 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b[76171]: ts=2024-09-21T09:33:46.563Z caller=node_exporter.go:117 level=info collector=netclass 2024-09-21T09:33:46.944 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:33:46 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b[76171]: ts=2024-09-21T09:33:46.564Z caller=node_exporter.go:117 level=info collector=netdev 2024-09-21T09:33:46.945 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:33:46 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b[76171]: ts=2024-09-21T09:33:46.564Z caller=node_exporter.go:117 level=info collector=netstat 2024-09-21T09:33:46.945 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:33:46 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b[76171]: ts=2024-09-21T09:33:46.564Z caller=node_exporter.go:117 level=info collector=nfs 2024-09-21T09:33:46.945 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:33:46 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b[76171]: ts=2024-09-21T09:33:46.564Z caller=node_exporter.go:117 level=info collector=nfsd 2024-09-21T09:33:46.945 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:33:46 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b[76171]: ts=2024-09-21T09:33:46.564Z caller=node_exporter.go:117 level=info collector=nvme 2024-09-21T09:33:46.945 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:33:46 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b[76171]: ts=2024-09-21T09:33:46.564Z caller=node_exporter.go:117 level=info collector=os 2024-09-21T09:33:46.945 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:33:46 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b[76171]: ts=2024-09-21T09:33:46.564Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-09-21T09:33:46.945 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:33:46 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b[76171]: ts=2024-09-21T09:33:46.564Z caller=node_exporter.go:117 level=info collector=pressure 2024-09-21T09:33:46.945 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:33:46 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b[76171]: ts=2024-09-21T09:33:46.564Z caller=node_exporter.go:117 level=info collector=rapl 2024-09-21T09:33:46.945 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:33:46 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b[76171]: ts=2024-09-21T09:33:46.564Z caller=node_exporter.go:117 level=info collector=schedstat 2024-09-21T09:33:46.945 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:33:46 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b[76171]: ts=2024-09-21T09:33:46.564Z caller=node_exporter.go:117 level=info collector=selinux 2024-09-21T09:33:46.945 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:33:46 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b[76171]: ts=2024-09-21T09:33:46.564Z caller=node_exporter.go:117 level=info collector=sockstat 2024-09-21T09:33:46.945 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:33:46 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b[76171]: ts=2024-09-21T09:33:46.564Z caller=node_exporter.go:117 level=info collector=softnet 2024-09-21T09:33:46.945 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:33:46 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b[76171]: ts=2024-09-21T09:33:46.564Z caller=node_exporter.go:117 level=info collector=stat 2024-09-21T09:33:46.945 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:33:46 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b[76171]: ts=2024-09-21T09:33:46.564Z caller=node_exporter.go:117 level=info collector=tapestats 2024-09-21T09:33:46.945 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:33:46 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b[76171]: ts=2024-09-21T09:33:46.564Z caller=node_exporter.go:117 level=info collector=textfile 2024-09-21T09:33:46.946 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:33:46 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b[76171]: ts=2024-09-21T09:33:46.564Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-09-21T09:33:46.946 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:33:46 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b[76171]: ts=2024-09-21T09:33:46.564Z caller=node_exporter.go:117 level=info collector=time 2024-09-21T09:33:46.946 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:33:46 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b[76171]: ts=2024-09-21T09:33:46.564Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-09-21T09:33:46.946 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:33:46 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b[76171]: ts=2024-09-21T09:33:46.564Z caller=node_exporter.go:117 level=info collector=uname 2024-09-21T09:33:46.946 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:33:46 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b[76171]: ts=2024-09-21T09:33:46.564Z caller=node_exporter.go:117 level=info collector=vmstat 2024-09-21T09:33:46.946 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:33:46 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b[76171]: ts=2024-09-21T09:33:46.564Z caller=node_exporter.go:117 level=info collector=xfs 2024-09-21T09:33:46.946 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:33:46 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b[76171]: ts=2024-09-21T09:33:46.564Z caller=node_exporter.go:117 level=info collector=zfs 2024-09-21T09:33:46.946 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:33:46 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b[76171]: ts=2024-09-21T09:33:46.564Z caller=tls_config.go:274 level=info msg="Listening on" address=[::]:9100 2024-09-21T09:33:46.946 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:33:46 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b[76171]: ts=2024-09-21T09:33:46.564Z caller=tls_config.go:277 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-09-21T09:33:46.946 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:33:46 smithi064 bash[76097]: a533420b0139f835486bd93cc492d96ac0333027426f3c475d08bf722a1f6e61 2024-09-21T09:33:46.946 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:33:46 smithi064 systemd[1]: Started Ceph node-exporter.b for d65e801e-77fa-11ef-baf6-efdc52797490. 2024-09-21T09:33:47.441 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:33:46 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[67503]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:33:46] "GET /metrics HTTP/1.1" 200 36128 "" "Prometheus/2.51.0" 2024-09-21T09:33:47.468 INFO:tasks.rados.rados.0.smithi022.stdout:794: done (2 left) 2024-09-21T09:33:47.468 INFO:tasks.rados.rados.0.smithi022.stdout:795: done (1 left) 2024-09-21T09:33:47.468 INFO:tasks.rados.rados.0.smithi022.stdout:796: done (0 left) 2024-09-21T09:33:47.468 INFO:tasks.rados.rados.0.smithi022.stdout:797: copy_from oid 48 from oid 7 current snap is 80 2024-09-21T09:33:47.468 INFO:tasks.rados.rados.0.smithi022.stdout:798: read oid 2 snap -1 2024-09-21T09:33:47.469 INFO:tasks.rados.rados.0.smithi022.stdout:798: expect deleted 2024-09-21T09:33:47.469 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 13 current snap is 80 2024-09-21T09:33:47.469 INFO:tasks.rados.rados.0.smithi022.stdout:799: seq_num 274 ranges {1949696=598016,2547712=499712,3047424=745472,3792896=360448} 2024-09-21T09:33:47.482 INFO:tasks.rados.rados.0.smithi022.stdout:799: writing smithi02284960-13 from 1949696 to 2547712 tid 1 2024-09-21T09:33:47.485 INFO:tasks.rados.rados.0.smithi022.stdout:799: writing smithi02284960-13 from 2547712 to 3047424 tid 2 2024-09-21T09:33:47.489 INFO:tasks.rados.rados.0.smithi022.stdout:799: writing smithi02284960-13 from 3047424 to 3792896 tid 3 2024-09-21T09:33:47.492 INFO:tasks.rados.rados.0.smithi022.stdout:799: writing smithi02284960-13 from 3792896 to 4153344 tid 4 2024-09-21T09:33:47.492 INFO:tasks.rados.rados.0.smithi022.stdout:800: read oid 47 snap -1 2024-09-21T09:33:47.492 INFO:tasks.rados.rados.0.smithi022.stdout:800: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-21T09:33:47.492 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 18 current snap is 80 2024-09-21T09:33:47.492 INFO:tasks.rados.rados.0.smithi022.stdout:797: finishing copy_from racing read to smithi02284960-48 2024-09-21T09:33:47.492 INFO:tasks.rados.rados.0.smithi022.stdout:797: finishing copy_from to smithi02284960-48 2024-09-21T09:33:47.492 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 48 v 367 (ObjNum 182 snap 47 seq_num 182) dirty exists 2024-09-21T09:33:47.493 INFO:tasks.rados.rados.0.smithi022.stdout:799: finishing write tid 1 to smithi02284960-13 2024-09-21T09:33:47.493 INFO:tasks.rados.rados.0.smithi022.stdout:799: finishing write tid 2 to smithi02284960-13 2024-09-21T09:33:47.493 INFO:tasks.rados.rados.0.smithi022.stdout:801: seq_num 275 ranges {1998848=614400,2613248=786432,3399680=499712,3899392=319488} 2024-09-21T09:33:47.506 INFO:tasks.rados.rados.0.smithi022.stdout:801: writing smithi02284960-18 from 1998848 to 2613248 tid 1 2024-09-21T09:33:47.511 INFO:tasks.rados.rados.0.smithi022.stdout:801: writing smithi02284960-18 from 2613248 to 3399680 tid 2 2024-09-21T09:33:47.514 INFO:tasks.rados.rados.0.smithi022.stdout:801: writing smithi02284960-18 from 3399680 to 3899392 tid 3 2024-09-21T09:33:47.516 INFO:tasks.rados.rados.0.smithi022.stdout:801: writing smithi02284960-18 from 3899392 to 4218880 tid 4 2024-09-21T09:33:47.516 INFO:tasks.rados.rados.0.smithi022.stdout:797: done (4 left) 2024-09-21T09:33:47.516 INFO:tasks.rados.rados.0.smithi022.stdout:798: done (3 left) 2024-09-21T09:33:47.516 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 49 current snap is 80 2024-09-21T09:33:47.516 INFO:tasks.rados.rados.0.smithi022.stdout:802: seq_num 276 ranges {0=630784,630784=671744,1302528=704512,2007040=229376} 2024-09-21T09:33:47.520 INFO:tasks.rados.rados.0.smithi022.stdout:802: writing smithi02284960-49 from 0 to 630784 tid 1 2024-09-21T09:33:47.523 INFO:tasks.rados.rados.0.smithi022.stdout:802: writing smithi02284960-49 from 630784 to 1302528 tid 2 2024-09-21T09:33:47.527 INFO:tasks.rados.rados.0.smithi022.stdout:802: writing smithi02284960-49 from 1302528 to 2007040 tid 3 2024-09-21T09:33:47.529 INFO:tasks.rados.rados.0.smithi022.stdout:802: writing smithi02284960-49 from 2007040 to 2236416 tid 4 2024-09-21T09:33:47.529 INFO:tasks.rados.rados.0.smithi022.stdout:803: read oid 14 snap -1 2024-09-21T09:33:47.529 INFO:tasks.rados.rados.0.smithi022.stdout:803: expect (ObjNum 144 snap 32 seq_num 144) 2024-09-21T09:33:47.529 INFO:tasks.rados.rados.0.smithi022.stdout:804: read oid 39 snap -1 2024-09-21T09:33:47.530 INFO:tasks.rados.rados.0.smithi022.stdout:804: expect (ObjNum 240 snap 65 seq_num 240) 2024-09-21T09:33:47.530 INFO:tasks.rados.rados.0.smithi022.stdout:805: setattr oid 6 current snap is 80 2024-09-21T09:33:47.530 INFO:tasks.rados.rados.0.smithi022.stdout:800: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-21T09:33:47.532 INFO:tasks.rados.rados.0.smithi022.stdout:799: finishing write tid 3 to smithi02284960-13 2024-09-21T09:33:47.532 INFO:tasks.rados.rados.0.smithi022.stdout:799: finishing write tid 4 to smithi02284960-13 2024-09-21T09:33:47.532 INFO:tasks.rados.rados.0.smithi022.stdout:799: finishing write tid 5 to smithi02284960-13 2024-09-21T09:33:47.532 INFO:tasks.rados.rados.0.smithi022.stdout:799: finishing write tid 6 to smithi02284960-13 2024-09-21T09:33:47.532 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 13 v 366 (ObjNum 274 snap 80 seq_num 274) dirty exists 2024-09-21T09:33:47.532 INFO:tasks.rados.rados.0.smithi022.stdout:799: left oid 13 (ObjNum 274 snap 80 seq_num 274) 2024-09-21T09:33:47.532 INFO:tasks.rados.rados.0.smithi022.stdout:801: finishing write tid 1 to smithi02284960-18 2024-09-21T09:33:47.532 INFO:tasks.rados.rados.0.smithi022.stdout:801: finishing write tid 2 to smithi02284960-18 2024-09-21T09:33:47.533 INFO:tasks.rados.rados.0.smithi022.stdout:801: finishing write tid 3 to smithi02284960-18 2024-09-21T09:33:47.533 INFO:tasks.rados.rados.0.smithi022.stdout:801: finishing write tid 4 to smithi02284960-18 2024-09-21T09:33:47.533 INFO:tasks.rados.rados.0.smithi022.stdout:801: finishing write tid 5 to smithi02284960-18 2024-09-21T09:33:47.533 INFO:tasks.rados.rados.0.smithi022.stdout:801: finishing write tid 6 to smithi02284960-18 2024-09-21T09:33:47.533 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 18 v 263 (ObjNum 275 snap 80 seq_num 275) dirty exists 2024-09-21T09:33:47.533 INFO:tasks.rados.rados.0.smithi022.stdout:801: left oid 18 (ObjNum 275 snap 80 seq_num 275) 2024-09-21T09:33:47.533 INFO:tasks.rados.rados.0.smithi022.stdout:802: finishing write tid 1 to smithi02284960-49 2024-09-21T09:33:47.533 INFO:tasks.rados.rados.0.smithi022.stdout:802: finishing write tid 2 to smithi02284960-49 2024-09-21T09:33:47.533 INFO:tasks.rados.rados.0.smithi022.stdout:799: done (6 left) 2024-09-21T09:33:47.533 INFO:tasks.rados.rados.0.smithi022.stdout:800: done (5 left) 2024-09-21T09:33:47.533 INFO:tasks.rados.rados.0.smithi022.stdout:801: done (4 left) 2024-09-21T09:33:47.533 INFO:tasks.rados.rados.0.smithi022.stdout:806: read oid 48 snap 78 2024-09-21T09:33:47.533 INFO:tasks.rados.rados.0.smithi022.stdout:806: expect (ObjNum 239 snap 65 seq_num 239) 2024-09-21T09:33:47.533 INFO:tasks.rados.rados.0.smithi022.stdout:807: snap_remove snap 74 2024-09-21T09:33:47.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:47 smithi022 ceph-mon[39525]: osdmap e216: 8 total, 8 up, 8 in 2024-09-21T09:33:47.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:47 smithi022 ceph-mon[39525]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:47.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:47 smithi022 ceph-mon[39525]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:47.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:47 smithi022 ceph-mon[39525]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-21T09:33:47.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:47 smithi022 ceph-mon[39525]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-21T09:33:47.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:47 smithi022 ceph-mon[39525]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T09:33:47.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:47 smithi022 ceph-mon[39525]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T09:33:47.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:47 smithi022 ceph-mon[39525]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T09:33:47.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:47 smithi022 ceph-mon[39525]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T09:33:47.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:47 smithi022 ceph-mon[39525]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:33:47.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:47 smithi022 ceph-mon[39525]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-21T09:33:47.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:47 smithi022 ceph-mon[39525]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-21T09:33:47.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:47 smithi022 ceph-mon[39525]: pgmap v81: 121 pgs: 3 active+clean+snaptrim_wait, 4 active+clean+snaptrim, 114 active+clean; 197 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.4 MiB/s rd, 6.4 MiB/s wr, 21 op/s 2024-09-21T09:33:47.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:47 smithi022 ceph-mon[27763]: osdmap e216: 8 total, 8 up, 8 in 2024-09-21T09:33:47.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:47 smithi022 ceph-mon[27763]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:47.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:47 smithi022 ceph-mon[27763]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:47.810 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:47 smithi022 ceph-mon[27763]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-21T09:33:47.810 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:47 smithi022 ceph-mon[27763]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-21T09:33:47.810 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:47 smithi022 ceph-mon[27763]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T09:33:47.810 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:47 smithi022 ceph-mon[27763]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T09:33:47.810 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:47 smithi022 ceph-mon[27763]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T09:33:47.810 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:47 smithi022 ceph-mon[27763]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T09:33:47.810 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:47 smithi022 ceph-mon[27763]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:33:47.810 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:47 smithi022 ceph-mon[27763]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-21T09:33:47.810 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:47 smithi022 ceph-mon[27763]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-21T09:33:47.810 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:47 smithi022 ceph-mon[27763]: pgmap v81: 121 pgs: 3 active+clean+snaptrim_wait, 4 active+clean+snaptrim, 114 active+clean; 197 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.4 MiB/s rd, 6.4 MiB/s wr, 21 op/s 2024-09-21T09:33:47.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:47 smithi064 ceph-mon[31124]: osdmap e216: 8 total, 8 up, 8 in 2024-09-21T09:33:47.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:47 smithi064 ceph-mon[31124]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:47.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:47 smithi064 ceph-mon[31124]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:47.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:47 smithi064 ceph-mon[31124]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-21T09:33:47.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:47 smithi064 ceph-mon[31124]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-21T09:33:47.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:47 smithi064 ceph-mon[31124]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T09:33:47.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:47 smithi064 ceph-mon[31124]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T09:33:47.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:47 smithi064 ceph-mon[31124]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T09:33:47.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:47 smithi064 ceph-mon[31124]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T09:33:47.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:47 smithi064 ceph-mon[31124]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:33:47.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:47 smithi064 ceph-mon[31124]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-21T09:33:47.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:47 smithi064 ceph-mon[31124]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-21T09:33:47.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:47 smithi064 ceph-mon[31124]: pgmap v81: 121 pgs: 3 active+clean+snaptrim_wait, 4 active+clean+snaptrim, 114 active+clean; 197 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.4 MiB/s rd, 6.4 MiB/s wr, 21 op/s 2024-09-21T09:33:48.474 INFO:tasks.rados.rados.0.smithi022.stdout:807: done (5 left) 2024-09-21T09:33:48.474 INFO:tasks.rados.rados.0.smithi022.stdout:808: copy_from oid 15 from oid 11 current snap is 80 2024-09-21T09:33:48.475 INFO:tasks.rados.rados.0.smithi022.stdout:802: finishing write tid 3 to smithi02284960-49 2024-09-21T09:33:48.475 INFO:tasks.rados.rados.0.smithi022.stdout:802: finishing write tid 4 to smithi02284960-49 2024-09-21T09:33:48.475 INFO:tasks.rados.rados.0.smithi022.stdout:802: finishing write tid 5 to smithi02284960-49 2024-09-21T09:33:48.475 INFO:tasks.rados.rados.0.smithi022.stdout:802: finishing write tid 6 to smithi02284960-49 2024-09-21T09:33:48.475 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 49 v 247 (ObjNum 276 snap 80 seq_num 276) dirty exists 2024-09-21T09:33:48.475 INFO:tasks.rados.rados.0.smithi022.stdout:802: left oid 49 (ObjNum 276 snap 80 seq_num 276) 2024-09-21T09:33:48.475 INFO:tasks.rados.rados.0.smithi022.stdout:804: expect (ObjNum 240 snap 65 seq_num 240) 2024-09-21T09:33:48.571 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 6 v 448 (ObjNum 142 snap 32 seq_num 142) dirty exists 2024-09-21T09:33:48.572 INFO:tasks.rados.rados.0.smithi022.stdout:803: expect (ObjNum 144 snap 32 seq_num 144) 2024-09-21T09:33:48.572 INFO:tasks.rados.rados.0.smithi022.stdout:806: expect (ObjNum 239 snap 65 seq_num 239) 2024-09-21T09:33:48.625 INFO:tasks.rados.rados.0.smithi022.stdout:808: finishing copy_from racing read to smithi02284960-15 2024-09-21T09:33:48.625 INFO:tasks.rados.rados.0.smithi022.stdout:808: finishing copy_from to smithi02284960-15 2024-09-21T09:33:48.625 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 15 v 346 (ObjNum 264 snap 76 seq_num 264) dirty exists 2024-09-21T09:33:48.625 INFO:tasks.rados.rados.0.smithi022.stdout:802: done (5 left) 2024-09-21T09:33:48.625 INFO:tasks.rados.rados.0.smithi022.stdout:803: done (4 left) 2024-09-21T09:33:48.625 INFO:tasks.rados.rados.0.smithi022.stdout:804: done (3 left) 2024-09-21T09:33:48.625 INFO:tasks.rados.rados.0.smithi022.stdout:805: done (2 left) 2024-09-21T09:33:48.625 INFO:tasks.rados.rados.0.smithi022.stdout:806: done (1 left) 2024-09-21T09:33:48.625 INFO:tasks.rados.rados.0.smithi022.stdout:808: done (0 left) 2024-09-21T09:33:48.625 INFO:tasks.rados.rados.0.smithi022.stdout:809: setattr oid 14 current snap is 80 2024-09-21T09:33:48.626 INFO:tasks.rados.rados.0.smithi022.stdout:810: setattr oid 46 current snap is 80 2024-09-21T09:33:48.627 INFO:tasks.rados.rados.0.smithi022.stdout:811: snap_create 2024-09-21T09:33:48.628 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 14 v 248 (ObjNum 144 snap 32 seq_num 144) dirty exists 2024-09-21T09:33:48.630 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 46 v 370 (ObjNum 204 snap 56 seq_num 204) dirty exists 2024-09-21T09:33:48.779 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:48 smithi022 ceph-mon[39525]: osdmap e217: 8 total, 8 up, 8 in 2024-09-21T09:33:48.779 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:48 smithi022 ceph-mon[39525]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:48.780 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:48 smithi022 ceph-mon[39525]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T09:33:48.780 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:48 smithi022 ceph-mon[39525]: from='mgr.24775 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T09:33:48.780 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:48 smithi022 ceph-mon[39525]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T09:33:48.780 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:48 smithi022 ceph-mon[39525]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:33:48.780 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:48 smithi022 ceph-mon[27763]: osdmap e217: 8 total, 8 up, 8 in 2024-09-21T09:33:48.780 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:48 smithi022 ceph-mon[27763]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:48.780 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:48 smithi022 ceph-mon[27763]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T09:33:48.780 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:48 smithi022 ceph-mon[27763]: from='mgr.24775 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T09:33:48.780 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:48 smithi022 ceph-mon[27763]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T09:33:48.780 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:48 smithi022 ceph-mon[27763]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:33:48.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:48 smithi064 ceph-mon[31124]: osdmap e217: 8 total, 8 up, 8 in 2024-09-21T09:33:48.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:48 smithi064 ceph-mon[31124]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:48.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:48 smithi064 ceph-mon[31124]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T09:33:48.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:48 smithi064 ceph-mon[31124]: from='mgr.24775 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T09:33:48.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:48 smithi064 ceph-mon[31124]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T09:33:48.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:48 smithi064 ceph-mon[31124]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:33:49.477 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:49 smithi022 ceph-mon[27763]: Upgrade: Updating mgr.y 2024-09-21T09:33:49.481 INFO:tasks.rados.rados.0.smithi022.stdout:809: done (2 left) 2024-09-21T09:33:49.481 INFO:tasks.rados.rados.0.smithi022.stdout:810: done (1 left) 2024-09-21T09:33:49.481 INFO:tasks.rados.rados.0.smithi022.stdout:811: done (0 left) 2024-09-21T09:33:49.481 INFO:tasks.rados.rados.0.smithi022.stdout:812: rollback oid 27 current snap is 81 2024-09-21T09:33:49.481 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 27 to 75 2024-09-21T09:33:49.481 INFO:tasks.rados.rados.0.smithi022.stdout:813: snap_remove snap 73 2024-09-21T09:33:49.742 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:49 smithi022 ceph-mon[39525]: Upgrade: Updating mgr.y 2024-09-21T09:33:49.742 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:49 smithi022 ceph-mon[39525]: Deploying daemon mgr.y on smithi022 2024-09-21T09:33:49.742 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:49 smithi022 ceph-mon[39525]: osdmap e218: 8 total, 8 up, 8 in 2024-09-21T09:33:49.742 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:49 smithi022 ceph-mon[39525]: pgmap v84: 121 pgs: 3 active+clean+snaptrim_wait, 4 active+clean+snaptrim, 114 active+clean; 197 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.4 MiB/s rd, 6.4 MiB/s wr, 21 op/s 2024-09-21T09:33:49.743 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:49 smithi022 ceph-mon[27763]: Deploying daemon mgr.y on smithi022 2024-09-21T09:33:49.743 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:49 smithi022 ceph-mon[27763]: osdmap e218: 8 total, 8 up, 8 in 2024-09-21T09:33:49.743 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:49 smithi022 ceph-mon[27763]: pgmap v84: 121 pgs: 3 active+clean+snaptrim_wait, 4 active+clean+snaptrim, 114 active+clean; 197 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.4 MiB/s rd, 6.4 MiB/s wr, 21 op/s 2024-09-21T09:33:49.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:49 smithi064 ceph-mon[31124]: Upgrade: Updating mgr.y 2024-09-21T09:33:49.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:49 smithi064 ceph-mon[31124]: Deploying daemon mgr.y on smithi022 2024-09-21T09:33:49.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:49 smithi064 ceph-mon[31124]: osdmap e218: 8 total, 8 up, 8 in 2024-09-21T09:33:49.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:49 smithi064 ceph-mon[31124]: pgmap v84: 121 pgs: 3 active+clean+snaptrim_wait, 4 active+clean+snaptrim, 114 active+clean; 197 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.4 MiB/s rd, 6.4 MiB/s wr, 21 op/s 2024-09-21T09:33:50.308 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:33:50 smithi022 systemd[1]: Stopping Ceph mgr.y for d65e801e-77fa-11ef-baf6-efdc52797490... 2024-09-21T09:33:50.481 INFO:tasks.rados.rados.0.smithi022.stdout:813: done (1 left) 2024-09-21T09:33:50.481 INFO:tasks.rados.rados.0.smithi022.stdout:814: copy_from oid 11 from oid 41 current snap is 81 2024-09-21T09:33:50.481 INFO:tasks.rados.rados.0.smithi022.stdout:812: finishing rollback tid 1 to smithi02284960-27 2024-09-21T09:33:50.481 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 27 v 298 (ObjNum 49 snap 0 seq_num 3570533776) dirty dne 2024-09-21T09:33:50.481 INFO:tasks.rados.rados.0.smithi022.stdout:812: done (1 left) 2024-09-21T09:33:50.481 INFO:tasks.rados.rados.0.smithi022.stdout:815: copy_from oid 46 from oid 17 current snap is 81 2024-09-21T09:33:50.481 INFO:tasks.rados.rados.0.smithi022.stdout:816: delete oid 29 current snap is 81 2024-09-21T09:33:50.484 INFO:tasks.rados.rados.0.smithi022.stdout:815: finishing copy_from to smithi02284960-46 2024-09-21T09:33:50.484 INFO:tasks.rados.rados.0.smithi022.stdout:815: got expected ENOENT (src dne) 2024-09-21T09:33:50.484 INFO:tasks.rados.rados.0.smithi022.stdout:815: finishing copy_from racing read to smithi02284960-46 2024-09-21T09:33:50.489 INFO:tasks.rados.rados.0.smithi022.stdout:815: done (2 left) 2024-09-21T09:33:50.489 INFO:tasks.rados.rados.0.smithi022.stdout:816: done (1 left) 2024-09-21T09:33:50.489 INFO:tasks.rados.rados.0.smithi022.stdout:817: snap_remove snap 64 2024-09-21T09:33:50.605 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:50 smithi022 ceph-mon[39525]: osdmap e219: 8 total, 8 up, 8 in 2024-09-21T09:33:50.605 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:33:50 smithi022 podman[88194]: 2024-09-21 09:33:50.309117853 +0000 UTC m=+0.124521570 container died 5bc449c0676ffbba20383552b4d91185ae197d22b0feaaa35cf8ba1ea83482b8 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, RELEASE=quincy-fbd1a6a, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-09-21T09:33:50.605 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:50 smithi022 ceph-mon[27763]: osdmap e219: 8 total, 8 up, 8 in 2024-09-21T09:33:50.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:50 smithi064 ceph-mon[31124]: osdmap e219: 8 total, 8 up, 8 in 2024-09-21T09:33:50.948 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:33:50 smithi022 podman[88194]: 2024-09-21 09:33:50.711776797 +0000 UTC m=+0.527180543 container cleanup 5bc449c0676ffbba20383552b4d91185ae197d22b0feaaa35cf8ba1ea83482b8 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y, RELEASE=quincy-fbd1a6a, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-09-21T09:33:50.948 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:33:50 smithi022 bash[88194]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y 2024-09-21T09:33:51.198 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:33:50 smithi022 podman[88213]: 2024-09-21 09:33:50.946756235 +0000 UTC m=+0.632038615 container remove 5bc449c0676ffbba20383552b4d91185ae197d22b0feaaa35cf8ba1ea83482b8 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, RELEASE=quincy-fbd1a6a, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0) 2024-09-21T09:33:51.198 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:33:50 smithi022 systemd[1]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490@mgr.y.service: Main process exited, code=exited, status=143/n/a 2024-09-21T09:33:51.198 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:33:51 smithi022 systemd[1]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490@mgr.y.service: Failed with result 'exit-code'. 2024-09-21T09:33:51.198 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:33:51 smithi022 systemd[1]: Stopped Ceph mgr.y for d65e801e-77fa-11ef-baf6-efdc52797490. 2024-09-21T09:33:51.198 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:33:51 smithi022 systemd[1]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490@mgr.y.service: Consumed 46.322s CPU time. 2024-09-21T09:33:51.489 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:33:51 smithi022 systemd[1]: Starting Ceph mgr.y for d65e801e-77fa-11ef-baf6-efdc52797490... 2024-09-21T09:33:51.496 INFO:tasks.rados.rados.0.smithi022.stdout:817: done (1 left) 2024-09-21T09:33:51.496 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 2 current snap is 81 2024-09-21T09:33:51.496 INFO:tasks.rados.rados.0.smithi022.stdout:818: seq_num 278 ranges {0=663552,663552=442368,1105920=720896,1826816=442368} 2024-09-21T09:33:51.500 INFO:tasks.rados.rados.0.smithi022.stdout:818: writing smithi02284960-2 from 0 to 663552 tid 1 2024-09-21T09:33:51.503 INFO:tasks.rados.rados.0.smithi022.stdout:818: writing smithi02284960-2 from 663552 to 1105920 tid 2 2024-09-21T09:33:51.509 INFO:tasks.rados.rados.0.smithi022.stdout:818: writing smithi02284960-2 from 1105920 to 1826816 tid 3 2024-09-21T09:33:51.512 INFO:tasks.rados.rados.0.smithi022.stdout:818: writing smithi02284960-2 from 1826816 to 2269184 tid 4 2024-09-21T09:33:51.512 INFO:tasks.rados.rados.0.smithi022.stdout:819: delete oid 5 current snap is 81 2024-09-21T09:33:51.513 INFO:tasks.rados.rados.0.smithi022.stdout:814: finishing copy_from racing read to smithi02284960-11 2024-09-21T09:33:51.513 INFO:tasks.rados.rados.0.smithi022.stdout:814: finishing copy_from to smithi02284960-11 2024-09-21T09:33:51.513 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 11 v 439 (ObjNum 247 snap 67 seq_num 247) dirty exists 2024-09-21T09:33:51.513 INFO:tasks.rados.rados.0.smithi022.stdout:818: finishing write tid 1 to smithi02284960-2 2024-09-21T09:33:51.513 INFO:tasks.rados.rados.0.smithi022.stdout:818: finishing write tid 2 to smithi02284960-2 2024-09-21T09:33:51.516 INFO:tasks.rados.rados.0.smithi022.stdout:818: finishing write tid 3 to smithi02284960-2 2024-09-21T09:33:51.516 INFO:tasks.rados.rados.0.smithi022.stdout:818: finishing write tid 4 to smithi02284960-2 2024-09-21T09:33:51.517 INFO:tasks.rados.rados.0.smithi022.stdout:814: done (2 left) 2024-09-21T09:33:51.517 INFO:tasks.rados.rados.0.smithi022.stdout:819: done (1 left) 2024-09-21T09:33:51.517 INFO:tasks.rados.rados.0.smithi022.stdout:820: snap_remove snap 77 2024-09-21T09:33:51.797 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:33:51 smithi022 podman[88347]: 2024-09-21 09:33:51.520034136 +0000 UTC m=+0.027622718 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:33:51.797 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:33:51 smithi022 podman[88347]: 2024-09-21 09:33:51.673602039 +0000 UTC m=+0.181190644 container create 6c5bee78f09b7a71fe7142f96cb10a468f15bf8e8a7f1a81a23e0a36feca7ca8 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, RELEASE=main-62da54b, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T09:33:51.798 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:51 smithi022 ceph-mon[39525]: osdmap e220: 8 total, 8 up, 8 in 2024-09-21T09:33:51.798 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:51 smithi022 ceph-mon[39525]: pgmap v87: 121 pgs: 3 active+clean+snaptrim_wait, 4 active+clean+snaptrim, 114 active+clean; 197 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:33:51.798 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:51 smithi022 ceph-mon[27763]: osdmap e220: 8 total, 8 up, 8 in 2024-09-21T09:33:51.798 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:51 smithi022 ceph-mon[27763]: pgmap v87: 121 pgs: 3 active+clean+snaptrim_wait, 4 active+clean+snaptrim, 114 active+clean; 197 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:33:51.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:51 smithi064 ceph-mon[31124]: osdmap e220: 8 total, 8 up, 8 in 2024-09-21T09:33:51.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:51 smithi064 ceph-mon[31124]: pgmap v87: 121 pgs: 3 active+clean+snaptrim_wait, 4 active+clean+snaptrim, 114 active+clean; 197 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:33:52.289 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:33:51 smithi022 podman[88347]: 2024-09-21 09:33:51.900910014 +0000 UTC m=+0.408498586 container init 6c5bee78f09b7a71fe7142f96cb10a468f15bf8e8a7f1a81a23e0a36feca7ca8 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, RELEASE=main-62da54b, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T09:33:52.289 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:33:51 smithi022 podman[88347]: 2024-09-21 09:33:51.905790229 +0000 UTC m=+0.413378793 container start 6c5bee78f09b7a71fe7142f96cb10a468f15bf8e8a7f1a81a23e0a36feca7ca8 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , ceph=True, RELEASE=main-62da54b, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2) 2024-09-21T09:33:52.289 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:33:51 smithi022 ceph-mgr[88384]: -- 172.21.15.22:0/3507095714 <== mon.2 v2:172.21.15.22:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x555bbba7f0e0 con 0x555bbba5c800 2024-09-21T09:33:52.289 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:33:52 smithi022 bash[88347]: 6c5bee78f09b7a71fe7142f96cb10a468f15bf8e8a7f1a81a23e0a36feca7ca8 2024-09-21T09:33:52.290 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:33:52 smithi022 systemd[1]: Started Ceph mgr.y for d65e801e-77fa-11ef-baf6-efdc52797490. 2024-09-21T09:33:52.506 INFO:tasks.rados.rados.0.smithi022.stdout:820: done (1 left) 2024-09-21T09:33:52.506 INFO:tasks.rados.rados.0.smithi022.stdout:821: read oid 9 snap -1 2024-09-21T09:33:52.506 INFO:tasks.rados.rados.0.smithi022.stdout:821: expect (ObjNum 137 snap 30 seq_num 137) 2024-09-21T09:33:52.506 INFO:tasks.rados.rados.0.smithi022.stdout:822: snap_remove snap 76 2024-09-21T09:33:52.552 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:52 smithi022 ceph-mon[27763]: osdmap e221: 8 total, 8 up, 8 in 2024-09-21T09:33:52.552 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:52 smithi022 ceph-mon[27763]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:52.552 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:52 smithi022 ceph-mon[27763]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:52.552 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:52 smithi022 ceph-mon[27763]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:33:52.552 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:52 smithi022 ceph-mon[27763]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:52.552 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:52 smithi022 ceph-mon[27763]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:52.552 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:52 smithi022 ceph-mon[39525]: osdmap e221: 8 total, 8 up, 8 in 2024-09-21T09:33:52.553 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:52 smithi022 ceph-mon[39525]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:52.553 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:52 smithi022 ceph-mon[39525]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:52.553 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:52 smithi022 ceph-mon[39525]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:33:52.553 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:52 smithi022 ceph-mon[39525]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:52.553 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:52 smithi022 ceph-mon[39525]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:52.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:52 smithi064 ceph-mon[31124]: osdmap e221: 8 total, 8 up, 8 in 2024-09-21T09:33:52.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:52 smithi064 ceph-mon[31124]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:52.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:52 smithi064 ceph-mon[31124]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:52.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:52 smithi064 ceph-mon[31124]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:33:52.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:52 smithi064 ceph-mon[31124]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:52.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:52 smithi064 ceph-mon[31124]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:53.507 INFO:tasks.rados.rados.0.smithi022.stdout:822: done (2 left) 2024-09-21T09:33:53.507 INFO:tasks.rados.rados.0.smithi022.stdout:823: read oid 47 snap -1 2024-09-21T09:33:53.508 INFO:tasks.rados.rados.0.smithi022.stdout:823: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-21T09:33:53.508 INFO:tasks.rados.rados.0.smithi022.stdout:824: snap_remove snap 53 2024-09-21T09:33:53.704 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:53 smithi022 ceph-mon[39525]: osdmap e222: 8 total, 8 up, 8 in 2024-09-21T09:33:53.704 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:53 smithi022 ceph-mon[39525]: pgmap v90: 121 pgs: 121 active+clean; 215 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 3.1 MiB/s rd, 4.8 MiB/s wr, 27 op/s 2024-09-21T09:33:53.704 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 21 09:33:53 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-alertmanager-a[87373]: ts=2024-09-21T09:33:53.508Z caller=notify.go:848 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] aggrGroup="{}/{}:{alertname=\"CephNodeDiskspaceWarning\"}" msg="Notify attempt failed, will retry later" attempts=1 err="Post \"\": dial tcp 172.21.15.22:8443: connect: connection refused" 2024-09-21T09:33:53.705 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:53 smithi022 ceph-mon[27763]: osdmap e222: 8 total, 8 up, 8 in 2024-09-21T09:33:53.705 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:53 smithi022 ceph-mon[27763]: pgmap v90: 121 pgs: 121 active+clean; 215 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 3.1 MiB/s rd, 4.8 MiB/s wr, 27 op/s 2024-09-21T09:33:53.821 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:53 smithi064 ceph-mon[31124]: osdmap e222: 8 total, 8 up, 8 in 2024-09-21T09:33:53.821 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:53 smithi064 ceph-mon[31124]: pgmap v90: 121 pgs: 121 active+clean; 215 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 3.1 MiB/s rd, 4.8 MiB/s wr, 27 op/s 2024-09-21T09:33:54.441 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:33:54 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[75413]: ts=2024-09-21T09:33:54.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:33:54.507 INFO:tasks.rados.rados.0.smithi022.stdout:824: done (3 left) 2024-09-21T09:33:54.507 INFO:tasks.rados.rados.0.smithi022.stdout:825: rollback oid 20 current snap is 81 2024-09-21T09:33:54.507 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 20 to 67 2024-09-21T09:33:54.507 INFO:tasks.rados.rados.0.smithi022.stdout:826: read oid 46 snap -1 2024-09-21T09:33:54.507 INFO:tasks.rados.rados.0.smithi022.stdout:826: expect (ObjNum 204 snap 56 seq_num 204) 2024-09-21T09:33:54.507 INFO:tasks.rados.rados.0.smithi022.stdout:827: rollback oid 32 current snap is 81 2024-09-21T09:33:54.507 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 32 to 62 2024-09-21T09:33:54.507 INFO:tasks.rados.rados.0.smithi022.stdout:828: copy_from oid 40 from oid 30 current snap is 81 2024-09-21T09:33:54.507 INFO:tasks.rados.rados.0.smithi022.stdout:818: finishing write tid 5 to smithi02284960-2 2024-09-21T09:33:54.507 INFO:tasks.rados.rados.0.smithi022.stdout:818: finishing write tid 6 to smithi02284960-2 2024-09-21T09:33:54.507 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 2 v 218 (ObjNum 278 snap 81 seq_num 278) dirty exists 2024-09-21T09:33:54.507 INFO:tasks.rados.rados.0.smithi022.stdout:818: left oid 2 (ObjNum 278 snap 81 seq_num 278) 2024-09-21T09:33:54.508 INFO:tasks.rados.rados.0.smithi022.stdout:821: expect (ObjNum 137 snap 30 seq_num 137) 2024-09-21T09:33:54.728 INFO:tasks.rados.rados.0.smithi022.stdout:823: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-21T09:33:54.730 INFO:tasks.rados.rados.0.smithi022.stdout:818: done (6 left) 2024-09-21T09:33:54.730 INFO:tasks.rados.rados.0.smithi022.stdout:821: done (5 left) 2024-09-21T09:33:54.730 INFO:tasks.rados.rados.0.smithi022.stdout:823: done (4 left) 2024-09-21T09:33:54.730 INFO:tasks.rados.rados.0.smithi022.stdout:829: delete oid 4 current snap is 81 2024-09-21T09:33:54.730 INFO:tasks.rados.rados.0.smithi022.stdout:828: finishing copy_from racing read to smithi02284960-40 2024-09-21T09:33:54.730 INFO:tasks.rados.rados.0.smithi022.stdout:826: expect (ObjNum 204 snap 56 seq_num 204) 2024-09-21T09:33:54.801 INFO:tasks.rados.rados.0.smithi022.stdout:825: finishing rollback tid 1 to smithi02284960-20 2024-09-21T09:33:54.801 INFO:tasks.rados.rados.0.smithi022.stdout:825: finishing rollback tid 2 to smithi02284960-20 2024-09-21T09:33:54.801 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 20 v 466 (ObjNum 132 snap 27 seq_num 132) dirty exists 2024-09-21T09:33:54.801 INFO:tasks.rados.rados.0.smithi022.stdout:827: finishing rollback tid 0 to smithi02284960-32 2024-09-21T09:33:54.801 INFO:tasks.rados.rados.0.smithi022.stdout:828: finishing copy_from to smithi02284960-40 2024-09-21T09:33:54.801 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 40 v 281 (ObjNum 214 snap 61 seq_num 214) dirty exists 2024-09-21T09:33:54.801 INFO:tasks.rados.rados.0.smithi022.stdout:827: finishing rollback tid 1 to smithi02284960-32 2024-09-21T09:33:54.801 INFO:tasks.rados.rados.0.smithi022.stdout:827: finishing rollback tid 2 to smithi02284960-32 2024-09-21T09:33:54.801 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 32 v 470 (ObjNum 143 snap 32 seq_num 143) dirty exists 2024-09-21T09:33:54.801 INFO:tasks.rados.rados.0.smithi022.stdout:825: done (4 left) 2024-09-21T09:33:54.802 INFO:tasks.rados.rados.0.smithi022.stdout:826: done (3 left) 2024-09-21T09:33:54.802 INFO:tasks.rados.rados.0.smithi022.stdout:827: done (2 left) 2024-09-21T09:33:54.802 INFO:tasks.rados.rados.0.smithi022.stdout:828: done (1 left) 2024-09-21T09:33:54.802 INFO:tasks.rados.rados.0.smithi022.stdout:829: done (0 left) 2024-09-21T09:33:54.802 INFO:tasks.rados.rados.0.smithi022.stdout:830: delete oid 15 current snap is 81 2024-09-21T09:33:54.803 INFO:tasks.rados.rados.0.smithi022.stdout:830: done (0 left) 2024-09-21T09:33:54.803 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 28 current snap is 81 2024-09-21T09:33:54.803 INFO:tasks.rados.rados.0.smithi022.stdout:831: seq_num 282 ranges {2605056=729088,3334144=425984,3760128=704512,4464640=475136} 2024-09-21T09:33:54.807 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:54 smithi022 ceph-mon[39525]: osdmap e223: 8 total, 8 up, 8 in 2024-09-21T09:33:54.807 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:54 smithi022 ceph-mon[27763]: osdmap e223: 8 total, 8 up, 8 in 2024-09-21T09:33:54.817 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:54 smithi064 ceph-mon[31124]: osdmap e223: 8 total, 8 up, 8 in 2024-09-21T09:33:54.821 INFO:tasks.rados.rados.0.smithi022.stdout:831: writing smithi02284960-28 from 2605056 to 3334144 tid 1 2024-09-21T09:33:54.824 INFO:tasks.rados.rados.0.smithi022.stdout:831: writing smithi02284960-28 from 3334144 to 3760128 tid 2 2024-09-21T09:33:54.828 INFO:tasks.rados.rados.0.smithi022.stdout:831: writing smithi02284960-28 from 3760128 to 4464640 tid 3 2024-09-21T09:33:54.831 INFO:tasks.rados.rados.0.smithi022.stdout:831: writing smithi02284960-28 from 4464640 to 4939776 tid 4 2024-09-21T09:33:54.831 INFO:tasks.rados.rados.0.smithi022.stdout:832: rmattr oid 34 current snap is 81 2024-09-21T09:33:54.831 INFO:tasks.rados.rados.0.smithi022.stdout:832: done (1 left) 2024-09-21T09:33:54.831 INFO:tasks.rados.rados.0.smithi022.stdout:833: snap_create 2024-09-21T09:33:54.832 INFO:tasks.rados.rados.0.smithi022.stdout:831: finishing write tid 1 to smithi02284960-28 2024-09-21T09:33:54.832 INFO:tasks.rados.rados.0.smithi022.stdout:831: finishing write tid 2 to smithi02284960-28 2024-09-21T09:33:54.832 INFO:tasks.rados.rados.0.smithi022.stdout:831: finishing write tid 3 to smithi02284960-28 2024-09-21T09:33:54.833 INFO:tasks.rados.rados.0.smithi022.stdout:831: finishing write tid 4 to smithi02284960-28 2024-09-21T09:33:54.834 INFO:tasks.rados.rados.0.smithi022.stdout:831: finishing write tid 5 to smithi02284960-28 2024-09-21T09:33:54.835 INFO:tasks.rados.rados.0.smithi022.stdout:831: finishing write tid 6 to smithi02284960-28 2024-09-21T09:33:54.835 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 28 v 351 (ObjNum 282 snap 81 seq_num 282) dirty exists 2024-09-21T09:33:54.835 INFO:tasks.rados.rados.0.smithi022.stdout:831: left oid 28 (ObjNum 282 snap 81 seq_num 282) 2024-09-21T09:33:55.451 INFO:tasks.rados.rados.0.smithi022.stdout:831: done (1 left) 2024-09-21T09:33:55.451 INFO:tasks.rados.rados.0.smithi022.stdout:833: done (0 left) 2024-09-21T09:33:55.451 INFO:tasks.rados.rados.0.smithi022.stdout:834: rmattr oid 35 current snap is 82 2024-09-21T09:33:55.451 INFO:tasks.rados.rados.0.smithi022.stdout:834: done (0 left) 2024-09-21T09:33:55.451 INFO:tasks.rados.rados.0.smithi022.stdout:835: read oid 27 snap -1 2024-09-21T09:33:55.451 INFO:tasks.rados.rados.0.smithi022.stdout:835: expect deleted 2024-09-21T09:33:55.451 INFO:tasks.rados.rados.0.smithi022.stdout:836: setattr oid 18 current snap is 82 2024-09-21T09:33:55.452 INFO:tasks.rados.rados.0.smithi022.stdout:837: rollback oid 25 current snap is 82 2024-09-21T09:33:55.452 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 25 to 59 2024-09-21T09:33:55.452 INFO:tasks.rados.rados.0.smithi022.stdout:838: snap_remove snap 71 2024-09-21T09:33:55.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:55 smithi022 ceph-mon[39525]: osdmap e224: 8 total, 8 up, 8 in 2024-09-21T09:33:55.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:55 smithi022 ceph-mon[39525]: pgmap v93: 121 pgs: 121 active+clean; 215 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 3.1 MiB/s rd, 4.8 MiB/s wr, 27 op/s 2024-09-21T09:33:55.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:55 smithi022 ceph-mon[39525]: osdmap e225: 8 total, 8 up, 8 in 2024-09-21T09:33:55.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:55 smithi022 ceph-mon[27763]: osdmap e224: 8 total, 8 up, 8 in 2024-09-21T09:33:55.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:55 smithi022 ceph-mon[27763]: pgmap v93: 121 pgs: 121 active+clean; 215 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 3.1 MiB/s rd, 4.8 MiB/s wr, 27 op/s 2024-09-21T09:33:55.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:55 smithi022 ceph-mon[27763]: osdmap e225: 8 total, 8 up, 8 in 2024-09-21T09:33:55.874 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:55 smithi064 ceph-mon[31124]: osdmap e224: 8 total, 8 up, 8 in 2024-09-21T09:33:55.875 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:55 smithi064 ceph-mon[31124]: pgmap v93: 121 pgs: 121 active+clean; 215 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 3.1 MiB/s rd, 4.8 MiB/s wr, 27 op/s 2024-09-21T09:33:55.875 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:55 smithi064 ceph-mon[31124]: osdmap e225: 8 total, 8 up, 8 in 2024-09-21T09:33:55.875 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:33:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[75413]: ts=2024-09-21T09:33:55.741Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:33:55.875 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:33:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[75413]: ts=2024-09-21T09:33:55.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:33:56.452 INFO:tasks.rados.rados.0.smithi022.stdout:838: done (3 left) 2024-09-21T09:33:56.452 INFO:tasks.rados.rados.0.smithi022.stdout:839: snap_create 2024-09-21T09:33:56.453 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 18 v 264 (ObjNum 275 snap 80 seq_num 275) dirty exists 2024-09-21T09:33:56.453 INFO:tasks.rados.rados.0.smithi022.stdout:837: finishing rollback tid 0 to smithi02284960-25 2024-09-21T09:33:56.453 INFO:tasks.rados.rados.0.smithi022.stdout:837: finishing rollback tid 1 to smithi02284960-25 2024-09-21T09:33:56.453 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 25 v 482 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-09-21T09:33:57.191 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:33:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[67503]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:33:56] "GET /metrics HTTP/1.1" 200 36133 "" "Prometheus/2.51.0" 2024-09-21T09:33:57.192 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:33:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[75413]: ts=2024-09-21T09:33:56.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi022\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:33:57.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:56 smithi064 ceph-mon[31124]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:57.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:56 smithi064 ceph-mon[31124]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:57.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:56 smithi064 ceph-mon[31124]: osdmap e226: 8 total, 8 up, 8 in 2024-09-21T09:33:57.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:56 smithi064 ceph-mon[31124]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:33:57.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:56 smithi064 ceph-mon[31124]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:57.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:56 smithi064 ceph-mon[31124]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:57.247 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:56 smithi022 ceph-mon[39525]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:57.247 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:56 smithi022 ceph-mon[39525]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:57.247 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:56 smithi022 ceph-mon[39525]: osdmap e226: 8 total, 8 up, 8 in 2024-09-21T09:33:57.247 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:56 smithi022 ceph-mon[39525]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:33:57.247 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:56 smithi022 ceph-mon[39525]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:57.247 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:56 smithi022 ceph-mon[39525]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:57.247 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:56 smithi022 ceph-mon[27763]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:57.247 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:56 smithi022 ceph-mon[27763]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:57.248 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:56 smithi022 ceph-mon[27763]: osdmap e226: 8 total, 8 up, 8 in 2024-09-21T09:33:57.248 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:56 smithi022 ceph-mon[27763]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:33:57.248 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:56 smithi022 ceph-mon[27763]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:57.248 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:56 smithi022 ceph-mon[27763]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:57.452 INFO:tasks.rados.rados.0.smithi022.stdout:835: done (3 left) 2024-09-21T09:33:57.453 INFO:tasks.rados.rados.0.smithi022.stdout:836: done (2 left) 2024-09-21T09:33:57.453 INFO:tasks.rados.rados.0.smithi022.stdout:837: done (1 left) 2024-09-21T09:33:57.453 INFO:tasks.rados.rados.0.smithi022.stdout:839: done (0 left) 2024-09-21T09:33:57.453 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 1 current snap is 83 2024-09-21T09:33:57.453 INFO:tasks.rados.rados.0.smithi022.stdout:840: seq_num 283 ranges {1818624=745472,2564096=712704,3276800=458752,3735552=434176} 2024-09-21T09:33:57.466 INFO:tasks.rados.rados.0.smithi022.stdout:840: writing smithi02284960-1 from 1818624 to 2564096 tid 1 2024-09-21T09:33:57.470 INFO:tasks.rados.rados.0.smithi022.stdout:840: writing smithi02284960-1 from 2564096 to 3276800 tid 2 2024-09-21T09:33:57.473 INFO:tasks.rados.rados.0.smithi022.stdout:840: writing smithi02284960-1 from 3276800 to 3735552 tid 3 2024-09-21T09:33:57.476 INFO:tasks.rados.rados.0.smithi022.stdout:840: writing smithi02284960-1 from 3735552 to 4169728 tid 4 2024-09-21T09:33:57.476 INFO:tasks.rados.rados.0.smithi022.stdout:841: copy_from oid 48 from oid 18 current snap is 83 2024-09-21T09:33:57.476 INFO:tasks.rados.rados.0.smithi022.stdout:840: finishing write tid 1 to smithi02284960-1 2024-09-21T09:33:57.476 INFO:tasks.rados.rados.0.smithi022.stdout:840: finishing write tid 2 to smithi02284960-1 2024-09-21T09:33:57.476 INFO:tasks.rados.rados.0.smithi022.stdout:842: read oid 33 snap -1 2024-09-21T09:33:57.477 INFO:tasks.rados.rados.0.smithi022.stdout:842: expect deleted 2024-09-21T09:33:57.477 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 7 current snap is 83 2024-09-21T09:33:57.477 INFO:tasks.rados.rados.0.smithi022.stdout:843: seq_num 284 ranges {1974272=761856,2736128=598016,3334144=671744,4005888=335872} 2024-09-21T09:33:57.492 INFO:tasks.rados.rados.0.smithi022.stdout:843: writing smithi02284960-7 from 1974272 to 2736128 tid 1 2024-09-21T09:33:57.496 INFO:tasks.rados.rados.0.smithi022.stdout:843: writing smithi02284960-7 from 2736128 to 3334144 tid 2 2024-09-21T09:33:57.500 INFO:tasks.rados.rados.0.smithi022.stdout:843: writing smithi02284960-7 from 3334144 to 4005888 tid 3 2024-09-21T09:33:57.503 INFO:tasks.rados.rados.0.smithi022.stdout:843: writing smithi02284960-7 from 4005888 to 4341760 tid 4 2024-09-21T09:33:57.503 INFO:tasks.rados.rados.0.smithi022.stdout:844: copy_from oid 19 from oid 22 current snap is 83 2024-09-21T09:33:57.503 INFO:tasks.rados.rados.0.smithi022.stdout:840: finishing write tid 3 to smithi02284960-1 2024-09-21T09:33:57.503 INFO:tasks.rados.rados.0.smithi022.stdout:840: finishing write tid 4 to smithi02284960-1 2024-09-21T09:33:57.503 INFO:tasks.rados.rados.0.smithi022.stdout:840: finishing write tid 5 to smithi02284960-1 2024-09-21T09:33:57.503 INFO:tasks.rados.rados.0.smithi022.stdout:840: finishing write tid 6 to smithi02284960-1 2024-09-21T09:33:57.503 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 1 v 444 (ObjNum 283 snap 83 seq_num 283) dirty exists 2024-09-21T09:33:57.503 INFO:tasks.rados.rados.0.smithi022.stdout:840: left oid 1 (ObjNum 283 snap 83 seq_num 283) 2024-09-21T09:33:57.503 INFO:tasks.rados.rados.0.smithi022.stdout:841: finishing copy_from racing read to smithi02284960-48 2024-09-21T09:33:57.503 INFO:tasks.rados.rados.0.smithi022.stdout:841: finishing copy_from to smithi02284960-48 2024-09-21T09:33:57.504 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 48 v 264 (ObjNum 275 snap 80 seq_num 275) dirty exists 2024-09-21T09:33:57.504 INFO:tasks.rados.rados.0.smithi022.stdout:843: finishing write tid 1 to smithi02284960-7 2024-09-21T09:33:57.504 INFO:tasks.rados.rados.0.smithi022.stdout:843: finishing write tid 2 to smithi02284960-7 2024-09-21T09:33:57.504 INFO:tasks.rados.rados.0.smithi022.stdout:840: done (4 left) 2024-09-21T09:33:57.504 INFO:tasks.rados.rados.0.smithi022.stdout:841: done (3 left) 2024-09-21T09:33:57.504 INFO:tasks.rados.rados.0.smithi022.stdout:842: done (2 left) 2024-09-21T09:33:57.504 INFO:tasks.rados.rados.0.smithi022.stdout:845: read oid 23 snap -1 2024-09-21T09:33:57.504 INFO:tasks.rados.rados.0.smithi022.stdout:845: expect (ObjNum 262 snap 73 seq_num 262) 2024-09-21T09:33:57.504 INFO:tasks.rados.rados.0.smithi022.stdout:846: delete oid 9 current snap is 83 2024-09-21T09:33:57.504 INFO:tasks.rados.rados.0.smithi022.stdout:844: finishing copy_from to smithi02284960-19 2024-09-21T09:33:57.504 INFO:tasks.rados.rados.0.smithi022.stdout:844: got expected ENOENT (src dne) 2024-09-21T09:33:57.505 INFO:tasks.rados.rados.0.smithi022.stdout:844: finishing copy_from racing read to smithi02284960-19 2024-09-21T09:33:57.505 INFO:tasks.rados.rados.0.smithi022.stdout:844: done (3 left) 2024-09-21T09:33:57.505 INFO:tasks.rados.rados.0.smithi022.stdout:846: done (2 left) 2024-09-21T09:33:57.505 INFO:tasks.rados.rados.0.smithi022.stdout:847: snap_remove snap 80 2024-09-21T09:33:57.808 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:33:57 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: [21/Sep/2024:09:33:57] ENGINE Bus STARTING 2024-09-21T09:33:57.808 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:33:57 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: CherryPy Checker: 2024-09-21T09:33:57.808 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:33:57 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: The Application mounted at '' has an empty config. 2024-09-21T09:33:57.808 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:33:57 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: 2024-09-21T09:33:57.808 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:33:57 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: [21/Sep/2024:09:33:57] ENGINE Serving on http://:::9283 2024-09-21T09:33:57.808 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:33:57 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: [21/Sep/2024:09:33:57] ENGINE Bus STARTED 2024-09-21T09:33:58.261 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 21 09:33:58 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-alertmanager-a[87373]: ts=2024-09-21T09:33:58.147Z caller=notify.go:860 level=info component=dispatcher receiver=ceph-dashboard integration=webhook[1] aggrGroup="{}/{}:{alertname=\"CephNodeDiskspaceWarning\"}" msg="Notify success" attempts=5 duration=5.962402ms 2024-09-21T09:33:58.261 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:57 smithi022 ceph-mon[39525]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:58.261 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:57 smithi022 ceph-mon[39525]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:58.261 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:57 smithi022 ceph-mon[39525]: pgmap v96: 121 pgs: 121 active+clean; 216 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.1 MiB/s rd, 1.4 MiB/s wr, 13 op/s 2024-09-21T09:33:58.261 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:57 smithi022 ceph-mon[39525]: osdmap e227: 8 total, 8 up, 8 in 2024-09-21T09:33:58.261 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:57 smithi022 ceph-mon[39525]: Standby manager daemon y restarted 2024-09-21T09:33:58.261 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:57 smithi022 ceph-mon[39525]: Standby manager daemon y started 2024-09-21T09:33:58.261 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:57 smithi022 ceph-mon[39525]: from='mgr.? 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-21T09:33:58.262 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:57 smithi022 ceph-mon[39525]: from='mgr.? 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:33:58.262 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:57 smithi022 ceph-mon[39525]: from='mgr.? 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-21T09:33:58.262 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:57 smithi022 ceph-mon[39525]: from='mgr.? 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:33:58.262 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:57 smithi022 ceph-mon[27763]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:58.262 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:57 smithi022 ceph-mon[27763]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:58.262 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:57 smithi022 ceph-mon[27763]: pgmap v96: 121 pgs: 121 active+clean; 216 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.1 MiB/s rd, 1.4 MiB/s wr, 13 op/s 2024-09-21T09:33:58.262 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:57 smithi022 ceph-mon[27763]: osdmap e227: 8 total, 8 up, 8 in 2024-09-21T09:33:58.262 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:57 smithi022 ceph-mon[27763]: Standby manager daemon y restarted 2024-09-21T09:33:58.262 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:57 smithi022 ceph-mon[27763]: Standby manager daemon y started 2024-09-21T09:33:58.262 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:57 smithi022 ceph-mon[27763]: from='mgr.? 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-21T09:33:58.262 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:57 smithi022 ceph-mon[27763]: from='mgr.? 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:33:58.262 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:57 smithi022 ceph-mon[27763]: from='mgr.? 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-21T09:33:58.263 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:57 smithi022 ceph-mon[27763]: from='mgr.? 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:33:58.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:57 smithi064 ceph-mon[31124]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:58.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:57 smithi064 ceph-mon[31124]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:33:58.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:57 smithi064 ceph-mon[31124]: pgmap v96: 121 pgs: 121 active+clean; 216 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.1 MiB/s rd, 1.4 MiB/s wr, 13 op/s 2024-09-21T09:33:58.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:57 smithi064 ceph-mon[31124]: osdmap e227: 8 total, 8 up, 8 in 2024-09-21T09:33:58.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:57 smithi064 ceph-mon[31124]: Standby manager daemon y restarted 2024-09-21T09:33:58.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:57 smithi064 ceph-mon[31124]: Standby manager daemon y started 2024-09-21T09:33:58.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:57 smithi064 ceph-mon[31124]: from='mgr.? 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-21T09:33:58.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:57 smithi064 ceph-mon[31124]: from='mgr.? 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:33:58.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:57 smithi064 ceph-mon[31124]: from='mgr.? 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-21T09:33:58.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:57 smithi064 ceph-mon[31124]: from='mgr.? 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:33:58.454 INFO:tasks.rados.rados.0.smithi022.stdout:847: done (2 left) 2024-09-21T09:33:58.454 INFO:tasks.rados.rados.0.smithi022.stdout:848: snap_remove snap 79 2024-09-21T09:33:59.456 INFO:tasks.rados.rados.0.smithi022.stdout:848: done (2 left) 2024-09-21T09:33:59.456 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 14 current snap is 83 2024-09-21T09:33:59.456 INFO:tasks.rados.rados.0.smithi022.stdout:849: seq_num 286 ranges {16384=794624,811008=761856,1572864=679936,2252800=163840} 2024-09-21T09:33:59.461 INFO:tasks.rados.rados.0.smithi022.stdout:849: writing smithi02284960-14 from 16384 to 811008 tid 1 2024-09-21T09:33:59.466 INFO:tasks.rados.rados.0.smithi022.stdout:849: writing smithi02284960-14 from 811008 to 1572864 tid 2 2024-09-21T09:33:59.469 INFO:tasks.rados.rados.0.smithi022.stdout:849: writing smithi02284960-14 from 1572864 to 2252800 tid 3 2024-09-21T09:33:59.471 INFO:tasks.rados.rados.0.smithi022.stdout:849: writing smithi02284960-14 from 2252800 to 2416640 tid 4 2024-09-21T09:33:59.471 INFO:tasks.rados.rados.0.smithi022.stdout:850: read oid 38 snap -1 2024-09-21T09:33:59.471 INFO:tasks.rados.rados.0.smithi022.stdout:850: expect (ObjNum 142 snap 32 seq_num 142) 2024-09-21T09:33:59.471 INFO:tasks.rados.rados.0.smithi022.stdout:851: rollback oid 42 current snap is 83 2024-09-21T09:33:59.471 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 42 to 81 2024-09-21T09:33:59.471 INFO:tasks.rados.rados.0.smithi022.stdout:843: finishing write tid 3 to smithi02284960-7 2024-09-21T09:33:59.471 INFO:tasks.rados.rados.0.smithi022.stdout:843: finishing write tid 4 to smithi02284960-7 2024-09-21T09:33:59.471 INFO:tasks.rados.rados.0.smithi022.stdout:843: finishing write tid 5 to smithi02284960-7 2024-09-21T09:33:59.471 INFO:tasks.rados.rados.0.smithi022.stdout:843: finishing write tid 6 to smithi02284960-7 2024-09-21T09:33:59.471 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 7 v 492 (ObjNum 284 snap 83 seq_num 284) dirty exists 2024-09-21T09:33:59.471 INFO:tasks.rados.rados.0.smithi022.stdout:843: left oid 7 (ObjNum 284 snap 83 seq_num 284) 2024-09-21T09:33:59.472 INFO:tasks.rados.rados.0.smithi022.stdout:845: expect (ObjNum 262 snap 73 seq_num 262) 2024-09-21T09:33:59.621 INFO:tasks.rados.rados.0.smithi022.stdout:849: finishing write tid 1 to smithi02284960-14 2024-09-21T09:33:59.622 INFO:tasks.rados.rados.0.smithi022.stdout:849: finishing write tid 2 to smithi02284960-14 2024-09-21T09:33:59.622 INFO:tasks.rados.rados.0.smithi022.stdout:849: finishing write tid 3 to smithi02284960-14 2024-09-21T09:33:59.622 INFO:tasks.rados.rados.0.smithi022.stdout:849: finishing write tid 4 to smithi02284960-14 2024-09-21T09:33:59.622 INFO:tasks.rados.rados.0.smithi022.stdout:851: finishing rollback tid 1 to smithi02284960-42 2024-09-21T09:33:59.622 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 42 v 0 (ObjNum 225 snap 0 seq_num 3569800592) dirty dne 2024-09-21T09:33:59.622 INFO:tasks.rados.rados.0.smithi022.stdout:849: finishing write tid 5 to smithi02284960-14 2024-09-21T09:33:59.622 INFO:tasks.rados.rados.0.smithi022.stdout:843: done (4 left) 2024-09-21T09:33:59.622 INFO:tasks.rados.rados.0.smithi022.stdout:845: done (3 left) 2024-09-21T09:33:59.622 INFO:tasks.rados.rados.0.smithi022.stdout:851: done (2 left) 2024-09-21T09:33:59.622 INFO:tasks.rados.rados.0.smithi022.stdout:852: rollback oid 17 current snap is 83 2024-09-21T09:33:59.622 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 17 to 66 2024-09-21T09:33:59.623 INFO:tasks.rados.rados.0.smithi022.stdout:849: finishing write tid 6 to smithi02284960-14 2024-09-21T09:33:59.623 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 14 v 253 (ObjNum 286 snap 83 seq_num 286) dirty exists 2024-09-21T09:33:59.623 INFO:tasks.rados.rados.0.smithi022.stdout:849: left oid 14 (ObjNum 286 snap 83 seq_num 286) 2024-09-21T09:33:59.623 INFO:tasks.rados.rados.0.smithi022.stdout:850: expect (ObjNum 142 snap 32 seq_num 142) 2024-09-21T09:33:59.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:59 smithi022 ceph-mon[39525]: osdmap e228: 8 total, 8 up, 8 in 2024-09-21T09:33:59.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:59 smithi022 ceph-mon[39525]: mgrmap e28: x(active, since 63s), standbys: y 2024-09-21T09:33:59.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:33:59 smithi022 ceph-mon[39525]: pgmap v99: 121 pgs: 121 active+clean; 216 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.1 MiB/s rd, 1.4 MiB/s wr, 13 op/s 2024-09-21T09:33:59.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:59 smithi022 ceph-mon[27763]: osdmap e228: 8 total, 8 up, 8 in 2024-09-21T09:33:59.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:59 smithi022 ceph-mon[27763]: mgrmap e28: x(active, since 63s), standbys: y 2024-09-21T09:33:59.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:33:59 smithi022 ceph-mon[27763]: pgmap v99: 121 pgs: 121 active+clean; 216 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.1 MiB/s rd, 1.4 MiB/s wr, 13 op/s 2024-09-21T09:33:59.919 INFO:tasks.rados.rados.0.smithi022.stdout:852: finishing rollback tid 1 to smithi02284960-17 2024-09-21T09:33:59.919 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 17 v 287 (ObjNum 289 snap 0 seq_num 1) dirty dne 2024-09-21T09:33:59.919 INFO:tasks.rados.rados.0.smithi022.stdout:849: done (2 left) 2024-09-21T09:33:59.919 INFO:tasks.rados.rados.0.smithi022.stdout:850: done (1 left) 2024-09-21T09:33:59.919 INFO:tasks.rados.rados.0.smithi022.stdout:852: done (0 left) 2024-09-21T09:33:59.919 INFO:tasks.rados.rados.0.smithi022.stdout:853: read oid 22 snap -1 2024-09-21T09:33:59.920 INFO:tasks.rados.rados.0.smithi022.stdout:853: expect deleted 2024-09-21T09:33:59.920 INFO:tasks.rados.rados.0.smithi022.stdout:854: read oid 2 snap 66 2024-09-21T09:33:59.920 INFO:tasks.rados.rados.0.smithi022.stdout:854: expect deleted 2024-09-21T09:33:59.920 INFO:tasks.rados.rados.0.smithi022.stdout:855: snap_create 2024-09-21T09:33:59.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:59 smithi064 ceph-mon[31124]: osdmap e228: 8 total, 8 up, 8 in 2024-09-21T09:33:59.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:59 smithi064 ceph-mon[31124]: mgrmap e28: x(active, since 63s), standbys: y 2024-09-21T09:33:59.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:33:59 smithi064 ceph-mon[31124]: pgmap v99: 121 pgs: 121 active+clean; 216 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.1 MiB/s rd, 1.4 MiB/s wr, 13 op/s 2024-09-21T09:34:00.112 INFO:tasks.rados.rados.0.smithi022.stdout:853: done (2 left) 2024-09-21T09:34:00.112 INFO:tasks.rados.rados.0.smithi022.stdout:854: done (1 left) 2024-09-21T09:34:00.112 INFO:tasks.rados.rados.0.smithi022.stdout:855: done (0 left) 2024-09-21T09:34:00.112 INFO:tasks.rados.rados.0.smithi022.stdout:856: read oid 27 snap -1 2024-09-21T09:34:00.112 INFO:tasks.rados.rados.0.smithi022.stdout:856: expect deleted 2024-09-21T09:34:00.112 INFO:tasks.rados.rados.0.smithi022.stdout:857: snap_remove snap 66 2024-09-21T09:34:00.307 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:34:00 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: [21/Sep/2024:09:34:00] ENGINE Bus STOPPING 2024-09-21T09:34:00.441 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:34:00 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[75413]: ts=2024-09-21T09:34:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:34:00.441 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:34:00 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[67503]: 2024-09-21T09:34:00.110+0000 7f1f89855640 -1 mgr handle_mgr_map I was active but no longer am 2024-09-21T09:34:00.442 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:34:00 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[67503]: ignoring --setuser ceph since I am not root 2024-09-21T09:34:00.442 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:34:00 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[67503]: ignoring --setgroup ceph since I am not root 2024-09-21T09:34:00.442 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:34:00 smithi064 ceph-mgr[67526]: -- 172.21.15.64:0/1583560745 <== mon.1 v2:172.21.15.64:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55c955c3d0e0 con 0x55c955c1b000 2024-09-21T09:34:00.452 INFO:tasks.rados.rados.0.smithi022.stdout:857: done (1 left) 2024-09-21T09:34:00.452 INFO:tasks.rados.rados.0.smithi022.stdout:858: snap_create 2024-09-21T09:34:00.808 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:34:00 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: [21/Sep/2024:09:34:00] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-21T09:34:00.808 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:34:00 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: [21/Sep/2024:09:34:00] ENGINE Bus STOPPED 2024-09-21T09:34:00.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:00 smithi022 ceph-mon[39525]: osdmap e229: 8 total, 8 up, 8 in 2024-09-21T09:34:00.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:00 smithi022 ceph-mon[39525]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:34:00.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:00 smithi022 ceph-mon[39525]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:34:00.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:00 smithi022 ceph-mon[39525]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:34:00.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:00 smithi022 ceph-mon[39525]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:34:00.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:00 smithi022 ceph-mon[39525]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:34:00.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:00 smithi022 ceph-mon[39525]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:34:00.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:00 smithi022 ceph-mon[39525]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-21T09:34:00.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:00 smithi022 ceph-mon[39525]: from='mgr.24775 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-21T09:34:00.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:00 smithi022 ceph-mon[39525]: Activating manager daemon y 2024-09-21T09:34:00.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:00 smithi022 ceph-mon[39525]: osdmap e230: 8 total, 8 up, 8 in 2024-09-21T09:34:00.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:00 smithi022 ceph-mon[39525]: from='mgr.24775 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-09-21T09:34:00.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:00 smithi022 ceph-mon[39525]: mgrmap e29: y(active, starting, since 0.00556727s) 2024-09-21T09:34:00.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:00 smithi022 ceph-mon[39525]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:34:00.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:00 smithi022 ceph-mon[39525]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:34:00.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:00 smithi022 ceph-mon[39525]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:34:00.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:00 smithi022 ceph-mon[39525]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-09-21T09:34:00.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:00 smithi022 ceph-mon[39525]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-09-21T09:34:00.810 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:00 smithi022 ceph-mon[39525]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-21T09:34:00.810 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:00 smithi022 ceph-mon[39525]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:34:00.810 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:00 smithi022 ceph-mon[39525]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:34:00.810 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:00 smithi022 ceph-mon[39525]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:34:00.810 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:00 smithi022 ceph-mon[39525]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:34:00.810 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:00 smithi022 ceph-mon[39525]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:34:00.810 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:00 smithi022 ceph-mon[39525]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:34:00.810 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:00 smithi022 ceph-mon[39525]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:34:00.810 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:00 smithi022 ceph-mon[39525]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:34:00.810 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:00 smithi022 ceph-mon[39525]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T09:34:00.810 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:00 smithi022 ceph-mon[39525]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T09:34:00.810 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:00 smithi022 ceph-mon[39525]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T09:34:00.810 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:00 smithi022 ceph-mon[39525]: osdmap e231: 8 total, 8 up, 8 in 2024-09-21T09:34:00.811 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:00 smithi022 ceph-mon[27763]: osdmap e229: 8 total, 8 up, 8 in 2024-09-21T09:34:00.811 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:00 smithi022 ceph-mon[27763]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:34:00.811 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:00 smithi022 ceph-mon[27763]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:34:00.811 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:00 smithi022 ceph-mon[27763]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:34:00.811 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:00 smithi022 ceph-mon[27763]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:34:00.811 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:00 smithi022 ceph-mon[27763]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:34:00.811 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:00 smithi022 ceph-mon[27763]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:34:00.811 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:00 smithi022 ceph-mon[27763]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-21T09:34:00.811 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:00 smithi022 ceph-mon[27763]: from='mgr.24775 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-21T09:34:00.811 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:00 smithi022 ceph-mon[27763]: Activating manager daemon y 2024-09-21T09:34:00.811 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:00 smithi022 ceph-mon[27763]: osdmap e230: 8 total, 8 up, 8 in 2024-09-21T09:34:00.811 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:00 smithi022 ceph-mon[27763]: from='mgr.24775 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-09-21T09:34:00.811 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:00 smithi022 ceph-mon[27763]: mgrmap e29: y(active, starting, since 0.00556727s) 2024-09-21T09:34:00.811 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:00 smithi022 ceph-mon[27763]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:34:00.812 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:00 smithi022 ceph-mon[27763]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:34:00.812 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:00 smithi022 ceph-mon[27763]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:34:00.812 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:00 smithi022 ceph-mon[27763]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-09-21T09:34:00.812 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:00 smithi022 ceph-mon[27763]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-09-21T09:34:00.812 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:00 smithi022 ceph-mon[27763]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-21T09:34:00.812 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:00 smithi022 ceph-mon[27763]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:34:00.812 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:00 smithi022 ceph-mon[27763]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:34:00.812 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:00 smithi022 ceph-mon[27763]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:34:00.812 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:00 smithi022 ceph-mon[27763]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:34:00.812 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:00 smithi022 ceph-mon[27763]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:34:00.812 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:00 smithi022 ceph-mon[27763]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:34:00.812 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:00 smithi022 ceph-mon[27763]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:34:00.812 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:00 smithi022 ceph-mon[27763]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:34:00.812 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:00 smithi022 ceph-mon[27763]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T09:34:00.812 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:00 smithi022 ceph-mon[27763]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T09:34:00.813 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:00 smithi022 ceph-mon[27763]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T09:34:00.813 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:00 smithi022 ceph-mon[27763]: osdmap e231: 8 total, 8 up, 8 in 2024-09-21T09:34:00.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:00 smithi064 ceph-mon[31124]: osdmap e229: 8 total, 8 up, 8 in 2024-09-21T09:34:00.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:00 smithi064 ceph-mon[31124]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:34:00.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:00 smithi064 ceph-mon[31124]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:34:00.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:00 smithi064 ceph-mon[31124]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:34:00.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:00 smithi064 ceph-mon[31124]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:34:00.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:00 smithi064 ceph-mon[31124]: from='mgr.24775 ' entity='mgr.x' 2024-09-21T09:34:00.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:00 smithi064 ceph-mon[31124]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:34:00.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:00 smithi064 ceph-mon[31124]: from='mgr.24775 172.21.15.64:0/2292065402' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-21T09:34:00.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:00 smithi064 ceph-mon[31124]: from='mgr.24775 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-21T09:34:00.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:00 smithi064 ceph-mon[31124]: Activating manager daemon y 2024-09-21T09:34:00.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:00 smithi064 ceph-mon[31124]: osdmap e230: 8 total, 8 up, 8 in 2024-09-21T09:34:00.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:00 smithi064 ceph-mon[31124]: from='mgr.24775 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-09-21T09:34:00.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:00 smithi064 ceph-mon[31124]: mgrmap e29: y(active, starting, since 0.00556727s) 2024-09-21T09:34:00.943 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:00 smithi064 ceph-mon[31124]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:34:00.943 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:00 smithi064 ceph-mon[31124]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:34:00.943 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:00 smithi064 ceph-mon[31124]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:34:00.943 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:00 smithi064 ceph-mon[31124]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-09-21T09:34:00.943 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:00 smithi064 ceph-mon[31124]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-09-21T09:34:00.943 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:00 smithi064 ceph-mon[31124]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-21T09:34:00.943 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:00 smithi064 ceph-mon[31124]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:34:00.943 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:00 smithi064 ceph-mon[31124]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:34:00.943 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:00 smithi064 ceph-mon[31124]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:34:00.943 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:00 smithi064 ceph-mon[31124]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:34:00.943 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:00 smithi064 ceph-mon[31124]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:34:00.943 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:00 smithi064 ceph-mon[31124]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:34:00.943 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:00 smithi064 ceph-mon[31124]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:34:00.944 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:00 smithi064 ceph-mon[31124]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:34:00.944 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:00 smithi064 ceph-mon[31124]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T09:34:00.944 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:00 smithi064 ceph-mon[31124]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T09:34:00.944 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:00 smithi064 ceph-mon[31124]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T09:34:00.944 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:00 smithi064 ceph-mon[31124]: osdmap e231: 8 total, 8 up, 8 in 2024-09-21T09:34:01.452 INFO:tasks.rados.rados.0.smithi022.stdout:856: done (1 left) 2024-09-21T09:34:01.452 INFO:tasks.rados.rados.0.smithi022.stdout:858: done (0 left) 2024-09-21T09:34:01.452 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 48 current snap is 85 2024-09-21T09:34:01.452 INFO:tasks.rados.rados.0.smithi022.stdout:859: seq_num 287 ranges {4218880=8192} 2024-09-21T09:34:01.457 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:34:01 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: 2024-09-21T09:34:01.117+0000 7f172e2c3640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:34:01.457 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:34:01 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: 2024-09-21T09:34:01.117+0000 7f172e2c3640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:34:01.457 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:34:01 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: 2024-09-21T09:34:01.117+0000 7f172e2c3640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:34:01.457 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:34:01 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: 2024-09-21T09:34:01.117+0000 7f172e2c3640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:34:01.457 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:34:01 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: 2024-09-21T09:34:01.117+0000 7f172e2c3640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:34:01.457 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:34:01 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: 2024-09-21T09:34:01.117+0000 7f172a2bb640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:34:01.457 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:34:01 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: 2024-09-21T09:34:01.117+0000 7f172a2bb640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:34:01.457 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:34:01 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: 2024-09-21T09:34:01.117+0000 7f172a2bb640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:34:01.457 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:34:01 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: 2024-09-21T09:34:01.117+0000 7f172a2bb640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:34:01.457 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:34:01 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: 2024-09-21T09:34:01.117+0000 7f172a2bb640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:34:01.476 INFO:tasks.rados.rados.0.smithi022.stdout:859: writing smithi02284960-48 from 4218880 to 4227072 tid 1 2024-09-21T09:34:01.476 INFO:tasks.rados.rados.0.smithi022.stdout:860: copy_from oid 23 from oid 35 current snap is 85 2024-09-21T09:34:01.476 INFO:tasks.rados.rados.0.smithi022.stdout:861: delete oid 34 current snap is 85 2024-09-21T09:34:01.483 INFO:tasks.rados.rados.0.smithi022.stdout:861: done (2 left) 2024-09-21T09:34:01.483 INFO:tasks.rados.rados.0.smithi022.stdout:862: snap_create 2024-09-21T09:34:01.485 INFO:tasks.rados.rados.0.smithi022.stdout:859: finishing write tid 1 to smithi02284960-48 2024-09-21T09:34:01.485 INFO:tasks.rados.rados.0.smithi022.stdout:859: finishing write tid 2 to smithi02284960-48 2024-09-21T09:34:01.487 INFO:teuthology.orchestra.run.smithi022.stdout:true 2024-09-21T09:34:01.489 INFO:tasks.rados.rados.0.smithi022.stdout:859: finishing write tid 3 to smithi02284960-48 2024-09-21T09:34:01.489 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 48 v 517 (ObjNum 287 snap 85 seq_num 287) dirty exists 2024-09-21T09:34:01.489 INFO:tasks.rados.rados.0.smithi022.stdout:859: left oid 48 (ObjNum 287 snap 85 seq_num 287) 2024-09-21T09:34:01.511 INFO:tasks.rados.rados.0.smithi022.stdout:860: finishing copy_from racing read to smithi02284960-23 2024-09-21T09:34:01.513 INFO:tasks.rados.rados.0.smithi022.stdout:860: finishing copy_from to smithi02284960-23 2024-09-21T09:34:01.513 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 23 v 346 (ObjNum 264 snap 76 seq_num 264) dirty exists 2024-09-21T09:34:01.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:01 smithi064 ceph-mon[31124]: Manager daemon y is now available 2024-09-21T09:34:01.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:01 smithi064 ceph-mon[31124]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:34:01.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:01 smithi064 ceph-mon[31124]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:34:01.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:01 smithi064 ceph-mon[31124]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T09:34:01.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:01 smithi064 ceph-mon[31124]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-21T09:34:01.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:01 smithi064 ceph-mon[31124]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-21T09:34:01.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:01 smithi064 ceph-mon[31124]: osdmap e232: 8 total, 8 up, 8 in 2024-09-21T09:34:01.808 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:34:01 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: [21/Sep/2024:09:34:01] ENGINE Bus STARTING 2024-09-21T09:34:01.808 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:34:01 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: [21/Sep/2024:09:34:01] ENGINE Serving on http://:::9283 2024-09-21T09:34:01.808 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:34:01 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: [21/Sep/2024:09:34:01] ENGINE Bus STARTED 2024-09-21T09:34:01.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:01 smithi022 ceph-mon[27763]: Manager daemon y is now available 2024-09-21T09:34:01.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:01 smithi022 ceph-mon[27763]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:34:01.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:01 smithi022 ceph-mon[27763]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:34:01.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:01 smithi022 ceph-mon[27763]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T09:34:01.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:01 smithi022 ceph-mon[27763]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-21T09:34:01.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:01 smithi022 ceph-mon[27763]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-21T09:34:01.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:01 smithi022 ceph-mon[27763]: osdmap e232: 8 total, 8 up, 8 in 2024-09-21T09:34:01.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:01 smithi022 ceph-mon[39525]: Manager daemon y is now available 2024-09-21T09:34:01.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:01 smithi022 ceph-mon[39525]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:34:01.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:01 smithi022 ceph-mon[39525]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:34:01.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:01 smithi022 ceph-mon[39525]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T09:34:01.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:01 smithi022 ceph-mon[39525]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-21T09:34:01.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:01 smithi022 ceph-mon[39525]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-21T09:34:01.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:01 smithi022 ceph-mon[39525]: osdmap e232: 8 total, 8 up, 8 in 2024-09-21T09:34:02.109 INFO:teuthology.orchestra.run.smithi022.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T09:34:02.110 INFO:teuthology.orchestra.run.smithi022.stdout:alertmanager.a smithi022 *:9093,9094 running (45s) 6s ago 5m 15.8M - 0.27.0 11f11916f8cd caced80ba544 2024-09-21T09:34:02.110 INFO:teuthology.orchestra.run.smithi022.stdout:grafana.a smithi064 *:3000 running (20s) 5s ago 5m 77.2M - 9.4.7 954c08fa6188 527bb657d70d 2024-09-21T09:34:02.110 INFO:teuthology.orchestra.run.smithi022.stdout:mds.a smithi022 running (6m) 6s ago 6m 16.2M - 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 34e4221f3a27 2024-09-21T09:34:02.110 INFO:teuthology.orchestra.run.smithi022.stdout:mds.b smithi064 running (6m) 5s ago 6m 19.8M - 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 0f846b3ec694 2024-09-21T09:34:02.110 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.x smithi064 *:8443,9283 running (86s) 5s ago 10m 519M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 c102c1678bcc 2024-09-21T09:34:02.110 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.y smithi022 *:8443,9283,8765 running (10s) 6s ago 11m 66.6M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 6c5bee78f09b 2024-09-21T09:34:02.110 INFO:teuthology.orchestra.run.smithi022.stdout:mon.a smithi022 running (11m) 6s ago 11m 53.9M 2048M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 1e0a2d74e66e 2024-09-21T09:34:02.110 INFO:teuthology.orchestra.run.smithi022.stdout:mon.b smithi064 running (10m) 5s ago 10m 46.5M 2048M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be ed126533cd64 2024-09-21T09:34:02.110 INFO:teuthology.orchestra.run.smithi022.stdout:mon.c smithi022 running (10m) 6s ago 10m 44.8M 2048M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 79f307e3b64a 2024-09-21T09:34:02.110 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.a smithi022 *:9100 running (39s) 6s ago 4m 8354k - 1.7.0 72c9c2088986 593c6158f524 2024-09-21T09:34:02.110 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.b smithi064 *:9100 running (15s) 5s ago 4m 7940k - 1.7.0 72c9c2088986 a533420b0139 2024-09-21T09:34:02.110 INFO:teuthology.orchestra.run.smithi022.stdout:osd.0 smithi022 running (9m) 6s ago 9m 161M 2481M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be fcd812003534 2024-09-21T09:34:02.110 INFO:teuthology.orchestra.run.smithi022.stdout:osd.1 smithi022 running (9m) 6s ago 9m 127M 2481M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 740462b5c304 2024-09-21T09:34:02.111 INFO:teuthology.orchestra.run.smithi022.stdout:osd.2 smithi022 running (9m) 6s ago 9m 142M 2481M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be fefc7c770c32 2024-09-21T09:34:02.111 INFO:teuthology.orchestra.run.smithi022.stdout:osd.3 smithi022 running (8m) 6s ago 8m 75.8M 2481M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be a56193b67a92 2024-09-21T09:34:02.111 INFO:teuthology.orchestra.run.smithi022.stdout:osd.4 smithi064 running (8m) 5s ago 8m 109M 2480M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 2a2937374fc4 2024-09-21T09:34:02.111 INFO:teuthology.orchestra.run.smithi022.stdout:osd.5 smithi064 running (7m) 5s ago 7m 134M 2480M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 1900c4af5d28 2024-09-21T09:34:02.111 INFO:teuthology.orchestra.run.smithi022.stdout:osd.6 smithi064 running (7m) 5s ago 7m 111M 2480M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 3cc4ec816e5b 2024-09-21T09:34:02.111 INFO:teuthology.orchestra.run.smithi022.stdout:osd.7 smithi064 running (7m) 5s ago 7m 119M 2480M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 18ef0be94104 2024-09-21T09:34:02.111 INFO:teuthology.orchestra.run.smithi022.stdout:prometheus.a smithi064 *:9095 running (27s) 5s ago 5m 41.6M - 2.51.0 1d3b7f56885b e96ed2646ce4 2024-09-21T09:34:02.475 INFO:tasks.rados.rados.0.smithi022.stdout:859: done (2 left) 2024-09-21T09:34:02.475 INFO:tasks.rados.rados.0.smithi022.stdout:860: done (1 left) 2024-09-21T09:34:02.475 INFO:tasks.rados.rados.0.smithi022.stdout:862: done (0 left) 2024-09-21T09:34:02.476 INFO:tasks.rados.rados.0.smithi022.stdout:863: read oid 28 snap -1 2024-09-21T09:34:02.476 INFO:tasks.rados.rados.0.smithi022.stdout:863: expect (ObjNum 282 snap 81 seq_num 282) 2024-09-21T09:34:02.476 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 19 current snap is 86 2024-09-21T09:34:02.476 INFO:tasks.rados.rados.0.smithi022.stdout:864: seq_num 289 ranges {4931584=40960} 2024-09-21T09:34:02.485 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-09-21T09:34:02.485 INFO:teuthology.orchestra.run.smithi022.stdout: "mon": { 2024-09-21T09:34:02.485 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 3 2024-09-21T09:34:02.485 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-21T09:34:02.485 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr": { 2024-09-21T09:34:02.485 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 1 2024-09-21T09:34:02.485 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-21T09:34:02.485 INFO:teuthology.orchestra.run.smithi022.stdout: "osd": { 2024-09-21T09:34:02.485 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 8 2024-09-21T09:34:02.485 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-21T09:34:02.485 INFO:teuthology.orchestra.run.smithi022.stdout: "mds": { 2024-09-21T09:34:02.486 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 2 2024-09-21T09:34:02.486 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-21T09:34:02.486 INFO:teuthology.orchestra.run.smithi022.stdout: "overall": { 2024-09-21T09:34:02.486 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 13, 2024-09-21T09:34:02.486 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 1 2024-09-21T09:34:02.486 INFO:teuthology.orchestra.run.smithi022.stdout: } 2024-09-21T09:34:02.486 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-09-21T09:34:02.503 INFO:tasks.rados.rados.0.smithi022.stdout:864: writing smithi02284960-19 from 4931584 to 4972544 tid 1 2024-09-21T09:34:02.503 INFO:tasks.rados.rados.0.smithi022.stdout:865: read oid 25 snap -1 2024-09-21T09:34:02.504 INFO:tasks.rados.rados.0.smithi022.stdout:865: expect deleted 2024-09-21T09:34:02.504 INFO:tasks.rados.rados.0.smithi022.stdout:866: rmattr oid 49 current snap is 86 2024-09-21T09:34:02.504 INFO:tasks.rados.rados.0.smithi022.stdout:866: done (3 left) 2024-09-21T09:34:02.504 INFO:tasks.rados.rados.0.smithi022.stdout:867: rmattr oid 30 current snap is 86 2024-09-21T09:34:02.504 INFO:tasks.rados.rados.0.smithi022.stdout:867: done (3 left) 2024-09-21T09:34:02.504 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 8 current snap is 86 2024-09-21T09:34:02.504 INFO:tasks.rados.rados.0.smithi022.stdout:868: seq_num 290 ranges {2097152=57344} 2024-09-21T09:34:02.515 INFO:tasks.rados.rados.0.smithi022.stdout:868: writing smithi02284960-8 from 2097152 to 2154496 tid 1 2024-09-21T09:34:02.515 INFO:tasks.rados.rados.0.smithi022.stdout:869: rollback oid 18 current snap is 86 2024-09-21T09:34:02.515 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 18 to 67 2024-09-21T09:34:02.515 INFO:tasks.rados.rados.0.smithi022.stdout:863: expect (ObjNum 282 snap 81 seq_num 282) 2024-09-21T09:34:02.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:02 smithi064 ceph-mon[31124]: from='client.15141 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:34:02.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:02 smithi064 ceph-mon[31124]: mgrmap e30: y(active, since 1.36414s) 2024-09-21T09:34:02.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:02 smithi064 ceph-mon[31124]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T09:34:02.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:02 smithi064 ceph-mon[31124]: pgmap v5: 121 pgs: 121 active+clean; 216 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:34:02.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:02 smithi064 ceph-mon[31124]: from='client.15171 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:34:02.703 INFO:tasks.rados.rados.0.smithi022.stdout:864: finishing write tid 1 to smithi02284960-19 2024-09-21T09:34:02.703 INFO:tasks.rados.rados.0.smithi022.stdout:863: done (4 left) 2024-09-21T09:34:02.703 INFO:tasks.rados.rados.0.smithi022.stdout:865: done (3 left) 2024-09-21T09:34:02.703 INFO:tasks.rados.rados.0.smithi022.stdout:870: delete oid 1 current snap is 86 2024-09-21T09:34:02.704 INFO:tasks.rados.rados.0.smithi022.stdout:864: finishing write tid 2 to smithi02284960-19 2024-09-21T09:34:02.704 INFO:tasks.rados.rados.0.smithi022.stdout:864: finishing write tid 3 to smithi02284960-19 2024-09-21T09:34:02.704 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 19 v 525 (ObjNum 289 snap 86 seq_num 289) dirty exists 2024-09-21T09:34:02.704 INFO:tasks.rados.rados.0.smithi022.stdout:864: left oid 19 (ObjNum 289 snap 86 seq_num 289) 2024-09-21T09:34:02.704 INFO:tasks.rados.rados.0.smithi022.stdout:868: finishing write tid 1 to smithi02284960-8 2024-09-21T09:34:02.704 INFO:tasks.rados.rados.0.smithi022.stdout:868: finishing write tid 2 to smithi02284960-8 2024-09-21T09:34:02.704 INFO:tasks.rados.rados.0.smithi022.stdout:869: finishing rollback tid 0 to smithi02284960-18 2024-09-21T09:34:02.704 INFO:tasks.rados.rados.0.smithi022.stdout:869: finishing rollback tid 1 to smithi02284960-18 2024-09-21T09:34:02.704 INFO:tasks.rados.rados.0.smithi022.stdout:869: finishing rollback tid 2 to smithi02284960-18 2024-09-21T09:34:02.704 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 18 v 267 (ObjNum 210 snap 58 seq_num 210) dirty exists 2024-09-21T09:34:02.704 INFO:tasks.rados.rados.0.smithi022.stdout:868: finishing write tid 3 to smithi02284960-8 2024-09-21T09:34:02.704 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 8 v 275 (ObjNum 290 snap 86 seq_num 290) dirty exists 2024-09-21T09:34:02.704 INFO:tasks.rados.rados.0.smithi022.stdout:868: left oid 8 (ObjNum 290 snap 86 seq_num 290) 2024-09-21T09:34:02.705 INFO:tasks.rados.rados.0.smithi022.stdout:864: done (3 left) 2024-09-21T09:34:02.705 INFO:tasks.rados.rados.0.smithi022.stdout:868: done (2 left) 2024-09-21T09:34:02.705 INFO:tasks.rados.rados.0.smithi022.stdout:869: done (1 left) 2024-09-21T09:34:02.705 INFO:tasks.rados.rados.0.smithi022.stdout:870: done (0 left) 2024-09-21T09:34:02.706 INFO:tasks.rados.rados.0.smithi022.stdout:871: rmattr oid 3 current snap is 86 2024-09-21T09:34:02.706 INFO:tasks.rados.rados.0.smithi022.stdout:871: done (0 left) 2024-09-21T09:34:02.706 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 44 current snap is 86 2024-09-21T09:34:02.706 INFO:tasks.rados.rados.0.smithi022.stdout:872: seq_num 292 ranges {1245184=98304} 2024-09-21T09:34:02.713 INFO:tasks.rados.rados.0.smithi022.stdout:872: writing smithi02284960-44 from 1245184 to 1343488 tid 1 2024-09-21T09:34:02.713 INFO:tasks.rados.rados.0.smithi022.stdout:873: rollback oid 13 current snap is 86 2024-09-21T09:34:02.713 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 13 to 84 2024-09-21T09:34:02.713 INFO:tasks.rados.rados.0.smithi022.stdout:874: read oid 37 snap -1 2024-09-21T09:34:02.713 INFO:tasks.rados.rados.0.smithi022.stdout:874: expect (ObjNum 218 snap 63 seq_num 218) 2024-09-21T09:34:02.713 INFO:tasks.rados.rados.0.smithi022.stdout:875: snap_create 2024-09-21T09:34:02.717 INFO:tasks.rados.rados.0.smithi022.stdout:873: finishing rollback tid 0 to smithi02284960-13 2024-09-21T09:34:02.717 INFO:tasks.rados.rados.0.smithi022.stdout:873: finishing rollback tid 1 to smithi02284960-13 2024-09-21T09:34:02.717 INFO:tasks.rados.rados.0.smithi022.stdout:872: finishing write tid 1 to smithi02284960-44 2024-09-21T09:34:02.717 INFO:tasks.rados.rados.0.smithi022.stdout:872: finishing write tid 2 to smithi02284960-44 2024-09-21T09:34:02.718 INFO:tasks.rados.rados.0.smithi022.stdout:873: finishing rollback tid 2 to smithi02284960-13 2024-09-21T09:34:02.719 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 13 v 417 (ObjNum 274 snap 80 seq_num 274) dirty exists 2024-09-21T09:34:02.719 INFO:tasks.rados.rados.0.smithi022.stdout:872: finishing write tid 3 to smithi02284960-44 2024-09-21T09:34:02.719 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 44 v 447 (ObjNum 292 snap 86 seq_num 292) dirty exists 2024-09-21T09:34:02.719 INFO:tasks.rados.rados.0.smithi022.stdout:872: left oid 44 (ObjNum 292 snap 86 seq_num 292) 2024-09-21T09:34:02.720 INFO:tasks.rados.rados.0.smithi022.stdout:874: expect (ObjNum 218 snap 63 seq_num 218) 2024-09-21T09:34:02.799 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-09-21T09:34:02.799 INFO:teuthology.orchestra.run.smithi022.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47", 2024-09-21T09:34:02.799 INFO:teuthology.orchestra.run.smithi022.stdout: "in_progress": true, 2024-09-21T09:34:02.799 INFO:teuthology.orchestra.run.smithi022.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T09:34:02.799 INFO:teuthology.orchestra.run.smithi022.stdout: "services_complete": [ 2024-09-21T09:34:02.799 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr" 2024-09-21T09:34:02.799 INFO:teuthology.orchestra.run.smithi022.stdout: ], 2024-09-21T09:34:02.799 INFO:teuthology.orchestra.run.smithi022.stdout: "progress": "2/20 daemons upgraded", 2024-09-21T09:34:02.799 INFO:teuthology.orchestra.run.smithi022.stdout: "message": "", 2024-09-21T09:34:02.800 INFO:teuthology.orchestra.run.smithi022.stdout: "is_paused": false 2024-09-21T09:34:02.800 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-09-21T09:34:02.800 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:02 smithi022 ceph-mon[27763]: from='client.15141 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:34:02.800 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:02 smithi022 ceph-mon[27763]: mgrmap e30: y(active, since 1.36414s) 2024-09-21T09:34:02.800 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:02 smithi022 ceph-mon[27763]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T09:34:02.800 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:02 smithi022 ceph-mon[27763]: pgmap v5: 121 pgs: 121 active+clean; 216 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:34:02.800 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:02 smithi022 ceph-mon[27763]: from='client.15171 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:34:02.800 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:02 smithi022 ceph-mon[39525]: from='client.15141 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:34:02.801 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:02 smithi022 ceph-mon[39525]: mgrmap e30: y(active, since 1.36414s) 2024-09-21T09:34:02.801 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:02 smithi022 ceph-mon[39525]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T09:34:02.801 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:02 smithi022 ceph-mon[39525]: pgmap v5: 121 pgs: 121 active+clean; 216 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:34:02.801 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:02 smithi022 ceph-mon[39525]: from='client.15171 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:34:02.801 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:34:02 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: 2024-09-21T09:34:02.547+0000 7f171104c640 -1 log_channel(cephadm) log [ERR] : [21/Sep/2024:09:34:02] ENGINE Error in HTTPServer.serve 2024-09-21T09:34:02.801 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:34:02 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: Traceback (most recent call last): 2024-09-21T09:34:02.801 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:34:02 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T09:34:02.801 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:34:02 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: self._connections.run(self.expiration_interval) 2024-09-21T09:34:02.801 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:34:02 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T09:34:02.801 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:34:02 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: self._run(expiration_interval) 2024-09-21T09:34:02.801 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:34:02 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T09:34:02.801 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:34:02 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T09:34:02.801 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:34:02 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T09:34:02.802 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:34:02 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T09:34:02.802 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:34:02 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T09:34:02.802 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:34:02 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: s = self.context.wrap_socket( 2024-09-21T09:34:02.802 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:34:02 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T09:34:02.802 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:34:02 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: return self.sslsocket_class._create( 2024-09-21T09:34:02.802 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:34:02 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T09:34:02.802 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:34:02 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: self.do_handshake() 2024-09-21T09:34:02.802 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:34:02 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T09:34:02.802 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:34:02 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: self._sslobj.do_handshake() 2024-09-21T09:34:02.802 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:34:02 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T09:34:02.802 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:34:02 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: 2024-09-21T09:34:03.477 INFO:tasks.rados.rados.0.smithi022.stdout:872: done (3 left) 2024-09-21T09:34:03.478 INFO:tasks.rados.rados.0.smithi022.stdout:873: done (2 left) 2024-09-21T09:34:03.478 INFO:tasks.rados.rados.0.smithi022.stdout:874: done (1 left) 2024-09-21T09:34:03.478 INFO:tasks.rados.rados.0.smithi022.stdout:875: done (0 left) 2024-09-21T09:34:03.478 INFO:tasks.rados.rados.0.smithi022.stdout:876: read oid 10 snap 82 2024-09-21T09:34:03.478 INFO:tasks.rados.rados.0.smithi022.stdout:876: expect deleted 2024-09-21T09:34:03.478 INFO:tasks.rados.rados.0.smithi022.stdout:877: rmattr oid 16 current snap is 87 2024-09-21T09:34:03.478 INFO:tasks.rados.rados.0.smithi022.stdout:877: done (1 left) 2024-09-21T09:34:03.478 INFO:tasks.rados.rados.0.smithi022.stdout:878: snap_remove snap 83 2024-09-21T09:34:03.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:03 smithi022 ceph-mon[39525]: [21/Sep/2024:09:34:02] ENGINE Bus STARTING 2024-09-21T09:34:03.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:03 smithi022 ceph-mon[39525]: osdmap e233: 8 total, 8 up, 8 in 2024-09-21T09:34:03.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:03 smithi022 ceph-mon[39525]: mgrmap e31: y(active, since 2s) 2024-09-21T09:34:03.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:03 smithi022 ceph-mon[39525]: from='client.? 172.21.15.22:0/3384784801' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:34:03.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:03 smithi022 ceph-mon[39525]: [21/Sep/2024:09:34:02] ENGINE Serving on https://172.21.15.22:7150 2024-09-21T09:34:03.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:03 smithi022 ceph-mon[39525]: [21/Sep/2024:09:34:02] ENGINE Error in HTTPServer.serve 2024-09-21T09:34:03.808 INFO:journalctl@ceph.mon.c.smithi022.stdout: Traceback (most recent call last): 2024-09-21T09:34:03.809 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T09:34:03.809 INFO:journalctl@ceph.mon.c.smithi022.stdout: self._connections.run(self.expiration_interval) 2024-09-21T09:34:03.809 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T09:34:03.809 INFO:journalctl@ceph.mon.c.smithi022.stdout: self._run(expiration_interval) 2024-09-21T09:34:03.809 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T09:34:03.809 INFO:journalctl@ceph.mon.c.smithi022.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T09:34:03.809 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T09:34:03.809 INFO:journalctl@ceph.mon.c.smithi022.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T09:34:03.809 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T09:34:03.809 INFO:journalctl@ceph.mon.c.smithi022.stdout: s = self.context.wrap_socket( 2024-09-21T09:34:03.809 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T09:34:03.809 INFO:journalctl@ceph.mon.c.smithi022.stdout: return self.sslsocket_class._create( 2024-09-21T09:34:03.810 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T09:34:03.810 INFO:journalctl@ceph.mon.c.smithi022.stdout: self.do_handshake() 2024-09-21T09:34:03.810 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T09:34:03.810 INFO:journalctl@ceph.mon.c.smithi022.stdout: self._sslobj.do_handshake() 2024-09-21T09:34:03.810 INFO:journalctl@ceph.mon.c.smithi022.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T09:34:03.810 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:03 smithi022 ceph-mon[39525]: [21/Sep/2024:09:34:02] ENGINE Serving on http://172.21.15.22:8765 2024-09-21T09:34:03.810 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:03 smithi022 ceph-mon[39525]: [21/Sep/2024:09:34:02] ENGINE Bus STARTED 2024-09-21T09:34:03.810 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:03 smithi022 ceph-mon[39525]: from='client.15189 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:34:03.810 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:03 smithi022 ceph-mon[27763]: [21/Sep/2024:09:34:02] ENGINE Bus STARTING 2024-09-21T09:34:03.810 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:03 smithi022 ceph-mon[27763]: osdmap e233: 8 total, 8 up, 8 in 2024-09-21T09:34:03.810 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:03 smithi022 ceph-mon[27763]: mgrmap e31: y(active, since 2s) 2024-09-21T09:34:03.811 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:03 smithi022 ceph-mon[27763]: from='client.? 172.21.15.22:0/3384784801' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:34:03.811 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:03 smithi022 ceph-mon[27763]: [21/Sep/2024:09:34:02] ENGINE Serving on https://172.21.15.22:7150 2024-09-21T09:34:03.811 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:03 smithi022 ceph-mon[27763]: [21/Sep/2024:09:34:02] ENGINE Error in HTTPServer.serve 2024-09-21T09:34:03.811 INFO:journalctl@ceph.mon.a.smithi022.stdout: Traceback (most recent call last): 2024-09-21T09:34:03.811 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T09:34:03.811 INFO:journalctl@ceph.mon.a.smithi022.stdout: self._connections.run(self.expiration_interval) 2024-09-21T09:34:03.811 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T09:34:03.811 INFO:journalctl@ceph.mon.a.smithi022.stdout: self._run(expiration_interval) 2024-09-21T09:34:03.811 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T09:34:03.811 INFO:journalctl@ceph.mon.a.smithi022.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T09:34:03.811 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T09:34:03.811 INFO:journalctl@ceph.mon.a.smithi022.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T09:34:03.812 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T09:34:03.812 INFO:journalctl@ceph.mon.a.smithi022.stdout: s = self.context.wrap_socket( 2024-09-21T09:34:03.812 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T09:34:03.812 INFO:journalctl@ceph.mon.a.smithi022.stdout: return self.sslsocket_class._create( 2024-09-21T09:34:03.812 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T09:34:03.812 INFO:journalctl@ceph.mon.a.smithi022.stdout: self.do_handshake() 2024-09-21T09:34:03.812 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T09:34:03.812 INFO:journalctl@ceph.mon.a.smithi022.stdout: self._sslobj.do_handshake() 2024-09-21T09:34:03.812 INFO:journalctl@ceph.mon.a.smithi022.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T09:34:03.812 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:03 smithi022 ceph-mon[27763]: [21/Sep/2024:09:34:02] ENGINE Serving on http://172.21.15.22:8765 2024-09-21T09:34:03.812 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:03 smithi022 ceph-mon[27763]: [21/Sep/2024:09:34:02] ENGINE Bus STARTED 2024-09-21T09:34:03.813 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:03 smithi022 ceph-mon[27763]: from='client.15189 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:34:03.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:03 smithi064 ceph-mon[31124]: [21/Sep/2024:09:34:02] ENGINE Bus STARTING 2024-09-21T09:34:03.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:03 smithi064 ceph-mon[31124]: osdmap e233: 8 total, 8 up, 8 in 2024-09-21T09:34:03.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:03 smithi064 ceph-mon[31124]: mgrmap e31: y(active, since 2s) 2024-09-21T09:34:03.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:03 smithi064 ceph-mon[31124]: from='client.? 172.21.15.22:0/3384784801' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:34:03.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:03 smithi064 ceph-mon[31124]: [21/Sep/2024:09:34:02] ENGINE Serving on https://172.21.15.22:7150 2024-09-21T09:34:03.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:03 smithi064 ceph-mon[31124]: [21/Sep/2024:09:34:02] ENGINE Error in HTTPServer.serve 2024-09-21T09:34:03.942 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-09-21T09:34:03.942 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T09:34:03.942 INFO:journalctl@ceph.mon.b.smithi064.stdout: self._connections.run(self.expiration_interval) 2024-09-21T09:34:03.942 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T09:34:03.942 INFO:journalctl@ceph.mon.b.smithi064.stdout: self._run(expiration_interval) 2024-09-21T09:34:03.942 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T09:34:03.943 INFO:journalctl@ceph.mon.b.smithi064.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T09:34:03.943 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T09:34:03.943 INFO:journalctl@ceph.mon.b.smithi064.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T09:34:03.943 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T09:34:03.943 INFO:journalctl@ceph.mon.b.smithi064.stdout: s = self.context.wrap_socket( 2024-09-21T09:34:03.943 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T09:34:03.943 INFO:journalctl@ceph.mon.b.smithi064.stdout: return self.sslsocket_class._create( 2024-09-21T09:34:03.943 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T09:34:03.943 INFO:journalctl@ceph.mon.b.smithi064.stdout: self.do_handshake() 2024-09-21T09:34:03.943 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T09:34:03.943 INFO:journalctl@ceph.mon.b.smithi064.stdout: self._sslobj.do_handshake() 2024-09-21T09:34:03.943 INFO:journalctl@ceph.mon.b.smithi064.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T09:34:03.944 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:03 smithi064 ceph-mon[31124]: [21/Sep/2024:09:34:02] ENGINE Serving on http://172.21.15.22:8765 2024-09-21T09:34:03.944 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:03 smithi064 ceph-mon[31124]: [21/Sep/2024:09:34:02] ENGINE Bus STARTED 2024-09-21T09:34:03.944 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:03 smithi064 ceph-mon[31124]: from='client.15189 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:34:04.441 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:34:04 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[75413]: ts=2024-09-21T09:34:04.148Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:34:04.484 INFO:tasks.rados.rados.0.smithi022.stdout:878: done (1 left) 2024-09-21T09:34:04.485 INFO:tasks.rados.rados.0.smithi022.stdout:879: read oid 50 snap -1 2024-09-21T09:34:04.485 INFO:tasks.rados.rados.0.smithi022.stdout:879: expect (ObjNum 256 snap 71 seq_num 256) 2024-09-21T09:34:04.485 INFO:tasks.rados.rados.0.smithi022.stdout:880: rmattr oid 38 current snap is 87 2024-09-21T09:34:04.485 INFO:tasks.rados.rados.0.smithi022.stdout:880: done (2 left) 2024-09-21T09:34:04.485 INFO:tasks.rados.rados.0.smithi022.stdout:881: snap_remove snap 62 2024-09-21T09:34:04.557 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:04 smithi022 ceph-mon[39525]: osdmap e234: 8 total, 8 up, 8 in 2024-09-21T09:34:04.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:04 smithi022 ceph-mon[27763]: osdmap e234: 8 total, 8 up, 8 in 2024-09-21T09:34:04.753 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:04 smithi064 ceph-mon[31124]: osdmap e234: 8 total, 8 up, 8 in 2024-09-21T09:34:05.453 INFO:tasks.rados.rados.0.smithi022.stdout:881: done (2 left) 2024-09-21T09:34:05.453 INFO:tasks.rados.rados.0.smithi022.stdout:882: copy_from oid 29 from oid 1 current snap is 87 2024-09-21T09:34:05.454 INFO:tasks.rados.rados.0.smithi022.stdout:879: expect (ObjNum 256 snap 71 seq_num 256) 2024-09-21T09:34:05.516 INFO:tasks.rados.rados.0.smithi022.stdout:882: finishing copy_from to smithi02284960-29 2024-09-21T09:34:05.516 INFO:tasks.rados.rados.0.smithi022.stdout:882: got expected ENOENT (src dne) 2024-09-21T09:34:05.517 INFO:tasks.rados.rados.0.smithi022.stdout:882: finishing copy_from racing read to smithi02284960-29 2024-09-21T09:34:05.517 INFO:tasks.rados.rados.0.smithi022.stdout:876: done (2 left) 2024-09-21T09:34:05.517 INFO:tasks.rados.rados.0.smithi022.stdout:879: done (1 left) 2024-09-21T09:34:05.517 INFO:tasks.rados.rados.0.smithi022.stdout:882: done (0 left) 2024-09-21T09:34:05.517 INFO:tasks.rados.rados.0.smithi022.stdout:883: read oid 17 snap -1 2024-09-21T09:34:05.517 INFO:tasks.rados.rados.0.smithi022.stdout:883: expect deleted 2024-09-21T09:34:05.517 INFO:tasks.rados.rados.0.smithi022.stdout:884: copy_from oid 4 from oid 24 current snap is 87 2024-09-21T09:34:05.517 INFO:tasks.rados.rados.0.smithi022.stdout:885: delete oid 30 current snap is 87 2024-09-21T09:34:05.518 INFO:tasks.rados.rados.0.smithi022.stdout:883: done (2 left) 2024-09-21T09:34:05.519 INFO:tasks.rados.rados.0.smithi022.stdout:885: done (1 left) 2024-09-21T09:34:05.519 INFO:tasks.rados.rados.0.smithi022.stdout:886: rollback oid 37 current snap is 87 2024-09-21T09:34:05.519 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 37 to 69 2024-09-21T09:34:05.519 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 9 current snap is 87 2024-09-21T09:34:05.519 INFO:tasks.rados.rados.0.smithi022.stdout:887: seq_num 294 ranges {0=131072} 2024-09-21T09:34:05.519 INFO:tasks.rados.rados.0.smithi022.stdout:887: writing smithi02284960-9 from 0 to 131072 tid 1 2024-09-21T09:34:05.519 INFO:tasks.rados.rados.0.smithi022.stdout:888: copy_from oid 20 from oid 25 current snap is 87 2024-09-21T09:34:05.520 INFO:tasks.rados.rados.0.smithi022.stdout:889: snap_remove snap 85 2024-09-21T09:34:05.742 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:05 smithi064 ceph-mon[31124]: pgmap v9: 121 pgs: 121 active+clean; 216 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:34:05.742 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:05 smithi064 ceph-mon[31124]: mgrmap e32: y(active, since 4s) 2024-09-21T09:34:05.742 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:05 smithi064 ceph-mon[31124]: osdmap e235: 8 total, 8 up, 8 in 2024-09-21T09:34:05.742 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:05 smithi064 ceph-mon[31124]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' 2024-09-21T09:34:05.742 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:05 smithi064 ceph-mon[31124]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' 2024-09-21T09:34:05.742 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:05 smithi064 ceph-mon[31124]: osdmap e236: 8 total, 8 up, 8 in 2024-09-21T09:34:05.742 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:34:05 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[67503]: [21/Sep/2024:09:34:05] ENGINE Bus STARTING 2024-09-21T09:34:05.743 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:34:05 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[67503]: CherryPy Checker: 2024-09-21T09:34:05.743 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:34:05 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[67503]: The Application mounted at '' has an empty config. 2024-09-21T09:34:05.743 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:34:05 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[67503]: 2024-09-21T09:34:05.807 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:05 smithi022 ceph-mon[39525]: pgmap v9: 121 pgs: 121 active+clean; 216 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:34:05.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:05 smithi022 ceph-mon[39525]: mgrmap e32: y(active, since 4s) 2024-09-21T09:34:05.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:05 smithi022 ceph-mon[39525]: osdmap e235: 8 total, 8 up, 8 in 2024-09-21T09:34:05.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:05 smithi022 ceph-mon[39525]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' 2024-09-21T09:34:05.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:05 smithi022 ceph-mon[39525]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' 2024-09-21T09:34:05.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:05 smithi022 ceph-mon[39525]: osdmap e236: 8 total, 8 up, 8 in 2024-09-21T09:34:05.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:05 smithi022 ceph-mon[27763]: pgmap v9: 121 pgs: 121 active+clean; 216 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:34:05.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:05 smithi022 ceph-mon[27763]: mgrmap e32: y(active, since 4s) 2024-09-21T09:34:05.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:05 smithi022 ceph-mon[27763]: osdmap e235: 8 total, 8 up, 8 in 2024-09-21T09:34:05.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:05 smithi022 ceph-mon[27763]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' 2024-09-21T09:34:05.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:05 smithi022 ceph-mon[27763]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' 2024-09-21T09:34:05.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:05 smithi022 ceph-mon[27763]: osdmap e236: 8 total, 8 up, 8 in 2024-09-21T09:34:05.998 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:34:05 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[67503]: [21/Sep/2024:09:34:05] ENGINE Serving on http://:::9283 2024-09-21T09:34:05.998 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:34:05 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[67503]: [21/Sep/2024:09:34:05] ENGINE Bus STARTED 2024-09-21T09:34:05.998 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:34:05 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[75413]: ts=2024-09-21T09:34:05.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:34:05.998 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:34:05 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[75413]: ts=2024-09-21T09:34:05.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:34:06.497 INFO:tasks.rados.rados.0.smithi022.stdout:889: done (4 left) 2024-09-21T09:34:06.497 INFO:tasks.rados.rados.0.smithi022.stdout:890: rollback oid 32 current snap is 87 2024-09-21T09:34:06.497 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 32 to 59 2024-09-21T09:34:06.497 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 44 current snap is 87 2024-09-21T09:34:06.497 INFO:tasks.rados.rados.0.smithi022.stdout:891: seq_num 295 ranges {1343488=147456} 2024-09-21T09:34:06.504 INFO:tasks.rados.rados.0.smithi022.stdout:891: writing smithi02284960-44 from 1343488 to 1490944 tid 1 2024-09-21T09:34:06.504 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 26 current snap is 87 2024-09-21T09:34:06.504 INFO:tasks.rados.rados.0.smithi022.stdout:888: finishing copy_from to smithi02284960-20 2024-09-21T09:34:06.505 INFO:tasks.rados.rados.0.smithi022.stdout:888: got expected ENOENT (src dne) 2024-09-21T09:34:06.505 INFO:tasks.rados.rados.0.smithi022.stdout:888: finishing copy_from racing read to smithi02284960-20 2024-09-21T09:34:06.505 INFO:tasks.rados.rados.0.smithi022.stdout:887: finishing write tid 1 to smithi02284960-9 2024-09-21T09:34:06.505 INFO:tasks.rados.rados.0.smithi022.stdout:887: finishing write tid 2 to smithi02284960-9 2024-09-21T09:34:06.505 INFO:tasks.rados.rados.0.smithi022.stdout:886: finishing rollback tid 0 to smithi02284960-37 2024-09-21T09:34:06.505 INFO:tasks.rados.rados.0.smithi022.stdout:886: finishing rollback tid 1 to smithi02284960-37 2024-09-21T09:34:06.505 INFO:tasks.rados.rados.0.smithi022.stdout:887: finishing write tid 3 to smithi02284960-9 2024-09-21T09:34:06.505 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 9 v 285 (ObjNum 294 snap 87 seq_num 294) dirty exists 2024-09-21T09:34:06.505 INFO:tasks.rados.rados.0.smithi022.stdout:887: left oid 9 (ObjNum 294 snap 87 seq_num 294) 2024-09-21T09:34:06.505 INFO:tasks.rados.rados.0.smithi022.stdout:886: finishing rollback tid 2 to smithi02284960-37 2024-09-21T09:34:06.505 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 37 v 539 (ObjNum 218 snap 63 seq_num 218) dirty exists 2024-09-21T09:34:06.505 INFO:tasks.rados.rados.0.smithi022.stdout:884: finishing copy_from racing read to smithi02284960-4 2024-09-21T09:34:06.505 INFO:tasks.rados.rados.0.smithi022.stdout:884: finishing copy_from to smithi02284960-4 2024-09-21T09:34:06.505 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 4 v 450 (ObjNum 253 snap 71 seq_num 253) dirty exists 2024-09-21T09:34:06.506 INFO:tasks.rados.rados.0.smithi022.stdout:892: seq_num 296 ranges {1261568=163840} 2024-09-21T09:34:06.514 INFO:tasks.rados.rados.0.smithi022.stdout:892: writing smithi02284960-26 from 1261568 to 1425408 tid 1 2024-09-21T09:34:06.514 INFO:tasks.rados.rados.0.smithi022.stdout:884: done (6 left) 2024-09-21T09:34:06.514 INFO:tasks.rados.rados.0.smithi022.stdout:886: done (5 left) 2024-09-21T09:34:06.514 INFO:tasks.rados.rados.0.smithi022.stdout:887: done (4 left) 2024-09-21T09:34:06.514 INFO:tasks.rados.rados.0.smithi022.stdout:888: done (3 left) 2024-09-21T09:34:06.514 INFO:tasks.rados.rados.0.smithi022.stdout:893: snap_create 2024-09-21T09:34:06.514 INFO:tasks.rados.rados.0.smithi022.stdout:890: finishing rollback tid 0 to smithi02284960-32 2024-09-21T09:34:06.514 INFO:tasks.rados.rados.0.smithi022.stdout:890: finishing rollback tid 1 to smithi02284960-32 2024-09-21T09:34:06.514 INFO:tasks.rados.rados.0.smithi022.stdout:890: finishing rollback tid 2 to smithi02284960-32 2024-09-21T09:34:06.514 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 32 v 531 (ObjNum 143 snap 32 seq_num 143) dirty exists 2024-09-21T09:34:06.515 INFO:tasks.rados.rados.0.smithi022.stdout:891: finishing write tid 1 to smithi02284960-44 2024-09-21T09:34:06.515 INFO:tasks.rados.rados.0.smithi022.stdout:891: finishing write tid 2 to smithi02284960-44 2024-09-21T09:34:06.515 INFO:tasks.rados.rados.0.smithi022.stdout:891: finishing write tid 3 to smithi02284960-44 2024-09-21T09:34:06.515 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 44 v 466 (ObjNum 295 snap 87 seq_num 295) dirty exists 2024-09-21T09:34:06.515 INFO:tasks.rados.rados.0.smithi022.stdout:891: left oid 44 (ObjNum 295 snap 87 seq_num 295) 2024-09-21T09:34:06.516 INFO:tasks.rados.rados.0.smithi022.stdout:892: finishing write tid 1 to smithi02284960-26 2024-09-21T09:34:06.516 INFO:tasks.rados.rados.0.smithi022.stdout:892: finishing write tid 2 to smithi02284960-26 2024-09-21T09:34:06.517 INFO:tasks.rados.rados.0.smithi022.stdout:892: finishing write tid 3 to smithi02284960-26 2024-09-21T09:34:06.517 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 26 v 432 (ObjNum 296 snap 87 seq_num 296) dirty exists 2024-09-21T09:34:06.517 INFO:tasks.rados.rados.0.smithi022.stdout:892: left oid 26 (ObjNum 296 snap 87 seq_num 296) 2024-09-21T09:34:06.690 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:06 smithi022 ceph-mon[39525]: Standby manager daemon x started 2024-09-21T09:34:06.690 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:06 smithi022 ceph-mon[39525]: from='mgr.? 172.21.15.64:0/1399933001' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-21T09:34:06.690 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:06 smithi022 ceph-mon[39525]: from='mgr.? 172.21.15.64:0/1399933001' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:34:06.690 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:06 smithi022 ceph-mon[39525]: from='mgr.? 172.21.15.64:0/1399933001' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-21T09:34:06.690 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:06 smithi022 ceph-mon[39525]: from='mgr.? 172.21.15.64:0/1399933001' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:34:06.690 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:06 smithi022 ceph-mon[39525]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' 2024-09-21T09:34:06.690 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:06 smithi022 ceph-mon[39525]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' 2024-09-21T09:34:06.690 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:06 smithi022 ceph-mon[39525]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' 2024-09-21T09:34:06.690 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:06 smithi022 ceph-mon[39525]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' 2024-09-21T09:34:06.691 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:06 smithi022 ceph-mon[27763]: Standby manager daemon x started 2024-09-21T09:34:06.691 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:06 smithi022 ceph-mon[27763]: from='mgr.? 172.21.15.64:0/1399933001' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-21T09:34:06.691 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:06 smithi022 ceph-mon[27763]: from='mgr.? 172.21.15.64:0/1399933001' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:34:06.691 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:06 smithi022 ceph-mon[27763]: from='mgr.? 172.21.15.64:0/1399933001' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-21T09:34:06.691 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:06 smithi022 ceph-mon[27763]: from='mgr.? 172.21.15.64:0/1399933001' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:34:06.691 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:06 smithi022 ceph-mon[27763]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' 2024-09-21T09:34:06.691 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:06 smithi022 ceph-mon[27763]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' 2024-09-21T09:34:06.691 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:06 smithi022 ceph-mon[27763]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' 2024-09-21T09:34:06.691 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:06 smithi022 ceph-mon[27763]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' 2024-09-21T09:34:06.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:06 smithi064 ceph-mon[31124]: Standby manager daemon x started 2024-09-21T09:34:06.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:06 smithi064 ceph-mon[31124]: from='mgr.? 172.21.15.64:0/1399933001' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-21T09:34:06.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:06 smithi064 ceph-mon[31124]: from='mgr.? 172.21.15.64:0/1399933001' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:34:06.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:06 smithi064 ceph-mon[31124]: from='mgr.? 172.21.15.64:0/1399933001' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-21T09:34:06.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:06 smithi064 ceph-mon[31124]: from='mgr.? 172.21.15.64:0/1399933001' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:34:06.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:06 smithi064 ceph-mon[31124]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' 2024-09-21T09:34:06.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:06 smithi064 ceph-mon[31124]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' 2024-09-21T09:34:06.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:06 smithi064 ceph-mon[31124]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' 2024-09-21T09:34:06.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:06 smithi064 ceph-mon[31124]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' 2024-09-21T09:34:07.441 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:34:06 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[67503]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:34:06] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.51.0" 2024-09-21T09:34:07.441 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:34:06 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[75413]: ts=2024-09-21T09:34:06.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi022\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:34:07.496 INFO:tasks.rados.rados.0.smithi022.stdout:890: done (3 left) 2024-09-21T09:34:07.496 INFO:tasks.rados.rados.0.smithi022.stdout:891: done (2 left) 2024-09-21T09:34:07.496 INFO:tasks.rados.rados.0.smithi022.stdout:892: done (1 left) 2024-09-21T09:34:07.496 INFO:tasks.rados.rados.0.smithi022.stdout:893: done (0 left) 2024-09-21T09:34:07.496 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 45 current snap is 88 2024-09-21T09:34:07.497 INFO:tasks.rados.rados.0.smithi022.stdout:894: seq_num 297 ranges {1277952=180224} 2024-09-21T09:34:07.504 INFO:tasks.rados.rados.0.smithi022.stdout:894: writing smithi02284960-45 from 1277952 to 1458176 tid 1 2024-09-21T09:34:07.504 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 10 current snap is 88 2024-09-21T09:34:07.504 INFO:tasks.rados.rados.0.smithi022.stdout:895: seq_num 298 ranges {0=196608} 2024-09-21T09:34:07.505 INFO:tasks.rados.rados.0.smithi022.stdout:895: writing smithi02284960-10 from 0 to 196608 tid 1 2024-09-21T09:34:07.505 INFO:tasks.rados.rados.0.smithi022.stdout:896: delete oid 7 current snap is 88 2024-09-21T09:34:07.508 INFO:tasks.rados.rados.0.smithi022.stdout:896: done (2 left) 2024-09-21T09:34:07.508 INFO:tasks.rados.rados.0.smithi022.stdout:897: rmattr oid 6 current snap is 88 2024-09-21T09:34:07.509 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 49 current snap is 88 2024-09-21T09:34:07.509 INFO:tasks.rados.rados.0.smithi022.stdout:898: seq_num 300 ranges {2236416=229376} 2024-09-21T09:34:07.523 INFO:tasks.rados.rados.0.smithi022.stdout:898: writing smithi02284960-49 from 2236416 to 2465792 tid 1 2024-09-21T09:34:07.523 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 2 current snap is 88 2024-09-21T09:34:07.523 INFO:tasks.rados.rados.0.smithi022.stdout:899: seq_num 301 ranges {2269184=245760} 2024-09-21T09:34:07.537 INFO:tasks.rados.rados.0.smithi022.stdout:899: writing smithi02284960-2 from 2269184 to 2514944 tid 1 2024-09-21T09:34:07.537 INFO:tasks.rados.rados.0.smithi022.stdout:900: delete oid 22 current snap is 88 2024-09-21T09:34:07.537 INFO:tasks.rados.rados.0.smithi022.stdout:894: finishing write tid 1 to smithi02284960-45 2024-09-21T09:34:07.537 INFO:tasks.rados.rados.0.smithi022.stdout:894: finishing write tid 2 to smithi02284960-45 2024-09-21T09:34:07.537 INFO:tasks.rados.rados.0.smithi022.stdout:895: finishing write tid 1 to smithi02284960-10 2024-09-21T09:34:07.538 INFO:tasks.rados.rados.0.smithi022.stdout:895: finishing write tid 2 to smithi02284960-10 2024-09-21T09:34:07.538 INFO:tasks.rados.rados.0.smithi022.stdout:894: finishing write tid 3 to smithi02284960-45 2024-09-21T09:34:07.538 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 45 v 283 (ObjNum 297 snap 88 seq_num 297) dirty exists 2024-09-21T09:34:07.538 INFO:tasks.rados.rados.0.smithi022.stdout:894: left oid 45 (ObjNum 297 snap 88 seq_num 297) 2024-09-21T09:34:07.538 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 6 v 535 (ObjNum 142 snap 32 seq_num 142) dirty exists 2024-09-21T09:34:07.538 INFO:tasks.rados.rados.0.smithi022.stdout:895: finishing write tid 3 to smithi02284960-10 2024-09-21T09:34:07.538 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 10 v 520 (ObjNum 298 snap 88 seq_num 298) dirty exists 2024-09-21T09:34:07.538 INFO:tasks.rados.rados.0.smithi022.stdout:895: left oid 10 (ObjNum 298 snap 88 seq_num 298) 2024-09-21T09:34:07.538 INFO:tasks.rados.rados.0.smithi022.stdout:898: finishing write tid 1 to smithi02284960-49 2024-09-21T09:34:07.538 INFO:tasks.rados.rados.0.smithi022.stdout:898: finishing write tid 2 to smithi02284960-49 2024-09-21T09:34:07.538 INFO:tasks.rados.rados.0.smithi022.stdout:898: finishing write tid 3 to smithi02284960-49 2024-09-21T09:34:07.539 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 49 v 255 (ObjNum 300 snap 88 seq_num 300) dirty exists 2024-09-21T09:34:07.539 INFO:tasks.rados.rados.0.smithi022.stdout:898: left oid 49 (ObjNum 300 snap 88 seq_num 300) 2024-09-21T09:34:07.539 INFO:tasks.rados.rados.0.smithi022.stdout:899: finishing write tid 1 to smithi02284960-2 2024-09-21T09:34:07.540 INFO:tasks.rados.rados.0.smithi022.stdout:899: finishing write tid 2 to smithi02284960-2 2024-09-21T09:34:07.541 INFO:tasks.rados.rados.0.smithi022.stdout:894: done (5 left) 2024-09-21T09:34:07.541 INFO:tasks.rados.rados.0.smithi022.stdout:895: done (4 left) 2024-09-21T09:34:07.541 INFO:tasks.rados.rados.0.smithi022.stdout:897: done (3 left) 2024-09-21T09:34:07.541 INFO:tasks.rados.rados.0.smithi022.stdout:898: done (2 left) 2024-09-21T09:34:07.541 INFO:tasks.rados.rados.0.smithi022.stdout:900: done (1 left) 2024-09-21T09:34:07.541 INFO:tasks.rados.rados.0.smithi022.stdout:901: snap_create 2024-09-21T09:34:07.541 INFO:tasks.rados.rados.0.smithi022.stdout:899: finishing write tid 3 to smithi02284960-2 2024-09-21T09:34:07.541 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 2 v 233 (ObjNum 301 snap 88 seq_num 301) dirty exists 2024-09-21T09:34:07.542 INFO:tasks.rados.rados.0.smithi022.stdout:899: left oid 2 (ObjNum 301 snap 88 seq_num 301) 2024-09-21T09:34:07.719 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:07 smithi022 ceph-mon[39525]: pgmap v12: 121 pgs: 121 active+clean; 216 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:34:07.719 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:07 smithi022 ceph-mon[39525]: mgrmap e33: y(active, since 6s), standbys: x 2024-09-21T09:34:07.719 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:07 smithi022 ceph-mon[39525]: osdmap e237: 8 total, 8 up, 8 in 2024-09-21T09:34:07.719 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:07 smithi022 ceph-mon[39525]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-21T09:34:07.719 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:07 smithi022 ceph-mon[39525]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' 2024-09-21T09:34:07.720 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:07 smithi022 ceph-mon[39525]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' 2024-09-21T09:34:07.720 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:07 smithi022 ceph-mon[27763]: pgmap v12: 121 pgs: 121 active+clean; 216 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:34:07.720 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:07 smithi022 ceph-mon[27763]: mgrmap e33: y(active, since 6s), standbys: x 2024-09-21T09:34:07.720 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:07 smithi022 ceph-mon[27763]: osdmap e237: 8 total, 8 up, 8 in 2024-09-21T09:34:07.720 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:07 smithi022 ceph-mon[27763]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-21T09:34:07.720 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:07 smithi022 ceph-mon[27763]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' 2024-09-21T09:34:07.720 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:07 smithi022 ceph-mon[27763]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' 2024-09-21T09:34:07.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:07 smithi064 ceph-mon[31124]: pgmap v12: 121 pgs: 121 active+clean; 216 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:34:07.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:07 smithi064 ceph-mon[31124]: mgrmap e33: y(active, since 6s), standbys: x 2024-09-21T09:34:07.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:07 smithi064 ceph-mon[31124]: osdmap e237: 8 total, 8 up, 8 in 2024-09-21T09:34:07.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:07 smithi064 ceph-mon[31124]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-21T09:34:07.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:07 smithi064 ceph-mon[31124]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' 2024-09-21T09:34:07.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:07 smithi064 ceph-mon[31124]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' 2024-09-21T09:34:08.501 INFO:tasks.rados.rados.0.smithi022.stdout:899: done (1 left) 2024-09-21T09:34:08.502 INFO:tasks.rados.rados.0.smithi022.stdout:901: done (0 left) 2024-09-21T09:34:08.502 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 13 current snap is 89 2024-09-21T09:34:08.502 INFO:tasks.rados.rados.0.smithi022.stdout:902: seq_num 303 ranges {4153344=278528} 2024-09-21T09:34:08.526 INFO:tasks.rados.rados.0.smithi022.stdout:902: writing smithi02284960-13 from 4153344 to 4431872 tid 1 2024-09-21T09:34:08.526 INFO:tasks.rados.rados.0.smithi022.stdout:903: delete oid 50 current snap is 89 2024-09-21T09:34:08.529 INFO:tasks.rados.rados.0.smithi022.stdout:903: done (1 left) 2024-09-21T09:34:08.529 INFO:tasks.rados.rados.0.smithi022.stdout:904: rollback oid 21 current snap is 89 2024-09-21T09:34:08.529 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 21 to 59 2024-09-21T09:34:08.530 INFO:tasks.rados.rados.0.smithi022.stdout:905: snap_remove snap 81 2024-09-21T09:34:08.807 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:08 smithi022 ceph-mon[39525]: osdmap e238: 8 total, 8 up, 8 in 2024-09-21T09:34:08.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:08 smithi022 ceph-mon[27763]: osdmap e238: 8 total, 8 up, 8 in 2024-09-21T09:34:08.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:08 smithi064 ceph-mon[31124]: osdmap e238: 8 total, 8 up, 8 in 2024-09-21T09:34:09.499 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:09 smithi022 ceph-mon[27763]: pgmap v15: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 225 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.9 MiB/s rd, 2.3 MiB/s wr, 27 op/s 2024-09-21T09:34:09.499 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:09 smithi022 ceph-mon[27763]: osdmap e239: 8 total, 8 up, 8 in 2024-09-21T09:34:09.499 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:09 smithi022 ceph-mon[27763]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' 2024-09-21T09:34:09.499 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:09 smithi022 ceph-mon[27763]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' 2024-09-21T09:34:09.499 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:09 smithi022 ceph-mon[27763]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:34:09.499 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:09 smithi022 ceph-mon[27763]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:34:09.506 INFO:tasks.rados.rados.0.smithi022.stdout:905: done (2 left) 2024-09-21T09:34:09.506 INFO:tasks.rados.rados.0.smithi022.stdout:906: snap_create 2024-09-21T09:34:09.506 INFO:tasks.rados.rados.0.smithi022.stdout:902: finishing write tid 1 to smithi02284960-13 2024-09-21T09:34:09.507 INFO:tasks.rados.rados.0.smithi022.stdout:902: finishing write tid 2 to smithi02284960-13 2024-09-21T09:34:09.507 INFO:tasks.rados.rados.0.smithi022.stdout:902: finishing write tid 3 to smithi02284960-13 2024-09-21T09:34:09.507 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 13 v 436 (ObjNum 303 snap 89 seq_num 303) dirty exists 2024-09-21T09:34:09.507 INFO:tasks.rados.rados.0.smithi022.stdout:902: left oid 13 (ObjNum 303 snap 89 seq_num 303) 2024-09-21T09:34:09.507 INFO:tasks.rados.rados.0.smithi022.stdout:904: finishing rollback tid 0 to smithi02284960-21 2024-09-21T09:34:09.507 INFO:tasks.rados.rados.0.smithi022.stdout:904: finishing rollback tid 1 to smithi02284960-21 2024-09-21T09:34:09.507 INFO:tasks.rados.rados.0.smithi022.stdout:904: finishing rollback tid 2 to smithi02284960-21 2024-09-21T09:34:09.507 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 21 v 237 (ObjNum 172 snap 43 seq_num 172) dirty exists 2024-09-21T09:34:09.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:09 smithi022 ceph-mon[39525]: pgmap v15: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 225 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.9 MiB/s rd, 2.3 MiB/s wr, 27 op/s 2024-09-21T09:34:09.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:09 smithi022 ceph-mon[39525]: osdmap e239: 8 total, 8 up, 8 in 2024-09-21T09:34:09.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:09 smithi022 ceph-mon[39525]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' 2024-09-21T09:34:09.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:09 smithi022 ceph-mon[39525]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' 2024-09-21T09:34:09.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:09 smithi022 ceph-mon[39525]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:34:09.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:09 smithi022 ceph-mon[39525]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:34:09.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:09 smithi064 ceph-mon[31124]: pgmap v15: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 225 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.9 MiB/s rd, 2.3 MiB/s wr, 27 op/s 2024-09-21T09:34:09.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:09 smithi064 ceph-mon[31124]: osdmap e239: 8 total, 8 up, 8 in 2024-09-21T09:34:09.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:09 smithi064 ceph-mon[31124]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' 2024-09-21T09:34:09.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:09 smithi064 ceph-mon[31124]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' 2024-09-21T09:34:09.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:09 smithi064 ceph-mon[31124]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:34:09.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:09 smithi064 ceph-mon[31124]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:34:10.455 INFO:tasks.rados.rados.0.smithi022.stdout:902: done (2 left) 2024-09-21T09:34:10.455 INFO:tasks.rados.rados.0.smithi022.stdout:904: done (1 left) 2024-09-21T09:34:10.455 INFO:tasks.rados.rados.0.smithi022.stdout:906: done (0 left) 2024-09-21T09:34:10.455 INFO:tasks.rados.rados.0.smithi022.stdout:907: read oid 46 snap -1 2024-09-21T09:34:10.455 INFO:tasks.rados.rados.0.smithi022.stdout:907: expect (ObjNum 204 snap 56 seq_num 204) 2024-09-21T09:34:10.455 INFO:tasks.rados.rados.0.smithi022.stdout:908: delete oid 8 current snap is 90 2024-09-21T09:34:10.458 INFO:tasks.rados.rados.0.smithi022.stdout:908: done (1 left) 2024-09-21T09:34:10.458 INFO:tasks.rados.rados.0.smithi022.stdout:909: snap_create 2024-09-21T09:34:10.461 INFO:tasks.rados.rados.0.smithi022.stdout:907: expect (ObjNum 204 snap 56 seq_num 204) 2024-09-21T09:34:10.503 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:34:10 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[75413]: ts=2024-09-21T09:34:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:34:10.810 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:10 smithi022 ceph-mon[39525]: Updating smithi022:/etc/ceph/ceph.conf 2024-09-21T09:34:10.810 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:10 smithi022 ceph-mon[39525]: Updating smithi064:/etc/ceph/ceph.conf 2024-09-21T09:34:10.810 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:10 smithi022 ceph-mon[39525]: Updating smithi022:/var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:34:10.810 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:10 smithi022 ceph-mon[39525]: Updating smithi064:/var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:34:10.810 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:10 smithi022 ceph-mon[39525]: Updating smithi022:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:34:10.810 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:10 smithi022 ceph-mon[39525]: Updating smithi064:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:34:10.810 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:10 smithi022 ceph-mon[39525]: osdmap e240: 8 total, 8 up, 8 in 2024-09-21T09:34:10.810 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:10 smithi022 ceph-mon[39525]: Updating smithi022:/var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:34:10.810 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:10 smithi022 ceph-mon[39525]: Updating smithi064:/var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:34:10.811 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:10 smithi022 ceph-mon[39525]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' 2024-09-21T09:34:10.811 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:10 smithi022 ceph-mon[39525]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' 2024-09-21T09:34:10.811 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:10 smithi022 ceph-mon[39525]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' 2024-09-21T09:34:10.811 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:10 smithi022 ceph-mon[39525]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' 2024-09-21T09:34:10.811 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:10 smithi022 ceph-mon[39525]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' 2024-09-21T09:34:10.811 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:10 smithi022 ceph-mon[39525]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-21T09:34:10.811 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:10 smithi022 ceph-mon[39525]: osdmap e241: 8 total, 8 up, 8 in 2024-09-21T09:34:10.811 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:10 smithi022 ceph-mon[27763]: Updating smithi022:/etc/ceph/ceph.conf 2024-09-21T09:34:10.811 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:10 smithi022 ceph-mon[27763]: Updating smithi064:/etc/ceph/ceph.conf 2024-09-21T09:34:10.812 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:10 smithi022 ceph-mon[27763]: Updating smithi022:/var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:34:10.812 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:10 smithi022 ceph-mon[27763]: Updating smithi064:/var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:34:10.812 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:10 smithi022 ceph-mon[27763]: Updating smithi022:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:34:10.812 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:10 smithi022 ceph-mon[27763]: Updating smithi064:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:34:10.812 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:10 smithi022 ceph-mon[27763]: osdmap e240: 8 total, 8 up, 8 in 2024-09-21T09:34:10.812 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:10 smithi022 ceph-mon[27763]: Updating smithi022:/var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:34:10.812 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:10 smithi022 ceph-mon[27763]: Updating smithi064:/var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:34:10.812 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:10 smithi022 ceph-mon[27763]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' 2024-09-21T09:34:10.812 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:10 smithi022 ceph-mon[27763]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' 2024-09-21T09:34:10.812 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:10 smithi022 ceph-mon[27763]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' 2024-09-21T09:34:10.812 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:10 smithi022 ceph-mon[27763]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' 2024-09-21T09:34:10.812 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:10 smithi022 ceph-mon[27763]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' 2024-09-21T09:34:10.813 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:10 smithi022 ceph-mon[27763]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-21T09:34:10.813 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:10 smithi022 ceph-mon[27763]: osdmap e241: 8 total, 8 up, 8 in 2024-09-21T09:34:10.874 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:10 smithi064 ceph-mon[31124]: Updating smithi022:/etc/ceph/ceph.conf 2024-09-21T09:34:10.874 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:10 smithi064 ceph-mon[31124]: Updating smithi064:/etc/ceph/ceph.conf 2024-09-21T09:34:10.874 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:10 smithi064 ceph-mon[31124]: Updating smithi022:/var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:34:10.874 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:10 smithi064 ceph-mon[31124]: Updating smithi064:/var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:34:10.874 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:10 smithi064 ceph-mon[31124]: Updating smithi022:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:34:10.874 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:10 smithi064 ceph-mon[31124]: Updating smithi064:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:34:10.874 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:10 smithi064 ceph-mon[31124]: osdmap e240: 8 total, 8 up, 8 in 2024-09-21T09:34:10.874 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:10 smithi064 ceph-mon[31124]: Updating smithi022:/var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:34:10.875 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:10 smithi064 ceph-mon[31124]: Updating smithi064:/var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:34:10.875 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:10 smithi064 ceph-mon[31124]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' 2024-09-21T09:34:10.875 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:10 smithi064 ceph-mon[31124]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' 2024-09-21T09:34:10.875 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:10 smithi064 ceph-mon[31124]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' 2024-09-21T09:34:10.875 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:10 smithi064 ceph-mon[31124]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' 2024-09-21T09:34:10.875 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:10 smithi064 ceph-mon[31124]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' 2024-09-21T09:34:10.875 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:10 smithi064 ceph-mon[31124]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-21T09:34:10.875 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:10 smithi064 ceph-mon[31124]: osdmap e241: 8 total, 8 up, 8 in 2024-09-21T09:34:11.457 INFO:tasks.rados.rados.0.smithi022.stdout:907: done (1 left) 2024-09-21T09:34:11.457 INFO:tasks.rados.rados.0.smithi022.stdout:909: done (0 left) 2024-09-21T09:34:11.457 INFO:tasks.rados.rados.0.smithi022.stdout:910: setattr oid 42 current snap is 91 2024-09-21T09:34:11.457 INFO:tasks.rados.rados.0.smithi022.stdout:911: delete oid 12 current snap is 91 2024-09-21T09:34:11.462 INFO:tasks.rados.rados.0.smithi022.stdout:911: done (1 left) 2024-09-21T09:34:11.462 INFO:tasks.rados.rados.0.smithi022.stdout:912: read oid 16 snap 87 2024-09-21T09:34:11.462 INFO:tasks.rados.rados.0.smithi022.stdout:912: expect deleted 2024-09-21T09:34:11.463 INFO:tasks.rados.rados.0.smithi022.stdout:913: rollback oid 44 current snap is 91 2024-09-21T09:34:11.463 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 44 to 89 2024-09-21T09:34:11.463 INFO:tasks.rados.rados.0.smithi022.stdout:914: rollback oid 1 current snap is 91 2024-09-21T09:34:11.463 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 1 to 89 2024-09-21T09:34:11.463 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 12 current snap is 91 2024-09-21T09:34:11.463 INFO:tasks.rados.rados.0.smithi022.stdout:915: seq_num 307 ranges {0=344064} 2024-09-21T09:34:11.465 INFO:tasks.rados.rados.0.smithi022.stdout:915: writing smithi02284960-12 from 0 to 344064 tid 1 2024-09-21T09:34:11.465 INFO:tasks.rados.rados.0.smithi022.stdout:916: rollback oid 35 current snap is 91 2024-09-21T09:34:11.465 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 35 to 84 2024-09-21T09:34:11.465 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 42 v 441 (ObjNum 161 snap 0 seq_num 1) dirty exists 2024-09-21T09:34:11.465 INFO:tasks.rados.rados.0.smithi022.stdout:910: done (5 left) 2024-09-21T09:34:11.465 INFO:tasks.rados.rados.0.smithi022.stdout:912: done (4 left) 2024-09-21T09:34:11.465 INFO:tasks.rados.rados.0.smithi022.stdout:917: read oid 48 snap -1 2024-09-21T09:34:11.465 INFO:tasks.rados.rados.0.smithi022.stdout:917: expect (ObjNum 287 snap 85 seq_num 287) 2024-09-21T09:34:11.465 INFO:tasks.rados.rados.0.smithi022.stdout:918: delete oid 26 current snap is 91 2024-09-21T09:34:11.467 INFO:tasks.rados.rados.0.smithi022.stdout:913: finishing rollback tid 0 to smithi02284960-44 2024-09-21T09:34:11.467 INFO:tasks.rados.rados.0.smithi022.stdout:913: finishing rollback tid 1 to smithi02284960-44 2024-09-21T09:34:11.469 INFO:tasks.rados.rados.0.smithi022.stdout:913: finishing rollback tid 2 to smithi02284960-44 2024-09-21T09:34:11.469 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 44 v 474 (ObjNum 295 snap 87 seq_num 295) dirty exists 2024-09-21T09:34:11.469 INFO:tasks.rados.rados.0.smithi022.stdout:915: finishing write tid 1 to smithi02284960-12 2024-09-21T09:34:11.469 INFO:tasks.rados.rados.0.smithi022.stdout:915: finishing write tid 2 to smithi02284960-12 2024-09-21T09:34:11.469 INFO:tasks.rados.rados.0.smithi022.stdout:913: done (5 left) 2024-09-21T09:34:11.469 INFO:tasks.rados.rados.0.smithi022.stdout:918: done (4 left) 2024-09-21T09:34:11.470 INFO:tasks.rados.rados.0.smithi022.stdout:919: setattr oid 19 current snap is 91 2024-09-21T09:34:11.470 INFO:tasks.rados.rados.0.smithi022.stdout:914: finishing rollback tid 1 to smithi02284960-1 2024-09-21T09:34:11.470 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 1 v 445 (ObjNum 161 snap 0 seq_num 1) dirty dne 2024-09-21T09:34:11.472 INFO:tasks.rados.rados.0.smithi022.stdout:914: done (4 left) 2024-09-21T09:34:11.472 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 44 current snap is 91 2024-09-21T09:34:11.472 INFO:tasks.rados.rados.0.smithi022.stdout:920: seq_num 309 ranges {1490944=376832} 2024-09-21T09:34:11.482 INFO:tasks.rados.rados.0.smithi022.stdout:920: writing smithi02284960-44 from 1490944 to 1867776 tid 1 2024-09-21T09:34:11.482 INFO:tasks.rados.rados.0.smithi022.stdout:921: read oid 9 snap -1 2024-09-21T09:34:11.482 INFO:tasks.rados.rados.0.smithi022.stdout:921: expect (ObjNum 294 snap 87 seq_num 294) 2024-09-21T09:34:11.482 INFO:tasks.rados.rados.0.smithi022.stdout:922: copy_from oid 17 from oid 25 current snap is 91 2024-09-21T09:34:11.482 INFO:tasks.rados.rados.0.smithi022.stdout:915: finishing write tid 3 to smithi02284960-12 2024-09-21T09:34:11.482 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 12 v 551 (ObjNum 307 snap 91 seq_num 307) dirty exists 2024-09-21T09:34:11.482 INFO:tasks.rados.rados.0.smithi022.stdout:915: left oid 12 (ObjNum 307 snap 91 seq_num 307) 2024-09-21T09:34:11.482 INFO:tasks.rados.rados.0.smithi022.stdout:916: finishing rollback tid 0 to smithi02284960-35 2024-09-21T09:34:11.483 INFO:tasks.rados.rados.0.smithi022.stdout:916: finishing rollback tid 1 to smithi02284960-35 2024-09-21T09:34:11.483 INFO:tasks.rados.rados.0.smithi022.stdout:916: finishing rollback tid 2 to smithi02284960-35 2024-09-21T09:34:11.483 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 35 v 354 (ObjNum 264 snap 76 seq_num 264) dirty exists 2024-09-21T09:34:11.483 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 19 v 553 (ObjNum 289 snap 86 seq_num 289) dirty exists 2024-09-21T09:34:11.483 INFO:tasks.rados.rados.0.smithi022.stdout:915: done (6 left) 2024-09-21T09:34:11.483 INFO:tasks.rados.rados.0.smithi022.stdout:916: done (5 left) 2024-09-21T09:34:11.483 INFO:tasks.rados.rados.0.smithi022.stdout:919: done (4 left) 2024-09-21T09:34:11.483 INFO:tasks.rados.rados.0.smithi022.stdout:923: snap_remove snap 67 2024-09-21T09:34:12.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:11 smithi022 ceph-mon[39525]: Reconfiguring daemon prometheus.a on smithi064 2024-09-21T09:34:12.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:11 smithi022 ceph-mon[39525]: pgmap v18: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 225 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.9 MiB/s rd, 2.3 MiB/s wr, 27 op/s 2024-09-21T09:34:12.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:11 smithi022 ceph-mon[39525]: osdmap e242: 8 total, 8 up, 8 in 2024-09-21T09:34:12.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:11 smithi022 ceph-mon[27763]: Reconfiguring daemon prometheus.a on smithi064 2024-09-21T09:34:12.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:11 smithi022 ceph-mon[27763]: pgmap v18: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 225 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.9 MiB/s rd, 2.3 MiB/s wr, 27 op/s 2024-09-21T09:34:12.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:11 smithi022 ceph-mon[27763]: osdmap e242: 8 total, 8 up, 8 in 2024-09-21T09:34:12.083 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:11 smithi064 ceph-mon[31124]: Reconfiguring daemon prometheus.a on smithi064 2024-09-21T09:34:12.084 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:11 smithi064 ceph-mon[31124]: pgmap v18: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 225 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.9 MiB/s rd, 2.3 MiB/s wr, 27 op/s 2024-09-21T09:34:12.084 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:11 smithi064 ceph-mon[31124]: osdmap e242: 8 total, 8 up, 8 in 2024-09-21T09:34:12.459 INFO:tasks.rados.rados.0.smithi022.stdout:917: expect (ObjNum 287 snap 85 seq_num 287) 2024-09-21T09:34:12.696 INFO:tasks.rados.rados.0.smithi022.stdout:922: finishing copy_from to smithi02284960-17 2024-09-21T09:34:12.696 INFO:tasks.rados.rados.0.smithi022.stdout:922: got expected ENOENT (src dne) 2024-09-21T09:34:12.696 INFO:tasks.rados.rados.0.smithi022.stdout:922: finishing copy_from racing read to smithi02284960-17 2024-09-21T09:34:12.696 INFO:tasks.rados.rados.0.smithi022.stdout:921: expect (ObjNum 294 snap 87 seq_num 294) 2024-09-21T09:34:12.700 INFO:tasks.rados.rados.0.smithi022.stdout:920: finishing write tid 1 to smithi02284960-44 2024-09-21T09:34:12.700 INFO:tasks.rados.rados.0.smithi022.stdout:920: finishing write tid 2 to smithi02284960-44 2024-09-21T09:34:12.700 INFO:tasks.rados.rados.0.smithi022.stdout:920: finishing write tid 3 to smithi02284960-44 2024-09-21T09:34:12.700 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 44 v 477 (ObjNum 309 snap 91 seq_num 309) dirty exists 2024-09-21T09:34:12.700 INFO:tasks.rados.rados.0.smithi022.stdout:920: left oid 44 (ObjNum 309 snap 91 seq_num 309) 2024-09-21T09:34:12.700 INFO:tasks.rados.rados.0.smithi022.stdout:917: done (4 left) 2024-09-21T09:34:12.700 INFO:tasks.rados.rados.0.smithi022.stdout:920: done (3 left) 2024-09-21T09:34:12.700 INFO:tasks.rados.rados.0.smithi022.stdout:921: done (2 left) 2024-09-21T09:34:12.700 INFO:tasks.rados.rados.0.smithi022.stdout:922: done (1 left) 2024-09-21T09:34:12.700 INFO:tasks.rados.rados.0.smithi022.stdout:923: done (0 left) 2024-09-21T09:34:12.700 INFO:tasks.rados.rados.0.smithi022.stdout:924: setattr oid 31 current snap is 91 2024-09-21T09:34:12.702 INFO:tasks.rados.rados.0.smithi022.stdout:925: snap_create 2024-09-21T09:34:12.709 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 31 v 555 (ObjNum 272 snap 79 seq_num 272) dirty exists 2024-09-21T09:34:13.461 INFO:tasks.rados.rados.0.smithi022.stdout:924: done (1 left) 2024-09-21T09:34:13.461 INFO:tasks.rados.rados.0.smithi022.stdout:925: done (0 left) 2024-09-21T09:34:13.461 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 37 current snap is 92 2024-09-21T09:34:13.461 INFO:tasks.rados.rados.0.smithi022.stdout:926: seq_num 310 ranges {1261568=401408} 2024-09-21T09:34:13.470 INFO:tasks.rados.rados.0.smithi022.stdout:926: writing smithi02284960-37 from 1261568 to 1662976 tid 1 2024-09-21T09:34:13.470 INFO:tasks.rados.rados.0.smithi022.stdout:927: snap_create 2024-09-21T09:34:13.475 INFO:tasks.rados.rados.0.smithi022.stdout:926: finishing write tid 1 to smithi02284960-37 2024-09-21T09:34:13.476 INFO:tasks.rados.rados.0.smithi022.stdout:926: finishing write tid 2 to smithi02284960-37 2024-09-21T09:34:13.477 INFO:tasks.rados.rados.0.smithi022.stdout:926: finishing write tid 3 to smithi02284960-37 2024-09-21T09:34:13.477 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 37 v 568 (ObjNum 310 snap 92 seq_num 310) dirty exists 2024-09-21T09:34:13.477 INFO:tasks.rados.rados.0.smithi022.stdout:926: left oid 37 (ObjNum 310 snap 92 seq_num 310) 2024-09-21T09:34:13.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:13 smithi064 ceph-mon[31124]: pgmap v21: 121 pgs: 121 active+clean; 233 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.5 MiB/s rd, 528 KiB/s wr, 25 op/s 2024-09-21T09:34:13.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:13 smithi064 ceph-mon[31124]: osdmap e243: 8 total, 8 up, 8 in 2024-09-21T09:34:13.692 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:34:13 smithi064 systemd[1]: Stopping Ceph prometheus.a for d65e801e-77fa-11ef-baf6-efdc52797490... 2024-09-21T09:34:13.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:13 smithi022 ceph-mon[39525]: pgmap v21: 121 pgs: 121 active+clean; 233 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.5 MiB/s rd, 528 KiB/s wr, 25 op/s 2024-09-21T09:34:13.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:13 smithi022 ceph-mon[39525]: osdmap e243: 8 total, 8 up, 8 in 2024-09-21T09:34:13.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:13 smithi022 ceph-mon[27763]: pgmap v21: 121 pgs: 121 active+clean; 233 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.5 MiB/s rd, 528 KiB/s wr, 25 op/s 2024-09-21T09:34:13.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:13 smithi022 ceph-mon[27763]: osdmap e243: 8 total, 8 up, 8 in 2024-09-21T09:34:14.118 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:34:13 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[75413]: ts=2024-09-21T09:34:13.793Z caller=main.go:964 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-09-21T09:34:14.119 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:34:13 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[75413]: ts=2024-09-21T09:34:13.793Z caller=main.go:988 level=info msg="Stopping scrape discovery manager..." 2024-09-21T09:34:14.119 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:34:13 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[75413]: ts=2024-09-21T09:34:13.793Z caller=main.go:1002 level=info msg="Stopping notify discovery manager..." 2024-09-21T09:34:14.119 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:34:13 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[75413]: ts=2024-09-21T09:34:13.793Z caller=manager.go:177 level=info component="rule manager" msg="Stopping rule manager..." 2024-09-21T09:34:14.119 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:34:13 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[75413]: ts=2024-09-21T09:34:13.793Z caller=main.go:998 level=info msg="Notify discovery manager stopped" 2024-09-21T09:34:14.119 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:34:13 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[75413]: ts=2024-09-21T09:34:13.793Z caller=main.go:984 level=info msg="Scrape discovery manager stopped" 2024-09-21T09:34:14.119 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:34:13 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[75413]: ts=2024-09-21T09:34:13.793Z caller=manager.go:187 level=info component="rule manager" msg="Rule manager stopped" 2024-09-21T09:34:14.119 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:34:13 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[75413]: ts=2024-09-21T09:34:13.793Z caller=main.go:1039 level=info msg="Stopping scrape manager..." 2024-09-21T09:34:14.119 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:34:13 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[75413]: ts=2024-09-21T09:34:13.793Z caller=main.go:1031 level=info msg="Scrape manager stopped" 2024-09-21T09:34:14.119 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:34:13 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[75413]: ts=2024-09-21T09:34:13.795Z caller=notifier.go:618 level=info component=notifier msg="Stopping notification manager..." 2024-09-21T09:34:14.119 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:34:13 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[75413]: ts=2024-09-21T09:34:13.795Z caller=main.go:1261 level=info msg="Notifier manager stopped" 2024-09-21T09:34:14.119 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:34:13 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[75413]: ts=2024-09-21T09:34:13.795Z caller=main.go:1273 level=info msg="See you next time!" 2024-09-21T09:34:14.119 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:34:13 smithi064 podman[78268]: 2024-09-21 09:34:13.803903669 +0000 UTC m=+0.195503270 container died e96ed2646ce4124317b10f6f75452885c3affcab89f3bb4cddb9ebc9d46a9ec0 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:34:14.427 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:34:14 smithi064 podman[78268]: 2024-09-21 09:34:14.216259037 +0000 UTC m=+0.607858639 container cleanup e96ed2646ce4124317b10f6f75452885c3affcab89f3bb4cddb9ebc9d46a9ec0 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:34:14.427 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:34:14 smithi064 bash[78268]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a 2024-09-21T09:34:14.467 INFO:tasks.rados.rados.0.smithi022.stdout:926: done (1 left) 2024-09-21T09:34:14.467 INFO:tasks.rados.rados.0.smithi022.stdout:927: done (0 left) 2024-09-21T09:34:14.467 INFO:tasks.rados.rados.0.smithi022.stdout:928: read oid 11 snap 56 2024-09-21T09:34:14.467 INFO:tasks.rados.rados.0.smithi022.stdout:928: expect (ObjNum 193 snap 49 seq_num 193) 2024-09-21T09:34:14.467 INFO:tasks.rados.rados.0.smithi022.stdout:929: snap_create 2024-09-21T09:34:14.480 INFO:tasks.rados.rados.0.smithi022.stdout:928: expect (ObjNum 193 snap 49 seq_num 193) 2024-09-21T09:34:14.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:14 smithi064 ceph-mon[31124]: osdmap e244: 8 total, 8 up, 8 in 2024-09-21T09:34:14.691 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:34:14 smithi064 podman[78287]: 2024-09-21 09:34:14.425862039 +0000 UTC m=+0.618195956 container remove e96ed2646ce4124317b10f6f75452885c3affcab89f3bb4cddb9ebc9d46a9ec0 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:34:14.692 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:34:14 smithi064 systemd[1]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490@prometheus.a.service: Deactivated successfully. 2024-09-21T09:34:14.692 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:34:14 smithi064 systemd[1]: Stopped Ceph prometheus.a for d65e801e-77fa-11ef-baf6-efdc52797490. 2024-09-21T09:34:14.692 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:34:14 smithi064 systemd[1]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490@prometheus.a.service: Consumed 1.148s CPU time. 2024-09-21T09:34:14.692 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:34:14 smithi064 systemd[1]: Starting Ceph prometheus.a for d65e801e-77fa-11ef-baf6-efdc52797490... 2024-09-21T09:34:14.807 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:14 smithi022 ceph-mon[39525]: osdmap e244: 8 total, 8 up, 8 in 2024-09-21T09:34:14.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:14 smithi022 ceph-mon[27763]: osdmap e244: 8 total, 8 up, 8 in 2024-09-21T09:34:15.130 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:34:14 smithi064 podman[78380]: 2024-09-21 09:34:14.775607356 +0000 UTC m=+0.017702180 image pull 1d3b7f56885b6dd623f1785be963aa9c195f86bc256ea454e8d02a7980b79c53 quay.io/prometheus/prometheus:v2.51.0 2024-09-21T09:34:15.130 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:34:14 smithi064 podman[78380]: 2024-09-21 09:34:14.87678335 +0000 UTC m=+0.118878163 container create 80629ffa21794f8a19c6f57c79913a56e5e53cf9661806f1399205594ad64556 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:34:15.130 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:34:15 smithi064 podman[78380]: 2024-09-21 09:34:15.103959089 +0000 UTC m=+0.346053911 container init 80629ffa21794f8a19c6f57c79913a56e5e53cf9661806f1399205594ad64556 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:34:15.130 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:34:15 smithi064 podman[78380]: 2024-09-21 09:34:15.109113762 +0000 UTC m=+0.351208575 container start 80629ffa21794f8a19c6f57c79913a56e5e53cf9661806f1399205594ad64556 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:34:15.442 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:34:15 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:34:15.128Z caller=main.go:617 level=info msg="Starting Prometheus Server" mode=server version="(version=2.51.0, branch=HEAD, revision=c05c15512acb675e3f6cd662a6727854e93fc024)" 2024-09-21T09:34:15.442 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:34:15 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:34:15.128Z caller=main.go:622 level=info build_context="(go=go1.22.1, platform=linux/amd64, user=root@b5723e458358, date=20240319-10:54:45, tags=netgo,builtinassets,stringlabels)" 2024-09-21T09:34:15.443 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:34:15 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:34:15.128Z caller=main.go:623 level=info host_details="(Linux 5.14.0-508.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024 x86_64 smithi064 (none))" 2024-09-21T09:34:15.443 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:34:15 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:34:15.128Z caller=main.go:624 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-09-21T09:34:15.443 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:34:15 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:34:15.128Z caller=main.go:625 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-21T09:34:15.443 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:34:15 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:34:15.130Z caller=web.go:568 level=info component=web msg="Start listening for connections" address=:9095 2024-09-21T09:34:15.443 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:34:15 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:34:15.131Z caller=main.go:1129 level=info msg="Starting TSDB ..." 2024-09-21T09:34:15.443 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:34:15 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:34:15.132Z caller=tls_config.go:313 level=info component=web msg="Listening on" address=[::]:9095 2024-09-21T09:34:15.443 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:34:15 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:34:15.132Z caller=tls_config.go:316 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-09-21T09:34:15.443 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:34:15 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:34:15.134Z caller=head.go:616 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-09-21T09:34:15.443 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:34:15 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:34:15.134Z caller=head.go:698 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.956µs 2024-09-21T09:34:15.443 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:34:15 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:34:15.134Z caller=head.go:706 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-09-21T09:34:15.443 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:34:15 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:34:15.134Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=3 2024-09-21T09:34:15.443 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:34:15 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:34:15.147Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=3 2024-09-21T09:34:15.443 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:34:15 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:34:15.152Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=3 2024-09-21T09:34:15.444 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:34:15 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:34:15.152Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=3 2024-09-21T09:34:15.444 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:34:15 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:34:15.152Z caller=head.go:815 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=29.294µs wal_replay_duration=17.68226ms wbl_replay_duration=184ns total_replay_duration=17.730274ms 2024-09-21T09:34:15.444 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:34:15 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:34:15.155Z caller=main.go:1150 level=info fs_type=XFS_SUPER_MAGIC 2024-09-21T09:34:15.444 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:34:15 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:34:15.155Z caller=main.go:1153 level=info msg="TSDB started" 2024-09-21T09:34:15.444 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:34:15 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:34:15.155Z caller=main.go:1335 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-21T09:34:15.444 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:34:15 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:34:15.171Z caller=main.go:1372 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=15.919118ms db_storage=946ns remote_storage=1.091µs web_handler=338ns query_engine=706ns scrape=764.019µs scrape_sd=200.144µs notify=23.393µs notify_sd=22.637µs rules=14.526026ms tracing=5.951µs 2024-09-21T09:34:15.444 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:34:15 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:34:15.171Z caller=main.go:1114 level=info msg="Server is ready to receive web requests." 2024-09-21T09:34:15.444 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:34:15 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:34:15.171Z caller=manager.go:163 level=info component="rule manager" msg="Starting rule manager..." 2024-09-21T09:34:15.444 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:34:15 smithi064 bash[78380]: 80629ffa21794f8a19c6f57c79913a56e5e53cf9661806f1399205594ad64556 2024-09-21T09:34:15.444 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:34:15 smithi064 systemd[1]: Started Ceph prometheus.a for d65e801e-77fa-11ef-baf6-efdc52797490. 2024-09-21T09:34:15.456 INFO:tasks.rados.rados.0.smithi022.stdout:928: done (1 left) 2024-09-21T09:34:15.456 INFO:tasks.rados.rados.0.smithi022.stdout:929: done (0 left) 2024-09-21T09:34:15.456 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 10 current snap is 94 2024-09-21T09:34:15.456 INFO:tasks.rados.rados.0.smithi022.stdout:930: seq_num 311 ranges {196608=417792} 2024-09-21T09:34:15.460 INFO:tasks.rados.rados.0.smithi022.stdout:930: writing smithi02284960-10 from 196608 to 614400 tid 1 2024-09-21T09:34:15.460 INFO:tasks.rados.rados.0.smithi022.stdout:931: copy_from oid 5 from oid 16 current snap is 94 2024-09-21T09:34:15.460 INFO:tasks.rados.rados.0.smithi022.stdout:932: copy_from oid 36 from oid 3 current snap is 94 2024-09-21T09:34:15.460 INFO:tasks.rados.rados.0.smithi022.stdout:933: read oid 6 snap -1 2024-09-21T09:34:15.460 INFO:tasks.rados.rados.0.smithi022.stdout:933: expect (ObjNum 142 snap 32 seq_num 142) 2024-09-21T09:34:15.460 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 8 current snap is 94 2024-09-21T09:34:15.460 INFO:tasks.rados.rados.0.smithi022.stdout:934: seq_num 312 ranges {0=434176} 2024-09-21T09:34:15.462 INFO:tasks.rados.rados.0.smithi022.stdout:934: writing smithi02284960-8 from 0 to 434176 tid 1 2024-09-21T09:34:15.462 INFO:tasks.rados.rados.0.smithi022.stdout:935: delete oid 30 current snap is 94 2024-09-21T09:34:15.466 INFO:tasks.rados.rados.0.smithi022.stdout:935: done (5 left) 2024-09-21T09:34:15.466 INFO:tasks.rados.rados.0.smithi022.stdout:936: snap_create 2024-09-21T09:34:15.466 INFO:tasks.rados.rados.0.smithi022.stdout:930: finishing write tid 1 to smithi02284960-10 2024-09-21T09:34:15.466 INFO:tasks.rados.rados.0.smithi022.stdout:931: finishing copy_from to smithi02284960-5 2024-09-21T09:34:15.467 INFO:tasks.rados.rados.0.smithi022.stdout:931: got expected ENOENT (src dne) 2024-09-21T09:34:15.467 INFO:tasks.rados.rados.0.smithi022.stdout:931: finishing copy_from racing read to smithi02284960-5 2024-09-21T09:34:15.467 INFO:tasks.rados.rados.0.smithi022.stdout:930: finishing write tid 2 to smithi02284960-10 2024-09-21T09:34:15.468 INFO:tasks.rados.rados.0.smithi022.stdout:934: finishing write tid 1 to smithi02284960-8 2024-09-21T09:34:15.469 INFO:tasks.rados.rados.0.smithi022.stdout:930: finishing write tid 3 to smithi02284960-10 2024-09-21T09:34:15.469 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 10 v 535 (ObjNum 311 snap 94 seq_num 311) dirty exists 2024-09-21T09:34:15.469 INFO:tasks.rados.rados.0.smithi022.stdout:930: left oid 10 (ObjNum 311 snap 94 seq_num 311) 2024-09-21T09:34:15.469 INFO:tasks.rados.rados.0.smithi022.stdout:934: finishing write tid 2 to smithi02284960-8 2024-09-21T09:34:15.472 INFO:tasks.rados.rados.0.smithi022.stdout:934: finishing write tid 3 to smithi02284960-8 2024-09-21T09:34:15.472 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 8 v 453 (ObjNum 312 snap 94 seq_num 312) dirty exists 2024-09-21T09:34:15.472 INFO:tasks.rados.rados.0.smithi022.stdout:934: left oid 8 (ObjNum 312 snap 94 seq_num 312) 2024-09-21T09:34:15.475 INFO:tasks.rados.rados.0.smithi022.stdout:932: finishing copy_from racing read to smithi02284960-36 2024-09-21T09:34:15.475 INFO:tasks.rados.rados.0.smithi022.stdout:932: finishing copy_from to smithi02284960-36 2024-09-21T09:34:15.475 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 36 v 93 (ObjNum 195 snap 50 seq_num 195) dirty exists 2024-09-21T09:34:15.491 INFO:tasks.rados.rados.0.smithi022.stdout:933: expect (ObjNum 142 snap 32 seq_num 142) 2024-09-21T09:34:15.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:15 smithi022 ceph-mon[39525]: pgmap v24: 121 pgs: 121 active+clean; 233 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.5 MiB/s rd, 528 KiB/s wr, 25 op/s 2024-09-21T09:34:15.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:15 smithi022 ceph-mon[39525]: osdmap e245: 8 total, 8 up, 8 in 2024-09-21T09:34:15.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:15 smithi022 ceph-mon[39525]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' 2024-09-21T09:34:15.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:15 smithi022 ceph-mon[39525]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' 2024-09-21T09:34:15.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:15 smithi022 ceph-mon[39525]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T09:34:15.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:15 smithi022 ceph-mon[39525]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:34:15.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:15 smithi022 ceph-mon[39525]: osdmap e246: 8 total, 8 up, 8 in 2024-09-21T09:34:15.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:15 smithi022 ceph-mon[27763]: pgmap v24: 121 pgs: 121 active+clean; 233 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.5 MiB/s rd, 528 KiB/s wr, 25 op/s 2024-09-21T09:34:15.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:15 smithi022 ceph-mon[27763]: osdmap e245: 8 total, 8 up, 8 in 2024-09-21T09:34:15.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:15 smithi022 ceph-mon[27763]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' 2024-09-21T09:34:15.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:15 smithi022 ceph-mon[27763]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' 2024-09-21T09:34:15.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:15 smithi022 ceph-mon[27763]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T09:34:15.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:15 smithi022 ceph-mon[27763]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:34:15.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:15 smithi022 ceph-mon[27763]: osdmap e246: 8 total, 8 up, 8 in 2024-09-21T09:34:15.828 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:15 smithi064 ceph-mon[31124]: pgmap v24: 121 pgs: 121 active+clean; 233 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.5 MiB/s rd, 528 KiB/s wr, 25 op/s 2024-09-21T09:34:15.828 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:15 smithi064 ceph-mon[31124]: osdmap e245: 8 total, 8 up, 8 in 2024-09-21T09:34:15.828 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:15 smithi064 ceph-mon[31124]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' 2024-09-21T09:34:15.828 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:15 smithi064 ceph-mon[31124]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' 2024-09-21T09:34:15.828 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:15 smithi064 ceph-mon[31124]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T09:34:15.828 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:15 smithi064 ceph-mon[31124]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:34:15.828 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:15 smithi064 ceph-mon[31124]: osdmap e246: 8 total, 8 up, 8 in 2024-09-21T09:34:16.458 INFO:tasks.rados.rados.0.smithi022.stdout:930: done (5 left) 2024-09-21T09:34:16.458 INFO:tasks.rados.rados.0.smithi022.stdout:931: done (4 left) 2024-09-21T09:34:16.458 INFO:tasks.rados.rados.0.smithi022.stdout:932: done (3 left) 2024-09-21T09:34:16.458 INFO:tasks.rados.rados.0.smithi022.stdout:933: done (2 left) 2024-09-21T09:34:16.458 INFO:tasks.rados.rados.0.smithi022.stdout:934: done (1 left) 2024-09-21T09:34:16.458 INFO:tasks.rados.rados.0.smithi022.stdout:936: done (0 left) 2024-09-21T09:34:16.458 INFO:tasks.rados.rados.0.smithi022.stdout:937: setattr oid 39 current snap is 95 2024-09-21T09:34:16.459 INFO:tasks.rados.rados.0.smithi022.stdout:938: read oid 37 snap -1 2024-09-21T09:34:16.459 INFO:tasks.rados.rados.0.smithi022.stdout:938: expect (ObjNum 310 snap 92 seq_num 310) 2024-09-21T09:34:16.459 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 33 current snap is 95 2024-09-21T09:34:16.459 INFO:tasks.rados.rados.0.smithi022.stdout:939: seq_num 314 ranges {0=466944} 2024-09-21T09:34:16.462 INFO:tasks.rados.rados.0.smithi022.stdout:939: writing smithi02284960-33 from 0 to 466944 tid 1 2024-09-21T09:34:16.462 INFO:tasks.rados.rados.0.smithi022.stdout:940: snap_remove snap 91 2024-09-21T09:34:16.755 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:16 smithi022 ceph-mon[39525]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T09:34:16.755 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:16 smithi022 ceph-mon[39525]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:34:16.755 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:16 smithi022 ceph-mon[39525]: osdmap e247: 8 total, 8 up, 8 in 2024-09-21T09:34:16.756 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:16 smithi022 ceph-mon[27763]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T09:34:16.756 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:16 smithi022 ceph-mon[27763]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:34:16.756 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:16 smithi022 ceph-mon[27763]: osdmap e247: 8 total, 8 up, 8 in 2024-09-21T09:34:16.838 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:16 smithi064 ceph-mon[31124]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T09:34:16.838 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:16 smithi064 ceph-mon[31124]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:34:16.838 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:16 smithi064 ceph-mon[31124]: osdmap e247: 8 total, 8 up, 8 in 2024-09-21T09:34:17.191 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:34:16 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:34:16.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi022\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:34:17.460 INFO:tasks.rados.rados.0.smithi022.stdout:940: done (3 left) 2024-09-21T09:34:17.460 INFO:tasks.rados.rados.0.smithi022.stdout:941: setattr oid 47 current snap is 95 2024-09-21T09:34:17.460 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 39 v 289 (ObjNum 240 snap 65 seq_num 240) dirty exists 2024-09-21T09:34:17.460 INFO:tasks.rados.rados.0.smithi022.stdout:939: finishing write tid 1 to smithi02284960-33 2024-09-21T09:34:17.460 INFO:tasks.rados.rados.0.smithi022.stdout:939: finishing write tid 2 to smithi02284960-33 2024-09-21T09:34:17.460 INFO:tasks.rados.rados.0.smithi022.stdout:939: finishing write tid 3 to smithi02284960-33 2024-09-21T09:34:17.460 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 33 v 553 (ObjNum 314 snap 95 seq_num 314) dirty exists 2024-09-21T09:34:17.460 INFO:tasks.rados.rados.0.smithi022.stdout:939: left oid 33 (ObjNum 314 snap 95 seq_num 314) 2024-09-21T09:34:17.460 INFO:tasks.rados.rados.0.smithi022.stdout:938: expect (ObjNum 310 snap 92 seq_num 310) 2024-09-21T09:34:17.529 INFO:tasks.rados.rados.0.smithi022.stdout:937: done (3 left) 2024-09-21T09:34:17.529 INFO:tasks.rados.rados.0.smithi022.stdout:938: done (2 left) 2024-09-21T09:34:17.529 INFO:tasks.rados.rados.0.smithi022.stdout:939: done (1 left) 2024-09-21T09:34:17.529 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 32 current snap is 95 2024-09-21T09:34:17.529 INFO:tasks.rados.rados.0.smithi022.stdout:942: seq_num 315 ranges {5767168=483328} 2024-09-21T09:34:17.563 INFO:tasks.rados.rados.0.smithi022.stdout:942: writing smithi02284960-32 from 5767168 to 6250496 tid 1 2024-09-21T09:34:17.563 INFO:tasks.rados.rados.0.smithi022.stdout:943: delete oid 42 current snap is 95 2024-09-21T09:34:17.563 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 47 v 487 (ObjNum 161 snap 0 seq_num 1) dirty exists 2024-09-21T09:34:17.565 INFO:tasks.rados.rados.0.smithi022.stdout:941: done (2 left) 2024-09-21T09:34:17.565 INFO:tasks.rados.rados.0.smithi022.stdout:943: done (1 left) 2024-09-21T09:34:17.566 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 48 current snap is 95 2024-09-21T09:34:17.566 INFO:tasks.rados.rados.0.smithi022.stdout:944: seq_num 317 ranges {4227072=516096} 2024-09-21T09:34:17.591 INFO:tasks.rados.rados.0.smithi022.stdout:944: writing smithi02284960-48 from 4227072 to 4743168 tid 1 2024-09-21T09:34:17.591 INFO:tasks.rados.rados.0.smithi022.stdout:945: snap_create 2024-09-21T09:34:17.592 INFO:tasks.rados.rados.0.smithi022.stdout:942: finishing write tid 1 to smithi02284960-32 2024-09-21T09:34:17.592 INFO:tasks.rados.rados.0.smithi022.stdout:942: finishing write tid 2 to smithi02284960-32 2024-09-21T09:34:17.592 INFO:tasks.rados.rados.0.smithi022.stdout:942: finishing write tid 3 to smithi02284960-32 2024-09-21T09:34:17.592 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 32 v 556 (ObjNum 315 snap 95 seq_num 315) dirty exists 2024-09-21T09:34:17.592 INFO:tasks.rados.rados.0.smithi022.stdout:942: left oid 32 (ObjNum 315 snap 95 seq_num 315) 2024-09-21T09:34:17.594 INFO:tasks.rados.rados.0.smithi022.stdout:944: finishing write tid 1 to smithi02284960-48 2024-09-21T09:34:17.595 INFO:tasks.rados.rados.0.smithi022.stdout:944: finishing write tid 2 to smithi02284960-48 2024-09-21T09:34:17.596 INFO:tasks.rados.rados.0.smithi022.stdout:944: finishing write tid 3 to smithi02284960-48 2024-09-21T09:34:17.596 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 48 v 559 (ObjNum 317 snap 95 seq_num 317) dirty exists 2024-09-21T09:34:17.596 INFO:tasks.rados.rados.0.smithi022.stdout:944: left oid 48 (ObjNum 317 snap 95 seq_num 317) 2024-09-21T09:34:17.758 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:17 smithi022 ceph-mon[39525]: pgmap v27: 121 pgs: 121 active+clean; 233 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:34:17.758 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:17 smithi022 ceph-mon[39525]: Upgrade: Updating mgr.x 2024-09-21T09:34:17.759 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:17 smithi022 ceph-mon[39525]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T09:34:17.759 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:17 smithi022 ceph-mon[39525]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T09:34:17.759 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:17 smithi022 ceph-mon[39525]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:34:17.759 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:17 smithi022 ceph-mon[39525]: Deploying daemon mgr.x on smithi064 2024-09-21T09:34:17.759 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:17 smithi022 ceph-mon[39525]: osdmap e248: 8 total, 8 up, 8 in 2024-09-21T09:34:17.759 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:17 smithi022 ceph-mon[27763]: pgmap v27: 121 pgs: 121 active+clean; 233 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:34:17.759 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:17 smithi022 ceph-mon[27763]: Upgrade: Updating mgr.x 2024-09-21T09:34:17.759 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:17 smithi022 ceph-mon[27763]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T09:34:17.759 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:17 smithi022 ceph-mon[27763]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T09:34:17.759 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:17 smithi022 ceph-mon[27763]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:34:17.760 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:17 smithi022 ceph-mon[27763]: Deploying daemon mgr.x on smithi064 2024-09-21T09:34:17.760 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:17 smithi022 ceph-mon[27763]: osdmap e248: 8 total, 8 up, 8 in 2024-09-21T09:34:17.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:17 smithi064 ceph-mon[31124]: pgmap v27: 121 pgs: 121 active+clean; 233 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:34:17.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:17 smithi064 ceph-mon[31124]: Upgrade: Updating mgr.x 2024-09-21T09:34:17.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:17 smithi064 ceph-mon[31124]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T09:34:17.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:17 smithi064 ceph-mon[31124]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T09:34:17.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:17 smithi064 ceph-mon[31124]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:34:17.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:17 smithi064 ceph-mon[31124]: Deploying daemon mgr.x on smithi064 2024-09-21T09:34:17.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:17 smithi064 ceph-mon[31124]: osdmap e248: 8 total, 8 up, 8 in 2024-09-21T09:34:18.485 INFO:tasks.rados.rados.0.smithi022.stdout:942: done (2 left) 2024-09-21T09:34:18.485 INFO:tasks.rados.rados.0.smithi022.stdout:944: done (1 left) 2024-09-21T09:34:18.486 INFO:tasks.rados.rados.0.smithi022.stdout:945: done (0 left) 2024-09-21T09:34:18.486 INFO:tasks.rados.rados.0.smithi022.stdout:946: snap_remove snap 82 2024-09-21T09:34:18.941 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:34:18 smithi064 systemd[1]: Stopping Ceph mgr.x for d65e801e-77fa-11ef-baf6-efdc52797490... 2024-09-21T09:34:19.191 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:34:18 smithi064 podman[78721]: 2024-09-21 09:34:18.939867699 +0000 UTC m=+0.146535533 container died c102c1678bcc148d502e16cf7be5992efdfd0cd8bae917048620aa9c3b28cbb8 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_CLEAN=True, RELEASE=main-62da54b, io.buildah.version=1.37.2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T09:34:19.489 INFO:tasks.rados.rados.0.smithi022.stdout:946: done (0 left) 2024-09-21T09:34:19.490 INFO:tasks.rados.rados.0.smithi022.stdout:947: rmattr oid 46 current snap is 96 2024-09-21T09:34:19.490 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 48 current snap is 96 2024-09-21T09:34:19.490 INFO:tasks.rados.rados.0.smithi022.stdout:948: seq_num 318 ranges {4743168=532480} 2024-09-21T09:34:19.492 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:34:19 smithi064 podman[78721]: 2024-09-21 09:34:19.314241128 +0000 UTC m=+0.520908962 container cleanup c102c1678bcc148d502e16cf7be5992efdfd0cd8bae917048620aa9c3b28cbb8 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , RELEASE=main-62da54b, org.label-schema.build-date=20240731, ceph=True, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2) 2024-09-21T09:34:19.492 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:34:19 smithi064 bash[78721]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x 2024-09-21T09:34:19.520 INFO:tasks.rados.rados.0.smithi022.stdout:948: writing smithi02284960-48 from 4743168 to 5275648 tid 1 2024-09-21T09:34:19.520 INFO:tasks.rados.rados.0.smithi022.stdout:949: delete oid 18 current snap is 96 2024-09-21T09:34:19.520 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 46 v 489 (ObjNum 204 snap 56 seq_num 204) dirty exists 2024-09-21T09:34:19.522 INFO:tasks.rados.rados.0.smithi022.stdout:947: done (2 left) 2024-09-21T09:34:19.523 INFO:tasks.rados.rados.0.smithi022.stdout:949: done (1 left) 2024-09-21T09:34:19.523 INFO:tasks.rados.rados.0.smithi022.stdout:950: read oid 49 snap -1 2024-09-21T09:34:19.523 INFO:tasks.rados.rados.0.smithi022.stdout:950: expect (ObjNum 300 snap 88 seq_num 300) 2024-09-21T09:34:19.523 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 27 current snap is 96 2024-09-21T09:34:19.523 INFO:tasks.rados.rados.0.smithi022.stdout:951: seq_num 320 ranges {0=565248} 2024-09-21T09:34:19.526 INFO:tasks.rados.rados.0.smithi022.stdout:951: writing smithi02284960-27 from 0 to 565248 tid 1 2024-09-21T09:34:19.527 INFO:tasks.rados.rados.0.smithi022.stdout:952: read oid 38 snap -1 2024-09-21T09:34:19.527 INFO:tasks.rados.rados.0.smithi022.stdout:952: expect (ObjNum 142 snap 32 seq_num 142) 2024-09-21T09:34:19.527 INFO:tasks.rados.rados.0.smithi022.stdout:953: rmattr oid 12 current snap is 96 2024-09-21T09:34:19.527 INFO:tasks.rados.rados.0.smithi022.stdout:953: done (4 left) 2024-09-21T09:34:19.527 INFO:tasks.rados.rados.0.smithi022.stdout:954: delete oid 29 current snap is 96 2024-09-21T09:34:19.527 INFO:tasks.rados.rados.0.smithi022.stdout:948: finishing write tid 1 to smithi02284960-48 2024-09-21T09:34:19.527 INFO:tasks.rados.rados.0.smithi022.stdout:948: finishing write tid 2 to smithi02284960-48 2024-09-21T09:34:19.527 INFO:tasks.rados.rados.0.smithi022.stdout:948: finishing write tid 3 to smithi02284960-48 2024-09-21T09:34:19.527 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 48 v 570 (ObjNum 318 snap 96 seq_num 318) dirty exists 2024-09-21T09:34:19.527 INFO:tasks.rados.rados.0.smithi022.stdout:948: left oid 48 (ObjNum 318 snap 96 seq_num 318) 2024-09-21T09:34:19.530 INFO:tasks.rados.rados.0.smithi022.stdout:951: finishing write tid 1 to smithi02284960-27 2024-09-21T09:34:19.530 INFO:tasks.rados.rados.0.smithi022.stdout:951: finishing write tid 2 to smithi02284960-27 2024-09-21T09:34:19.532 INFO:tasks.rados.rados.0.smithi022.stdout:951: finishing write tid 3 to smithi02284960-27 2024-09-21T09:34:19.532 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 27 v 455 (ObjNum 320 snap 96 seq_num 320) dirty exists 2024-09-21T09:34:19.532 INFO:tasks.rados.rados.0.smithi022.stdout:951: left oid 27 (ObjNum 320 snap 96 seq_num 320) 2024-09-21T09:34:19.536 INFO:tasks.rados.rados.0.smithi022.stdout:950: expect (ObjNum 300 snap 88 seq_num 300) 2024-09-21T09:34:19.642 INFO:tasks.rados.rados.0.smithi022.stdout:952: expect (ObjNum 142 snap 32 seq_num 142) 2024-09-21T09:34:19.755 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:19 smithi064 ceph-mon[31124]: pgmap v30: 121 pgs: 121 active+clean; 232 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 8.1 MiB/s rd, 634 KiB/s wr, 16 op/s 2024-09-21T09:34:19.756 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:19 smithi064 ceph-mon[31124]: osdmap e249: 8 total, 8 up, 8 in 2024-09-21T09:34:19.756 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:34:19 smithi064 podman[78738]: 2024-09-21 09:34:19.499059073 +0000 UTC m=+0.558376203 container remove c102c1678bcc148d502e16cf7be5992efdfd0cd8bae917048620aa9c3b28cbb8 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, RELEASE=main-62da54b, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, ceph=True, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T09:34:19.756 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:34:19 smithi064 systemd[1]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-09-21T09:34:19.756 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:34:19 smithi064 systemd[1]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490@mgr.x.service: Failed with result 'exit-code'. 2024-09-21T09:34:19.756 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:34:19 smithi064 systemd[1]: Stopped Ceph mgr.x for d65e801e-77fa-11ef-baf6-efdc52797490. 2024-09-21T09:34:19.756 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:34:19 smithi064 systemd[1]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490@mgr.x.service: Consumed 19.219s CPU time. 2024-09-21T09:34:19.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:19 smithi022 ceph-mon[39525]: pgmap v30: 121 pgs: 121 active+clean; 232 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 8.1 MiB/s rd, 634 KiB/s wr, 16 op/s 2024-09-21T09:34:19.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:19 smithi022 ceph-mon[39525]: osdmap e249: 8 total, 8 up, 8 in 2024-09-21T09:34:19.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:19 smithi022 ceph-mon[27763]: pgmap v30: 121 pgs: 121 active+clean; 232 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 8.1 MiB/s rd, 634 KiB/s wr, 16 op/s 2024-09-21T09:34:19.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:19 smithi022 ceph-mon[27763]: osdmap e249: 8 total, 8 up, 8 in 2024-09-21T09:34:19.936 INFO:tasks.rados.rados.0.smithi022.stdout:948: done (4 left) 2024-09-21T09:34:19.936 INFO:tasks.rados.rados.0.smithi022.stdout:950: done (3 left) 2024-09-21T09:34:19.936 INFO:tasks.rados.rados.0.smithi022.stdout:951: done (2 left) 2024-09-21T09:34:19.936 INFO:tasks.rados.rados.0.smithi022.stdout:952: done (1 left) 2024-09-21T09:34:19.937 INFO:tasks.rados.rados.0.smithi022.stdout:954: done (0 left) 2024-09-21T09:34:19.937 INFO:tasks.rados.rados.0.smithi022.stdout:955: copy_from oid 50 from oid 30 current snap is 96 2024-09-21T09:34:19.937 INFO:tasks.rados.rados.0.smithi022.stdout:956: snap_remove snap 94 2024-09-21T09:34:20.159 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:34:19 smithi064 systemd[1]: Starting Ceph mgr.x for d65e801e-77fa-11ef-baf6-efdc52797490... 2024-09-21T09:34:20.434 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:34:20 smithi064 podman[78869]: 2024-09-21 09:34:20.061077118 +0000 UTC m=+0.016315465 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:34:20.434 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:34:20 smithi064 podman[78869]: 2024-09-21 09:34:20.175236454 +0000 UTC m=+0.130474831 container create 61abc61fea8d38822833e6734ef06390abf36052535b0473748ff5f90aae7d43 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-62da54b, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS) 2024-09-21T09:34:20.434 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:34:20 smithi064 podman[78869]: 2024-09-21 09:34:20.402911216 +0000 UTC m=+0.358149563 container init 61abc61fea8d38822833e6734ef06390abf36052535b0473748ff5f90aae7d43 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-62da54b, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, ceph=True) 2024-09-21T09:34:20.434 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:34:20 smithi064 podman[78869]: 2024-09-21 09:34:20.407679203 +0000 UTC m=+0.362917533 container start 61abc61fea8d38822833e6734ef06390abf36052535b0473748ff5f90aae7d43 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-62da54b, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD) 2024-09-21T09:34:20.435 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:34:20 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:34:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:34:20.462 INFO:tasks.rados.rados.0.smithi022.stdout:956: done (1 left) 2024-09-21T09:34:20.463 INFO:tasks.rados.rados.0.smithi022.stdout:957: snap_create 2024-09-21T09:34:20.463 INFO:tasks.rados.rados.0.smithi022.stdout:955: finishing copy_from to smithi02284960-50 2024-09-21T09:34:20.463 INFO:tasks.rados.rados.0.smithi022.stdout:955: got expected ENOENT (src dne) 2024-09-21T09:34:20.463 INFO:tasks.rados.rados.0.smithi022.stdout:955: finishing copy_from racing read to smithi02284960-50 2024-09-21T09:34:20.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:20 smithi064 ceph-mon[31124]: osdmap e250: 8 total, 8 up, 8 in 2024-09-21T09:34:20.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:20 smithi064 ceph-mon[31124]: osdmap e251: 8 total, 8 up, 8 in 2024-09-21T09:34:20.691 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:34:20 smithi064 ceph-mgr[78907]: -- 172.21.15.64:0/1517980807 <== mon.1 v2:172.21.15.64:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5629502e10e0 con 0x5629502bf000 2024-09-21T09:34:20.692 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:34:20 smithi064 bash[78869]: 61abc61fea8d38822833e6734ef06390abf36052535b0473748ff5f90aae7d43 2024-09-21T09:34:20.692 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:34:20 smithi064 systemd[1]: Started Ceph mgr.x for d65e801e-77fa-11ef-baf6-efdc52797490. 2024-09-21T09:34:20.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:20 smithi022 ceph-mon[39525]: osdmap e250: 8 total, 8 up, 8 in 2024-09-21T09:34:20.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:20 smithi022 ceph-mon[39525]: osdmap e251: 8 total, 8 up, 8 in 2024-09-21T09:34:20.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:20 smithi022 ceph-mon[27763]: osdmap e250: 8 total, 8 up, 8 in 2024-09-21T09:34:20.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:20 smithi022 ceph-mon[27763]: osdmap e251: 8 total, 8 up, 8 in 2024-09-21T09:34:21.460 INFO:tasks.rados.rados.0.smithi022.stdout:955: done (1 left) 2024-09-21T09:34:21.460 INFO:tasks.rados.rados.0.smithi022.stdout:957: done (0 left) 2024-09-21T09:34:21.460 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 21 current snap is 97 2024-09-21T09:34:21.460 INFO:tasks.rados.rados.0.smithi022.stdout:958: seq_num 322 ranges {483328=598016} 2024-09-21T09:34:21.466 INFO:tasks.rados.rados.0.smithi022.stdout:958: writing smithi02284960-21 from 483328 to 1081344 tid 1 2024-09-21T09:34:21.466 INFO:tasks.rados.rados.0.smithi022.stdout:959: rollback oid 42 current snap is 97 2024-09-21T09:34:21.466 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 42 to 72 2024-09-21T09:34:21.466 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 47 current snap is 97 2024-09-21T09:34:21.466 INFO:tasks.rados.rados.0.smithi022.stdout:960: seq_num 323 ranges {0=614400} 2024-09-21T09:34:21.469 INFO:tasks.rados.rados.0.smithi022.stdout:960: writing smithi02284960-47 from 0 to 614400 tid 1 2024-09-21T09:34:21.469 INFO:tasks.rados.rados.0.smithi022.stdout:961: copy_from oid 13 from oid 7 current snap is 97 2024-09-21T09:34:21.469 INFO:tasks.rados.rados.0.smithi022.stdout:962: snap_create 2024-09-21T09:34:21.469 INFO:tasks.rados.rados.0.smithi022.stdout:959: finishing rollback tid 1 to smithi02284960-42 2024-09-21T09:34:21.470 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 42 v 449 (ObjNum 129 snap 0 seq_num 3569853608) dirty dne 2024-09-21T09:34:21.472 INFO:tasks.rados.rados.0.smithi022.stdout:958: finishing write tid 1 to smithi02284960-21 2024-09-21T09:34:21.473 INFO:tasks.rados.rados.0.smithi022.stdout:961: finishing copy_from to smithi02284960-13 2024-09-21T09:34:21.473 INFO:tasks.rados.rados.0.smithi022.stdout:961: got expected ENOENT (src dne) 2024-09-21T09:34:21.473 INFO:tasks.rados.rados.0.smithi022.stdout:958: finishing write tid 2 to smithi02284960-21 2024-09-21T09:34:21.473 INFO:tasks.rados.rados.0.smithi022.stdout:961: finishing copy_from racing read to smithi02284960-13 2024-09-21T09:34:21.474 INFO:tasks.rados.rados.0.smithi022.stdout:958: finishing write tid 3 to smithi02284960-21 2024-09-21T09:34:21.474 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 21 v 256 (ObjNum 322 snap 97 seq_num 322) dirty exists 2024-09-21T09:34:21.474 INFO:tasks.rados.rados.0.smithi022.stdout:958: left oid 21 (ObjNum 322 snap 97 seq_num 322) 2024-09-21T09:34:21.476 INFO:tasks.rados.rados.0.smithi022.stdout:960: finishing write tid 1 to smithi02284960-47 2024-09-21T09:34:21.476 INFO:tasks.rados.rados.0.smithi022.stdout:960: finishing write tid 2 to smithi02284960-47 2024-09-21T09:34:21.477 INFO:tasks.rados.rados.0.smithi022.stdout:960: finishing write tid 3 to smithi02284960-47 2024-09-21T09:34:21.478 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 47 v 502 (ObjNum 323 snap 97 seq_num 323) dirty exists 2024-09-21T09:34:21.478 INFO:tasks.rados.rados.0.smithi022.stdout:960: left oid 47 (ObjNum 323 snap 97 seq_num 323) 2024-09-21T09:34:21.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:21 smithi064 ceph-mon[31124]: pgmap v33: 121 pgs: 121 active+clean; 232 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 8.1 MiB/s rd, 634 KiB/s wr, 16 op/s 2024-09-21T09:34:21.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:21 smithi064 ceph-mon[31124]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' 2024-09-21T09:34:21.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:21 smithi064 ceph-mon[31124]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' 2024-09-21T09:34:21.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:21 smithi064 ceph-mon[31124]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:34:21.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:21 smithi064 ceph-mon[31124]: osdmap e252: 8 total, 8 up, 8 in 2024-09-21T09:34:22.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:21 smithi022 ceph-mon[39525]: pgmap v33: 121 pgs: 121 active+clean; 232 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 8.1 MiB/s rd, 634 KiB/s wr, 16 op/s 2024-09-21T09:34:22.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:21 smithi022 ceph-mon[39525]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' 2024-09-21T09:34:22.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:21 smithi022 ceph-mon[39525]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' 2024-09-21T09:34:22.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:21 smithi022 ceph-mon[39525]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:34:22.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:21 smithi022 ceph-mon[39525]: osdmap e252: 8 total, 8 up, 8 in 2024-09-21T09:34:22.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:21 smithi022 ceph-mon[27763]: pgmap v33: 121 pgs: 121 active+clean; 232 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 8.1 MiB/s rd, 634 KiB/s wr, 16 op/s 2024-09-21T09:34:22.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:21 smithi022 ceph-mon[27763]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' 2024-09-21T09:34:22.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:21 smithi022 ceph-mon[27763]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' 2024-09-21T09:34:22.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:21 smithi022 ceph-mon[27763]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:34:22.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:21 smithi022 ceph-mon[27763]: osdmap e252: 8 total, 8 up, 8 in 2024-09-21T09:34:22.461 INFO:tasks.rados.rados.0.smithi022.stdout:958: done (4 left) 2024-09-21T09:34:22.461 INFO:tasks.rados.rados.0.smithi022.stdout:959: done (3 left) 2024-09-21T09:34:22.461 INFO:tasks.rados.rados.0.smithi022.stdout:960: done (2 left) 2024-09-21T09:34:22.461 INFO:tasks.rados.rados.0.smithi022.stdout:961: done (1 left) 2024-09-21T09:34:22.461 INFO:tasks.rados.rados.0.smithi022.stdout:962: done (0 left) 2024-09-21T09:34:22.461 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 15 current snap is 98 2024-09-21T09:34:22.462 INFO:tasks.rados.rados.0.smithi022.stdout:963: seq_num 324 ranges {0=630784} 2024-09-21T09:34:22.464 INFO:tasks.rados.rados.0.smithi022.stdout:963: writing smithi02284960-15 from 0 to 630784 tid 1 2024-09-21T09:34:22.464 INFO:tasks.rados.rados.0.smithi022.stdout:964: delete oid 31 current snap is 98 2024-09-21T09:34:22.471 INFO:tasks.rados.rados.0.smithi022.stdout:963: finishing write tid 1 to smithi02284960-15 2024-09-21T09:34:22.471 INFO:tasks.rados.rados.0.smithi022.stdout:963: finishing write tid 2 to smithi02284960-15 2024-09-21T09:34:22.472 INFO:tasks.rados.rados.0.smithi022.stdout:963: finishing write tid 3 to smithi02284960-15 2024-09-21T09:34:22.472 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 15 v 580 (ObjNum 324 snap 98 seq_num 324) dirty exists 2024-09-21T09:34:22.472 INFO:tasks.rados.rados.0.smithi022.stdout:963: left oid 15 (ObjNum 324 snap 98 seq_num 324) 2024-09-21T09:34:22.472 INFO:tasks.rados.rados.0.smithi022.stdout:963: done (1 left) 2024-09-21T09:34:22.472 INFO:tasks.rados.rados.0.smithi022.stdout:964: done (0 left) 2024-09-21T09:34:22.473 INFO:tasks.rados.rados.0.smithi022.stdout:965: snap_create 2024-09-21T09:34:23.463 INFO:tasks.rados.rados.0.smithi022.stdout:965: done (0 left) 2024-09-21T09:34:23.463 INFO:tasks.rados.rados.0.smithi022.stdout:966: copy_from oid 33 from oid 44 current snap is 99 2024-09-21T09:34:23.463 INFO:tasks.rados.rados.0.smithi022.stdout:967: delete oid 3 current snap is 99 2024-09-21T09:34:23.470 INFO:tasks.rados.rados.0.smithi022.stdout:967: done (1 left) 2024-09-21T09:34:23.470 INFO:tasks.rados.rados.0.smithi022.stdout:968: snap_remove snap 59 2024-09-21T09:34:23.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:23 smithi022 ceph-mon[39525]: pgmap v36: 121 pgs: 121 active+clean; 233 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.8 MiB/s rd, 808 KiB/s wr, 13 op/s 2024-09-21T09:34:23.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:23 smithi022 ceph-mon[39525]: osdmap e253: 8 total, 8 up, 8 in 2024-09-21T09:34:23.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:23 smithi022 ceph-mon[27763]: pgmap v36: 121 pgs: 121 active+clean; 233 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.8 MiB/s rd, 808 KiB/s wr, 13 op/s 2024-09-21T09:34:23.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:23 smithi022 ceph-mon[27763]: osdmap e253: 8 total, 8 up, 8 in 2024-09-21T09:34:23.830 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:23 smithi064 ceph-mon[31124]: pgmap v36: 121 pgs: 121 active+clean; 233 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.8 MiB/s rd, 808 KiB/s wr, 13 op/s 2024-09-21T09:34:23.830 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:23 smithi064 ceph-mon[31124]: osdmap e253: 8 total, 8 up, 8 in 2024-09-21T09:34:24.474 INFO:tasks.rados.rados.0.smithi022.stdout:966: finishing copy_from racing read to smithi02284960-33 2024-09-21T09:34:24.475 INFO:tasks.rados.rados.0.smithi022.stdout:966: finishing copy_from to smithi02284960-33 2024-09-21T09:34:24.475 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 33 v 477 (ObjNum 309 snap 91 seq_num 309) dirty exists 2024-09-21T09:34:24.475 INFO:tasks.rados.rados.0.smithi022.stdout:966: done (1 left) 2024-09-21T09:34:24.475 INFO:tasks.rados.rados.0.smithi022.stdout:968: done (0 left) 2024-09-21T09:34:24.475 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 34 current snap is 99 2024-09-21T09:34:24.475 INFO:tasks.rados.rados.0.smithi022.stdout:969: seq_num 327 ranges {0=679936} 2024-09-21T09:34:24.480 INFO:tasks.rados.rados.0.smithi022.stdout:969: writing smithi02284960-34 from 0 to 679936 tid 1 2024-09-21T09:34:24.480 INFO:tasks.rados.rados.0.smithi022.stdout:970: read oid 25 snap -1 2024-09-21T09:34:24.480 INFO:tasks.rados.rados.0.smithi022.stdout:970: expect deleted 2024-09-21T09:34:24.480 INFO:tasks.rados.rados.0.smithi022.stdout:971: read oid 39 snap -1 2024-09-21T09:34:24.480 INFO:tasks.rados.rados.0.smithi022.stdout:971: expect (ObjNum 240 snap 65 seq_num 240) 2024-09-21T09:34:24.480 INFO:tasks.rados.rados.0.smithi022.stdout:972: copy_from oid 9 from oid 43 current snap is 99 2024-09-21T09:34:24.480 INFO:tasks.rados.rados.0.smithi022.stdout:973: rollback oid 5 current snap is 99 2024-09-21T09:34:24.480 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 5 to 72 2024-09-21T09:34:24.480 INFO:tasks.rados.rados.0.smithi022.stdout:974: snap_create 2024-09-21T09:34:24.487 INFO:tasks.rados.rados.0.smithi022.stdout:969: finishing write tid 1 to smithi02284960-34 2024-09-21T09:34:24.487 INFO:tasks.rados.rados.0.smithi022.stdout:969: finishing write tid 2 to smithi02284960-34 2024-09-21T09:34:24.488 INFO:tasks.rados.rados.0.smithi022.stdout:973: finishing rollback tid 1 to smithi02284960-5 2024-09-21T09:34:24.488 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 5 v 244 (ObjNum 161 snap 0 seq_num 1) dirty dne 2024-09-21T09:34:24.491 INFO:tasks.rados.rados.0.smithi022.stdout:969: finishing write tid 3 to smithi02284960-34 2024-09-21T09:34:24.491 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 34 v 262 (ObjNum 327 snap 99 seq_num 327) dirty exists 2024-09-21T09:34:24.491 INFO:tasks.rados.rados.0.smithi022.stdout:969: left oid 34 (ObjNum 327 snap 99 seq_num 327) 2024-09-21T09:34:24.495 INFO:tasks.rados.rados.0.smithi022.stdout:971: expect (ObjNum 240 snap 65 seq_num 240) 2024-09-21T09:34:24.592 INFO:tasks.rados.rados.0.smithi022.stdout:972: finishing copy_from racing read to smithi02284960-9 2024-09-21T09:34:24.592 INFO:tasks.rados.rados.0.smithi022.stdout:972: finishing copy_from to smithi02284960-9 2024-09-21T09:34:24.592 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 9 v 286 (ObjNum 240 snap 65 seq_num 240) dirty exists 2024-09-21T09:34:24.807 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:24 smithi022 ceph-mon[39525]: osdmap e254: 8 total, 8 up, 8 in 2024-09-21T09:34:24.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:24 smithi022 ceph-mon[27763]: osdmap e254: 8 total, 8 up, 8 in 2024-09-21T09:34:24.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:24 smithi064 ceph-mon[31124]: osdmap e254: 8 total, 8 up, 8 in 2024-09-21T09:34:25.461 INFO:tasks.rados.rados.0.smithi022.stdout:969: done (5 left) 2024-09-21T09:34:25.461 INFO:tasks.rados.rados.0.smithi022.stdout:970: done (4 left) 2024-09-21T09:34:25.461 INFO:tasks.rados.rados.0.smithi022.stdout:971: done (3 left) 2024-09-21T09:34:25.461 INFO:tasks.rados.rados.0.smithi022.stdout:972: done (2 left) 2024-09-21T09:34:25.461 INFO:tasks.rados.rados.0.smithi022.stdout:973: done (1 left) 2024-09-21T09:34:25.461 INFO:tasks.rados.rados.0.smithi022.stdout:974: done (0 left) 2024-09-21T09:34:25.461 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 1 current snap is 100 2024-09-21T09:34:25.461 INFO:tasks.rados.rados.0.smithi022.stdout:975: seq_num 328 ranges {0=696320} 2024-09-21T09:34:25.464 INFO:tasks.rados.rados.0.smithi022.stdout:975: writing smithi02284960-1 from 0 to 696320 tid 1 2024-09-21T09:34:25.464 INFO:tasks.rados.rados.0.smithi022.stdout:976: read oid 35 snap -1 2024-09-21T09:34:25.464 INFO:tasks.rados.rados.0.smithi022.stdout:976: expect (ObjNum 264 snap 76 seq_num 264) 2024-09-21T09:34:25.464 INFO:tasks.rados.rados.0.smithi022.stdout:977: snap_create 2024-09-21T09:34:25.471 INFO:tasks.rados.rados.0.smithi022.stdout:975: finishing write tid 1 to smithi02284960-1 2024-09-21T09:34:25.471 INFO:tasks.rados.rados.0.smithi022.stdout:975: finishing write tid 2 to smithi02284960-1 2024-09-21T09:34:25.474 INFO:tasks.rados.rados.0.smithi022.stdout:975: finishing write tid 3 to smithi02284960-1 2024-09-21T09:34:25.474 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 1 v 514 (ObjNum 328 snap 100 seq_num 328) dirty exists 2024-09-21T09:34:25.474 INFO:tasks.rados.rados.0.smithi022.stdout:975: left oid 1 (ObjNum 328 snap 100 seq_num 328) 2024-09-21T09:34:25.490 INFO:tasks.rados.rados.0.smithi022.stdout:976: expect (ObjNum 264 snap 76 seq_num 264) 2024-09-21T09:34:25.741 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:25 smithi064 ceph-mon[31124]: pgmap v39: 121 pgs: 121 active+clean; 233 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.8 MiB/s rd, 808 KiB/s wr, 13 op/s 2024-09-21T09:34:25.742 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:25 smithi064 ceph-mon[31124]: osdmap e255: 8 total, 8 up, 8 in 2024-09-21T09:34:25.742 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:25 smithi064 ceph-mon[31124]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' 2024-09-21T09:34:25.742 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:25 smithi064 ceph-mon[31124]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' 2024-09-21T09:34:25.742 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:25 smithi064 ceph-mon[31124]: osdmap e256: 8 total, 8 up, 8 in 2024-09-21T09:34:25.808 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:34:25 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:34:25] "GET /metrics HTTP/1.1" 200 36129 "" "Prometheus/2.51.0" 2024-09-21T09:34:25.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:25 smithi022 ceph-mon[27763]: pgmap v39: 121 pgs: 121 active+clean; 233 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.8 MiB/s rd, 808 KiB/s wr, 13 op/s 2024-09-21T09:34:25.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:25 smithi022 ceph-mon[27763]: osdmap e255: 8 total, 8 up, 8 in 2024-09-21T09:34:25.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:25 smithi022 ceph-mon[27763]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' 2024-09-21T09:34:25.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:25 smithi022 ceph-mon[27763]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' 2024-09-21T09:34:25.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:25 smithi022 ceph-mon[27763]: osdmap e256: 8 total, 8 up, 8 in 2024-09-21T09:34:25.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:25 smithi022 ceph-mon[39525]: pgmap v39: 121 pgs: 121 active+clean; 233 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.8 MiB/s rd, 808 KiB/s wr, 13 op/s 2024-09-21T09:34:25.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:25 smithi022 ceph-mon[39525]: osdmap e255: 8 total, 8 up, 8 in 2024-09-21T09:34:25.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:25 smithi022 ceph-mon[39525]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' 2024-09-21T09:34:25.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:25 smithi022 ceph-mon[39525]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' 2024-09-21T09:34:25.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:25 smithi022 ceph-mon[39525]: osdmap e256: 8 total, 8 up, 8 in 2024-09-21T09:34:26.016 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:34:25 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:34:25.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:34:26.016 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:34:25 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:34:25.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:34:26.017 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:34:25 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[78884]: [21/Sep/2024:09:34:25] ENGINE Bus STARTING 2024-09-21T09:34:26.017 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:34:25 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[78884]: CherryPy Checker: 2024-09-21T09:34:26.017 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:34:25 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[78884]: The Application mounted at '' has an empty config. 2024-09-21T09:34:26.017 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:34:25 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[78884]: 2024-09-21T09:34:26.269 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:34:26 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[78884]: [21/Sep/2024:09:34:26] ENGINE Serving on http://:::9283 2024-09-21T09:34:26.269 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:34:26 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[78884]: [21/Sep/2024:09:34:26] ENGINE Bus STARTED 2024-09-21T09:34:26.461 INFO:tasks.rados.rados.0.smithi022.stdout:975: done (2 left) 2024-09-21T09:34:26.462 INFO:tasks.rados.rados.0.smithi022.stdout:976: done (1 left) 2024-09-21T09:34:26.462 INFO:tasks.rados.rados.0.smithi022.stdout:977: done (0 left) 2024-09-21T09:34:26.462 INFO:tasks.rados.rados.0.smithi022.stdout:978: rollback oid 37 current snap is 101 2024-09-21T09:34:26.462 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 37 to 75 2024-09-21T09:34:26.462 INFO:tasks.rados.rados.0.smithi022.stdout:979: delete oid 44 current snap is 101 2024-09-21T09:34:26.464 INFO:tasks.rados.rados.0.smithi022.stdout:979: done (1 left) 2024-09-21T09:34:26.464 INFO:tasks.rados.rados.0.smithi022.stdout:980: copy_from oid 39 from oid 8 current snap is 101 2024-09-21T09:34:26.465 INFO:tasks.rados.rados.0.smithi022.stdout:981: read oid 46 snap 56 2024-09-21T09:34:26.465 INFO:tasks.rados.rados.0.smithi022.stdout:981: expect (ObjNum 45 snap 0 seq_num 45) 2024-09-21T09:34:26.465 INFO:tasks.rados.rados.0.smithi022.stdout:982: rollback oid 29 current snap is 101 2024-09-21T09:34:26.465 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 29 to 88 2024-09-21T09:34:26.465 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 50 current snap is 101 2024-09-21T09:34:26.465 INFO:tasks.rados.rados.0.smithi022.stdout:983: seq_num 330 ranges {0=737280} 2024-09-21T09:34:26.469 INFO:tasks.rados.rados.0.smithi022.stdout:983: writing smithi02284960-50 from 0 to 737280 tid 1 2024-09-21T09:34:26.469 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 24 current snap is 101 2024-09-21T09:34:26.470 INFO:tasks.rados.rados.0.smithi022.stdout:978: finishing rollback tid 0 to smithi02284960-37 2024-09-21T09:34:26.470 INFO:tasks.rados.rados.0.smithi022.stdout:978: finishing rollback tid 1 to smithi02284960-37 2024-09-21T09:34:26.470 INFO:tasks.rados.rados.0.smithi022.stdout:982: finishing rollback tid 1 to smithi02284960-29 2024-09-21T09:34:26.470 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 29 v 162 (ObjNum 33 snap 0 seq_num 3562300792) dirty dne 2024-09-21T09:34:26.470 INFO:tasks.rados.rados.0.smithi022.stdout:978: finishing rollback tid 2 to smithi02284960-37 2024-09-21T09:34:26.470 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 37 v 592 (ObjNum 218 snap 63 seq_num 218) dirty exists 2024-09-21T09:34:26.470 INFO:tasks.rados.rados.0.smithi022.stdout:984: seq_num 331 ranges {3416064=753664} 2024-09-21T09:34:26.493 INFO:tasks.rados.rados.0.smithi022.stdout:984: writing smithi02284960-24 from 3416064 to 4169728 tid 1 2024-09-21T09:34:26.493 INFO:tasks.rados.rados.0.smithi022.stdout:978: done (5 left) 2024-09-21T09:34:26.493 INFO:tasks.rados.rados.0.smithi022.stdout:982: done (4 left) 2024-09-21T09:34:26.493 INFO:tasks.rados.rados.0.smithi022.stdout:985: snap_remove snap 93 2024-09-21T09:34:26.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:26 smithi064 ceph-mon[31124]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' 2024-09-21T09:34:26.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:26 smithi064 ceph-mon[31124]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' 2024-09-21T09:34:26.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:26 smithi064 ceph-mon[31124]: Standby manager daemon x restarted 2024-09-21T09:34:26.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:26 smithi064 ceph-mon[31124]: Standby manager daemon x started 2024-09-21T09:34:26.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:26 smithi064 ceph-mon[31124]: from='mgr.? 172.21.15.64:0/3156531021' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-21T09:34:26.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:26 smithi064 ceph-mon[31124]: from='mgr.? 172.21.15.64:0/3156531021' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:34:26.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:26 smithi064 ceph-mon[31124]: from='mgr.? 172.21.15.64:0/3156531021' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-21T09:34:26.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:26 smithi064 ceph-mon[31124]: from='mgr.? 172.21.15.64:0/3156531021' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:34:26.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:26 smithi064 ceph-mon[31124]: osdmap e257: 8 total, 8 up, 8 in 2024-09-21T09:34:26.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:26 smithi022 ceph-mon[39525]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' 2024-09-21T09:34:26.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:26 smithi022 ceph-mon[39525]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' 2024-09-21T09:34:26.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:26 smithi022 ceph-mon[39525]: Standby manager daemon x restarted 2024-09-21T09:34:26.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:26 smithi022 ceph-mon[39525]: Standby manager daemon x started 2024-09-21T09:34:26.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:26 smithi022 ceph-mon[39525]: from='mgr.? 172.21.15.64:0/3156531021' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-21T09:34:26.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:26 smithi022 ceph-mon[39525]: from='mgr.? 172.21.15.64:0/3156531021' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:34:26.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:26 smithi022 ceph-mon[39525]: from='mgr.? 172.21.15.64:0/3156531021' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-21T09:34:26.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:26 smithi022 ceph-mon[39525]: from='mgr.? 172.21.15.64:0/3156531021' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:34:26.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:26 smithi022 ceph-mon[39525]: osdmap e257: 8 total, 8 up, 8 in 2024-09-21T09:34:26.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:26 smithi022 ceph-mon[27763]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' 2024-09-21T09:34:26.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:26 smithi022 ceph-mon[27763]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' 2024-09-21T09:34:26.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:26 smithi022 ceph-mon[27763]: Standby manager daemon x restarted 2024-09-21T09:34:26.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:26 smithi022 ceph-mon[27763]: Standby manager daemon x started 2024-09-21T09:34:26.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:26 smithi022 ceph-mon[27763]: from='mgr.? 172.21.15.64:0/3156531021' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-21T09:34:26.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:26 smithi022 ceph-mon[27763]: from='mgr.? 172.21.15.64:0/3156531021' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:34:26.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:26 smithi022 ceph-mon[27763]: from='mgr.? 172.21.15.64:0/3156531021' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-21T09:34:26.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:26 smithi022 ceph-mon[27763]: from='mgr.? 172.21.15.64:0/3156531021' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:34:26.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:26 smithi022 ceph-mon[27763]: osdmap e257: 8 total, 8 up, 8 in 2024-09-21T09:34:26.972 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:34:26 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:34:26.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi022\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:34:27.464 INFO:tasks.rados.rados.0.smithi022.stdout:985: done (4 left) 2024-09-21T09:34:27.465 INFO:tasks.rados.rados.0.smithi022.stdout:986: rmattr oid 20 current snap is 101 2024-09-21T09:34:27.465 INFO:tasks.rados.rados.0.smithi022.stdout:980: finishing copy_from racing read to smithi02284960-39 2024-09-21T09:34:27.465 INFO:tasks.rados.rados.0.smithi022.stdout:981: expect (ObjNum 45 snap 0 seq_num 45) 2024-09-21T09:34:27.490 INFO:tasks.rados.rados.0.smithi022.stdout:980: finishing copy_from to smithi02284960-39 2024-09-21T09:34:27.491 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 39 v 453 (ObjNum 312 snap 94 seq_num 312) dirty exists 2024-09-21T09:34:27.491 INFO:tasks.rados.rados.0.smithi022.stdout:983: finishing write tid 1 to smithi02284960-50 2024-09-21T09:34:27.491 INFO:tasks.rados.rados.0.smithi022.stdout:983: finishing write tid 2 to smithi02284960-50 2024-09-21T09:34:27.491 INFO:tasks.rados.rados.0.smithi022.stdout:983: finishing write tid 3 to smithi02284960-50 2024-09-21T09:34:27.491 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 50 v 305 (ObjNum 330 snap 101 seq_num 330) dirty exists 2024-09-21T09:34:27.491 INFO:tasks.rados.rados.0.smithi022.stdout:983: left oid 50 (ObjNum 330 snap 101 seq_num 330) 2024-09-21T09:34:27.491 INFO:tasks.rados.rados.0.smithi022.stdout:984: finishing write tid 1 to smithi02284960-24 2024-09-21T09:34:27.491 INFO:tasks.rados.rados.0.smithi022.stdout:984: finishing write tid 2 to smithi02284960-24 2024-09-21T09:34:27.491 INFO:tasks.rados.rados.0.smithi022.stdout:984: finishing write tid 3 to smithi02284960-24 2024-09-21T09:34:27.491 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 24 v 595 (ObjNum 331 snap 101 seq_num 331) dirty exists 2024-09-21T09:34:27.491 INFO:tasks.rados.rados.0.smithi022.stdout:984: left oid 24 (ObjNum 331 snap 101 seq_num 331) 2024-09-21T09:34:27.491 INFO:tasks.rados.rados.0.smithi022.stdout:980: done (4 left) 2024-09-21T09:34:27.492 INFO:tasks.rados.rados.0.smithi022.stdout:981: done (3 left) 2024-09-21T09:34:27.492 INFO:tasks.rados.rados.0.smithi022.stdout:983: done (2 left) 2024-09-21T09:34:27.492 INFO:tasks.rados.rados.0.smithi022.stdout:984: done (1 left) 2024-09-21T09:34:27.492 INFO:tasks.rados.rados.0.smithi022.stdout:987: setattr oid 16 current snap is 101 2024-09-21T09:34:27.492 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 20 v 590 (ObjNum 132 snap 27 seq_num 132) dirty exists 2024-09-21T09:34:27.493 INFO:tasks.rados.rados.0.smithi022.stdout:986: done (1 left) 2024-09-21T09:34:27.493 INFO:tasks.rados.rados.0.smithi022.stdout:988: snap_remove snap 90 2024-09-21T09:34:27.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:27 smithi022 ceph-mon[39525]: pgmap v42: 121 pgs: 121 active+clean; 233 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:34:27.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:27 smithi022 ceph-mon[39525]: mgrmap e34: y(active, since 26s), standbys: x 2024-09-21T09:34:27.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:27 smithi022 ceph-mon[39525]: osdmap e258: 8 total, 8 up, 8 in 2024-09-21T09:34:27.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:27 smithi022 ceph-mon[27763]: pgmap v42: 121 pgs: 121 active+clean; 233 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:34:27.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:27 smithi022 ceph-mon[27763]: mgrmap e34: y(active, since 26s), standbys: x 2024-09-21T09:34:27.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:27 smithi022 ceph-mon[27763]: osdmap e258: 8 total, 8 up, 8 in 2024-09-21T09:34:27.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:27 smithi064 ceph-mon[31124]: pgmap v42: 121 pgs: 121 active+clean; 233 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:34:27.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:27 smithi064 ceph-mon[31124]: mgrmap e34: y(active, since 26s), standbys: x 2024-09-21T09:34:27.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:27 smithi064 ceph-mon[31124]: osdmap e258: 8 total, 8 up, 8 in 2024-09-21T09:34:28.466 INFO:tasks.rados.rados.0.smithi022.stdout:988: done (1 left) 2024-09-21T09:34:28.467 INFO:tasks.rados.rados.0.smithi022.stdout:989: rollback oid 17 current snap is 101 2024-09-21T09:34:28.467 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 17 to 72 2024-09-21T09:34:28.467 INFO:tasks.rados.rados.0.smithi022.stdout:990: snap_remove snap 88 2024-09-21T09:34:29.473 INFO:tasks.rados.rados.0.smithi022.stdout:990: done (2 left) 2024-09-21T09:34:29.473 INFO:tasks.rados.rados.0.smithi022.stdout:991: read oid 28 snap -1 2024-09-21T09:34:29.473 INFO:tasks.rados.rados.0.smithi022.stdout:991: expect (ObjNum 282 snap 81 seq_num 282) 2024-09-21T09:34:29.473 INFO:tasks.rados.rados.0.smithi022.stdout:992: copy_from oid 13 from oid 7 current snap is 101 2024-09-21T09:34:29.473 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 40 current snap is 101 2024-09-21T09:34:29.473 INFO:tasks.rados.rados.0.smithi022.stdout:993: seq_num 332 ranges {1196032=770048} 2024-09-21T09:34:29.484 INFO:tasks.rados.rados.0.smithi022.stdout:993: writing smithi02284960-40 from 1196032 to 1966080 tid 1 2024-09-21T09:34:29.485 INFO:tasks.rados.rados.0.smithi022.stdout:994: read oid 48 snap 98 2024-09-21T09:34:29.485 INFO:tasks.rados.rados.0.smithi022.stdout:994: expect (ObjNum 318 snap 96 seq_num 318) 2024-09-21T09:34:29.485 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 44 current snap is 101 2024-09-21T09:34:29.485 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 16 v 286 (ObjNum 0 snap 0 seq_num 304) dirty exists 2024-09-21T09:34:29.485 INFO:tasks.rados.rados.0.smithi022.stdout:989: finishing rollback tid 1 to smithi02284960-17 2024-09-21T09:34:29.486 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 17 v 287 (ObjNum 97 snap 0 seq_num 2955649868) dirty dne 2024-09-21T09:34:29.486 INFO:tasks.rados.rados.0.smithi022.stdout:992: finishing copy_from to smithi02284960-13 2024-09-21T09:34:29.486 INFO:tasks.rados.rados.0.smithi022.stdout:992: got expected ENOENT (src dne) 2024-09-21T09:34:29.486 INFO:tasks.rados.rados.0.smithi022.stdout:992: finishing copy_from racing read to smithi02284960-13 2024-09-21T09:34:29.486 INFO:tasks.rados.rados.0.smithi022.stdout:995: seq_num 333 ranges {0=786432} 2024-09-21T09:34:29.491 INFO:tasks.rados.rados.0.smithi022.stdout:995: writing smithi02284960-44 from 0 to 786432 tid 1 2024-09-21T09:34:29.491 INFO:tasks.rados.rados.0.smithi022.stdout:987: done (6 left) 2024-09-21T09:34:29.491 INFO:tasks.rados.rados.0.smithi022.stdout:989: done (5 left) 2024-09-21T09:34:29.491 INFO:tasks.rados.rados.0.smithi022.stdout:992: done (4 left) 2024-09-21T09:34:29.492 INFO:tasks.rados.rados.0.smithi022.stdout:996: copy_from oid 35 from oid 10 current snap is 101 2024-09-21T09:34:29.492 INFO:tasks.rados.rados.0.smithi022.stdout:997: snap_remove snap 78 2024-09-21T09:34:29.802 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:29 smithi022 ceph-mon[39525]: pgmap v45: 121 pgs: 121 active+clean; 233 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.6 MiB/s rd, 1.8 MiB/s wr, 9 op/s 2024-09-21T09:34:29.802 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:29 smithi022 ceph-mon[39525]: osdmap e259: 8 total, 8 up, 8 in 2024-09-21T09:34:29.802 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:29 smithi022 ceph-mon[39525]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' 2024-09-21T09:34:29.802 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:29 smithi022 ceph-mon[39525]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' 2024-09-21T09:34:29.802 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:29 smithi022 ceph-mon[39525]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:34:29.802 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:29 smithi022 ceph-mon[39525]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:34:29.802 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:29 smithi022 ceph-mon[39525]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' 2024-09-21T09:34:29.802 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:29 smithi022 ceph-mon[39525]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:34:29.803 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:29 smithi022 ceph-mon[39525]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:34:29.803 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:29 smithi022 ceph-mon[39525]: Upgrade: Setting container_image for all mgr 2024-09-21T09:34:29.803 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:29 smithi022 ceph-mon[39525]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' 2024-09-21T09:34:29.803 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:29 smithi022 ceph-mon[39525]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-09-21T09:34:29.803 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:29 smithi022 ceph-mon[39525]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-09-21T09:34:29.803 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:29 smithi022 ceph-mon[39525]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-09-21T09:34:29.803 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:29 smithi022 ceph-mon[39525]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-09-21T09:34:29.803 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:29 smithi022 ceph-mon[39525]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-21T09:34:29.803 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:29 smithi022 ceph-mon[39525]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-09-21T09:34:29.803 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:29 smithi022 ceph-mon[39525]: Upgrade: It appears safe to stop mon.a 2024-09-21T09:34:29.803 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:29 smithi022 ceph-mon[27763]: pgmap v45: 121 pgs: 121 active+clean; 233 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.6 MiB/s rd, 1.8 MiB/s wr, 9 op/s 2024-09-21T09:34:29.803 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:29 smithi022 ceph-mon[27763]: osdmap e259: 8 total, 8 up, 8 in 2024-09-21T09:34:29.804 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:29 smithi022 ceph-mon[27763]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' 2024-09-21T09:34:29.804 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:29 smithi022 ceph-mon[27763]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' 2024-09-21T09:34:29.804 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:29 smithi022 ceph-mon[27763]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:34:29.804 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:29 smithi022 ceph-mon[27763]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:34:29.804 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:29 smithi022 ceph-mon[27763]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' 2024-09-21T09:34:29.804 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:29 smithi022 ceph-mon[27763]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:34:29.804 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:29 smithi022 ceph-mon[27763]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:34:29.804 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:29 smithi022 ceph-mon[27763]: Upgrade: Setting container_image for all mgr 2024-09-21T09:34:29.804 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:29 smithi022 ceph-mon[27763]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' 2024-09-21T09:34:29.804 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:29 smithi022 ceph-mon[27763]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-09-21T09:34:29.804 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:29 smithi022 ceph-mon[27763]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-09-21T09:34:29.804 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:29 smithi022 ceph-mon[27763]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-09-21T09:34:29.804 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:29 smithi022 ceph-mon[27763]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-09-21T09:34:29.805 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:29 smithi022 ceph-mon[27763]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-21T09:34:29.805 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:29 smithi022 ceph-mon[27763]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-09-21T09:34:29.805 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:29 smithi022 ceph-mon[27763]: Upgrade: It appears safe to stop mon.a 2024-09-21T09:34:29.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:29 smithi064 ceph-mon[31124]: pgmap v45: 121 pgs: 121 active+clean; 233 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.6 MiB/s rd, 1.8 MiB/s wr, 9 op/s 2024-09-21T09:34:29.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:29 smithi064 ceph-mon[31124]: osdmap e259: 8 total, 8 up, 8 in 2024-09-21T09:34:29.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:29 smithi064 ceph-mon[31124]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' 2024-09-21T09:34:29.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:29 smithi064 ceph-mon[31124]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' 2024-09-21T09:34:29.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:29 smithi064 ceph-mon[31124]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:34:29.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:29 smithi064 ceph-mon[31124]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:34:29.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:29 smithi064 ceph-mon[31124]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' 2024-09-21T09:34:29.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:29 smithi064 ceph-mon[31124]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:34:29.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:29 smithi064 ceph-mon[31124]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:34:29.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:29 smithi064 ceph-mon[31124]: Upgrade: Setting container_image for all mgr 2024-09-21T09:34:29.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:29 smithi064 ceph-mon[31124]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' 2024-09-21T09:34:29.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:29 smithi064 ceph-mon[31124]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-09-21T09:34:29.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:29 smithi064 ceph-mon[31124]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-09-21T09:34:29.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:29 smithi064 ceph-mon[31124]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-09-21T09:34:29.943 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:29 smithi064 ceph-mon[31124]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-09-21T09:34:29.943 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:29 smithi064 ceph-mon[31124]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-21T09:34:29.943 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:29 smithi064 ceph-mon[31124]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-09-21T09:34:29.943 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:29 smithi064 ceph-mon[31124]: Upgrade: It appears safe to stop mon.a 2024-09-21T09:34:30.461 INFO:tasks.rados.rados.0.smithi022.stdout:997: done (5 left) 2024-09-21T09:34:30.461 INFO:tasks.rados.rados.0.smithi022.stdout:998: snap_remove snap 99 2024-09-21T09:34:30.475 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:34:30 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:34:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:34:30.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:30 smithi022 ceph-mon[39525]: osdmap e260: 8 total, 8 up, 8 in 2024-09-21T09:34:30.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:30 smithi022 ceph-mon[39525]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' 2024-09-21T09:34:30.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:30 smithi022 ceph-mon[39525]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T09:34:30.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:30 smithi022 ceph-mon[39525]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T09:34:30.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:30 smithi022 ceph-mon[39525]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:34:30.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:30 smithi022 ceph-mon[39525]: osdmap e261: 8 total, 8 up, 8 in 2024-09-21T09:34:30.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:30 smithi022 ceph-mon[27763]: osdmap e260: 8 total, 8 up, 8 in 2024-09-21T09:34:30.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:30 smithi022 ceph-mon[27763]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' 2024-09-21T09:34:30.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:30 smithi022 ceph-mon[27763]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T09:34:30.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:30 smithi022 ceph-mon[27763]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T09:34:30.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:30 smithi022 ceph-mon[27763]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:34:30.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:30 smithi022 ceph-mon[27763]: osdmap e261: 8 total, 8 up, 8 in 2024-09-21T09:34:30.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:30 smithi064 ceph-mon[31124]: osdmap e260: 8 total, 8 up, 8 in 2024-09-21T09:34:30.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:30 smithi064 ceph-mon[31124]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' 2024-09-21T09:34:30.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:30 smithi064 ceph-mon[31124]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T09:34:30.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:30 smithi064 ceph-mon[31124]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T09:34:30.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:30 smithi064 ceph-mon[31124]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:34:30.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:30 smithi064 ceph-mon[31124]: osdmap e261: 8 total, 8 up, 8 in 2024-09-21T09:34:31.463 INFO:tasks.rados.rados.0.smithi022.stdout:998: done (5 left) 2024-09-21T09:34:31.463 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 13 current snap is 101 2024-09-21T09:34:31.463 INFO:tasks.rados.rados.0.smithi022.stdout:999: seq_num 334 ranges {4431872=802816} 2024-09-21T09:34:31.492 INFO:tasks.rados.rados.0.smithi022.stdout:999: writing smithi02284960-13 from 4431872 to 5234688 tid 1 2024-09-21T09:34:31.492 INFO:tasks.rados.rados.0.smithi022.stdout:1000: read oid 18 snap -1 2024-09-21T09:34:31.492 INFO:tasks.rados.rados.0.smithi022.stdout:1000: expect deleted 2024-09-21T09:34:31.492 INFO:tasks.rados.rados.0.smithi022.stdout:1001: setattr oid 9 current snap is 101 2024-09-21T09:34:31.492 INFO:tasks.rados.rados.0.smithi022.stdout:993: finishing write tid 1 to smithi02284960-40 2024-09-21T09:34:31.493 INFO:tasks.rados.rados.0.smithi022.stdout:993: finishing write tid 2 to smithi02284960-40 2024-09-21T09:34:31.493 INFO:tasks.rados.rados.0.smithi022.stdout:995: finishing write tid 1 to smithi02284960-44 2024-09-21T09:34:31.493 INFO:tasks.rados.rados.0.smithi022.stdout:993: finishing write tid 3 to smithi02284960-40 2024-09-21T09:34:31.493 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 40 v 554 (ObjNum 332 snap 101 seq_num 332) dirty exists 2024-09-21T09:34:31.493 INFO:tasks.rados.rados.0.smithi022.stdout:993: left oid 40 (ObjNum 332 snap 101 seq_num 332) 2024-09-21T09:34:31.493 INFO:tasks.rados.rados.0.smithi022.stdout:991: expect (ObjNum 282 snap 81 seq_num 282) 2024-09-21T09:34:31.679 INFO:tasks.rados.rados.0.smithi022.stdout:996: finishing copy_from racing read to smithi02284960-35 2024-09-21T09:34:31.680 INFO:tasks.rados.rados.0.smithi022.stdout:996: finishing copy_from to smithi02284960-35 2024-09-21T09:34:31.680 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 35 v 535 (ObjNum 311 snap 94 seq_num 311) dirty exists 2024-09-21T09:34:31.680 INFO:tasks.rados.rados.0.smithi022.stdout:995: finishing write tid 2 to smithi02284960-44 2024-09-21T09:34:31.680 INFO:tasks.rados.rados.0.smithi022.stdout:995: finishing write tid 3 to smithi02284960-44 2024-09-21T09:34:31.680 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 44 v 534 (ObjNum 333 snap 101 seq_num 333) dirty exists 2024-09-21T09:34:31.680 INFO:tasks.rados.rados.0.smithi022.stdout:995: left oid 44 (ObjNum 333 snap 101 seq_num 333) 2024-09-21T09:34:31.680 INFO:tasks.rados.rados.0.smithi022.stdout:994: expect (ObjNum 318 snap 96 seq_num 318) 2024-09-21T09:34:31.753 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:31 smithi022 ceph-mon[39525]: pgmap v48: 121 pgs: 121 active+clean; 233 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.6 MiB/s rd, 1.8 MiB/s wr, 9 op/s 2024-09-21T09:34:31.753 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:31 smithi022 ceph-mon[39525]: Upgrade: Updating mon.a 2024-09-21T09:34:31.753 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:31 smithi022 ceph-mon[39525]: Deploying daemon mon.a on smithi022 2024-09-21T09:34:31.754 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:31 smithi022 ceph-mon[39525]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:34:31.754 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:31 smithi022 ceph-mon[39525]: osdmap e262: 8 total, 8 up, 8 in 2024-09-21T09:34:31.754 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:31 smithi022 ceph-mon[27763]: pgmap v48: 121 pgs: 121 active+clean; 233 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.6 MiB/s rd, 1.8 MiB/s wr, 9 op/s 2024-09-21T09:34:31.754 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:31 smithi022 ceph-mon[27763]: Upgrade: Updating mon.a 2024-09-21T09:34:31.754 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:31 smithi022 ceph-mon[27763]: Deploying daemon mon.a on smithi022 2024-09-21T09:34:31.754 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:31 smithi022 ceph-mon[27763]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:34:31.754 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:31 smithi022 ceph-mon[27763]: osdmap e262: 8 total, 8 up, 8 in 2024-09-21T09:34:31.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:31 smithi064 ceph-mon[31124]: pgmap v48: 121 pgs: 121 active+clean; 233 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.6 MiB/s rd, 1.8 MiB/s wr, 9 op/s 2024-09-21T09:34:31.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:31 smithi064 ceph-mon[31124]: Upgrade: Updating mon.a 2024-09-21T09:34:31.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:31 smithi064 ceph-mon[31124]: Deploying daemon mon.a on smithi022 2024-09-21T09:34:31.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:31 smithi064 ceph-mon[31124]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:34:31.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:31 smithi064 ceph-mon[31124]: osdmap e262: 8 total, 8 up, 8 in 2024-09-21T09:34:32.023 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:32 smithi022 systemd[1]: Stopping Ceph mon.a for d65e801e-77fa-11ef-baf6-efdc52797490... 2024-09-21T09:34:32.095 INFO:tasks.rados.rados.0.smithi022.stdout:999: finishing write tid 1 to smithi02284960-13 2024-09-21T09:34:32.095 INFO:tasks.rados.rados.0.smithi022.stdout:999: finishing write tid 2 to smithi02284960-13 2024-09-21T09:34:32.095 INFO:tasks.rados.rados.0.smithi022.stdout:999: finishing write tid 3 to smithi02284960-13 2024-09-21T09:34:32.095 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 13 v 481 (ObjNum 334 snap 101 seq_num 334) dirty exists 2024-09-21T09:34:32.095 INFO:tasks.rados.rados.0.smithi022.stdout:999: left oid 13 (ObjNum 334 snap 101 seq_num 334) 2024-09-21T09:34:32.095 INFO:tasks.rados.rados.0.smithi022.stdout:991: done (7 left) 2024-09-21T09:34:32.095 INFO:tasks.rados.rados.0.smithi022.stdout:993: done (6 left) 2024-09-21T09:34:32.095 INFO:tasks.rados.rados.0.smithi022.stdout:994: done (5 left) 2024-09-21T09:34:32.095 INFO:tasks.rados.rados.0.smithi022.stdout:995: done (4 left) 2024-09-21T09:34:32.095 INFO:tasks.rados.rados.0.smithi022.stdout:996: done (3 left) 2024-09-21T09:34:32.095 INFO:tasks.rados.rados.0.smithi022.stdout:999: done (2 left) 2024-09-21T09:34:32.095 INFO:tasks.rados.rados.0.smithi022.stdout:1000: done (1 left) 2024-09-21T09:34:32.096 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 4 current snap is 101 2024-09-21T09:34:32.096 INFO:tasks.rados.rados.0.smithi022.stdout:1002: seq_num 335 ranges {3416064=417792,3833856=401408} 2024-09-21T09:34:32.115 INFO:tasks.rados.rados.0.smithi022.stdout:1002: writing smithi02284960-4 from 3416064 to 3833856 tid 1 2024-09-21T09:34:32.118 INFO:tasks.rados.rados.0.smithi022.stdout:1002: writing smithi02284960-4 from 3833856 to 4235264 tid 2 2024-09-21T09:34:32.118 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 11 current snap is 101 2024-09-21T09:34:32.118 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 9 v 454 (ObjNum 240 snap 65 seq_num 240) dirty exists 2024-09-21T09:34:32.118 INFO:tasks.rados.rados.0.smithi022.stdout:1003: seq_num 336 ranges {3293184=434176,3727360=401408} 2024-09-21T09:34:32.138 INFO:tasks.rados.rados.0.smithi022.stdout:1003: writing smithi02284960-11 from 3293184 to 3727360 tid 1 2024-09-21T09:34:32.141 INFO:tasks.rados.rados.0.smithi022.stdout:1003: writing smithi02284960-11 from 3727360 to 4128768 tid 2 2024-09-21T09:34:32.141 INFO:tasks.rados.rados.0.smithi022.stdout:1001: done (2 left) 2024-09-21T09:34:32.141 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 44 current snap is 101 2024-09-21T09:34:32.141 INFO:tasks.rados.rados.0.smithi022.stdout:1002: finishing write tid 1 to smithi02284960-4 2024-09-21T09:34:32.141 INFO:tasks.rados.rados.0.smithi022.stdout:1004: seq_num 337 ranges {786432=450560,1236992=401408} 2024-09-21T09:34:32.147 INFO:tasks.rados.rados.0.smithi022.stdout:1004: writing smithi02284960-44 from 786432 to 1236992 tid 1 2024-09-21T09:34:32.150 INFO:tasks.rados.rados.0.smithi022.stdout:1004: writing smithi02284960-44 from 1236992 to 1638400 tid 2 2024-09-21T09:34:32.150 INFO:tasks.rados.rados.0.smithi022.stdout:1005: snap_remove snap 97 2024-09-21T09:34:32.558 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:34:32 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: [21/Sep/2024:09:34:32] ENGINE Bus STOPPING 2024-09-21T09:34:32.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:32 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mon-a[27740]: 2024-09-21T09:34:32.282+0000 7fdfcea62640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T09:34:32.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:32 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mon-a[27740]: 2024-09-21T09:34:32.282+0000 7fdfcea62640 -1 mon.a@0(leader) e3 *** Got Signal Terminated *** 2024-09-21T09:34:32.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:32 smithi022 podman[96444]: 2024-09-21 09:34:32.355360719 +0000 UTC m=+0.178260122 container died 1e0a2d74e66ed0875bbbe448f9186ef3a086f3147ceee15bc593a101f91895e0 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mon-a, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=quincy-fbd1a6a, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS) 2024-09-21T09:34:32.926 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:32 smithi022 podman[96444]: 2024-09-21 09:34:32.690180989 +0000 UTC m=+0.513080389 container cleanup 1e0a2d74e66ed0875bbbe448f9186ef3a086f3147ceee15bc593a101f91895e0 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mon-a, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, RELEASE=quincy-fbd1a6a, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.license=GPLv2) 2024-09-21T09:34:32.926 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:32 smithi022 bash[96444]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mon-a 2024-09-21T09:34:32.927 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:34:32 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: [21/Sep/2024:09:34:32] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-21T09:34:32.927 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:34:32 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: [21/Sep/2024:09:34:32] ENGINE Bus STOPPED 2024-09-21T09:34:32.927 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:34:32 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: [21/Sep/2024:09:34:32] ENGINE Bus STARTING 2024-09-21T09:34:32.927 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:34:32 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: [21/Sep/2024:09:34:32] ENGINE Serving on http://:::9283 2024-09-21T09:34:32.927 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:34:32 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: [21/Sep/2024:09:34:32] ENGINE Bus STARTED 2024-09-21T09:34:33.145 INFO:teuthology.orchestra.run.smithi022.stdout:true 2024-09-21T09:34:33.187 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:32 smithi022 podman[96460]: 2024-09-21 09:34:32.924793977 +0000 UTC m=+0.567415764 container remove 1e0a2d74e66ed0875bbbe448f9186ef3a086f3147ceee15bc593a101f91895e0 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mon-a, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=quincy-fbd1a6a, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True) 2024-09-21T09:34:33.557 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 systemd[1]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490@mon.a.service: Deactivated successfully. 2024-09-21T09:34:33.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 systemd[1]: Stopped Ceph mon.a for d65e801e-77fa-11ef-baf6-efdc52797490. 2024-09-21T09:34:33.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 systemd[1]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490@mon.a.service: Consumed 15.279s CPU time. 2024-09-21T09:34:33.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 systemd[1]: Starting Ceph mon.a for d65e801e-77fa-11ef-baf6-efdc52797490... 2024-09-21T09:34:33.772 INFO:teuthology.orchestra.run.smithi022.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T09:34:33.772 INFO:teuthology.orchestra.run.smithi022.stdout:alertmanager.a smithi022 *:9093,9094 running (76s) 28s ago 5m 18.4M - 0.27.0 11f11916f8cd caced80ba544 2024-09-21T09:34:33.773 INFO:teuthology.orchestra.run.smithi022.stdout:grafana.a smithi064 *:3000 running (52s) 8s ago 5m 77.6M - 9.4.7 954c08fa6188 527bb657d70d 2024-09-21T09:34:33.773 INFO:teuthology.orchestra.run.smithi022.stdout:mds.a smithi022 running (7m) 28s ago 7m 16.4M - 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 34e4221f3a27 2024-09-21T09:34:33.773 INFO:teuthology.orchestra.run.smithi022.stdout:mds.b smithi064 running (7m) 8s ago 7m 20.2M - 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 0f846b3ec694 2024-09-21T09:34:33.773 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.x smithi064 *:8443,9283,8765 running (13s) 8s ago 10m 64.1M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 61abc61fea8d 2024-09-21T09:34:33.773 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.y smithi022 *:8443,9283,8765 running (42s) 28s ago 12m 491M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 6c5bee78f09b 2024-09-21T09:34:33.773 INFO:teuthology.orchestra.run.smithi022.stdout:mon.a smithi022 running (12m) 28s ago 12m 56.9M 2048M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 1e0a2d74e66e 2024-09-21T09:34:33.773 INFO:teuthology.orchestra.run.smithi022.stdout:mon.b smithi064 running (11m) 8s ago 11m 52.3M 2048M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be ed126533cd64 2024-09-21T09:34:33.773 INFO:teuthology.orchestra.run.smithi022.stdout:mon.c smithi022 running (10m) 28s ago 10m 48.5M 2048M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 79f307e3b64a 2024-09-21T09:34:33.773 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.a smithi022 *:9100 running (71s) 28s ago 5m 7952k - 1.7.0 72c9c2088986 593c6158f524 2024-09-21T09:34:33.773 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.b smithi064 *:9100 running (47s) 8s ago 5m 9.96M - 1.7.0 72c9c2088986 a533420b0139 2024-09-21T09:34:33.773 INFO:teuthology.orchestra.run.smithi022.stdout:osd.0 smithi022 running (10m) 28s ago 10m 170M 2481M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be fcd812003534 2024-09-21T09:34:33.773 INFO:teuthology.orchestra.run.smithi022.stdout:osd.1 smithi022 running (9m) 28s ago 9m 133M 2481M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 740462b5c304 2024-09-21T09:34:33.774 INFO:teuthology.orchestra.run.smithi022.stdout:osd.2 smithi022 running (9m) 28s ago 9m 148M 2481M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be fefc7c770c32 2024-09-21T09:34:33.774 INFO:teuthology.orchestra.run.smithi022.stdout:osd.3 smithi022 running (9m) 28s ago 9m 102M 2481M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be a56193b67a92 2024-09-21T09:34:33.774 INFO:teuthology.orchestra.run.smithi022.stdout:osd.4 smithi064 running (8m) 8s ago 8m 118M 2480M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 2a2937374fc4 2024-09-21T09:34:33.774 INFO:teuthology.orchestra.run.smithi022.stdout:osd.5 smithi064 running (8m) 8s ago 8m 152M 2480M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 1900c4af5d28 2024-09-21T09:34:33.774 INFO:teuthology.orchestra.run.smithi022.stdout:osd.6 smithi064 running (7m) 8s ago 7m 117M 2480M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 3cc4ec816e5b 2024-09-21T09:34:33.774 INFO:teuthology.orchestra.run.smithi022.stdout:osd.7 smithi064 running (7m) 8s ago 7m 130M 2480M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 18ef0be94104 2024-09-21T09:34:33.774 INFO:teuthology.orchestra.run.smithi022.stdout:prometheus.a smithi064 *:9095 running (18s) 8s ago 6m 37.2M - 2.51.0 1d3b7f56885b 80629ffa2179 2024-09-21T09:34:33.879 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 podman[96657]: 2024-09-21 09:34:33.52654865 +0000 UTC m=+0.017530384 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:34:33.879 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 podman[96657]: 2024-09-21 09:34:33.659365504 +0000 UTC m=+0.150347238 container create 0513395c784d2ca33f510b41dec5e34718168c717e222018230be292f24d09d1 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mon-a, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240731, RELEASE=main-62da54b, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T09:34:33.879 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 podman[96657]: 2024-09-21 09:34:33.871069996 +0000 UTC m=+0.362051734 container init 0513395c784d2ca33f510b41dec5e34718168c717e222018230be292f24d09d1 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mon-a, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=main-62da54b, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True) 2024-09-21T09:34:34.129 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-09-21T09:34:34.129 INFO:teuthology.orchestra.run.smithi022.stdout: "mon": { 2024-09-21T09:34:34.129 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 3 2024-09-21T09:34:34.129 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-21T09:34:34.129 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr": { 2024-09-21T09:34:34.129 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 2 2024-09-21T09:34:34.129 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-21T09:34:34.129 INFO:teuthology.orchestra.run.smithi022.stdout: "osd": { 2024-09-21T09:34:34.129 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 8 2024-09-21T09:34:34.129 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-21T09:34:34.129 INFO:teuthology.orchestra.run.smithi022.stdout: "mds": { 2024-09-21T09:34:34.129 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 2 2024-09-21T09:34:34.129 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-21T09:34:34.130 INFO:teuthology.orchestra.run.smithi022.stdout: "overall": { 2024-09-21T09:34:34.130 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 13, 2024-09-21T09:34:34.130 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 2 2024-09-21T09:34:34.130 INFO:teuthology.orchestra.run.smithi022.stdout: } 2024-09-21T09:34:34.130 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-09-21T09:34:34.131 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 podman[96657]: 2024-09-21 09:34:33.877611652 +0000 UTC m=+0.368593383 container start 0513395c784d2ca33f510b41dec5e34718168c717e222018230be292f24d09d1 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mon-a, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, RELEASE=main-62da54b, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T09:34:34.131 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: set uid:gid to 167:167 (ceph:ceph) 2024-09-21T09:34:34.131 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev), process ceph-mon, pid 7 2024-09-21T09:34:34.131 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: pidfile_write: ignore empty --pid-file 2024-09-21T09:34:34.131 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: load: jerasure load: lrc 2024-09-21T09:34:34.131 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: RocksDB version: 7.9.2 2024-09-21T09:34:34.131 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Git sha 0 2024-09-21T09:34:34.132 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Compile date 2024-09-20 20:32:28 2024-09-21T09:34:34.132 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: DB SUMMARY 2024-09-21T09:34:34.132 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: DB Session ID: 9ZJVPDCFN8VGI5MORAZ8 2024-09-21T09:34:34.132 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: CURRENT file: CURRENT 2024-09-21T09:34:34.132 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: IDENTITY file: IDENTITY 2024-09-21T09:34:34.132 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: MANIFEST file: MANIFEST-000015 size: 1173 Bytes 2024-09-21T09:34:34.132 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000033.sst 2024-09-21T09:34:34.132 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000031.log size: 5079964 ; 2024-09-21T09:34:34.132 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.error_if_exists: 0 2024-09-21T09:34:34.133 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.create_if_missing: 0 2024-09-21T09:34:34.133 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.paranoid_checks: 1 2024-09-21T09:34:34.133 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-21T09:34:34.133 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-21T09:34:34.133 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-21T09:34:34.133 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.env: 0x558934891120 2024-09-21T09:34:34.133 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.fs: PosixFileSystem 2024-09-21T09:34:34.133 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.info_log: 0x5589359b7700 2024-09-21T09:34:34.134 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.max_file_opening_threads: 16 2024-09-21T09:34:34.134 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.statistics: (nil) 2024-09-21T09:34:34.134 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.use_fsync: 0 2024-09-21T09:34:34.134 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.max_log_file_size: 0 2024-09-21T09:34:34.134 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-21T09:34:34.134 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-21T09:34:34.134 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.keep_log_file_num: 1000 2024-09-21T09:34:34.134 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.recycle_log_file_num: 0 2024-09-21T09:34:34.134 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.allow_fallocate: 1 2024-09-21T09:34:34.135 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.allow_mmap_reads: 0 2024-09-21T09:34:34.135 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.allow_mmap_writes: 0 2024-09-21T09:34:34.135 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.use_direct_reads: 0 2024-09-21T09:34:34.135 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-21T09:34:34.135 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.create_missing_column_families: 0 2024-09-21T09:34:34.135 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.db_log_dir: 2024-09-21T09:34:34.135 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.wal_dir: 2024-09-21T09:34:34.135 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-21T09:34:34.136 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-21T09:34:34.136 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-21T09:34:34.136 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-21T09:34:34.136 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-21T09:34:34.136 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-21T09:34:34.136 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.advise_random_on_open: 1 2024-09-21T09:34:34.136 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.db_write_buffer_size: 0 2024-09-21T09:34:34.136 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.write_buffer_manager: 0x5589359bbb80 2024-09-21T09:34:34.136 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-21T09:34:34.137 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-21T09:34:34.137 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-21T09:34:34.137 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.rate_limiter: (nil) 2024-09-21T09:34:34.137 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-21T09:34:34.137 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.wal_recovery_mode: 2 2024-09-21T09:34:34.137 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.enable_thread_tracking: 0 2024-09-21T09:34:34.137 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.enable_pipelined_write: 0 2024-09-21T09:34:34.137 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.unordered_write: 0 2024-09-21T09:34:34.137 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-21T09:34:34.137 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-21T09:34:34.138 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-21T09:34:34.138 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-21T09:34:34.138 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.row_cache: None 2024-09-21T09:34:34.138 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.wal_filter: None 2024-09-21T09:34:34.138 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-21T09:34:34.138 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.allow_ingest_behind: 0 2024-09-21T09:34:34.138 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.two_write_queues: 0 2024-09-21T09:34:34.138 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.manual_wal_flush: 0 2024-09-21T09:34:34.139 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.wal_compression: 0 2024-09-21T09:34:34.139 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.atomic_flush: 0 2024-09-21T09:34:34.139 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-21T09:34:34.139 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-21T09:34:34.139 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-21T09:34:34.141 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.log_readahead_size: 0 2024-09-21T09:34:34.141 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-21T09:34:34.141 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.best_efforts_recovery: 0 2024-09-21T09:34:34.141 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-21T09:34:34.141 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-21T09:34:34.141 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.allow_data_in_errors: 0 2024-09-21T09:34:34.141 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.db_host_id: __hostname__ 2024-09-21T09:34:34.141 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-21T09:34:34.142 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.max_background_jobs: 2 2024-09-21T09:34:34.142 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.max_background_compactions: -1 2024-09-21T09:34:34.142 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.max_subcompactions: 1 2024-09-21T09:34:34.142 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-21T09:34:34.142 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-21T09:34:34.142 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-21T09:34:34.142 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.max_total_wal_size: 0 2024-09-21T09:34:34.142 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-21T09:34:34.142 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-21T09:34:34.143 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-21T09:34:34.143 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-21T09:34:34.143 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.max_open_files: -1 2024-09-21T09:34:34.143 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.bytes_per_sync: 0 2024-09-21T09:34:34.143 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-21T09:34:34.143 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-21T09:34:34.143 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.compaction_readahead_size: 0 2024-09-21T09:34:34.143 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.max_background_flushes: -1 2024-09-21T09:34:34.143 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Compression algorithms supported: 2024-09-21T09:34:34.143 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: kZSTD supported: 0 2024-09-21T09:34:34.143 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: kXpressCompression supported: 0 2024-09-21T09:34:34.143 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: kBZip2Compression supported: 0 2024-09-21T09:34:34.144 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-21T09:34:34.144 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: kLZ4Compression supported: 1 2024-09-21T09:34:34.144 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: kZlibCompression supported: 1 2024-09-21T09:34:34.144 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: kLZ4HCCompression supported: 1 2024-09-21T09:34:34.144 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: kSnappyCompression supported: 1 2024-09-21T09:34:34.144 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-21T09:34:34.144 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-21T09:34:34.144 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 2024-09-21T09:34:34.144 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-21T09:34:34.144 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-21T09:34:34.144 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.merge_operator: 2024-09-21T09:34:34.144 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.compaction_filter: None 2024-09-21T09:34:34.144 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.compaction_filter_factory: None 2024-09-21T09:34:34.144 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.sst_partitioner_factory: None 2024-09-21T09:34:34.144 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-21T09:34:34.145 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-21T09:34:34.145 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5589359b7320) 2024-09-21T09:34:34.145 INFO:journalctl@ceph.mon.a.smithi022.stdout: cache_index_and_filter_blocks: 1 2024-09-21T09:34:34.145 INFO:journalctl@ceph.mon.a.smithi022.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-21T09:34:34.145 INFO:journalctl@ceph.mon.a.smithi022.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-21T09:34:34.145 INFO:journalctl@ceph.mon.a.smithi022.stdout: pin_top_level_index_and_filter: 1 2024-09-21T09:34:34.145 INFO:journalctl@ceph.mon.a.smithi022.stdout: index_type: 0 2024-09-21T09:34:34.145 INFO:journalctl@ceph.mon.a.smithi022.stdout: data_block_index_type: 0 2024-09-21T09:34:34.145 INFO:journalctl@ceph.mon.a.smithi022.stdout: index_shortening: 1 2024-09-21T09:34:34.145 INFO:journalctl@ceph.mon.a.smithi022.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-21T09:34:34.145 INFO:journalctl@ceph.mon.a.smithi022.stdout: checksum: 4 2024-09-21T09:34:34.145 INFO:journalctl@ceph.mon.a.smithi022.stdout: no_block_cache: 0 2024-09-21T09:34:34.145 INFO:journalctl@ceph.mon.a.smithi022.stdout: block_cache: 0x5589359ab610 2024-09-21T09:34:34.145 INFO:journalctl@ceph.mon.a.smithi022.stdout: block_cache_name: BinnedLRUCache 2024-09-21T09:34:34.145 INFO:journalctl@ceph.mon.a.smithi022.stdout: block_cache_options: 2024-09-21T09:34:34.145 INFO:journalctl@ceph.mon.a.smithi022.stdout: capacity : 536870912 2024-09-21T09:34:34.146 INFO:journalctl@ceph.mon.a.smithi022.stdout: num_shard_bits : 4 2024-09-21T09:34:34.146 INFO:journalctl@ceph.mon.a.smithi022.stdout: strict_capacity_limit : 0 2024-09-21T09:34:34.146 INFO:journalctl@ceph.mon.a.smithi022.stdout: high_pri_pool_ratio: 0.000 2024-09-21T09:34:34.146 INFO:journalctl@ceph.mon.a.smithi022.stdout: block_cache_compressed: (nil) 2024-09-21T09:34:34.146 INFO:journalctl@ceph.mon.a.smithi022.stdout: persistent_cache: (nil) 2024-09-21T09:34:34.146 INFO:journalctl@ceph.mon.a.smithi022.stdout: block_size: 4096 2024-09-21T09:34:34.146 INFO:journalctl@ceph.mon.a.smithi022.stdout: block_size_deviation: 10 2024-09-21T09:34:34.146 INFO:journalctl@ceph.mon.a.smithi022.stdout: block_restart_interval: 16 2024-09-21T09:34:34.146 INFO:journalctl@ceph.mon.a.smithi022.stdout: index_block_restart_interval: 1 2024-09-21T09:34:34.146 INFO:journalctl@ceph.mon.a.smithi022.stdout: metadata_block_size: 4096 2024-09-21T09:34:34.146 INFO:journalctl@ceph.mon.a.smithi022.stdout: partition_filters: 0 2024-09-21T09:34:34.146 INFO:journalctl@ceph.mon.a.smithi022.stdout: use_delta_encoding: 1 2024-09-21T09:34:34.147 INFO:journalctl@ceph.mon.a.smithi022.stdout: filter_policy: bloomfilter 2024-09-21T09:34:34.147 INFO:journalctl@ceph.mon.a.smithi022.stdout: whole_key_filtering: 1 2024-09-21T09:34:34.147 INFO:journalctl@ceph.mon.a.smithi022.stdout: verify_compression: 0 2024-09-21T09:34:34.147 INFO:journalctl@ceph.mon.a.smithi022.stdout: read_amp_bytes_per_bit: 0 2024-09-21T09:34:34.147 INFO:journalctl@ceph.mon.a.smithi022.stdout: format_version: 5 2024-09-21T09:34:34.147 INFO:journalctl@ceph.mon.a.smithi022.stdout: enable_index_compression: 1 2024-09-21T09:34:34.147 INFO:journalctl@ceph.mon.a.smithi022.stdout: block_align: 0 2024-09-21T09:34:34.147 INFO:journalctl@ceph.mon.a.smithi022.stdout: max_auto_readahead_size: 262144 2024-09-21T09:34:34.147 INFO:journalctl@ceph.mon.a.smithi022.stdout: prepopulate_block_cache: 0 2024-09-21T09:34:34.147 INFO:journalctl@ceph.mon.a.smithi022.stdout: initial_auto_readahead_size: 8192 2024-09-21T09:34:34.147 INFO:journalctl@ceph.mon.a.smithi022.stdout: num_file_reads_for_auto_readahead: 2 2024-09-21T09:34:34.147 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.write_buffer_size: 33554432 2024-09-21T09:34:34.147 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.max_write_buffer_number: 2 2024-09-21T09:34:34.147 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.compression: NoCompression 2024-09-21T09:34:34.148 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.bottommost_compression: Disabled 2024-09-21T09:34:34.148 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.prefix_extractor: nullptr 2024-09-21T09:34:34.148 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-21T09:34:34.148 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.num_levels: 7 2024-09-21T09:34:34.148 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-21T09:34:34.148 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-21T09:34:34.148 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-21T09:34:34.148 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-21T09:34:34.148 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-21T09:34:34.148 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-21T09:34:34.148 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-21T09:34:34.148 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-21T09:34:34.148 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-21T09:34:34.148 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-21T09:34:34.149 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-21T09:34:34.149 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-21T09:34:34.149 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-21T09:34:34.149 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.compression_opts.level: 32767 2024-09-21T09:34:34.149 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.compression_opts.strategy: 0 2024-09-21T09:34:34.149 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-21T09:34:34.149 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-21T09:34:34.149 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-21T09:34:34.149 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-21T09:34:34.149 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.compression_opts.enabled: false 2024-09-21T09:34:34.149 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-21T09:34:34.149 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-21T09:34:34.149 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-21T09:34:34.149 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-21T09:34:34.149 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.target_file_size_base: 67108864 2024-09-21T09:34:34.150 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-21T09:34:34.150 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-21T09:34:34.150 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-21T09:34:34.150 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-21T09:34:34.150 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-21T09:34:34.150 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-21T09:34:34.150 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-21T09:34:34.150 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-21T09:34:34.150 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-21T09:34:34.150 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-21T09:34:34.150 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-21T09:34:34.150 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-21T09:34:34.150 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-21T09:34:34.150 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-21T09:34:34.151 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.arena_block_size: 1048576 2024-09-21T09:34:34.151 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-21T09:34:34.151 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-21T09:34:34.151 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.disable_auto_compactions: 0 2024-09-21T09:34:34.151 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-21T09:34:34.151 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-21T09:34:34.151 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-21T09:34:34.151 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-21T09:34:34.151 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-21T09:34:34.151 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-21T09:34:34.151 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-21T09:34:34.151 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-21T09:34:34.151 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-21T09:34:34.151 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-21T09:34:34.152 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-21T09:34:34.152 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.inplace_update_support: 0 2024-09-21T09:34:34.152 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-21T09:34:34.152 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-21T09:34:34.152 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-21T09:34:34.152 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-21T09:34:34.152 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.bloom_locality: 0 2024-09-21T09:34:34.152 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.max_successive_merges: 0 2024-09-21T09:34:34.152 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-21T09:34:34.152 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.paranoid_file_checks: 0 2024-09-21T09:34:34.152 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.force_consistency_checks: 1 2024-09-21T09:34:34.152 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.report_bg_io_stats: 0 2024-09-21T09:34:34.152 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.ttl: 2592000 2024-09-21T09:34:34.152 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-21T09:34:34.153 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-21T09:34:34.153 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-21T09:34:34.153 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.enable_blob_files: false 2024-09-21T09:34:34.153 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.min_blob_size: 0 2024-09-21T09:34:34.153 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.blob_file_size: 268435456 2024-09-21T09:34:34.153 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-21T09:34:34.153 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-21T09:34:34.153 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-21T09:34:34.153 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-21T09:34:34.153 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-21T09:34:34.153 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.blob_file_starting_level: 0 2024-09-21T09:34:34.153 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-21T09:34:34.153 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 33.sst 2024-09-21T09:34:34.153 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-21T09:34:34.154 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 succeeded,manifest_file_number is 15, next_file_number is 35, last_sequence is 12769, log_number is 31,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-21T09:34:34.154 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 31 2024-09-21T09:34:34.154 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: b9b914da-1973-42af-b0a1-d06ce239017b 2024-09-21T09:34:34.154 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726911273904729, "job": 1, "event": "recovery_started", "wal_files": [31]} 2024-09-21T09:34:34.154 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #31 mode 2 2024-09-21T09:34:34.154 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726911273919502, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 36, "file_size": 4736025, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12770, "largest_seqno": 15621, "table_properties": {"data_size": 4722107, "index_size": 9415, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 3461, "raw_key_size": 31316, "raw_average_key_size": 23, "raw_value_size": 4693505, "raw_average_value_size": 3451, "num_data_blocks": 410, "num_entries": 1360, "num_filter_entries": 1360, "num_deletions": 307, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726911273, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "b9b914da-1973-42af-b0a1-d06ce239017b", "db_session_id": "9ZJVPDCFN8VGI5MORAZ8", "orig_file_number": 36, "seqno_to_time_mapping": "N/A"}} 2024-09-21T09:34:34.154 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726911273919663, "job": 1, "event": "recovery_finished"} 2024-09-21T09:34:34.154 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: [db/version_set.cc:5047] Creating manifest 38 2024-09-21T09:34:34.154 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-21T09:34:34.154 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000031.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-21T09:34:34.154 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x5589359dae00 2024-09-21T09:34:34.154 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: DB pointer 0x558935adc000 2024-09-21T09:34:34.154 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-21T09:34:34.155 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-21T09:34:34.155 INFO:journalctl@ceph.mon.a.smithi022.stdout: ** DB Stats ** 2024-09-21T09:34:34.155 INFO:journalctl@ceph.mon.a.smithi022.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-21T09:34:34.155 INFO:journalctl@ceph.mon.a.smithi022.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-21T09:34:34.155 INFO:journalctl@ceph.mon.a.smithi022.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-21T09:34:34.155 INFO:journalctl@ceph.mon.a.smithi022.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-21T09:34:34.155 INFO:journalctl@ceph.mon.a.smithi022.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-21T09:34:34.155 INFO:journalctl@ceph.mon.a.smithi022.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-21T09:34:34.155 INFO:journalctl@ceph.mon.a.smithi022.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-21T09:34:34.155 INFO:journalctl@ceph.mon.a.smithi022.stdout: 2024-09-21T09:34:34.155 INFO:journalctl@ceph.mon.a.smithi022.stdout: ** Compaction Stats [default] ** 2024-09-21T09:34:34.155 INFO:journalctl@ceph.mon.a.smithi022.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-21T09:34:34.155 INFO:journalctl@ceph.mon.a.smithi022.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-21T09:34:34.155 INFO:journalctl@ceph.mon.a.smithi022.stdout: L0 1/0 4.52 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 624.9 0.01 0.00 1 0.007 0 0 0.0 0.0 2024-09-21T09:34:34.156 INFO:journalctl@ceph.mon.a.smithi022.stdout: L6 1/0 7.42 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-21T09:34:34.156 INFO:journalctl@ceph.mon.a.smithi022.stdout: Sum 2/0 11.94 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 624.9 0.01 0.00 1 0.007 0 0 0.0 0.0 2024-09-21T09:34:34.156 INFO:journalctl@ceph.mon.a.smithi022.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 624.9 0.01 0.00 1 0.007 0 0 0.0 0.0 2024-09-21T09:34:34.156 INFO:journalctl@ceph.mon.a.smithi022.stdout: 2024-09-21T09:34:34.156 INFO:journalctl@ceph.mon.a.smithi022.stdout: ** Compaction Stats [default] ** 2024-09-21T09:34:34.156 INFO:journalctl@ceph.mon.a.smithi022.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-21T09:34:34.156 INFO:journalctl@ceph.mon.a.smithi022.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-21T09:34:34.156 INFO:journalctl@ceph.mon.a.smithi022.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 624.9 0.01 0.00 1 0.007 0 0 0.0 0.0 2024-09-21T09:34:34.156 INFO:journalctl@ceph.mon.a.smithi022.stdout: 2024-09-21T09:34:34.156 INFO:journalctl@ceph.mon.a.smithi022.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-21T09:34:34.156 INFO:journalctl@ceph.mon.a.smithi022.stdout: 2024-09-21T09:34:34.156 INFO:journalctl@ceph.mon.a.smithi022.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-21T09:34:34.157 INFO:journalctl@ceph.mon.a.smithi022.stdout: Flush(GB): cumulative 0.004, interval 0.004 2024-09-21T09:34:34.157 INFO:journalctl@ceph.mon.a.smithi022.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-21T09:34:34.157 INFO:journalctl@ceph.mon.a.smithi022.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-21T09:34:34.157 INFO:journalctl@ceph.mon.a.smithi022.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-21T09:34:34.157 INFO:journalctl@ceph.mon.a.smithi022.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-21T09:34:34.157 INFO:journalctl@ceph.mon.a.smithi022.stdout: Cumulative compaction: 0.00 GB write, 258.30 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T09:34:34.157 INFO:journalctl@ceph.mon.a.smithi022.stdout: Interval compaction: 0.00 GB write, 258.30 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T09:34:34.157 INFO:journalctl@ceph.mon.a.smithi022.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-21T09:34:34.157 INFO:journalctl@ceph.mon.a.smithi022.stdout: Block cache BinnedLRUCache@0x5589359ab610#7 capacity: 512.00 MB usage: 14.11 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1e-05 secs_since: 0 2024-09-21T09:34:34.157 INFO:journalctl@ceph.mon.a.smithi022.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,4.03 KB,0.0007689%) IndexBlock(1,10.08 KB,0.00192225%) Misc(1,0.00 KB,0%) 2024-09-21T09:34:34.157 INFO:journalctl@ceph.mon.a.smithi022.stdout: 2024-09-21T09:34:34.157 INFO:journalctl@ceph.mon.a.smithi022.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-21T09:34:34.157 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: starting mon.a rank 0 at public addrs [v2:172.21.15.22:3300/0,v1:172.21.15.22:6789/0] at bind addrs [v2:172.21.15.22:3300/0,v1:172.21.15.22:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid d65e801e-77fa-11ef-baf6-efdc52797490 2024-09-21T09:34:34.157 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: mon.a@-1(???) e3 preinit fsid d65e801e-77fa-11ef-baf6-efdc52797490 2024-09-21T09:34:34.157 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: mon.a@-1(???).mds e8 new map 2024-09-21T09:34:34.158 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: mon.a@-1(???).mds e8 print_map 2024-09-21T09:34:34.158 INFO:journalctl@ceph.mon.a.smithi022.stdout: e8 2024-09-21T09:34:34.158 INFO:journalctl@ceph.mon.a.smithi022.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-09-21T09:34:34.158 INFO:journalctl@ceph.mon.a.smithi022.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-21T09:34:34.158 INFO:journalctl@ceph.mon.a.smithi022.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-21T09:34:34.158 INFO:journalctl@ceph.mon.a.smithi022.stdout: legacy client fscid: 1 2024-09-21T09:34:34.158 INFO:journalctl@ceph.mon.a.smithi022.stdout: 2024-09-21T09:34:34.158 INFO:journalctl@ceph.mon.a.smithi022.stdout: Filesystem 'cephfs' (1) 2024-09-21T09:34:34.158 INFO:journalctl@ceph.mon.a.smithi022.stdout: fs_name cephfs 2024-09-21T09:34:34.158 INFO:journalctl@ceph.mon.a.smithi022.stdout: epoch 8 2024-09-21T09:34:34.158 INFO:journalctl@ceph.mon.a.smithi022.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-09-21T09:34:34.158 INFO:journalctl@ceph.mon.a.smithi022.stdout: created 2024-09-21T09:27:37.584397+0000 2024-09-21T09:34:34.158 INFO:journalctl@ceph.mon.a.smithi022.stdout: modified 2024-09-21T09:27:50.029809+0000 2024-09-21T09:34:34.158 INFO:journalctl@ceph.mon.a.smithi022.stdout: tableserver 0 2024-09-21T09:34:34.159 INFO:journalctl@ceph.mon.a.smithi022.stdout: root 0 2024-09-21T09:34:34.159 INFO:journalctl@ceph.mon.a.smithi022.stdout: session_timeout 60 2024-09-21T09:34:34.159 INFO:journalctl@ceph.mon.a.smithi022.stdout: session_autoclose 300 2024-09-21T09:34:34.159 INFO:journalctl@ceph.mon.a.smithi022.stdout: max_file_size 1099511627776 2024-09-21T09:34:34.159 INFO:journalctl@ceph.mon.a.smithi022.stdout: max_xattr_size 65536 2024-09-21T09:34:34.159 INFO:journalctl@ceph.mon.a.smithi022.stdout: required_client_features {} 2024-09-21T09:34:34.159 INFO:journalctl@ceph.mon.a.smithi022.stdout: last_failure 0 2024-09-21T09:34:34.159 INFO:journalctl@ceph.mon.a.smithi022.stdout: last_failure_osd_epoch 0 2024-09-21T09:34:34.159 INFO:journalctl@ceph.mon.a.smithi022.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-21T09:34:34.159 INFO:journalctl@ceph.mon.a.smithi022.stdout: max_mds 1 2024-09-21T09:34:34.159 INFO:journalctl@ceph.mon.a.smithi022.stdout: in 0 2024-09-21T09:34:34.159 INFO:journalctl@ceph.mon.a.smithi022.stdout: up {0=24325} 2024-09-21T09:34:34.159 INFO:journalctl@ceph.mon.a.smithi022.stdout: failed 2024-09-21T09:34:34.160 INFO:journalctl@ceph.mon.a.smithi022.stdout: damaged 2024-09-21T09:34:34.160 INFO:journalctl@ceph.mon.a.smithi022.stdout: stopped 2024-09-21T09:34:34.160 INFO:journalctl@ceph.mon.a.smithi022.stdout: data_pools [3] 2024-09-21T09:34:34.160 INFO:journalctl@ceph.mon.a.smithi022.stdout: metadata_pool 2 2024-09-21T09:34:34.160 INFO:journalctl@ceph.mon.a.smithi022.stdout: inline_data disabled 2024-09-21T09:34:34.160 INFO:journalctl@ceph.mon.a.smithi022.stdout: balancer 2024-09-21T09:34:34.160 INFO:journalctl@ceph.mon.a.smithi022.stdout: bal_rank_mask -1 2024-09-21T09:34:34.160 INFO:journalctl@ceph.mon.a.smithi022.stdout: standby_count_wanted 0 2024-09-21T09:34:34.160 INFO:journalctl@ceph.mon.a.smithi022.stdout: qdb_cluster leader: 0 members: 2024-09-21T09:34:34.160 INFO:journalctl@ceph.mon.a.smithi022.stdout: [mds.b{0:24325} state up:active seq 5 addr [v2:172.21.15.64:6832/3589447551,v1:172.21.15.64:6833/3589447551] compat {c=[1],r=[1],i=[7ff]}] 2024-09-21T09:34:34.160 INFO:journalctl@ceph.mon.a.smithi022.stdout: 2024-09-21T09:34:34.160 INFO:journalctl@ceph.mon.a.smithi022.stdout: 2024-09-21T09:34:34.161 INFO:journalctl@ceph.mon.a.smithi022.stdout: Standby daemons: 2024-09-21T09:34:34.161 INFO:journalctl@ceph.mon.a.smithi022.stdout: 2024-09-21T09:34:34.161 INFO:journalctl@ceph.mon.a.smithi022.stdout: [mds.a{-1:14424} state up:standby seq 1 addr [v2:172.21.15.22:6834/3558312740,v1:172.21.15.22:6835/3558312740] compat {c=[1],r=[1],i=[7ff]}] 2024-09-21T09:34:34.161 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: mon.a@-1(???).osd e262 crush map has features 3314933069573799936, adjusting msgr requires 2024-09-21T09:34:34.161 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: mon.a@-1(???).osd e262 crush map has features 432629308056666112, adjusting msgr requires 2024-09-21T09:34:34.161 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: mon.a@-1(???).osd e262 crush map has features 432629308056666112, adjusting msgr requires 2024-09-21T09:34:34.161 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: mon.a@-1(???).osd e262 crush map has features 432629308056666112, adjusting msgr requires 2024-09-21T09:34:34.161 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 ceph-mon[96736]: mon.a@-1(???).paxosservice(auth 1..19) refresh upgraded, format 0 -> 3 2024-09-21T09:34:34.161 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 bash[96657]: 0513395c784d2ca33f510b41dec5e34718168c717e222018230be292f24d09d1 2024-09-21T09:34:34.161 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:33 smithi022 systemd[1]: Started Ceph mon.a for d65e801e-77fa-11ef-baf6-efdc52797490. 2024-09-21T09:34:34.427 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-09-21T09:34:34.427 INFO:teuthology.orchestra.run.smithi022.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47", 2024-09-21T09:34:34.428 INFO:teuthology.orchestra.run.smithi022.stdout: "in_progress": true, 2024-09-21T09:34:34.428 INFO:teuthology.orchestra.run.smithi022.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T09:34:34.428 INFO:teuthology.orchestra.run.smithi022.stdout: "services_complete": [ 2024-09-21T09:34:34.428 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr" 2024-09-21T09:34:34.428 INFO:teuthology.orchestra.run.smithi022.stdout: ], 2024-09-21T09:34:34.428 INFO:teuthology.orchestra.run.smithi022.stdout: "progress": "2/20 daemons upgraded", 2024-09-21T09:34:34.428 INFO:teuthology.orchestra.run.smithi022.stdout: "message": "Currently upgrading mon daemons", 2024-09-21T09:34:34.428 INFO:teuthology.orchestra.run.smithi022.stdout: "is_paused": false 2024-09-21T09:34:34.428 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-09-21T09:34:34.441 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:34:34 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:34:34.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:34:36.058 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:34:35 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:34:35] "GET /metrics HTTP/1.1" 200 36131 "" "Prometheus/2.51.0" 2024-09-21T09:34:36.192 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:34:35 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:34:35.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:34:36.192 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:34:35 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:34:35.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:34:37.011 INFO:tasks.rados.rados.0.smithi022.stdout:1005: done (3 left) 2024-09-21T09:34:37.011 INFO:tasks.rados.rados.0.smithi022.stdout:1006: setattr oid 24 current snap is 101 2024-09-21T09:34:37.011 INFO:tasks.rados.rados.0.smithi022.stdout:1002: finishing write tid 2 to smithi02284960-4 2024-09-21T09:34:37.011 INFO:tasks.rados.rados.0.smithi022.stdout:1002: finishing write tid 3 to smithi02284960-4 2024-09-21T09:34:37.011 INFO:tasks.rados.rados.0.smithi022.stdout:1002: finishing write tid 4 to smithi02284960-4 2024-09-21T09:34:37.011 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 4 v 457 (ObjNum 335 snap 101 seq_num 335) dirty exists 2024-09-21T09:34:37.011 INFO:tasks.rados.rados.0.smithi022.stdout:1002: left oid 4 (ObjNum 335 snap 101 seq_num 335) 2024-09-21T09:34:37.012 INFO:tasks.rados.rados.0.smithi022.stdout:1003: finishing write tid 1 to smithi02284960-11 2024-09-21T09:34:37.012 INFO:tasks.rados.rados.0.smithi022.stdout:1003: finishing write tid 2 to smithi02284960-11 2024-09-21T09:34:37.012 INFO:tasks.rados.rados.0.smithi022.stdout:1003: finishing write tid 3 to smithi02284960-11 2024-09-21T09:34:37.012 INFO:tasks.rados.rados.0.smithi022.stdout:1003: finishing write tid 4 to smithi02284960-11 2024-09-21T09:34:37.012 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 11 v 552 (ObjNum 336 snap 101 seq_num 336) dirty exists 2024-09-21T09:34:37.012 INFO:tasks.rados.rados.0.smithi022.stdout:1003: left oid 11 (ObjNum 336 snap 101 seq_num 336) 2024-09-21T09:34:37.012 INFO:tasks.rados.rados.0.smithi022.stdout:1004: finishing write tid 1 to smithi02284960-44 2024-09-21T09:34:37.012 INFO:tasks.rados.rados.0.smithi022.stdout:1004: finishing write tid 2 to smithi02284960-44 2024-09-21T09:34:37.012 INFO:tasks.rados.rados.0.smithi022.stdout:1004: finishing write tid 3 to smithi02284960-44 2024-09-21T09:34:37.013 INFO:tasks.rados.rados.0.smithi022.stdout:1004: finishing write tid 4 to smithi02284960-44 2024-09-21T09:34:37.013 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 44 v 555 (ObjNum 337 snap 101 seq_num 337) dirty exists 2024-09-21T09:34:37.013 INFO:tasks.rados.rados.0.smithi022.stdout:1004: left oid 44 (ObjNum 337 snap 101 seq_num 337) 2024-09-21T09:34:37.014 INFO:tasks.rados.rados.0.smithi022.stdout:1002: done (3 left) 2024-09-21T09:34:37.014 INFO:tasks.rados.rados.0.smithi022.stdout:1003: done (2 left) 2024-09-21T09:34:37.014 INFO:tasks.rados.rados.0.smithi022.stdout:1004: done (1 left) 2024-09-21T09:34:37.014 INFO:tasks.rados.rados.0.smithi022.stdout:1007: rollback oid 25 current snap is 101 2024-09-21T09:34:37.014 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 25 to 95 2024-09-21T09:34:37.014 INFO:tasks.rados.rados.0.smithi022.stdout:1008: read oid 33 snap -1 2024-09-21T09:34:37.015 INFO:tasks.rados.rados.0.smithi022.stdout:1008: expect (ObjNum 309 snap 91 seq_num 309) 2024-09-21T09:34:37.015 INFO:tasks.rados.rados.0.smithi022.stdout:1009: read oid 1 snap -1 2024-09-21T09:34:37.015 INFO:tasks.rados.rados.0.smithi022.stdout:1009: expect (ObjNum 328 snap 100 seq_num 328) 2024-09-21T09:34:37.015 INFO:tasks.rados.rados.0.smithi022.stdout:1010: rmattr oid 32 current snap is 101 2024-09-21T09:34:37.015 INFO:tasks.rados.rados.0.smithi022.stdout:1011: rmattr oid 22 current snap is 101 2024-09-21T09:34:37.015 INFO:tasks.rados.rados.0.smithi022.stdout:1011: done (5 left) 2024-09-21T09:34:37.015 INFO:tasks.rados.rados.0.smithi022.stdout:1012: setattr oid 2 current snap is 101 2024-09-21T09:34:37.015 INFO:tasks.rados.rados.0.smithi022.stdout:1013: read oid 30 snap 96 2024-09-21T09:34:37.016 INFO:tasks.rados.rados.0.smithi022.stdout:1013: expect deleted 2024-09-21T09:34:37.016 INFO:tasks.rados.rados.0.smithi022.stdout:1014: rollback oid 50 current snap is 101 2024-09-21T09:34:37.016 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 50 to 69 2024-09-21T09:34:37.017 INFO:tasks.rados.rados.0.smithi022.stdout:1015: setattr oid 40 current snap is 101 2024-09-21T09:34:37.017 INFO:tasks.rados.rados.0.smithi022.stdout:1016: rollback oid 6 current snap is 101 2024-09-21T09:34:37.017 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 6 to 75 2024-09-21T09:34:37.017 INFO:tasks.rados.rados.0.smithi022.stdout:1017: rollback oid 14 current snap is 101 2024-09-21T09:34:37.017 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 14 to 95 2024-09-21T09:34:37.017 INFO:tasks.rados.rados.0.smithi022.stdout:1018: read oid 18 snap -1 2024-09-21T09:34:37.017 INFO:tasks.rados.rados.0.smithi022.stdout:1018: expect deleted 2024-09-21T09:34:37.017 INFO:tasks.rados.rados.0.smithi022.stdout:1019: snap_create 2024-09-21T09:34:37.025 INFO:tasks.rados.rados.0.smithi022.stdout:1009: expect (ObjNum 328 snap 100 seq_num 328) 2024-09-21T09:34:37.058 INFO:tasks.rados.rados.0.smithi022.stdout:1007: finishing rollback tid 1 to smithi02284960-25 2024-09-21T09:34:37.058 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 25 v 482 (ObjNum 289 snap 0 seq_num 0) dirty dne 2024-09-21T09:34:37.058 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 24 v 636 (ObjNum 331 snap 101 seq_num 331) dirty exists 2024-09-21T09:34:37.058 INFO:tasks.rados.rados.0.smithi022.stdout:1014: finishing rollback tid 0 to smithi02284960-50 2024-09-21T09:34:37.058 INFO:tasks.rados.rados.0.smithi022.stdout:1014: finishing rollback tid 1 to smithi02284960-50 2024-09-21T09:34:37.059 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 50 v 321 (ObjNum 65 snap 0 seq_num 1649767256) dirty dne 2024-09-21T09:34:37.059 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 40 v 566 (ObjNum 332 snap 101 seq_num 332) dirty exists 2024-09-21T09:34:37.059 INFO:tasks.rados.rados.0.smithi022.stdout:1017: finishing rollback tid 0 to smithi02284960-14 2024-09-21T09:34:37.059 INFO:tasks.rados.rados.0.smithi022.stdout:1017: finishing rollback tid 1 to smithi02284960-14 2024-09-21T09:34:37.059 INFO:tasks.rados.rados.0.smithi022.stdout:1017: finishing rollback tid 2 to smithi02284960-14 2024-09-21T09:34:37.059 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 14 v 258 (ObjNum 286 snap 83 seq_num 286) dirty exists 2024-09-21T09:34:37.059 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 2 v 276 (ObjNum 301 snap 88 seq_num 301) dirty exists 2024-09-21T09:34:37.059 INFO:tasks.rados.rados.0.smithi022.stdout:1008: expect (ObjNum 309 snap 91 seq_num 309) 2024-09-21T09:34:37.180 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 32 v 612 (ObjNum 315 snap 95 seq_num 315) dirty exists 2024-09-21T09:34:37.180 INFO:tasks.rados.rados.0.smithi022.stdout:1016: finishing rollback tid 0 to smithi02284960-6 2024-09-21T09:34:37.180 INFO:tasks.rados.rados.0.smithi022.stdout:1016: finishing rollback tid 1 to smithi02284960-6 2024-09-21T09:34:37.180 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 6 v 617 (ObjNum 161 snap 0 seq_num 2956027908) dirty dne 2024-09-21T09:34:37.308 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:34:37 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: ignoring --setuser ceph since I am not root 2024-09-21T09:34:37.308 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:34:37 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: ignoring --setgroup ceph since I am not root 2024-09-21T09:34:37.308 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:34:37 smithi022 ceph-mgr[88384]: -- 172.21.15.22:0/456063228 <== mon.2 v2:172.21.15.22:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55b6bf6150e0 con 0x55b6bf5f3400 2024-09-21T09:34:37.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:36 smithi022 ceph-mon[39525]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:34:37.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:36 smithi022 ceph-mon[39525]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:34:37.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:36 smithi022 ceph-mon[39525]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:34:37.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:36 smithi022 ceph-mon[39525]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T09:34:37.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:36 smithi022 ceph-mon[39525]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T09:34:37.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:36 smithi022 ceph-mon[39525]: from='client.25109 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:34:37.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:36 smithi022 ceph-mon[39525]: from='client.25115 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:34:37.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:36 smithi022 ceph-mon[39525]: from='client.25121 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:34:37.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:36 smithi022 ceph-mon[39525]: pgmap v52: 121 pgs: 121 active+clean; 228 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.2 MiB/s rd, 504 KiB/s wr, 8 op/s 2024-09-21T09:34:37.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:36 smithi022 ceph-mon[39525]: from='client.? 172.21.15.22:0/3181526778' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:34:37.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:36 smithi022 ceph-mon[39525]: from='client.25133 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:34:37.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:36 smithi022 ceph-mon[39525]: mon.a calling monitor election 2024-09-21T09:34:37.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:36 smithi022 ceph-mon[39525]: pgmap v53: 121 pgs: 121 active+clean; 228 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.4 MiB/s rd, 429 KiB/s wr, 7 op/s 2024-09-21T09:34:37.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:37 smithi022 ceph-mon[39525]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-21T09:34:37.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:37 smithi022 ceph-mon[39525]: monmap epoch 3 2024-09-21T09:34:37.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:37 smithi022 ceph-mon[39525]: fsid d65e801e-77fa-11ef-baf6-efdc52797490 2024-09-21T09:34:37.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:37 smithi022 ceph-mon[39525]: last_changed 2024-09-21T09:23:35.984741+0000 2024-09-21T09:34:37.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:37 smithi022 ceph-mon[39525]: created 2024-09-21T09:22:01.045656+0000 2024-09-21T09:34:37.310 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:37 smithi022 ceph-mon[39525]: min_mon_release 17 (quincy) 2024-09-21T09:34:37.310 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:37 smithi022 ceph-mon[39525]: election_strategy: 1 2024-09-21T09:34:37.310 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:37 smithi022 ceph-mon[39525]: 0: [v2:172.21.15.22:3300/0,v1:172.21.15.22:6789/0] mon.a 2024-09-21T09:34:37.310 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:37 smithi022 ceph-mon[39525]: 1: [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] mon.b 2024-09-21T09:34:37.310 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:37 smithi022 ceph-mon[39525]: 2: [v2:172.21.15.22:3301/0,v1:172.21.15.22:6790/0] mon.c 2024-09-21T09:34:37.310 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:37 smithi022 ceph-mon[39525]: fsmap cephfs:1 {0=b=up:active} 1 up:standby 2024-09-21T09:34:37.310 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:37 smithi022 ceph-mon[39525]: osdmap e262: 8 total, 8 up, 8 in 2024-09-21T09:34:37.310 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:37 smithi022 ceph-mon[39525]: mgrmap e34: y(active, since 36s), standbys: x 2024-09-21T09:34:37.310 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:37 smithi022 ceph-mon[39525]: overall HEALTH_OK 2024-09-21T09:34:37.310 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:37 smithi022 ceph-mon[39525]: from='mgr.25028 ' entity='' 2024-09-21T09:34:37.310 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:37 smithi022 ceph-mon[39525]: mgrmap e35: y(active, since 36s), standbys: x 2024-09-21T09:34:37.311 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:36 smithi022 ceph-mon[96736]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:34:37.311 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:36 smithi022 ceph-mon[96736]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:34:37.311 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:36 smithi022 ceph-mon[96736]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:34:37.311 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:36 smithi022 ceph-mon[96736]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T09:34:37.311 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:36 smithi022 ceph-mon[96736]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T09:34:37.311 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:36 smithi022 ceph-mon[96736]: from='client.25109 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:34:37.311 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:36 smithi022 ceph-mon[96736]: from='client.25115 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:34:37.311 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:36 smithi022 ceph-mon[96736]: from='client.25121 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:34:37.311 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:36 smithi022 ceph-mon[96736]: pgmap v52: 121 pgs: 121 active+clean; 228 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.2 MiB/s rd, 504 KiB/s wr, 8 op/s 2024-09-21T09:34:37.311 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:36 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/3181526778' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:34:37.311 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:36 smithi022 ceph-mon[96736]: from='client.25133 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:34:37.312 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:36 smithi022 ceph-mon[96736]: mon.a calling monitor election 2024-09-21T09:34:37.312 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:36 smithi022 ceph-mon[96736]: pgmap v53: 121 pgs: 121 active+clean; 228 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.4 MiB/s rd, 429 KiB/s wr, 7 op/s 2024-09-21T09:34:37.312 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:36 smithi022 ceph-mon[96736]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-21T09:34:37.312 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:36 smithi022 ceph-mon[96736]: monmap epoch 3 2024-09-21T09:34:37.312 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:36 smithi022 ceph-mon[96736]: fsid d65e801e-77fa-11ef-baf6-efdc52797490 2024-09-21T09:34:37.312 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:37 smithi022 ceph-mon[96736]: last_changed 2024-09-21T09:23:35.984741+0000 2024-09-21T09:34:37.312 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:37 smithi022 ceph-mon[96736]: created 2024-09-21T09:22:01.045656+0000 2024-09-21T09:34:37.312 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:37 smithi022 ceph-mon[96736]: min_mon_release 17 (quincy) 2024-09-21T09:34:37.312 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:37 smithi022 ceph-mon[96736]: election_strategy: 1 2024-09-21T09:34:37.312 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:37 smithi022 ceph-mon[96736]: 0: [v2:172.21.15.22:3300/0,v1:172.21.15.22:6789/0] mon.a 2024-09-21T09:34:37.312 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:37 smithi022 ceph-mon[96736]: 1: [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] mon.b 2024-09-21T09:34:37.312 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:37 smithi022 ceph-mon[96736]: 2: [v2:172.21.15.22:3301/0,v1:172.21.15.22:6790/0] mon.c 2024-09-21T09:34:37.312 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:37 smithi022 ceph-mon[96736]: fsmap cephfs:1 {0=b=up:active} 1 up:standby 2024-09-21T09:34:37.312 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:37 smithi022 ceph-mon[96736]: osdmap e262: 8 total, 8 up, 8 in 2024-09-21T09:34:37.313 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:37 smithi022 ceph-mon[96736]: mgrmap e34: y(active, since 36s), standbys: x 2024-09-21T09:34:37.313 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:37 smithi022 ceph-mon[96736]: overall HEALTH_OK 2024-09-21T09:34:37.313 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:37 smithi022 ceph-mon[96736]: from='mgr.25028 ' entity='' 2024-09-21T09:34:37.313 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:37 smithi022 ceph-mon[96736]: mgrmap e35: y(active, since 36s), standbys: x 2024-09-21T09:34:37.441 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:34:36 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:34:36.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi022\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:34:37.442 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:34:37 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[78884]: ignoring --setuser ceph since I am not root 2024-09-21T09:34:37.442 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:34:37 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[78884]: ignoring --setgroup ceph since I am not root 2024-09-21T09:34:37.442 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:34:37 smithi064 ceph-mgr[78907]: -- 172.21.15.64:0/4207139379 <== mon.1 v2:172.21.15.64:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55d2f57630e0 con 0x55d2f5741000 2024-09-21T09:34:37.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:37 smithi064 ceph-mon[31124]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:34:37.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:37 smithi064 ceph-mon[31124]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:34:37.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:37 smithi064 ceph-mon[31124]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:34:37.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:37 smithi064 ceph-mon[31124]: from='mgr.25028 172.21.15.22:0/2928229956' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T09:34:37.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:37 smithi064 ceph-mon[31124]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T09:34:37.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:37 smithi064 ceph-mon[31124]: from='client.25109 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:34:37.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:37 smithi064 ceph-mon[31124]: from='client.25115 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:34:37.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:37 smithi064 ceph-mon[31124]: from='client.25121 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:34:37.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:37 smithi064 ceph-mon[31124]: pgmap v52: 121 pgs: 121 active+clean; 228 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.2 MiB/s rd, 504 KiB/s wr, 8 op/s 2024-09-21T09:34:37.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:37 smithi064 ceph-mon[31124]: from='client.? 172.21.15.22:0/3181526778' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:34:37.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:37 smithi064 ceph-mon[31124]: from='client.25133 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:34:37.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:37 smithi064 ceph-mon[31124]: mon.a calling monitor election 2024-09-21T09:34:37.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:37 smithi064 ceph-mon[31124]: pgmap v53: 121 pgs: 121 active+clean; 228 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.4 MiB/s rd, 429 KiB/s wr, 7 op/s 2024-09-21T09:34:37.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:37 smithi064 ceph-mon[31124]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-21T09:34:37.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:37 smithi064 ceph-mon[31124]: monmap epoch 3 2024-09-21T09:34:37.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:37 smithi064 ceph-mon[31124]: fsid d65e801e-77fa-11ef-baf6-efdc52797490 2024-09-21T09:34:37.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:37 smithi064 ceph-mon[31124]: last_changed 2024-09-21T09:23:35.984741+0000 2024-09-21T09:34:37.444 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:37 smithi064 ceph-mon[31124]: created 2024-09-21T09:22:01.045656+0000 2024-09-21T09:34:37.444 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:37 smithi064 ceph-mon[31124]: min_mon_release 17 (quincy) 2024-09-21T09:34:37.444 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:37 smithi064 ceph-mon[31124]: election_strategy: 1 2024-09-21T09:34:37.444 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:37 smithi064 ceph-mon[31124]: 0: [v2:172.21.15.22:3300/0,v1:172.21.15.22:6789/0] mon.a 2024-09-21T09:34:37.444 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:37 smithi064 ceph-mon[31124]: 1: [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] mon.b 2024-09-21T09:34:37.444 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:37 smithi064 ceph-mon[31124]: 2: [v2:172.21.15.22:3301/0,v1:172.21.15.22:6790/0] mon.c 2024-09-21T09:34:37.444 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:37 smithi064 ceph-mon[31124]: fsmap cephfs:1 {0=b=up:active} 1 up:standby 2024-09-21T09:34:37.444 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:37 smithi064 ceph-mon[31124]: osdmap e262: 8 total, 8 up, 8 in 2024-09-21T09:34:37.444 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:37 smithi064 ceph-mon[31124]: mgrmap e34: y(active, since 36s), standbys: x 2024-09-21T09:34:37.444 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:37 smithi064 ceph-mon[31124]: overall HEALTH_OK 2024-09-21T09:34:37.444 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:37 smithi064 ceph-mon[31124]: from='mgr.25028 ' entity='' 2024-09-21T09:34:37.445 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:37 smithi064 ceph-mon[31124]: mgrmap e35: y(active, since 36s), standbys: x 2024-09-21T09:34:38.014 INFO:tasks.rados.rados.0.smithi022.stdout:1006: done (12 left) 2024-09-21T09:34:38.014 INFO:tasks.rados.rados.0.smithi022.stdout:1007: done (11 left) 2024-09-21T09:34:38.014 INFO:tasks.rados.rados.0.smithi022.stdout:1008: done (10 left) 2024-09-21T09:34:38.015 INFO:tasks.rados.rados.0.smithi022.stdout:1009: done (9 left) 2024-09-21T09:34:38.015 INFO:tasks.rados.rados.0.smithi022.stdout:1010: done (8 left) 2024-09-21T09:34:38.015 INFO:tasks.rados.rados.0.smithi022.stdout:1012: done (7 left) 2024-09-21T09:34:38.015 INFO:tasks.rados.rados.0.smithi022.stdout:1013: done (6 left) 2024-09-21T09:34:38.015 INFO:tasks.rados.rados.0.smithi022.stdout:1014: done (5 left) 2024-09-21T09:34:38.015 INFO:tasks.rados.rados.0.smithi022.stdout:1015: done (4 left) 2024-09-21T09:34:38.015 INFO:tasks.rados.rados.0.smithi022.stdout:1016: done (3 left) 2024-09-21T09:34:38.015 INFO:tasks.rados.rados.0.smithi022.stdout:1017: done (2 left) 2024-09-21T09:34:38.015 INFO:tasks.rados.rados.0.smithi022.stdout:1018: done (1 left) 2024-09-21T09:34:38.015 INFO:tasks.rados.rados.0.smithi022.stdout:1019: done (0 left) 2024-09-21T09:34:38.015 INFO:tasks.rados.rados.0.smithi022.stdout:1020: copy_from oid 2 from oid 18 current snap is 102 2024-09-21T09:34:38.015 INFO:tasks.rados.rados.0.smithi022.stdout:1021: snap_create 2024-09-21T09:34:38.018 INFO:tasks.rados.rados.0.smithi022.stdout:1020: finishing copy_from to smithi02284960-2 2024-09-21T09:34:38.018 INFO:tasks.rados.rados.0.smithi022.stdout:1020: got expected ENOENT (src dne) 2024-09-21T09:34:38.018 INFO:tasks.rados.rados.0.smithi022.stdout:1020: finishing copy_from racing read to smithi02284960-2 2024-09-21T09:34:38.307 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:38 smithi022 ceph-mon[96736]: osdmap e263: 8 total, 8 up, 8 in 2024-09-21T09:34:38.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:38 smithi022 ceph-mon[39525]: osdmap e263: 8 total, 8 up, 8 in 2024-09-21T09:34:38.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:38 smithi064 ceph-mon[31124]: osdmap e263: 8 total, 8 up, 8 in 2024-09-21T09:34:38.933 INFO:tasks.rados.rados.0.smithi022.stdout:1020: done (1 left) 2024-09-21T09:34:38.934 INFO:tasks.rados.rados.0.smithi022.stdout:1021: done (0 left) 2024-09-21T09:34:38.934 INFO:tasks.rados.rados.0.smithi022.stdout:1022: read oid 19 snap -1 2024-09-21T09:34:38.934 INFO:tasks.rados.rados.0.smithi022.stdout:1022: expect (ObjNum 289 snap 86 seq_num 289) 2024-09-21T09:34:38.934 INFO:tasks.rados.rados.0.smithi022.stdout:1023: read oid 26 snap 72 2024-09-21T09:34:38.934 INFO:tasks.rados.rados.0.smithi022.stdout:1023: expect (ObjNum 204 snap 56 seq_num 204) 2024-09-21T09:34:38.934 INFO:tasks.rados.rados.0.smithi022.stdout:1024: setattr oid 37 current snap is 103 2024-09-21T09:34:38.934 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 1 current snap is 103 2024-09-21T09:34:38.934 INFO:tasks.rados.rados.0.smithi022.stdout:1025: seq_num 338 ranges {696320=466944,1163264=401408} 2024-09-21T09:34:38.941 INFO:tasks.rados.rados.0.smithi022.stdout:1025: writing smithi02284960-1 from 696320 to 1163264 tid 1 2024-09-21T09:34:38.944 INFO:tasks.rados.rados.0.smithi022.stdout:1025: writing smithi02284960-1 from 1163264 to 1564672 tid 2 2024-09-21T09:34:38.944 INFO:tasks.rados.rados.0.smithi022.stdout:1026: delete oid 32 current snap is 103 2024-09-21T09:34:38.946 INFO:tasks.rados.rados.0.smithi022.stdout:1023: expect (ObjNum 204 snap 56 seq_num 204) 2024-09-21T09:34:39.017 INFO:tasks.rados.rados.0.smithi022.stdout:1025: finishing write tid 1 to smithi02284960-1 2024-09-21T09:34:39.017 INFO:tasks.rados.rados.0.smithi022.stdout:1025: finishing write tid 2 to smithi02284960-1 2024-09-21T09:34:39.017 INFO:tasks.rados.rados.0.smithi022.stdout:1025: finishing write tid 3 to smithi02284960-1 2024-09-21T09:34:39.017 INFO:tasks.rados.rados.0.smithi022.stdout:1023: done (4 left) 2024-09-21T09:34:39.017 INFO:tasks.rados.rados.0.smithi022.stdout:1026: done (3 left) 2024-09-21T09:34:39.017 INFO:tasks.rados.rados.0.smithi022.stdout:1027: setattr oid 17 current snap is 103 2024-09-21T09:34:39.017 INFO:tasks.rados.rados.0.smithi022.stdout:1025: finishing write tid 4 to smithi02284960-1 2024-09-21T09:34:39.017 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 1 v 563 (ObjNum 338 snap 103 seq_num 338) dirty exists 2024-09-21T09:34:39.017 INFO:tasks.rados.rados.0.smithi022.stdout:1025: left oid 1 (ObjNum 338 snap 103 seq_num 338) 2024-09-21T09:34:39.018 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 37 v 644 (ObjNum 218 snap 63 seq_num 218) dirty exists 2024-09-21T09:34:39.018 INFO:tasks.rados.rados.0.smithi022.stdout:1022: expect (ObjNum 289 snap 86 seq_num 289) 2024-09-21T09:34:39.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:39 smithi022 ceph-mon[39525]: osdmap e264: 8 total, 8 up, 8 in 2024-09-21T09:34:39.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:39 smithi022 ceph-mon[39525]: osdmap e265: 8 total, 8 up, 8 in 2024-09-21T09:34:39.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:39 smithi022 ceph-mon[96736]: osdmap e264: 8 total, 8 up, 8 in 2024-09-21T09:34:39.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:39 smithi022 ceph-mon[96736]: osdmap e265: 8 total, 8 up, 8 in 2024-09-21T09:34:39.308 INFO:tasks.rados.rados.0.smithi022.stdout:1022: done (3 left) 2024-09-21T09:34:39.309 INFO:tasks.rados.rados.0.smithi022.stdout:1024: done (2 left) 2024-09-21T09:34:39.309 INFO:tasks.rados.rados.0.smithi022.stdout:1025: done (1 left) 2024-09-21T09:34:39.309 INFO:tasks.rados.rados.0.smithi022.stdout:1028: rollback oid 3 current snap is 103 2024-09-21T09:34:39.309 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 3 to 92 2024-09-21T09:34:39.310 INFO:tasks.rados.rados.0.smithi022.stdout:1029: snap_create 2024-09-21T09:34:39.312 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 17 v 454 (ObjNum 177 snap 0 seq_num 0) dirty exists 2024-09-21T09:34:39.312 INFO:tasks.rados.rados.0.smithi022.stdout:1028: finishing rollback tid 1 to smithi02284960-3 2024-09-21T09:34:39.312 INFO:tasks.rados.rados.0.smithi022.stdout:1028: finishing rollback tid 2 to smithi02284960-3 2024-09-21T09:34:39.312 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 3 v 123 (ObjNum 195 snap 50 seq_num 195) dirty exists 2024-09-21T09:34:39.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:39 smithi064 ceph-mon[31124]: osdmap e264: 8 total, 8 up, 8 in 2024-09-21T09:34:39.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:39 smithi064 ceph-mon[31124]: osdmap e265: 8 total, 8 up, 8 in 2024-09-21T09:34:40.016 INFO:tasks.rados.rados.0.smithi022.stdout:1027: done (2 left) 2024-09-21T09:34:40.016 INFO:tasks.rados.rados.0.smithi022.stdout:1028: done (1 left) 2024-09-21T09:34:40.016 INFO:tasks.rados.rados.0.smithi022.stdout:1029: done (0 left) 2024-09-21T09:34:40.016 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 36 current snap is 104 2024-09-21T09:34:40.017 INFO:tasks.rados.rados.0.smithi022.stdout:1030: seq_num 340 ranges {876544=499712,1376256=401408} 2024-09-21T09:34:40.023 INFO:tasks.rados.rados.0.smithi022.stdout:1030: writing smithi02284960-36 from 876544 to 1376256 tid 1 2024-09-21T09:34:40.027 INFO:tasks.rados.rados.0.smithi022.stdout:1030: writing smithi02284960-36 from 1376256 to 1777664 tid 2 2024-09-21T09:34:40.027 INFO:tasks.rados.rados.0.smithi022.stdout:1031: read oid 24 snap -1 2024-09-21T09:34:40.027 INFO:tasks.rados.rados.0.smithi022.stdout:1031: expect (ObjNum 331 snap 101 seq_num 331) 2024-09-21T09:34:40.027 INFO:tasks.rados.rados.0.smithi022.stdout:1032: read oid 30 snap -1 2024-09-21T09:34:40.027 INFO:tasks.rados.rados.0.smithi022.stdout:1032: expect deleted 2024-09-21T09:34:40.027 INFO:tasks.rados.rados.0.smithi022.stdout:1033: rmattr oid 9 current snap is 104 2024-09-21T09:34:40.027 INFO:tasks.rados.rados.0.smithi022.stdout:1034: rollback oid 14 current snap is 104 2024-09-21T09:34:40.027 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 14 to 87 2024-09-21T09:34:40.027 INFO:tasks.rados.rados.0.smithi022.stdout:1035: delete oid 22 current snap is 104 2024-09-21T09:34:40.034 INFO:tasks.rados.rados.0.smithi022.stdout:1030: finishing write tid 1 to smithi02284960-36 2024-09-21T09:34:40.034 INFO:tasks.rados.rados.0.smithi022.stdout:1030: finishing write tid 2 to smithi02284960-36 2024-09-21T09:34:40.034 INFO:tasks.rados.rados.0.smithi022.stdout:1034: finishing rollback tid 0 to smithi02284960-14 2024-09-21T09:34:40.034 INFO:tasks.rados.rados.0.smithi022.stdout:1034: finishing rollback tid 1 to smithi02284960-14 2024-09-21T09:34:40.034 INFO:tasks.rados.rados.0.smithi022.stdout:1034: finishing rollback tid 2 to smithi02284960-14 2024-09-21T09:34:40.034 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 14 v 261 (ObjNum 286 snap 83 seq_num 286) dirty exists 2024-09-21T09:34:40.034 INFO:tasks.rados.rados.0.smithi022.stdout:1032: done (5 left) 2024-09-21T09:34:40.034 INFO:tasks.rados.rados.0.smithi022.stdout:1034: done (4 left) 2024-09-21T09:34:40.034 INFO:tasks.rados.rados.0.smithi022.stdout:1035: done (3 left) 2024-09-21T09:34:40.034 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 30 current snap is 104 2024-09-21T09:34:40.035 INFO:tasks.rados.rados.0.smithi022.stdout:1036: seq_num 342 ranges {0=532480,532480=401408} 2024-09-21T09:34:40.037 INFO:tasks.rados.rados.0.smithi022.stdout:1036: writing smithi02284960-30 from 0 to 532480 tid 1 2024-09-21T09:34:40.039 INFO:tasks.rados.rados.0.smithi022.stdout:1036: writing smithi02284960-30 from 532480 to 933888 tid 2 2024-09-21T09:34:40.040 INFO:tasks.rados.rados.0.smithi022.stdout:1037: read oid 46 snap -1 2024-09-21T09:34:40.040 INFO:tasks.rados.rados.0.smithi022.stdout:1037: expect (ObjNum 204 snap 56 seq_num 204) 2024-09-21T09:34:40.040 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 22 current snap is 104 2024-09-21T09:34:40.040 INFO:tasks.rados.rados.0.smithi022.stdout:1030: finishing write tid 3 to smithi02284960-36 2024-09-21T09:34:40.040 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 9 v 458 (ObjNum 240 snap 65 seq_num 240) dirty exists 2024-09-21T09:34:40.040 INFO:tasks.rados.rados.0.smithi022.stdout:1030: finishing write tid 4 to smithi02284960-36 2024-09-21T09:34:40.040 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 36 v 325 (ObjNum 340 snap 104 seq_num 340) dirty exists 2024-09-21T09:34:40.040 INFO:tasks.rados.rados.0.smithi022.stdout:1030: left oid 36 (ObjNum 340 snap 104 seq_num 340) 2024-09-21T09:34:40.040 INFO:tasks.rados.rados.0.smithi022.stdout:1038: seq_num 343 ranges {0=548864,548864=401408} 2024-09-21T09:34:40.043 INFO:tasks.rados.rados.0.smithi022.stdout:1038: writing smithi02284960-22 from 0 to 548864 tid 1 2024-09-21T09:34:40.046 INFO:tasks.rados.rados.0.smithi022.stdout:1038: writing smithi02284960-22 from 548864 to 950272 tid 2 2024-09-21T09:34:40.046 INFO:tasks.rados.rados.0.smithi022.stdout:1030: done (5 left) 2024-09-21T09:34:40.046 INFO:tasks.rados.rados.0.smithi022.stdout:1033: done (4 left) 2024-09-21T09:34:40.046 INFO:tasks.rados.rados.0.smithi022.stdout:1039: read oid 7 snap -1 2024-09-21T09:34:40.046 INFO:tasks.rados.rados.0.smithi022.stdout:1039: expect deleted 2024-09-21T09:34:40.046 INFO:tasks.rados.rados.0.smithi022.stdout:1040: setattr oid 41 current snap is 104 2024-09-21T09:34:40.046 INFO:tasks.rados.rados.0.smithi022.stdout:1036: finishing write tid 1 to smithi02284960-30 2024-09-21T09:34:40.046 INFO:tasks.rados.rados.0.smithi022.stdout:1036: finishing write tid 2 to smithi02284960-30 2024-09-21T09:34:40.046 INFO:tasks.rados.rados.0.smithi022.stdout:1036: finishing write tid 3 to smithi02284960-30 2024-09-21T09:34:40.047 INFO:tasks.rados.rados.0.smithi022.stdout:1036: finishing write tid 4 to smithi02284960-30 2024-09-21T09:34:40.047 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 30 v 457 (ObjNum 342 snap 104 seq_num 342) dirty exists 2024-09-21T09:34:40.047 INFO:tasks.rados.rados.0.smithi022.stdout:1036: left oid 30 (ObjNum 342 snap 104 seq_num 342) 2024-09-21T09:34:40.048 INFO:tasks.rados.rados.0.smithi022.stdout:1036: done (5 left) 2024-09-21T09:34:40.048 INFO:tasks.rados.rados.0.smithi022.stdout:1041: read oid 31 snap -1 2024-09-21T09:34:40.048 INFO:tasks.rados.rados.0.smithi022.stdout:1041: expect deleted 2024-09-21T09:34:40.048 INFO:tasks.rados.rados.0.smithi022.stdout:1042: copy_from oid 18 from oid 14 current snap is 104 2024-09-21T09:34:40.048 INFO:tasks.rados.rados.0.smithi022.stdout:1043: rollback oid 48 current snap is 104 2024-09-21T09:34:40.048 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 48 to 98 2024-09-21T09:34:40.048 INFO:tasks.rados.rados.0.smithi022.stdout:1044: copy_from oid 49 from oid 21 current snap is 104 2024-09-21T09:34:40.048 INFO:tasks.rados.rados.0.smithi022.stdout:1045: read oid 25 snap 84 2024-09-21T09:34:40.048 INFO:tasks.rados.rados.0.smithi022.stdout:1045: expect deleted 2024-09-21T09:34:40.048 INFO:tasks.rados.rados.0.smithi022.stdout:1046: copy_from oid 19 from oid 5 current snap is 104 2024-09-21T09:34:40.048 INFO:tasks.rados.rados.0.smithi022.stdout:1047: snap_create 2024-09-21T09:34:40.052 INFO:tasks.rados.rados.0.smithi022.stdout:1037: expect (ObjNum 204 snap 56 seq_num 204) 2024-09-21T09:34:40.124 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 41 v 568 (ObjNum 247 snap 67 seq_num 247) dirty exists 2024-09-21T09:34:40.124 INFO:tasks.rados.rados.0.smithi022.stdout:1043: finishing rollback tid 0 to smithi02284960-48 2024-09-21T09:34:40.124 INFO:tasks.rados.rados.0.smithi022.stdout:1043: finishing rollback tid 1 to smithi02284960-48 2024-09-21T09:34:40.124 INFO:tasks.rados.rados.0.smithi022.stdout:1043: finishing rollback tid 2 to smithi02284960-48 2024-09-21T09:34:40.124 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 48 v 625 (ObjNum 318 snap 96 seq_num 318) dirty exists 2024-09-21T09:34:40.124 INFO:tasks.rados.rados.0.smithi022.stdout:1031: expect (ObjNum 331 snap 101 seq_num 331) 2024-09-21T09:34:40.318 INFO:tasks.rados.rados.0.smithi022.stdout:1046: finishing copy_from to smithi02284960-19 2024-09-21T09:34:40.318 INFO:tasks.rados.rados.0.smithi022.stdout:1046: got expected ENOENT (src dne) 2024-09-21T09:34:40.318 INFO:tasks.rados.rados.0.smithi022.stdout:1046: finishing copy_from racing read to smithi02284960-19 2024-09-21T09:34:40.318 INFO:tasks.rados.rados.0.smithi022.stdout:1038: finishing write tid 1 to smithi02284960-22 2024-09-21T09:34:40.319 INFO:tasks.rados.rados.0.smithi022.stdout:1038: finishing write tid 2 to smithi02284960-22 2024-09-21T09:34:40.319 INFO:tasks.rados.rados.0.smithi022.stdout:1038: finishing write tid 3 to smithi02284960-22 2024-09-21T09:34:40.319 INFO:tasks.rados.rados.0.smithi022.stdout:1038: finishing write tid 4 to smithi02284960-22 2024-09-21T09:34:40.319 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 22 v 487 (ObjNum 343 snap 104 seq_num 343) dirty exists 2024-09-21T09:34:40.319 INFO:tasks.rados.rados.0.smithi022.stdout:1038: left oid 22 (ObjNum 343 snap 104 seq_num 343) 2024-09-21T09:34:40.319 INFO:tasks.rados.rados.0.smithi022.stdout:1044: finishing copy_from racing read to smithi02284960-49 2024-09-21T09:34:40.319 INFO:tasks.rados.rados.0.smithi022.stdout:1044: finishing copy_from to smithi02284960-49 2024-09-21T09:34:40.319 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 49 v 256 (ObjNum 322 snap 97 seq_num 322) dirty exists 2024-09-21T09:34:40.319 INFO:tasks.rados.rados.0.smithi022.stdout:1042: finishing copy_from racing read to smithi02284960-18 2024-09-21T09:34:40.319 INFO:tasks.rados.rados.0.smithi022.stdout:1042: finishing copy_from to smithi02284960-18 2024-09-21T09:34:40.319 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 18 v 261 (ObjNum 286 snap 83 seq_num 286) dirty exists 2024-09-21T09:34:40.691 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:34:40 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:34:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:34:41.021 INFO:tasks.rados.rados.0.smithi022.stdout:1031: done (11 left) 2024-09-21T09:34:41.022 INFO:tasks.rados.rados.0.smithi022.stdout:1037: done (10 left) 2024-09-21T09:34:41.022 INFO:tasks.rados.rados.0.smithi022.stdout:1038: done (9 left) 2024-09-21T09:34:41.022 INFO:tasks.rados.rados.0.smithi022.stdout:1039: done (8 left) 2024-09-21T09:34:41.022 INFO:tasks.rados.rados.0.smithi022.stdout:1040: done (7 left) 2024-09-21T09:34:41.022 INFO:tasks.rados.rados.0.smithi022.stdout:1041: done (6 left) 2024-09-21T09:34:41.022 INFO:tasks.rados.rados.0.smithi022.stdout:1042: done (5 left) 2024-09-21T09:34:41.022 INFO:tasks.rados.rados.0.smithi022.stdout:1043: done (4 left) 2024-09-21T09:34:41.022 INFO:tasks.rados.rados.0.smithi022.stdout:1044: done (3 left) 2024-09-21T09:34:41.022 INFO:tasks.rados.rados.0.smithi022.stdout:1045: done (2 left) 2024-09-21T09:34:41.022 INFO:tasks.rados.rados.0.smithi022.stdout:1046: done (1 left) 2024-09-21T09:34:41.022 INFO:tasks.rados.rados.0.smithi022.stdout:1047: done (0 left) 2024-09-21T09:34:41.023 INFO:tasks.rados.rados.0.smithi022.stdout:1048: read oid 48 snap -1 2024-09-21T09:34:41.023 INFO:tasks.rados.rados.0.smithi022.stdout:1048: expect (ObjNum 318 snap 96 seq_num 318) 2024-09-21T09:34:41.023 INFO:tasks.rados.rados.0.smithi022.stdout:1049: snap_remove snap 98 2024-09-21T09:34:41.307 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:41 smithi022 ceph-mon[39525]: osdmap e266: 8 total, 8 up, 8 in 2024-09-21T09:34:41.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:41 smithi022 ceph-mon[96736]: osdmap e266: 8 total, 8 up, 8 in 2024-09-21T09:34:41.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:41 smithi064 ceph-mon[31124]: osdmap e266: 8 total, 8 up, 8 in 2024-09-21T09:34:42.023 INFO:tasks.rados.rados.0.smithi022.stdout:1049: done (1 left) 2024-09-21T09:34:42.023 INFO:tasks.rados.rados.0.smithi022.stdout:1050: snap_create 2024-09-21T09:34:42.023 INFO:tasks.rados.rados.0.smithi022.stdout:1048: expect (ObjNum 318 snap 96 seq_num 318) 2024-09-21T09:34:42.279 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:42 smithi064 ceph-mon[31124]: osdmap e267: 8 total, 8 up, 8 in 2024-09-21T09:34:42.307 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:42 smithi022 ceph-mon[39525]: osdmap e267: 8 total, 8 up, 8 in 2024-09-21T09:34:42.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:42 smithi022 ceph-mon[96736]: osdmap e267: 8 total, 8 up, 8 in 2024-09-21T09:34:42.442 INFO:tasks.rados.rados.0.smithi022.stdout:1048: done (1 left) 2024-09-21T09:34:42.442 INFO:tasks.rados.rados.0.smithi022.stdout:1050: done (0 left) 2024-09-21T09:34:42.442 INFO:tasks.rados.rados.0.smithi022.stdout:1051: copy_from oid 20 from oid 32 current snap is 106 2024-09-21T09:34:42.442 INFO:tasks.rados.rados.0.smithi022.stdout:1052: delete oid 3 current snap is 106 2024-09-21T09:34:42.443 INFO:tasks.rados.rados.0.smithi022.stdout:1052: done (1 left) 2024-09-21T09:34:42.444 INFO:tasks.rados.rados.0.smithi022.stdout:1053: copy_from oid 45 from oid 37 current snap is 106 2024-09-21T09:34:42.444 INFO:tasks.rados.rados.0.smithi022.stdout:1054: snap_remove snap 105 2024-09-21T09:34:42.691 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:34:42 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[78884]: [21/Sep/2024:09:34:42] ENGINE Bus STARTING 2024-09-21T09:34:42.691 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:34:42 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[78884]: CherryPy Checker: 2024-09-21T09:34:42.692 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:34:42 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[78884]: The Application mounted at '' has an empty config. 2024-09-21T09:34:42.692 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:34:42 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[78884]: 2024-09-21T09:34:42.692 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:34:42 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[78884]: [21/Sep/2024:09:34:42] ENGINE Serving on http://:::9283 2024-09-21T09:34:42.692 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:34:42 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x[78884]: [21/Sep/2024:09:34:42] ENGINE Bus STARTED 2024-09-21T09:34:43.197 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:34:42 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: 2024-09-21T09:34:42.827+0000 7fee8f81e640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:34:43.197 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:34:42 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: 2024-09-21T09:34:42.827+0000 7fee8f81e640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:34:43.197 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:34:42 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: 2024-09-21T09:34:42.827+0000 7fee8f81e640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:34:43.197 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:34:42 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: 2024-09-21T09:34:42.827+0000 7fee8f81e640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:34:43.197 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:34:42 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: 2024-09-21T09:34:42.827+0000 7fee8f81e640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:34:43.197 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:34:42 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: 2024-09-21T09:34:42.827+0000 7fee91822640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:34:43.198 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:34:42 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: 2024-09-21T09:34:42.827+0000 7fee91822640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:34:43.198 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:34:42 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: 2024-09-21T09:34:42.828+0000 7fee91822640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:34:43.198 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:34:42 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: 2024-09-21T09:34:42.828+0000 7fee91822640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:34:43.198 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:34:42 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: 2024-09-21T09:34:42.828+0000 7fee91822640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:34:43.198 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:43 smithi022 ceph-mon[39525]: osdmap e268: 8 total, 8 up, 8 in 2024-09-21T09:34:43.198 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:43 smithi022 ceph-mon[39525]: Standby manager daemon x restarted 2024-09-21T09:34:43.198 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:43 smithi022 ceph-mon[39525]: Standby manager daemon x started 2024-09-21T09:34:43.198 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:43 smithi022 ceph-mon[39525]: from='mgr.? 172.21.15.64:0/3553284809' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-21T09:34:43.198 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:43 smithi022 ceph-mon[39525]: from='mgr.? 172.21.15.64:0/3553284809' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:34:43.198 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:43 smithi022 ceph-mon[39525]: from='mgr.? 172.21.15.64:0/3553284809' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-21T09:34:43.199 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:43 smithi022 ceph-mon[39525]: from='mgr.? 172.21.15.64:0/3553284809' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:34:43.199 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:43 smithi022 ceph-mon[39525]: Active manager daemon y restarted 2024-09-21T09:34:43.199 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:43 smithi022 ceph-mon[39525]: Activating manager daemon y 2024-09-21T09:34:43.199 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:43 smithi022 ceph-mon[39525]: osdmap e269: 8 total, 8 up, 8 in 2024-09-21T09:34:43.199 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:43 smithi022 ceph-mon[39525]: mgrmap e36: y(active, starting, since 0.00664555s), standbys: x 2024-09-21T09:34:43.199 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:43 smithi022 ceph-mon[39525]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-09-21T09:34:43.199 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:43 smithi022 ceph-mon[39525]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-09-21T09:34:43.199 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:43 smithi022 ceph-mon[39525]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-09-21T09:34:43.199 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:43 smithi022 ceph-mon[39525]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "a"} : dispatch 2024-09-21T09:34:43.199 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:43 smithi022 ceph-mon[39525]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "b"} : dispatch 2024-09-21T09:34:43.199 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:43 smithi022 ceph-mon[39525]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "mgr metadata", "who": "y", "id": "y"} : dispatch 2024-09-21T09:34:43.199 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:43 smithi022 ceph-mon[39525]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "mgr metadata", "who": "x", "id": "x"} : dispatch 2024-09-21T09:34:43.199 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:43 smithi022 ceph-mon[39525]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-21T09:34:43.200 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:43 smithi022 ceph-mon[39525]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-21T09:34:43.200 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:43 smithi022 ceph-mon[39525]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-21T09:34:43.200 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:43 smithi022 ceph-mon[39525]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-21T09:34:43.200 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:43 smithi022 ceph-mon[39525]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-09-21T09:34:43.200 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:43 smithi022 ceph-mon[39525]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-09-21T09:34:43.200 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:43 smithi022 ceph-mon[39525]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-09-21T09:34:43.200 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:43 smithi022 ceph-mon[39525]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-09-21T09:34:43.200 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:43 smithi022 ceph-mon[39525]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "mds metadata"} : dispatch 2024-09-21T09:34:43.200 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:43 smithi022 ceph-mon[39525]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd metadata"} : dispatch 2024-09-21T09:34:43.200 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:43 smithi022 ceph-mon[39525]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "mon metadata"} : dispatch 2024-09-21T09:34:43.200 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:43 smithi022 ceph-mon[39525]: Manager daemon y is now available 2024-09-21T09:34:43.201 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:43 smithi022 ceph-mon[96736]: osdmap e268: 8 total, 8 up, 8 in 2024-09-21T09:34:43.201 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:43 smithi022 ceph-mon[96736]: Standby manager daemon x restarted 2024-09-21T09:34:43.201 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:43 smithi022 ceph-mon[96736]: Standby manager daemon x started 2024-09-21T09:34:43.201 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:43 smithi022 ceph-mon[96736]: from='mgr.? 172.21.15.64:0/3553284809' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-21T09:34:43.201 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:43 smithi022 ceph-mon[96736]: from='mgr.? 172.21.15.64:0/3553284809' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:34:43.201 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:43 smithi022 ceph-mon[96736]: from='mgr.? 172.21.15.64:0/3553284809' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-21T09:34:43.201 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:43 smithi022 ceph-mon[96736]: from='mgr.? 172.21.15.64:0/3553284809' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:34:43.201 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:43 smithi022 ceph-mon[96736]: Active manager daemon y restarted 2024-09-21T09:34:43.201 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:43 smithi022 ceph-mon[96736]: Activating manager daemon y 2024-09-21T09:34:43.201 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:43 smithi022 ceph-mon[96736]: osdmap e269: 8 total, 8 up, 8 in 2024-09-21T09:34:43.201 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:43 smithi022 ceph-mon[96736]: mgrmap e36: y(active, starting, since 0.00664555s), standbys: x 2024-09-21T09:34:43.201 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:43 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-09-21T09:34:43.201 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:43 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-09-21T09:34:43.201 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:43 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-09-21T09:34:43.201 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:43 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "a"} : dispatch 2024-09-21T09:34:43.201 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:43 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "b"} : dispatch 2024-09-21T09:34:43.202 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:43 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "mgr metadata", "who": "y", "id": "y"} : dispatch 2024-09-21T09:34:43.202 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:43 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "mgr metadata", "who": "x", "id": "x"} : dispatch 2024-09-21T09:34:43.202 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:43 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-21T09:34:43.202 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:43 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-21T09:34:43.202 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:43 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-21T09:34:43.202 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:43 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-21T09:34:43.202 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:43 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-09-21T09:34:43.202 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:43 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-09-21T09:34:43.202 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:43 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-09-21T09:34:43.202 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:43 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-09-21T09:34:43.202 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:43 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "mds metadata"} : dispatch 2024-09-21T09:34:43.202 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:43 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd metadata"} : dispatch 2024-09-21T09:34:43.203 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:43 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "mon metadata"} : dispatch 2024-09-21T09:34:43.203 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:43 smithi022 ceph-mon[96736]: Manager daemon y is now available 2024-09-21T09:34:43.203 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:43 smithi022 ceph-mon[39525]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:34:43.203 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:43 smithi022 ceph-mon[39525]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:34:43.203 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:43 smithi022 ceph-mon[39525]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-09-21T09:34:43.203 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:43 smithi022 ceph-mon[39525]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"} : dispatch 2024-09-21T09:34:43.203 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:43 smithi022 ceph-mon[39525]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"} : dispatch 2024-09-21T09:34:43.203 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:43 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:34:43.203 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:43 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:34:43.204 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:43 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-09-21T09:34:43.204 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:43 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"} : dispatch 2024-09-21T09:34:43.204 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:43 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"} : dispatch 2024-09-21T09:34:43.333 INFO:tasks.rados.rados.0.smithi022.stdout:1054: done (2 left) 2024-09-21T09:34:43.333 INFO:tasks.rados.rados.0.smithi022.stdout:1055: snap_create 2024-09-21T09:34:43.333 INFO:tasks.rados.rados.0.smithi022.stdout:1051: finishing copy_from to smithi02284960-20 2024-09-21T09:34:43.333 INFO:tasks.rados.rados.0.smithi022.stdout:1051: got expected ENOENT (src dne) 2024-09-21T09:34:43.333 INFO:tasks.rados.rados.0.smithi022.stdout:1051: finishing copy_from racing read to smithi02284960-20 2024-09-21T09:34:43.333 INFO:tasks.rados.rados.0.smithi022.stdout:1053: finishing copy_from racing read to smithi02284960-45 2024-09-21T09:34:43.333 INFO:tasks.rados.rados.0.smithi022.stdout:1053: finishing copy_from to smithi02284960-45 2024-09-21T09:34:43.333 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 45 v 644 (ObjNum 218 snap 63 seq_num 218) dirty exists 2024-09-21T09:34:43.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:43 smithi064 ceph-mon[31124]: osdmap e268: 8 total, 8 up, 8 in 2024-09-21T09:34:43.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:43 smithi064 ceph-mon[31124]: Standby manager daemon x restarted 2024-09-21T09:34:43.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:43 smithi064 ceph-mon[31124]: Standby manager daemon x started 2024-09-21T09:34:43.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:43 smithi064 ceph-mon[31124]: from='mgr.? 172.21.15.64:0/3553284809' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-21T09:34:43.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:43 smithi064 ceph-mon[31124]: from='mgr.? 172.21.15.64:0/3553284809' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:34:43.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:43 smithi064 ceph-mon[31124]: from='mgr.? 172.21.15.64:0/3553284809' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-21T09:34:43.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:43 smithi064 ceph-mon[31124]: from='mgr.? 172.21.15.64:0/3553284809' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:34:43.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:43 smithi064 ceph-mon[31124]: Active manager daemon y restarted 2024-09-21T09:34:43.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:43 smithi064 ceph-mon[31124]: Activating manager daemon y 2024-09-21T09:34:43.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:43 smithi064 ceph-mon[31124]: osdmap e269: 8 total, 8 up, 8 in 2024-09-21T09:34:43.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:43 smithi064 ceph-mon[31124]: mgrmap e36: y(active, starting, since 0.00664555s), standbys: x 2024-09-21T09:34:43.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:43 smithi064 ceph-mon[31124]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-09-21T09:34:43.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:43 smithi064 ceph-mon[31124]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-09-21T09:34:43.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:43 smithi064 ceph-mon[31124]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-09-21T09:34:43.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:43 smithi064 ceph-mon[31124]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "a"} : dispatch 2024-09-21T09:34:43.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:43 smithi064 ceph-mon[31124]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "b"} : dispatch 2024-09-21T09:34:43.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:43 smithi064 ceph-mon[31124]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "mgr metadata", "who": "y", "id": "y"} : dispatch 2024-09-21T09:34:43.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:43 smithi064 ceph-mon[31124]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "mgr metadata", "who": "x", "id": "x"} : dispatch 2024-09-21T09:34:43.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:43 smithi064 ceph-mon[31124]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-21T09:34:43.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:43 smithi064 ceph-mon[31124]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-21T09:34:43.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:43 smithi064 ceph-mon[31124]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-21T09:34:43.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:43 smithi064 ceph-mon[31124]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-21T09:34:43.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:43 smithi064 ceph-mon[31124]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-09-21T09:34:43.444 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:43 smithi064 ceph-mon[31124]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-09-21T09:34:43.444 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:43 smithi064 ceph-mon[31124]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-09-21T09:34:43.444 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:43 smithi064 ceph-mon[31124]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-09-21T09:34:43.444 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:43 smithi064 ceph-mon[31124]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "mds metadata"} : dispatch 2024-09-21T09:34:43.444 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:43 smithi064 ceph-mon[31124]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd metadata"} : dispatch 2024-09-21T09:34:43.444 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:43 smithi064 ceph-mon[31124]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "mon metadata"} : dispatch 2024-09-21T09:34:43.444 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:43 smithi064 ceph-mon[31124]: Manager daemon y is now available 2024-09-21T09:34:43.444 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:43 smithi064 ceph-mon[31124]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:34:43.444 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:43 smithi064 ceph-mon[31124]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:34:43.444 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:43 smithi064 ceph-mon[31124]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-09-21T09:34:43.444 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:43 smithi064 ceph-mon[31124]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"} : dispatch 2024-09-21T09:34:43.444 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:43 smithi064 ceph-mon[31124]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"} : dispatch 2024-09-21T09:34:43.558 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:34:43 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: 2024-09-21T09:34:43.337+0000 7feec1b50640 -1 mgr.server handle_report got status from non-daemon mon.a 2024-09-21T09:34:43.558 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:34:43 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: [21/Sep/2024:09:34:43] ENGINE Bus STARTING 2024-09-21T09:34:43.558 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:34:43 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: CherryPy Checker: 2024-09-21T09:34:43.558 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:34:43 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: The Application mounted at '' has an empty config. 2024-09-21T09:34:43.558 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:34:43 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: 2024-09-21T09:34:43.558 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:34:43 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: [21/Sep/2024:09:34:43] ENGINE Serving on http://:::9283 2024-09-21T09:34:43.558 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:34:43 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: [21/Sep/2024:09:34:43] ENGINE Bus STARTED 2024-09-21T09:34:44.343 INFO:tasks.rados.rados.0.smithi022.stdout:1051: done (2 left) 2024-09-21T09:34:44.343 INFO:tasks.rados.rados.0.smithi022.stdout:1053: done (1 left) 2024-09-21T09:34:44.343 INFO:tasks.rados.rados.0.smithi022.stdout:1055: done (0 left) 2024-09-21T09:34:44.343 INFO:tasks.rados.rados.0.smithi022.stdout:1056: snap_remove snap 92 2024-09-21T09:34:44.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:44 smithi064 ceph-mon[31124]: mgrmap e37: y(active, since 1.01264s), standbys: x 2024-09-21T09:34:44.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:44 smithi064 ceph-mon[31124]: osdmap e270: 8 total, 8 up, 8 in 2024-09-21T09:34:44.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:44 smithi064 ceph-mon[31124]: [21/Sep/2024:09:34:44] ENGINE Bus STARTING 2024-09-21T09:34:44.442 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:34:44 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:34:44.148Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:34:44.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:44 smithi022 ceph-mon[39525]: mgrmap e37: y(active, since 1.01264s), standbys: x 2024-09-21T09:34:44.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:44 smithi022 ceph-mon[39525]: osdmap e270: 8 total, 8 up, 8 in 2024-09-21T09:34:44.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:44 smithi022 ceph-mon[39525]: [21/Sep/2024:09:34:44] ENGINE Bus STARTING 2024-09-21T09:34:44.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:44 smithi022 ceph-mon[96736]: mgrmap e37: y(active, since 1.01264s), standbys: x 2024-09-21T09:34:44.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:44 smithi022 ceph-mon[96736]: osdmap e270: 8 total, 8 up, 8 in 2024-09-21T09:34:44.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:44 smithi022 ceph-mon[96736]: [21/Sep/2024:09:34:44] ENGINE Bus STARTING 2024-09-21T09:34:44.558 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:34:44 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: 2024-09-21T09:34:44.133+0000 7fee75630640 -1 log_channel(cephadm) log [ERR] : [21/Sep/2024:09:34:44] ENGINE Error in HTTPServer.serve 2024-09-21T09:34:44.559 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:34:44 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: Traceback (most recent call last): 2024-09-21T09:34:44.559 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:34:44 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T09:34:44.559 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:34:44 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: self._connections.run(self.expiration_interval) 2024-09-21T09:34:44.559 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:34:44 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T09:34:44.559 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:34:44 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: self._run(expiration_interval) 2024-09-21T09:34:44.559 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:34:44 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T09:34:44.559 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:34:44 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T09:34:44.559 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:34:44 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T09:34:44.559 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:34:44 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T09:34:44.559 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:34:44 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T09:34:44.559 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:34:44 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: s = self.context.wrap_socket( 2024-09-21T09:34:44.559 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:34:44 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T09:34:44.559 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:34:44 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: return self.sslsocket_class._create( 2024-09-21T09:34:44.559 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:34:44 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T09:34:44.560 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:34:44 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: self.do_handshake() 2024-09-21T09:34:44.560 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:34:44 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T09:34:44.560 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:34:44 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: self._sslobj.do_handshake() 2024-09-21T09:34:44.560 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:34:44 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T09:34:44.560 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:34:44 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: 2024-09-21T09:34:45.345 INFO:tasks.rados.rados.0.smithi022.stdout:1056: done (0 left) 2024-09-21T09:34:45.345 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 25 current snap is 107 2024-09-21T09:34:45.345 INFO:tasks.rados.rados.0.smithi022.stdout:1057: seq_num 345 ranges {0=581632,581632=401408} 2024-09-21T09:34:45.348 INFO:tasks.rados.rados.0.smithi022.stdout:1057: writing smithi02284960-25 from 0 to 581632 tid 1 2024-09-21T09:34:45.352 INFO:tasks.rados.rados.0.smithi022.stdout:1057: writing smithi02284960-25 from 581632 to 983040 tid 2 2024-09-21T09:34:45.353 INFO:tasks.rados.rados.0.smithi022.stdout:1058: copy_from oid 1 from oid 10 current snap is 107 2024-09-21T09:34:45.353 INFO:tasks.rados.rados.0.smithi022.stdout:1059: read oid 46 snap -1 2024-09-21T09:34:45.353 INFO:tasks.rados.rados.0.smithi022.stdout:1059: expect (ObjNum 204 snap 56 seq_num 204) 2024-09-21T09:34:45.353 INFO:tasks.rados.rados.0.smithi022.stdout:1060: rmattr oid 50 current snap is 107 2024-09-21T09:34:45.353 INFO:tasks.rados.rados.0.smithi022.stdout:1060: done (3 left) 2024-09-21T09:34:45.353 INFO:tasks.rados.rados.0.smithi022.stdout:1061: snap_create 2024-09-21T09:34:45.357 INFO:tasks.rados.rados.0.smithi022.stdout:1057: finishing write tid 1 to smithi02284960-25 2024-09-21T09:34:45.357 INFO:tasks.rados.rados.0.smithi022.stdout:1057: finishing write tid 2 to smithi02284960-25 2024-09-21T09:34:45.361 INFO:tasks.rados.rados.0.smithi022.stdout:1057: finishing write tid 3 to smithi02284960-25 2024-09-21T09:34:45.363 INFO:tasks.rados.rados.0.smithi022.stdout:1057: finishing write tid 4 to smithi02284960-25 2024-09-21T09:34:45.363 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 25 v 579 (ObjNum 345 snap 107 seq_num 345) dirty exists 2024-09-21T09:34:45.363 INFO:tasks.rados.rados.0.smithi022.stdout:1057: left oid 25 (ObjNum 345 snap 107 seq_num 345) 2024-09-21T09:34:45.366 INFO:tasks.rados.rados.0.smithi022.stdout:1059: expect (ObjNum 204 snap 56 seq_num 204) 2024-09-21T09:34:45.442 INFO:tasks.rados.rados.0.smithi022.stdout:1058: finishing copy_from racing read to smithi02284960-1 2024-09-21T09:34:45.442 INFO:tasks.rados.rados.0.smithi022.stdout:1058: finishing copy_from to smithi02284960-1 2024-09-21T09:34:45.442 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 1 v 535 (ObjNum 311 snap 94 seq_num 311) dirty exists 2024-09-21T09:34:45.491 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:45 smithi064 ceph-mon[31124]: [21/Sep/2024:09:34:44] ENGINE Serving on https://172.21.15.22:7150 2024-09-21T09:34:45.491 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:45 smithi064 ceph-mon[31124]: [21/Sep/2024:09:34:44] ENGINE Error in HTTPServer.serve 2024-09-21T09:34:45.491 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-09-21T09:34:45.491 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T09:34:45.491 INFO:journalctl@ceph.mon.b.smithi064.stdout: self._connections.run(self.expiration_interval) 2024-09-21T09:34:45.492 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T09:34:45.492 INFO:journalctl@ceph.mon.b.smithi064.stdout: self._run(expiration_interval) 2024-09-21T09:34:45.492 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T09:34:45.492 INFO:journalctl@ceph.mon.b.smithi064.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T09:34:45.492 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T09:34:45.492 INFO:journalctl@ceph.mon.b.smithi064.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T09:34:45.492 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T09:34:45.492 INFO:journalctl@ceph.mon.b.smithi064.stdout: s = self.context.wrap_socket( 2024-09-21T09:34:45.492 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T09:34:45.492 INFO:journalctl@ceph.mon.b.smithi064.stdout: return self.sslsocket_class._create( 2024-09-21T09:34:45.492 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T09:34:45.492 INFO:journalctl@ceph.mon.b.smithi064.stdout: self.do_handshake() 2024-09-21T09:34:45.492 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T09:34:45.492 INFO:journalctl@ceph.mon.b.smithi064.stdout: self._sslobj.do_handshake() 2024-09-21T09:34:45.493 INFO:journalctl@ceph.mon.b.smithi064.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T09:34:45.493 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:45 smithi064 ceph-mon[31124]: [21/Sep/2024:09:34:44] ENGINE Serving on http://172.21.15.22:8765 2024-09-21T09:34:45.493 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:45 smithi064 ceph-mon[31124]: [21/Sep/2024:09:34:44] ENGINE Bus STARTED 2024-09-21T09:34:45.493 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:45 smithi064 ceph-mon[31124]: pgmap v5: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 239 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:34:45.493 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:45 smithi064 ceph-mon[31124]: mgrmap e38: y(active, since 2s), standbys: x 2024-09-21T09:34:45.493 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:45 smithi064 ceph-mon[31124]: osdmap e271: 8 total, 8 up, 8 in 2024-09-21T09:34:45.682 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:45 smithi022 ceph-mon[39525]: [21/Sep/2024:09:34:44] ENGINE Serving on https://172.21.15.22:7150 2024-09-21T09:34:45.682 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:45 smithi022 ceph-mon[39525]: [21/Sep/2024:09:34:44] ENGINE Error in HTTPServer.serve 2024-09-21T09:34:45.683 INFO:journalctl@ceph.mon.c.smithi022.stdout: Traceback (most recent call last): 2024-09-21T09:34:45.683 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T09:34:45.683 INFO:journalctl@ceph.mon.c.smithi022.stdout: self._connections.run(self.expiration_interval) 2024-09-21T09:34:45.683 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T09:34:45.683 INFO:journalctl@ceph.mon.c.smithi022.stdout: self._run(expiration_interval) 2024-09-21T09:34:45.683 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T09:34:45.683 INFO:journalctl@ceph.mon.c.smithi022.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T09:34:45.683 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T09:34:45.683 INFO:journalctl@ceph.mon.c.smithi022.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T09:34:45.683 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T09:34:45.683 INFO:journalctl@ceph.mon.c.smithi022.stdout: s = self.context.wrap_socket( 2024-09-21T09:34:45.683 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T09:34:45.683 INFO:journalctl@ceph.mon.c.smithi022.stdout: return self.sslsocket_class._create( 2024-09-21T09:34:45.683 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T09:34:45.684 INFO:journalctl@ceph.mon.c.smithi022.stdout: self.do_handshake() 2024-09-21T09:34:45.684 INFO:journalctl@ceph.mon.c.smithi022.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T09:34:45.684 INFO:journalctl@ceph.mon.c.smithi022.stdout: self._sslobj.do_handshake() 2024-09-21T09:34:45.684 INFO:journalctl@ceph.mon.c.smithi022.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T09:34:45.684 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:45 smithi022 ceph-mon[39525]: [21/Sep/2024:09:34:44] ENGINE Serving on http://172.21.15.22:8765 2024-09-21T09:34:45.684 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:45 smithi022 ceph-mon[39525]: [21/Sep/2024:09:34:44] ENGINE Bus STARTED 2024-09-21T09:34:45.684 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:45 smithi022 ceph-mon[39525]: pgmap v5: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 239 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:34:45.684 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:45 smithi022 ceph-mon[39525]: mgrmap e38: y(active, since 2s), standbys: x 2024-09-21T09:34:45.684 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:45 smithi022 ceph-mon[39525]: osdmap e271: 8 total, 8 up, 8 in 2024-09-21T09:34:45.684 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:45 smithi022 ceph-mon[96736]: [21/Sep/2024:09:34:44] ENGINE Serving on https://172.21.15.22:7150 2024-09-21T09:34:45.684 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:45 smithi022 ceph-mon[96736]: [21/Sep/2024:09:34:44] ENGINE Error in HTTPServer.serve 2024-09-21T09:34:45.685 INFO:journalctl@ceph.mon.a.smithi022.stdout: Traceback (most recent call last): 2024-09-21T09:34:45.685 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T09:34:45.685 INFO:journalctl@ceph.mon.a.smithi022.stdout: self._connections.run(self.expiration_interval) 2024-09-21T09:34:45.685 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T09:34:45.685 INFO:journalctl@ceph.mon.a.smithi022.stdout: self._run(expiration_interval) 2024-09-21T09:34:45.685 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T09:34:45.685 INFO:journalctl@ceph.mon.a.smithi022.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T09:34:45.685 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T09:34:45.685 INFO:journalctl@ceph.mon.a.smithi022.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T09:34:45.685 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T09:34:45.685 INFO:journalctl@ceph.mon.a.smithi022.stdout: s = self.context.wrap_socket( 2024-09-21T09:34:45.685 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T09:34:45.685 INFO:journalctl@ceph.mon.a.smithi022.stdout: return self.sslsocket_class._create( 2024-09-21T09:34:45.685 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T09:34:45.685 INFO:journalctl@ceph.mon.a.smithi022.stdout: self.do_handshake() 2024-09-21T09:34:45.686 INFO:journalctl@ceph.mon.a.smithi022.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T09:34:45.686 INFO:journalctl@ceph.mon.a.smithi022.stdout: self._sslobj.do_handshake() 2024-09-21T09:34:45.686 INFO:journalctl@ceph.mon.a.smithi022.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T09:34:45.686 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:45 smithi022 ceph-mon[96736]: [21/Sep/2024:09:34:44] ENGINE Serving on http://172.21.15.22:8765 2024-09-21T09:34:45.686 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:45 smithi022 ceph-mon[96736]: [21/Sep/2024:09:34:44] ENGINE Bus STARTED 2024-09-21T09:34:45.686 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:45 smithi022 ceph-mon[96736]: pgmap v5: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 239 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:34:45.686 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:45 smithi022 ceph-mon[96736]: mgrmap e38: y(active, since 2s), standbys: x 2024-09-21T09:34:45.686 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:45 smithi022 ceph-mon[96736]: osdmap e271: 8 total, 8 up, 8 in 2024-09-21T09:34:46.058 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:34:45 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:34:45] "GET /metrics HTTP/1.1" 200 33376 "" "Prometheus/2.51.0" 2024-09-21T09:34:46.191 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:34:45 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:34:45.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:34:46.191 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:34:45 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:34:45.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:34:46.355 INFO:tasks.rados.rados.0.smithi022.stdout:1057: done (3 left) 2024-09-21T09:34:46.355 INFO:tasks.rados.rados.0.smithi022.stdout:1058: done (2 left) 2024-09-21T09:34:46.355 INFO:tasks.rados.rados.0.smithi022.stdout:1059: done (1 left) 2024-09-21T09:34:46.356 INFO:tasks.rados.rados.0.smithi022.stdout:1061: done (0 left) 2024-09-21T09:34:46.356 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 39 current snap is 108 2024-09-21T09:34:46.356 INFO:tasks.rados.rados.0.smithi022.stdout:1062: seq_num 346 ranges {434176=598016,1032192=401408} 2024-09-21T09:34:46.361 INFO:tasks.rados.rados.0.smithi022.stdout:1062: writing smithi02284960-39 from 434176 to 1032192 tid 1 2024-09-21T09:34:46.364 INFO:tasks.rados.rados.0.smithi022.stdout:1062: writing smithi02284960-39 from 1032192 to 1433600 tid 2 2024-09-21T09:34:46.364 INFO:tasks.rados.rados.0.smithi022.stdout:1063: read oid 25 snap 56 2024-09-21T09:34:46.364 INFO:tasks.rados.rados.0.smithi022.stdout:1063: expect deleted 2024-09-21T09:34:46.364 INFO:tasks.rados.rados.0.smithi022.stdout:1064: snap_create 2024-09-21T09:34:46.367 INFO:tasks.rados.rados.0.smithi022.stdout:1062: finishing write tid 1 to smithi02284960-39 2024-09-21T09:34:46.367 INFO:tasks.rados.rados.0.smithi022.stdout:1062: finishing write tid 2 to smithi02284960-39 2024-09-21T09:34:46.367 INFO:tasks.rados.rados.0.smithi022.stdout:1062: finishing write tid 3 to smithi02284960-39 2024-09-21T09:34:46.368 INFO:tasks.rados.rados.0.smithi022.stdout:1062: finishing write tid 4 to smithi02284960-39 2024-09-21T09:34:46.368 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 39 v 460 (ObjNum 346 snap 108 seq_num 346) dirty exists 2024-09-21T09:34:46.368 INFO:tasks.rados.rados.0.smithi022.stdout:1062: left oid 39 (ObjNum 346 snap 108 seq_num 346) 2024-09-21T09:34:46.664 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:46 smithi022 ceph-mon[39525]: osdmap e272: 8 total, 8 up, 8 in 2024-09-21T09:34:46.664 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:46 smithi022 ceph-mon[96736]: osdmap e272: 8 total, 8 up, 8 in 2024-09-21T09:34:46.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:46 smithi064 ceph-mon[31124]: osdmap e272: 8 total, 8 up, 8 in 2024-09-21T09:34:47.257 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:34:46 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:34:46.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi022\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:34:47.352 INFO:tasks.rados.rados.0.smithi022.stdout:1062: done (2 left) 2024-09-21T09:34:47.352 INFO:tasks.rados.rados.0.smithi022.stdout:1063: done (1 left) 2024-09-21T09:34:47.352 INFO:tasks.rados.rados.0.smithi022.stdout:1064: done (0 left) 2024-09-21T09:34:47.352 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 6 current snap is 109 2024-09-21T09:34:47.352 INFO:tasks.rados.rados.0.smithi022.stdout:1065: seq_num 347 ranges {0=614400,614400=401408} 2024-09-21T09:34:47.355 INFO:tasks.rados.rados.0.smithi022.stdout:1065: writing smithi02284960-6 from 0 to 614400 tid 1 2024-09-21T09:34:47.358 INFO:tasks.rados.rados.0.smithi022.stdout:1065: writing smithi02284960-6 from 614400 to 1015808 tid 2 2024-09-21T09:34:47.358 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 44 current snap is 109 2024-09-21T09:34:47.358 INFO:tasks.rados.rados.0.smithi022.stdout:1066: seq_num 348 ranges {1638400=630784,2269184=401408} 2024-09-21T09:34:47.370 INFO:tasks.rados.rados.0.smithi022.stdout:1066: writing smithi02284960-44 from 1638400 to 2269184 tid 1 2024-09-21T09:34:47.373 INFO:tasks.rados.rados.0.smithi022.stdout:1066: writing smithi02284960-44 from 2269184 to 2670592 tid 2 2024-09-21T09:34:47.373 INFO:tasks.rados.rados.0.smithi022.stdout:1067: read oid 50 snap -1 2024-09-21T09:34:47.373 INFO:tasks.rados.rados.0.smithi022.stdout:1067: expect deleted 2024-09-21T09:34:47.373 INFO:tasks.rados.rados.0.smithi022.stdout:1068: snap_create 2024-09-21T09:34:47.373 INFO:tasks.rados.rados.0.smithi022.stdout:1065: finishing write tid 1 to smithi02284960-6 2024-09-21T09:34:47.373 INFO:tasks.rados.rados.0.smithi022.stdout:1065: finishing write tid 2 to smithi02284960-6 2024-09-21T09:34:47.373 INFO:tasks.rados.rados.0.smithi022.stdout:1065: finishing write tid 3 to smithi02284960-6 2024-09-21T09:34:47.373 INFO:tasks.rados.rados.0.smithi022.stdout:1065: finishing write tid 4 to smithi02284960-6 2024-09-21T09:34:47.374 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 6 v 646 (ObjNum 347 snap 109 seq_num 347) dirty exists 2024-09-21T09:34:47.374 INFO:tasks.rados.rados.0.smithi022.stdout:1065: left oid 6 (ObjNum 347 snap 109 seq_num 347) 2024-09-21T09:34:47.375 INFO:tasks.rados.rados.0.smithi022.stdout:1066: finishing write tid 1 to smithi02284960-44 2024-09-21T09:34:47.376 INFO:tasks.rados.rados.0.smithi022.stdout:1066: finishing write tid 2 to smithi02284960-44 2024-09-21T09:34:47.376 INFO:tasks.rados.rados.0.smithi022.stdout:1066: finishing write tid 3 to smithi02284960-44 2024-09-21T09:34:47.376 INFO:tasks.rados.rados.0.smithi022.stdout:1066: finishing write tid 4 to smithi02284960-44 2024-09-21T09:34:47.376 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 44 v 581 (ObjNum 348 snap 109 seq_num 348) dirty exists 2024-09-21T09:34:47.377 INFO:tasks.rados.rados.0.smithi022.stdout:1066: left oid 44 (ObjNum 348 snap 109 seq_num 348) 2024-09-21T09:34:47.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:47 smithi064 ceph-mon[31124]: pgmap v8: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 239 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:34:47.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:47 smithi064 ceph-mon[31124]: mgrmap e39: y(active, since 4s), standbys: x 2024-09-21T09:34:47.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:47 smithi064 ceph-mon[31124]: osdmap e273: 8 total, 8 up, 8 in 2024-09-21T09:34:47.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:47 smithi064 ceph-mon[31124]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:34:47.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:47 smithi064 ceph-mon[31124]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:34:47.807 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:47 smithi022 ceph-mon[39525]: pgmap v8: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 239 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:34:47.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:47 smithi022 ceph-mon[39525]: mgrmap e39: y(active, since 4s), standbys: x 2024-09-21T09:34:47.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:47 smithi022 ceph-mon[39525]: osdmap e273: 8 total, 8 up, 8 in 2024-09-21T09:34:47.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:47 smithi022 ceph-mon[39525]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:34:47.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:47 smithi022 ceph-mon[39525]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:34:47.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:47 smithi022 ceph-mon[96736]: pgmap v8: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 239 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:34:47.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:47 smithi022 ceph-mon[96736]: mgrmap e39: y(active, since 4s), standbys: x 2024-09-21T09:34:47.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:47 smithi022 ceph-mon[96736]: osdmap e273: 8 total, 8 up, 8 in 2024-09-21T09:34:47.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:47 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:34:47.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:47 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:34:48.360 INFO:tasks.rados.rados.0.smithi022.stdout:1065: done (3 left) 2024-09-21T09:34:48.360 INFO:tasks.rados.rados.0.smithi022.stdout:1066: done (2 left) 2024-09-21T09:34:48.361 INFO:tasks.rados.rados.0.smithi022.stdout:1067: done (1 left) 2024-09-21T09:34:48.361 INFO:tasks.rados.rados.0.smithi022.stdout:1068: done (0 left) 2024-09-21T09:34:48.361 INFO:tasks.rados.rados.0.smithi022.stdout:1069: snap_create 2024-09-21T09:34:48.671 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:48 smithi064 ceph-mon[31124]: osdmap e274: 8 total, 8 up, 8 in 2024-09-21T09:34:48.671 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:48 smithi064 ceph-mon[31124]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:34:48.671 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:48 smithi064 ceph-mon[31124]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:34:48.671 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:48 smithi064 ceph-mon[31124]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:34:48.671 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:48 smithi064 ceph-mon[31124]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:34:48.671 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:48 smithi064 ceph-mon[31124]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:34:48.671 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:48 smithi064 ceph-mon[31124]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:34:48.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:48 smithi022 ceph-mon[39525]: osdmap e274: 8 total, 8 up, 8 in 2024-09-21T09:34:48.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:48 smithi022 ceph-mon[39525]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:34:48.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:48 smithi022 ceph-mon[39525]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:34:48.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:48 smithi022 ceph-mon[39525]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:34:48.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:48 smithi022 ceph-mon[39525]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:34:48.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:48 smithi022 ceph-mon[39525]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:34:48.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:48 smithi022 ceph-mon[39525]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:34:48.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:48 smithi022 ceph-mon[96736]: osdmap e274: 8 total, 8 up, 8 in 2024-09-21T09:34:48.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:48 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:34:48.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:48 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:34:48.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:48 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:34:48.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:48 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:34:48.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:48 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:34:48.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:48 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:34:48.936 INFO:tasks.rados.rados.0.smithi022.stdout:1069: done (0 left) 2024-09-21T09:34:48.936 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 5 current snap is 111 2024-09-21T09:34:48.936 INFO:tasks.rados.rados.0.smithi022.stdout:1070: seq_num 349 ranges {0=655360,655360=401408} 2024-09-21T09:34:48.940 INFO:tasks.rados.rados.0.smithi022.stdout:1070: writing smithi02284960-5 from 0 to 655360 tid 1 2024-09-21T09:34:48.942 INFO:tasks.rados.rados.0.smithi022.stdout:1070: writing smithi02284960-5 from 655360 to 1056768 tid 2 2024-09-21T09:34:48.942 INFO:tasks.rados.rados.0.smithi022.stdout:1071: read oid 22 snap -1 2024-09-21T09:34:48.943 INFO:tasks.rados.rados.0.smithi022.stdout:1071: expect (ObjNum 343 snap 104 seq_num 343) 2024-09-21T09:34:48.943 INFO:tasks.rados.rados.0.smithi022.stdout:1072: rollback oid 47 current snap is 111 2024-09-21T09:34:48.943 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 47 to 96 2024-09-21T09:34:48.943 INFO:tasks.rados.rados.0.smithi022.stdout:1073: delete oid 15 current snap is 111 2024-09-21T09:34:48.948 INFO:tasks.rados.rados.0.smithi022.stdout:1072: finishing rollback tid 0 to smithi02284960-47 2024-09-21T09:34:48.949 INFO:tasks.rados.rados.0.smithi022.stdout:1072: finishing rollback tid 1 to smithi02284960-47 2024-09-21T09:34:48.950 INFO:tasks.rados.rados.0.smithi022.stdout:1070: finishing write tid 1 to smithi02284960-5 2024-09-21T09:34:48.950 INFO:tasks.rados.rados.0.smithi022.stdout:1072: finishing rollback tid 2 to smithi02284960-47 2024-09-21T09:34:48.950 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 47 v 585 (ObjNum 289 snap 0 seq_num 1) dirty exists 2024-09-21T09:34:48.950 INFO:tasks.rados.rados.0.smithi022.stdout:1070: finishing write tid 2 to smithi02284960-5 2024-09-21T09:34:48.950 INFO:tasks.rados.rados.0.smithi022.stdout:1070: finishing write tid 3 to smithi02284960-5 2024-09-21T09:34:48.950 INFO:tasks.rados.rados.0.smithi022.stdout:1072: done (3 left) 2024-09-21T09:34:48.950 INFO:tasks.rados.rados.0.smithi022.stdout:1073: done (2 left) 2024-09-21T09:34:48.950 INFO:tasks.rados.rados.0.smithi022.stdout:1074: rollback oid 17 current snap is 111 2024-09-21T09:34:48.951 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 17 to 75 2024-09-21T09:34:48.951 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 49 current snap is 111 2024-09-21T09:34:48.951 INFO:tasks.rados.rados.0.smithi022.stdout:1075: seq_num 351 ranges {1081344=688128,1769472=401408} 2024-09-21T09:34:48.960 INFO:tasks.rados.rados.0.smithi022.stdout:1075: writing smithi02284960-49 from 1081344 to 1769472 tid 1 2024-09-21T09:34:48.962 INFO:tasks.rados.rados.0.smithi022.stdout:1075: writing smithi02284960-49 from 1769472 to 2170880 tid 2 2024-09-21T09:34:48.962 INFO:tasks.rados.rados.0.smithi022.stdout:1076: delete oid 37 current snap is 111 2024-09-21T09:34:48.962 INFO:tasks.rados.rados.0.smithi022.stdout:1070: finishing write tid 4 to smithi02284960-5 2024-09-21T09:34:48.962 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 5 v 461 (ObjNum 349 snap 111 seq_num 349) dirty exists 2024-09-21T09:34:48.963 INFO:tasks.rados.rados.0.smithi022.stdout:1070: left oid 5 (ObjNum 349 snap 111 seq_num 349) 2024-09-21T09:34:48.963 INFO:tasks.rados.rados.0.smithi022.stdout:1071: expect (ObjNum 343 snap 104 seq_num 343) 2024-09-21T09:34:48.988 INFO:tasks.rados.rados.0.smithi022.stdout:1074: finishing rollback tid 0 to smithi02284960-17 2024-09-21T09:34:48.988 INFO:tasks.rados.rados.0.smithi022.stdout:1074: finishing rollback tid 1 to smithi02284960-17 2024-09-21T09:34:48.988 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 17 v 462 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-09-21T09:34:48.989 INFO:tasks.rados.rados.0.smithi022.stdout:1075: finishing write tid 1 to smithi02284960-49 2024-09-21T09:34:48.989 INFO:tasks.rados.rados.0.smithi022.stdout:1075: finishing write tid 2 to smithi02284960-49 2024-09-21T09:34:48.989 INFO:tasks.rados.rados.0.smithi022.stdout:1075: finishing write tid 3 to smithi02284960-49 2024-09-21T09:34:48.989 INFO:tasks.rados.rados.0.smithi022.stdout:1070: done (4 left) 2024-09-21T09:34:48.989 INFO:tasks.rados.rados.0.smithi022.stdout:1071: done (3 left) 2024-09-21T09:34:48.989 INFO:tasks.rados.rados.0.smithi022.stdout:1074: done (2 left) 2024-09-21T09:34:48.989 INFO:tasks.rados.rados.0.smithi022.stdout:1076: done (1 left) 2024-09-21T09:34:48.989 INFO:tasks.rados.rados.0.smithi022.stdout:1077: rollback oid 46 current snap is 111 2024-09-21T09:34:48.989 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 46 to 69 2024-09-21T09:34:48.989 INFO:tasks.rados.rados.0.smithi022.stdout:1075: finishing write tid 4 to smithi02284960-49 2024-09-21T09:34:48.989 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 49 v 265 (ObjNum 351 snap 111 seq_num 351) dirty exists 2024-09-21T09:34:48.989 INFO:tasks.rados.rados.0.smithi022.stdout:1075: left oid 49 (ObjNum 351 snap 111 seq_num 351) 2024-09-21T09:34:48.989 INFO:tasks.rados.rados.0.smithi022.stdout:1075: done (1 left) 2024-09-21T09:34:48.989 INFO:tasks.rados.rados.0.smithi022.stdout:1078: read oid 7 snap -1 2024-09-21T09:34:48.990 INFO:tasks.rados.rados.0.smithi022.stdout:1078: expect deleted 2024-09-21T09:34:48.990 INFO:tasks.rados.rados.0.smithi022.stdout:1079: setattr oid 49 current snap is 111 2024-09-21T09:34:48.990 INFO:tasks.rados.rados.0.smithi022.stdout:1078: done (2 left) 2024-09-21T09:34:48.990 INFO:tasks.rados.rados.0.smithi022.stdout:1080: read oid 31 snap 108 2024-09-21T09:34:48.990 INFO:tasks.rados.rados.0.smithi022.stdout:1080: expect deleted 2024-09-21T09:34:48.990 INFO:tasks.rados.rados.0.smithi022.stdout:1081: snap_create 2024-09-21T09:34:48.991 INFO:tasks.rados.rados.0.smithi022.stdout:1077: finishing rollback tid 0 to smithi02284960-46 2024-09-21T09:34:48.991 INFO:tasks.rados.rados.0.smithi022.stdout:1077: finishing rollback tid 1 to smithi02284960-46 2024-09-21T09:34:48.991 INFO:tasks.rados.rados.0.smithi022.stdout:1077: finishing rollback tid 2 to smithi02284960-46 2024-09-21T09:34:48.991 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 46 v 589 (ObjNum 187 snap 47 seq_num 187) dirty exists 2024-09-21T09:34:48.992 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 49 v 266 (ObjNum 351 snap 111 seq_num 351) dirty exists 2024-09-21T09:34:49.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:49 smithi064 ceph-mon[31124]: pgmap v11: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 239 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:34:49.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:49 smithi064 ceph-mon[31124]: osdmap e275: 8 total, 8 up, 8 in 2024-09-21T09:34:49.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:49 smithi064 ceph-mon[31124]: osdmap e276: 8 total, 8 up, 8 in 2024-09-21T09:34:49.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:49 smithi022 ceph-mon[39525]: pgmap v11: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 239 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:34:49.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:49 smithi022 ceph-mon[39525]: osdmap e275: 8 total, 8 up, 8 in 2024-09-21T09:34:49.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:49 smithi022 ceph-mon[39525]: osdmap e276: 8 total, 8 up, 8 in 2024-09-21T09:34:49.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:49 smithi022 ceph-mon[96736]: pgmap v11: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 239 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:34:49.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:49 smithi022 ceph-mon[96736]: osdmap e275: 8 total, 8 up, 8 in 2024-09-21T09:34:49.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:49 smithi022 ceph-mon[96736]: osdmap e276: 8 total, 8 up, 8 in 2024-09-21T09:34:49.938 INFO:tasks.rados.rados.0.smithi022.stdout:1077: done (3 left) 2024-09-21T09:34:49.939 INFO:tasks.rados.rados.0.smithi022.stdout:1079: done (2 left) 2024-09-21T09:34:49.939 INFO:tasks.rados.rados.0.smithi022.stdout:1080: done (1 left) 2024-09-21T09:34:49.939 INFO:tasks.rados.rados.0.smithi022.stdout:1081: done (0 left) 2024-09-21T09:34:49.939 INFO:tasks.rados.rados.0.smithi022.stdout:1082: read oid 50 snap -1 2024-09-21T09:34:49.939 INFO:tasks.rados.rados.0.smithi022.stdout:1082: expect deleted 2024-09-21T09:34:49.939 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 49 current snap is 112 2024-09-21T09:34:49.939 INFO:tasks.rados.rados.0.smithi022.stdout:1083: seq_num 353 ranges {2170880=720896,2891776=401408} 2024-09-21T09:34:49.954 INFO:tasks.rados.rados.0.smithi022.stdout:1083: writing smithi02284960-49 from 2170880 to 2891776 tid 1 2024-09-21T09:34:49.956 INFO:tasks.rados.rados.0.smithi022.stdout:1083: writing smithi02284960-49 from 2891776 to 3293184 tid 2 2024-09-21T09:34:49.956 INFO:tasks.rados.rados.0.smithi022.stdout:1084: rollback oid 32 current snap is 112 2024-09-21T09:34:49.956 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 32 to 110 2024-09-21T09:34:49.957 INFO:tasks.rados.rados.0.smithi022.stdout:1082: done (2 left) 2024-09-21T09:34:49.957 INFO:tasks.rados.rados.0.smithi022.stdout:1085: delete oid 28 current snap is 112 2024-09-21T09:34:49.961 INFO:tasks.rados.rados.0.smithi022.stdout:1084: finishing rollback tid 1 to smithi02284960-32 2024-09-21T09:34:49.961 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 32 v 621 (ObjNum 65 snap 0 seq_num 1449223769) dirty dne 2024-09-21T09:34:49.961 INFO:tasks.rados.rados.0.smithi022.stdout:1084: done (2 left) 2024-09-21T09:34:49.961 INFO:tasks.rados.rados.0.smithi022.stdout:1085: done (1 left) 2024-09-21T09:34:49.961 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 30 current snap is 112 2024-09-21T09:34:49.962 INFO:tasks.rados.rados.0.smithi022.stdout:1086: seq_num 355 ranges {933888=753664,1687552=401408} 2024-09-21T09:34:49.970 INFO:tasks.rados.rados.0.smithi022.stdout:1086: writing smithi02284960-30 from 933888 to 1687552 tid 1 2024-09-21T09:34:49.973 INFO:tasks.rados.rados.0.smithi022.stdout:1086: writing smithi02284960-30 from 1687552 to 2088960 tid 2 2024-09-21T09:34:49.973 INFO:tasks.rados.rados.0.smithi022.stdout:1087: delete oid 27 current snap is 112 2024-09-21T09:34:49.973 INFO:tasks.rados.rados.0.smithi022.stdout:1083: finishing write tid 1 to smithi02284960-49 2024-09-21T09:34:49.973 INFO:tasks.rados.rados.0.smithi022.stdout:1083: finishing write tid 2 to smithi02284960-49 2024-09-21T09:34:49.973 INFO:tasks.rados.rados.0.smithi022.stdout:1083: finishing write tid 3 to smithi02284960-49 2024-09-21T09:34:49.974 INFO:tasks.rados.rados.0.smithi022.stdout:1083: finishing write tid 4 to smithi02284960-49 2024-09-21T09:34:49.974 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 49 v 270 (ObjNum 353 snap 112 seq_num 353) dirty exists 2024-09-21T09:34:49.974 INFO:tasks.rados.rados.0.smithi022.stdout:1083: left oid 49 (ObjNum 353 snap 112 seq_num 353) 2024-09-21T09:34:49.975 INFO:tasks.rados.rados.0.smithi022.stdout:1086: finishing write tid 1 to smithi02284960-30 2024-09-21T09:34:49.975 INFO:tasks.rados.rados.0.smithi022.stdout:1086: finishing write tid 2 to smithi02284960-30 2024-09-21T09:34:49.975 INFO:tasks.rados.rados.0.smithi022.stdout:1086: finishing write tid 3 to smithi02284960-30 2024-09-21T09:34:49.977 INFO:tasks.rados.rados.0.smithi022.stdout:1083: done (2 left) 2024-09-21T09:34:49.977 INFO:tasks.rados.rados.0.smithi022.stdout:1087: done (1 left) 2024-09-21T09:34:49.977 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 3 current snap is 112 2024-09-21T09:34:49.977 INFO:tasks.rados.rados.0.smithi022.stdout:1086: finishing write tid 4 to smithi02284960-30 2024-09-21T09:34:49.977 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 30 v 465 (ObjNum 355 snap 112 seq_num 355) dirty exists 2024-09-21T09:34:49.977 INFO:tasks.rados.rados.0.smithi022.stdout:1086: left oid 30 (ObjNum 355 snap 112 seq_num 355) 2024-09-21T09:34:49.977 INFO:tasks.rados.rados.0.smithi022.stdout:1088: seq_num 357 ranges {0=786432,786432=401408} 2024-09-21T09:34:49.981 INFO:tasks.rados.rados.0.smithi022.stdout:1088: writing smithi02284960-3 from 0 to 786432 tid 1 2024-09-21T09:34:49.983 INFO:tasks.rados.rados.0.smithi022.stdout:1088: writing smithi02284960-3 from 786432 to 1187840 tid 2 2024-09-21T09:34:49.983 INFO:tasks.rados.rados.0.smithi022.stdout:1086: done (1 left) 2024-09-21T09:34:49.984 INFO:tasks.rados.rados.0.smithi022.stdout:1089: read oid 11 snap -1 2024-09-21T09:34:49.984 INFO:tasks.rados.rados.0.smithi022.stdout:1089: expect (ObjNum 336 snap 101 seq_num 336) 2024-09-21T09:34:49.984 INFO:tasks.rados.rados.0.smithi022.stdout:1090: snap_create 2024-09-21T09:34:49.985 INFO:tasks.rados.rados.0.smithi022.stdout:1088: finishing write tid 1 to smithi02284960-3 2024-09-21T09:34:49.986 INFO:tasks.rados.rados.0.smithi022.stdout:1088: finishing write tid 2 to smithi02284960-3 2024-09-21T09:34:49.986 INFO:tasks.rados.rados.0.smithi022.stdout:1088: finishing write tid 3 to smithi02284960-3 2024-09-21T09:34:49.988 INFO:tasks.rados.rados.0.smithi022.stdout:1088: finishing write tid 4 to smithi02284960-3 2024-09-21T09:34:49.988 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 3 v 134 (ObjNum 357 snap 112 seq_num 357) dirty exists 2024-09-21T09:34:49.988 INFO:tasks.rados.rados.0.smithi022.stdout:1088: left oid 3 (ObjNum 357 snap 112 seq_num 357) 2024-09-21T09:34:50.000 INFO:tasks.rados.rados.0.smithi022.stdout:1089: expect (ObjNum 336 snap 101 seq_num 336) 2024-09-21T09:34:50.441 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:34:50 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:34:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:34:50.944 INFO:tasks.rados.rados.0.smithi022.stdout:1088: done (2 left) 2024-09-21T09:34:50.944 INFO:tasks.rados.rados.0.smithi022.stdout:1089: done (1 left) 2024-09-21T09:34:50.944 INFO:tasks.rados.rados.0.smithi022.stdout:1090: done (0 left) 2024-09-21T09:34:50.944 INFO:tasks.rados.rados.0.smithi022.stdout:1091: read oid 12 snap -1 2024-09-21T09:34:50.944 INFO:tasks.rados.rados.0.smithi022.stdout:1091: expect (ObjNum 307 snap 91 seq_num 307) 2024-09-21T09:34:50.944 INFO:tasks.rados.rados.0.smithi022.stdout:1092: rollback oid 47 current snap is 113 2024-09-21T09:34:50.944 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 47 to 89 2024-09-21T09:34:50.944 INFO:tasks.rados.rados.0.smithi022.stdout:1093: copy_from oid 22 from oid 35 current snap is 113 2024-09-21T09:34:50.944 INFO:tasks.rados.rados.0.smithi022.stdout:1094: delete oid 3 current snap is 113 2024-09-21T09:34:50.950 INFO:tasks.rados.rados.0.smithi022.stdout:1091: expect (ObjNum 307 snap 91 seq_num 307) 2024-09-21T09:34:50.960 INFO:tasks.rados.rados.0.smithi022.stdout:1092: finishing rollback tid 0 to smithi02284960-47 2024-09-21T09:34:50.960 INFO:tasks.rados.rados.0.smithi022.stdout:1092: finishing rollback tid 1 to smithi02284960-47 2024-09-21T09:34:50.960 INFO:tasks.rados.rados.0.smithi022.stdout:1092: finishing rollback tid 2 to smithi02284960-47 2024-09-21T09:34:50.960 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 47 v 593 (ObjNum 97 snap 0 seq_num 3570698976) dirty exists 2024-09-21T09:34:50.960 INFO:tasks.rados.rados.0.smithi022.stdout:1093: finishing copy_from racing read to smithi02284960-22 2024-09-21T09:34:50.960 INFO:tasks.rados.rados.0.smithi022.stdout:1093: finishing copy_from to smithi02284960-22 2024-09-21T09:34:50.960 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 22 v 535 (ObjNum 311 snap 94 seq_num 311) dirty exists 2024-09-21T09:34:50.960 INFO:tasks.rados.rados.0.smithi022.stdout:1091: done (3 left) 2024-09-21T09:34:50.960 INFO:tasks.rados.rados.0.smithi022.stdout:1092: done (2 left) 2024-09-21T09:34:50.960 INFO:tasks.rados.rados.0.smithi022.stdout:1093: done (1 left) 2024-09-21T09:34:50.960 INFO:tasks.rados.rados.0.smithi022.stdout:1094: done (0 left) 2024-09-21T09:34:50.960 INFO:tasks.rados.rados.0.smithi022.stdout:1095: rmattr oid 42 current snap is 113 2024-09-21T09:34:50.960 INFO:tasks.rados.rados.0.smithi022.stdout:1095: done (0 left) 2024-09-21T09:34:50.960 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 36 current snap is 113 2024-09-21T09:34:50.961 INFO:tasks.rados.rados.0.smithi022.stdout:1096: seq_num 359 ranges {1777664=417792,2195456=417792,2613248=385024} 2024-09-21T09:34:50.971 INFO:tasks.rados.rados.0.smithi022.stdout:1096: writing smithi02284960-36 from 1777664 to 2195456 tid 1 2024-09-21T09:34:50.974 INFO:tasks.rados.rados.0.smithi022.stdout:1096: writing smithi02284960-36 from 2195456 to 2613248 tid 2 2024-09-21T09:34:50.976 INFO:tasks.rados.rados.0.smithi022.stdout:1096: writing smithi02284960-36 from 2613248 to 2998272 tid 3 2024-09-21T09:34:50.976 INFO:tasks.rados.rados.0.smithi022.stdout:1097: snap_create 2024-09-21T09:34:50.976 INFO:tasks.rados.rados.0.smithi022.stdout:1096: finishing write tid 1 to smithi02284960-36 2024-09-21T09:34:50.977 INFO:tasks.rados.rados.0.smithi022.stdout:1096: finishing write tid 2 to smithi02284960-36 2024-09-21T09:34:50.979 INFO:tasks.rados.rados.0.smithi022.stdout:1096: finishing write tid 3 to smithi02284960-36 2024-09-21T09:34:50.979 INFO:tasks.rados.rados.0.smithi022.stdout:1096: finishing write tid 4 to smithi02284960-36 2024-09-21T09:34:50.980 INFO:tasks.rados.rados.0.smithi022.stdout:1096: finishing write tid 5 to smithi02284960-36 2024-09-21T09:34:50.980 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 36 v 648 (ObjNum 359 snap 113 seq_num 359) dirty exists 2024-09-21T09:34:50.980 INFO:tasks.rados.rados.0.smithi022.stdout:1096: left oid 36 (ObjNum 359 snap 113 seq_num 359) 2024-09-21T09:34:51.123 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:50 smithi064 ceph-mon[31124]: osdmap e277: 8 total, 8 up, 8 in 2024-09-21T09:34:51.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:50 smithi022 ceph-mon[39525]: osdmap e277: 8 total, 8 up, 8 in 2024-09-21T09:34:51.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:50 smithi022 ceph-mon[96736]: osdmap e277: 8 total, 8 up, 8 in 2024-09-21T09:34:51.948 INFO:tasks.rados.rados.0.smithi022.stdout:1096: done (1 left) 2024-09-21T09:34:51.948 INFO:tasks.rados.rados.0.smithi022.stdout:1097: done (0 left) 2024-09-21T09:34:51.948 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 39 current snap is 114 2024-09-21T09:34:51.948 INFO:tasks.rados.rados.0.smithi022.stdout:1098: seq_num 360 ranges {1433600=434176,1867776=704512,2572288=98304} 2024-09-21T09:34:51.958 INFO:tasks.rados.rados.0.smithi022.stdout:1098: writing smithi02284960-39 from 1433600 to 1867776 tid 1 2024-09-21T09:34:51.962 INFO:tasks.rados.rados.0.smithi022.stdout:1098: writing smithi02284960-39 from 1867776 to 2572288 tid 2 2024-09-21T09:34:51.963 INFO:tasks.rados.rados.0.smithi022.stdout:1098: writing smithi02284960-39 from 2572288 to 2670592 tid 3 2024-09-21T09:34:51.963 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 43 current snap is 114 2024-09-21T09:34:51.963 INFO:tasks.rados.rados.0.smithi022.stdout:1099: seq_num 361 ranges {2400256=450560,2850816=581632,3432448=221184} 2024-09-21T09:34:51.979 INFO:tasks.rados.rados.0.smithi022.stdout:1099: writing smithi02284960-43 from 2400256 to 2850816 tid 1 2024-09-21T09:34:51.982 INFO:tasks.rados.rados.0.smithi022.stdout:1099: writing smithi02284960-43 from 2850816 to 3432448 tid 2 2024-09-21T09:34:51.984 INFO:tasks.rados.rados.0.smithi022.stdout:1099: writing smithi02284960-43 from 3432448 to 3653632 tid 3 2024-09-21T09:34:51.984 INFO:tasks.rados.rados.0.smithi022.stdout:1100: read oid 2 snap -1 2024-09-21T09:34:51.984 INFO:tasks.rados.rados.0.smithi022.stdout:1100: expect (ObjNum 301 snap 88 seq_num 301) 2024-09-21T09:34:51.984 INFO:tasks.rados.rados.0.smithi022.stdout:1101: delete oid 13 current snap is 114 2024-09-21T09:34:51.984 INFO:tasks.rados.rados.0.smithi022.stdout:1098: finishing write tid 1 to smithi02284960-39 2024-09-21T09:34:51.984 INFO:tasks.rados.rados.0.smithi022.stdout:1098: finishing write tid 2 to smithi02284960-39 2024-09-21T09:34:51.984 INFO:tasks.rados.rados.0.smithi022.stdout:1098: finishing write tid 3 to smithi02284960-39 2024-09-21T09:34:51.984 INFO:tasks.rados.rados.0.smithi022.stdout:1098: finishing write tid 4 to smithi02284960-39 2024-09-21T09:34:51.985 INFO:tasks.rados.rados.0.smithi022.stdout:1098: finishing write tid 5 to smithi02284960-39 2024-09-21T09:34:51.985 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 39 v 469 (ObjNum 360 snap 114 seq_num 360) dirty exists 2024-09-21T09:34:51.985 INFO:tasks.rados.rados.0.smithi022.stdout:1098: left oid 39 (ObjNum 360 snap 114 seq_num 360) 2024-09-21T09:34:51.989 INFO:tasks.rados.rados.0.smithi022.stdout:1099: finishing write tid 1 to smithi02284960-43 2024-09-21T09:34:51.989 INFO:tasks.rados.rados.0.smithi022.stdout:1099: finishing write tid 2 to smithi02284960-43 2024-09-21T09:34:51.989 INFO:tasks.rados.rados.0.smithi022.stdout:1099: finishing write tid 3 to smithi02284960-43 2024-09-21T09:34:51.990 INFO:tasks.rados.rados.0.smithi022.stdout:1099: finishing write tid 4 to smithi02284960-43 2024-09-21T09:34:51.990 INFO:tasks.rados.rados.0.smithi022.stdout:1098: done (3 left) 2024-09-21T09:34:51.990 INFO:tasks.rados.rados.0.smithi022.stdout:1101: done (2 left) 2024-09-21T09:34:51.990 INFO:tasks.rados.rados.0.smithi022.stdout:1102: copy_from oid 49 from oid 41 current snap is 114 2024-09-21T09:34:51.990 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 5 current snap is 114 2024-09-21T09:34:51.990 INFO:tasks.rados.rados.0.smithi022.stdout:1103: seq_num 363 ranges {1056768=483328,1540096=753664,2293760=49152} 2024-09-21T09:34:51.999 INFO:tasks.rados.rados.0.smithi022.stdout:1103: writing smithi02284960-5 from 1056768 to 1540096 tid 1 2024-09-21T09:34:52.005 INFO:tasks.rados.rados.0.smithi022.stdout:1103: writing smithi02284960-5 from 1540096 to 2293760 tid 2 2024-09-21T09:34:52.005 INFO:tasks.rados.rados.0.smithi022.stdout:1103: writing smithi02284960-5 from 2293760 to 2342912 tid 3 2024-09-21T09:34:52.006 INFO:tasks.rados.rados.0.smithi022.stdout:1104: delete oid 48 current snap is 114 2024-09-21T09:34:52.006 INFO:tasks.rados.rados.0.smithi022.stdout:1099: finishing write tid 5 to smithi02284960-43 2024-09-21T09:34:52.006 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 43 v 586 (ObjNum 361 snap 114 seq_num 361) dirty exists 2024-09-21T09:34:52.006 INFO:tasks.rados.rados.0.smithi022.stdout:1099: left oid 43 (ObjNum 361 snap 114 seq_num 361) 2024-09-21T09:34:52.006 INFO:tasks.rados.rados.0.smithi022.stdout:1100: expect (ObjNum 301 snap 88 seq_num 301) 2024-09-21T09:34:52.141 INFO:tasks.rados.rados.0.smithi022.stdout:1103: finishing write tid 1 to smithi02284960-5 2024-09-21T09:34:52.141 INFO:tasks.rados.rados.0.smithi022.stdout:1102: finishing copy_from racing read to smithi02284960-49 2024-09-21T09:34:52.141 INFO:tasks.rados.rados.0.smithi022.stdout:1103: finishing write tid 2 to smithi02284960-5 2024-09-21T09:34:52.141 INFO:tasks.rados.rados.0.smithi022.stdout:1099: done (4 left) 2024-09-21T09:34:52.141 INFO:tasks.rados.rados.0.smithi022.stdout:1100: done (3 left) 2024-09-21T09:34:52.141 INFO:tasks.rados.rados.0.smithi022.stdout:1104: done (2 left) 2024-09-21T09:34:52.141 INFO:tasks.rados.rados.0.smithi022.stdout:1105: snap_create 2024-09-21T09:34:52.141 INFO:tasks.rados.rados.0.smithi022.stdout:1102: finishing copy_from to smithi02284960-49 2024-09-21T09:34:52.142 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 49 v 568 (ObjNum 247 snap 67 seq_num 247) dirty exists 2024-09-21T09:34:52.142 INFO:tasks.rados.rados.0.smithi022.stdout:1103: finishing write tid 3 to smithi02284960-5 2024-09-21T09:34:52.142 INFO:tasks.rados.rados.0.smithi022.stdout:1103: finishing write tid 4 to smithi02284960-5 2024-09-21T09:34:52.142 INFO:tasks.rados.rados.0.smithi022.stdout:1103: finishing write tid 5 to smithi02284960-5 2024-09-21T09:34:52.142 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 5 v 465 (ObjNum 363 snap 114 seq_num 363) dirty exists 2024-09-21T09:34:52.142 INFO:tasks.rados.rados.0.smithi022.stdout:1103: left oid 5 (ObjNum 363 snap 114 seq_num 363) 2024-09-21T09:34:52.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:51 smithi064 ceph-mon[31124]: pgmap v15: 121 pgs: 121 active+clean; 243 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 1.4 MiB/s rd, 1.1 MiB/s wr, 23 op/s 2024-09-21T09:34:52.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:51 smithi064 ceph-mon[31124]: osdmap e278: 8 total, 8 up, 8 in 2024-09-21T09:34:52.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:51 smithi064 ceph-mon[31124]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:34:52.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:51 smithi064 ceph-mon[31124]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:34:52.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:51 smithi064 ceph-mon[31124]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:34:52.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:51 smithi064 ceph-mon[31124]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:34:52.307 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:51 smithi022 ceph-mon[39525]: pgmap v15: 121 pgs: 121 active+clean; 243 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 1.4 MiB/s rd, 1.1 MiB/s wr, 23 op/s 2024-09-21T09:34:52.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:51 smithi022 ceph-mon[39525]: osdmap e278: 8 total, 8 up, 8 in 2024-09-21T09:34:52.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:51 smithi022 ceph-mon[39525]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:34:52.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:51 smithi022 ceph-mon[39525]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:34:52.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:51 smithi022 ceph-mon[39525]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:34:52.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:51 smithi022 ceph-mon[39525]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:34:52.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:51 smithi022 ceph-mon[96736]: pgmap v15: 121 pgs: 121 active+clean; 243 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 1.4 MiB/s rd, 1.1 MiB/s wr, 23 op/s 2024-09-21T09:34:52.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:51 smithi022 ceph-mon[96736]: osdmap e278: 8 total, 8 up, 8 in 2024-09-21T09:34:52.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:51 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:34:52.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:51 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:34:52.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:51 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:34:52.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:51 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:34:53.054 INFO:tasks.rados.rados.0.smithi022.stdout:1102: done (2 left) 2024-09-21T09:34:53.055 INFO:tasks.rados.rados.0.smithi022.stdout:1103: done (1 left) 2024-09-21T09:34:53.055 INFO:tasks.rados.rados.0.smithi022.stdout:1105: done (0 left) 2024-09-21T09:34:53.055 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 47 current snap is 115 2024-09-21T09:34:53.055 INFO:tasks.rados.rados.0.smithi022.stdout:1106: seq_num 365 ranges {0=516096,516096=573440,1089536=229376} 2024-09-21T09:34:53.057 INFO:tasks.rados.rados.0.smithi022.stdout:1106: writing smithi02284960-47 from 0 to 516096 tid 1 2024-09-21T09:34:53.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:52 smithi022 ceph-mon[96736]: Updating smithi022:/etc/ceph/ceph.conf 2024-09-21T09:34:53.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:52 smithi022 ceph-mon[96736]: Updating smithi064:/etc/ceph/ceph.conf 2024-09-21T09:34:53.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:52 smithi022 ceph-mon[96736]: Updating smithi022:/var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:34:53.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:52 smithi022 ceph-mon[96736]: Updating smithi064:/var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:34:53.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:52 smithi022 ceph-mon[96736]: Updating smithi022:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:34:53.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:52 smithi022 ceph-mon[96736]: Updating smithi064:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:34:53.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:52 smithi022 ceph-mon[96736]: Updating smithi022:/var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:34:53.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:52 smithi022 ceph-mon[96736]: Updating smithi064:/var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:34:53.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:52 smithi022 ceph-mon[96736]: osdmap e279: 8 total, 8 up, 8 in 2024-09-21T09:34:53.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:52 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:34:53.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:52 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:34:53.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:52 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:34:53.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:52 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:34:53.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:52 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:34:53.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:52 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:34:53.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:52 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:34:53.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:52 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "quorum_status"} : dispatch 2024-09-21T09:34:53.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:52 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "mon ok-to-stop", "ids": ["c"]} : dispatch 2024-09-21T09:34:53.059 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:52 smithi022 ceph-mon[39525]: Updating smithi022:/etc/ceph/ceph.conf 2024-09-21T09:34:53.059 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:52 smithi022 ceph-mon[39525]: Updating smithi064:/etc/ceph/ceph.conf 2024-09-21T09:34:53.059 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:52 smithi022 ceph-mon[39525]: Updating smithi022:/var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:34:53.059 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:52 smithi022 ceph-mon[39525]: Updating smithi064:/var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:34:53.060 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:52 smithi022 ceph-mon[39525]: Updating smithi022:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:34:53.060 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:52 smithi022 ceph-mon[39525]: Updating smithi064:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:34:53.060 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:52 smithi022 ceph-mon[39525]: Updating smithi022:/var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:34:53.060 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:52 smithi022 ceph-mon[39525]: Updating smithi064:/var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:34:53.060 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:52 smithi022 ceph-mon[39525]: osdmap e279: 8 total, 8 up, 8 in 2024-09-21T09:34:53.060 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:52 smithi022 ceph-mon[39525]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:34:53.060 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:52 smithi022 ceph-mon[39525]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:34:53.060 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:52 smithi022 ceph-mon[39525]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:34:53.060 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:52 smithi022 ceph-mon[39525]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:34:53.060 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:52 smithi022 ceph-mon[39525]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:34:53.060 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:52 smithi022 ceph-mon[39525]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:34:53.060 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:52 smithi022 ceph-mon[39525]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:34:53.060 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:52 smithi022 ceph-mon[39525]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "quorum_status"} : dispatch 2024-09-21T09:34:53.060 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:52 smithi022 ceph-mon[39525]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "mon ok-to-stop", "ids": ["c"]} : dispatch 2024-09-21T09:34:53.061 INFO:tasks.rados.rados.0.smithi022.stdout:1106: writing smithi02284960-47 from 516096 to 1089536 tid 2 2024-09-21T09:34:53.063 INFO:tasks.rados.rados.0.smithi022.stdout:1106: writing smithi02284960-47 from 1089536 to 1318912 tid 3 2024-09-21T09:34:53.063 INFO:tasks.rados.rados.0.smithi022.stdout:1107: snap_remove snap 108 2024-09-21T09:34:53.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:52 smithi064 ceph-mon[31124]: Updating smithi022:/etc/ceph/ceph.conf 2024-09-21T09:34:53.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:52 smithi064 ceph-mon[31124]: Updating smithi064:/etc/ceph/ceph.conf 2024-09-21T09:34:53.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:52 smithi064 ceph-mon[31124]: Updating smithi022:/var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:34:53.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:52 smithi064 ceph-mon[31124]: Updating smithi064:/var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:34:53.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:52 smithi064 ceph-mon[31124]: Updating smithi022:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:34:53.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:52 smithi064 ceph-mon[31124]: Updating smithi064:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:34:53.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:52 smithi064 ceph-mon[31124]: Updating smithi022:/var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:34:53.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:52 smithi064 ceph-mon[31124]: Updating smithi064:/var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:34:53.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:52 smithi064 ceph-mon[31124]: osdmap e279: 8 total, 8 up, 8 in 2024-09-21T09:34:53.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:52 smithi064 ceph-mon[31124]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:34:53.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:52 smithi064 ceph-mon[31124]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:34:53.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:52 smithi064 ceph-mon[31124]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:34:53.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:52 smithi064 ceph-mon[31124]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:34:53.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:52 smithi064 ceph-mon[31124]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:34:53.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:52 smithi064 ceph-mon[31124]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:34:53.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:52 smithi064 ceph-mon[31124]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:34:53.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:52 smithi064 ceph-mon[31124]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "quorum_status"} : dispatch 2024-09-21T09:34:53.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:52 smithi064 ceph-mon[31124]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "mon ok-to-stop", "ids": ["c"]} : dispatch 2024-09-21T09:34:53.937 INFO:tasks.rados.rados.0.smithi022.stdout:1107: done (1 left) 2024-09-21T09:34:53.937 INFO:tasks.rados.rados.0.smithi022.stdout:1108: delete oid 9 current snap is 115 2024-09-21T09:34:53.937 INFO:tasks.rados.rados.0.smithi022.stdout:1106: finishing write tid 1 to smithi02284960-47 2024-09-21T09:34:53.937 INFO:tasks.rados.rados.0.smithi022.stdout:1106: finishing write tid 2 to smithi02284960-47 2024-09-21T09:34:53.937 INFO:tasks.rados.rados.0.smithi022.stdout:1106: finishing write tid 3 to smithi02284960-47 2024-09-21T09:34:53.937 INFO:tasks.rados.rados.0.smithi022.stdout:1106: finishing write tid 4 to smithi02284960-47 2024-09-21T09:34:53.937 INFO:tasks.rados.rados.0.smithi022.stdout:1106: finishing write tid 5 to smithi02284960-47 2024-09-21T09:34:53.937 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 47 v 598 (ObjNum 365 snap 115 seq_num 365) dirty exists 2024-09-21T09:34:53.937 INFO:tasks.rados.rados.0.smithi022.stdout:1106: left oid 47 (ObjNum 365 snap 115 seq_num 365) 2024-09-21T09:34:53.949 INFO:tasks.rados.rados.0.smithi022.stdout:1106: done (1 left) 2024-09-21T09:34:53.949 INFO:tasks.rados.rados.0.smithi022.stdout:1108: done (0 left) 2024-09-21T09:34:53.949 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 46 current snap is 115 2024-09-21T09:34:53.949 INFO:tasks.rados.rados.0.smithi022.stdout:1109: seq_num 367 ranges {7028736=548864,7577600=737280,8314880=65536} 2024-09-21T09:34:53.990 INFO:tasks.rados.rados.0.smithi022.stdout:1109: writing smithi02284960-46 from 7028736 to 7577600 tid 1 2024-09-21T09:34:53.994 INFO:tasks.rados.rados.0.smithi022.stdout:1109: writing smithi02284960-46 from 7577600 to 8314880 tid 2 2024-09-21T09:34:53.994 INFO:tasks.rados.rados.0.smithi022.stdout:1109: writing smithi02284960-46 from 8314880 to 8380416 tid 3 2024-09-21T09:34:53.994 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 15 current snap is 115 2024-09-21T09:34:53.994 INFO:tasks.rados.rados.0.smithi022.stdout:1109: finishing write tid 1 to smithi02284960-46 2024-09-21T09:34:53.994 INFO:tasks.rados.rados.0.smithi022.stdout:1110: seq_num 368 ranges {0=573440,573440=622592,1196032=180224} 2024-09-21T09:34:53.997 INFO:tasks.rados.rados.0.smithi022.stdout:1110: writing smithi02284960-15 from 0 to 573440 tid 1 2024-09-21T09:34:54.001 INFO:tasks.rados.rados.0.smithi022.stdout:1110: writing smithi02284960-15 from 573440 to 1196032 tid 2 2024-09-21T09:34:54.002 INFO:tasks.rados.rados.0.smithi022.stdout:1110: writing smithi02284960-15 from 1196032 to 1376256 tid 3 2024-09-21T09:34:54.002 INFO:tasks.rados.rados.0.smithi022.stdout:1111: read oid 31 snap -1 2024-09-21T09:34:54.003 INFO:tasks.rados.rados.0.smithi022.stdout:1111: expect deleted 2024-09-21T09:34:54.003 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 23 current snap is 115 2024-09-21T09:34:54.003 INFO:tasks.rados.rados.0.smithi022.stdout:1109: finishing write tid 2 to smithi02284960-46 2024-09-21T09:34:54.003 INFO:tasks.rados.rados.0.smithi022.stdout:1109: finishing write tid 3 to smithi02284960-46 2024-09-21T09:34:54.003 INFO:tasks.rados.rados.0.smithi022.stdout:1112: seq_num 369 ranges {6045696=589824,6635520=507904,7143424=294912} 2024-09-21T09:34:54.038 INFO:tasks.rados.rados.0.smithi022.stdout:1112: writing smithi02284960-23 from 6045696 to 6635520 tid 1 2024-09-21T09:34:54.042 INFO:tasks.rados.rados.0.smithi022.stdout:1112: writing smithi02284960-23 from 6635520 to 7143424 tid 2 2024-09-21T09:34:54.044 INFO:tasks.rados.rados.0.smithi022.stdout:1112: writing smithi02284960-23 from 7143424 to 7438336 tid 3 2024-09-21T09:34:54.044 INFO:tasks.rados.rados.0.smithi022.stdout:1113: snap_remove snap 111 2024-09-21T09:34:54.159 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:53 smithi022 ceph-mon[39525]: Upgrade: It appears safe to stop mon.c 2024-09-21T09:34:54.159 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:53 smithi022 ceph-mon[39525]: pgmap v18: 121 pgs: 121 active+clean; 243 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:34:54.159 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:53 smithi022 ceph-mon[39525]: osdmap e280: 8 total, 8 up, 8 in 2024-09-21T09:34:54.159 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:53 smithi022 ceph-mon[39525]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:34:54.160 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:53 smithi022 ceph-mon[39525]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-21T09:34:54.160 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:53 smithi022 ceph-mon[39525]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-21T09:34:54.160 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:53 smithi022 ceph-mon[39525]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:34:54.160 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:53 smithi022 ceph-mon[39525]: osdmap e281: 8 total, 8 up, 8 in 2024-09-21T09:34:54.160 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:53 smithi022 ceph-mon[96736]: Upgrade: It appears safe to stop mon.c 2024-09-21T09:34:54.160 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:53 smithi022 ceph-mon[96736]: pgmap v18: 121 pgs: 121 active+clean; 243 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:34:54.160 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:53 smithi022 ceph-mon[96736]: osdmap e280: 8 total, 8 up, 8 in 2024-09-21T09:34:54.160 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:53 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:34:54.160 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:53 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-21T09:34:54.160 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:53 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-21T09:34:54.160 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:53 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:34:54.160 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:53 smithi022 ceph-mon[96736]: osdmap e281: 8 total, 8 up, 8 in 2024-09-21T09:34:54.441 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:34:54 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:34:54.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:34:54.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:53 smithi064 ceph-mon[31124]: Upgrade: It appears safe to stop mon.c 2024-09-21T09:34:54.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:53 smithi064 ceph-mon[31124]: pgmap v18: 121 pgs: 121 active+clean; 243 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:34:54.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:53 smithi064 ceph-mon[31124]: osdmap e280: 8 total, 8 up, 8 in 2024-09-21T09:34:54.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:53 smithi064 ceph-mon[31124]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:34:54.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:53 smithi064 ceph-mon[31124]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-21T09:34:54.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:53 smithi064 ceph-mon[31124]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-21T09:34:54.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:53 smithi064 ceph-mon[31124]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:34:54.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:53 smithi064 ceph-mon[31124]: osdmap e281: 8 total, 8 up, 8 in 2024-09-21T09:34:54.958 INFO:tasks.rados.rados.0.smithi022.stdout:1109: finishing write tid 4 to smithi02284960-46 2024-09-21T09:34:54.958 INFO:tasks.rados.rados.0.smithi022.stdout:1109: finishing write tid 5 to smithi02284960-46 2024-09-21T09:34:54.959 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 46 v 609 (ObjNum 367 snap 115 seq_num 367) dirty exists 2024-09-21T09:34:54.959 INFO:tasks.rados.rados.0.smithi022.stdout:1109: left oid 46 (ObjNum 367 snap 115 seq_num 367) 2024-09-21T09:34:54.959 INFO:tasks.rados.rados.0.smithi022.stdout:1110: finishing write tid 1 to smithi02284960-15 2024-09-21T09:34:54.959 INFO:tasks.rados.rados.0.smithi022.stdout:1110: finishing write tid 2 to smithi02284960-15 2024-09-21T09:34:54.959 INFO:tasks.rados.rados.0.smithi022.stdout:1110: finishing write tid 3 to smithi02284960-15 2024-09-21T09:34:54.959 INFO:tasks.rados.rados.0.smithi022.stdout:1110: finishing write tid 4 to smithi02284960-15 2024-09-21T09:34:54.959 INFO:tasks.rados.rados.0.smithi022.stdout:1110: finishing write tid 5 to smithi02284960-15 2024-09-21T09:34:54.959 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 15 v 666 (ObjNum 368 snap 115 seq_num 368) dirty exists 2024-09-21T09:34:54.959 INFO:tasks.rados.rados.0.smithi022.stdout:1110: left oid 15 (ObjNum 368 snap 115 seq_num 368) 2024-09-21T09:34:54.959 INFO:tasks.rados.rados.0.smithi022.stdout:1112: finishing write tid 1 to smithi02284960-23 2024-09-21T09:34:54.959 INFO:tasks.rados.rados.0.smithi022.stdout:1112: finishing write tid 2 to smithi02284960-23 2024-09-21T09:34:54.959 INFO:tasks.rados.rados.0.smithi022.stdout:1112: finishing write tid 3 to smithi02284960-23 2024-09-21T09:34:54.959 INFO:tasks.rados.rados.0.smithi022.stdout:1112: finishing write tid 4 to smithi02284960-23 2024-09-21T09:34:54.959 INFO:tasks.rados.rados.0.smithi022.stdout:1112: finishing write tid 5 to smithi02284960-23 2024-09-21T09:34:54.960 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 23 v 350 (ObjNum 369 snap 115 seq_num 369) dirty exists 2024-09-21T09:34:54.960 INFO:tasks.rados.rados.0.smithi022.stdout:1112: left oid 23 (ObjNum 369 snap 115 seq_num 369) 2024-09-21T09:34:54.960 INFO:tasks.rados.rados.0.smithi022.stdout:1109: done (4 left) 2024-09-21T09:34:54.960 INFO:tasks.rados.rados.0.smithi022.stdout:1110: done (3 left) 2024-09-21T09:34:54.960 INFO:tasks.rados.rados.0.smithi022.stdout:1111: done (2 left) 2024-09-21T09:34:54.960 INFO:tasks.rados.rados.0.smithi022.stdout:1112: done (1 left) 2024-09-21T09:34:54.960 INFO:tasks.rados.rados.0.smithi022.stdout:1113: done (0 left) 2024-09-21T09:34:54.960 INFO:tasks.rados.rados.0.smithi022.stdout:1114: rmattr oid 30 current snap is 115 2024-09-21T09:34:54.960 INFO:tasks.rados.rados.0.smithi022.stdout:1114: done (0 left) 2024-09-21T09:34:54.960 INFO:tasks.rados.rados.0.smithi022.stdout:1115: snap_remove snap 112 2024-09-21T09:34:55.212 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:54 smithi022 ceph-mon[39525]: Upgrade: Updating mon.c 2024-09-21T09:34:55.212 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:54 smithi022 ceph-mon[39525]: Deploying daemon mon.c on smithi022 2024-09-21T09:34:55.212 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:54 smithi022 ceph-mon[96736]: Upgrade: Updating mon.c 2024-09-21T09:34:55.213 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:34:54 smithi022 ceph-mon[96736]: Deploying daemon mon.c on smithi022 2024-09-21T09:34:55.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:54 smithi064 ceph-mon[31124]: Upgrade: Updating mon.c 2024-09-21T09:34:55.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:34:54 smithi064 ceph-mon[31124]: Deploying daemon mon.c on smithi022 2024-09-21T09:34:55.471 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:55 smithi022 systemd[1]: Stopping Ceph mon.c for d65e801e-77fa-11ef-baf6-efdc52797490... 2024-09-21T09:34:55.808 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:34:55 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:34:55] "GET /metrics HTTP/1.1" 200 33376 "" "Prometheus/2.51.0" 2024-09-21T09:34:55.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:55 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mon-c[39502]: 2024-09-21T09:34:55.468+0000 7fa47dd06640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T09:34:55.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:55 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mon-c[39502]: 2024-09-21T09:34:55.468+0000 7fa47dd06640 -1 mon.c@2(peon) e3 *** Got Signal Terminated *** 2024-09-21T09:34:55.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:55 smithi022 podman[98424]: 2024-09-21 09:34:55.550794748 +0000 UTC m=+0.191802354 container died 79f307e3b64ae5b1c912931bbf795d4a334d924c2b0dc757e944822a4c6e984f (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mon-c, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, RELEASE=quincy-fbd1a6a, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, GIT_CLEAN=True, ceph=True) 2024-09-21T09:34:56.186 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:55 smithi022 podman[98424]: 2024-09-21 09:34:55.933251701 +0000 UTC m=+0.574259297 container cleanup 79f307e3b64ae5b1c912931bbf795d4a334d924c2b0dc757e944822a4c6e984f (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mon-c, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=quincy-fbd1a6a, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.37.2) 2024-09-21T09:34:56.186 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:55 smithi022 bash[98424]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mon-c 2024-09-21T09:34:56.191 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:34:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:34:55.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:34:56.192 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:34:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:34:55.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:34:56.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:56 smithi022 podman[98443]: 2024-09-21 09:34:56.184629404 +0000 UTC m=+0.632119915 container remove 79f307e3b64ae5b1c912931bbf795d4a334d924c2b0dc757e944822a4c6e984f (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mon-c, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=quincy-fbd1a6a, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T09:34:56.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:56 smithi022 systemd[1]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490@mon.c.service: Deactivated successfully. 2024-09-21T09:34:56.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:56 smithi022 systemd[1]: Stopped Ceph mon.c for d65e801e-77fa-11ef-baf6-efdc52797490. 2024-09-21T09:34:56.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:56 smithi022 systemd[1]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490@mon.c.service: Consumed 8.925s CPU time. 2024-09-21T09:34:56.970 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:56 smithi022 systemd[1]: Starting Ceph mon.c for d65e801e-77fa-11ef-baf6-efdc52797490... 2024-09-21T09:34:56.970 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:56 smithi022 podman[98577]: 2024-09-21 09:34:56.690149353 +0000 UTC m=+0.016839984 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:34:56.970 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:56 smithi022 podman[98577]: 2024-09-21 09:34:56.811357976 +0000 UTC m=+0.138048604 container create c74b58b67bd6f4da50b76ef30d2472b035ec784c9d751309124c186a279eca21 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mon-c, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-62da54b, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.vendor=CentOS) 2024-09-21T09:34:57.441 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:34:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:34:56.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi022\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:34:57.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 podman[98577]: 2024-09-21 09:34:57.072437572 +0000 UTC m=+0.399128218 container init c74b58b67bd6f4da50b76ef30d2472b035ec784c9d751309124c186a279eca21 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mon-c, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , RELEASE=main-62da54b, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=) 2024-09-21T09:34:57.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 podman[98577]: 2024-09-21 09:34:57.077389502 +0000 UTC m=+0.404080132 container start c74b58b67bd6f4da50b76ef30d2472b035ec784c9d751309124c186a279eca21 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mon-c, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-62da54b, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, ceph=True) 2024-09-21T09:34:57.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: set uid:gid to 167:167 (ceph:ceph) 2024-09-21T09:34:57.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev), process ceph-mon, pid 7 2024-09-21T09:34:57.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: pidfile_write: ignore empty --pid-file 2024-09-21T09:34:57.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: load: jerasure load: lrc 2024-09-21T09:34:57.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: RocksDB version: 7.9.2 2024-09-21T09:34:57.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Git sha 0 2024-09-21T09:34:57.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Compile date 2024-09-20 20:32:28 2024-09-21T09:34:57.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: DB SUMMARY 2024-09-21T09:34:57.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: DB Session ID: TGMMNFDU3LYCL1RO2LQE 2024-09-21T09:34:57.560 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: CURRENT file: CURRENT 2024-09-21T09:34:57.560 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: IDENTITY file: IDENTITY 2024-09-21T09:34:57.560 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: MANIFEST file: MANIFEST-000009 size: 1097 Bytes 2024-09-21T09:34:57.560 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 1, files: 000027.sst 2024-09-21T09:34:57.560 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000025.log size: 10971772 ; 2024-09-21T09:34:57.560 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.error_if_exists: 0 2024-09-21T09:34:57.561 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.create_if_missing: 0 2024-09-21T09:34:57.561 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.paranoid_checks: 1 2024-09-21T09:34:57.561 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-21T09:34:57.561 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-21T09:34:57.561 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-21T09:34:57.561 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.env: 0x5615548e6120 2024-09-21T09:34:57.561 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.fs: PosixFileSystem 2024-09-21T09:34:57.562 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.info_log: 0x561555fb9700 2024-09-21T09:34:57.562 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.max_file_opening_threads: 16 2024-09-21T09:34:57.562 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.statistics: (nil) 2024-09-21T09:34:57.564 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.use_fsync: 0 2024-09-21T09:34:57.564 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.max_log_file_size: 0 2024-09-21T09:34:57.564 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-21T09:34:57.564 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-21T09:34:57.565 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.keep_log_file_num: 1000 2024-09-21T09:34:57.565 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.recycle_log_file_num: 0 2024-09-21T09:34:57.565 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.allow_fallocate: 1 2024-09-21T09:34:57.565 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.allow_mmap_reads: 0 2024-09-21T09:34:57.565 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.allow_mmap_writes: 0 2024-09-21T09:34:57.565 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.use_direct_reads: 0 2024-09-21T09:34:57.565 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-21T09:34:57.565 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.create_missing_column_families: 0 2024-09-21T09:34:57.566 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.db_log_dir: 2024-09-21T09:34:57.566 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.wal_dir: 2024-09-21T09:34:57.566 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-21T09:34:57.566 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-21T09:34:57.566 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-21T09:34:57.566 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-21T09:34:57.566 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-21T09:34:57.566 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-21T09:34:57.566 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.advise_random_on_open: 1 2024-09-21T09:34:57.566 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.db_write_buffer_size: 0 2024-09-21T09:34:57.566 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.write_buffer_manager: 0x561555fbdb80 2024-09-21T09:34:57.566 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-21T09:34:57.566 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-21T09:34:57.566 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-21T09:34:57.567 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.rate_limiter: (nil) 2024-09-21T09:34:57.567 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-21T09:34:57.567 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.wal_recovery_mode: 2 2024-09-21T09:34:57.567 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.enable_thread_tracking: 0 2024-09-21T09:34:57.567 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.enable_pipelined_write: 0 2024-09-21T09:34:57.567 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.unordered_write: 0 2024-09-21T09:34:57.567 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-21T09:34:57.567 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-21T09:34:57.567 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-21T09:34:57.567 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-21T09:34:57.567 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.row_cache: None 2024-09-21T09:34:57.567 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.wal_filter: None 2024-09-21T09:34:57.567 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-21T09:34:57.567 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.allow_ingest_behind: 0 2024-09-21T09:34:57.567 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.two_write_queues: 0 2024-09-21T09:34:57.568 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.manual_wal_flush: 0 2024-09-21T09:34:57.568 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.wal_compression: 0 2024-09-21T09:34:57.568 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.atomic_flush: 0 2024-09-21T09:34:57.568 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-21T09:34:57.568 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-21T09:34:57.568 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-21T09:34:57.568 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.log_readahead_size: 0 2024-09-21T09:34:57.568 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-21T09:34:57.568 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.best_efforts_recovery: 0 2024-09-21T09:34:57.568 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-21T09:34:57.568 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-21T09:34:57.569 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.allow_data_in_errors: 0 2024-09-21T09:34:57.569 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.db_host_id: __hostname__ 2024-09-21T09:34:57.569 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-21T09:34:57.569 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.max_background_jobs: 2 2024-09-21T09:34:57.569 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.max_background_compactions: -1 2024-09-21T09:34:57.569 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.max_subcompactions: 1 2024-09-21T09:34:57.569 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-21T09:34:57.569 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-21T09:34:57.569 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-21T09:34:57.569 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.max_total_wal_size: 0 2024-09-21T09:34:57.569 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-21T09:34:57.569 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-21T09:34:57.569 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-21T09:34:57.569 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-21T09:34:57.570 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.max_open_files: -1 2024-09-21T09:34:57.570 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.bytes_per_sync: 0 2024-09-21T09:34:57.570 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-21T09:34:57.570 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-21T09:34:57.570 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.compaction_readahead_size: 0 2024-09-21T09:34:57.570 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.max_background_flushes: -1 2024-09-21T09:34:57.570 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Compression algorithms supported: 2024-09-21T09:34:57.570 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: kZSTD supported: 0 2024-09-21T09:34:57.570 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: kXpressCompression supported: 0 2024-09-21T09:34:57.570 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: kBZip2Compression supported: 0 2024-09-21T09:34:57.570 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-21T09:34:57.571 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: kLZ4Compression supported: 1 2024-09-21T09:34:57.571 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: kZlibCompression supported: 1 2024-09-21T09:34:57.571 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: kLZ4HCCompression supported: 1 2024-09-21T09:34:57.571 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: kSnappyCompression supported: 1 2024-09-21T09:34:57.571 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-21T09:34:57.571 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-21T09:34:57.571 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 2024-09-21T09:34:57.571 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-21T09:34:57.571 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-21T09:34:57.571 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.merge_operator: 2024-09-21T09:34:57.571 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.compaction_filter: None 2024-09-21T09:34:57.571 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.compaction_filter_factory: None 2024-09-21T09:34:57.572 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.sst_partitioner_factory: None 2024-09-21T09:34:57.572 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-21T09:34:57.572 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-21T09:34:57.572 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x561555fb9320) 2024-09-21T09:34:57.572 INFO:journalctl@ceph.mon.c.smithi022.stdout: cache_index_and_filter_blocks: 1 2024-09-21T09:34:57.572 INFO:journalctl@ceph.mon.c.smithi022.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-21T09:34:57.572 INFO:journalctl@ceph.mon.c.smithi022.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-21T09:34:57.572 INFO:journalctl@ceph.mon.c.smithi022.stdout: pin_top_level_index_and_filter: 1 2024-09-21T09:34:57.573 INFO:journalctl@ceph.mon.c.smithi022.stdout: index_type: 0 2024-09-21T09:34:57.573 INFO:journalctl@ceph.mon.c.smithi022.stdout: data_block_index_type: 0 2024-09-21T09:34:57.573 INFO:journalctl@ceph.mon.c.smithi022.stdout: index_shortening: 1 2024-09-21T09:34:57.573 INFO:journalctl@ceph.mon.c.smithi022.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-21T09:34:57.574 INFO:journalctl@ceph.mon.c.smithi022.stdout: checksum: 4 2024-09-21T09:34:57.574 INFO:journalctl@ceph.mon.c.smithi022.stdout: no_block_cache: 0 2024-09-21T09:34:57.574 INFO:journalctl@ceph.mon.c.smithi022.stdout: block_cache: 0x561555fad610 2024-09-21T09:34:57.574 INFO:journalctl@ceph.mon.c.smithi022.stdout: block_cache_name: BinnedLRUCache 2024-09-21T09:34:57.574 INFO:journalctl@ceph.mon.c.smithi022.stdout: block_cache_options: 2024-09-21T09:34:57.575 INFO:journalctl@ceph.mon.c.smithi022.stdout: capacity : 536870912 2024-09-21T09:34:57.575 INFO:journalctl@ceph.mon.c.smithi022.stdout: num_shard_bits : 4 2024-09-21T09:34:57.575 INFO:journalctl@ceph.mon.c.smithi022.stdout: strict_capacity_limit : 0 2024-09-21T09:34:57.575 INFO:journalctl@ceph.mon.c.smithi022.stdout: high_pri_pool_ratio: 0.000 2024-09-21T09:34:57.575 INFO:journalctl@ceph.mon.c.smithi022.stdout: block_cache_compressed: (nil) 2024-09-21T09:34:57.576 INFO:journalctl@ceph.mon.c.smithi022.stdout: persistent_cache: (nil) 2024-09-21T09:34:57.576 INFO:journalctl@ceph.mon.c.smithi022.stdout: block_size: 4096 2024-09-21T09:34:57.576 INFO:journalctl@ceph.mon.c.smithi022.stdout: block_size_deviation: 10 2024-09-21T09:34:57.576 INFO:journalctl@ceph.mon.c.smithi022.stdout: block_restart_interval: 16 2024-09-21T09:34:57.576 INFO:journalctl@ceph.mon.c.smithi022.stdout: index_block_restart_interval: 1 2024-09-21T09:34:57.576 INFO:journalctl@ceph.mon.c.smithi022.stdout: metadata_block_size: 4096 2024-09-21T09:34:57.576 INFO:journalctl@ceph.mon.c.smithi022.stdout: partition_filters: 0 2024-09-21T09:34:57.577 INFO:journalctl@ceph.mon.c.smithi022.stdout: use_delta_encoding: 1 2024-09-21T09:34:57.577 INFO:journalctl@ceph.mon.c.smithi022.stdout: filter_policy: bloomfilter 2024-09-21T09:34:57.577 INFO:journalctl@ceph.mon.c.smithi022.stdout: whole_key_filtering: 1 2024-09-21T09:34:57.577 INFO:journalctl@ceph.mon.c.smithi022.stdout: verify_compression: 0 2024-09-21T09:34:57.577 INFO:journalctl@ceph.mon.c.smithi022.stdout: read_amp_bytes_per_bit: 0 2024-09-21T09:34:57.577 INFO:journalctl@ceph.mon.c.smithi022.stdout: format_version: 5 2024-09-21T09:34:57.578 INFO:journalctl@ceph.mon.c.smithi022.stdout: enable_index_compression: 1 2024-09-21T09:34:57.578 INFO:journalctl@ceph.mon.c.smithi022.stdout: block_align: 0 2024-09-21T09:34:57.578 INFO:journalctl@ceph.mon.c.smithi022.stdout: max_auto_readahead_size: 262144 2024-09-21T09:34:57.578 INFO:journalctl@ceph.mon.c.smithi022.stdout: prepopulate_block_cache: 0 2024-09-21T09:34:57.578 INFO:journalctl@ceph.mon.c.smithi022.stdout: initial_auto_readahead_size: 8192 2024-09-21T09:34:57.578 INFO:journalctl@ceph.mon.c.smithi022.stdout: num_file_reads_for_auto_readahead: 2 2024-09-21T09:34:57.579 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.write_buffer_size: 33554432 2024-09-21T09:34:57.579 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.max_write_buffer_number: 2 2024-09-21T09:34:57.579 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.compression: NoCompression 2024-09-21T09:34:57.579 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.bottommost_compression: Disabled 2024-09-21T09:34:57.579 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.prefix_extractor: nullptr 2024-09-21T09:34:57.579 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-21T09:34:57.580 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.num_levels: 7 2024-09-21T09:34:57.580 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-21T09:34:57.580 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-21T09:34:57.580 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-21T09:34:57.580 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-21T09:34:57.580 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-21T09:34:57.580 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-21T09:34:57.580 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-21T09:34:57.580 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-21T09:34:57.581 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-21T09:34:57.581 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-21T09:34:57.581 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-21T09:34:57.581 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-21T09:34:57.581 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-21T09:34:57.581 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.compression_opts.level: 32767 2024-09-21T09:34:57.581 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.compression_opts.strategy: 0 2024-09-21T09:34:57.581 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-21T09:34:57.581 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-21T09:34:57.581 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-21T09:34:57.581 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-21T09:34:57.581 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.compression_opts.enabled: false 2024-09-21T09:34:57.581 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-21T09:34:57.581 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-21T09:34:57.581 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-21T09:34:57.582 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-21T09:34:57.582 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.target_file_size_base: 67108864 2024-09-21T09:34:57.582 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-21T09:34:57.582 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-21T09:34:57.582 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-21T09:34:57.582 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-21T09:34:57.582 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-21T09:34:57.582 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-21T09:34:57.583 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-21T09:34:57.583 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-21T09:34:57.583 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-21T09:34:57.583 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-21T09:34:57.583 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-21T09:34:57.583 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-21T09:34:57.584 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-21T09:34:57.584 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-21T09:34:57.584 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.arena_block_size: 1048576 2024-09-21T09:34:57.584 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-21T09:34:57.584 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-21T09:34:57.584 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.disable_auto_compactions: 0 2024-09-21T09:34:57.585 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-21T09:34:57.585 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-21T09:34:57.585 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-21T09:34:57.585 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-21T09:34:57.585 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-21T09:34:57.585 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-21T09:34:57.586 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-21T09:34:57.586 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-21T09:34:57.586 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-21T09:34:57.586 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-21T09:34:57.586 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-21T09:34:57.586 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.inplace_update_support: 0 2024-09-21T09:34:57.586 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-21T09:34:57.586 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-21T09:34:57.586 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-21T09:34:57.586 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-21T09:34:57.586 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.bloom_locality: 0 2024-09-21T09:34:57.586 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.max_successive_merges: 0 2024-09-21T09:34:57.586 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-21T09:34:57.587 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.paranoid_file_checks: 0 2024-09-21T09:34:57.587 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.force_consistency_checks: 1 2024-09-21T09:34:57.587 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.report_bg_io_stats: 0 2024-09-21T09:34:57.587 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.ttl: 2592000 2024-09-21T09:34:57.587 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-21T09:34:57.587 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-21T09:34:57.587 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-21T09:34:57.587 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.enable_blob_files: false 2024-09-21T09:34:57.587 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.min_blob_size: 0 2024-09-21T09:34:57.587 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.blob_file_size: 268435456 2024-09-21T09:34:57.587 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-21T09:34:57.587 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-21T09:34:57.587 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-21T09:34:57.588 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-21T09:34:57.588 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-21T09:34:57.588 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.blob_file_starting_level: 0 2024-09-21T09:34:57.588 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-21T09:34:57.588 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 27.sst 2024-09-21T09:34:57.588 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-21T09:34:57.588 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 29, last_sequence is 13362, log_number is 25,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-21T09:34:57.588 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 25 2024-09-21T09:34:57.588 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 5ff9eb41-e42e-4fb5-87fa-84f29ffa0ff0 2024-09-21T09:34:57.588 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726911297105310, "job": 1, "event": "recovery_started", "wal_files": [25]} 2024-09-21T09:34:57.588 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #25 mode 2 2024-09-21T09:34:57.588 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726911297129606, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 30, "file_size": 6759997, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 13367, "largest_seqno": 17167, "table_properties": {"data_size": 6742985, "index_size": 11805, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 4165, "raw_key_size": 38125, "raw_average_key_size": 23, "raw_value_size": 6708037, "raw_average_value_size": 4063, "num_data_blocks": 512, "num_entries": 1651, "num_filter_entries": 1651, "num_deletions": 310, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726911297, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "5ff9eb41-e42e-4fb5-87fa-84f29ffa0ff0", "db_session_id": "TGMMNFDU3LYCL1RO2LQE", "orig_file_number": 30, "seqno_to_time_mapping": "N/A"}} 2024-09-21T09:34:57.588 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726911297129735, "job": 1, "event": "recovery_finished"} 2024-09-21T09:34:57.588 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: [db/version_set.cc:5047] Creating manifest 32 2024-09-21T09:34:57.589 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-21T09:34:57.589 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000025.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-21T09:34:57.589 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x561555fdce00 2024-09-21T09:34:57.589 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: DB pointer 0x5615560de000 2024-09-21T09:34:57.589 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-21T09:34:57.589 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-21T09:34:57.589 INFO:journalctl@ceph.mon.c.smithi022.stdout: ** DB Stats ** 2024-09-21T09:34:57.589 INFO:journalctl@ceph.mon.c.smithi022.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-21T09:34:57.589 INFO:journalctl@ceph.mon.c.smithi022.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-21T09:34:57.589 INFO:journalctl@ceph.mon.c.smithi022.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-21T09:34:57.589 INFO:journalctl@ceph.mon.c.smithi022.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-21T09:34:57.589 INFO:journalctl@ceph.mon.c.smithi022.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-21T09:34:57.589 INFO:journalctl@ceph.mon.c.smithi022.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-21T09:34:57.590 INFO:journalctl@ceph.mon.c.smithi022.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-21T09:34:57.590 INFO:journalctl@ceph.mon.c.smithi022.stdout: 2024-09-21T09:34:57.590 INFO:journalctl@ceph.mon.c.smithi022.stdout: ** Compaction Stats [default] ** 2024-09-21T09:34:57.590 INFO:journalctl@ceph.mon.c.smithi022.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-21T09:34:57.590 INFO:journalctl@ceph.mon.c.smithi022.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-21T09:34:57.590 INFO:journalctl@ceph.mon.c.smithi022.stdout: L0 1/0 6.45 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 702.3 0.01 0.00 1 0.009 0 0 0.0 0.0 2024-09-21T09:34:57.590 INFO:journalctl@ceph.mon.c.smithi022.stdout: L6 1/0 7.43 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-21T09:34:57.590 INFO:journalctl@ceph.mon.c.smithi022.stdout: Sum 2/0 13.87 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 702.3 0.01 0.00 1 0.009 0 0 0.0 0.0 2024-09-21T09:34:57.590 INFO:journalctl@ceph.mon.c.smithi022.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 702.3 0.01 0.00 1 0.009 0 0 0.0 0.0 2024-09-21T09:34:57.590 INFO:journalctl@ceph.mon.c.smithi022.stdout: 2024-09-21T09:34:57.590 INFO:journalctl@ceph.mon.c.smithi022.stdout: ** Compaction Stats [default] ** 2024-09-21T09:34:57.590 INFO:journalctl@ceph.mon.c.smithi022.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-21T09:34:57.590 INFO:journalctl@ceph.mon.c.smithi022.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-21T09:34:57.591 INFO:journalctl@ceph.mon.c.smithi022.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 702.3 0.01 0.00 1 0.009 0 0 0.0 0.0 2024-09-21T09:34:57.591 INFO:journalctl@ceph.mon.c.smithi022.stdout: 2024-09-21T09:34:57.591 INFO:journalctl@ceph.mon.c.smithi022.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-21T09:34:57.591 INFO:journalctl@ceph.mon.c.smithi022.stdout: 2024-09-21T09:34:57.591 INFO:journalctl@ceph.mon.c.smithi022.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-21T09:34:57.591 INFO:journalctl@ceph.mon.c.smithi022.stdout: Flush(GB): cumulative 0.006, interval 0.006 2024-09-21T09:34:57.591 INFO:journalctl@ceph.mon.c.smithi022.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-21T09:34:57.591 INFO:journalctl@ceph.mon.c.smithi022.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-21T09:34:57.591 INFO:journalctl@ceph.mon.c.smithi022.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-21T09:34:57.591 INFO:journalctl@ceph.mon.c.smithi022.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-21T09:34:57.591 INFO:journalctl@ceph.mon.c.smithi022.stdout: Cumulative compaction: 0.01 GB write, 235.11 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T09:34:57.591 INFO:journalctl@ceph.mon.c.smithi022.stdout: Interval compaction: 0.01 GB write, 235.11 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T09:34:57.591 INFO:journalctl@ceph.mon.c.smithi022.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-21T09:34:57.591 INFO:journalctl@ceph.mon.c.smithi022.stdout: Block cache BinnedLRUCache@0x561555fad610#7 capacity: 512.00 MB usage: 16.61 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 2.1e-05 secs_since: 0 2024-09-21T09:34:57.592 INFO:journalctl@ceph.mon.c.smithi022.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,4.53 KB,0.000864267%) IndexBlock(1,12.08 KB,0.00230372%) Misc(1,0.00 KB,0%) 2024-09-21T09:34:57.592 INFO:journalctl@ceph.mon.c.smithi022.stdout: 2024-09-21T09:34:57.592 INFO:journalctl@ceph.mon.c.smithi022.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-21T09:34:57.592 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: starting mon.c rank 2 at public addrs [v2:172.21.15.22:3301/0,v1:172.21.15.22:6790/0] at bind addrs [v2:172.21.15.22:3301/0,v1:172.21.15.22:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid d65e801e-77fa-11ef-baf6-efdc52797490 2024-09-21T09:34:57.592 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: mon.c@-1(???) e3 preinit fsid d65e801e-77fa-11ef-baf6-efdc52797490 2024-09-21T09:34:57.592 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: mon.c@-1(???).mds e8 new map 2024-09-21T09:34:57.592 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: mon.c@-1(???).mds e8 print_map 2024-09-21T09:34:57.592 INFO:journalctl@ceph.mon.c.smithi022.stdout: e8 2024-09-21T09:34:57.592 INFO:journalctl@ceph.mon.c.smithi022.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-09-21T09:34:57.592 INFO:journalctl@ceph.mon.c.smithi022.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-21T09:34:57.592 INFO:journalctl@ceph.mon.c.smithi022.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-21T09:34:57.592 INFO:journalctl@ceph.mon.c.smithi022.stdout: legacy client fscid: 1 2024-09-21T09:34:57.592 INFO:journalctl@ceph.mon.c.smithi022.stdout: 2024-09-21T09:34:57.592 INFO:journalctl@ceph.mon.c.smithi022.stdout: Filesystem 'cephfs' (1) 2024-09-21T09:34:57.592 INFO:journalctl@ceph.mon.c.smithi022.stdout: fs_name cephfs 2024-09-21T09:34:57.593 INFO:journalctl@ceph.mon.c.smithi022.stdout: epoch 8 2024-09-21T09:34:57.593 INFO:journalctl@ceph.mon.c.smithi022.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-09-21T09:34:57.593 INFO:journalctl@ceph.mon.c.smithi022.stdout: created 2024-09-21T09:27:37.584397+0000 2024-09-21T09:34:57.593 INFO:journalctl@ceph.mon.c.smithi022.stdout: modified 2024-09-21T09:27:50.029809+0000 2024-09-21T09:34:57.593 INFO:journalctl@ceph.mon.c.smithi022.stdout: tableserver 0 2024-09-21T09:34:57.593 INFO:journalctl@ceph.mon.c.smithi022.stdout: root 0 2024-09-21T09:34:57.593 INFO:journalctl@ceph.mon.c.smithi022.stdout: session_timeout 60 2024-09-21T09:34:57.593 INFO:journalctl@ceph.mon.c.smithi022.stdout: session_autoclose 300 2024-09-21T09:34:57.593 INFO:journalctl@ceph.mon.c.smithi022.stdout: max_file_size 1099511627776 2024-09-21T09:34:57.593 INFO:journalctl@ceph.mon.c.smithi022.stdout: max_xattr_size 65536 2024-09-21T09:34:57.593 INFO:journalctl@ceph.mon.c.smithi022.stdout: required_client_features {} 2024-09-21T09:34:57.593 INFO:journalctl@ceph.mon.c.smithi022.stdout: last_failure 0 2024-09-21T09:34:57.593 INFO:journalctl@ceph.mon.c.smithi022.stdout: last_failure_osd_epoch 0 2024-09-21T09:34:57.593 INFO:journalctl@ceph.mon.c.smithi022.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-21T09:34:57.593 INFO:journalctl@ceph.mon.c.smithi022.stdout: max_mds 1 2024-09-21T09:34:57.593 INFO:journalctl@ceph.mon.c.smithi022.stdout: in 0 2024-09-21T09:34:57.594 INFO:journalctl@ceph.mon.c.smithi022.stdout: up {0=24325} 2024-09-21T09:34:57.594 INFO:journalctl@ceph.mon.c.smithi022.stdout: failed 2024-09-21T09:34:57.594 INFO:journalctl@ceph.mon.c.smithi022.stdout: damaged 2024-09-21T09:34:57.594 INFO:journalctl@ceph.mon.c.smithi022.stdout: stopped 2024-09-21T09:34:57.594 INFO:journalctl@ceph.mon.c.smithi022.stdout: data_pools [3] 2024-09-21T09:34:57.594 INFO:journalctl@ceph.mon.c.smithi022.stdout: metadata_pool 2 2024-09-21T09:34:57.594 INFO:journalctl@ceph.mon.c.smithi022.stdout: inline_data disabled 2024-09-21T09:34:57.594 INFO:journalctl@ceph.mon.c.smithi022.stdout: balancer 2024-09-21T09:34:57.594 INFO:journalctl@ceph.mon.c.smithi022.stdout: bal_rank_mask -1 2024-09-21T09:34:57.594 INFO:journalctl@ceph.mon.c.smithi022.stdout: standby_count_wanted 0 2024-09-21T09:34:57.594 INFO:journalctl@ceph.mon.c.smithi022.stdout: qdb_cluster leader: 0 members: 2024-09-21T09:34:57.594 INFO:journalctl@ceph.mon.c.smithi022.stdout: [mds.b{0:24325} state up:active seq 5 addr [v2:172.21.15.64:6832/3589447551,v1:172.21.15.64:6833/3589447551] compat {c=[1],r=[1],i=[7ff]}] 2024-09-21T09:34:57.594 INFO:journalctl@ceph.mon.c.smithi022.stdout: 2024-09-21T09:34:57.594 INFO:journalctl@ceph.mon.c.smithi022.stdout: 2024-09-21T09:34:57.594 INFO:journalctl@ceph.mon.c.smithi022.stdout: Standby daemons: 2024-09-21T09:34:57.595 INFO:journalctl@ceph.mon.c.smithi022.stdout: 2024-09-21T09:34:57.595 INFO:journalctl@ceph.mon.c.smithi022.stdout: [mds.a{-1:14424} state up:standby seq 1 addr [v2:172.21.15.22:6834/3558312740,v1:172.21.15.22:6835/3558312740] compat {c=[1],r=[1],i=[7ff]}] 2024-09-21T09:34:57.595 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: mon.c@-1(???).osd e282 crush map has features 3314933069573799936, adjusting msgr requires 2024-09-21T09:34:57.595 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: mon.c@-1(???).osd e282 crush map has features 432629308056666112, adjusting msgr requires 2024-09-21T09:34:57.595 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: mon.c@-1(???).osd e282 crush map has features 432629308056666112, adjusting msgr requires 2024-09-21T09:34:57.595 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: mon.c@-1(???).osd e282 crush map has features 432629308056666112, adjusting msgr requires 2024-09-21T09:34:57.595 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 ceph-mon[98615]: mon.c@-1(???).paxosservice(auth 1..20) refresh upgraded, format 0 -> 3 2024-09-21T09:34:57.595 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 bash[98577]: c74b58b67bd6f4da50b76ef30d2472b035ec784c9d751309124c186a279eca21 2024-09-21T09:34:57.595 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:34:57 smithi022 systemd[1]: Started Ceph mon.c for d65e801e-77fa-11ef-baf6-efdc52797490. 2024-09-21T09:35:00.691 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:35:00 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:35:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:35:02.357 INFO:tasks.rados.rados.0.smithi022.stdout:1115: done (0 left) 2024-09-21T09:35:02.357 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 19 current snap is 115 2024-09-21T09:35:02.357 INFO:tasks.rados.rados.0.smithi022.stdout:1116: seq_num 370 ranges {4972544=606208,5578752=794624,6373376=8192} 2024-09-21T09:35:02.388 INFO:tasks.rados.rados.0.smithi022.stdout:1116: writing smithi02284960-19 from 4972544 to 5578752 tid 1 2024-09-21T09:35:02.393 INFO:tasks.rados.rados.0.smithi022.stdout:1116: writing smithi02284960-19 from 5578752 to 6373376 tid 2 2024-09-21T09:35:02.393 INFO:tasks.rados.rados.0.smithi022.stdout:1116: writing smithi02284960-19 from 6373376 to 6381568 tid 3 2024-09-21T09:35:02.393 INFO:tasks.rados.rados.0.smithi022.stdout:1117: read oid 41 snap -1 2024-09-21T09:35:02.393 INFO:tasks.rados.rados.0.smithi022.stdout:1117: expect (ObjNum 247 snap 67 seq_num 247) 2024-09-21T09:35:02.393 INFO:tasks.rados.rados.0.smithi022.stdout:1118: delete oid 48 current snap is 115 2024-09-21T09:35:02.398 INFO:tasks.rados.rados.0.smithi022.stdout:1118: done (2 left) 2024-09-21T09:35:02.398 INFO:tasks.rados.rados.0.smithi022.stdout:1119: snap_remove snap 107 2024-09-21T09:35:03.377 INFO:tasks.rados.rados.0.smithi022.stdout:1119: done (2 left) 2024-09-21T09:35:03.377 INFO:tasks.rados.rados.0.smithi022.stdout:1120: snap_create 2024-09-21T09:35:03.377 INFO:tasks.rados.rados.0.smithi022.stdout:1116: finishing write tid 1 to smithi02284960-19 2024-09-21T09:35:03.377 INFO:tasks.rados.rados.0.smithi022.stdout:1116: finishing write tid 2 to smithi02284960-19 2024-09-21T09:35:03.377 INFO:tasks.rados.rados.0.smithi022.stdout:1116: finishing write tid 3 to smithi02284960-19 2024-09-21T09:35:03.377 INFO:tasks.rados.rados.0.smithi022.stdout:1116: finishing write tid 4 to smithi02284960-19 2024-09-21T09:35:03.377 INFO:tasks.rados.rados.0.smithi022.stdout:1116: finishing write tid 5 to smithi02284960-19 2024-09-21T09:35:03.377 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 19 v 671 (ObjNum 370 snap 115 seq_num 370) dirty exists 2024-09-21T09:35:03.377 INFO:tasks.rados.rados.0.smithi022.stdout:1116: left oid 19 (ObjNum 370 snap 115 seq_num 370) 2024-09-21T09:35:03.377 INFO:tasks.rados.rados.0.smithi022.stdout:1117: expect (ObjNum 247 snap 67 seq_num 247) 2024-09-21T09:35:03.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:03 smithi022 ceph-mon[98615]: mon.c calling monitor election 2024-09-21T09:35:03.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:03 smithi022 ceph-mon[98615]: mon.a calling monitor election 2024-09-21T09:35:03.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:03 smithi022 ceph-mon[98615]: mon.b calling monitor election 2024-09-21T09:35:03.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:03 smithi022 ceph-mon[98615]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-21T09:35:03.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:03 smithi022 ceph-mon[98615]: monmap epoch 3 2024-09-21T09:35:03.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:03 smithi022 ceph-mon[98615]: fsid d65e801e-77fa-11ef-baf6-efdc52797490 2024-09-21T09:35:03.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:03 smithi022 ceph-mon[98615]: last_changed 2024-09-21T09:23:35.984741+0000 2024-09-21T09:35:03.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:03 smithi022 ceph-mon[98615]: created 2024-09-21T09:22:01.045656+0000 2024-09-21T09:35:03.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:03 smithi022 ceph-mon[98615]: min_mon_release 17 (quincy) 2024-09-21T09:35:03.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:03 smithi022 ceph-mon[98615]: election_strategy: 1 2024-09-21T09:35:03.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:03 smithi022 ceph-mon[98615]: 0: [v2:172.21.15.22:3300/0,v1:172.21.15.22:6789/0] mon.a 2024-09-21T09:35:03.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:03 smithi022 ceph-mon[98615]: 1: [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] mon.b 2024-09-21T09:35:03.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:03 smithi022 ceph-mon[98615]: 2: [v2:172.21.15.22:3301/0,v1:172.21.15.22:6790/0] mon.c 2024-09-21T09:35:03.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:03 smithi022 ceph-mon[98615]: fsmap cephfs:1 {0=b=up:active} 1 up:standby 2024-09-21T09:35:03.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:03 smithi022 ceph-mon[98615]: osdmap e283: 8 total, 8 up, 8 in 2024-09-21T09:35:03.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:03 smithi022 ceph-mon[98615]: mgrmap e39: y(active, since 21s), standbys: x 2024-09-21T09:35:03.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:03 smithi022 ceph-mon[98615]: overall HEALTH_OK 2024-09-21T09:35:03.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:03 smithi022 ceph-mon[96736]: mon.c calling monitor election 2024-09-21T09:35:03.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:03 smithi022 ceph-mon[96736]: mon.a calling monitor election 2024-09-21T09:35:03.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:03 smithi022 ceph-mon[96736]: mon.b calling monitor election 2024-09-21T09:35:03.560 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:03 smithi022 ceph-mon[96736]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-21T09:35:03.560 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:03 smithi022 ceph-mon[96736]: monmap epoch 3 2024-09-21T09:35:03.560 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:03 smithi022 ceph-mon[96736]: fsid d65e801e-77fa-11ef-baf6-efdc52797490 2024-09-21T09:35:03.560 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:03 smithi022 ceph-mon[96736]: last_changed 2024-09-21T09:23:35.984741+0000 2024-09-21T09:35:03.560 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:03 smithi022 ceph-mon[96736]: created 2024-09-21T09:22:01.045656+0000 2024-09-21T09:35:03.560 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:03 smithi022 ceph-mon[96736]: min_mon_release 17 (quincy) 2024-09-21T09:35:03.560 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:03 smithi022 ceph-mon[96736]: election_strategy: 1 2024-09-21T09:35:03.560 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:03 smithi022 ceph-mon[96736]: 0: [v2:172.21.15.22:3300/0,v1:172.21.15.22:6789/0] mon.a 2024-09-21T09:35:03.560 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:03 smithi022 ceph-mon[96736]: 1: [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] mon.b 2024-09-21T09:35:03.560 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:03 smithi022 ceph-mon[96736]: 2: [v2:172.21.15.22:3301/0,v1:172.21.15.22:6790/0] mon.c 2024-09-21T09:35:03.560 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:03 smithi022 ceph-mon[96736]: fsmap cephfs:1 {0=b=up:active} 1 up:standby 2024-09-21T09:35:03.560 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:03 smithi022 ceph-mon[96736]: osdmap e283: 8 total, 8 up, 8 in 2024-09-21T09:35:03.560 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:03 smithi022 ceph-mon[96736]: mgrmap e39: y(active, since 21s), standbys: x 2024-09-21T09:35:03.560 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:03 smithi022 ceph-mon[96736]: overall HEALTH_OK 2024-09-21T09:35:03.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:03 smithi064 ceph-mon[31124]: mon.c calling monitor election 2024-09-21T09:35:03.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:03 smithi064 ceph-mon[31124]: mon.a calling monitor election 2024-09-21T09:35:03.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:03 smithi064 ceph-mon[31124]: mon.b calling monitor election 2024-09-21T09:35:03.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:03 smithi064 ceph-mon[31124]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-21T09:35:03.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:03 smithi064 ceph-mon[31124]: monmap epoch 3 2024-09-21T09:35:03.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:03 smithi064 ceph-mon[31124]: fsid d65e801e-77fa-11ef-baf6-efdc52797490 2024-09-21T09:35:03.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:03 smithi064 ceph-mon[31124]: last_changed 2024-09-21T09:23:35.984741+0000 2024-09-21T09:35:03.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:03 smithi064 ceph-mon[31124]: created 2024-09-21T09:22:01.045656+0000 2024-09-21T09:35:03.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:03 smithi064 ceph-mon[31124]: min_mon_release 17 (quincy) 2024-09-21T09:35:03.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:03 smithi064 ceph-mon[31124]: election_strategy: 1 2024-09-21T09:35:03.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:03 smithi064 ceph-mon[31124]: 0: [v2:172.21.15.22:3300/0,v1:172.21.15.22:6789/0] mon.a 2024-09-21T09:35:03.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:03 smithi064 ceph-mon[31124]: 1: [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] mon.b 2024-09-21T09:35:03.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:03 smithi064 ceph-mon[31124]: 2: [v2:172.21.15.22:3301/0,v1:172.21.15.22:6790/0] mon.c 2024-09-21T09:35:03.693 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:03 smithi064 ceph-mon[31124]: fsmap cephfs:1 {0=b=up:active} 1 up:standby 2024-09-21T09:35:03.693 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:03 smithi064 ceph-mon[31124]: osdmap e283: 8 total, 8 up, 8 in 2024-09-21T09:35:03.693 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:03 smithi064 ceph-mon[31124]: mgrmap e39: y(active, since 21s), standbys: x 2024-09-21T09:35:03.693 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:03 smithi064 ceph-mon[31124]: overall HEALTH_OK 2024-09-21T09:35:04.386 INFO:tasks.rados.rados.0.smithi022.stdout:1116: done (2 left) 2024-09-21T09:35:04.386 INFO:tasks.rados.rados.0.smithi022.stdout:1117: done (1 left) 2024-09-21T09:35:04.386 INFO:tasks.rados.rados.0.smithi022.stdout:1120: done (0 left) 2024-09-21T09:35:04.386 INFO:tasks.rados.rados.0.smithi022.stdout:1121: delete oid 13 current snap is 116 2024-09-21T09:35:04.389 INFO:tasks.rados.rados.0.smithi022.stdout:1121: done (0 left) 2024-09-21T09:35:04.389 INFO:tasks.rados.rados.0.smithi022.stdout:1122: read oid 27 snap -1 2024-09-21T09:35:04.390 INFO:tasks.rados.rados.0.smithi022.stdout:1122: expect deleted 2024-09-21T09:35:04.390 INFO:tasks.rados.rados.0.smithi022.stdout:1123: snap_remove snap 106 2024-09-21T09:35:04.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:04 smithi064 ceph-mon[31124]: osdmap e284: 8 total, 8 up, 8 in 2024-09-21T09:35:04.441 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:35:04 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:35:04.148Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:35:04.768 INFO:teuthology.orchestra.run.smithi022.stdout:true 2024-09-21T09:35:04.769 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:04 smithi022 ceph-mon[98615]: osdmap e284: 8 total, 8 up, 8 in 2024-09-21T09:35:04.769 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:04 smithi022 ceph-mon[96736]: osdmap e284: 8 total, 8 up, 8 in 2024-09-21T09:35:05.372 INFO:teuthology.orchestra.run.smithi022.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T09:35:05.372 INFO:teuthology.orchestra.run.smithi022.stdout:alertmanager.a smithi022 *:9093,9094 running (108s) 18s ago 6m 18.4M - 0.27.0 11f11916f8cd caced80ba544 2024-09-21T09:35:05.372 INFO:teuthology.orchestra.run.smithi022.stdout:grafana.a smithi064 *:3000 running (84s) 17s ago 6m 77.7M - 9.4.7 954c08fa6188 527bb657d70d 2024-09-21T09:35:05.372 INFO:teuthology.orchestra.run.smithi022.stdout:mds.a smithi022 running (7m) 18s ago 7m 17.1M - 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 34e4221f3a27 2024-09-21T09:35:05.372 INFO:teuthology.orchestra.run.smithi022.stdout:mds.b smithi064 running (7m) 17s ago 7m 20.6M - 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 0f846b3ec694 2024-09-21T09:35:05.372 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.x smithi064 *:8443,9283,8765 running (45s) 17s ago 11m 451M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 61abc61fea8d 2024-09-21T09:35:05.373 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.y smithi022 *:8443,9283,8765 running (73s) 18s ago 12m 490M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 6c5bee78f09b 2024-09-21T09:35:05.373 INFO:teuthology.orchestra.run.smithi022.stdout:mon.a smithi022 running (31s) 18s ago 13m 35.6M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 0513395c784d 2024-09-21T09:35:05.373 INFO:teuthology.orchestra.run.smithi022.stdout:mon.b smithi064 running (11m) 17s ago 11m 59.9M 2048M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be ed126533cd64 2024-09-21T09:35:05.373 INFO:teuthology.orchestra.run.smithi022.stdout:mon.c smithi022 starting - - - 2048M 2024-09-21T09:35:05.373 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.a smithi022 *:9100 running (102s) 18s ago 5m 8124k - 1.7.0 72c9c2088986 593c6158f524 2024-09-21T09:35:05.373 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.b smithi064 *:9100 running (79s) 17s ago 5m 9684k - 1.7.0 72c9c2088986 a533420b0139 2024-09-21T09:35:05.373 INFO:teuthology.orchestra.run.smithi022.stdout:osd.0 smithi022 running (10m) 18s ago 10m 190M 2481M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be fcd812003534 2024-09-21T09:35:05.373 INFO:teuthology.orchestra.run.smithi022.stdout:osd.1 smithi022 running (10m) 18s ago 10m 147M 2481M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 740462b5c304 2024-09-21T09:35:05.373 INFO:teuthology.orchestra.run.smithi022.stdout:osd.2 smithi022 running (10m) 18s ago 10m 176M 2481M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be fefc7c770c32 2024-09-21T09:35:05.373 INFO:teuthology.orchestra.run.smithi022.stdout:osd.3 smithi022 running (9m) 18s ago 9m 94.7M 2481M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be a56193b67a92 2024-09-21T09:35:05.373 INFO:teuthology.orchestra.run.smithi022.stdout:osd.4 smithi064 running (9m) 17s ago 9m 130M 2480M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 2a2937374fc4 2024-09-21T09:35:05.373 INFO:teuthology.orchestra.run.smithi022.stdout:osd.5 smithi064 running (8m) 17s ago 8m 167M 2480M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 1900c4af5d28 2024-09-21T09:35:05.373 INFO:teuthology.orchestra.run.smithi022.stdout:osd.6 smithi064 running (8m) 17s ago 8m 119M 2480M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 3cc4ec816e5b 2024-09-21T09:35:05.373 INFO:teuthology.orchestra.run.smithi022.stdout:osd.7 smithi064 running (8m) 17s ago 8m 138M 2480M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 18ef0be94104 2024-09-21T09:35:05.373 INFO:teuthology.orchestra.run.smithi022.stdout:prometheus.a smithi064 *:9095 running (50s) 17s ago 6m 41.7M - 2.51.0 1d3b7f56885b 80629ffa2179 2024-09-21T09:35:05.394 INFO:tasks.rados.rados.0.smithi022.stdout:1123: done (1 left) 2024-09-21T09:35:05.396 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 43 current snap is 116 2024-09-21T09:35:05.396 INFO:tasks.rados.rados.0.smithi022.stdout:1124: seq_num 373 ranges {3653632=655360,4308992=499712,4808704=303104} 2024-09-21T09:35:05.421 INFO:tasks.rados.rados.0.smithi022.stdout:1124: writing smithi02284960-43 from 3653632 to 4308992 tid 1 2024-09-21T09:35:05.424 INFO:tasks.rados.rados.0.smithi022.stdout:1124: writing smithi02284960-43 from 4308992 to 4808704 tid 2 2024-09-21T09:35:05.426 INFO:tasks.rados.rados.0.smithi022.stdout:1124: writing smithi02284960-43 from 4808704 to 5111808 tid 3 2024-09-21T09:35:05.426 INFO:tasks.rados.rados.0.smithi022.stdout:1125: read oid 38 snap -1 2024-09-21T09:35:05.427 INFO:tasks.rados.rados.0.smithi022.stdout:1125: expect (ObjNum 142 snap 32 seq_num 142) 2024-09-21T09:35:05.427 INFO:tasks.rados.rados.0.smithi022.stdout:1126: rollback oid 25 current snap is 116 2024-09-21T09:35:05.427 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 25 to 84 2024-09-21T09:35:05.427 INFO:tasks.rados.rados.0.smithi022.stdout:1122: done (3 left) 2024-09-21T09:35:05.427 INFO:tasks.rados.rados.0.smithi022.stdout:1127: snap_create 2024-09-21T09:35:05.429 INFO:tasks.rados.rados.0.smithi022.stdout:1124: finishing write tid 1 to smithi02284960-43 2024-09-21T09:35:05.429 INFO:tasks.rados.rados.0.smithi022.stdout:1124: finishing write tid 2 to smithi02284960-43 2024-09-21T09:35:05.431 INFO:tasks.rados.rados.0.smithi022.stdout:1124: finishing write tid 3 to smithi02284960-43 2024-09-21T09:35:05.431 INFO:tasks.rados.rados.0.smithi022.stdout:1124: finishing write tid 4 to smithi02284960-43 2024-09-21T09:35:05.431 INFO:tasks.rados.rados.0.smithi022.stdout:1126: finishing rollback tid 0 to smithi02284960-25 2024-09-21T09:35:05.432 INFO:tasks.rados.rados.0.smithi022.stdout:1126: finishing rollback tid 1 to smithi02284960-25 2024-09-21T09:35:05.432 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 25 v 604 (ObjNum 161 snap 0 seq_num 1) dirty dne 2024-09-21T09:35:05.432 INFO:tasks.rados.rados.0.smithi022.stdout:1124: finishing write tid 5 to smithi02284960-43 2024-09-21T09:35:05.432 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 43 v 601 (ObjNum 373 snap 116 seq_num 373) dirty exists 2024-09-21T09:35:05.432 INFO:tasks.rados.rados.0.smithi022.stdout:1124: left oid 43 (ObjNum 373 snap 116 seq_num 373) 2024-09-21T09:35:05.447 INFO:tasks.rados.rados.0.smithi022.stdout:1125: expect (ObjNum 142 snap 32 seq_num 142) 2024-09-21T09:35:05.680 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:05 smithi022 ceph-mon[98615]: pgmap v29: 121 pgs: 121 active+clean; 269 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 1.2 MiB/s rd, 675 KiB/s wr, 3 op/s 2024-09-21T09:35:05.680 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:05 smithi022 ceph-mon[98615]: osdmap e285: 8 total, 8 up, 8 in 2024-09-21T09:35:05.680 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:05 smithi022 ceph-mon[98615]: from='client.34146 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:35:05.681 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:05 smithi022 ceph-mon[96736]: pgmap v29: 121 pgs: 121 active+clean; 269 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 1.2 MiB/s rd, 675 KiB/s wr, 3 op/s 2024-09-21T09:35:05.681 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:05 smithi022 ceph-mon[96736]: osdmap e285: 8 total, 8 up, 8 in 2024-09-21T09:35:05.681 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:05 smithi022 ceph-mon[96736]: from='client.34146 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:35:05.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:05 smithi064 ceph-mon[31124]: pgmap v29: 121 pgs: 121 active+clean; 269 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 1.2 MiB/s rd, 675 KiB/s wr, 3 op/s 2024-09-21T09:35:05.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:05 smithi064 ceph-mon[31124]: osdmap e285: 8 total, 8 up, 8 in 2024-09-21T09:35:05.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:05 smithi064 ceph-mon[31124]: from='client.34146 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:35:05.753 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-09-21T09:35:05.754 INFO:teuthology.orchestra.run.smithi022.stdout: "mon": { 2024-09-21T09:35:05.754 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 1, 2024-09-21T09:35:05.754 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 2 2024-09-21T09:35:05.754 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-21T09:35:05.754 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr": { 2024-09-21T09:35:05.754 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 2 2024-09-21T09:35:05.754 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-21T09:35:05.754 INFO:teuthology.orchestra.run.smithi022.stdout: "osd": { 2024-09-21T09:35:05.754 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 8 2024-09-21T09:35:05.754 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-21T09:35:05.754 INFO:teuthology.orchestra.run.smithi022.stdout: "mds": { 2024-09-21T09:35:05.754 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 2 2024-09-21T09:35:05.755 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-21T09:35:05.755 INFO:teuthology.orchestra.run.smithi022.stdout: "overall": { 2024-09-21T09:35:05.755 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 11, 2024-09-21T09:35:05.755 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 4 2024-09-21T09:35:05.755 INFO:teuthology.orchestra.run.smithi022.stdout: } 2024-09-21T09:35:05.755 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-09-21T09:35:06.057 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-09-21T09:35:06.057 INFO:teuthology.orchestra.run.smithi022.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47", 2024-09-21T09:35:06.058 INFO:teuthology.orchestra.run.smithi022.stdout: "in_progress": true, 2024-09-21T09:35:06.058 INFO:teuthology.orchestra.run.smithi022.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T09:35:06.058 INFO:teuthology.orchestra.run.smithi022.stdout: "services_complete": [ 2024-09-21T09:35:06.058 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr" 2024-09-21T09:35:06.058 INFO:teuthology.orchestra.run.smithi022.stdout: ], 2024-09-21T09:35:06.058 INFO:teuthology.orchestra.run.smithi022.stdout: "progress": "3/20 daemons upgraded", 2024-09-21T09:35:06.058 INFO:teuthology.orchestra.run.smithi022.stdout: "message": "Currently upgrading mon daemons", 2024-09-21T09:35:06.058 INFO:teuthology.orchestra.run.smithi022.stdout: "is_paused": false 2024-09-21T09:35:06.058 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-09-21T09:35:06.058 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:35:05 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:35:05] "GET /metrics HTTP/1.1" 200 36121 "" "Prometheus/2.51.0" 2024-09-21T09:35:06.191 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:35:05 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:35:05.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:35:06.191 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:35:05 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:35:05.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:35:06.389 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:06 smithi022 ceph-mon[96736]: from='client.34152 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:35:06.389 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:06 smithi022 ceph-mon[96736]: from='client.34158 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:35:06.389 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:06 smithi022 ceph-mon[96736]: osdmap e286: 8 total, 8 up, 8 in 2024-09-21T09:35:06.389 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:06 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/4241601094' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:35:06.396 INFO:tasks.rados.rados.0.smithi022.stdout:1124: done (3 left) 2024-09-21T09:35:06.396 INFO:tasks.rados.rados.0.smithi022.stdout:1125: done (2 left) 2024-09-21T09:35:06.396 INFO:tasks.rados.rados.0.smithi022.stdout:1126: done (1 left) 2024-09-21T09:35:06.396 INFO:tasks.rados.rados.0.smithi022.stdout:1127: done (0 left) 2024-09-21T09:35:06.396 INFO:tasks.rados.rados.0.smithi022.stdout:1128: rollback oid 29 current snap is 117 2024-09-21T09:35:06.396 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 29 to 89 2024-09-21T09:35:06.396 INFO:tasks.rados.rados.0.smithi022.stdout:1129: snap_remove snap 109 2024-09-21T09:35:06.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:06 smithi064 ceph-mon[31124]: from='client.34152 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:35:06.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:06 smithi064 ceph-mon[31124]: from='client.34158 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:35:06.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:06 smithi064 ceph-mon[31124]: osdmap e286: 8 total, 8 up, 8 in 2024-09-21T09:35:06.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:06 smithi064 ceph-mon[31124]: from='client.? 172.21.15.22:0/4241601094' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:35:06.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:06 smithi064 ceph-mon[31124]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:06.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:06 smithi064 ceph-mon[31124]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:06.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:06 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:06.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:06 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:06.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:06 smithi022 ceph-mon[98615]: from='client.34152 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:35:06.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:06 smithi022 ceph-mon[98615]: from='client.34158 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:35:06.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:06 smithi022 ceph-mon[98615]: osdmap e286: 8 total, 8 up, 8 in 2024-09-21T09:35:06.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:06 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/4241601094' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:35:06.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:06 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:06.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:06 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:07.191 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:35:06 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:35:06.950Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi022\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:35:07.397 INFO:tasks.rados.rados.0.smithi022.stdout:1129: done (1 left) 2024-09-21T09:35:07.397 INFO:tasks.rados.rados.0.smithi022.stdout:1130: read oid 4 snap -1 2024-09-21T09:35:07.397 INFO:tasks.rados.rados.0.smithi022.stdout:1130: expect (ObjNum 335 snap 101 seq_num 335) 2024-09-21T09:35:07.397 INFO:tasks.rados.rados.0.smithi022.stdout:1131: delete oid 20 current snap is 117 2024-09-21T09:35:07.397 INFO:tasks.rados.rados.0.smithi022.stdout:1128: finishing rollback tid 1 to smithi02284960-29 2024-09-21T09:35:07.397 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 29 v 162 (ObjNum 33 snap 0 seq_num 3570498208) dirty dne 2024-09-21T09:35:07.398 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:35:07 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: 2024-09-21T09:35:07.139+0000 7feec1b50640 -1 mgr.server handle_report got status from non-daemon mon.c 2024-09-21T09:35:07.408 INFO:tasks.rados.rados.0.smithi022.stdout:1128: done (2 left) 2024-09-21T09:35:07.408 INFO:tasks.rados.rados.0.smithi022.stdout:1131: done (1 left) 2024-09-21T09:35:07.408 INFO:tasks.rados.rados.0.smithi022.stdout:1132: snap_remove snap 116 2024-09-21T09:35:07.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:07 smithi064 ceph-mon[31124]: from='client.34170 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:35:07.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:07 smithi064 ceph-mon[31124]: pgmap v32: 121 pgs: 121 active+clean; 269 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 2.4 MiB/s rd, 344 KiB/s wr, 2 op/s 2024-09-21T09:35:07.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:07 smithi064 ceph-mon[31124]: osdmap e287: 8 total, 8 up, 8 in 2024-09-21T09:35:07.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:07 smithi064 ceph-mon[31124]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:07.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:07 smithi064 ceph-mon[31124]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:07.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:07 smithi022 ceph-mon[96736]: from='client.34170 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:35:07.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:07 smithi022 ceph-mon[96736]: pgmap v32: 121 pgs: 121 active+clean; 269 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 2.4 MiB/s rd, 344 KiB/s wr, 2 op/s 2024-09-21T09:35:07.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:07 smithi022 ceph-mon[96736]: osdmap e287: 8 total, 8 up, 8 in 2024-09-21T09:35:07.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:07 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:07.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:07 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:07.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:07 smithi022 ceph-mon[98615]: from='client.34170 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:35:07.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:07 smithi022 ceph-mon[98615]: pgmap v32: 121 pgs: 121 active+clean; 269 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 2.4 MiB/s rd, 344 KiB/s wr, 2 op/s 2024-09-21T09:35:07.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:07 smithi022 ceph-mon[98615]: osdmap e287: 8 total, 8 up, 8 in 2024-09-21T09:35:07.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:07 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:07.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:07 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:08.407 INFO:tasks.rados.rados.0.smithi022.stdout:1132: done (1 left) 2024-09-21T09:35:08.407 INFO:tasks.rados.rados.0.smithi022.stdout:1133: delete oid 11 current snap is 117 2024-09-21T09:35:08.407 INFO:tasks.rados.rados.0.smithi022.stdout:1130: expect (ObjNum 335 snap 101 seq_num 335) 2024-09-21T09:35:08.605 INFO:tasks.rados.rados.0.smithi022.stdout:1130: done (1 left) 2024-09-21T09:35:08.605 INFO:tasks.rados.rados.0.smithi022.stdout:1133: done (0 left) 2024-09-21T09:35:08.605 INFO:tasks.rados.rados.0.smithi022.stdout:1134: snap_remove snap 110 2024-09-21T09:35:08.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:08 smithi064 ceph-mon[31124]: osdmap e288: 8 total, 8 up, 8 in 2024-09-21T09:35:08.697 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:08 smithi022 ceph-mon[96736]: osdmap e288: 8 total, 8 up, 8 in 2024-09-21T09:35:08.697 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:08 smithi022 ceph-mon[98615]: osdmap e288: 8 total, 8 up, 8 in 2024-09-21T09:35:08.940 INFO:tasks.rados.rados.0.smithi022.stdout:1134: done (0 left) 2024-09-21T09:35:08.940 INFO:tasks.rados.rados.0.smithi022.stdout:1135: read oid 6 snap -1 2024-09-21T09:35:08.940 INFO:tasks.rados.rados.0.smithi022.stdout:1135: expect (ObjNum 347 snap 109 seq_num 347) 2024-09-21T09:35:08.941 INFO:tasks.rados.rados.0.smithi022.stdout:1136: copy_from oid 50 from oid 30 current snap is 117 2024-09-21T09:35:08.941 INFO:tasks.rados.rados.0.smithi022.stdout:1137: rmattr oid 49 current snap is 117 2024-09-21T09:35:08.941 INFO:tasks.rados.rados.0.smithi022.stdout:1138: delete oid 45 current snap is 117 2024-09-21T09:35:08.947 INFO:tasks.rados.rados.0.smithi022.stdout:1138: done (3 left) 2024-09-21T09:35:08.947 INFO:tasks.rados.rados.0.smithi022.stdout:1139: delete oid 36 current snap is 117 2024-09-21T09:35:08.949 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 49 v 569 (ObjNum 247 snap 67 seq_num 247) dirty exists 2024-09-21T09:35:08.953 INFO:tasks.rados.rados.0.smithi022.stdout:1135: expect (ObjNum 347 snap 109 seq_num 347) 2024-09-21T09:35:09.004 INFO:tasks.rados.rados.0.smithi022.stdout:1136: finishing copy_from racing read to smithi02284960-50 2024-09-21T09:35:09.004 INFO:tasks.rados.rados.0.smithi022.stdout:1136: finishing copy_from to smithi02284960-50 2024-09-21T09:35:09.004 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 50 v 465 (ObjNum 355 snap 112 seq_num 355) dirty exists 2024-09-21T09:35:09.004 INFO:tasks.rados.rados.0.smithi022.stdout:1135: done (3 left) 2024-09-21T09:35:09.004 INFO:tasks.rados.rados.0.smithi022.stdout:1136: done (2 left) 2024-09-21T09:35:09.004 INFO:tasks.rados.rados.0.smithi022.stdout:1137: done (1 left) 2024-09-21T09:35:09.004 INFO:tasks.rados.rados.0.smithi022.stdout:1139: done (0 left) 2024-09-21T09:35:09.004 INFO:tasks.rados.rados.0.smithi022.stdout:1140: read oid 23 snap 89 2024-09-21T09:35:09.004 INFO:tasks.rados.rados.0.smithi022.stdout:1140: expect (ObjNum 264 snap 76 seq_num 264) 2024-09-21T09:35:09.005 INFO:tasks.rados.rados.0.smithi022.stdout:1141: copy_from oid 20 from oid 30 current snap is 117 2024-09-21T09:35:09.005 INFO:tasks.rados.rados.0.smithi022.stdout:1142: copy_from oid 7 from oid 1 current snap is 117 2024-09-21T09:35:09.005 INFO:tasks.rados.rados.0.smithi022.stdout:1143: snap_remove snap 102 2024-09-21T09:35:09.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:09 smithi064 ceph-mon[31124]: pgmap v35: 121 pgs: 121 active+clean; 269 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:35:09.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:09 smithi064 ceph-mon[31124]: osdmap e289: 8 total, 8 up, 8 in 2024-09-21T09:35:09.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:09 smithi064 ceph-mon[31124]: osdmap e290: 8 total, 8 up, 8 in 2024-09-21T09:35:09.773 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:09 smithi022 ceph-mon[98615]: pgmap v35: 121 pgs: 121 active+clean; 269 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:35:09.773 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:09 smithi022 ceph-mon[98615]: osdmap e289: 8 total, 8 up, 8 in 2024-09-21T09:35:09.773 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:09 smithi022 ceph-mon[98615]: osdmap e290: 8 total, 8 up, 8 in 2024-09-21T09:35:09.774 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:09 smithi022 ceph-mon[96736]: pgmap v35: 121 pgs: 121 active+clean; 269 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:35:09.774 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:09 smithi022 ceph-mon[96736]: osdmap e289: 8 total, 8 up, 8 in 2024-09-21T09:35:09.774 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:09 smithi022 ceph-mon[96736]: osdmap e290: 8 total, 8 up, 8 in 2024-09-21T09:35:09.942 INFO:tasks.rados.rados.0.smithi022.stdout:1143: done (3 left) 2024-09-21T09:35:09.942 INFO:tasks.rados.rados.0.smithi022.stdout:1144: read oid 4 snap 75 2024-09-21T09:35:09.942 INFO:tasks.rados.rados.0.smithi022.stdout:1144: expect (ObjNum 250 snap 70 seq_num 250) 2024-09-21T09:35:09.942 INFO:tasks.rados.rados.0.smithi022.stdout:1145: snap_remove snap 56 2024-09-21T09:35:10.691 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:35:10 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:35:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:35:10.949 INFO:tasks.rados.rados.0.smithi022.stdout:1145: done (4 left) 2024-09-21T09:35:10.949 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 8 current snap is 117 2024-09-21T09:35:10.949 INFO:tasks.rados.rados.0.smithi022.stdout:1146: seq_num 378 ranges {434176=737280,1171456=720896,1892352=81920} 2024-09-21T09:35:10.956 INFO:tasks.rados.rados.0.smithi022.stdout:1146: writing smithi02284960-8 from 434176 to 1171456 tid 1 2024-09-21T09:35:10.961 INFO:tasks.rados.rados.0.smithi022.stdout:1146: writing smithi02284960-8 from 1171456 to 1892352 tid 2 2024-09-21T09:35:10.963 INFO:tasks.rados.rados.0.smithi022.stdout:1146: writing smithi02284960-8 from 1892352 to 1974272 tid 3 2024-09-21T09:35:10.963 INFO:tasks.rados.rados.0.smithi022.stdout:1147: read oid 47 snap -1 2024-09-21T09:35:10.963 INFO:tasks.rados.rados.0.smithi022.stdout:1147: expect (ObjNum 365 snap 115 seq_num 365) 2024-09-21T09:35:10.963 INFO:tasks.rados.rados.0.smithi022.stdout:1148: copy_from oid 37 from oid 50 current snap is 117 2024-09-21T09:35:10.963 INFO:tasks.rados.rados.0.smithi022.stdout:1141: finishing copy_from racing read to smithi02284960-20 2024-09-21T09:35:10.963 INFO:tasks.rados.rados.0.smithi022.stdout:1142: finishing copy_from racing read to smithi02284960-7 2024-09-21T09:35:10.963 INFO:tasks.rados.rados.0.smithi022.stdout:1142: finishing copy_from to smithi02284960-7 2024-09-21T09:35:10.963 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 7 v 535 (ObjNum 311 snap 94 seq_num 311) dirty exists 2024-09-21T09:35:10.963 INFO:tasks.rados.rados.0.smithi022.stdout:1141: finishing copy_from to smithi02284960-20 2024-09-21T09:35:10.963 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 20 v 465 (ObjNum 355 snap 112 seq_num 355) dirty exists 2024-09-21T09:35:10.963 INFO:tasks.rados.rados.0.smithi022.stdout:1140: expect (ObjNum 264 snap 76 seq_num 264) 2024-09-21T09:35:11.062 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:10 smithi064 ceph-mon[31124]: osdmap e291: 8 total, 8 up, 8 in 2024-09-21T09:35:11.062 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:10 smithi064 ceph-mon[31124]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:11.062 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:10 smithi064 ceph-mon[31124]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:11.062 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:10 smithi064 ceph-mon[31124]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:35:11.062 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:10 smithi064 ceph-mon[31124]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:35:11.062 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:10 smithi064 ceph-mon[31124]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:11.063 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:10 smithi064 ceph-mon[31124]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:35:11.063 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:10 smithi064 ceph-mon[31124]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:35:11.063 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:10 smithi064 ceph-mon[31124]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "quorum_status"} : dispatch 2024-09-21T09:35:11.063 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:10 smithi064 ceph-mon[31124]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "mon ok-to-stop", "ids": ["b"]} : dispatch 2024-09-21T09:35:11.265 INFO:tasks.rados.rados.0.smithi022.stdout:1144: expect (ObjNum 250 snap 70 seq_num 250) 2024-09-21T09:35:11.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:10 smithi022 ceph-mon[98615]: osdmap e291: 8 total, 8 up, 8 in 2024-09-21T09:35:11.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:10 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:11.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:10 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:11.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:10 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:35:11.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:10 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:35:11.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:10 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:11.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:10 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:35:11.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:10 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:35:11.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:10 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "quorum_status"} : dispatch 2024-09-21T09:35:11.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:10 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "mon ok-to-stop", "ids": ["b"]} : dispatch 2024-09-21T09:35:11.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:10 smithi022 ceph-mon[96736]: osdmap e291: 8 total, 8 up, 8 in 2024-09-21T09:35:11.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:10 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:11.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:10 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:11.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:10 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:35:11.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:10 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:35:11.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:10 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:11.310 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:10 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:35:11.310 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:10 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:35:11.310 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:10 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "quorum_status"} : dispatch 2024-09-21T09:35:11.310 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:10 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "mon ok-to-stop", "ids": ["b"]} : dispatch 2024-09-21T09:35:11.377 INFO:tasks.rados.rados.0.smithi022.stdout:1146: finishing write tid 1 to smithi02284960-8 2024-09-21T09:35:11.377 INFO:tasks.rados.rados.0.smithi022.stdout:1146: finishing write tid 2 to smithi02284960-8 2024-09-21T09:35:11.377 INFO:tasks.rados.rados.0.smithi022.stdout:1146: finishing write tid 3 to smithi02284960-8 2024-09-21T09:35:11.378 INFO:tasks.rados.rados.0.smithi022.stdout:1146: finishing write tid 4 to smithi02284960-8 2024-09-21T09:35:11.378 INFO:tasks.rados.rados.0.smithi022.stdout:1146: finishing write tid 5 to smithi02284960-8 2024-09-21T09:35:11.378 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 8 v 470 (ObjNum 378 snap 117 seq_num 378) dirty exists 2024-09-21T09:35:11.378 INFO:tasks.rados.rados.0.smithi022.stdout:1146: left oid 8 (ObjNum 378 snap 117 seq_num 378) 2024-09-21T09:35:11.378 INFO:tasks.rados.rados.0.smithi022.stdout:1147: expect (ObjNum 365 snap 115 seq_num 365) 2024-09-21T09:35:11.414 INFO:tasks.rados.rados.0.smithi022.stdout:1148: finishing copy_from racing read to smithi02284960-37 2024-09-21T09:35:11.414 INFO:tasks.rados.rados.0.smithi022.stdout:1148: finishing copy_from to smithi02284960-37 2024-09-21T09:35:11.414 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 37 v 465 (ObjNum 355 snap 112 seq_num 355) dirty exists 2024-09-21T09:35:11.414 INFO:tasks.rados.rados.0.smithi022.stdout:1140: done (6 left) 2024-09-21T09:35:11.414 INFO:tasks.rados.rados.0.smithi022.stdout:1141: done (5 left) 2024-09-21T09:35:11.414 INFO:tasks.rados.rados.0.smithi022.stdout:1142: done (4 left) 2024-09-21T09:35:11.414 INFO:tasks.rados.rados.0.smithi022.stdout:1144: done (3 left) 2024-09-21T09:35:11.415 INFO:tasks.rados.rados.0.smithi022.stdout:1146: done (2 left) 2024-09-21T09:35:11.415 INFO:tasks.rados.rados.0.smithi022.stdout:1147: done (1 left) 2024-09-21T09:35:11.415 INFO:tasks.rados.rados.0.smithi022.stdout:1148: done (0 left) 2024-09-21T09:35:11.415 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 23 current snap is 117 2024-09-21T09:35:11.415 INFO:tasks.rados.rados.0.smithi022.stdout:1149: seq_num 379 ranges {7438336=753664,8192000=606208,8798208=196608} 2024-09-21T09:35:11.459 INFO:tasks.rados.rados.0.smithi022.stdout:1149: writing smithi02284960-23 from 7438336 to 8192000 tid 1 2024-09-21T09:35:11.462 INFO:tasks.rados.rados.0.smithi022.stdout:1149: writing smithi02284960-23 from 8192000 to 8798208 tid 2 2024-09-21T09:35:11.464 INFO:tasks.rados.rados.0.smithi022.stdout:1149: writing smithi02284960-23 from 8798208 to 8994816 tid 3 2024-09-21T09:35:11.464 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 36 current snap is 117 2024-09-21T09:35:11.464 INFO:tasks.rados.rados.0.smithi022.stdout:1149: finishing write tid 1 to smithi02284960-23 2024-09-21T09:35:11.464 INFO:tasks.rados.rados.0.smithi022.stdout:1150: seq_num 380 ranges {0=770048,770048=483328,1253376=319488} 2024-09-21T09:35:11.468 INFO:tasks.rados.rados.0.smithi022.stdout:1150: writing smithi02284960-36 from 0 to 770048 tid 1 2024-09-21T09:35:11.471 INFO:tasks.rados.rados.0.smithi022.stdout:1150: writing smithi02284960-36 from 770048 to 1253376 tid 2 2024-09-21T09:35:11.474 INFO:tasks.rados.rados.0.smithi022.stdout:1150: writing smithi02284960-36 from 1253376 to 1572864 tid 3 2024-09-21T09:35:11.474 INFO:tasks.rados.rados.0.smithi022.stdout:1149: finishing write tid 2 to smithi02284960-23 2024-09-21T09:35:11.474 INFO:tasks.rados.rados.0.smithi022.stdout:1149: finishing write tid 3 to smithi02284960-23 2024-09-21T09:35:11.474 INFO:tasks.rados.rados.0.smithi022.stdout:1149: finishing write tid 4 to smithi02284960-23 2024-09-21T09:35:11.474 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 12 current snap is 117 2024-09-21T09:35:11.474 INFO:tasks.rados.rados.0.smithi022.stdout:1149: finishing write tid 5 to smithi02284960-23 2024-09-21T09:35:11.474 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 23 v 354 (ObjNum 379 snap 117 seq_num 379) dirty exists 2024-09-21T09:35:11.474 INFO:tasks.rados.rados.0.smithi022.stdout:1149: left oid 23 (ObjNum 379 snap 117 seq_num 379) 2024-09-21T09:35:11.474 INFO:tasks.rados.rados.0.smithi022.stdout:1150: finishing write tid 1 to smithi02284960-36 2024-09-21T09:35:11.474 INFO:tasks.rados.rados.0.smithi022.stdout:1151: seq_num 381 ranges {344064=786432,1130496=417792,1548288=385024} 2024-09-21T09:35:11.480 INFO:tasks.rados.rados.0.smithi022.stdout:1151: writing smithi02284960-12 from 344064 to 1130496 tid 1 2024-09-21T09:35:11.482 INFO:tasks.rados.rados.0.smithi022.stdout:1151: writing smithi02284960-12 from 1130496 to 1548288 tid 2 2024-09-21T09:35:11.485 INFO:tasks.rados.rados.0.smithi022.stdout:1151: writing smithi02284960-12 from 1548288 to 1933312 tid 3 2024-09-21T09:35:11.485 INFO:tasks.rados.rados.0.smithi022.stdout:1149: done (2 left) 2024-09-21T09:35:11.485 INFO:tasks.rados.rados.0.smithi022.stdout:1152: snap_create 2024-09-21T09:35:11.485 INFO:tasks.rados.rados.0.smithi022.stdout:1150: finishing write tid 2 to smithi02284960-36 2024-09-21T09:35:11.485 INFO:tasks.rados.rados.0.smithi022.stdout:1150: finishing write tid 3 to smithi02284960-36 2024-09-21T09:35:11.485 INFO:tasks.rados.rados.0.smithi022.stdout:1150: finishing write tid 4 to smithi02284960-36 2024-09-21T09:35:11.485 INFO:tasks.rados.rados.0.smithi022.stdout:1150: finishing write tid 5 to smithi02284960-36 2024-09-21T09:35:11.485 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 36 v 654 (ObjNum 380 snap 117 seq_num 380) dirty exists 2024-09-21T09:35:11.485 INFO:tasks.rados.rados.0.smithi022.stdout:1150: left oid 36 (ObjNum 380 snap 117 seq_num 380) 2024-09-21T09:35:11.485 INFO:tasks.rados.rados.0.smithi022.stdout:1151: finishing write tid 1 to smithi02284960-12 2024-09-21T09:35:11.485 INFO:tasks.rados.rados.0.smithi022.stdout:1151: finishing write tid 2 to smithi02284960-12 2024-09-21T09:35:11.487 INFO:tasks.rados.rados.0.smithi022.stdout:1151: finishing write tid 3 to smithi02284960-12 2024-09-21T09:35:11.487 INFO:tasks.rados.rados.0.smithi022.stdout:1151: finishing write tid 4 to smithi02284960-12 2024-09-21T09:35:11.488 INFO:tasks.rados.rados.0.smithi022.stdout:1151: finishing write tid 5 to smithi02284960-12 2024-09-21T09:35:11.488 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 12 v 713 (ObjNum 381 snap 117 seq_num 381) dirty exists 2024-09-21T09:35:11.488 INFO:tasks.rados.rados.0.smithi022.stdout:1151: left oid 12 (ObjNum 381 snap 117 seq_num 381) 2024-09-21T09:35:12.118 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:11 smithi064 ceph-mon[31124]: pgmap v39: 121 pgs: 121 active+clean; 271 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 8.2 MiB/s rd, 361 KiB/s wr, 4 op/s 2024-09-21T09:35:12.118 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:11 smithi064 ceph-mon[31124]: Upgrade: It appears safe to stop mon.b 2024-09-21T09:35:12.118 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:11 smithi064 ceph-mon[31124]: osdmap e292: 8 total, 8 up, 8 in 2024-09-21T09:35:12.118 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:11 smithi064 ceph-mon[31124]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:12.118 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:11 smithi064 ceph-mon[31124]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-21T09:35:12.118 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:11 smithi064 ceph-mon[31124]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-21T09:35:12.119 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:11 smithi064 ceph-mon[31124]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:35:12.307 INFO:tasks.rados.rados.0.smithi022.stdout:1150: done (2 left) 2024-09-21T09:35:12.307 INFO:tasks.rados.rados.0.smithi022.stdout:1151: done (1 left) 2024-09-21T09:35:12.307 INFO:tasks.rados.rados.0.smithi022.stdout:1152: done (0 left) 2024-09-21T09:35:12.307 INFO:tasks.rados.rados.0.smithi022.stdout:1153: read oid 16 snap -1 2024-09-21T09:35:12.307 INFO:tasks.rados.rados.0.smithi022.stdout:1153: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-21T09:35:12.307 INFO:tasks.rados.rados.0.smithi022.stdout:1154: snap_create 2024-09-21T09:35:12.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:11 smithi022 ceph-mon[98615]: pgmap v39: 121 pgs: 121 active+clean; 271 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 8.2 MiB/s rd, 361 KiB/s wr, 4 op/s 2024-09-21T09:35:12.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:11 smithi022 ceph-mon[98615]: Upgrade: It appears safe to stop mon.b 2024-09-21T09:35:12.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:11 smithi022 ceph-mon[98615]: osdmap e292: 8 total, 8 up, 8 in 2024-09-21T09:35:12.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:11 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:12.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:11 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-21T09:35:12.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:11 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-21T09:35:12.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:11 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:35:12.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:11 smithi022 ceph-mon[96736]: pgmap v39: 121 pgs: 121 active+clean; 271 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 8.2 MiB/s rd, 361 KiB/s wr, 4 op/s 2024-09-21T09:35:12.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:11 smithi022 ceph-mon[96736]: Upgrade: It appears safe to stop mon.b 2024-09-21T09:35:12.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:11 smithi022 ceph-mon[96736]: osdmap e292: 8 total, 8 up, 8 in 2024-09-21T09:35:12.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:11 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:12.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:11 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-21T09:35:12.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:11 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-21T09:35:12.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:11 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:35:12.309 INFO:tasks.rados.rados.0.smithi022.stdout:1153: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-21T09:35:13.163 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:12 smithi064 ceph-mon[31124]: Upgrade: Updating mon.b 2024-09-21T09:35:13.163 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:12 smithi064 ceph-mon[31124]: Deploying daemon mon.b on smithi064 2024-09-21T09:35:13.163 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:12 smithi064 ceph-mon[31124]: osdmap e293: 8 total, 8 up, 8 in 2024-09-21T09:35:13.163 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:12 smithi064 ceph-mon[31124]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:35:13.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:12 smithi022 ceph-mon[98615]: Upgrade: Updating mon.b 2024-09-21T09:35:13.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:12 smithi022 ceph-mon[98615]: Deploying daemon mon.b on smithi064 2024-09-21T09:35:13.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:12 smithi022 ceph-mon[98615]: osdmap e293: 8 total, 8 up, 8 in 2024-09-21T09:35:13.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:12 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:35:13.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:12 smithi022 ceph-mon[96736]: Upgrade: Updating mon.b 2024-09-21T09:35:13.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:12 smithi022 ceph-mon[96736]: Deploying daemon mon.b on smithi064 2024-09-21T09:35:13.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:12 smithi022 ceph-mon[96736]: osdmap e293: 8 total, 8 up, 8 in 2024-09-21T09:35:13.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:12 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:35:13.312 INFO:tasks.rados.rados.0.smithi022.stdout:1153: done (1 left) 2024-09-21T09:35:13.312 INFO:tasks.rados.rados.0.smithi022.stdout:1154: done (0 left) 2024-09-21T09:35:13.312 INFO:tasks.rados.rados.0.smithi022.stdout:1155: rmattr oid 40 current snap is 119 2024-09-21T09:35:13.312 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 32 current snap is 119 2024-09-21T09:35:13.312 INFO:tasks.rados.rados.0.smithi022.stdout:1156: seq_num 382 ranges {0=802816,802816=704512,1507328=98304} 2024-09-21T09:35:13.316 INFO:tasks.rados.rados.0.smithi022.stdout:1156: writing smithi02284960-32 from 0 to 802816 tid 1 2024-09-21T09:35:13.321 INFO:tasks.rados.rados.0.smithi022.stdout:1156: writing smithi02284960-32 from 802816 to 1507328 tid 2 2024-09-21T09:35:13.322 INFO:tasks.rados.rados.0.smithi022.stdout:1156: writing smithi02284960-32 from 1507328 to 1605632 tid 3 2024-09-21T09:35:13.322 INFO:tasks.rados.rados.0.smithi022.stdout:1157: snap_create 2024-09-21T09:35:13.322 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 40 v 626 (ObjNum 332 snap 101 seq_num 332) dirty exists 2024-09-21T09:35:13.326 INFO:tasks.rados.rados.0.smithi022.stdout:1156: finishing write tid 1 to smithi02284960-32 2024-09-21T09:35:13.327 INFO:tasks.rados.rados.0.smithi022.stdout:1156: finishing write tid 2 to smithi02284960-32 2024-09-21T09:35:13.328 INFO:tasks.rados.rados.0.smithi022.stdout:1156: finishing write tid 3 to smithi02284960-32 2024-09-21T09:35:13.328 INFO:tasks.rados.rados.0.smithi022.stdout:1156: finishing write tid 4 to smithi02284960-32 2024-09-21T09:35:13.329 INFO:tasks.rados.rados.0.smithi022.stdout:1156: finishing write tid 5 to smithi02284960-32 2024-09-21T09:35:13.329 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 32 v 690 (ObjNum 382 snap 119 seq_num 382) dirty exists 2024-09-21T09:35:13.329 INFO:tasks.rados.rados.0.smithi022.stdout:1156: left oid 32 (ObjNum 382 snap 119 seq_num 382) 2024-09-21T09:35:13.729 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:13 smithi064 systemd[1]: Stopping Ceph mon.b for d65e801e-77fa-11ef-baf6-efdc52797490... 2024-09-21T09:35:14.068 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:13 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mon-b[31101]: 2024-09-21T09:35:13.725+0000 7f41ce3d5640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T09:35:14.068 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:13 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mon-b[31101]: 2024-09-21T09:35:13.725+0000 7f41ce3d5640 -1 mon.b@1(peon) e3 *** Got Signal Terminated *** 2024-09-21T09:35:14.068 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:13 smithi064 podman[86522]: 2024-09-21 09:35:13.798373772 +0000 UTC m=+0.189136416 container died ed126533cd64ff6a282f3cb73136801422203655f4f16fbb5683e4986c899996 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mon-b, org.label-schema.schema-version=1.0, RELEASE=quincy-fbd1a6a, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS) 2024-09-21T09:35:14.393 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:14 smithi064 podman[86522]: 2024-09-21 09:35:14.16611729 +0000 UTC m=+0.556879927 container cleanup ed126533cd64ff6a282f3cb73136801422203655f4f16fbb5683e4986c899996 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mon-b, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_CLEAN=True, RELEASE=quincy-fbd1a6a, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux ) 2024-09-21T09:35:14.393 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:14 smithi064 bash[86522]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mon-b 2024-09-21T09:35:14.393 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:35:14 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:35:14.148Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:35:14.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:14 smithi064 podman[86539]: 2024-09-21 09:35:14.392003498 +0000 UTC m=+0.594875001 container remove ed126533cd64ff6a282f3cb73136801422203655f4f16fbb5683e4986c899996 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mon-b, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, org.label-schema.build-date=20240731, RELEASE=quincy-fbd1a6a, org.label-schema.schema-version=1.0) 2024-09-21T09:35:14.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:14 smithi064 systemd[1]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490@mon.b.service: Deactivated successfully. 2024-09-21T09:35:14.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:14 smithi064 systemd[1]: Stopped Ceph mon.b for d65e801e-77fa-11ef-baf6-efdc52797490. 2024-09-21T09:35:14.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:14 smithi064 systemd[1]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490@mon.b.service: Consumed 9.923s CPU time. 2024-09-21T09:35:15.076 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:14 smithi064 systemd[1]: Starting Ceph mon.b for d65e801e-77fa-11ef-baf6-efdc52797490... 2024-09-21T09:35:15.076 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:14 smithi064 podman[86671]: 2024-09-21 09:35:14.959683277 +0000 UTC m=+0.113584436 container create 9dab26b6a9e74bd6f1779d35c9af92a426b9e4512d0b6dfb28eb894dab7b1ff7 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mon-b, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-62da54b, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True) 2024-09-21T09:35:15.076 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:14 smithi064 podman[86671]: 2024-09-21 09:35:14.862456994 +0000 UTC m=+0.016358170 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:35:15.223 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 podman[86671]: 2024-09-21 09:35:15.187054386 +0000 UTC m=+0.340955555 container init 9dab26b6a9e74bd6f1779d35c9af92a426b9e4512d0b6dfb28eb894dab7b1ff7 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mon-b, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, RELEASE=main-62da54b, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731) 2024-09-21T09:35:15.223 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 podman[86671]: 2024-09-21 09:35:15.192083422 +0000 UTC m=+0.345984591 container start 9dab26b6a9e74bd6f1779d35c9af92a426b9e4512d0b6dfb28eb894dab7b1ff7 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mon-b, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, RELEASE=main-62da54b, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, ceph=True, org.label-schema.vendor=CentOS) 2024-09-21T09:35:15.223 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: set uid:gid to 167:167 (ceph:ceph) 2024-09-21T09:35:15.223 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev), process ceph-mon, pid 7 2024-09-21T09:35:15.224 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: pidfile_write: ignore empty --pid-file 2024-09-21T09:35:15.224 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: load: jerasure load: lrc 2024-09-21T09:35:15.224 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: RocksDB version: 7.9.2 2024-09-21T09:35:15.224 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Git sha 0 2024-09-21T09:35:15.224 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Compile date 2024-09-20 20:32:28 2024-09-21T09:35:15.224 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: DB SUMMARY 2024-09-21T09:35:15.224 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: DB Session ID: SG9JZKOOL1NMOAH99HQ2 2024-09-21T09:35:15.224 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: CURRENT file: CURRENT 2024-09-21T09:35:15.224 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: IDENTITY file: IDENTITY 2024-09-21T09:35:15.224 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: MANIFEST file: MANIFEST-000009 size: 1097 Bytes 2024-09-21T09:35:15.224 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 1, files: 000027.sst 2024-09-21T09:35:15.224 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000025.log size: 11843798 ; 2024-09-21T09:35:15.225 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.error_if_exists: 0 2024-09-21T09:35:15.225 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.create_if_missing: 0 2024-09-21T09:35:15.225 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.paranoid_checks: 1 2024-09-21T09:35:15.225 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-21T09:35:15.225 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-21T09:35:15.225 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-21T09:35:15.225 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.env: 0x561453d01120 2024-09-21T09:35:15.225 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.fs: PosixFileSystem 2024-09-21T09:35:15.225 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.info_log: 0x56145481f700 2024-09-21T09:35:15.225 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.max_file_opening_threads: 16 2024-09-21T09:35:15.225 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.statistics: (nil) 2024-09-21T09:35:15.226 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.use_fsync: 0 2024-09-21T09:35:15.226 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.max_log_file_size: 0 2024-09-21T09:35:15.226 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-21T09:35:15.227 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-21T09:35:15.227 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.keep_log_file_num: 1000 2024-09-21T09:35:15.227 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.recycle_log_file_num: 0 2024-09-21T09:35:15.227 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.allow_fallocate: 1 2024-09-21T09:35:15.227 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.allow_mmap_reads: 0 2024-09-21T09:35:15.227 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.allow_mmap_writes: 0 2024-09-21T09:35:15.227 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.use_direct_reads: 0 2024-09-21T09:35:15.227 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-21T09:35:15.227 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.create_missing_column_families: 0 2024-09-21T09:35:15.227 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.db_log_dir: 2024-09-21T09:35:15.227 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.wal_dir: 2024-09-21T09:35:15.227 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-21T09:35:15.228 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-21T09:35:15.228 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-21T09:35:15.228 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-21T09:35:15.228 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-21T09:35:15.228 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-21T09:35:15.228 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.advise_random_on_open: 1 2024-09-21T09:35:15.228 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.db_write_buffer_size: 0 2024-09-21T09:35:15.228 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.write_buffer_manager: 0x561454823b80 2024-09-21T09:35:15.228 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-21T09:35:15.228 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-21T09:35:15.228 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-21T09:35:15.228 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.rate_limiter: (nil) 2024-09-21T09:35:15.228 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-21T09:35:15.229 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.wal_recovery_mode: 2 2024-09-21T09:35:15.229 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.enable_thread_tracking: 0 2024-09-21T09:35:15.229 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.enable_pipelined_write: 0 2024-09-21T09:35:15.229 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.unordered_write: 0 2024-09-21T09:35:15.229 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-21T09:35:15.229 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-21T09:35:15.229 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-21T09:35:15.229 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-21T09:35:15.229 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.row_cache: None 2024-09-21T09:35:15.229 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.wal_filter: None 2024-09-21T09:35:15.229 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-21T09:35:15.229 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.allow_ingest_behind: 0 2024-09-21T09:35:15.230 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.two_write_queues: 0 2024-09-21T09:35:15.230 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.manual_wal_flush: 0 2024-09-21T09:35:15.230 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.wal_compression: 0 2024-09-21T09:35:15.230 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.atomic_flush: 0 2024-09-21T09:35:15.230 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-21T09:35:15.230 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-21T09:35:15.230 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-21T09:35:15.230 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.log_readahead_size: 0 2024-09-21T09:35:15.230 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-21T09:35:15.230 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.best_efforts_recovery: 0 2024-09-21T09:35:15.230 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-21T09:35:15.231 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-21T09:35:15.231 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.allow_data_in_errors: 0 2024-09-21T09:35:15.231 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.db_host_id: __hostname__ 2024-09-21T09:35:15.231 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-21T09:35:15.231 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.max_background_jobs: 2 2024-09-21T09:35:15.231 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.max_background_compactions: -1 2024-09-21T09:35:15.231 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.max_subcompactions: 1 2024-09-21T09:35:15.231 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-21T09:35:15.231 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-21T09:35:15.231 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-21T09:35:15.231 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.max_total_wal_size: 0 2024-09-21T09:35:15.231 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-21T09:35:15.231 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-21T09:35:15.232 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-21T09:35:15.232 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-21T09:35:15.232 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.max_open_files: -1 2024-09-21T09:35:15.232 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.bytes_per_sync: 0 2024-09-21T09:35:15.232 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-21T09:35:15.232 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-21T09:35:15.232 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.compaction_readahead_size: 0 2024-09-21T09:35:15.232 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.max_background_flushes: -1 2024-09-21T09:35:15.232 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Compression algorithms supported: 2024-09-21T09:35:15.232 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: kZSTD supported: 0 2024-09-21T09:35:15.232 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: kXpressCompression supported: 0 2024-09-21T09:35:15.232 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: kBZip2Compression supported: 0 2024-09-21T09:35:15.233 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-21T09:35:15.233 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: kLZ4Compression supported: 1 2024-09-21T09:35:15.233 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: kZlibCompression supported: 1 2024-09-21T09:35:15.233 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: kLZ4HCCompression supported: 1 2024-09-21T09:35:15.233 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: kSnappyCompression supported: 1 2024-09-21T09:35:15.233 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-21T09:35:15.233 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-21T09:35:15.233 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 2024-09-21T09:35:15.233 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-21T09:35:15.233 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-21T09:35:15.234 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.merge_operator: 2024-09-21T09:35:15.234 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.compaction_filter: None 2024-09-21T09:35:15.234 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.compaction_filter_factory: None 2024-09-21T09:35:15.234 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.sst_partitioner_factory: None 2024-09-21T09:35:15.234 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-21T09:35:15.234 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-21T09:35:15.234 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x56145481f320) 2024-09-21T09:35:15.234 INFO:journalctl@ceph.mon.b.smithi064.stdout: cache_index_and_filter_blocks: 1 2024-09-21T09:35:15.234 INFO:journalctl@ceph.mon.b.smithi064.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-21T09:35:15.234 INFO:journalctl@ceph.mon.b.smithi064.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-21T09:35:15.234 INFO:journalctl@ceph.mon.b.smithi064.stdout: pin_top_level_index_and_filter: 1 2024-09-21T09:35:15.234 INFO:journalctl@ceph.mon.b.smithi064.stdout: index_type: 0 2024-09-21T09:35:15.235 INFO:journalctl@ceph.mon.b.smithi064.stdout: data_block_index_type: 0 2024-09-21T09:35:15.235 INFO:journalctl@ceph.mon.b.smithi064.stdout: index_shortening: 1 2024-09-21T09:35:15.235 INFO:journalctl@ceph.mon.b.smithi064.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-21T09:35:15.235 INFO:journalctl@ceph.mon.b.smithi064.stdout: checksum: 4 2024-09-21T09:35:15.235 INFO:journalctl@ceph.mon.b.smithi064.stdout: no_block_cache: 0 2024-09-21T09:35:15.235 INFO:journalctl@ceph.mon.b.smithi064.stdout: block_cache: 0x561454813610 2024-09-21T09:35:15.235 INFO:journalctl@ceph.mon.b.smithi064.stdout: block_cache_name: BinnedLRUCache 2024-09-21T09:35:15.235 INFO:journalctl@ceph.mon.b.smithi064.stdout: block_cache_options: 2024-09-21T09:35:15.235 INFO:journalctl@ceph.mon.b.smithi064.stdout: capacity : 536870912 2024-09-21T09:35:15.235 INFO:journalctl@ceph.mon.b.smithi064.stdout: num_shard_bits : 4 2024-09-21T09:35:15.235 INFO:journalctl@ceph.mon.b.smithi064.stdout: strict_capacity_limit : 0 2024-09-21T09:35:15.235 INFO:journalctl@ceph.mon.b.smithi064.stdout: high_pri_pool_ratio: 0.000 2024-09-21T09:35:15.236 INFO:journalctl@ceph.mon.b.smithi064.stdout: block_cache_compressed: (nil) 2024-09-21T09:35:15.236 INFO:journalctl@ceph.mon.b.smithi064.stdout: persistent_cache: (nil) 2024-09-21T09:35:15.236 INFO:journalctl@ceph.mon.b.smithi064.stdout: block_size: 4096 2024-09-21T09:35:15.236 INFO:journalctl@ceph.mon.b.smithi064.stdout: block_size_deviation: 10 2024-09-21T09:35:15.236 INFO:journalctl@ceph.mon.b.smithi064.stdout: block_restart_interval: 16 2024-09-21T09:35:15.236 INFO:journalctl@ceph.mon.b.smithi064.stdout: index_block_restart_interval: 1 2024-09-21T09:35:15.236 INFO:journalctl@ceph.mon.b.smithi064.stdout: metadata_block_size: 4096 2024-09-21T09:35:15.236 INFO:journalctl@ceph.mon.b.smithi064.stdout: partition_filters: 0 2024-09-21T09:35:15.236 INFO:journalctl@ceph.mon.b.smithi064.stdout: use_delta_encoding: 1 2024-09-21T09:35:15.236 INFO:journalctl@ceph.mon.b.smithi064.stdout: filter_policy: bloomfilter 2024-09-21T09:35:15.236 INFO:journalctl@ceph.mon.b.smithi064.stdout: whole_key_filtering: 1 2024-09-21T09:35:15.236 INFO:journalctl@ceph.mon.b.smithi064.stdout: verify_compression: 0 2024-09-21T09:35:15.236 INFO:journalctl@ceph.mon.b.smithi064.stdout: read_amp_bytes_per_bit: 0 2024-09-21T09:35:15.237 INFO:journalctl@ceph.mon.b.smithi064.stdout: format_version: 5 2024-09-21T09:35:15.237 INFO:journalctl@ceph.mon.b.smithi064.stdout: enable_index_compression: 1 2024-09-21T09:35:15.237 INFO:journalctl@ceph.mon.b.smithi064.stdout: block_align: 0 2024-09-21T09:35:15.237 INFO:journalctl@ceph.mon.b.smithi064.stdout: max_auto_readahead_size: 262144 2024-09-21T09:35:15.237 INFO:journalctl@ceph.mon.b.smithi064.stdout: prepopulate_block_cache: 0 2024-09-21T09:35:15.237 INFO:journalctl@ceph.mon.b.smithi064.stdout: initial_auto_readahead_size: 8192 2024-09-21T09:35:15.237 INFO:journalctl@ceph.mon.b.smithi064.stdout: num_file_reads_for_auto_readahead: 2 2024-09-21T09:35:15.237 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.write_buffer_size: 33554432 2024-09-21T09:35:15.237 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.max_write_buffer_number: 2 2024-09-21T09:35:15.237 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.compression: NoCompression 2024-09-21T09:35:15.237 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.bottommost_compression: Disabled 2024-09-21T09:35:15.237 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.prefix_extractor: nullptr 2024-09-21T09:35:15.237 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-21T09:35:15.238 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.num_levels: 7 2024-09-21T09:35:15.238 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-21T09:35:15.238 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-21T09:35:15.238 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-21T09:35:15.238 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-21T09:35:15.238 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-21T09:35:15.238 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-21T09:35:15.238 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-21T09:35:15.238 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-21T09:35:15.238 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-21T09:35:15.238 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-21T09:35:15.238 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-21T09:35:15.238 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-21T09:35:15.239 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-21T09:35:15.239 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.compression_opts.level: 32767 2024-09-21T09:35:15.239 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.compression_opts.strategy: 0 2024-09-21T09:35:15.239 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-21T09:35:15.239 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-21T09:35:15.239 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-21T09:35:15.239 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-21T09:35:15.239 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.compression_opts.enabled: false 2024-09-21T09:35:15.239 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-21T09:35:15.239 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-21T09:35:15.239 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-21T09:35:15.239 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-21T09:35:15.239 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.target_file_size_base: 67108864 2024-09-21T09:35:15.240 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-21T09:35:15.240 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-21T09:35:15.240 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-21T09:35:15.240 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-21T09:35:15.240 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-21T09:35:15.240 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-21T09:35:15.240 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-21T09:35:15.240 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-21T09:35:15.240 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-21T09:35:15.240 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-21T09:35:15.240 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-21T09:35:15.240 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-21T09:35:15.240 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-21T09:35:15.240 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-21T09:35:15.241 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.arena_block_size: 1048576 2024-09-21T09:35:15.241 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-21T09:35:15.241 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-21T09:35:15.241 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.disable_auto_compactions: 0 2024-09-21T09:35:15.241 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-21T09:35:15.241 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-21T09:35:15.241 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-21T09:35:15.241 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-21T09:35:15.241 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-21T09:35:15.242 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-21T09:35:15.242 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-21T09:35:15.242 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-21T09:35:15.242 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-21T09:35:15.242 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-21T09:35:15.242 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-21T09:35:15.242 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.inplace_update_support: 0 2024-09-21T09:35:15.242 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-21T09:35:15.242 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-21T09:35:15.242 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-21T09:35:15.242 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-21T09:35:15.243 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.bloom_locality: 0 2024-09-21T09:35:15.243 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.max_successive_merges: 0 2024-09-21T09:35:15.243 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-21T09:35:15.243 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.paranoid_file_checks: 0 2024-09-21T09:35:15.243 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.force_consistency_checks: 1 2024-09-21T09:35:15.243 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.report_bg_io_stats: 0 2024-09-21T09:35:15.243 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.ttl: 2592000 2024-09-21T09:35:15.243 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-21T09:35:15.243 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-21T09:35:15.243 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-21T09:35:15.243 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.enable_blob_files: false 2024-09-21T09:35:15.243 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.min_blob_size: 0 2024-09-21T09:35:15.244 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.blob_file_size: 268435456 2024-09-21T09:35:15.244 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-21T09:35:15.244 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-21T09:35:15.244 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-21T09:35:15.244 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-21T09:35:15.244 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-21T09:35:15.244 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.blob_file_starting_level: 0 2024-09-21T09:35:15.244 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-21T09:35:15.245 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 27.sst 2024-09-21T09:35:15.245 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-21T09:35:15.245 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 29, last_sequence is 13438, log_number is 25,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-21T09:35:15.245 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 25 2024-09-21T09:35:15.245 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 5b01eb0b-2464-4205-99f7-9da5ccd663ef 2024-09-21T09:35:15.245 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726911315219351, "job": 1, "event": "recovery_started", "wal_files": [25]} 2024-09-21T09:35:15.245 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #25 mode 2 2024-09-21T09:35:15.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726911315248538, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 30, "file_size": 7269259, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 13443, "largest_seqno": 17663, "table_properties": {"data_size": 7250535, "index_size": 13133, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 4549, "raw_key_size": 41634, "raw_average_key_size": 23, "raw_value_size": 7212110, "raw_average_value_size": 4002, "num_data_blocks": 569, "num_entries": 1802, "num_filter_entries": 1802, "num_deletions": 310, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726911315, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "5b01eb0b-2464-4205-99f7-9da5ccd663ef", "db_session_id": "SG9JZKOOL1NMOAH99HQ2", "orig_file_number": 30, "seqno_to_time_mapping": "N/A"}} 2024-09-21T09:35:15.502 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726911315248673, "job": 1, "event": "recovery_finished"} 2024-09-21T09:35:15.502 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: [db/version_set.cc:5047] Creating manifest 32 2024-09-21T09:35:15.502 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-21T09:35:15.502 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000025.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-21T09:35:15.502 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x561454842e00 2024-09-21T09:35:15.502 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: DB pointer 0x561454944000 2024-09-21T09:35:15.502 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-21T09:35:15.502 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-21T09:35:15.502 INFO:journalctl@ceph.mon.b.smithi064.stdout: ** DB Stats ** 2024-09-21T09:35:15.503 INFO:journalctl@ceph.mon.b.smithi064.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-21T09:35:15.503 INFO:journalctl@ceph.mon.b.smithi064.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-21T09:35:15.503 INFO:journalctl@ceph.mon.b.smithi064.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-21T09:35:15.503 INFO:journalctl@ceph.mon.b.smithi064.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-21T09:35:15.503 INFO:journalctl@ceph.mon.b.smithi064.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-21T09:35:15.503 INFO:journalctl@ceph.mon.b.smithi064.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-21T09:35:15.503 INFO:journalctl@ceph.mon.b.smithi064.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-21T09:35:15.503 INFO:journalctl@ceph.mon.b.smithi064.stdout: 2024-09-21T09:35:15.503 INFO:journalctl@ceph.mon.b.smithi064.stdout: ** Compaction Stats [default] ** 2024-09-21T09:35:15.503 INFO:journalctl@ceph.mon.b.smithi064.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-21T09:35:15.503 INFO:journalctl@ceph.mon.b.smithi064.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-21T09:35:15.503 INFO:journalctl@ceph.mon.b.smithi064.stdout: L0 1/0 6.93 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 534.8 0.01 0.00 1 0.013 0 0 0.0 0.0 2024-09-21T09:35:15.503 INFO:journalctl@ceph.mon.b.smithi064.stdout: L6 1/0 7.42 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-21T09:35:15.503 INFO:journalctl@ceph.mon.b.smithi064.stdout: Sum 2/0 14.36 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 534.8 0.01 0.00 1 0.013 0 0 0.0 0.0 2024-09-21T09:35:15.504 INFO:journalctl@ceph.mon.b.smithi064.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 534.8 0.01 0.00 1 0.013 0 0 0.0 0.0 2024-09-21T09:35:15.504 INFO:journalctl@ceph.mon.b.smithi064.stdout: 2024-09-21T09:35:15.504 INFO:journalctl@ceph.mon.b.smithi064.stdout: ** Compaction Stats [default] ** 2024-09-21T09:35:15.504 INFO:journalctl@ceph.mon.b.smithi064.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-21T09:35:15.504 INFO:journalctl@ceph.mon.b.smithi064.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-21T09:35:15.504 INFO:journalctl@ceph.mon.b.smithi064.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 534.8 0.01 0.00 1 0.013 0 0 0.0 0.0 2024-09-21T09:35:15.504 INFO:journalctl@ceph.mon.b.smithi064.stdout: 2024-09-21T09:35:15.504 INFO:journalctl@ceph.mon.b.smithi064.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-21T09:35:15.504 INFO:journalctl@ceph.mon.b.smithi064.stdout: 2024-09-21T09:35:15.504 INFO:journalctl@ceph.mon.b.smithi064.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-21T09:35:15.504 INFO:journalctl@ceph.mon.b.smithi064.stdout: Flush(GB): cumulative 0.007, interval 0.007 2024-09-21T09:35:15.504 INFO:journalctl@ceph.mon.b.smithi064.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-21T09:35:15.505 INFO:journalctl@ceph.mon.b.smithi064.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-21T09:35:15.505 INFO:journalctl@ceph.mon.b.smithi064.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-21T09:35:15.505 INFO:journalctl@ceph.mon.b.smithi064.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-21T09:35:15.505 INFO:journalctl@ceph.mon.b.smithi064.stdout: Cumulative compaction: 0.01 GB write, 215.34 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T09:35:15.505 INFO:journalctl@ceph.mon.b.smithi064.stdout: Interval compaction: 0.01 GB write, 215.34 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T09:35:15.505 INFO:journalctl@ceph.mon.b.smithi064.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-21T09:35:15.505 INFO:journalctl@ceph.mon.b.smithi064.stdout: Block cache BinnedLRUCache@0x561454813610#7 capacity: 512.00 MB usage: 17.61 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.1e-05 secs_since: 0 2024-09-21T09:35:15.505 INFO:journalctl@ceph.mon.b.smithi064.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,4.53 KB,0.000864267%) IndexBlock(1,13.08 KB,0.00249445%) Misc(1,0.00 KB,0%) 2024-09-21T09:35:15.505 INFO:journalctl@ceph.mon.b.smithi064.stdout: 2024-09-21T09:35:15.505 INFO:journalctl@ceph.mon.b.smithi064.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-21T09:35:15.505 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: starting mon.b rank 1 at public addrs [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] at bind addrs [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid d65e801e-77fa-11ef-baf6-efdc52797490 2024-09-21T09:35:15.505 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: mon.b@-1(???) e3 preinit fsid d65e801e-77fa-11ef-baf6-efdc52797490 2024-09-21T09:35:15.505 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: mon.b@-1(???).mds e8 new map 2024-09-21T09:35:15.505 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: mon.b@-1(???).mds e8 print_map 2024-09-21T09:35:15.506 INFO:journalctl@ceph.mon.b.smithi064.stdout: e8 2024-09-21T09:35:15.506 INFO:journalctl@ceph.mon.b.smithi064.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-09-21T09:35:15.506 INFO:journalctl@ceph.mon.b.smithi064.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-21T09:35:15.506 INFO:journalctl@ceph.mon.b.smithi064.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-21T09:35:15.506 INFO:journalctl@ceph.mon.b.smithi064.stdout: legacy client fscid: 1 2024-09-21T09:35:15.506 INFO:journalctl@ceph.mon.b.smithi064.stdout: 2024-09-21T09:35:15.506 INFO:journalctl@ceph.mon.b.smithi064.stdout: Filesystem 'cephfs' (1) 2024-09-21T09:35:15.506 INFO:journalctl@ceph.mon.b.smithi064.stdout: fs_name cephfs 2024-09-21T09:35:15.506 INFO:journalctl@ceph.mon.b.smithi064.stdout: epoch 8 2024-09-21T09:35:15.506 INFO:journalctl@ceph.mon.b.smithi064.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-09-21T09:35:15.506 INFO:journalctl@ceph.mon.b.smithi064.stdout: created 2024-09-21T09:27:37.584397+0000 2024-09-21T09:35:15.506 INFO:journalctl@ceph.mon.b.smithi064.stdout: modified 2024-09-21T09:27:50.029809+0000 2024-09-21T09:35:15.506 INFO:journalctl@ceph.mon.b.smithi064.stdout: tableserver 0 2024-09-21T09:35:15.507 INFO:journalctl@ceph.mon.b.smithi064.stdout: root 0 2024-09-21T09:35:15.507 INFO:journalctl@ceph.mon.b.smithi064.stdout: session_timeout 60 2024-09-21T09:35:15.507 INFO:journalctl@ceph.mon.b.smithi064.stdout: session_autoclose 300 2024-09-21T09:35:15.507 INFO:journalctl@ceph.mon.b.smithi064.stdout: max_file_size 1099511627776 2024-09-21T09:35:15.507 INFO:journalctl@ceph.mon.b.smithi064.stdout: max_xattr_size 65536 2024-09-21T09:35:15.507 INFO:journalctl@ceph.mon.b.smithi064.stdout: required_client_features {} 2024-09-21T09:35:15.507 INFO:journalctl@ceph.mon.b.smithi064.stdout: last_failure 0 2024-09-21T09:35:15.507 INFO:journalctl@ceph.mon.b.smithi064.stdout: last_failure_osd_epoch 0 2024-09-21T09:35:15.507 INFO:journalctl@ceph.mon.b.smithi064.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-21T09:35:15.507 INFO:journalctl@ceph.mon.b.smithi064.stdout: max_mds 1 2024-09-21T09:35:15.507 INFO:journalctl@ceph.mon.b.smithi064.stdout: in 0 2024-09-21T09:35:15.507 INFO:journalctl@ceph.mon.b.smithi064.stdout: up {0=24325} 2024-09-21T09:35:15.507 INFO:journalctl@ceph.mon.b.smithi064.stdout: failed 2024-09-21T09:35:15.507 INFO:journalctl@ceph.mon.b.smithi064.stdout: damaged 2024-09-21T09:35:15.508 INFO:journalctl@ceph.mon.b.smithi064.stdout: stopped 2024-09-21T09:35:15.508 INFO:journalctl@ceph.mon.b.smithi064.stdout: data_pools [3] 2024-09-21T09:35:15.508 INFO:journalctl@ceph.mon.b.smithi064.stdout: metadata_pool 2 2024-09-21T09:35:15.508 INFO:journalctl@ceph.mon.b.smithi064.stdout: inline_data disabled 2024-09-21T09:35:15.508 INFO:journalctl@ceph.mon.b.smithi064.stdout: balancer 2024-09-21T09:35:15.508 INFO:journalctl@ceph.mon.b.smithi064.stdout: bal_rank_mask -1 2024-09-21T09:35:15.508 INFO:journalctl@ceph.mon.b.smithi064.stdout: standby_count_wanted 0 2024-09-21T09:35:15.508 INFO:journalctl@ceph.mon.b.smithi064.stdout: qdb_cluster leader: 0 members: 2024-09-21T09:35:15.508 INFO:journalctl@ceph.mon.b.smithi064.stdout: [mds.b{0:24325} state up:active seq 5 addr [v2:172.21.15.64:6832/3589447551,v1:172.21.15.64:6833/3589447551] compat {c=[1],r=[1],i=[7ff]}] 2024-09-21T09:35:15.508 INFO:journalctl@ceph.mon.b.smithi064.stdout: 2024-09-21T09:35:15.508 INFO:journalctl@ceph.mon.b.smithi064.stdout: 2024-09-21T09:35:15.508 INFO:journalctl@ceph.mon.b.smithi064.stdout: Standby daemons: 2024-09-21T09:35:15.508 INFO:journalctl@ceph.mon.b.smithi064.stdout: 2024-09-21T09:35:15.509 INFO:journalctl@ceph.mon.b.smithi064.stdout: [mds.a{-1:14424} state up:standby seq 1 addr [v2:172.21.15.22:6834/3558312740,v1:172.21.15.22:6835/3558312740] compat {c=[1],r=[1],i=[7ff]}] 2024-09-21T09:35:15.509 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: mon.b@-1(???).osd e294 crush map has features 3314933069573799936, adjusting msgr requires 2024-09-21T09:35:15.509 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: mon.b@-1(???).osd e294 crush map has features 432629308056666112, adjusting msgr requires 2024-09-21T09:35:15.509 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: mon.b@-1(???).osd e294 crush map has features 432629308056666112, adjusting msgr requires 2024-09-21T09:35:15.509 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: mon.b@-1(???).osd e294 crush map has features 432629308056666112, adjusting msgr requires 2024-09-21T09:35:15.509 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: mon.b@-1(???).paxosservice(auth 1..21) refresh upgraded, format 0 -> 3 2024-09-21T09:35:15.509 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 bash[86671]: 9dab26b6a9e74bd6f1779d35c9af92a426b9e4512d0b6dfb28eb894dab7b1ff7 2024-09-21T09:35:15.509 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 systemd[1]: Started Ceph mon.b for d65e801e-77fa-11ef-baf6-efdc52797490. 2024-09-21T09:35:15.509 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: pgmap v42: 121 pgs: 121 active+clean; 271 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:35:15.509 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-mon[86709]: osdmap e294: 8 total, 8 up, 8 in 2024-09-21T09:35:15.717 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:15 smithi022 ceph-mon[98615]: pgmap v42: 121 pgs: 121 active+clean; 271 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:35:15.718 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:15 smithi022 ceph-mon[98615]: osdmap e294: 8 total, 8 up, 8 in 2024-09-21T09:35:15.718 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:15 smithi022 ceph-mon[96736]: pgmap v42: 121 pgs: 121 active+clean; 271 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:35:15.718 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:15 smithi022 ceph-mon[96736]: osdmap e294: 8 total, 8 up, 8 in 2024-09-21T09:35:15.999 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:35:15.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:35:15.999 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:35:15 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:35:15.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:35:16.058 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:35:15 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:35:15] "GET /metrics HTTP/1.1" 200 36117 "" "Prometheus/2.51.0" 2024-09-21T09:35:16.372 INFO:tasks.rados.rados.0.smithi022.stdout:1155: done (2 left) 2024-09-21T09:35:16.372 INFO:tasks.rados.rados.0.smithi022.stdout:1156: done (1 left) 2024-09-21T09:35:16.372 INFO:tasks.rados.rados.0.smithi022.stdout:1157: done (0 left) 2024-09-21T09:35:16.372 INFO:tasks.rados.rados.0.smithi022.stdout:1158: snap_remove snap 119 2024-09-21T09:35:16.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:16 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-09-21T09:35:16.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:16 smithi064 ceph-mon[86709]: mon.a calling monitor election 2024-09-21T09:35:16.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:16 smithi064 ceph-mon[86709]: mon.b calling monitor election 2024-09-21T09:35:16.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:16 smithi064 ceph-mon[86709]: mon.c calling monitor election 2024-09-21T09:35:16.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:16 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-09-21T09:35:16.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:16 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-09-21T09:35:16.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:16 smithi064 ceph-mon[86709]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-21T09:35:16.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:16 smithi064 ceph-mon[86709]: monmap epoch 4 2024-09-21T09:35:16.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:16 smithi064 ceph-mon[86709]: fsid d65e801e-77fa-11ef-baf6-efdc52797490 2024-09-21T09:35:16.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:16 smithi064 ceph-mon[86709]: last_changed 2024-09-21T09:35:15.361054+0000 2024-09-21T09:35:16.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:16 smithi064 ceph-mon[86709]: created 2024-09-21T09:22:01.045656+0000 2024-09-21T09:35:16.693 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:16 smithi064 ceph-mon[86709]: min_mon_release 19 (squid) 2024-09-21T09:35:16.693 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:16 smithi064 ceph-mon[86709]: election_strategy: 1 2024-09-21T09:35:16.693 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:16 smithi064 ceph-mon[86709]: 0: [v2:172.21.15.22:3300/0,v1:172.21.15.22:6789/0] mon.a 2024-09-21T09:35:16.693 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:16 smithi064 ceph-mon[86709]: 1: [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] mon.b 2024-09-21T09:35:16.693 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:16 smithi064 ceph-mon[86709]: 2: [v2:172.21.15.22:3301/0,v1:172.21.15.22:6790/0] mon.c 2024-09-21T09:35:16.693 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:16 smithi064 ceph-mon[86709]: fsmap cephfs:1 {0=b=up:active} 1 up:standby 2024-09-21T09:35:16.693 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:16 smithi064 ceph-mon[86709]: osdmap e294: 8 total, 8 up, 8 in 2024-09-21T09:35:16.693 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:16 smithi064 ceph-mon[86709]: mgrmap e39: y(active, since 33s), standbys: x 2024-09-21T09:35:16.693 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:16 smithi064 ceph-mon[86709]: overall HEALTH_OK 2024-09-21T09:35:16.693 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:16 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:16.693 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:16 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:16.693 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:16 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:35:16.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:16 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-09-21T09:35:16.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:16 smithi022 ceph-mon[98615]: mon.a calling monitor election 2024-09-21T09:35:16.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:16 smithi022 ceph-mon[98615]: mon.b calling monitor election 2024-09-21T09:35:16.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:16 smithi022 ceph-mon[98615]: mon.c calling monitor election 2024-09-21T09:35:16.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:16 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-09-21T09:35:16.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:16 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-09-21T09:35:16.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:16 smithi022 ceph-mon[98615]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-21T09:35:16.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:16 smithi022 ceph-mon[98615]: monmap epoch 4 2024-09-21T09:35:16.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:16 smithi022 ceph-mon[98615]: fsid d65e801e-77fa-11ef-baf6-efdc52797490 2024-09-21T09:35:16.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:16 smithi022 ceph-mon[98615]: last_changed 2024-09-21T09:35:15.361054+0000 2024-09-21T09:35:16.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:16 smithi022 ceph-mon[98615]: created 2024-09-21T09:22:01.045656+0000 2024-09-21T09:35:16.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:16 smithi022 ceph-mon[98615]: min_mon_release 19 (squid) 2024-09-21T09:35:16.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:16 smithi022 ceph-mon[98615]: election_strategy: 1 2024-09-21T09:35:16.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:16 smithi022 ceph-mon[98615]: 0: [v2:172.21.15.22:3300/0,v1:172.21.15.22:6789/0] mon.a 2024-09-21T09:35:16.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:16 smithi022 ceph-mon[98615]: 1: [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] mon.b 2024-09-21T09:35:16.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:16 smithi022 ceph-mon[98615]: 2: [v2:172.21.15.22:3301/0,v1:172.21.15.22:6790/0] mon.c 2024-09-21T09:35:16.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:16 smithi022 ceph-mon[98615]: fsmap cephfs:1 {0=b=up:active} 1 up:standby 2024-09-21T09:35:16.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:16 smithi022 ceph-mon[98615]: osdmap e294: 8 total, 8 up, 8 in 2024-09-21T09:35:16.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:16 smithi022 ceph-mon[98615]: mgrmap e39: y(active, since 33s), standbys: x 2024-09-21T09:35:16.810 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:16 smithi022 ceph-mon[98615]: overall HEALTH_OK 2024-09-21T09:35:16.810 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:16 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:16.810 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:16 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:16.810 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:16 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:35:16.810 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:16 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-09-21T09:35:16.810 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:16 smithi022 ceph-mon[96736]: mon.a calling monitor election 2024-09-21T09:35:16.810 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:16 smithi022 ceph-mon[96736]: mon.b calling monitor election 2024-09-21T09:35:16.810 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:16 smithi022 ceph-mon[96736]: mon.c calling monitor election 2024-09-21T09:35:16.810 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:16 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-09-21T09:35:16.811 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:16 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-09-21T09:35:16.811 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:16 smithi022 ceph-mon[96736]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-21T09:35:16.811 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:16 smithi022 ceph-mon[96736]: monmap epoch 4 2024-09-21T09:35:16.811 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:16 smithi022 ceph-mon[96736]: fsid d65e801e-77fa-11ef-baf6-efdc52797490 2024-09-21T09:35:16.811 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:16 smithi022 ceph-mon[96736]: last_changed 2024-09-21T09:35:15.361054+0000 2024-09-21T09:35:16.811 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:16 smithi022 ceph-mon[96736]: created 2024-09-21T09:22:01.045656+0000 2024-09-21T09:35:16.811 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:16 smithi022 ceph-mon[96736]: min_mon_release 19 (squid) 2024-09-21T09:35:16.811 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:16 smithi022 ceph-mon[96736]: election_strategy: 1 2024-09-21T09:35:16.811 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:16 smithi022 ceph-mon[96736]: 0: [v2:172.21.15.22:3300/0,v1:172.21.15.22:6789/0] mon.a 2024-09-21T09:35:16.811 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:16 smithi022 ceph-mon[96736]: 1: [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] mon.b 2024-09-21T09:35:16.811 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:16 smithi022 ceph-mon[96736]: 2: [v2:172.21.15.22:3301/0,v1:172.21.15.22:6790/0] mon.c 2024-09-21T09:35:16.811 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:16 smithi022 ceph-mon[96736]: fsmap cephfs:1 {0=b=up:active} 1 up:standby 2024-09-21T09:35:16.811 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:16 smithi022 ceph-mon[96736]: osdmap e294: 8 total, 8 up, 8 in 2024-09-21T09:35:16.811 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:16 smithi022 ceph-mon[96736]: mgrmap e39: y(active, since 33s), standbys: x 2024-09-21T09:35:16.812 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:16 smithi022 ceph-mon[96736]: overall HEALTH_OK 2024-09-21T09:35:16.812 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:16 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:16.812 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:16 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:16.812 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:16 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:35:17.129 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:35:16 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:35:16.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi022\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:35:17.382 INFO:tasks.rados.rados.0.smithi022.stdout:1158: done (0 left) 2024-09-21T09:35:17.382 INFO:tasks.rados.rados.0.smithi022.stdout:1159: copy_from oid 33 from oid 1 current snap is 120 2024-09-21T09:35:17.383 INFO:tasks.rados.rados.0.smithi022.stdout:1160: read oid 46 snap -1 2024-09-21T09:35:17.383 INFO:tasks.rados.rados.0.smithi022.stdout:1160: expect (ObjNum 367 snap 115 seq_num 367) 2024-09-21T09:35:17.383 INFO:tasks.rados.rados.0.smithi022.stdout:1161: snap_remove snap 87 2024-09-21T09:35:17.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:17 smithi064 ceph-mon[86709]: pgmap v45: 121 pgs: 121 active+clean; 260 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 5.6 MiB/s rd, 2.3 MiB/s wr, 13 op/s 2024-09-21T09:35:17.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:17 smithi064 ceph-mon[86709]: osdmap e295: 8 total, 8 up, 8 in 2024-09-21T09:35:17.807 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:17 smithi022 ceph-mon[96736]: pgmap v45: 121 pgs: 121 active+clean; 260 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 5.6 MiB/s rd, 2.3 MiB/s wr, 13 op/s 2024-09-21T09:35:17.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:17 smithi022 ceph-mon[96736]: osdmap e295: 8 total, 8 up, 8 in 2024-09-21T09:35:17.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:17 smithi022 ceph-mon[98615]: pgmap v45: 121 pgs: 121 active+clean; 260 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 5.6 MiB/s rd, 2.3 MiB/s wr, 13 op/s 2024-09-21T09:35:17.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:17 smithi022 ceph-mon[98615]: osdmap e295: 8 total, 8 up, 8 in 2024-09-21T09:35:18.389 INFO:tasks.rados.rados.0.smithi022.stdout:1160: expect (ObjNum 367 snap 115 seq_num 367) 2024-09-21T09:35:18.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:18 smithi064 ceph-mon[86709]: osdmap e296: 8 total, 8 up, 8 in 2024-09-21T09:35:18.807 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:18 smithi022 ceph-mon[98615]: osdmap e296: 8 total, 8 up, 8 in 2024-09-21T09:35:18.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:18 smithi022 ceph-mon[96736]: osdmap e296: 8 total, 8 up, 8 in 2024-09-21T09:35:19.091 INFO:tasks.rados.rados.0.smithi022.stdout:1159: finishing copy_from racing read to smithi02284960-33 2024-09-21T09:35:19.091 INFO:tasks.rados.rados.0.smithi022.stdout:1159: finishing copy_from to smithi02284960-33 2024-09-21T09:35:19.091 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 33 v 535 (ObjNum 311 snap 94 seq_num 311) dirty exists 2024-09-21T09:35:19.091 INFO:tasks.rados.rados.0.smithi022.stdout:1159: done (2 left) 2024-09-21T09:35:19.091 INFO:tasks.rados.rados.0.smithi022.stdout:1160: done (1 left) 2024-09-21T09:35:19.091 INFO:tasks.rados.rados.0.smithi022.stdout:1161: done (0 left) 2024-09-21T09:35:19.091 INFO:tasks.rados.rados.0.smithi022.stdout:1162: snap_remove snap 117 2024-09-21T09:35:19.564 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:19 smithi064 ceph-mon[86709]: pgmap v48: 121 pgs: 121 active+clean; 260 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 9.6 MiB/s rd, 2.4 MiB/s wr, 14 op/s 2024-09-21T09:35:19.565 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:19 smithi064 ceph-mon[86709]: osdmap e297: 8 total, 8 up, 8 in 2024-09-21T09:35:19.565 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:19 smithi064 ceph-mon[86709]: osdmap e298: 8 total, 8 up, 8 in 2024-09-21T09:35:19.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:19 smithi022 ceph-mon[98615]: pgmap v48: 121 pgs: 121 active+clean; 260 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 9.6 MiB/s rd, 2.4 MiB/s wr, 14 op/s 2024-09-21T09:35:19.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:19 smithi022 ceph-mon[98615]: osdmap e297: 8 total, 8 up, 8 in 2024-09-21T09:35:19.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:19 smithi022 ceph-mon[98615]: osdmap e298: 8 total, 8 up, 8 in 2024-09-21T09:35:19.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:19 smithi022 ceph-mon[96736]: pgmap v48: 121 pgs: 121 active+clean; 260 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 9.6 MiB/s rd, 2.4 MiB/s wr, 14 op/s 2024-09-21T09:35:19.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:19 smithi022 ceph-mon[96736]: osdmap e297: 8 total, 8 up, 8 in 2024-09-21T09:35:19.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:19 smithi022 ceph-mon[96736]: osdmap e298: 8 total, 8 up, 8 in 2024-09-21T09:35:19.944 INFO:tasks.rados.rados.0.smithi022.stdout:1162: done (0 left) 2024-09-21T09:35:19.944 INFO:tasks.rados.rados.0.smithi022.stdout:1163: snap_create 2024-09-21T09:35:20.558 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:35:20 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: 2024-09-21T09:35:20.260+0000 7feec1b50640 -1 mgr.server handle_report got status from non-daemon mon.b 2024-09-21T09:35:20.691 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:35:20 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:35:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:35:20.944 INFO:tasks.rados.rados.0.smithi022.stdout:1163: done (0 left) 2024-09-21T09:35:20.945 INFO:tasks.rados.rados.0.smithi022.stdout:1164: rmattr oid 26 current snap is 121 2024-09-21T09:35:20.945 INFO:tasks.rados.rados.0.smithi022.stdout:1164: done (0 left) 2024-09-21T09:35:20.945 INFO:tasks.rados.rados.0.smithi022.stdout:1165: rmattr oid 47 current snap is 121 2024-09-21T09:35:20.945 INFO:tasks.rados.rados.0.smithi022.stdout:1166: rmattr oid 11 current snap is 121 2024-09-21T09:35:20.945 INFO:tasks.rados.rados.0.smithi022.stdout:1166: done (1 left) 2024-09-21T09:35:20.945 INFO:tasks.rados.rados.0.smithi022.stdout:1167: snap_remove snap 113 2024-09-21T09:35:21.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:20 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:21.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:20 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:21.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:20 smithi022 ceph-mon[98615]: osdmap e299: 8 total, 8 up, 8 in 2024-09-21T09:35:21.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:20 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:21.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:20 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:21.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:20 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:21.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:20 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:21.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:20 smithi022 ceph-mon[96736]: osdmap e299: 8 total, 8 up, 8 in 2024-09-21T09:35:21.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:20 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:21.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:20 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:21.120 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:20 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:21.121 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:20 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:21.121 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:20 smithi064 ceph-mon[86709]: osdmap e299: 8 total, 8 up, 8 in 2024-09-21T09:35:21.121 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:20 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:21.121 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:20 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:21.949 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 47 v 673 (ObjNum 365 snap 115 seq_num 365) dirty exists 2024-09-21T09:35:21.949 INFO:tasks.rados.rados.0.smithi022.stdout:1165: done (1 left) 2024-09-21T09:35:21.949 INFO:tasks.rados.rados.0.smithi022.stdout:1167: done (0 left) 2024-09-21T09:35:21.949 INFO:tasks.rados.rados.0.smithi022.stdout:1168: delete oid 38 current snap is 121 2024-09-21T09:35:21.960 INFO:tasks.rados.rados.0.smithi022.stdout:1168: done (0 left) 2024-09-21T09:35:21.960 INFO:tasks.rados.rados.0.smithi022.stdout:1169: rollback oid 3 current snap is 121 2024-09-21T09:35:21.960 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 3 to 115 2024-09-21T09:35:21.960 INFO:tasks.rados.rados.0.smithi022.stdout:1170: setattr oid 37 current snap is 121 2024-09-21T09:35:21.963 INFO:tasks.rados.rados.0.smithi022.stdout:1169: finishing rollback tid 1 to smithi02284960-3 2024-09-21T09:35:21.963 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 3 v 136 (ObjNum 161 snap 0 seq_num 1) dirty dne 2024-09-21T09:35:21.963 INFO:tasks.rados.rados.0.smithi022.stdout:1169: done (1 left) 2024-09-21T09:35:21.963 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 14 current snap is 121 2024-09-21T09:35:21.963 INFO:tasks.rados.rados.0.smithi022.stdout:1171: seq_num 384 ranges {2416640=434176,2850816=475136,3325952=630784,3956736=98304} 2024-09-21T09:35:21.980 INFO:tasks.rados.rados.0.smithi022.stdout:1171: writing smithi02284960-14 from 2416640 to 2850816 tid 1 2024-09-21T09:35:21.983 INFO:tasks.rados.rados.0.smithi022.stdout:1171: writing smithi02284960-14 from 2850816 to 3325952 tid 2 2024-09-21T09:35:21.988 INFO:tasks.rados.rados.0.smithi022.stdout:1171: writing smithi02284960-14 from 3325952 to 3956736 tid 3 2024-09-21T09:35:21.990 INFO:tasks.rados.rados.0.smithi022.stdout:1171: writing smithi02284960-14 from 3956736 to 4055040 tid 4 2024-09-21T09:35:21.990 INFO:tasks.rados.rados.0.smithi022.stdout:1172: snap_create 2024-09-21T09:35:21.990 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 37 v 729 (ObjNum 355 snap 112 seq_num 355) dirty exists 2024-09-21T09:35:21.990 INFO:tasks.rados.rados.0.smithi022.stdout:1171: finishing write tid 1 to smithi02284960-14 2024-09-21T09:35:21.990 INFO:tasks.rados.rados.0.smithi022.stdout:1171: finishing write tid 2 to smithi02284960-14 2024-09-21T09:35:21.992 INFO:tasks.rados.rados.0.smithi022.stdout:1171: finishing write tid 3 to smithi02284960-14 2024-09-21T09:35:21.992 INFO:tasks.rados.rados.0.smithi022.stdout:1171: finishing write tid 4 to smithi02284960-14 2024-09-21T09:35:21.992 INFO:tasks.rados.rados.0.smithi022.stdout:1171: finishing write tid 5 to smithi02284960-14 2024-09-21T09:35:21.995 INFO:tasks.rados.rados.0.smithi022.stdout:1171: finishing write tid 6 to smithi02284960-14 2024-09-21T09:35:21.995 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 14 v 574 (ObjNum 384 snap 121 seq_num 384) dirty exists 2024-09-21T09:35:21.995 INFO:tasks.rados.rados.0.smithi022.stdout:1171: left oid 14 (ObjNum 384 snap 121 seq_num 384) 2024-09-21T09:35:22.158 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:21 smithi064 ceph-mon[86709]: pgmap v52: 121 pgs: 121 active+clean; 260 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 6.0 MiB/s rd, 151 KiB/s wr, 1 op/s 2024-09-21T09:35:22.159 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:21 smithi064 ceph-mon[86709]: osdmap e300: 8 total, 8 up, 8 in 2024-09-21T09:35:22.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:21 smithi022 ceph-mon[98615]: pgmap v52: 121 pgs: 121 active+clean; 260 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 6.0 MiB/s rd, 151 KiB/s wr, 1 op/s 2024-09-21T09:35:22.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:21 smithi022 ceph-mon[98615]: osdmap e300: 8 total, 8 up, 8 in 2024-09-21T09:35:22.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:21 smithi022 ceph-mon[96736]: pgmap v52: 121 pgs: 121 active+clean; 260 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 6.0 MiB/s rd, 151 KiB/s wr, 1 op/s 2024-09-21T09:35:22.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:21 smithi022 ceph-mon[96736]: osdmap e300: 8 total, 8 up, 8 in 2024-09-21T09:35:22.955 INFO:tasks.rados.rados.0.smithi022.stdout:1170: done (2 left) 2024-09-21T09:35:22.955 INFO:tasks.rados.rados.0.smithi022.stdout:1171: done (1 left) 2024-09-21T09:35:22.955 INFO:tasks.rados.rados.0.smithi022.stdout:1172: done (0 left) 2024-09-21T09:35:22.955 INFO:tasks.rados.rados.0.smithi022.stdout:1173: read oid 7 snap 100 2024-09-21T09:35:22.955 INFO:tasks.rados.rados.0.smithi022.stdout:1173: expect deleted 2024-09-21T09:35:22.955 INFO:tasks.rados.rados.0.smithi022.stdout:1174: rmattr oid 2 current snap is 122 2024-09-21T09:35:22.955 INFO:tasks.rados.rados.0.smithi022.stdout:1175: copy_from oid 46 from oid 47 current snap is 122 2024-09-21T09:35:22.955 INFO:tasks.rados.rados.0.smithi022.stdout:1176: copy_from oid 38 from oid 20 current snap is 122 2024-09-21T09:35:22.955 INFO:tasks.rados.rados.0.smithi022.stdout:1177: copy_from oid 44 from oid 39 current snap is 122 2024-09-21T09:35:22.955 INFO:tasks.rados.rados.0.smithi022.stdout:1178: delete oid 25 current snap is 122 2024-09-21T09:35:22.958 INFO:tasks.rados.rados.0.smithi022.stdout:1173: done (5 left) 2024-09-21T09:35:22.958 INFO:tasks.rados.rados.0.smithi022.stdout:1178: done (4 left) 2024-09-21T09:35:22.959 INFO:tasks.rados.rados.0.smithi022.stdout:1179: rollback oid 5 current snap is 122 2024-09-21T09:35:22.959 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 5 to 120 2024-09-21T09:35:22.959 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 16 current snap is 122 2024-09-21T09:35:22.959 INFO:tasks.rados.rados.0.smithi022.stdout:1180: seq_num 386 ranges {0=466944,466944=638976,1105920=565248} 2024-09-21T09:35:22.962 INFO:tasks.rados.rados.0.smithi022.stdout:1180: writing smithi02284960-16 from 0 to 466944 tid 1 2024-09-21T09:35:22.966 INFO:tasks.rados.rados.0.smithi022.stdout:1180: writing smithi02284960-16 from 466944 to 1105920 tid 2 2024-09-21T09:35:22.970 INFO:tasks.rados.rados.0.smithi022.stdout:1180: writing smithi02284960-16 from 1105920 to 1671168 tid 3 2024-09-21T09:35:22.971 INFO:tasks.rados.rados.0.smithi022.stdout:1181: snap_remove snap 72 2024-09-21T09:35:23.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:22 smithi064 ceph-mon[86709]: osdmap e301: 8 total, 8 up, 8 in 2024-09-21T09:35:23.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:22 smithi022 ceph-mon[98615]: osdmap e301: 8 total, 8 up, 8 in 2024-09-21T09:35:23.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:22 smithi022 ceph-mon[96736]: osdmap e301: 8 total, 8 up, 8 in 2024-09-21T09:35:23.961 INFO:tasks.rados.rados.0.smithi022.stdout:1181: done (6 left) 2024-09-21T09:35:23.961 INFO:tasks.rados.rados.0.smithi022.stdout:1182: snap_remove snap 118 2024-09-21T09:35:24.260 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:23 smithi022 ceph-mon[96736]: pgmap v55: 121 pgs: 121 active+clean; 260 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:35:24.261 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:23 smithi022 ceph-mon[96736]: osdmap e302: 8 total, 8 up, 8 in 2024-09-21T09:35:24.261 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:23 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:24.261 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:23 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:24.261 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:23 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:35:24.261 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:23 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:35:24.261 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:23 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:24.261 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:23 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-21T09:35:24.261 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:23 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-21T09:35:24.261 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:23 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:35:24.262 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:23 smithi022 ceph-mon[98615]: pgmap v55: 121 pgs: 121 active+clean; 260 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:35:24.262 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:23 smithi022 ceph-mon[98615]: osdmap e302: 8 total, 8 up, 8 in 2024-09-21T09:35:24.262 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:23 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:24.262 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:23 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:24.262 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:23 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:35:24.262 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:23 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:35:24.262 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:23 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:24.262 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:23 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-21T09:35:24.262 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:23 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-21T09:35:24.262 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:23 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:35:24.442 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:35:24 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:35:24.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:35:24.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:23 smithi064 ceph-mon[86709]: pgmap v55: 121 pgs: 121 active+clean; 260 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:35:24.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:23 smithi064 ceph-mon[86709]: osdmap e302: 8 total, 8 up, 8 in 2024-09-21T09:35:24.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:23 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:24.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:23 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:24.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:23 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:35:24.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:23 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:35:24.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:23 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:24.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:23 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-21T09:35:24.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:23 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-21T09:35:24.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:23 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:35:24.967 INFO:tasks.rados.rados.0.smithi022.stdout:1182: done (6 left) 2024-09-21T09:35:24.967 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 30 current snap is 122 2024-09-21T09:35:24.967 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 2 v 294 (ObjNum 301 snap 88 seq_num 301) dirty exists 2024-09-21T09:35:24.967 INFO:tasks.rados.rados.0.smithi022.stdout:1183: seq_num 387 ranges {2088960=483328,2572288=524288,3096576=679936} 2024-09-21T09:35:24.980 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:24 smithi022 ceph-mon[98615]: Reconfiguring mon.a (monmap changed)... 2024-09-21T09:35:24.981 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:24 smithi022 ceph-mon[98615]: Reconfiguring daemon mon.a on smithi022 2024-09-21T09:35:24.981 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:24 smithi022 ceph-mon[98615]: osdmap e303: 8 total, 8 up, 8 in 2024-09-21T09:35:24.981 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:24 smithi022 ceph-mon[96736]: Reconfiguring mon.a (monmap changed)... 2024-09-21T09:35:24.981 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:24 smithi022 ceph-mon[96736]: Reconfiguring daemon mon.a on smithi022 2024-09-21T09:35:24.981 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:24 smithi022 ceph-mon[96736]: osdmap e303: 8 total, 8 up, 8 in 2024-09-21T09:35:24.981 INFO:tasks.rados.rados.0.smithi022.stdout:1183: writing smithi02284960-30 from 2088960 to 2572288 tid 1 2024-09-21T09:35:24.984 INFO:tasks.rados.rados.0.smithi022.stdout:1183: writing smithi02284960-30 from 2572288 to 3096576 tid 2 2024-09-21T09:35:24.989 INFO:tasks.rados.rados.0.smithi022.stdout:1183: writing smithi02284960-30 from 3096576 to 3776512 tid 3 2024-09-21T09:35:24.989 INFO:tasks.rados.rados.0.smithi022.stdout:1174: done (6 left) 2024-09-21T09:35:24.990 INFO:tasks.rados.rados.0.smithi022.stdout:1184: read oid 11 snap -1 2024-09-21T09:35:24.990 INFO:tasks.rados.rados.0.smithi022.stdout:1184: expect deleted 2024-09-21T09:35:24.990 INFO:tasks.rados.rados.0.smithi022.stdout:1185: delete oid 15 current snap is 122 2024-09-21T09:35:24.990 INFO:tasks.rados.rados.0.smithi022.stdout:1179: finishing rollback tid 0 to smithi02284960-5 2024-09-21T09:35:24.990 INFO:tasks.rados.rados.0.smithi022.stdout:1179: finishing rollback tid 1 to smithi02284960-5 2024-09-21T09:35:24.990 INFO:tasks.rados.rados.0.smithi022.stdout:1179: finishing rollback tid 2 to smithi02284960-5 2024-09-21T09:35:24.990 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 5 v 473 (ObjNum 363 snap 114 seq_num 363) dirty exists 2024-09-21T09:35:24.990 INFO:tasks.rados.rados.0.smithi022.stdout:1180: finishing write tid 1 to smithi02284960-16 2024-09-21T09:35:24.990 INFO:tasks.rados.rados.0.smithi022.stdout:1175: finishing copy_from racing read to smithi02284960-46 2024-09-21T09:35:24.990 INFO:tasks.rados.rados.0.smithi022.stdout:1176: finishing copy_from racing read to smithi02284960-38 2024-09-21T09:35:24.990 INFO:tasks.rados.rados.0.smithi022.stdout:1180: finishing write tid 2 to smithi02284960-16 2024-09-21T09:35:24.990 INFO:tasks.rados.rados.0.smithi022.stdout:1180: finishing write tid 3 to smithi02284960-16 2024-09-21T09:35:24.990 INFO:tasks.rados.rados.0.smithi022.stdout:1176: finishing copy_from to smithi02284960-38 2024-09-21T09:35:24.991 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 38 v 465 (ObjNum 355 snap 112 seq_num 355) dirty exists 2024-09-21T09:35:24.991 INFO:tasks.rados.rados.0.smithi022.stdout:1180: finishing write tid 4 to smithi02284960-16 2024-09-21T09:35:24.991 INFO:tasks.rados.rados.0.smithi022.stdout:1180: finishing write tid 5 to smithi02284960-16 2024-09-21T09:35:24.991 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 16 v 306 (ObjNum 386 snap 122 seq_num 386) dirty exists 2024-09-21T09:35:24.991 INFO:tasks.rados.rados.0.smithi022.stdout:1180: left oid 16 (ObjNum 386 snap 122 seq_num 386) 2024-09-21T09:35:24.991 INFO:tasks.rados.rados.0.smithi022.stdout:1177: finishing copy_from racing read to smithi02284960-44 2024-09-21T09:35:24.991 INFO:tasks.rados.rados.0.smithi022.stdout:1175: finishing copy_from to smithi02284960-46 2024-09-21T09:35:24.991 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 46 v 673 (ObjNum 365 snap 115 seq_num 365) dirty exists 2024-09-21T09:35:24.991 INFO:tasks.rados.rados.0.smithi022.stdout:1177: finishing copy_from to smithi02284960-44 2024-09-21T09:35:24.991 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 44 v 469 (ObjNum 360 snap 114 seq_num 360) dirty exists 2024-09-21T09:35:24.991 INFO:tasks.rados.rados.0.smithi022.stdout:1183: finishing write tid 1 to smithi02284960-30 2024-09-21T09:35:24.991 INFO:tasks.rados.rados.0.smithi022.stdout:1183: finishing write tid 2 to smithi02284960-30 2024-09-21T09:35:24.994 INFO:tasks.rados.rados.0.smithi022.stdout:1183: finishing write tid 3 to smithi02284960-30 2024-09-21T09:35:24.994 INFO:tasks.rados.rados.0.smithi022.stdout:1183: finishing write tid 4 to smithi02284960-30 2024-09-21T09:35:24.996 INFO:tasks.rados.rados.0.smithi022.stdout:1183: finishing write tid 5 to smithi02284960-30 2024-09-21T09:35:24.996 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 30 v 473 (ObjNum 387 snap 122 seq_num 387) dirty exists 2024-09-21T09:35:24.996 INFO:tasks.rados.rados.0.smithi022.stdout:1183: left oid 30 (ObjNum 387 snap 122 seq_num 387) 2024-09-21T09:35:24.997 INFO:tasks.rados.rados.0.smithi022.stdout:1175: done (7 left) 2024-09-21T09:35:24.998 INFO:tasks.rados.rados.0.smithi022.stdout:1176: done (6 left) 2024-09-21T09:35:24.998 INFO:tasks.rados.rados.0.smithi022.stdout:1177: done (5 left) 2024-09-21T09:35:24.998 INFO:tasks.rados.rados.0.smithi022.stdout:1179: done (4 left) 2024-09-21T09:35:24.998 INFO:tasks.rados.rados.0.smithi022.stdout:1180: done (3 left) 2024-09-21T09:35:24.998 INFO:tasks.rados.rados.0.smithi022.stdout:1183: done (2 left) 2024-09-21T09:35:24.998 INFO:tasks.rados.rados.0.smithi022.stdout:1184: done (1 left) 2024-09-21T09:35:24.998 INFO:tasks.rados.rados.0.smithi022.stdout:1185: done (0 left) 2024-09-21T09:35:24.998 INFO:tasks.rados.rados.0.smithi022.stdout:1186: rollback oid 41 current snap is 122 2024-09-21T09:35:24.998 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 41 to 69 2024-09-21T09:35:24.998 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 25 current snap is 122 2024-09-21T09:35:24.998 INFO:tasks.rados.rados.0.smithi022.stdout:1187: seq_num 389 ranges {0=524288,524288=745472,1269760=458752} 2024-09-21T09:35:25.001 INFO:tasks.rados.rados.0.smithi022.stdout:1187: writing smithi02284960-25 from 0 to 524288 tid 1 2024-09-21T09:35:25.006 INFO:tasks.rados.rados.0.smithi022.stdout:1187: writing smithi02284960-25 from 524288 to 1269760 tid 2 2024-09-21T09:35:25.010 INFO:tasks.rados.rados.0.smithi022.stdout:1187: writing smithi02284960-25 from 1269760 to 1728512 tid 3 2024-09-21T09:35:25.010 INFO:tasks.rados.rados.0.smithi022.stdout:1188: rollback oid 49 current snap is 122 2024-09-21T09:35:25.010 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 49 to 104 2024-09-21T09:35:25.011 INFO:tasks.rados.rados.0.smithi022.stdout:1186: finishing rollback tid 0 to smithi02284960-41 2024-09-21T09:35:25.011 INFO:tasks.rados.rados.0.smithi022.stdout:1186: finishing rollback tid 1 to smithi02284960-41 2024-09-21T09:35:25.011 INFO:tasks.rados.rados.0.smithi022.stdout:1186: finishing rollback tid 2 to smithi02284960-41 2024-09-21T09:35:25.011 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 41 v 657 (ObjNum 247 snap 67 seq_num 247) dirty exists 2024-09-21T09:35:25.011 INFO:tasks.rados.rados.0.smithi022.stdout:1187: finishing write tid 1 to smithi02284960-25 2024-09-21T09:35:25.011 INFO:tasks.rados.rados.0.smithi022.stdout:1187: finishing write tid 2 to smithi02284960-25 2024-09-21T09:35:25.011 INFO:tasks.rados.rados.0.smithi022.stdout:1186: done (2 left) 2024-09-21T09:35:25.012 INFO:tasks.rados.rados.0.smithi022.stdout:1189: delete oid 21 current snap is 122 2024-09-21T09:35:25.012 INFO:tasks.rados.rados.0.smithi022.stdout:1189: done (2 left) 2024-09-21T09:35:25.012 INFO:tasks.rados.rados.0.smithi022.stdout:1190: delete oid 41 current snap is 122 2024-09-21T09:35:25.013 INFO:tasks.rados.rados.0.smithi022.stdout:1188: finishing rollback tid 0 to smithi02284960-49 2024-09-21T09:35:25.013 INFO:tasks.rados.rados.0.smithi022.stdout:1188: finishing rollback tid 1 to smithi02284960-49 2024-09-21T09:35:25.014 INFO:tasks.rados.rados.0.smithi022.stdout:1188: finishing rollback tid 2 to smithi02284960-49 2024-09-21T09:35:25.014 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 49 v 577 (ObjNum 322 snap 97 seq_num 322) dirty exists 2024-09-21T09:35:25.014 INFO:tasks.rados.rados.0.smithi022.stdout:1187: finishing write tid 3 to smithi02284960-25 2024-09-21T09:35:25.014 INFO:tasks.rados.rados.0.smithi022.stdout:1187: finishing write tid 4 to smithi02284960-25 2024-09-21T09:35:25.015 INFO:tasks.rados.rados.0.smithi022.stdout:1188: done (2 left) 2024-09-21T09:35:25.015 INFO:tasks.rados.rados.0.smithi022.stdout:1190: done (1 left) 2024-09-21T09:35:25.015 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 28 current snap is 122 2024-09-21T09:35:25.015 INFO:tasks.rados.rados.0.smithi022.stdout:1191: seq_num 392 ranges {0=573440,573440=802816,1376256=401408} 2024-09-21T09:35:25.018 INFO:tasks.rados.rados.0.smithi022.stdout:1191: writing smithi02284960-28 from 0 to 573440 tid 1 2024-09-21T09:35:25.023 INFO:tasks.rados.rados.0.smithi022.stdout:1191: writing smithi02284960-28 from 573440 to 1376256 tid 2 2024-09-21T09:35:25.026 INFO:tasks.rados.rados.0.smithi022.stdout:1191: writing smithi02284960-28 from 1376256 to 1777664 tid 3 2024-09-21T09:35:25.026 INFO:tasks.rados.rados.0.smithi022.stdout:1192: rmattr oid 19 current snap is 122 2024-09-21T09:35:25.026 INFO:tasks.rados.rados.0.smithi022.stdout:1187: finishing write tid 5 to smithi02284960-25 2024-09-21T09:35:25.026 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 25 v 661 (ObjNum 389 snap 122 seq_num 389) dirty exists 2024-09-21T09:35:25.026 INFO:tasks.rados.rados.0.smithi022.stdout:1187: left oid 25 (ObjNum 389 snap 122 seq_num 389) 2024-09-21T09:35:25.026 INFO:tasks.rados.rados.0.smithi022.stdout:1191: finishing write tid 1 to smithi02284960-28 2024-09-21T09:35:25.026 INFO:tasks.rados.rados.0.smithi022.stdout:1187: done (2 left) 2024-09-21T09:35:25.026 INFO:tasks.rados.rados.0.smithi022.stdout:1193: read oid 36 snap 69 2024-09-21T09:35:25.026 INFO:tasks.rados.rados.0.smithi022.stdout:1193: expect (ObjNum 188 snap 47 seq_num 188) 2024-09-21T09:35:25.026 INFO:tasks.rados.rados.0.smithi022.stdout:1194: snap_remove snap 114 2024-09-21T09:35:25.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:24 smithi064 ceph-mon[86709]: Reconfiguring mon.a (monmap changed)... 2024-09-21T09:35:25.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:24 smithi064 ceph-mon[86709]: Reconfiguring daemon mon.a on smithi022 2024-09-21T09:35:25.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:24 smithi064 ceph-mon[86709]: osdmap e303: 8 total, 8 up, 8 in 2024-09-21T09:35:25.964 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:35:25 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:35:25] "GET /metrics HTTP/1.1" 200 36117 "" "Prometheus/2.51.0" 2024-09-21T09:35:25.964 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:25 smithi022 ceph-mon[96736]: pgmap v58: 121 pgs: 121 active+clean; 267 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 511 B/s rd, 2.2 MiB/s wr, 13 op/s 2024-09-21T09:35:25.965 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:25 smithi022 ceph-mon[96736]: osdmap e304: 8 total, 8 up, 8 in 2024-09-21T09:35:25.965 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:25 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:25.965 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:25 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:25.972 INFO:tasks.rados.rados.0.smithi022.stdout:1194: done (3 left) 2024-09-21T09:35:25.972 INFO:tasks.rados.rados.0.smithi022.stdout:1195: read oid 29 snap -1 2024-09-21T09:35:25.972 INFO:tasks.rados.rados.0.smithi022.stdout:1195: expect deleted 2024-09-21T09:35:25.972 INFO:tasks.rados.rados.0.smithi022.stdout:1196: snap_remove snap 101 2024-09-21T09:35:26.191 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:35:25 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:35:25.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:35:26.192 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:35:25 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:35:25.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:35:26.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:25 smithi064 ceph-mon[86709]: pgmap v58: 121 pgs: 121 active+clean; 267 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 511 B/s rd, 2.2 MiB/s wr, 13 op/s 2024-09-21T09:35:26.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:25 smithi064 ceph-mon[86709]: osdmap e304: 8 total, 8 up, 8 in 2024-09-21T09:35:26.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:25 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:26.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:25 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:26.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:25 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-21T09:35:26.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:25 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-09-21T09:35:26.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:25 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:35:26.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:25 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-21T09:35:26.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:25 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-09-21T09:35:26.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:25 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:35:26.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:25 smithi022 ceph-mon[98615]: pgmap v58: 121 pgs: 121 active+clean; 267 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 511 B/s rd, 2.2 MiB/s wr, 13 op/s 2024-09-21T09:35:26.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:25 smithi022 ceph-mon[98615]: osdmap e304: 8 total, 8 up, 8 in 2024-09-21T09:35:26.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:25 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:26.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:25 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:26.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:25 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-21T09:35:26.310 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:25 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-09-21T09:35:26.310 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:25 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:35:26.977 INFO:tasks.rados.rados.0.smithi022.stdout:1196: done (4 left) 2024-09-21T09:35:26.978 INFO:tasks.rados.rados.0.smithi022.stdout:1197: rollback oid 22 current snap is 122 2024-09-21T09:35:26.978 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 22 to 84 2024-09-21T09:35:26.978 INFO:tasks.rados.rados.0.smithi022.stdout:1198: snap_create 2024-09-21T09:35:26.978 INFO:tasks.rados.rados.0.smithi022.stdout:1191: finishing write tid 2 to smithi02284960-28 2024-09-21T09:35:26.978 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 19 v 745 (ObjNum 370 snap 115 seq_num 370) dirty exists 2024-09-21T09:35:26.978 INFO:tasks.rados.rados.0.smithi022.stdout:1191: finishing write tid 3 to smithi02284960-28 2024-09-21T09:35:26.978 INFO:tasks.rados.rados.0.smithi022.stdout:1191: finishing write tid 4 to smithi02284960-28 2024-09-21T09:35:26.978 INFO:tasks.rados.rados.0.smithi022.stdout:1193: expect (ObjNum 188 snap 47 seq_num 188) 2024-09-21T09:35:27.032 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:26 smithi022 ceph-mon[98615]: Reconfiguring mgr.y (monmap changed)... 2024-09-21T09:35:27.032 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:26 smithi022 ceph-mon[98615]: Reconfiguring daemon mgr.y on smithi022 2024-09-21T09:35:27.032 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:26 smithi022 ceph-mon[98615]: osdmap e305: 8 total, 8 up, 8 in 2024-09-21T09:35:27.032 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:26 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:27.033 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:26 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:27.033 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:26 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-21T09:35:27.033 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:26 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-21T09:35:27.033 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:26 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:35:27.033 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:26 smithi022 ceph-mon[96736]: Reconfiguring mgr.y (monmap changed)... 2024-09-21T09:35:27.033 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:26 smithi022 ceph-mon[96736]: Reconfiguring daemon mgr.y on smithi022 2024-09-21T09:35:27.033 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:26 smithi022 ceph-mon[96736]: osdmap e305: 8 total, 8 up, 8 in 2024-09-21T09:35:27.033 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:26 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:27.033 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:26 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:27.033 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:26 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-21T09:35:27.033 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:26 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-21T09:35:27.033 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:26 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:35:27.052 INFO:tasks.rados.rados.0.smithi022.stdout:1191: finishing write tid 5 to smithi02284960-28 2024-09-21T09:35:27.052 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 28 v 540 (ObjNum 392 snap 122 seq_num 392) dirty exists 2024-09-21T09:35:27.052 INFO:tasks.rados.rados.0.smithi022.stdout:1191: left oid 28 (ObjNum 392 snap 122 seq_num 392) 2024-09-21T09:35:27.052 INFO:tasks.rados.rados.0.smithi022.stdout:1197: finishing rollback tid 0 to smithi02284960-22 2024-09-21T09:35:27.052 INFO:tasks.rados.rados.0.smithi022.stdout:1197: finishing rollback tid 1 to smithi02284960-22 2024-09-21T09:35:27.053 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 22 v 565 (ObjNum 4113 snap 0 seq_num 3570707360) dirty dne 2024-09-21T09:35:27.441 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:35:26 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:35:26.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi022\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:35:27.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:26 smithi064 ceph-mon[86709]: Reconfiguring mgr.y (monmap changed)... 2024-09-21T09:35:27.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:26 smithi064 ceph-mon[86709]: Reconfiguring daemon mgr.y on smithi022 2024-09-21T09:35:27.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:26 smithi064 ceph-mon[86709]: osdmap e305: 8 total, 8 up, 8 in 2024-09-21T09:35:27.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:26 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:27.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:26 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:27.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:26 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-21T09:35:27.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:26 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-21T09:35:27.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:26 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:35:27.977 INFO:tasks.rados.rados.0.smithi022.stdout:1191: done (5 left) 2024-09-21T09:35:27.977 INFO:tasks.rados.rados.0.smithi022.stdout:1192: done (4 left) 2024-09-21T09:35:27.977 INFO:tasks.rados.rados.0.smithi022.stdout:1193: done (3 left) 2024-09-21T09:35:27.977 INFO:tasks.rados.rados.0.smithi022.stdout:1195: done (2 left) 2024-09-21T09:35:27.977 INFO:tasks.rados.rados.0.smithi022.stdout:1197: done (1 left) 2024-09-21T09:35:27.977 INFO:tasks.rados.rados.0.smithi022.stdout:1198: done (0 left) 2024-09-21T09:35:27.977 INFO:tasks.rados.rados.0.smithi022.stdout:1199: snap_create 2024-09-21T09:35:28.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:27 smithi022 ceph-mon[96736]: pgmap v61: 121 pgs: 121 active+clean; 267 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 511 B/s rd, 2.2 MiB/s wr, 13 op/s 2024-09-21T09:35:28.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:27 smithi022 ceph-mon[96736]: Reconfiguring mon.c (monmap changed)... 2024-09-21T09:35:28.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:27 smithi022 ceph-mon[96736]: Reconfiguring daemon mon.c on smithi022 2024-09-21T09:35:28.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:27 smithi022 ceph-mon[96736]: osdmap e306: 8 total, 8 up, 8 in 2024-09-21T09:35:28.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:27 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:35:28.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:27 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:28.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:27 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:28.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:27 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-21T09:35:28.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:27 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:35:28.059 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:27 smithi022 ceph-mon[98615]: pgmap v61: 121 pgs: 121 active+clean; 267 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 511 B/s rd, 2.2 MiB/s wr, 13 op/s 2024-09-21T09:35:28.059 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:27 smithi022 ceph-mon[98615]: Reconfiguring mon.c (monmap changed)... 2024-09-21T09:35:28.059 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:27 smithi022 ceph-mon[98615]: Reconfiguring daemon mon.c on smithi022 2024-09-21T09:35:28.059 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:27 smithi022 ceph-mon[98615]: osdmap e306: 8 total, 8 up, 8 in 2024-09-21T09:35:28.059 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:27 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:35:28.059 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:27 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:28.059 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:27 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:28.059 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:27 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-21T09:35:28.059 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:27 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:35:28.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:27 smithi064 ceph-mon[86709]: pgmap v61: 121 pgs: 121 active+clean; 267 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 511 B/s rd, 2.2 MiB/s wr, 13 op/s 2024-09-21T09:35:28.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:27 smithi064 ceph-mon[86709]: Reconfiguring mon.c (monmap changed)... 2024-09-21T09:35:28.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:27 smithi064 ceph-mon[86709]: Reconfiguring daemon mon.c on smithi022 2024-09-21T09:35:28.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:27 smithi064 ceph-mon[86709]: osdmap e306: 8 total, 8 up, 8 in 2024-09-21T09:35:28.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:27 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:35:28.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:27 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:28.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:27 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:28.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:27 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-21T09:35:28.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:27 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:35:28.944 INFO:tasks.rados.rados.0.smithi022.stdout:1199: done (0 left) 2024-09-21T09:35:28.944 INFO:tasks.rados.rados.0.smithi022.stdout:1200: delete oid 39 current snap is 124 2024-09-21T09:35:28.949 INFO:tasks.rados.rados.0.smithi022.stdout:1200: done (0 left) 2024-09-21T09:35:28.949 INFO:tasks.rados.rados.0.smithi022.stdout:1201: rollback oid 33 current snap is 124 2024-09-21T09:35:28.949 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 33 to 100 2024-09-21T09:35:28.949 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 12 current snap is 124 2024-09-21T09:35:28.949 INFO:tasks.rados.rados.0.smithi022.stdout:1202: seq_num 394 ranges {1933312=606208,2539520=565248,3104768=557056,3661824=81920} 2024-09-21T09:35:28.962 INFO:tasks.rados.rados.0.smithi022.stdout:1202: writing smithi02284960-12 from 1933312 to 2539520 tid 1 2024-09-21T09:35:28.966 INFO:tasks.rados.rados.0.smithi022.stdout:1202: writing smithi02284960-12 from 2539520 to 3104768 tid 2 2024-09-21T09:35:28.969 INFO:tasks.rados.rados.0.smithi022.stdout:1202: writing smithi02284960-12 from 3104768 to 3661824 tid 3 2024-09-21T09:35:28.970 INFO:tasks.rados.rados.0.smithi022.stdout:1202: writing smithi02284960-12 from 3661824 to 3743744 tid 4 2024-09-21T09:35:28.970 INFO:tasks.rados.rados.0.smithi022.stdout:1203: rollback oid 42 current snap is 124 2024-09-21T09:35:28.970 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 42 to 121 2024-09-21T09:35:28.970 INFO:tasks.rados.rados.0.smithi022.stdout:1201: finishing rollback tid 0 to smithi02284960-33 2024-09-21T09:35:28.970 INFO:tasks.rados.rados.0.smithi022.stdout:1201: finishing rollback tid 1 to smithi02284960-33 2024-09-21T09:35:28.970 INFO:tasks.rados.rados.0.smithi022.stdout:1201: finishing rollback tid 2 to smithi02284960-33 2024-09-21T09:35:28.970 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 33 v 736 (ObjNum 309 snap 91 seq_num 309) dirty exists 2024-09-21T09:35:28.970 INFO:tasks.rados.rados.0.smithi022.stdout:1201: done (2 left) 2024-09-21T09:35:28.971 INFO:tasks.rados.rados.0.smithi022.stdout:1204: snap_remove snap 86 2024-09-21T09:35:29.057 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:28 smithi022 ceph-mon[96736]: Reconfiguring osd.0 (monmap changed)... 2024-09-21T09:35:29.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:28 smithi022 ceph-mon[96736]: Reconfiguring daemon osd.0 on smithi022 2024-09-21T09:35:29.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:28 smithi022 ceph-mon[96736]: osdmap e307: 8 total, 8 up, 8 in 2024-09-21T09:35:29.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:28 smithi022 ceph-mon[96736]: osdmap e308: 8 total, 8 up, 8 in 2024-09-21T09:35:29.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:28 smithi022 ceph-mon[98615]: Reconfiguring osd.0 (monmap changed)... 2024-09-21T09:35:29.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:28 smithi022 ceph-mon[98615]: Reconfiguring daemon osd.0 on smithi022 2024-09-21T09:35:29.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:28 smithi022 ceph-mon[98615]: osdmap e307: 8 total, 8 up, 8 in 2024-09-21T09:35:29.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:28 smithi022 ceph-mon[98615]: osdmap e308: 8 total, 8 up, 8 in 2024-09-21T09:35:29.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:28 smithi064 ceph-mon[86709]: Reconfiguring osd.0 (monmap changed)... 2024-09-21T09:35:29.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:28 smithi064 ceph-mon[86709]: Reconfiguring daemon osd.0 on smithi022 2024-09-21T09:35:29.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:28 smithi064 ceph-mon[86709]: osdmap e307: 8 total, 8 up, 8 in 2024-09-21T09:35:29.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:28 smithi064 ceph-mon[86709]: osdmap e308: 8 total, 8 up, 8 in 2024-09-21T09:35:29.945 INFO:tasks.rados.rados.0.smithi022.stdout:1204: done (2 left) 2024-09-21T09:35:29.945 INFO:tasks.rados.rados.0.smithi022.stdout:1205: read oid 8 snap -1 2024-09-21T09:35:29.945 INFO:tasks.rados.rados.0.smithi022.stdout:1205: expect (ObjNum 378 snap 117 seq_num 378) 2024-09-21T09:35:29.946 INFO:tasks.rados.rados.0.smithi022.stdout:1206: setattr oid 25 current snap is 124 2024-09-21T09:35:29.946 INFO:tasks.rados.rados.0.smithi022.stdout:1202: finishing write tid 1 to smithi02284960-12 2024-09-21T09:35:29.946 INFO:tasks.rados.rados.0.smithi022.stdout:1203: finishing rollback tid 1 to smithi02284960-42 2024-09-21T09:35:29.946 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 42 v 0 (ObjNum 161 snap 0 seq_num 3570056064) dirty dne 2024-09-21T09:35:29.946 INFO:tasks.rados.rados.0.smithi022.stdout:1202: finishing write tid 2 to smithi02284960-12 2024-09-21T09:35:29.946 INFO:tasks.rados.rados.0.smithi022.stdout:1202: finishing write tid 3 to smithi02284960-12 2024-09-21T09:35:29.946 INFO:tasks.rados.rados.0.smithi022.stdout:1202: finishing write tid 4 to smithi02284960-12 2024-09-21T09:35:29.946 INFO:tasks.rados.rados.0.smithi022.stdout:1202: finishing write tid 5 to smithi02284960-12 2024-09-21T09:35:29.946 INFO:tasks.rados.rados.0.smithi022.stdout:1202: finishing write tid 6 to smithi02284960-12 2024-09-21T09:35:29.946 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 12 v 763 (ObjNum 394 snap 124 seq_num 394) dirty exists 2024-09-21T09:35:29.946 INFO:tasks.rados.rados.0.smithi022.stdout:1202: left oid 12 (ObjNum 394 snap 124 seq_num 394) 2024-09-21T09:35:29.947 INFO:tasks.rados.rados.0.smithi022.stdout:1202: done (3 left) 2024-09-21T09:35:29.947 INFO:tasks.rados.rados.0.smithi022.stdout:1203: done (2 left) 2024-09-21T09:35:29.947 INFO:tasks.rados.rados.0.smithi022.stdout:1207: delete oid 3 current snap is 124 2024-09-21T09:35:29.952 INFO:tasks.rados.rados.0.smithi022.stdout:1207: done (2 left) 2024-09-21T09:35:29.952 INFO:tasks.rados.rados.0.smithi022.stdout:1208: snap_create 2024-09-21T09:35:29.960 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 25 v 678 (ObjNum 389 snap 122 seq_num 389) dirty exists 2024-09-21T09:35:29.964 INFO:tasks.rados.rados.0.smithi022.stdout:1205: expect (ObjNum 378 snap 117 seq_num 378) 2024-09-21T09:35:30.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:29 smithi022 ceph-mon[98615]: pgmap v64: 121 pgs: 121 active+clean; 267 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:35:30.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:29 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:30.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:29 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:30.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:29 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-21T09:35:30.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:29 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:35:30.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:29 smithi022 ceph-mon[98615]: osdmap e309: 8 total, 8 up, 8 in 2024-09-21T09:35:30.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:29 smithi022 ceph-mon[96736]: pgmap v64: 121 pgs: 121 active+clean; 267 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:35:30.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:29 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:30.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:29 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:30.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:29 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-21T09:35:30.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:29 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:35:30.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:29 smithi022 ceph-mon[96736]: osdmap e309: 8 total, 8 up, 8 in 2024-09-21T09:35:30.441 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:35:30 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:35:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:35:30.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:29 smithi064 ceph-mon[86709]: pgmap v64: 121 pgs: 121 active+clean; 267 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:35:30.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:29 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:30.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:29 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:30.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:29 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-21T09:35:30.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:29 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:35:30.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:29 smithi064 ceph-mon[86709]: osdmap e309: 8 total, 8 up, 8 in 2024-09-21T09:35:30.946 INFO:tasks.rados.rados.0.smithi022.stdout:1205: done (2 left) 2024-09-21T09:35:30.946 INFO:tasks.rados.rados.0.smithi022.stdout:1206: done (1 left) 2024-09-21T09:35:30.946 INFO:tasks.rados.rados.0.smithi022.stdout:1208: done (0 left) 2024-09-21T09:35:30.946 INFO:tasks.rados.rados.0.smithi022.stdout:1209: rollback oid 41 current snap is 125 2024-09-21T09:35:30.946 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 41 to 122 2024-09-21T09:35:30.946 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 4 current snap is 125 2024-09-21T09:35:30.946 INFO:tasks.rados.rados.0.smithi022.stdout:1210: seq_num 396 ranges {4235264=638976,4874240=786432,5660672=417792} 2024-09-21T09:35:30.972 INFO:tasks.rados.rados.0.smithi022.stdout:1210: writing smithi02284960-4 from 4235264 to 4874240 tid 1 2024-09-21T09:35:30.977 INFO:tasks.rados.rados.0.smithi022.stdout:1210: writing smithi02284960-4 from 4874240 to 5660672 tid 2 2024-09-21T09:35:30.980 INFO:tasks.rados.rados.0.smithi022.stdout:1210: writing smithi02284960-4 from 5660672 to 6078464 tid 3 2024-09-21T09:35:30.980 INFO:tasks.rados.rados.0.smithi022.stdout:1211: delete oid 19 current snap is 125 2024-09-21T09:35:30.980 INFO:tasks.rados.rados.0.smithi022.stdout:1209: finishing rollback tid 1 to smithi02284960-41 2024-09-21T09:35:30.980 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 41 v 662 (ObjNum 161 snap 0 seq_num 0) dirty dne 2024-09-21T09:35:30.981 INFO:tasks.rados.rados.0.smithi022.stdout:1210: finishing write tid 1 to smithi02284960-4 2024-09-21T09:35:30.982 INFO:tasks.rados.rados.0.smithi022.stdout:1210: finishing write tid 2 to smithi02284960-4 2024-09-21T09:35:30.983 INFO:tasks.rados.rados.0.smithi022.stdout:1210: finishing write tid 3 to smithi02284960-4 2024-09-21T09:35:30.983 INFO:tasks.rados.rados.0.smithi022.stdout:1210: finishing write tid 4 to smithi02284960-4 2024-09-21T09:35:30.985 INFO:tasks.rados.rados.0.smithi022.stdout:1210: finishing write tid 5 to smithi02284960-4 2024-09-21T09:35:30.985 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 4 v 477 (ObjNum 396 snap 125 seq_num 396) dirty exists 2024-09-21T09:35:30.985 INFO:tasks.rados.rados.0.smithi022.stdout:1210: left oid 4 (ObjNum 396 snap 125 seq_num 396) 2024-09-21T09:35:30.986 INFO:tasks.rados.rados.0.smithi022.stdout:1209: done (2 left) 2024-09-21T09:35:30.986 INFO:tasks.rados.rados.0.smithi022.stdout:1210: done (1 left) 2024-09-21T09:35:30.986 INFO:tasks.rados.rados.0.smithi022.stdout:1211: done (0 left) 2024-09-21T09:35:30.986 INFO:tasks.rados.rados.0.smithi022.stdout:1212: rollback oid 25 current snap is 125 2024-09-21T09:35:30.986 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 25 to 115 2024-09-21T09:35:30.986 INFO:tasks.rados.rados.0.smithi022.stdout:1213: snap_remove snap 75 2024-09-21T09:35:31.271 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:30 smithi022 ceph-mon[98615]: Reconfiguring osd.1 (monmap changed)... 2024-09-21T09:35:31.271 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:30 smithi022 ceph-mon[98615]: Reconfiguring daemon osd.1 on smithi022 2024-09-21T09:35:31.271 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:30 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:31.271 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:30 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:31.271 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:30 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-21T09:35:31.271 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:30 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:35:31.271 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:30 smithi022 ceph-mon[98615]: osdmap e310: 8 total, 8 up, 8 in 2024-09-21T09:35:31.271 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:30 smithi022 ceph-mon[96736]: Reconfiguring osd.1 (monmap changed)... 2024-09-21T09:35:31.272 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:30 smithi022 ceph-mon[96736]: Reconfiguring daemon osd.1 on smithi022 2024-09-21T09:35:31.272 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:30 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:31.272 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:30 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:31.272 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:30 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-21T09:35:31.272 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:30 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:35:31.272 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:30 smithi022 ceph-mon[96736]: osdmap e310: 8 total, 8 up, 8 in 2024-09-21T09:35:31.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:30 smithi064 ceph-mon[86709]: Reconfiguring osd.1 (monmap changed)... 2024-09-21T09:35:31.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:30 smithi064 ceph-mon[86709]: Reconfiguring daemon osd.1 on smithi022 2024-09-21T09:35:31.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:30 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:31.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:30 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:31.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:30 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-21T09:35:31.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:30 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:35:31.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:30 smithi064 ceph-mon[86709]: osdmap e310: 8 total, 8 up, 8 in 2024-09-21T09:35:31.989 INFO:tasks.rados.rados.0.smithi022.stdout:1213: done (1 left) 2024-09-21T09:35:31.990 INFO:tasks.rados.rados.0.smithi022.stdout:1214: read oid 20 snap -1 2024-09-21T09:35:31.990 INFO:tasks.rados.rados.0.smithi022.stdout:1214: expect (ObjNum 355 snap 112 seq_num 355) 2024-09-21T09:35:31.990 INFO:tasks.rados.rados.0.smithi022.stdout:1215: delete oid 4 current snap is 125 2024-09-21T09:35:31.990 INFO:tasks.rados.rados.0.smithi022.stdout:1212: finishing rollback tid 0 to smithi02284960-25 2024-09-21T09:35:31.990 INFO:tasks.rados.rados.0.smithi022.stdout:1212: finishing rollback tid 1 to smithi02284960-25 2024-09-21T09:35:31.990 INFO:tasks.rados.rados.0.smithi022.stdout:1212: finishing rollback tid 2 to smithi02284960-25 2024-09-21T09:35:31.990 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 25 v 689 (ObjNum 345 snap 107 seq_num 345) dirty exists 2024-09-21T09:35:32.000 INFO:tasks.rados.rados.0.smithi022.stdout:1212: done (2 left) 2024-09-21T09:35:32.000 INFO:tasks.rados.rados.0.smithi022.stdout:1215: done (1 left) 2024-09-21T09:35:32.000 INFO:tasks.rados.rados.0.smithi022.stdout:1216: rollback oid 16 current snap is 125 2024-09-21T09:35:32.000 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 16 to 104 2024-09-21T09:35:32.000 INFO:tasks.rados.rados.0.smithi022.stdout:1217: rmattr oid 44 current snap is 125 2024-09-21T09:35:32.000 INFO:tasks.rados.rados.0.smithi022.stdout:1217: done (2 left) 2024-09-21T09:35:32.000 INFO:tasks.rados.rados.0.smithi022.stdout:1218: delete oid 8 current snap is 125 2024-09-21T09:35:32.006 INFO:tasks.rados.rados.0.smithi022.stdout:1214: expect (ObjNum 355 snap 112 seq_num 355) 2024-09-21T09:35:32.082 INFO:tasks.rados.rados.0.smithi022.stdout:1216: finishing rollback tid 0 to smithi02284960-16 2024-09-21T09:35:32.082 INFO:tasks.rados.rados.0.smithi022.stdout:1216: finishing rollback tid 1 to smithi02284960-16 2024-09-21T09:35:32.082 INFO:tasks.rados.rados.0.smithi022.stdout:1216: finishing rollback tid 2 to smithi02284960-16 2024-09-21T09:35:32.082 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 16 v 322 (ObjNum 129 snap 0 seq_num 1693273484) dirty exists 2024-09-21T09:35:32.082 INFO:tasks.rados.rados.0.smithi022.stdout:1214: done (2 left) 2024-09-21T09:35:32.082 INFO:tasks.rados.rados.0.smithi022.stdout:1216: done (1 left) 2024-09-21T09:35:32.082 INFO:tasks.rados.rados.0.smithi022.stdout:1218: done (0 left) 2024-09-21T09:35:32.083 INFO:tasks.rados.rados.0.smithi022.stdout:1219: read oid 26 snap -1 2024-09-21T09:35:32.083 INFO:tasks.rados.rados.0.smithi022.stdout:1219: expect deleted 2024-09-21T09:35:32.083 INFO:tasks.rados.rados.0.smithi022.stdout:1220: snap_create 2024-09-21T09:35:32.184 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:31 smithi022 ceph-mon[98615]: pgmap v67: 121 pgs: 121 active+clean; 273 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 1.1 MiB/s rd, 1.2 MiB/s wr, 8 op/s 2024-09-21T09:35:32.184 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:31 smithi022 ceph-mon[98615]: Reconfiguring osd.2 (monmap changed)... 2024-09-21T09:35:32.184 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:31 smithi022 ceph-mon[98615]: Reconfiguring daemon osd.2 on smithi022 2024-09-21T09:35:32.184 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:31 smithi022 ceph-mon[96736]: pgmap v67: 121 pgs: 121 active+clean; 273 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 1.1 MiB/s rd, 1.2 MiB/s wr, 8 op/s 2024-09-21T09:35:32.184 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:31 smithi022 ceph-mon[96736]: Reconfiguring osd.2 (monmap changed)... 2024-09-21T09:35:32.184 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:31 smithi022 ceph-mon[96736]: Reconfiguring daemon osd.2 on smithi022 2024-09-21T09:35:32.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:31 smithi064 ceph-mon[86709]: pgmap v67: 121 pgs: 121 active+clean; 273 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 1.1 MiB/s rd, 1.2 MiB/s wr, 8 op/s 2024-09-21T09:35:32.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:31 smithi064 ceph-mon[86709]: Reconfiguring osd.2 (monmap changed)... 2024-09-21T09:35:32.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:31 smithi064 ceph-mon[86709]: Reconfiguring daemon osd.2 on smithi022 2024-09-21T09:35:33.001 INFO:tasks.rados.rados.0.smithi022.stdout:1219: done (1 left) 2024-09-21T09:35:33.002 INFO:tasks.rados.rados.0.smithi022.stdout:1220: done (0 left) 2024-09-21T09:35:33.002 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 40 current snap is 126 2024-09-21T09:35:33.002 INFO:tasks.rados.rados.0.smithi022.stdout:1221: seq_num 400 ranges {1966080=704512,2670592=720896,3391488=483328} 2024-09-21T09:35:33.016 INFO:tasks.rados.rados.0.smithi022.stdout:1221: writing smithi02284960-40 from 1966080 to 2670592 tid 1 2024-09-21T09:35:33.020 INFO:tasks.rados.rados.0.smithi022.stdout:1221: writing smithi02284960-40 from 2670592 to 3391488 tid 2 2024-09-21T09:35:33.025 INFO:tasks.rados.rados.0.smithi022.stdout:1221: writing smithi02284960-40 from 3391488 to 3874816 tid 3 2024-09-21T09:35:33.025 INFO:tasks.rados.rados.0.smithi022.stdout:1222: delete oid 2 current snap is 126 2024-09-21T09:35:33.027 INFO:tasks.rados.rados.0.smithi022.stdout:1222: done (1 left) 2024-09-21T09:35:33.027 INFO:tasks.rados.rados.0.smithi022.stdout:1223: snap_remove snap 124 2024-09-21T09:35:33.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:32 smithi022 ceph-mon[96736]: osdmap e311: 8 total, 8 up, 8 in 2024-09-21T09:35:33.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:32 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:33.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:32 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:33.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:32 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-21T09:35:33.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:32 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:35:33.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:32 smithi022 ceph-mon[98615]: osdmap e311: 8 total, 8 up, 8 in 2024-09-21T09:35:33.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:32 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:33.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:32 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:33.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:32 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-21T09:35:33.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:32 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:35:33.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:32 smithi064 ceph-mon[86709]: osdmap e311: 8 total, 8 up, 8 in 2024-09-21T09:35:33.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:32 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:33.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:32 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:33.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:32 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-21T09:35:33.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:32 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:35:33.944 INFO:tasks.rados.rados.0.smithi022.stdout:1223: done (1 left) 2024-09-21T09:35:33.945 INFO:tasks.rados.rados.0.smithi022.stdout:1224: copy_from oid 43 from oid 39 current snap is 126 2024-09-21T09:35:33.945 INFO:tasks.rados.rados.0.smithi022.stdout:1221: finishing write tid 1 to smithi02284960-40 2024-09-21T09:35:33.945 INFO:tasks.rados.rados.0.smithi022.stdout:1221: finishing write tid 2 to smithi02284960-40 2024-09-21T09:35:33.945 INFO:tasks.rados.rados.0.smithi022.stdout:1221: finishing write tid 3 to smithi02284960-40 2024-09-21T09:35:33.945 INFO:tasks.rados.rados.0.smithi022.stdout:1221: finishing write tid 4 to smithi02284960-40 2024-09-21T09:35:33.945 INFO:tasks.rados.rados.0.smithi022.stdout:1221: finishing write tid 5 to smithi02284960-40 2024-09-21T09:35:33.945 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 40 v 702 (ObjNum 400 snap 126 seq_num 400) dirty exists 2024-09-21T09:35:33.945 INFO:tasks.rados.rados.0.smithi022.stdout:1221: left oid 40 (ObjNum 400 snap 126 seq_num 400) 2024-09-21T09:35:33.945 INFO:tasks.rados.rados.0.smithi022.stdout:1221: done (1 left) 2024-09-21T09:35:33.945 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 49 current snap is 126 2024-09-21T09:35:33.945 INFO:tasks.rados.rados.0.smithi022.stdout:1225: seq_num 402 ranges {1081344=737280,1818624=491520,2310144=516096,2826240=196608} 2024-09-21T09:35:33.954 INFO:tasks.rados.rados.0.smithi022.stdout:1225: writing smithi02284960-49 from 1081344 to 1818624 tid 1 2024-09-21T09:35:33.957 INFO:tasks.rados.rados.0.smithi022.stdout:1225: writing smithi02284960-49 from 1818624 to 2310144 tid 2 2024-09-21T09:35:33.960 INFO:tasks.rados.rados.0.smithi022.stdout:1225: writing smithi02284960-49 from 2310144 to 2826240 tid 3 2024-09-21T09:35:33.962 INFO:tasks.rados.rados.0.smithi022.stdout:1225: writing smithi02284960-49 from 2826240 to 3022848 tid 4 2024-09-21T09:35:33.962 INFO:tasks.rados.rados.0.smithi022.stdout:1226: delete oid 26 current snap is 126 2024-09-21T09:35:33.962 INFO:tasks.rados.rados.0.smithi022.stdout:1224: finishing copy_from to smithi02284960-43 2024-09-21T09:35:33.962 INFO:tasks.rados.rados.0.smithi022.stdout:1224: got expected ENOENT (src dne) 2024-09-21T09:35:33.962 INFO:tasks.rados.rados.0.smithi022.stdout:1224: finishing copy_from racing read to smithi02284960-43 2024-09-21T09:35:33.962 INFO:tasks.rados.rados.0.smithi022.stdout:1225: finishing write tid 1 to smithi02284960-49 2024-09-21T09:35:33.962 INFO:tasks.rados.rados.0.smithi022.stdout:1225: finishing write tid 2 to smithi02284960-49 2024-09-21T09:35:33.964 INFO:tasks.rados.rados.0.smithi022.stdout:1225: finishing write tid 3 to smithi02284960-49 2024-09-21T09:35:33.965 INFO:tasks.rados.rados.0.smithi022.stdout:1225: finishing write tid 4 to smithi02284960-49 2024-09-21T09:35:33.966 INFO:tasks.rados.rados.0.smithi022.stdout:1225: finishing write tid 5 to smithi02284960-49 2024-09-21T09:35:33.967 INFO:tasks.rados.rados.0.smithi022.stdout:1224: done (2 left) 2024-09-21T09:35:33.967 INFO:tasks.rados.rados.0.smithi022.stdout:1226: done (1 left) 2024-09-21T09:35:33.967 INFO:tasks.rados.rados.0.smithi022.stdout:1227: read oid 7 snap 100 2024-09-21T09:35:33.967 INFO:tasks.rados.rados.0.smithi022.stdout:1227: expect deleted 2024-09-21T09:35:33.967 INFO:tasks.rados.rados.0.smithi022.stdout:1228: rollback oid 18 current snap is 126 2024-09-21T09:35:33.967 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 18 to 69 2024-09-21T09:35:33.967 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 40 current snap is 126 2024-09-21T09:35:33.968 INFO:tasks.rados.rados.0.smithi022.stdout:1229: seq_num 404 ranges {3874816=770048,4644864=712704,5357568=491520} 2024-09-21T09:35:33.993 INFO:tasks.rados.rados.0.smithi022.stdout:1229: writing smithi02284960-40 from 3874816 to 4644864 tid 1 2024-09-21T09:35:33.997 INFO:tasks.rados.rados.0.smithi022.stdout:1229: writing smithi02284960-40 from 4644864 to 5357568 tid 2 2024-09-21T09:35:34.001 INFO:tasks.rados.rados.0.smithi022.stdout:1229: writing smithi02284960-40 from 5357568 to 5849088 tid 3 2024-09-21T09:35:34.001 INFO:tasks.rados.rados.0.smithi022.stdout:1225: finishing write tid 6 to smithi02284960-49 2024-09-21T09:35:34.001 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 49 v 582 (ObjNum 402 snap 126 seq_num 402) dirty exists 2024-09-21T09:35:34.001 INFO:tasks.rados.rados.0.smithi022.stdout:1225: left oid 49 (ObjNum 402 snap 126 seq_num 402) 2024-09-21T09:35:34.001 INFO:tasks.rados.rados.0.smithi022.stdout:1225: done (3 left) 2024-09-21T09:35:34.001 INFO:tasks.rados.rados.0.smithi022.stdout:1230: read oid 8 snap -1 2024-09-21T09:35:34.002 INFO:tasks.rados.rados.0.smithi022.stdout:1230: expect deleted 2024-09-21T09:35:34.002 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 33 current snap is 126 2024-09-21T09:35:34.002 INFO:tasks.rados.rados.0.smithi022.stdout:1231: seq_num 405 ranges {1867776=786432,2654208=589824,3244032=614400} 2024-09-21T09:35:34.015 INFO:tasks.rados.rados.0.smithi022.stdout:1231: writing smithi02284960-33 from 1867776 to 2654208 tid 1 2024-09-21T09:35:34.019 INFO:tasks.rados.rados.0.smithi022.stdout:1231: writing smithi02284960-33 from 2654208 to 3244032 tid 2 2024-09-21T09:35:34.024 INFO:tasks.rados.rados.0.smithi022.stdout:1231: writing smithi02284960-33 from 3244032 to 3858432 tid 3 2024-09-21T09:35:34.024 INFO:tasks.rados.rados.0.smithi022.stdout:1232: read oid 25 snap -1 2024-09-21T09:35:34.024 INFO:tasks.rados.rados.0.smithi022.stdout:1232: expect (ObjNum 345 snap 107 seq_num 345) 2024-09-21T09:35:34.024 INFO:tasks.rados.rados.0.smithi022.stdout:1233: snap_create 2024-09-21T09:35:34.024 INFO:tasks.rados.rados.0.smithi022.stdout:1228: finishing rollback tid 0 to smithi02284960-18 2024-09-21T09:35:34.024 INFO:tasks.rados.rados.0.smithi022.stdout:1228: finishing rollback tid 1 to smithi02284960-18 2024-09-21T09:35:34.024 INFO:tasks.rados.rados.0.smithi022.stdout:1228: finishing rollback tid 2 to smithi02284960-18 2024-09-21T09:35:34.024 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 18 v 326 (ObjNum 210 snap 58 seq_num 210) dirty exists 2024-09-21T09:35:34.024 INFO:tasks.rados.rados.0.smithi022.stdout:1229: finishing write tid 1 to smithi02284960-40 2024-09-21T09:35:34.024 INFO:tasks.rados.rados.0.smithi022.stdout:1229: finishing write tid 2 to smithi02284960-40 2024-09-21T09:35:34.024 INFO:tasks.rados.rados.0.smithi022.stdout:1229: finishing write tid 3 to smithi02284960-40 2024-09-21T09:35:34.024 INFO:tasks.rados.rados.0.smithi022.stdout:1229: finishing write tid 4 to smithi02284960-40 2024-09-21T09:35:34.025 INFO:tasks.rados.rados.0.smithi022.stdout:1229: finishing write tid 5 to smithi02284960-40 2024-09-21T09:35:34.025 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 40 v 710 (ObjNum 404 snap 126 seq_num 404) dirty exists 2024-09-21T09:35:34.025 INFO:tasks.rados.rados.0.smithi022.stdout:1229: left oid 40 (ObjNum 404 snap 126 seq_num 404) 2024-09-21T09:35:34.025 INFO:tasks.rados.rados.0.smithi022.stdout:1231: finishing write tid 1 to smithi02284960-33 2024-09-21T09:35:34.025 INFO:tasks.rados.rados.0.smithi022.stdout:1231: finishing write tid 2 to smithi02284960-33 2024-09-21T09:35:34.026 INFO:tasks.rados.rados.0.smithi022.stdout:1231: finishing write tid 3 to smithi02284960-33 2024-09-21T09:35:34.026 INFO:tasks.rados.rados.0.smithi022.stdout:1231: finishing write tid 4 to smithi02284960-33 2024-09-21T09:35:34.028 INFO:tasks.rados.rados.0.smithi022.stdout:1231: finishing write tid 5 to smithi02284960-33 2024-09-21T09:35:34.028 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 33 v 759 (ObjNum 405 snap 126 seq_num 405) dirty exists 2024-09-21T09:35:34.028 INFO:tasks.rados.rados.0.smithi022.stdout:1231: left oid 33 (ObjNum 405 snap 126 seq_num 405) 2024-09-21T09:35:34.029 INFO:tasks.rados.rados.0.smithi022.stdout:1232: expect (ObjNum 345 snap 107 seq_num 345) 2024-09-21T09:35:34.057 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:33 smithi022 ceph-mon[96736]: Reconfiguring osd.3 (monmap changed)... 2024-09-21T09:35:34.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:33 smithi022 ceph-mon[96736]: Reconfiguring daemon osd.3 on smithi022 2024-09-21T09:35:34.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:33 smithi022 ceph-mon[96736]: pgmap v70: 121 pgs: 121 active+clean; 273 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 1.1 MiB/s rd, 1.2 MiB/s wr, 8 op/s 2024-09-21T09:35:34.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:33 smithi022 ceph-mon[96736]: osdmap e312: 8 total, 8 up, 8 in 2024-09-21T09:35:34.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:33 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:34.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:33 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:34.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:33 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-21T09:35:34.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:33 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:35:34.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:33 smithi022 ceph-mon[96736]: osdmap e313: 8 total, 8 up, 8 in 2024-09-21T09:35:34.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:33 smithi022 ceph-mon[98615]: Reconfiguring osd.3 (monmap changed)... 2024-09-21T09:35:34.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:33 smithi022 ceph-mon[98615]: Reconfiguring daemon osd.3 on smithi022 2024-09-21T09:35:34.059 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:33 smithi022 ceph-mon[98615]: pgmap v70: 121 pgs: 121 active+clean; 273 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 1.1 MiB/s rd, 1.2 MiB/s wr, 8 op/s 2024-09-21T09:35:34.059 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:33 smithi022 ceph-mon[98615]: osdmap e312: 8 total, 8 up, 8 in 2024-09-21T09:35:34.059 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:33 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:34.059 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:33 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:34.059 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:33 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-21T09:35:34.059 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:33 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:35:34.059 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:33 smithi022 ceph-mon[98615]: osdmap e313: 8 total, 8 up, 8 in 2024-09-21T09:35:34.343 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:35:34 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:35:34.148Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:35:34.343 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:33 smithi064 ceph-mon[86709]: Reconfiguring osd.3 (monmap changed)... 2024-09-21T09:35:34.343 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:33 smithi064 ceph-mon[86709]: Reconfiguring daemon osd.3 on smithi022 2024-09-21T09:35:34.343 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:33 smithi064 ceph-mon[86709]: pgmap v70: 121 pgs: 121 active+clean; 273 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 1.1 MiB/s rd, 1.2 MiB/s wr, 8 op/s 2024-09-21T09:35:34.343 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:33 smithi064 ceph-mon[86709]: osdmap e312: 8 total, 8 up, 8 in 2024-09-21T09:35:34.344 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:33 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:34.344 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:33 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:34.344 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:33 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-21T09:35:34.344 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:33 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:35:34.344 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:33 smithi064 ceph-mon[86709]: osdmap e313: 8 total, 8 up, 8 in 2024-09-21T09:35:35.009 INFO:tasks.rados.rados.0.smithi022.stdout:1227: done (6 left) 2024-09-21T09:35:35.009 INFO:tasks.rados.rados.0.smithi022.stdout:1228: done (5 left) 2024-09-21T09:35:35.009 INFO:tasks.rados.rados.0.smithi022.stdout:1229: done (4 left) 2024-09-21T09:35:35.009 INFO:tasks.rados.rados.0.smithi022.stdout:1230: done (3 left) 2024-09-21T09:35:35.009 INFO:tasks.rados.rados.0.smithi022.stdout:1231: done (2 left) 2024-09-21T09:35:35.009 INFO:tasks.rados.rados.0.smithi022.stdout:1232: done (1 left) 2024-09-21T09:35:35.010 INFO:tasks.rados.rados.0.smithi022.stdout:1233: done (0 left) 2024-09-21T09:35:35.010 INFO:tasks.rados.rados.0.smithi022.stdout:1234: read oid 18 snap -1 2024-09-21T09:35:35.010 INFO:tasks.rados.rados.0.smithi022.stdout:1234: expect (ObjNum 210 snap 58 seq_num 210) 2024-09-21T09:35:35.010 INFO:tasks.rados.rados.0.smithi022.stdout:1235: setattr oid 1 current snap is 127 2024-09-21T09:35:35.010 INFO:tasks.rados.rados.0.smithi022.stdout:1236: snap_create 2024-09-21T09:35:35.014 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 1 v 737 (ObjNum 311 snap 94 seq_num 311) dirty exists 2024-09-21T09:35:35.018 INFO:tasks.rados.rados.0.smithi022.stdout:1234: expect (ObjNum 210 snap 58 seq_num 210) 2024-09-21T09:35:35.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:35 smithi022 ceph-mon[98615]: Reconfiguring mds.a (monmap changed)... 2024-09-21T09:35:35.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:35 smithi022 ceph-mon[98615]: Reconfiguring daemon mds.a on smithi022 2024-09-21T09:35:35.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:35 smithi022 ceph-mon[96736]: Reconfiguring mds.a (monmap changed)... 2024-09-21T09:35:35.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:35 smithi022 ceph-mon[96736]: Reconfiguring daemon mds.a on smithi022 2024-09-21T09:35:35.358 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:35 smithi064 ceph-mon[86709]: Reconfiguring mds.a (monmap changed)... 2024-09-21T09:35:35.358 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:35 smithi064 ceph-mon[86709]: Reconfiguring daemon mds.a on smithi022 2024-09-21T09:35:35.941 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:35:35 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:35:35.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:35:35.941 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:35:35 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:35:35.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:35:36.010 INFO:tasks.rados.rados.0.smithi022.stdout:1234: done (2 left) 2024-09-21T09:35:36.010 INFO:tasks.rados.rados.0.smithi022.stdout:1235: done (1 left) 2024-09-21T09:35:36.010 INFO:tasks.rados.rados.0.smithi022.stdout:1236: done (0 left) 2024-09-21T09:35:36.010 INFO:tasks.rados.rados.0.smithi022.stdout:1237: setattr oid 40 current snap is 128 2024-09-21T09:35:36.011 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:35:35 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:35:35] "GET /metrics HTTP/1.1" 200 36109 "" "Prometheus/2.51.0" 2024-09-21T09:35:36.011 INFO:tasks.rados.rados.0.smithi022.stdout:1238: read oid 8 snap -1 2024-09-21T09:35:36.012 INFO:tasks.rados.rados.0.smithi022.stdout:1238: expect deleted 2024-09-21T09:35:36.012 INFO:tasks.rados.rados.0.smithi022.stdout:1239: snap_remove snap 115 2024-09-21T09:35:36.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:36 smithi022 ceph-mon[98615]: pgmap v73: 121 pgs: 121 active+clean; 276 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 2.9 MiB/s rd, 1.3 MiB/s wr, 13 op/s 2024-09-21T09:35:36.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:36 smithi022 ceph-mon[98615]: osdmap e314: 8 total, 8 up, 8 in 2024-09-21T09:35:36.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:36 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:36.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:36 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:36.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:36 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-21T09:35:36.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:36 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-21T09:35:36.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:36 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:35:36.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:36 smithi022 ceph-mon[96736]: pgmap v73: 121 pgs: 121 active+clean; 276 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 2.9 MiB/s rd, 1.3 MiB/s wr, 13 op/s 2024-09-21T09:35:36.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:36 smithi022 ceph-mon[96736]: osdmap e314: 8 total, 8 up, 8 in 2024-09-21T09:35:36.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:36 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:36.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:36 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:36.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:36 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-21T09:35:36.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:36 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-21T09:35:36.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:36 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:35:36.381 INFO:teuthology.orchestra.run.smithi022.stdout:true 2024-09-21T09:35:36.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:36 smithi064 ceph-mon[86709]: pgmap v73: 121 pgs: 121 active+clean; 276 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 2.9 MiB/s rd, 1.3 MiB/s wr, 13 op/s 2024-09-21T09:35:36.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:36 smithi064 ceph-mon[86709]: osdmap e314: 8 total, 8 up, 8 in 2024-09-21T09:35:36.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:36 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:36.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:36 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:36.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:36 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-21T09:35:36.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:36 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-21T09:35:36.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:36 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:35:36.972 INFO:teuthology.orchestra.run.smithi022.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T09:35:36.972 INFO:teuthology.orchestra.run.smithi022.stdout:alertmanager.a smithi022 *:9093,9094 running (2m) 30s ago 6m 18.4M - 0.27.0 11f11916f8cd caced80ba544 2024-09-21T09:35:36.972 INFO:teuthology.orchestra.run.smithi022.stdout:grafana.a smithi064 *:3000 running (115s) 17s ago 6m 77.9M - 9.4.7 954c08fa6188 527bb657d70d 2024-09-21T09:35:36.972 INFO:teuthology.orchestra.run.smithi022.stdout:mds.a smithi022 running (8m) 30s ago 8m 17.5M - 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 34e4221f3a27 2024-09-21T09:35:36.972 INFO:teuthology.orchestra.run.smithi022.stdout:mds.b smithi064 running (8m) 17s ago 8m 20.9M - 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 0f846b3ec694 2024-09-21T09:35:36.972 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.x smithi064 *:8443,9283,8765 running (76s) 17s ago 11m 453M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 61abc61fea8d 2024-09-21T09:35:36.972 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.y smithi022 *:8443,9283,8765 running (105s) 30s ago 13m 504M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 6c5bee78f09b 2024-09-21T09:35:36.972 INFO:teuthology.orchestra.run.smithi022.stdout:mon.a smithi022 running (63s) 30s ago 13m 40.7M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 0513395c784d 2024-09-21T09:35:36.972 INFO:teuthology.orchestra.run.smithi022.stdout:mon.b smithi064 running (22s) 17s ago 12m 31.5M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 9dab26b6a9e7 2024-09-21T09:35:36.972 INFO:teuthology.orchestra.run.smithi022.stdout:mon.c smithi022 running (40s) 30s ago 12m 27.6M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 c74b58b67bd6 2024-09-21T09:35:36.973 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.a smithi022 *:9100 running (2m) 30s ago 6m 7902k - 1.7.0 72c9c2088986 593c6158f524 2024-09-21T09:35:36.973 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.b smithi064 *:9100 running (110s) 17s ago 6m 9.94M - 1.7.0 72c9c2088986 a533420b0139 2024-09-21T09:35:36.973 INFO:teuthology.orchestra.run.smithi022.stdout:osd.0 smithi022 running (11m) 30s ago 11m 199M 2481M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be fcd812003534 2024-09-21T09:35:36.973 INFO:teuthology.orchestra.run.smithi022.stdout:osd.1 smithi022 running (11m) 30s ago 11m 153M 2481M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 740462b5c304 2024-09-21T09:35:36.973 INFO:teuthology.orchestra.run.smithi022.stdout:osd.2 smithi022 running (10m) 30s ago 10m 186M 2481M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be fefc7c770c32 2024-09-21T09:35:36.973 INFO:teuthology.orchestra.run.smithi022.stdout:osd.3 smithi022 running (10m) 30s ago 10m 95.9M 2481M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be a56193b67a92 2024-09-21T09:35:36.973 INFO:teuthology.orchestra.run.smithi022.stdout:osd.4 smithi064 running (9m) 17s ago 9m 141M 2480M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 2a2937374fc4 2024-09-21T09:35:36.973 INFO:teuthology.orchestra.run.smithi022.stdout:osd.5 smithi064 running (9m) 17s ago 9m 192M 2480M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 1900c4af5d28 2024-09-21T09:35:36.973 INFO:teuthology.orchestra.run.smithi022.stdout:osd.6 smithi064 running (9m) 17s ago 9m 126M 2480M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 3cc4ec816e5b 2024-09-21T09:35:36.973 INFO:teuthology.orchestra.run.smithi022.stdout:osd.7 smithi064 running (8m) 17s ago 8m 148M 2480M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 18ef0be94104 2024-09-21T09:35:36.973 INFO:teuthology.orchestra.run.smithi022.stdout:prometheus.a smithi064 *:9095 running (82s) 17s ago 7m 44.0M - 2.51.0 1d3b7f56885b 80629ffa2179 2024-09-21T09:35:37.019 INFO:tasks.rados.rados.0.smithi022.stdout:1239: done (2 left) 2024-09-21T09:35:37.020 INFO:tasks.rados.rados.0.smithi022.stdout:1240: read oid 6 snap -1 2024-09-21T09:35:37.020 INFO:tasks.rados.rados.0.smithi022.stdout:1240: expect (ObjNum 347 snap 109 seq_num 347) 2024-09-21T09:35:37.020 INFO:tasks.rados.rados.0.smithi022.stdout:1241: snap_remove snap 84 2024-09-21T09:35:37.191 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:35:36 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:35:36.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi022\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:35:37.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:37 smithi064 ceph-mon[86709]: Reconfiguring mon.b (monmap changed)... 2024-09-21T09:35:37.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:37 smithi064 ceph-mon[86709]: Reconfiguring daemon mon.b on smithi064 2024-09-21T09:35:37.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:37 smithi064 ceph-mon[86709]: osdmap e315: 8 total, 8 up, 8 in 2024-09-21T09:35:37.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:37 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:37.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:37 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:37.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:37 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-21T09:35:37.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:37 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-09-21T09:35:37.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:37 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:35:37.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:37 smithi022 ceph-mon[98615]: Reconfiguring mon.b (monmap changed)... 2024-09-21T09:35:37.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:37 smithi022 ceph-mon[98615]: Reconfiguring daemon mon.b on smithi064 2024-09-21T09:35:37.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:37 smithi022 ceph-mon[98615]: osdmap e315: 8 total, 8 up, 8 in 2024-09-21T09:35:37.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:37 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:37.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:37 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:37.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:37 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-21T09:35:37.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:37 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-09-21T09:35:37.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:37 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:35:37.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:37 smithi022 ceph-mon[96736]: Reconfiguring mon.b (monmap changed)... 2024-09-21T09:35:37.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:37 smithi022 ceph-mon[96736]: Reconfiguring daemon mon.b on smithi064 2024-09-21T09:35:37.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:37 smithi022 ceph-mon[96736]: osdmap e315: 8 total, 8 up, 8 in 2024-09-21T09:35:37.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:37 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:37.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:37 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:37.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:37 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-21T09:35:37.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:37 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-09-21T09:35:37.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:37 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:35:37.338 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-09-21T09:35:37.338 INFO:teuthology.orchestra.run.smithi022.stdout: "mon": { 2024-09-21T09:35:37.339 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 3 2024-09-21T09:35:37.339 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-21T09:35:37.339 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr": { 2024-09-21T09:35:37.339 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 2 2024-09-21T09:35:37.339 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-21T09:35:37.339 INFO:teuthology.orchestra.run.smithi022.stdout: "osd": { 2024-09-21T09:35:37.339 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 8 2024-09-21T09:35:37.339 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-21T09:35:37.339 INFO:teuthology.orchestra.run.smithi022.stdout: "mds": { 2024-09-21T09:35:37.339 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 2 2024-09-21T09:35:37.339 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-21T09:35:37.339 INFO:teuthology.orchestra.run.smithi022.stdout: "overall": { 2024-09-21T09:35:37.339 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 10, 2024-09-21T09:35:37.340 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 5 2024-09-21T09:35:37.340 INFO:teuthology.orchestra.run.smithi022.stdout: } 2024-09-21T09:35:37.340 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-09-21T09:35:37.635 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-09-21T09:35:37.636 INFO:teuthology.orchestra.run.smithi022.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47", 2024-09-21T09:35:37.636 INFO:teuthology.orchestra.run.smithi022.stdout: "in_progress": true, 2024-09-21T09:35:37.636 INFO:teuthology.orchestra.run.smithi022.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T09:35:37.636 INFO:teuthology.orchestra.run.smithi022.stdout: "services_complete": [ 2024-09-21T09:35:37.636 INFO:teuthology.orchestra.run.smithi022.stdout: "mon", 2024-09-21T09:35:37.636 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr" 2024-09-21T09:35:37.636 INFO:teuthology.orchestra.run.smithi022.stdout: ], 2024-09-21T09:35:37.636 INFO:teuthology.orchestra.run.smithi022.stdout: "progress": "5/20 daemons upgraded", 2024-09-21T09:35:37.636 INFO:teuthology.orchestra.run.smithi022.stdout: "message": "Currently upgrading mon daemons", 2024-09-21T09:35:37.636 INFO:teuthology.orchestra.run.smithi022.stdout: "is_paused": false 2024-09-21T09:35:37.636 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-09-21T09:35:38.022 INFO:tasks.rados.rados.0.smithi022.stdout:1241: done (3 left) 2024-09-21T09:35:38.022 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 19 current snap is 128 2024-09-21T09:35:38.022 INFO:tasks.rados.rados.0.smithi022.stdout:1242: seq_num 406 ranges {0=802816,802816=475136,1277952=499712,1777664=229376} 2024-09-21T09:35:38.027 INFO:tasks.rados.rados.0.smithi022.stdout:1242: writing smithi02284960-19 from 0 to 802816 tid 1 2024-09-21T09:35:38.031 INFO:tasks.rados.rados.0.smithi022.stdout:1242: writing smithi02284960-19 from 802816 to 1277952 tid 2 2024-09-21T09:35:38.035 INFO:tasks.rados.rados.0.smithi022.stdout:1242: writing smithi02284960-19 from 1277952 to 1777664 tid 3 2024-09-21T09:35:38.037 INFO:tasks.rados.rados.0.smithi022.stdout:1242: writing smithi02284960-19 from 1777664 to 2007040 tid 4 2024-09-21T09:35:38.037 INFO:tasks.rados.rados.0.smithi022.stdout:1243: setattr oid 11 current snap is 128 2024-09-21T09:35:38.040 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 40 v 712 (ObjNum 404 snap 126 seq_num 404) dirty exists 2024-09-21T09:35:38.040 INFO:tasks.rados.rados.0.smithi022.stdout:1240: expect (ObjNum 347 snap 109 seq_num 347) 2024-09-21T09:35:38.071 INFO:tasks.rados.rados.0.smithi022.stdout:1242: finishing write tid 1 to smithi02284960-19 2024-09-21T09:35:38.071 INFO:tasks.rados.rados.0.smithi022.stdout:1242: finishing write tid 2 to smithi02284960-19 2024-09-21T09:35:38.071 INFO:tasks.rados.rados.0.smithi022.stdout:1242: finishing write tid 3 to smithi02284960-19 2024-09-21T09:35:38.071 INFO:tasks.rados.rados.0.smithi022.stdout:1242: finishing write tid 4 to smithi02284960-19 2024-09-21T09:35:38.071 INFO:tasks.rados.rados.0.smithi022.stdout:1242: finishing write tid 5 to smithi02284960-19 2024-09-21T09:35:38.071 INFO:tasks.rados.rados.0.smithi022.stdout:1242: finishing write tid 6 to smithi02284960-19 2024-09-21T09:35:38.072 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 19 v 806 (ObjNum 406 snap 128 seq_num 406) dirty exists 2024-09-21T09:35:38.072 INFO:tasks.rados.rados.0.smithi022.stdout:1242: left oid 19 (ObjNum 406 snap 128 seq_num 406) 2024-09-21T09:35:38.072 INFO:tasks.rados.rados.0.smithi022.stdout:1237: done (4 left) 2024-09-21T09:35:38.072 INFO:tasks.rados.rados.0.smithi022.stdout:1238: done (3 left) 2024-09-21T09:35:38.072 INFO:tasks.rados.rados.0.smithi022.stdout:1240: done (2 left) 2024-09-21T09:35:38.072 INFO:tasks.rados.rados.0.smithi022.stdout:1242: done (1 left) 2024-09-21T09:35:38.072 INFO:tasks.rados.rados.0.smithi022.stdout:1244: rollback oid 9 current snap is 128 2024-09-21T09:35:38.072 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 9 to 125 2024-09-21T09:35:38.072 INFO:tasks.rados.rados.0.smithi022.stdout:1245: snap_remove snap 122 2024-09-21T09:35:38.244 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:38 smithi064 ceph-mon[86709]: pgmap v76: 121 pgs: 121 active+clean; 276 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 2.9 MiB/s rd, 1.3 MiB/s wr, 13 op/s 2024-09-21T09:35:38.244 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:38 smithi064 ceph-mon[86709]: from='client.34176 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:35:38.244 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:38 smithi064 ceph-mon[86709]: from='client.44108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:35:38.244 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:38 smithi064 ceph-mon[86709]: Reconfiguring mgr.x (monmap changed)... 2024-09-21T09:35:38.244 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:38 smithi064 ceph-mon[86709]: Reconfiguring daemon mgr.x on smithi064 2024-09-21T09:35:38.244 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:38 smithi064 ceph-mon[86709]: from='client.34188 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:35:38.244 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:38 smithi064 ceph-mon[86709]: osdmap e316: 8 total, 8 up, 8 in 2024-09-21T09:35:38.244 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:38 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/1501048034' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:35:38.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:38 smithi022 ceph-mon[96736]: pgmap v76: 121 pgs: 121 active+clean; 276 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 2.9 MiB/s rd, 1.3 MiB/s wr, 13 op/s 2024-09-21T09:35:38.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:38 smithi022 ceph-mon[96736]: from='client.34176 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:35:38.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:38 smithi022 ceph-mon[96736]: from='client.44108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:35:38.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:38 smithi022 ceph-mon[96736]: Reconfiguring mgr.x (monmap changed)... 2024-09-21T09:35:38.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:38 smithi022 ceph-mon[96736]: Reconfiguring daemon mgr.x on smithi064 2024-09-21T09:35:38.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:38 smithi022 ceph-mon[96736]: from='client.34188 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:35:38.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:38 smithi022 ceph-mon[96736]: osdmap e316: 8 total, 8 up, 8 in 2024-09-21T09:35:38.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:38 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/1501048034' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:35:38.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:38 smithi022 ceph-mon[98615]: pgmap v76: 121 pgs: 121 active+clean; 276 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 2.9 MiB/s rd, 1.3 MiB/s wr, 13 op/s 2024-09-21T09:35:38.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:38 smithi022 ceph-mon[98615]: from='client.34176 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:35:38.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:38 smithi022 ceph-mon[98615]: from='client.44108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:35:38.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:38 smithi022 ceph-mon[98615]: Reconfiguring mgr.x (monmap changed)... 2024-09-21T09:35:38.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:38 smithi022 ceph-mon[98615]: Reconfiguring daemon mgr.x on smithi064 2024-09-21T09:35:38.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:38 smithi022 ceph-mon[98615]: from='client.34188 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:35:38.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:38 smithi022 ceph-mon[98615]: osdmap e316: 8 total, 8 up, 8 in 2024-09-21T09:35:38.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:38 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/1501048034' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:35:39.029 INFO:tasks.rados.rados.0.smithi022.stdout:1245: done (2 left) 2024-09-21T09:35:39.029 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 29 current snap is 128 2024-09-21T09:35:39.029 INFO:tasks.rados.rados.0.smithi022.stdout:1244: finishing rollback tid 1 to smithi02284960-9 2024-09-21T09:35:39.029 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 9 v 466 (ObjNum 833 snap 0 seq_num 3412041744) dirty dne 2024-09-21T09:35:39.029 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 11 v 758 (ObjNum 289 snap 0 seq_num 0) dirty exists 2024-09-21T09:35:39.029 INFO:tasks.rados.rados.0.smithi022.stdout:1246: seq_num 407 ranges {0=425984,425984=761856,1187840=704512,1892352=139264} 2024-09-21T09:35:39.031 INFO:tasks.rados.rados.0.smithi022.stdout:1246: writing smithi02284960-29 from 0 to 425984 tid 1 2024-09-21T09:35:39.037 INFO:tasks.rados.rados.0.smithi022.stdout:1246: writing smithi02284960-29 from 425984 to 1187840 tid 2 2024-09-21T09:35:39.042 INFO:tasks.rados.rados.0.smithi022.stdout:1246: writing smithi02284960-29 from 1187840 to 1892352 tid 3 2024-09-21T09:35:39.043 INFO:tasks.rados.rados.0.smithi022.stdout:1246: writing smithi02284960-29 from 1892352 to 2031616 tid 4 2024-09-21T09:35:39.043 INFO:tasks.rados.rados.0.smithi022.stdout:1243: done (2 left) 2024-09-21T09:35:39.043 INFO:tasks.rados.rados.0.smithi022.stdout:1244: done (1 left) 2024-09-21T09:35:39.043 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 6 current snap is 128 2024-09-21T09:35:39.043 INFO:tasks.rados.rados.0.smithi022.stdout:1246: finishing write tid 1 to smithi02284960-29 2024-09-21T09:35:39.043 INFO:tasks.rados.rados.0.smithi022.stdout:1246: finishing write tid 2 to smithi02284960-29 2024-09-21T09:35:39.043 INFO:tasks.rados.rados.0.smithi022.stdout:1247: seq_num 408 ranges {1015808=442368,1458176=647168,2105344=507904,2613248=450560} 2024-09-21T09:35:39.051 INFO:tasks.rados.rados.0.smithi022.stdout:1247: writing smithi02284960-6 from 1015808 to 1458176 tid 1 2024-09-21T09:35:39.055 INFO:tasks.rados.rados.0.smithi022.stdout:1247: writing smithi02284960-6 from 1458176 to 2105344 tid 2 2024-09-21T09:35:39.058 INFO:tasks.rados.rados.0.smithi022.stdout:1247: writing smithi02284960-6 from 2105344 to 2613248 tid 3 2024-09-21T09:35:39.062 INFO:tasks.rados.rados.0.smithi022.stdout:1247: writing smithi02284960-6 from 2613248 to 3063808 tid 4 2024-09-21T09:35:39.062 INFO:tasks.rados.rados.0.smithi022.stdout:1248: read oid 28 snap -1 2024-09-21T09:35:39.062 INFO:tasks.rados.rados.0.smithi022.stdout:1248: expect (ObjNum 392 snap 122 seq_num 392) 2024-09-21T09:35:39.062 INFO:tasks.rados.rados.0.smithi022.stdout:1249: copy_from oid 18 from oid 4 current snap is 128 2024-09-21T09:35:39.062 INFO:tasks.rados.rados.0.smithi022.stdout:1246: finishing write tid 3 to smithi02284960-29 2024-09-21T09:35:39.062 INFO:tasks.rados.rados.0.smithi022.stdout:1246: finishing write tid 4 to smithi02284960-29 2024-09-21T09:35:39.063 INFO:tasks.rados.rados.0.smithi022.stdout:1246: finishing write tid 5 to smithi02284960-29 2024-09-21T09:35:39.063 INFO:tasks.rados.rados.0.smithi022.stdout:1246: finishing write tid 6 to smithi02284960-29 2024-09-21T09:35:39.063 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 29 v 470 (ObjNum 407 snap 128 seq_num 407) dirty exists 2024-09-21T09:35:39.063 INFO:tasks.rados.rados.0.smithi022.stdout:1246: left oid 29 (ObjNum 407 snap 128 seq_num 407) 2024-09-21T09:35:39.063 INFO:tasks.rados.rados.0.smithi022.stdout:1247: finishing write tid 1 to smithi02284960-6 2024-09-21T09:35:39.063 INFO:tasks.rados.rados.0.smithi022.stdout:1246: done (3 left) 2024-09-21T09:35:39.063 INFO:tasks.rados.rados.0.smithi022.stdout:1250: read oid 43 snap -1 2024-09-21T09:35:39.063 INFO:tasks.rados.rados.0.smithi022.stdout:1250: expect (ObjNum 373 snap 116 seq_num 373) 2024-09-21T09:35:39.063 INFO:tasks.rados.rados.0.smithi022.stdout:1251: read oid 1 snap 104 2024-09-21T09:35:39.063 INFO:tasks.rados.rados.0.smithi022.stdout:1251: expect (ObjNum 338 snap 103 seq_num 338) 2024-09-21T09:35:39.063 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 8 current snap is 128 2024-09-21T09:35:39.063 INFO:tasks.rados.rados.0.smithi022.stdout:1252: seq_num 409 ranges {0=458752,458752=532480,991232=720896,1712128=352256} 2024-09-21T09:35:39.065 INFO:tasks.rados.rados.0.smithi022.stdout:1252: writing smithi02284960-8 from 0 to 458752 tid 1 2024-09-21T09:35:39.069 INFO:tasks.rados.rados.0.smithi022.stdout:1252: writing smithi02284960-8 from 458752 to 991232 tid 2 2024-09-21T09:35:39.075 INFO:tasks.rados.rados.0.smithi022.stdout:1252: writing smithi02284960-8 from 991232 to 1712128 tid 3 2024-09-21T09:35:39.078 INFO:tasks.rados.rados.0.smithi022.stdout:1252: writing smithi02284960-8 from 1712128 to 2064384 tid 4 2024-09-21T09:35:39.078 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 38 current snap is 128 2024-09-21T09:35:39.078 INFO:tasks.rados.rados.0.smithi022.stdout:1247: finishing write tid 2 to smithi02284960-6 2024-09-21T09:35:39.078 INFO:tasks.rados.rados.0.smithi022.stdout:1253: seq_num 410 ranges {2088960=475136,2564096=417792,2981888=475136,3457024=425984,3883008=286720} 2024-09-21T09:35:39.092 INFO:tasks.rados.rados.0.smithi022.stdout:1253: writing smithi02284960-38 from 2088960 to 2564096 tid 1 2024-09-21T09:35:39.094 INFO:tasks.rados.rados.0.smithi022.stdout:1253: writing smithi02284960-38 from 2564096 to 2981888 tid 2 2024-09-21T09:35:39.097 INFO:tasks.rados.rados.0.smithi022.stdout:1253: writing smithi02284960-38 from 2981888 to 3457024 tid 3 2024-09-21T09:35:39.100 INFO:tasks.rados.rados.0.smithi022.stdout:1253: writing smithi02284960-38 from 3457024 to 3883008 tid 4 2024-09-21T09:35:39.102 INFO:tasks.rados.rados.0.smithi022.stdout:1253: writing smithi02284960-38 from 3883008 to 4169728 tid 5 2024-09-21T09:35:39.102 INFO:tasks.rados.rados.0.smithi022.stdout:1254: snap_remove snap 95 2024-09-21T09:35:39.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:39 smithi022 ceph-mon[98615]: from='client.44126 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:35:39.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:39 smithi022 ceph-mon[98615]: osdmap e317: 8 total, 8 up, 8 in 2024-09-21T09:35:39.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:39 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:39.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:39 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:39.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:39 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-21T09:35:39.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:39 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:35:39.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:39 smithi022 ceph-mon[96736]: from='client.44126 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:35:39.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:39 smithi022 ceph-mon[96736]: osdmap e317: 8 total, 8 up, 8 in 2024-09-21T09:35:39.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:39 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:39.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:39 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:39.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:39 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-21T09:35:39.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:39 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:35:39.369 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:39 smithi064 ceph-mon[86709]: from='client.44126 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:35:39.369 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:39 smithi064 ceph-mon[86709]: osdmap e317: 8 total, 8 up, 8 in 2024-09-21T09:35:39.369 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:39 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:39.369 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:39 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:39.370 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:39 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-21T09:35:39.370 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:39 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:35:40.037 INFO:tasks.rados.rados.0.smithi022.stdout:1254: done (7 left) 2024-09-21T09:35:40.037 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 26 current snap is 128 2024-09-21T09:35:40.037 INFO:tasks.rados.rados.0.smithi022.stdout:1255: seq_num 411 ranges {0=491520,491520=753664,1245184=688128,1933312=163840} 2024-09-21T09:35:40.040 INFO:tasks.rados.rados.0.smithi022.stdout:1255: writing smithi02284960-26 from 0 to 491520 tid 1 2024-09-21T09:35:40.046 INFO:tasks.rados.rados.0.smithi022.stdout:1255: writing smithi02284960-26 from 491520 to 1245184 tid 2 2024-09-21T09:35:40.051 INFO:tasks.rados.rados.0.smithi022.stdout:1255: writing smithi02284960-26 from 1245184 to 1933312 tid 3 2024-09-21T09:35:40.053 INFO:tasks.rados.rados.0.smithi022.stdout:1255: writing smithi02284960-26 from 1933312 to 2097152 tid 4 2024-09-21T09:35:40.053 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 41 current snap is 128 2024-09-21T09:35:40.053 INFO:tasks.rados.rados.0.smithi022.stdout:1247: finishing write tid 3 to smithi02284960-6 2024-09-21T09:35:40.053 INFO:tasks.rados.rados.0.smithi022.stdout:1247: finishing write tid 4 to smithi02284960-6 2024-09-21T09:35:40.053 INFO:tasks.rados.rados.0.smithi022.stdout:1247: finishing write tid 5 to smithi02284960-6 2024-09-21T09:35:40.053 INFO:tasks.rados.rados.0.smithi022.stdout:1248: expect (ObjNum 392 snap 122 seq_num 392) 2024-09-21T09:35:40.106 INFO:tasks.rados.rados.0.smithi022.stdout:1249: finishing copy_from to smithi02284960-18 2024-09-21T09:35:40.106 INFO:tasks.rados.rados.0.smithi022.stdout:1249: got expected ENOENT (src dne) 2024-09-21T09:35:40.106 INFO:tasks.rados.rados.0.smithi022.stdout:1249: finishing copy_from racing read to smithi02284960-18 2024-09-21T09:35:40.106 INFO:tasks.rados.rados.0.smithi022.stdout:1252: finishing write tid 1 to smithi02284960-8 2024-09-21T09:35:40.106 INFO:tasks.rados.rados.0.smithi022.stdout:1247: finishing write tid 6 to smithi02284960-6 2024-09-21T09:35:40.106 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 6 v 781 (ObjNum 408 snap 128 seq_num 408) dirty exists 2024-09-21T09:35:40.106 INFO:tasks.rados.rados.0.smithi022.stdout:1247: left oid 6 (ObjNum 408 snap 128 seq_num 408) 2024-09-21T09:35:40.107 INFO:tasks.rados.rados.0.smithi022.stdout:1256: seq_num 412 ranges {0=507904,507904=630784,1138688=491520,1630208=483328} 2024-09-21T09:35:40.109 INFO:tasks.rados.rados.0.smithi022.stdout:1256: writing smithi02284960-41 from 0 to 507904 tid 1 2024-09-21T09:35:40.112 INFO:tasks.rados.rados.0.smithi022.stdout:1256: writing smithi02284960-41 from 507904 to 1138688 tid 2 2024-09-21T09:35:40.115 INFO:tasks.rados.rados.0.smithi022.stdout:1256: writing smithi02284960-41 from 1138688 to 1630208 tid 3 2024-09-21T09:35:40.118 INFO:tasks.rados.rados.0.smithi022.stdout:1256: writing smithi02284960-41 from 1630208 to 2113536 tid 4 2024-09-21T09:35:40.118 INFO:tasks.rados.rados.0.smithi022.stdout:1247: done (8 left) 2024-09-21T09:35:40.118 INFO:tasks.rados.rados.0.smithi022.stdout:1248: done (7 left) 2024-09-21T09:35:40.118 INFO:tasks.rados.rados.0.smithi022.stdout:1249: done (6 left) 2024-09-21T09:35:40.119 INFO:tasks.rados.rados.0.smithi022.stdout:1257: snap_remove snap 69 2024-09-21T09:35:40.286 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:35:40 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:35:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:35:40.287 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:40 smithi064 ceph-mon[86709]: Reconfiguring osd.4 (monmap changed)... 2024-09-21T09:35:40.287 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:40 smithi064 ceph-mon[86709]: Reconfiguring daemon osd.4 on smithi064 2024-09-21T09:35:40.287 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:40 smithi064 ceph-mon[86709]: pgmap v79: 121 pgs: 121 active+clean; 267 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 2.2 MiB/s rd, 486 KiB/s wr, 13 op/s 2024-09-21T09:35:40.287 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:40 smithi064 ceph-mon[86709]: osdmap e318: 8 total, 8 up, 8 in 2024-09-21T09:35:40.287 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:40 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:40.287 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:40 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:40.287 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:40 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-21T09:35:40.287 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:40 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:35:40.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:40 smithi022 ceph-mon[98615]: Reconfiguring osd.4 (monmap changed)... 2024-09-21T09:35:40.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:40 smithi022 ceph-mon[98615]: Reconfiguring daemon osd.4 on smithi064 2024-09-21T09:35:40.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:40 smithi022 ceph-mon[98615]: pgmap v79: 121 pgs: 121 active+clean; 267 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 2.2 MiB/s rd, 486 KiB/s wr, 13 op/s 2024-09-21T09:35:40.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:40 smithi022 ceph-mon[98615]: osdmap e318: 8 total, 8 up, 8 in 2024-09-21T09:35:40.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:40 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:40.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:40 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:40.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:40 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-21T09:35:40.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:40 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:35:40.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:40 smithi022 ceph-mon[96736]: Reconfiguring osd.4 (monmap changed)... 2024-09-21T09:35:40.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:40 smithi022 ceph-mon[96736]: Reconfiguring daemon osd.4 on smithi064 2024-09-21T09:35:40.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:40 smithi022 ceph-mon[96736]: pgmap v79: 121 pgs: 121 active+clean; 267 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 2.2 MiB/s rd, 486 KiB/s wr, 13 op/s 2024-09-21T09:35:40.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:40 smithi022 ceph-mon[96736]: osdmap e318: 8 total, 8 up, 8 in 2024-09-21T09:35:40.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:40 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:40.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:40 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:40.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:40 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-21T09:35:40.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:40 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:35:41.049 INFO:tasks.rados.rados.0.smithi022.stdout:1257: done (6 left) 2024-09-21T09:35:41.049 INFO:tasks.rados.rados.0.smithi022.stdout:1258: rollback oid 13 current snap is 128 2024-09-21T09:35:41.049 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 13 to 103 2024-09-21T09:35:41.049 INFO:tasks.rados.rados.0.smithi022.stdout:1259: read oid 40 snap -1 2024-09-21T09:35:41.050 INFO:tasks.rados.rados.0.smithi022.stdout:1259: expect (ObjNum 404 snap 126 seq_num 404) 2024-09-21T09:35:41.050 INFO:tasks.rados.rados.0.smithi022.stdout:1260: read oid 34 snap 104 2024-09-21T09:35:41.050 INFO:tasks.rados.rados.0.smithi022.stdout:1260: expect (ObjNum 327 snap 99 seq_num 327) 2024-09-21T09:35:41.050 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 42 current snap is 128 2024-09-21T09:35:41.050 INFO:tasks.rados.rados.0.smithi022.stdout:1261: seq_num 413 ranges {0=524288,524288=516096,1040384=696320,1736704=393216} 2024-09-21T09:35:41.053 INFO:tasks.rados.rados.0.smithi022.stdout:1261: writing smithi02284960-42 from 0 to 524288 tid 1 2024-09-21T09:35:41.057 INFO:tasks.rados.rados.0.smithi022.stdout:1261: writing smithi02284960-42 from 524288 to 1040384 tid 2 2024-09-21T09:35:41.062 INFO:tasks.rados.rados.0.smithi022.stdout:1261: writing smithi02284960-42 from 1040384 to 1736704 tid 3 2024-09-21T09:35:41.065 INFO:tasks.rados.rados.0.smithi022.stdout:1261: writing smithi02284960-42 from 1736704 to 2129920 tid 4 2024-09-21T09:35:41.066 INFO:tasks.rados.rados.0.smithi022.stdout:1251: expect (ObjNum 338 snap 103 seq_num 338) 2024-09-21T09:35:41.123 INFO:tasks.rados.rados.0.smithi022.stdout:1252: finishing write tid 2 to smithi02284960-8 2024-09-21T09:35:41.123 INFO:tasks.rados.rados.0.smithi022.stdout:1252: finishing write tid 3 to smithi02284960-8 2024-09-21T09:35:41.123 INFO:tasks.rados.rados.0.smithi022.stdout:1252: finishing write tid 4 to smithi02284960-8 2024-09-21T09:35:41.123 INFO:tasks.rados.rados.0.smithi022.stdout:1251: done (9 left) 2024-09-21T09:35:41.123 INFO:tasks.rados.rados.0.smithi022.stdout:1262: read oid 32 snap -1 2024-09-21T09:35:41.123 INFO:tasks.rados.rados.0.smithi022.stdout:1262: expect (ObjNum 382 snap 119 seq_num 382) 2024-09-21T09:35:41.123 INFO:tasks.rados.rados.0.smithi022.stdout:1263: delete oid 31 current snap is 128 2024-09-21T09:35:41.124 INFO:tasks.rados.rados.0.smithi022.stdout:1252: finishing write tid 5 to smithi02284960-8 2024-09-21T09:35:41.124 INFO:tasks.rados.rados.0.smithi022.stdout:1252: finishing write tid 6 to smithi02284960-8 2024-09-21T09:35:41.124 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 8 v 484 (ObjNum 409 snap 128 seq_num 409) dirty exists 2024-09-21T09:35:41.124 INFO:tasks.rados.rados.0.smithi022.stdout:1252: left oid 8 (ObjNum 409 snap 128 seq_num 409) 2024-09-21T09:35:41.124 INFO:tasks.rados.rados.0.smithi022.stdout:1250: expect (ObjNum 373 snap 116 seq_num 373) 2024-09-21T09:35:41.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:41 smithi064 ceph-mon[86709]: Reconfiguring osd.5 (monmap changed)... 2024-09-21T09:35:41.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:41 smithi064 ceph-mon[86709]: Reconfiguring daemon osd.5 on smithi064 2024-09-21T09:35:41.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:41 smithi064 ceph-mon[86709]: osdmap e319: 8 total, 8 up, 8 in 2024-09-21T09:35:41.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:41 smithi022 ceph-mon[98615]: Reconfiguring osd.5 (monmap changed)... 2024-09-21T09:35:41.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:41 smithi022 ceph-mon[98615]: Reconfiguring daemon osd.5 on smithi064 2024-09-21T09:35:41.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:41 smithi022 ceph-mon[98615]: osdmap e319: 8 total, 8 up, 8 in 2024-09-21T09:35:41.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:41 smithi022 ceph-mon[96736]: Reconfiguring osd.5 (monmap changed)... 2024-09-21T09:35:41.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:41 smithi022 ceph-mon[96736]: Reconfiguring daemon osd.5 on smithi064 2024-09-21T09:35:41.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:41 smithi022 ceph-mon[96736]: osdmap e319: 8 total, 8 up, 8 in 2024-09-21T09:35:41.386 INFO:tasks.rados.rados.0.smithi022.stdout:1253: finishing write tid 1 to smithi02284960-38 2024-09-21T09:35:41.386 INFO:tasks.rados.rados.0.smithi022.stdout:1253: finishing write tid 2 to smithi02284960-38 2024-09-21T09:35:41.386 INFO:tasks.rados.rados.0.smithi022.stdout:1253: finishing write tid 3 to smithi02284960-38 2024-09-21T09:35:41.386 INFO:tasks.rados.rados.0.smithi022.stdout:1253: finishing write tid 4 to smithi02284960-38 2024-09-21T09:35:41.386 INFO:tasks.rados.rados.0.smithi022.stdout:1253: finishing write tid 5 to smithi02284960-38 2024-09-21T09:35:41.386 INFO:tasks.rados.rados.0.smithi022.stdout:1253: finishing write tid 6 to smithi02284960-38 2024-09-21T09:35:41.387 INFO:tasks.rados.rados.0.smithi022.stdout:1253: finishing write tid 7 to smithi02284960-38 2024-09-21T09:35:41.387 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 38 v 476 (ObjNum 410 snap 128 seq_num 410) dirty exists 2024-09-21T09:35:41.387 INFO:tasks.rados.rados.0.smithi022.stdout:1253: left oid 38 (ObjNum 410 snap 128 seq_num 410) 2024-09-21T09:35:41.387 INFO:tasks.rados.rados.0.smithi022.stdout:1255: finishing write tid 1 to smithi02284960-26 2024-09-21T09:35:41.387 INFO:tasks.rados.rados.0.smithi022.stdout:1255: finishing write tid 2 to smithi02284960-26 2024-09-21T09:35:41.387 INFO:tasks.rados.rados.0.smithi022.stdout:1255: finishing write tid 3 to smithi02284960-26 2024-09-21T09:35:41.387 INFO:tasks.rados.rados.0.smithi022.stdout:1255: finishing write tid 4 to smithi02284960-26 2024-09-21T09:35:41.387 INFO:tasks.rados.rados.0.smithi022.stdout:1255: finishing write tid 5 to smithi02284960-26 2024-09-21T09:35:41.387 INFO:tasks.rados.rados.0.smithi022.stdout:1255: finishing write tid 6 to smithi02284960-26 2024-09-21T09:35:41.387 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 26 v 591 (ObjNum 411 snap 128 seq_num 411) dirty exists 2024-09-21T09:35:41.387 INFO:tasks.rados.rados.0.smithi022.stdout:1255: left oid 26 (ObjNum 411 snap 128 seq_num 411) 2024-09-21T09:35:41.387 INFO:tasks.rados.rados.0.smithi022.stdout:1250: done (10 left) 2024-09-21T09:35:41.387 INFO:tasks.rados.rados.0.smithi022.stdout:1252: done (9 left) 2024-09-21T09:35:41.388 INFO:tasks.rados.rados.0.smithi022.stdout:1253: done (8 left) 2024-09-21T09:35:41.388 INFO:tasks.rados.rados.0.smithi022.stdout:1255: done (7 left) 2024-09-21T09:35:41.388 INFO:tasks.rados.rados.0.smithi022.stdout:1263: done (6 left) 2024-09-21T09:35:41.388 INFO:tasks.rados.rados.0.smithi022.stdout:1264: snap_create 2024-09-21T09:35:41.388 INFO:tasks.rados.rados.0.smithi022.stdout:1256: finishing write tid 1 to smithi02284960-41 2024-09-21T09:35:41.388 INFO:tasks.rados.rados.0.smithi022.stdout:1256: finishing write tid 2 to smithi02284960-41 2024-09-21T09:35:41.388 INFO:tasks.rados.rados.0.smithi022.stdout:1256: finishing write tid 3 to smithi02284960-41 2024-09-21T09:35:41.388 INFO:tasks.rados.rados.0.smithi022.stdout:1256: finishing write tid 4 to smithi02284960-41 2024-09-21T09:35:41.388 INFO:tasks.rados.rados.0.smithi022.stdout:1256: finishing write tid 5 to smithi02284960-41 2024-09-21T09:35:41.388 INFO:tasks.rados.rados.0.smithi022.stdout:1256: finishing write tid 6 to smithi02284960-41 2024-09-21T09:35:41.388 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 41 v 741 (ObjNum 412 snap 128 seq_num 412) dirty exists 2024-09-21T09:35:41.388 INFO:tasks.rados.rados.0.smithi022.stdout:1256: left oid 41 (ObjNum 412 snap 128 seq_num 412) 2024-09-21T09:35:41.388 INFO:tasks.rados.rados.0.smithi022.stdout:1258: finishing rollback tid 1 to smithi02284960-13 2024-09-21T09:35:41.388 INFO:tasks.rados.rados.0.smithi022.stdout:1258: finishing rollback tid 2 to smithi02284960-13 2024-09-21T09:35:41.388 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 13 v 593 (ObjNum 334 snap 101 seq_num 334) dirty exists 2024-09-21T09:35:41.389 INFO:tasks.rados.rados.0.smithi022.stdout:1260: expect (ObjNum 327 snap 99 seq_num 327) 2024-09-21T09:35:41.403 INFO:tasks.rados.rados.0.smithi022.stdout:1261: finishing write tid 1 to smithi02284960-42 2024-09-21T09:35:41.403 INFO:tasks.rados.rados.0.smithi022.stdout:1261: finishing write tid 2 to smithi02284960-42 2024-09-21T09:35:41.403 INFO:tasks.rados.rados.0.smithi022.stdout:1261: finishing write tid 3 to smithi02284960-42 2024-09-21T09:35:41.403 INFO:tasks.rados.rados.0.smithi022.stdout:1261: finishing write tid 4 to smithi02284960-42 2024-09-21T09:35:41.403 INFO:tasks.rados.rados.0.smithi022.stdout:1261: finishing write tid 5 to smithi02284960-42 2024-09-21T09:35:41.403 INFO:tasks.rados.rados.0.smithi022.stdout:1261: finishing write tid 6 to smithi02284960-42 2024-09-21T09:35:41.403 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 42 v 602 (ObjNum 413 snap 128 seq_num 413) dirty exists 2024-09-21T09:35:41.403 INFO:tasks.rados.rados.0.smithi022.stdout:1261: left oid 42 (ObjNum 413 snap 128 seq_num 413) 2024-09-21T09:35:41.404 INFO:tasks.rados.rados.0.smithi022.stdout:1259: expect (ObjNum 404 snap 126 seq_num 404) 2024-09-21T09:35:41.675 INFO:tasks.rados.rados.0.smithi022.stdout:1262: expect (ObjNum 382 snap 119 seq_num 382) 2024-09-21T09:35:42.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:42 smithi022 ceph-mon[98615]: pgmap v82: 121 pgs: 121 active+clean; 264 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 2.9 MiB/s rd, 1.9 MiB/s wr, 24 op/s 2024-09-21T09:35:42.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:42 smithi022 ceph-mon[98615]: osdmap e320: 8 total, 8 up, 8 in 2024-09-21T09:35:42.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:42 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:42.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:42 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:42.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:42 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-21T09:35:42.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:42 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:35:42.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:42 smithi022 ceph-mon[96736]: pgmap v82: 121 pgs: 121 active+clean; 264 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 2.9 MiB/s rd, 1.9 MiB/s wr, 24 op/s 2024-09-21T09:35:42.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:42 smithi022 ceph-mon[96736]: osdmap e320: 8 total, 8 up, 8 in 2024-09-21T09:35:42.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:42 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:42.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:42 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:42.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:42 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-21T09:35:42.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:42 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:35:42.316 INFO:tasks.rados.rados.0.smithi022.stdout:1256: done (6 left) 2024-09-21T09:35:42.316 INFO:tasks.rados.rados.0.smithi022.stdout:1258: done (5 left) 2024-09-21T09:35:42.316 INFO:tasks.rados.rados.0.smithi022.stdout:1259: done (4 left) 2024-09-21T09:35:42.316 INFO:tasks.rados.rados.0.smithi022.stdout:1260: done (3 left) 2024-09-21T09:35:42.316 INFO:tasks.rados.rados.0.smithi022.stdout:1261: done (2 left) 2024-09-21T09:35:42.316 INFO:tasks.rados.rados.0.smithi022.stdout:1262: done (1 left) 2024-09-21T09:35:42.316 INFO:tasks.rados.rados.0.smithi022.stdout:1264: done (0 left) 2024-09-21T09:35:42.316 INFO:tasks.rados.rados.0.smithi022.stdout:1265: copy_from oid 38 from oid 8 current snap is 129 2024-09-21T09:35:42.316 INFO:tasks.rados.rados.0.smithi022.stdout:1266: delete oid 42 current snap is 129 2024-09-21T09:35:42.319 INFO:tasks.rados.rados.0.smithi022.stdout:1266: done (1 left) 2024-09-21T09:35:42.319 INFO:tasks.rados.rados.0.smithi022.stdout:1267: delete oid 29 current snap is 129 2024-09-21T09:35:42.322 INFO:tasks.rados.rados.0.smithi022.stdout:1267: done (1 left) 2024-09-21T09:35:42.322 INFO:tasks.rados.rados.0.smithi022.stdout:1268: rollback oid 27 current snap is 129 2024-09-21T09:35:42.322 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 27 to 125 2024-09-21T09:35:42.322 INFO:tasks.rados.rados.0.smithi022.stdout:1269: snap_remove snap 121 2024-09-21T09:35:42.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:42 smithi064 ceph-mon[86709]: pgmap v82: 121 pgs: 121 active+clean; 264 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 2.9 MiB/s rd, 1.9 MiB/s wr, 24 op/s 2024-09-21T09:35:42.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:42 smithi064 ceph-mon[86709]: osdmap e320: 8 total, 8 up, 8 in 2024-09-21T09:35:42.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:42 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:42.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:42 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:42.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:42 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-21T09:35:42.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:42 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:35:43.300 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:43 smithi064 ceph-mon[86709]: Reconfiguring osd.6 (monmap changed)... 2024-09-21T09:35:43.301 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:43 smithi064 ceph-mon[86709]: Reconfiguring daemon osd.6 on smithi064 2024-09-21T09:35:43.301 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:43 smithi064 ceph-mon[86709]: osdmap e321: 8 total, 8 up, 8 in 2024-09-21T09:35:43.301 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:43 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:35:43.301 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:43 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:43.301 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:43 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:43.301 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:43 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-21T09:35:43.301 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:43 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:35:43.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:43 smithi022 ceph-mon[98615]: Reconfiguring osd.6 (monmap changed)... 2024-09-21T09:35:43.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:43 smithi022 ceph-mon[98615]: Reconfiguring daemon osd.6 on smithi064 2024-09-21T09:35:43.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:43 smithi022 ceph-mon[98615]: osdmap e321: 8 total, 8 up, 8 in 2024-09-21T09:35:43.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:43 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:35:43.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:43 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:43.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:43 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:43.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:43 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-21T09:35:43.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:43 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:35:43.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:43 smithi022 ceph-mon[96736]: Reconfiguring osd.6 (monmap changed)... 2024-09-21T09:35:43.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:43 smithi022 ceph-mon[96736]: Reconfiguring daemon osd.6 on smithi064 2024-09-21T09:35:43.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:43 smithi022 ceph-mon[96736]: osdmap e321: 8 total, 8 up, 8 in 2024-09-21T09:35:43.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:43 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:35:43.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:43 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:43.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:43 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:43.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:43 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-21T09:35:43.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:43 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:35:43.323 INFO:tasks.rados.rados.0.smithi022.stdout:1269: done (2 left) 2024-09-21T09:35:43.323 INFO:tasks.rados.rados.0.smithi022.stdout:1270: copy_from oid 3 from oid 44 current snap is 129 2024-09-21T09:35:43.323 INFO:tasks.rados.rados.0.smithi022.stdout:1268: finishing rollback tid 1 to smithi02284960-27 2024-09-21T09:35:43.323 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 27 v 495 (ObjNum 4113 snap 0 seq_num 260136574) dirty dne 2024-09-21T09:35:43.323 INFO:tasks.rados.rados.0.smithi022.stdout:1265: finishing copy_from racing read to smithi02284960-38 2024-09-21T09:35:43.323 INFO:tasks.rados.rados.0.smithi022.stdout:1265: finishing copy_from to smithi02284960-38 2024-09-21T09:35:43.323 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 38 v 484 (ObjNum 409 snap 128 seq_num 409) dirty exists 2024-09-21T09:35:43.323 INFO:tasks.rados.rados.0.smithi022.stdout:1265: done (2 left) 2024-09-21T09:35:43.323 INFO:tasks.rados.rados.0.smithi022.stdout:1268: done (1 left) 2024-09-21T09:35:43.323 INFO:tasks.rados.rados.0.smithi022.stdout:1271: rollback oid 23 current snap is 129 2024-09-21T09:35:43.324 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 23 to 123 2024-09-21T09:35:43.324 INFO:tasks.rados.rados.0.smithi022.stdout:1272: rollback oid 49 current snap is 129 2024-09-21T09:35:43.324 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 49 to 125 2024-09-21T09:35:43.324 INFO:tasks.rados.rados.0.smithi022.stdout:1273: snap_remove snap 120 2024-09-21T09:35:43.949 INFO:tasks.rados.rados.0.smithi022.stdout:1271: finishing rollback tid 0 to smithi02284960-23 2024-09-21T09:35:43.950 INFO:tasks.rados.rados.0.smithi022.stdout:1271: finishing rollback tid 1 to smithi02284960-23 2024-09-21T09:35:43.950 INFO:tasks.rados.rados.0.smithi022.stdout:1271: finishing rollback tid 2 to smithi02284960-23 2024-09-21T09:35:43.950 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 23 v 487 (ObjNum 379 snap 117 seq_num 379) dirty exists 2024-09-21T09:35:43.950 INFO:tasks.rados.rados.0.smithi022.stdout:1272: finishing rollback tid 0 to smithi02284960-49 2024-09-21T09:35:43.950 INFO:tasks.rados.rados.0.smithi022.stdout:1272: finishing rollback tid 1 to smithi02284960-49 2024-09-21T09:35:43.950 INFO:tasks.rados.rados.0.smithi022.stdout:1272: finishing rollback tid 2 to smithi02284960-49 2024-09-21T09:35:43.950 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 49 v 585 (ObjNum 322 snap 97 seq_num 322) dirty exists 2024-09-21T09:35:43.950 INFO:tasks.rados.rados.0.smithi022.stdout:1270: finishing copy_from racing read to smithi02284960-3 2024-09-21T09:35:43.950 INFO:tasks.rados.rados.0.smithi022.stdout:1270: finishing copy_from to smithi02284960-3 2024-09-21T09:35:43.950 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 3 v 469 (ObjNum 360 snap 114 seq_num 360) dirty exists 2024-09-21T09:35:43.950 INFO:tasks.rados.rados.0.smithi022.stdout:1270: done (3 left) 2024-09-21T09:35:43.950 INFO:tasks.rados.rados.0.smithi022.stdout:1271: done (2 left) 2024-09-21T09:35:43.951 INFO:tasks.rados.rados.0.smithi022.stdout:1272: done (1 left) 2024-09-21T09:35:43.951 INFO:tasks.rados.rados.0.smithi022.stdout:1273: done (0 left) 2024-09-21T09:35:43.951 INFO:tasks.rados.rados.0.smithi022.stdout:1274: copy_from oid 33 from oid 48 current snap is 129 2024-09-21T09:35:43.951 INFO:tasks.rados.rados.0.smithi022.stdout:1275: snap_create 2024-09-21T09:35:43.959 INFO:tasks.rados.rados.0.smithi022.stdout:1274: finishing copy_from to smithi02284960-33 2024-09-21T09:35:43.959 INFO:tasks.rados.rados.0.smithi022.stdout:1274: got expected ENOENT (src dne) 2024-09-21T09:35:43.960 INFO:tasks.rados.rados.0.smithi022.stdout:1274: finishing copy_from racing read to smithi02284960-33 2024-09-21T09:35:44.149 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:44 smithi064 ceph-mon[86709]: pgmap v85: 121 pgs: 121 active+clean; 264 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 721 KiB/s rd, 1.4 MiB/s wr, 10 op/s 2024-09-21T09:35:44.149 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:44 smithi064 ceph-mon[86709]: Reconfiguring osd.7 (monmap changed)... 2024-09-21T09:35:44.149 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:44 smithi064 ceph-mon[86709]: Reconfiguring daemon osd.7 on smithi064 2024-09-21T09:35:44.149 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:44 smithi064 ceph-mon[86709]: osdmap e322: 8 total, 8 up, 8 in 2024-09-21T09:35:44.149 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:44 smithi064 ceph-mon[86709]: osdmap e323: 8 total, 8 up, 8 in 2024-09-21T09:35:44.307 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:44 smithi022 ceph-mon[98615]: pgmap v85: 121 pgs: 121 active+clean; 264 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 721 KiB/s rd, 1.4 MiB/s wr, 10 op/s 2024-09-21T09:35:44.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:44 smithi022 ceph-mon[98615]: Reconfiguring osd.7 (monmap changed)... 2024-09-21T09:35:44.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:44 smithi022 ceph-mon[98615]: Reconfiguring daemon osd.7 on smithi064 2024-09-21T09:35:44.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:44 smithi022 ceph-mon[98615]: osdmap e322: 8 total, 8 up, 8 in 2024-09-21T09:35:44.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:44 smithi022 ceph-mon[98615]: osdmap e323: 8 total, 8 up, 8 in 2024-09-21T09:35:44.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:44 smithi022 ceph-mon[96736]: pgmap v85: 121 pgs: 121 active+clean; 264 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 721 KiB/s rd, 1.4 MiB/s wr, 10 op/s 2024-09-21T09:35:44.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:44 smithi022 ceph-mon[96736]: Reconfiguring osd.7 (monmap changed)... 2024-09-21T09:35:44.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:44 smithi022 ceph-mon[96736]: Reconfiguring daemon osd.7 on smithi064 2024-09-21T09:35:44.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:44 smithi022 ceph-mon[96736]: osdmap e322: 8 total, 8 up, 8 in 2024-09-21T09:35:44.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:44 smithi022 ceph-mon[96736]: osdmap e323: 8 total, 8 up, 8 in 2024-09-21T09:35:44.441 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:35:44 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:35:44.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:35:44.949 INFO:tasks.rados.rados.0.smithi022.stdout:1274: done (1 left) 2024-09-21T09:35:44.950 INFO:tasks.rados.rados.0.smithi022.stdout:1275: done (0 left) 2024-09-21T09:35:44.950 INFO:tasks.rados.rados.0.smithi022.stdout:1276: copy_from oid 35 from oid 22 current snap is 130 2024-09-21T09:35:44.950 INFO:tasks.rados.rados.0.smithi022.stdout:1277: rollback oid 20 current snap is 130 2024-09-21T09:35:44.950 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 20 to 100 2024-09-21T09:35:44.950 INFO:tasks.rados.rados.0.smithi022.stdout:1278: delete oid 27 current snap is 130 2024-09-21T09:35:44.952 INFO:tasks.rados.rados.0.smithi022.stdout:1278: done (2 left) 2024-09-21T09:35:44.952 INFO:tasks.rados.rados.0.smithi022.stdout:1279: rollback oid 36 current snap is 130 2024-09-21T09:35:44.952 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 36 to 126 2024-09-21T09:35:44.952 INFO:tasks.rados.rados.0.smithi022.stdout:1280: delete oid 7 current snap is 130 2024-09-21T09:35:44.953 INFO:tasks.rados.rados.0.smithi022.stdout:1276: finishing copy_from to smithi02284960-35 2024-09-21T09:35:44.954 INFO:tasks.rados.rados.0.smithi022.stdout:1276: got expected ENOENT (src dne) 2024-09-21T09:35:44.954 INFO:tasks.rados.rados.0.smithi022.stdout:1276: finishing copy_from racing read to smithi02284960-35 2024-09-21T09:35:44.957 INFO:tasks.rados.rados.0.smithi022.stdout:1279: finishing rollback tid 0 to smithi02284960-36 2024-09-21T09:35:44.957 INFO:tasks.rados.rados.0.smithi022.stdout:1279: finishing rollback tid 1 to smithi02284960-36 2024-09-21T09:35:44.957 INFO:tasks.rados.rados.0.smithi022.stdout:1279: finishing rollback tid 2 to smithi02284960-36 2024-09-21T09:35:44.957 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 36 v 657 (ObjNum 380 snap 117 seq_num 380) dirty exists 2024-09-21T09:35:44.959 INFO:tasks.rados.rados.0.smithi022.stdout:1277: finishing rollback tid 0 to smithi02284960-20 2024-09-21T09:35:44.959 INFO:tasks.rados.rados.0.smithi022.stdout:1277: finishing rollback tid 1 to smithi02284960-20 2024-09-21T09:35:44.960 INFO:tasks.rados.rados.0.smithi022.stdout:1277: finishing rollback tid 2 to smithi02284960-20 2024-09-21T09:35:44.960 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 20 v 811 (ObjNum 132 snap 27 seq_num 132) dirty exists 2024-09-21T09:35:44.960 INFO:tasks.rados.rados.0.smithi022.stdout:1276: done (3 left) 2024-09-21T09:35:44.960 INFO:tasks.rados.rados.0.smithi022.stdout:1277: done (2 left) 2024-09-21T09:35:44.960 INFO:tasks.rados.rados.0.smithi022.stdout:1279: done (1 left) 2024-09-21T09:35:44.960 INFO:tasks.rados.rados.0.smithi022.stdout:1280: done (0 left) 2024-09-21T09:35:44.960 INFO:tasks.rados.rados.0.smithi022.stdout:1281: copy_from oid 20 from oid 27 current snap is 130 2024-09-21T09:35:44.961 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 12 current snap is 130 2024-09-21T09:35:44.961 INFO:tasks.rados.rados.0.smithi022.stdout:1282: seq_num 419 ranges {3743744=622592,4366336=671744,5038080=647168,5685248=286720} 2024-09-21T09:35:44.984 INFO:tasks.rados.rados.0.smithi022.stdout:1282: writing smithi02284960-12 from 3743744 to 4366336 tid 1 2024-09-21T09:35:44.987 INFO:tasks.rados.rados.0.smithi022.stdout:1282: writing smithi02284960-12 from 4366336 to 5038080 tid 2 2024-09-21T09:35:44.991 INFO:tasks.rados.rados.0.smithi022.stdout:1282: writing smithi02284960-12 from 5038080 to 5685248 tid 3 2024-09-21T09:35:44.993 INFO:tasks.rados.rados.0.smithi022.stdout:1282: writing smithi02284960-12 from 5685248 to 5971968 tid 4 2024-09-21T09:35:44.993 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 19 current snap is 130 2024-09-21T09:35:44.993 INFO:tasks.rados.rados.0.smithi022.stdout:1281: finishing copy_from to smithi02284960-20 2024-09-21T09:35:44.993 INFO:tasks.rados.rados.0.smithi022.stdout:1281: got expected ENOENT (src dne) 2024-09-21T09:35:44.993 INFO:tasks.rados.rados.0.smithi022.stdout:1281: finishing copy_from racing read to smithi02284960-20 2024-09-21T09:35:44.993 INFO:tasks.rados.rados.0.smithi022.stdout:1282: finishing write tid 1 to smithi02284960-12 2024-09-21T09:35:44.993 INFO:tasks.rados.rados.0.smithi022.stdout:1282: finishing write tid 2 to smithi02284960-12 2024-09-21T09:35:44.993 INFO:tasks.rados.rados.0.smithi022.stdout:1283: seq_num 420 ranges {2007040=638976,2646016=557056,3203072=450560,3653632=598016} 2024-09-21T09:35:45.007 INFO:tasks.rados.rados.0.smithi022.stdout:1283: writing smithi02284960-19 from 2007040 to 2646016 tid 1 2024-09-21T09:35:45.010 INFO:tasks.rados.rados.0.smithi022.stdout:1283: writing smithi02284960-19 from 2646016 to 3203072 tid 2 2024-09-21T09:35:45.012 INFO:tasks.rados.rados.0.smithi022.stdout:1283: writing smithi02284960-19 from 3203072 to 3653632 tid 3 2024-09-21T09:35:45.017 INFO:tasks.rados.rados.0.smithi022.stdout:1283: writing smithi02284960-19 from 3653632 to 4251648 tid 4 2024-09-21T09:35:45.017 INFO:tasks.rados.rados.0.smithi022.stdout:1281: done (2 left) 2024-09-21T09:35:45.017 INFO:tasks.rados.rados.0.smithi022.stdout:1284: rollback oid 33 current snap is 130 2024-09-21T09:35:45.017 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 33 to 123 2024-09-21T09:35:45.017 INFO:tasks.rados.rados.0.smithi022.stdout:1282: finishing write tid 3 to smithi02284960-12 2024-09-21T09:35:45.017 INFO:tasks.rados.rados.0.smithi022.stdout:1282: finishing write tid 4 to smithi02284960-12 2024-09-21T09:35:45.017 INFO:tasks.rados.rados.0.smithi022.stdout:1282: finishing write tid 5 to smithi02284960-12 2024-09-21T09:35:45.018 INFO:tasks.rados.rados.0.smithi022.stdout:1282: finishing write tid 6 to smithi02284960-12 2024-09-21T09:35:45.018 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 12 v 853 (ObjNum 419 snap 130 seq_num 419) dirty exists 2024-09-21T09:35:45.018 INFO:tasks.rados.rados.0.smithi022.stdout:1282: left oid 12 (ObjNum 419 snap 130 seq_num 419) 2024-09-21T09:35:45.018 INFO:tasks.rados.rados.0.smithi022.stdout:1282: done (2 left) 2024-09-21T09:35:45.018 INFO:tasks.rados.rados.0.smithi022.stdout:1285: read oid 10 snap -1 2024-09-21T09:35:45.018 INFO:tasks.rados.rados.0.smithi022.stdout:1285: expect (ObjNum 311 snap 94 seq_num 311) 2024-09-21T09:35:45.018 INFO:tasks.rados.rados.0.smithi022.stdout:1286: snap_remove snap 129 2024-09-21T09:35:45.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:45 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:45.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:45 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:45.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:45 smithi022 ceph-mon[98615]: Reconfiguring mds.b (monmap changed)... 2024-09-21T09:35:45.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:45 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-21T09:35:45.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:45 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:35:45.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:45 smithi022 ceph-mon[98615]: Reconfiguring daemon mds.b on smithi064 2024-09-21T09:35:45.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:45 smithi022 ceph-mon[98615]: pgmap v88: 121 pgs: 1 active+clean+snaptrim_wait, 2 active+clean+snaptrim, 118 active+clean; 226 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 4.0 MiB/s wr, 19 op/s 2024-09-21T09:35:45.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:45 smithi022 ceph-mon[98615]: osdmap e324: 8 total, 8 up, 8 in 2024-09-21T09:35:45.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:45 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:45.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:45 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:45.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:45 smithi022 ceph-mon[96736]: Reconfiguring mds.b (monmap changed)... 2024-09-21T09:35:45.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:45 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-21T09:35:45.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:45 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:35:45.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:45 smithi022 ceph-mon[96736]: Reconfiguring daemon mds.b on smithi064 2024-09-21T09:35:45.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:45 smithi022 ceph-mon[96736]: pgmap v88: 121 pgs: 1 active+clean+snaptrim_wait, 2 active+clean+snaptrim, 118 active+clean; 226 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 4.0 MiB/s wr, 19 op/s 2024-09-21T09:35:45.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:45 smithi022 ceph-mon[96736]: osdmap e324: 8 total, 8 up, 8 in 2024-09-21T09:35:45.566 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:45 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:45.566 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:45 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:45.566 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:45 smithi064 ceph-mon[86709]: Reconfiguring mds.b (monmap changed)... 2024-09-21T09:35:45.566 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:45 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-21T09:35:45.566 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:45 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:35:45.566 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:45 smithi064 ceph-mon[86709]: Reconfiguring daemon mds.b on smithi064 2024-09-21T09:35:45.566 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:45 smithi064 ceph-mon[86709]: pgmap v88: 121 pgs: 1 active+clean+snaptrim_wait, 2 active+clean+snaptrim, 118 active+clean; 226 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 4.0 MiB/s wr, 19 op/s 2024-09-21T09:35:45.566 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:45 smithi064 ceph-mon[86709]: osdmap e324: 8 total, 8 up, 8 in 2024-09-21T09:35:45.941 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:35:45 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:35:45.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:35:45.942 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:35:45 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:35:45.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:35:45.950 INFO:tasks.rados.rados.0.smithi022.stdout:1286: done (3 left) 2024-09-21T09:35:45.951 INFO:tasks.rados.rados.0.smithi022.stdout:1287: delete oid 30 current snap is 130 2024-09-21T09:35:45.951 INFO:tasks.rados.rados.0.smithi022.stdout:1283: finishing write tid 1 to smithi02284960-19 2024-09-21T09:35:45.951 INFO:tasks.rados.rados.0.smithi022.stdout:1283: finishing write tid 2 to smithi02284960-19 2024-09-21T09:35:45.951 INFO:tasks.rados.rados.0.smithi022.stdout:1283: finishing write tid 3 to smithi02284960-19 2024-09-21T09:35:45.951 INFO:tasks.rados.rados.0.smithi022.stdout:1283: finishing write tid 4 to smithi02284960-19 2024-09-21T09:35:45.951 INFO:tasks.rados.rados.0.smithi022.stdout:1283: finishing write tid 5 to smithi02284960-19 2024-09-21T09:35:45.951 INFO:tasks.rados.rados.0.smithi022.stdout:1284: finishing rollback tid 0 to smithi02284960-33 2024-09-21T09:35:45.951 INFO:tasks.rados.rados.0.smithi022.stdout:1284: finishing rollback tid 1 to smithi02284960-33 2024-09-21T09:35:45.951 INFO:tasks.rados.rados.0.smithi022.stdout:1284: finishing rollback tid 2 to smithi02284960-33 2024-09-21T09:35:45.951 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 33 v 817 (ObjNum 311 snap 94 seq_num 311) dirty exists 2024-09-21T09:35:45.951 INFO:tasks.rados.rados.0.smithi022.stdout:1283: finishing write tid 6 to smithi02284960-19 2024-09-21T09:35:45.951 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 19 v 859 (ObjNum 420 snap 130 seq_num 420) dirty exists 2024-09-21T09:35:45.952 INFO:tasks.rados.rados.0.smithi022.stdout:1283: left oid 19 (ObjNum 420 snap 130 seq_num 420) 2024-09-21T09:35:45.952 INFO:tasks.rados.rados.0.smithi022.stdout:1285: expect (ObjNum 311 snap 94 seq_num 311) 2024-09-21T09:35:45.972 INFO:tasks.rados.rados.0.smithi022.stdout:1283: done (3 left) 2024-09-21T09:35:45.972 INFO:tasks.rados.rados.0.smithi022.stdout:1284: done (2 left) 2024-09-21T09:35:45.972 INFO:tasks.rados.rados.0.smithi022.stdout:1285: done (1 left) 2024-09-21T09:35:45.972 INFO:tasks.rados.rados.0.smithi022.stdout:1287: done (0 left) 2024-09-21T09:35:45.972 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 19 current snap is 130 2024-09-21T09:35:45.973 INFO:tasks.rados.rados.0.smithi022.stdout:1288: seq_num 422 ranges {4251648=671744,4923392=729088,5652480=417792,6070272=458752} 2024-09-21T09:35:45.999 INFO:tasks.rados.rados.0.smithi022.stdout:1288: writing smithi02284960-19 from 4251648 to 4923392 tid 1 2024-09-21T09:35:46.003 INFO:tasks.rados.rados.0.smithi022.stdout:1288: writing smithi02284960-19 from 4923392 to 5652480 tid 2 2024-09-21T09:35:46.006 INFO:tasks.rados.rados.0.smithi022.stdout:1288: writing smithi02284960-19 from 5652480 to 6070272 tid 3 2024-09-21T09:35:46.009 INFO:tasks.rados.rados.0.smithi022.stdout:1288: writing smithi02284960-19 from 6070272 to 6529024 tid 4 2024-09-21T09:35:46.010 INFO:tasks.rados.rados.0.smithi022.stdout:1289: snap_create 2024-09-21T09:35:46.010 INFO:tasks.rados.rados.0.smithi022.stdout:1288: finishing write tid 1 to smithi02284960-19 2024-09-21T09:35:46.010 INFO:tasks.rados.rados.0.smithi022.stdout:1288: finishing write tid 2 to smithi02284960-19 2024-09-21T09:35:46.010 INFO:tasks.rados.rados.0.smithi022.stdout:1288: finishing write tid 3 to smithi02284960-19 2024-09-21T09:35:46.013 INFO:tasks.rados.rados.0.smithi022.stdout:1288: finishing write tid 4 to smithi02284960-19 2024-09-21T09:35:46.013 INFO:tasks.rados.rados.0.smithi022.stdout:1288: finishing write tid 5 to smithi02284960-19 2024-09-21T09:35:46.014 INFO:tasks.rados.rados.0.smithi022.stdout:1288: finishing write tid 6 to smithi02284960-19 2024-09-21T09:35:46.014 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 19 v 868 (ObjNum 422 snap 130 seq_num 422) dirty exists 2024-09-21T09:35:46.014 INFO:tasks.rados.rados.0.smithi022.stdout:1288: left oid 19 (ObjNum 422 snap 130 seq_num 422) 2024-09-21T09:35:46.056 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:35:45 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:35:45] "GET /metrics HTTP/1.1" 200 36112 "" "Prometheus/2.51.0" 2024-09-21T09:35:46.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:46 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:46.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:46 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:46.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:46 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:35:46.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:46 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:35:46.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:46 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:35:46.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:46 smithi064 ceph-mon[86709]: Upgrade: Setting container_image for all mon 2024-09-21T09:35:46.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:46 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:46.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:46 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-21T09:35:46.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:46 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-09-21T09:35:46.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:46 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-21T09:35:46.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:46 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-09-21T09:35:46.943 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:46 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-21T09:35:46.943 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:46 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-09-21T09:35:46.943 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:46 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:35:46.943 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:46 smithi064 ceph-mon[86709]: Upgrade: Setting container_image for all crash 2024-09-21T09:35:46.943 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:46 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:46.943 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:46 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16} : dispatch 2024-09-21T09:35:46.943 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:46 smithi064 ceph-mon[86709]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-21T09:35:46.943 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:46 smithi064 ceph-mon[86709]: Upgrade: osd.0 is safe to restart 2024-09-21T09:35:46.943 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:46 smithi064 ceph-mon[86709]: osdmap e325: 8 total, 8 up, 8 in 2024-09-21T09:35:46.954 INFO:tasks.rados.rados.0.smithi022.stdout:1288: done (1 left) 2024-09-21T09:35:46.954 INFO:tasks.rados.rados.0.smithi022.stdout:1289: done (0 left) 2024-09-21T09:35:46.954 INFO:tasks.rados.rados.0.smithi022.stdout:1290: snap_create 2024-09-21T09:35:47.047 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:46 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:47.047 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:46 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:47.047 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:46 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:35:47.048 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:46 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:35:47.048 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:46 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:35:47.048 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:46 smithi022 ceph-mon[98615]: Upgrade: Setting container_image for all mon 2024-09-21T09:35:47.048 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:46 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:47.048 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:46 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-21T09:35:47.048 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:46 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-09-21T09:35:47.048 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:46 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-21T09:35:47.048 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:46 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-09-21T09:35:47.048 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:46 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-21T09:35:47.048 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:46 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-09-21T09:35:47.048 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:46 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:35:47.048 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:46 smithi022 ceph-mon[98615]: Upgrade: Setting container_image for all crash 2024-09-21T09:35:47.048 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:46 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:47.048 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:46 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16} : dispatch 2024-09-21T09:35:47.048 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:46 smithi022 ceph-mon[98615]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-21T09:35:47.049 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:46 smithi022 ceph-mon[98615]: Upgrade: osd.0 is safe to restart 2024-09-21T09:35:47.049 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:46 smithi022 ceph-mon[98615]: osdmap e325: 8 total, 8 up, 8 in 2024-09-21T09:35:47.049 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:46 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:47.049 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:46 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:47.049 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:46 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:35:47.049 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:46 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:35:47.049 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:46 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:35:47.049 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:46 smithi022 ceph-mon[96736]: Upgrade: Setting container_image for all mon 2024-09-21T09:35:47.049 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:46 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:47.049 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:46 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-21T09:35:47.049 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:46 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-09-21T09:35:47.050 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:46 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-21T09:35:47.050 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:46 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-09-21T09:35:47.050 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:46 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-21T09:35:47.050 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:46 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-09-21T09:35:47.050 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:46 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:35:47.050 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:46 smithi022 ceph-mon[96736]: Upgrade: Setting container_image for all crash 2024-09-21T09:35:47.050 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:46 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:47.050 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:46 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16} : dispatch 2024-09-21T09:35:47.050 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:46 smithi022 ceph-mon[96736]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-21T09:35:47.050 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:46 smithi022 ceph-mon[96736]: Upgrade: osd.0 is safe to restart 2024-09-21T09:35:47.050 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:46 smithi022 ceph-mon[96736]: osdmap e325: 8 total, 8 up, 8 in 2024-09-21T09:35:47.441 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:35:46 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:35:46.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi022\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:35:47.955 INFO:tasks.rados.rados.0.smithi022.stdout:1290: done (0 left) 2024-09-21T09:35:47.955 INFO:tasks.rados.rados.0.smithi022.stdout:1291: rollback oid 44 current snap is 132 2024-09-21T09:35:47.956 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 44 to 128 2024-09-21T09:35:47.956 INFO:tasks.rados.rados.0.smithi022.stdout:1292: read oid 36 snap -1 2024-09-21T09:35:47.956 INFO:tasks.rados.rados.0.smithi022.stdout:1292: expect (ObjNum 380 snap 117 seq_num 380) 2024-09-21T09:35:47.956 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 1 current snap is 132 2024-09-21T09:35:47.956 INFO:tasks.rados.rados.0.smithi022.stdout:1293: seq_num 423 ranges {614400=688128,1302528=614400,1916928=622592,2539520=368640} 2024-09-21T09:35:47.958 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:47 smithi022 ceph-mon[98615]: pgmap v91: 121 pgs: 1 active+clean+snaptrim_wait, 2 active+clean+snaptrim, 118 active+clean; 226 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 4.0 MiB/s wr, 19 op/s 2024-09-21T09:35:47.959 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:47 smithi022 ceph-mon[98615]: osdmap e326: 8 total, 8 up, 8 in 2024-09-21T09:35:47.959 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:47 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:47.959 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:47 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-21T09:35:47.959 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:47 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:35:47.959 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:47 smithi022 ceph-mon[96736]: pgmap v91: 121 pgs: 1 active+clean+snaptrim_wait, 2 active+clean+snaptrim, 118 active+clean; 226 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 4.0 MiB/s wr, 19 op/s 2024-09-21T09:35:47.959 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:47 smithi022 ceph-mon[96736]: osdmap e326: 8 total, 8 up, 8 in 2024-09-21T09:35:47.959 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:47 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:47.959 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:47 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-21T09:35:47.959 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:47 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:35:47.963 INFO:tasks.rados.rados.0.smithi022.stdout:1293: writing smithi02284960-1 from 614400 to 1302528 tid 1 2024-09-21T09:35:47.967 INFO:tasks.rados.rados.0.smithi022.stdout:1293: writing smithi02284960-1 from 1302528 to 1916928 tid 2 2024-09-21T09:35:47.971 INFO:tasks.rados.rados.0.smithi022.stdout:1293: writing smithi02284960-1 from 1916928 to 2539520 tid 3 2024-09-21T09:35:47.973 INFO:tasks.rados.rados.0.smithi022.stdout:1293: writing smithi02284960-1 from 2539520 to 2908160 tid 4 2024-09-21T09:35:47.974 INFO:tasks.rados.rados.0.smithi022.stdout:1294: delete oid 50 current snap is 132 2024-09-21T09:35:47.974 INFO:tasks.rados.rados.0.smithi022.stdout:1291: finishing rollback tid 0 to smithi02284960-44 2024-09-21T09:35:47.974 INFO:tasks.rados.rados.0.smithi022.stdout:1291: finishing rollback tid 1 to smithi02284960-44 2024-09-21T09:35:47.974 INFO:tasks.rados.rados.0.smithi022.stdout:1291: finishing rollback tid 2 to smithi02284960-44 2024-09-21T09:35:47.974 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 44 v 796 (ObjNum 360 snap 114 seq_num 360) dirty exists 2024-09-21T09:35:47.974 INFO:tasks.rados.rados.0.smithi022.stdout:1292: expect (ObjNum 380 snap 117 seq_num 380) 2024-09-21T09:35:48.017 INFO:tasks.rados.rados.0.smithi022.stdout:1293: finishing write tid 1 to smithi02284960-1 2024-09-21T09:35:48.017 INFO:tasks.rados.rados.0.smithi022.stdout:1293: finishing write tid 2 to smithi02284960-1 2024-09-21T09:35:48.017 INFO:tasks.rados.rados.0.smithi022.stdout:1293: finishing write tid 3 to smithi02284960-1 2024-09-21T09:35:48.017 INFO:tasks.rados.rados.0.smithi022.stdout:1293: finishing write tid 4 to smithi02284960-1 2024-09-21T09:35:48.017 INFO:tasks.rados.rados.0.smithi022.stdout:1293: finishing write tid 5 to smithi02284960-1 2024-09-21T09:35:48.017 INFO:tasks.rados.rados.0.smithi022.stdout:1293: finishing write tid 6 to smithi02284960-1 2024-09-21T09:35:48.017 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 1 v 802 (ObjNum 423 snap 132 seq_num 423) dirty exists 2024-09-21T09:35:48.017 INFO:tasks.rados.rados.0.smithi022.stdout:1293: left oid 1 (ObjNum 423 snap 132 seq_num 423) 2024-09-21T09:35:48.018 INFO:tasks.rados.rados.0.smithi022.stdout:1291: done (3 left) 2024-09-21T09:35:48.018 INFO:tasks.rados.rados.0.smithi022.stdout:1292: done (2 left) 2024-09-21T09:35:48.018 INFO:tasks.rados.rados.0.smithi022.stdout:1293: done (1 left) 2024-09-21T09:35:48.018 INFO:tasks.rados.rados.0.smithi022.stdout:1294: done (0 left) 2024-09-21T09:35:48.018 INFO:tasks.rados.rados.0.smithi022.stdout:1295: read oid 15 snap -1 2024-09-21T09:35:48.018 INFO:tasks.rados.rados.0.smithi022.stdout:1295: expect deleted 2024-09-21T09:35:48.018 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 44 current snap is 132 2024-09-21T09:35:48.018 INFO:tasks.rados.rados.0.smithi022.stdout:1296: seq_num 425 ranges {2670592=720896,3391488=778240,4169728=638976,4808704=188416} 2024-09-21T09:35:48.035 INFO:tasks.rados.rados.0.smithi022.stdout:1296: writing smithi02284960-44 from 2670592 to 3391488 tid 1 2024-09-21T09:35:48.039 INFO:tasks.rados.rados.0.smithi022.stdout:1296: writing smithi02284960-44 from 3391488 to 4169728 tid 2 2024-09-21T09:35:48.043 INFO:tasks.rados.rados.0.smithi022.stdout:1296: writing smithi02284960-44 from 4169728 to 4808704 tid 3 2024-09-21T09:35:48.044 INFO:tasks.rados.rados.0.smithi022.stdout:1296: writing smithi02284960-44 from 4808704 to 4997120 tid 4 2024-09-21T09:35:48.045 INFO:tasks.rados.rados.0.smithi022.stdout:1297: snap_create 2024-09-21T09:35:48.045 INFO:tasks.rados.rados.0.smithi022.stdout:1296: finishing write tid 1 to smithi02284960-44 2024-09-21T09:35:48.045 INFO:tasks.rados.rados.0.smithi022.stdout:1296: finishing write tid 2 to smithi02284960-44 2024-09-21T09:35:48.046 INFO:tasks.rados.rados.0.smithi022.stdout:1296: finishing write tid 3 to smithi02284960-44 2024-09-21T09:35:48.046 INFO:tasks.rados.rados.0.smithi022.stdout:1296: finishing write tid 4 to smithi02284960-44 2024-09-21T09:35:48.046 INFO:tasks.rados.rados.0.smithi022.stdout:1296: finishing write tid 5 to smithi02284960-44 2024-09-21T09:35:48.047 INFO:tasks.rados.rados.0.smithi022.stdout:1296: finishing write tid 6 to smithi02284960-44 2024-09-21T09:35:48.048 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 44 v 807 (ObjNum 425 snap 132 seq_num 425) dirty exists 2024-09-21T09:35:48.048 INFO:tasks.rados.rados.0.smithi022.stdout:1296: left oid 44 (ObjNum 425 snap 132 seq_num 425) 2024-09-21T09:35:48.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:47 smithi064 ceph-mon[86709]: pgmap v91: 121 pgs: 1 active+clean+snaptrim_wait, 2 active+clean+snaptrim, 118 active+clean; 226 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 4.0 MiB/s wr, 19 op/s 2024-09-21T09:35:48.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:47 smithi064 ceph-mon[86709]: osdmap e326: 8 total, 8 up, 8 in 2024-09-21T09:35:48.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:47 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:48.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:47 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-21T09:35:48.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:47 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:35:48.949 INFO:tasks.rados.rados.0.smithi022.stdout:1295: done (2 left) 2024-09-21T09:35:48.949 INFO:tasks.rados.rados.0.smithi022.stdout:1296: done (1 left) 2024-09-21T09:35:48.950 INFO:tasks.rados.rados.0.smithi022.stdout:1297: done (0 left) 2024-09-21T09:35:48.950 INFO:tasks.rados.rados.0.smithi022.stdout:1298: read oid 49 snap -1 2024-09-21T09:35:48.950 INFO:tasks.rados.rados.0.smithi022.stdout:1298: expect (ObjNum 322 snap 97 seq_num 322) 2024-09-21T09:35:48.950 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 45 current snap is 133 2024-09-21T09:35:48.950 INFO:tasks.rados.rados.0.smithi022.stdout:1299: seq_num 426 ranges {0=737280,737280=712704,1449984=802816,2252800=90112} 2024-09-21T09:35:48.953 INFO:tasks.rados.rados.0.smithi022.stdout:1299: writing smithi02284960-45 from 0 to 737280 tid 1 2024-09-21T09:35:48.958 INFO:tasks.rados.rados.0.smithi022.stdout:1299: writing smithi02284960-45 from 737280 to 1449984 tid 2 2024-09-21T09:35:48.964 INFO:tasks.rados.rados.0.smithi022.stdout:1299: writing smithi02284960-45 from 1449984 to 2252800 tid 3 2024-09-21T09:35:48.965 INFO:tasks.rados.rados.0.smithi022.stdout:1299: writing smithi02284960-45 from 2252800 to 2342912 tid 4 2024-09-21T09:35:48.965 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 18 current snap is 133 2024-09-21T09:35:48.965 INFO:tasks.rados.rados.0.smithi022.stdout:1300: seq_num 427 ranges {1998848=761856,2760704=598016,3358720=606208,3964928=401408} 2024-09-21T09:35:48.980 INFO:tasks.rados.rados.0.smithi022.stdout:1300: writing smithi02284960-18 from 1998848 to 2760704 tid 1 2024-09-21T09:35:48.983 INFO:tasks.rados.rados.0.smithi022.stdout:1300: writing smithi02284960-18 from 2760704 to 3358720 tid 2 2024-09-21T09:35:48.987 INFO:tasks.rados.rados.0.smithi022.stdout:1300: writing smithi02284960-18 from 3358720 to 3964928 tid 3 2024-09-21T09:35:48.990 INFO:tasks.rados.rados.0.smithi022.stdout:1300: writing smithi02284960-18 from 3964928 to 4366336 tid 4 2024-09-21T09:35:48.990 INFO:tasks.rados.rados.0.smithi022.stdout:1301: rollback oid 27 current snap is 133 2024-09-21T09:35:48.990 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 27 to 131 2024-09-21T09:35:48.990 INFO:tasks.rados.rados.0.smithi022.stdout:1298: expect (ObjNum 322 snap 97 seq_num 322) 2024-09-21T09:35:49.013 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:48 smithi022 ceph-mon[98615]: Upgrade: Updating osd.0 2024-09-21T09:35:49.013 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:48 smithi022 ceph-mon[98615]: Deploying daemon osd.0 on smithi022 2024-09-21T09:35:49.013 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:48 smithi022 ceph-mon[98615]: osdmap e327: 8 total, 8 up, 8 in 2024-09-21T09:35:49.014 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:48 smithi022 ceph-mon[98615]: osdmap e328: 8 total, 8 up, 8 in 2024-09-21T09:35:49.014 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:48 smithi022 ceph-mon[96736]: Upgrade: Updating osd.0 2024-09-21T09:35:49.014 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:48 smithi022 ceph-mon[96736]: Deploying daemon osd.0 on smithi022 2024-09-21T09:35:49.014 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:48 smithi022 ceph-mon[96736]: osdmap e327: 8 total, 8 up, 8 in 2024-09-21T09:35:49.014 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:48 smithi022 ceph-mon[96736]: osdmap e328: 8 total, 8 up, 8 in 2024-09-21T09:35:49.028 INFO:tasks.rados.rados.0.smithi022.stdout:1299: finishing write tid 1 to smithi02284960-45 2024-09-21T09:35:49.028 INFO:tasks.rados.rados.0.smithi022.stdout:1299: finishing write tid 2 to smithi02284960-45 2024-09-21T09:35:49.028 INFO:tasks.rados.rados.0.smithi022.stdout:1298: done (3 left) 2024-09-21T09:35:49.028 INFO:tasks.rados.rados.0.smithi022.stdout:1302: read oid 34 snap -1 2024-09-21T09:35:49.028 INFO:tasks.rados.rados.0.smithi022.stdout:1302: expect (ObjNum 327 snap 99 seq_num 327) 2024-09-21T09:35:49.028 INFO:tasks.rados.rados.0.smithi022.stdout:1303: read oid 49 snap -1 2024-09-21T09:35:49.028 INFO:tasks.rados.rados.0.smithi022.stdout:1303: expect (ObjNum 322 snap 97 seq_num 322) 2024-09-21T09:35:49.028 INFO:tasks.rados.rados.0.smithi022.stdout:1304: read oid 32 snap 123 2024-09-21T09:35:49.028 INFO:tasks.rados.rados.0.smithi022.stdout:1304: expect (ObjNum 382 snap 119 seq_num 382) 2024-09-21T09:35:49.028 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 25 current snap is 133 2024-09-21T09:35:49.029 INFO:tasks.rados.rados.0.smithi022.stdout:1299: finishing write tid 3 to smithi02284960-45 2024-09-21T09:35:49.029 INFO:tasks.rados.rados.0.smithi022.stdout:1299: finishing write tid 4 to smithi02284960-45 2024-09-21T09:35:49.029 INFO:tasks.rados.rados.0.smithi022.stdout:1299: finishing write tid 5 to smithi02284960-45 2024-09-21T09:35:49.029 INFO:tasks.rados.rados.0.smithi022.stdout:1299: finishing write tid 6 to smithi02284960-45 2024-09-21T09:35:49.029 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 45 v 663 (ObjNum 426 snap 133 seq_num 426) dirty exists 2024-09-21T09:35:49.029 INFO:tasks.rados.rados.0.smithi022.stdout:1299: left oid 45 (ObjNum 426 snap 133 seq_num 426) 2024-09-21T09:35:49.029 INFO:tasks.rados.rados.0.smithi022.stdout:1300: finishing write tid 1 to smithi02284960-18 2024-09-21T09:35:49.029 INFO:tasks.rados.rados.0.smithi022.stdout:1300: finishing write tid 2 to smithi02284960-18 2024-09-21T09:35:49.029 INFO:tasks.rados.rados.0.smithi022.stdout:1300: finishing write tid 3 to smithi02284960-18 2024-09-21T09:35:49.029 INFO:tasks.rados.rados.0.smithi022.stdout:1305: seq_num 428 ranges {983040=778240,1761280=483328,2244608=409600,2654208=417792,3072000=294912} 2024-09-21T09:35:49.037 INFO:tasks.rados.rados.0.smithi022.stdout:1305: writing smithi02284960-25 from 983040 to 1761280 tid 1 2024-09-21T09:35:49.040 INFO:tasks.rados.rados.0.smithi022.stdout:1305: writing smithi02284960-25 from 1761280 to 2244608 tid 2 2024-09-21T09:35:49.042 INFO:tasks.rados.rados.0.smithi022.stdout:1305: writing smithi02284960-25 from 2244608 to 2654208 tid 3 2024-09-21T09:35:49.044 INFO:tasks.rados.rados.0.smithi022.stdout:1305: writing smithi02284960-25 from 2654208 to 3072000 tid 4 2024-09-21T09:35:49.047 INFO:tasks.rados.rados.0.smithi022.stdout:1305: writing smithi02284960-25 from 3072000 to 3366912 tid 5 2024-09-21T09:35:49.047 INFO:tasks.rados.rados.0.smithi022.stdout:1299: done (6 left) 2024-09-21T09:35:49.047 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 38 current snap is 133 2024-09-21T09:35:49.047 INFO:tasks.rados.rados.0.smithi022.stdout:1300: finishing write tid 4 to smithi02284960-18 2024-09-21T09:35:49.047 INFO:tasks.rados.rados.0.smithi022.stdout:1300: finishing write tid 5 to smithi02284960-18 2024-09-21T09:35:49.047 INFO:tasks.rados.rados.0.smithi022.stdout:1300: finishing write tid 6 to smithi02284960-18 2024-09-21T09:35:49.047 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 18 v 358 (ObjNum 427 snap 133 seq_num 427) dirty exists 2024-09-21T09:35:49.047 INFO:tasks.rados.rados.0.smithi022.stdout:1300: left oid 18 (ObjNum 427 snap 133 seq_num 427) 2024-09-21T09:35:49.047 INFO:tasks.rados.rados.0.smithi022.stdout:1301: finishing rollback tid 1 to smithi02284960-27 2024-09-21T09:35:49.047 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 27 v 495 (ObjNum 161 snap 0 seq_num 1) dirty dne 2024-09-21T09:35:49.047 INFO:tasks.rados.rados.0.smithi022.stdout:1306: seq_num 429 ranges {2064384=794624,2859008=770048,3629056=622592,4251648=212992} 2024-09-21T09:35:49.062 INFO:tasks.rados.rados.0.smithi022.stdout:1306: writing smithi02284960-38 from 2064384 to 2859008 tid 1 2024-09-21T09:35:49.066 INFO:tasks.rados.rados.0.smithi022.stdout:1306: writing smithi02284960-38 from 2859008 to 3629056 tid 2 2024-09-21T09:35:49.070 INFO:tasks.rados.rados.0.smithi022.stdout:1306: writing smithi02284960-38 from 3629056 to 4251648 tid 3 2024-09-21T09:35:49.071 INFO:tasks.rados.rados.0.smithi022.stdout:1306: writing smithi02284960-38 from 4251648 to 4464640 tid 4 2024-09-21T09:35:49.071 INFO:tasks.rados.rados.0.smithi022.stdout:1300: done (6 left) 2024-09-21T09:35:49.072 INFO:tasks.rados.rados.0.smithi022.stdout:1301: done (5 left) 2024-09-21T09:35:49.072 INFO:tasks.rados.rados.0.smithi022.stdout:1307: read oid 46 snap -1 2024-09-21T09:35:49.072 INFO:tasks.rados.rados.0.smithi022.stdout:1307: expect (ObjNum 365 snap 115 seq_num 365) 2024-09-21T09:35:49.072 INFO:tasks.rados.rados.0.smithi022.stdout:1308: snap_create 2024-09-21T09:35:49.072 INFO:tasks.rados.rados.0.smithi022.stdout:1304: expect (ObjNum 382 snap 119 seq_num 382) 2024-09-21T09:35:49.115 INFO:tasks.rados.rados.0.smithi022.stdout:1302: expect (ObjNum 327 snap 99 seq_num 327) 2024-09-21T09:35:49.133 INFO:tasks.rados.rados.0.smithi022.stdout:1303: expect (ObjNum 322 snap 97 seq_num 322) 2024-09-21T09:35:49.169 INFO:tasks.rados.rados.0.smithi022.stdout:1305: finishing write tid 1 to smithi02284960-25 2024-09-21T09:35:49.170 INFO:tasks.rados.rados.0.smithi022.stdout:1305: finishing write tid 2 to smithi02284960-25 2024-09-21T09:35:49.170 INFO:tasks.rados.rados.0.smithi022.stdout:1305: finishing write tid 3 to smithi02284960-25 2024-09-21T09:35:49.170 INFO:tasks.rados.rados.0.smithi022.stdout:1305: finishing write tid 4 to smithi02284960-25 2024-09-21T09:35:49.170 INFO:tasks.rados.rados.0.smithi022.stdout:1305: finishing write tid 5 to smithi02284960-25 2024-09-21T09:35:49.170 INFO:tasks.rados.rados.0.smithi022.stdout:1305: finishing write tid 6 to smithi02284960-25 2024-09-21T09:35:49.170 INFO:tasks.rados.rados.0.smithi022.stdout:1305: finishing write tid 7 to smithi02284960-25 2024-09-21T09:35:49.170 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 25 v 766 (ObjNum 428 snap 133 seq_num 428) dirty exists 2024-09-21T09:35:49.170 INFO:tasks.rados.rados.0.smithi022.stdout:1305: left oid 25 (ObjNum 428 snap 133 seq_num 428) 2024-09-21T09:35:49.170 INFO:tasks.rados.rados.0.smithi022.stdout:1306: finishing write tid 1 to smithi02284960-38 2024-09-21T09:35:49.170 INFO:tasks.rados.rados.0.smithi022.stdout:1306: finishing write tid 2 to smithi02284960-38 2024-09-21T09:35:49.170 INFO:tasks.rados.rados.0.smithi022.stdout:1306: finishing write tid 3 to smithi02284960-38 2024-09-21T09:35:49.170 INFO:tasks.rados.rados.0.smithi022.stdout:1306: finishing write tid 4 to smithi02284960-38 2024-09-21T09:35:49.170 INFO:tasks.rados.rados.0.smithi022.stdout:1306: finishing write tid 5 to smithi02284960-38 2024-09-21T09:35:49.170 INFO:tasks.rados.rados.0.smithi022.stdout:1306: finishing write tid 6 to smithi02284960-38 2024-09-21T09:35:49.171 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 38 v 492 (ObjNum 429 snap 133 seq_num 429) dirty exists 2024-09-21T09:35:49.171 INFO:tasks.rados.rados.0.smithi022.stdout:1306: left oid 38 (ObjNum 429 snap 133 seq_num 429) 2024-09-21T09:35:49.171 INFO:tasks.rados.rados.0.smithi022.stdout:1307: expect (ObjNum 365 snap 115 seq_num 365) 2024-09-21T09:35:49.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:48 smithi064 ceph-mon[86709]: Upgrade: Updating osd.0 2024-09-21T09:35:49.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:48 smithi064 ceph-mon[86709]: Deploying daemon osd.0 on smithi022 2024-09-21T09:35:49.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:48 smithi064 ceph-mon[86709]: osdmap e327: 8 total, 8 up, 8 in 2024-09-21T09:35:49.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:48 smithi064 ceph-mon[86709]: osdmap e328: 8 total, 8 up, 8 in 2024-09-21T09:35:49.969 INFO:tasks.rados.rados.0.smithi022.stdout:1302: done (6 left) 2024-09-21T09:35:49.969 INFO:tasks.rados.rados.0.smithi022.stdout:1303: done (5 left) 2024-09-21T09:35:49.969 INFO:tasks.rados.rados.0.smithi022.stdout:1304: done (4 left) 2024-09-21T09:35:49.969 INFO:tasks.rados.rados.0.smithi022.stdout:1305: done (3 left) 2024-09-21T09:35:49.969 INFO:tasks.rados.rados.0.smithi022.stdout:1306: done (2 left) 2024-09-21T09:35:49.969 INFO:tasks.rados.rados.0.smithi022.stdout:1307: done (1 left) 2024-09-21T09:35:49.969 INFO:tasks.rados.rados.0.smithi022.stdout:1308: done (0 left) 2024-09-21T09:35:49.969 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 42 current snap is 134 2024-09-21T09:35:49.970 INFO:tasks.rados.rados.0.smithi022.stdout:1309: seq_num 430 ranges {0=8192} 2024-09-21T09:35:49.970 INFO:tasks.rados.rados.0.smithi022.stdout:1309: writing smithi02284960-42 from 0 to 8192 tid 1 2024-09-21T09:35:49.970 INFO:tasks.rados.rados.0.smithi022.stdout:1310: rmattr oid 36 current snap is 134 2024-09-21T09:35:49.970 INFO:tasks.rados.rados.0.smithi022.stdout:1310: done (1 left) 2024-09-21T09:35:49.970 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 32 current snap is 134 2024-09-21T09:35:49.970 INFO:tasks.rados.rados.0.smithi022.stdout:1311: seq_num 431 ranges {1605632=24576} 2024-09-21T09:35:49.978 INFO:tasks.rados.rados.0.smithi022.stdout:1311: writing smithi02284960-32 from 1605632 to 1630208 tid 1 2024-09-21T09:35:49.978 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 5 current snap is 134 2024-09-21T09:35:49.978 INFO:tasks.rados.rados.0.smithi022.stdout:1312: seq_num 432 ranges {2342912=40960} 2024-09-21T09:35:49.991 INFO:tasks.rados.rados.0.smithi022.stdout:1312: writing smithi02284960-5 from 2342912 to 2383872 tid 1 2024-09-21T09:35:49.991 INFO:tasks.rados.rados.0.smithi022.stdout:1313: snap_remove snap 103 2024-09-21T09:35:50.302 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:49 smithi022 ceph-mon[98615]: pgmap v94: 121 pgs: 121 active+clean; 226 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s rd, 1.1 MiB/s wr, 7 op/s 2024-09-21T09:35:50.303 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:49 smithi022 ceph-mon[96736]: pgmap v94: 121 pgs: 121 active+clean; 226 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s rd, 1.1 MiB/s wr, 7 op/s 2024-09-21T09:35:50.303 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 21 09:35:50 smithi022 systemd[1]: Stopping Ceph osd.0 for d65e801e-77fa-11ef-baf6-efdc52797490... 2024-09-21T09:35:50.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:49 smithi064 ceph-mon[86709]: pgmap v94: 121 pgs: 121 active+clean; 226 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s rd, 1.1 MiB/s wr, 7 op/s 2024-09-21T09:35:50.442 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:35:50 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:35:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:35:50.557 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 21 09:35:50 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-0[43934]: 2024-09-21T09:35:50.299+0000 7ff75b8cf640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T09:35:50.558 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 21 09:35:50 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-0[43934]: 2024-09-21T09:35:50.299+0000 7ff75b8cf640 -1 osd.0 329 *** Got signal Terminated *** 2024-09-21T09:35:50.558 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 21 09:35:50 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-0[43934]: 2024-09-21T09:35:50.299+0000 7ff75b8cf640 -1 osd.0 329 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T09:35:50.981 INFO:tasks.rados.rados.0.smithi022.stdout:1309: finishing write tid 1 to smithi02284960-42 2024-09-21T09:35:50.982 INFO:tasks.rados.rados.0.smithi022.stdout:1309: finishing write tid 2 to smithi02284960-42 2024-09-21T09:35:50.982 INFO:tasks.rados.rados.0.smithi022.stdout:1309: finishing write tid 3 to smithi02284960-42 2024-09-21T09:35:50.982 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 42 v 613 (ObjNum 430 snap 134 seq_num 430) dirty exists 2024-09-21T09:35:50.982 INFO:tasks.rados.rados.0.smithi022.stdout:1309: left oid 42 (ObjNum 430 snap 134 seq_num 430) 2024-09-21T09:35:50.982 INFO:tasks.rados.rados.0.smithi022.stdout:1311: finishing write tid 1 to smithi02284960-32 2024-09-21T09:35:50.982 INFO:tasks.rados.rados.0.smithi022.stdout:1311: finishing write tid 2 to smithi02284960-32 2024-09-21T09:35:50.982 INFO:tasks.rados.rados.0.smithi022.stdout:1311: finishing write tid 3 to smithi02284960-32 2024-09-21T09:35:50.982 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 32 v 826 (ObjNum 431 snap 134 seq_num 431) dirty exists 2024-09-21T09:35:50.982 INFO:tasks.rados.rados.0.smithi022.stdout:1311: left oid 32 (ObjNum 431 snap 134 seq_num 431) 2024-09-21T09:35:50.982 INFO:tasks.rados.rados.0.smithi022.stdout:1312: finishing write tid 1 to smithi02284960-5 2024-09-21T09:35:50.982 INFO:tasks.rados.rados.0.smithi022.stdout:1312: finishing write tid 2 to smithi02284960-5 2024-09-21T09:35:50.983 INFO:tasks.rados.rados.0.smithi022.stdout:1312: finishing write tid 3 to smithi02284960-5 2024-09-21T09:35:50.983 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 5 v 487 (ObjNum 432 snap 134 seq_num 432) dirty exists 2024-09-21T09:35:50.983 INFO:tasks.rados.rados.0.smithi022.stdout:1312: left oid 5 (ObjNum 432 snap 134 seq_num 432) 2024-09-21T09:35:50.983 INFO:tasks.rados.rados.0.smithi022.stdout:1309: done (3 left) 2024-09-21T09:35:50.983 INFO:tasks.rados.rados.0.smithi022.stdout:1311: done (2 left) 2024-09-21T09:35:50.983 INFO:tasks.rados.rados.0.smithi022.stdout:1312: done (1 left) 2024-09-21T09:35:50.983 INFO:tasks.rados.rados.0.smithi022.stdout:1313: done (0 left) 2024-09-21T09:35:50.983 INFO:tasks.rados.rados.0.smithi022.stdout:1314: snap_remove snap 104 2024-09-21T09:35:51.307 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:50 smithi022 ceph-mon[98615]: osdmap e329: 8 total, 8 up, 8 in 2024-09-21T09:35:51.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:50 smithi022 ceph-mon[98615]: osd.0 marked itself down and dead 2024-09-21T09:35:51.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:50 smithi022 ceph-mon[96736]: osdmap e329: 8 total, 8 up, 8 in 2024-09-21T09:35:51.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:50 smithi022 ceph-mon[96736]: osd.0 marked itself down and dead 2024-09-21T09:35:51.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:50 smithi064 ceph-mon[86709]: osdmap e329: 8 total, 8 up, 8 in 2024-09-21T09:35:51.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:50 smithi064 ceph-mon[86709]: osd.0 marked itself down and dead 2024-09-21T09:35:51.603 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 21 09:35:51 smithi022 podman[103331]: 2024-09-21 09:35:51.540896039 +0000 UTC m=+1.365345098 container died fcd81200353496381827fec3569e5afe4b0f9c4d9c74b7744c05171ff121f133 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-0, RELEASE=quincy-fbd1a6a, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731) 2024-09-21T09:35:51.988 INFO:tasks.rados.rados.0.smithi022.stdout:1314: done (0 left) 2024-09-21T09:35:51.988 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 12 current snap is 134 2024-09-21T09:35:51.988 INFO:tasks.rados.rados.0.smithi022.stdout:1315: seq_num 433 ranges {5971968=57344} 2024-09-21T09:35:52.023 INFO:tasks.rados.rados.0.smithi022.stdout:1315: writing smithi02284960-12 from 5971968 to 6029312 tid 1 2024-09-21T09:35:52.023 INFO:tasks.rados.rados.0.smithi022.stdout:1316: read oid 41 snap 130 2024-09-21T09:35:52.023 INFO:tasks.rados.rados.0.smithi022.stdout:1316: expect (ObjNum 412 snap 128 seq_num 412) 2024-09-21T09:35:52.023 INFO:tasks.rados.rados.0.smithi022.stdout:1317: rmattr oid 31 current snap is 134 2024-09-21T09:35:52.023 INFO:tasks.rados.rados.0.smithi022.stdout:1317: done (2 left) 2024-09-21T09:35:52.023 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 29 current snap is 134 2024-09-21T09:35:52.023 INFO:tasks.rados.rados.0.smithi022.stdout:1318: seq_num 434 ranges {0=73728} 2024-09-21T09:35:52.023 INFO:tasks.rados.rados.0.smithi022.stdout:1318: writing smithi02284960-29 from 0 to 73728 tid 1 2024-09-21T09:35:52.023 INFO:tasks.rados.rados.0.smithi022.stdout:1319: read oid 25 snap -1 2024-09-21T09:35:52.023 INFO:tasks.rados.rados.0.smithi022.stdout:1319: expect (ObjNum 428 snap 133 seq_num 428) 2024-09-21T09:35:52.024 INFO:tasks.rados.rados.0.smithi022.stdout:1320: delete oid 27 current snap is 134 2024-09-21T09:35:52.025 INFO:tasks.rados.rados.0.smithi022.stdout:1320: done (4 left) 2024-09-21T09:35:52.025 INFO:tasks.rados.rados.0.smithi022.stdout:1321: rollback oid 40 current snap is 134 2024-09-21T09:35:52.025 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 40 to 131 2024-09-21T09:35:52.025 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 14 current snap is 134 2024-09-21T09:35:52.025 INFO:tasks.rados.rados.0.smithi022.stdout:1322: seq_num 436 ranges {4055040=106496} 2024-09-21T09:35:52.051 INFO:tasks.rados.rados.0.smithi022.stdout:1322: writing smithi02284960-14 from 4055040 to 4161536 tid 1 2024-09-21T09:35:52.051 INFO:tasks.rados.rados.0.smithi022.stdout:1323: read oid 5 snap -1 2024-09-21T09:35:52.051 INFO:tasks.rados.rados.0.smithi022.stdout:1323: expect (ObjNum 432 snap 134 seq_num 432) 2024-09-21T09:35:52.051 INFO:tasks.rados.rados.0.smithi022.stdout:1324: read oid 45 snap 133 2024-09-21T09:35:52.051 INFO:tasks.rados.rados.0.smithi022.stdout:1324: expect (ObjNum 426 snap 133 seq_num 426) 2024-09-21T09:35:52.051 INFO:tasks.rados.rados.0.smithi022.stdout:1325: snap_remove snap 126 2024-09-21T09:35:52.307 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 21 09:35:52 smithi022 podman[103331]: 2024-09-21 09:35:52.00737841 +0000 UTC m=+1.831827459 container cleanup fcd81200353496381827fec3569e5afe4b0f9c4d9c74b7744c05171ff121f133 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-0, CEPH_POINT_RELEASE=, RELEASE=quincy-fbd1a6a, org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T09:35:52.308 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 21 09:35:52 smithi022 bash[103331]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-0 2024-09-21T09:35:52.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:51 smithi022 ceph-mon[98615]: pgmap v97: 121 pgs: 121 active+clean; 230 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 1.6 MiB/s rd, 3.3 MiB/s wr, 13 op/s 2024-09-21T09:35:52.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:51 smithi022 ceph-mon[98615]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T09:35:52.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:51 smithi022 ceph-mon[98615]: osdmap e330: 8 total, 7 up, 8 in 2024-09-21T09:35:52.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:51 smithi022 ceph-mon[96736]: pgmap v97: 121 pgs: 121 active+clean; 230 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 1.6 MiB/s rd, 3.3 MiB/s wr, 13 op/s 2024-09-21T09:35:52.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:51 smithi022 ceph-mon[96736]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T09:35:52.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:51 smithi022 ceph-mon[96736]: osdmap e330: 8 total, 7 up, 8 in 2024-09-21T09:35:52.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:51 smithi064 ceph-mon[86709]: pgmap v97: 121 pgs: 121 active+clean; 230 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 1.6 MiB/s rd, 3.3 MiB/s wr, 13 op/s 2024-09-21T09:35:52.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:51 smithi064 ceph-mon[86709]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T09:35:52.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:51 smithi064 ceph-mon[86709]: osdmap e330: 8 total, 7 up, 8 in 2024-09-21T09:35:52.656 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 21 09:35:52 smithi022 podman[103353]: 2024-09-21 09:35:52.308927776 +0000 UTC m=+0.769502198 container remove fcd81200353496381827fec3569e5afe4b0f9c4d9c74b7744c05171ff121f133 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, ceph=True, RELEASE=quincy-fbd1a6a, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux ) 2024-09-21T09:35:52.911 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 21 09:35:52 smithi022 podman[103454]: 2024-09-21 09:35:52.55830454 +0000 UTC m=+0.022773914 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:35:52.911 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 21 09:35:52 smithi022 podman[103454]: 2024-09-21 09:35:52.676738547 +0000 UTC m=+0.141207908 container create 820b1f3bfdf63fd9b4dd379c9af9b6820ff08f182e5a9e8433e5ebd5e61f38da (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-0-deactivate, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, RELEASE=main-62da54b) 2024-09-21T09:35:52.911 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 21 09:35:52 smithi022 podman[103454]: 2024-09-21 09:35:52.904250185 +0000 UTC m=+0.368719554 container init 820b1f3bfdf63fd9b4dd379c9af9b6820ff08f182e5a9e8433e5ebd5e61f38da (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-0-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=main-62da54b, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0) 2024-09-21T09:35:52.911 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 21 09:35:52 smithi022 podman[103454]: 2024-09-21 09:35:52.909718941 +0000 UTC m=+0.374188307 container start 820b1f3bfdf63fd9b4dd379c9af9b6820ff08f182e5a9e8433e5ebd5e61f38da (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-0-deactivate, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.build-date=20240731, ceph=True, RELEASE=main-62da54b, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T09:35:52.989 INFO:tasks.rados.rados.0.smithi022.stdout:1325: done (8 left) 2024-09-21T09:35:52.989 INFO:tasks.rados.rados.0.smithi022.stdout:1326: read oid 37 snap -1 2024-09-21T09:35:52.990 INFO:tasks.rados.rados.0.smithi022.stdout:1326: expect (ObjNum 355 snap 112 seq_num 355) 2024-09-21T09:35:52.990 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 8 current snap is 134 2024-09-21T09:35:52.990 INFO:tasks.rados.rados.0.smithi022.stdout:1318: finishing write tid 1 to smithi02284960-29 2024-09-21T09:35:52.990 INFO:tasks.rados.rados.0.smithi022.stdout:1327: seq_num 437 ranges {2064384=122880} 2024-09-21T09:35:53.002 INFO:tasks.rados.rados.0.smithi022.stdout:1327: writing smithi02284960-8 from 2064384 to 2187264 tid 1 2024-09-21T09:35:53.002 INFO:tasks.rados.rados.0.smithi022.stdout:1328: rollback oid 39 current snap is 134 2024-09-21T09:35:53.002 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 39 to 123 2024-09-21T09:35:53.002 INFO:tasks.rados.rados.0.smithi022.stdout:1318: finishing write tid 2 to smithi02284960-29 2024-09-21T09:35:53.002 INFO:tasks.rados.rados.0.smithi022.stdout:1315: finishing write tid 1 to smithi02284960-12 2024-09-21T09:35:53.002 INFO:tasks.rados.rados.0.smithi022.stdout:1329: rollback oid 33 current snap is 134 2024-09-21T09:35:53.003 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 33 to 125 2024-09-21T09:35:53.003 INFO:tasks.rados.rados.0.smithi022.stdout:1315: finishing write tid 2 to smithi02284960-12 2024-09-21T09:35:53.003 INFO:tasks.rados.rados.0.smithi022.stdout:1318: finishing write tid 3 to smithi02284960-29 2024-09-21T09:35:53.003 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 29 v 494 (ObjNum 434 snap 134 seq_num 434) dirty exists 2024-09-21T09:35:53.003 INFO:tasks.rados.rados.0.smithi022.stdout:1318: left oid 29 (ObjNum 434 snap 134 seq_num 434) 2024-09-21T09:35:53.003 INFO:tasks.rados.rados.0.smithi022.stdout:1315: finishing write tid 3 to smithi02284960-12 2024-09-21T09:35:53.003 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 12 v 887 (ObjNum 433 snap 134 seq_num 433) dirty exists 2024-09-21T09:35:53.003 INFO:tasks.rados.rados.0.smithi022.stdout:1315: left oid 12 (ObjNum 433 snap 134 seq_num 433) 2024-09-21T09:35:53.003 INFO:tasks.rados.rados.0.smithi022.stdout:1316: expect (ObjNum 412 snap 128 seq_num 412) 2024-09-21T09:35:53.093 INFO:tasks.rados.rados.0.smithi022.stdout:1319: expect (ObjNum 428 snap 133 seq_num 428) 2024-09-21T09:35:53.206 INFO:tasks.rados.rados.0.smithi022.stdout:1321: finishing rollback tid 0 to smithi02284960-40 2024-09-21T09:35:53.206 INFO:tasks.rados.rados.0.smithi022.stdout:1321: finishing rollback tid 1 to smithi02284960-40 2024-09-21T09:35:53.206 INFO:tasks.rados.rados.0.smithi022.stdout:1315: done (11 left) 2024-09-21T09:35:53.206 INFO:tasks.rados.rados.0.smithi022.stdout:1316: done (10 left) 2024-09-21T09:35:53.206 INFO:tasks.rados.rados.0.smithi022.stdout:1318: done (9 left) 2024-09-21T09:35:53.206 INFO:tasks.rados.rados.0.smithi022.stdout:1319: done (8 left) 2024-09-21T09:35:53.206 INFO:tasks.rados.rados.0.smithi022.stdout:1330: rmattr oid 12 current snap is 134 2024-09-21T09:35:53.207 INFO:tasks.rados.rados.0.smithi022.stdout:1321: finishing rollback tid 2 to smithi02284960-40 2024-09-21T09:35:53.207 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 40 v 770 (ObjNum 404 snap 126 seq_num 404) dirty exists 2024-09-21T09:35:53.207 INFO:tasks.rados.rados.0.smithi022.stdout:1322: finishing write tid 1 to smithi02284960-14 2024-09-21T09:35:53.207 INFO:tasks.rados.rados.0.smithi022.stdout:1322: finishing write tid 2 to smithi02284960-14 2024-09-21T09:35:53.207 INFO:tasks.rados.rados.0.smithi022.stdout:1322: finishing write tid 3 to smithi02284960-14 2024-09-21T09:35:53.207 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 14 v 587 (ObjNum 436 snap 134 seq_num 436) dirty exists 2024-09-21T09:35:53.207 INFO:tasks.rados.rados.0.smithi022.stdout:1322: left oid 14 (ObjNum 436 snap 134 seq_num 436) 2024-09-21T09:35:53.207 INFO:tasks.rados.rados.0.smithi022.stdout:1321: done (8 left) 2024-09-21T09:35:53.207 INFO:tasks.rados.rados.0.smithi022.stdout:1322: done (7 left) 2024-09-21T09:35:53.207 INFO:tasks.rados.rados.0.smithi022.stdout:1331: rmattr oid 30 current snap is 134 2024-09-21T09:35:53.207 INFO:tasks.rados.rados.0.smithi022.stdout:1331: done (7 left) 2024-09-21T09:35:53.207 INFO:tasks.rados.rados.0.smithi022.stdout:1332: delete oid 38 current snap is 134 2024-09-21T09:35:53.207 INFO:tasks.rados.rados.0.smithi022.stdout:1323: expect (ObjNum 432 snap 134 seq_num 432) 2024-09-21T09:35:53.307 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:52 smithi022 ceph-mon[98615]: osdmap e331: 8 total, 7 up, 8 in 2024-09-21T09:35:53.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:52 smithi022 ceph-mon[96736]: osdmap e331: 8 total, 7 up, 8 in 2024-09-21T09:35:53.308 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 21 09:35:53 smithi022 podman[103454]: 2024-09-21 09:35:53.021088458 +0000 UTC m=+0.485557827 container attach 820b1f3bfdf63fd9b4dd379c9af9b6820ff08f182e5a9e8433e5ebd5e61f38da (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-0-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , RELEASE=main-62da54b, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, ceph=True) 2024-09-21T09:35:53.308 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 21 09:35:53 smithi022 podman[103454]: 2024-09-21 09:35:53.144892879 +0000 UTC m=+0.609362253 container died 820b1f3bfdf63fd9b4dd379c9af9b6820ff08f182e5a9e8433e5ebd5e61f38da (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-0-deactivate, GIT_CLEAN=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, RELEASE=main-62da54b, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS) 2024-09-21T09:35:53.324 INFO:tasks.rados.rados.0.smithi022.stdout:1324: expect (ObjNum 426 snap 133 seq_num 426) 2024-09-21T09:35:53.387 INFO:tasks.rados.rados.0.smithi022.stdout:1328: finishing rollback tid 1 to smithi02284960-39 2024-09-21T09:35:53.387 INFO:tasks.rados.rados.0.smithi022.stdout:1326: expect (ObjNum 355 snap 112 seq_num 355) 2024-09-21T09:35:53.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:52 smithi064 ceph-mon[86709]: osdmap e331: 8 total, 7 up, 8 in 2024-09-21T09:35:53.460 INFO:tasks.rados.rados.0.smithi022.stdout:1328: finishing rollback tid 2 to smithi02284960-39 2024-09-21T09:35:53.460 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 39 v 477 (ObjNum 360 snap 114 seq_num 360) dirty exists 2024-09-21T09:35:53.460 INFO:tasks.rados.rados.0.smithi022.stdout:1327: finishing write tid 1 to smithi02284960-8 2024-09-21T09:35:53.460 INFO:tasks.rados.rados.0.smithi022.stdout:1323: done (7 left) 2024-09-21T09:35:53.460 INFO:tasks.rados.rados.0.smithi022.stdout:1324: done (6 left) 2024-09-21T09:35:53.460 INFO:tasks.rados.rados.0.smithi022.stdout:1326: done (5 left) 2024-09-21T09:35:53.460 INFO:tasks.rados.rados.0.smithi022.stdout:1328: done (4 left) 2024-09-21T09:35:53.461 INFO:tasks.rados.rados.0.smithi022.stdout:1332: done (3 left) 2024-09-21T09:35:53.461 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 31 current snap is 134 2024-09-21T09:35:53.461 INFO:tasks.rados.rados.0.smithi022.stdout:1329: finishing rollback tid 0 to smithi02284960-33 2024-09-21T09:35:53.461 INFO:tasks.rados.rados.0.smithi022.stdout:1333: seq_num 439 ranges {0=155648} 2024-09-21T09:35:53.462 INFO:tasks.rados.rados.0.smithi022.stdout:1333: writing smithi02284960-31 from 0 to 155648 tid 1 2024-09-21T09:35:53.462 INFO:tasks.rados.rados.0.smithi022.stdout:1334: snap_remove snap 130 2024-09-21T09:35:53.996 INFO:tasks.rados.rados.0.smithi022.stdout:1334: done (4 left) 2024-09-21T09:35:53.996 INFO:tasks.rados.rados.0.smithi022.stdout:1335: read oid 3 snap -1 2024-09-21T09:35:53.996 INFO:tasks.rados.rados.0.smithi022.stdout:1335: expect (ObjNum 360 snap 114 seq_num 360) 2024-09-21T09:35:53.996 INFO:tasks.rados.rados.0.smithi022.stdout:1336: copy_from oid 49 from oid 11 current snap is 134 2024-09-21T09:35:53.996 INFO:tasks.rados.rados.0.smithi022.stdout:1327: finishing write tid 2 to smithi02284960-8 2024-09-21T09:35:53.996 INFO:tasks.rados.rados.0.smithi022.stdout:1329: finishing rollback tid 1 to smithi02284960-33 2024-09-21T09:35:53.997 INFO:tasks.rados.rados.0.smithi022.stdout:1329: finishing rollback tid 2 to smithi02284960-33 2024-09-21T09:35:53.997 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 33 v 830 (ObjNum 309 snap 91 seq_num 309) dirty exists 2024-09-21T09:35:53.997 INFO:tasks.rados.rados.0.smithi022.stdout:1329: done (5 left) 2024-09-21T09:35:53.997 INFO:tasks.rados.rados.0.smithi022.stdout:1337: delete oid 36 current snap is 134 2024-09-21T09:35:53.997 INFO:tasks.rados.rados.0.smithi022.stdout:1327: finishing write tid 3 to smithi02284960-8 2024-09-21T09:35:53.997 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 8 v 504 (ObjNum 437 snap 134 seq_num 437) dirty exists 2024-09-21T09:35:53.997 INFO:tasks.rados.rados.0.smithi022.stdout:1327: left oid 8 (ObjNum 437 snap 134 seq_num 437) 2024-09-21T09:35:53.997 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 12 v 0 (ObjNum 433 snap 134 seq_num 433) dirty exists 2024-09-21T09:35:53.997 INFO:tasks.rados.rados.0.smithi022.stdout:1333: finishing write tid 1 to smithi02284960-31 2024-09-21T09:35:53.997 INFO:tasks.rados.rados.0.smithi022.stdout:1333: finishing write tid 2 to smithi02284960-31 2024-09-21T09:35:53.997 INFO:tasks.rados.rados.0.smithi022.stdout:1333: finishing write tid 3 to smithi02284960-31 2024-09-21T09:35:53.997 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 31 v 891 (ObjNum 439 snap 134 seq_num 439) dirty exists 2024-09-21T09:35:53.997 INFO:tasks.rados.rados.0.smithi022.stdout:1333: left oid 31 (ObjNum 439 snap 134 seq_num 439) 2024-09-21T09:35:54.000 INFO:tasks.rados.rados.0.smithi022.stdout:1327: done (5 left) 2024-09-21T09:35:54.000 INFO:tasks.rados.rados.0.smithi022.stdout:1330: done (4 left) 2024-09-21T09:35:54.000 INFO:tasks.rados.rados.0.smithi022.stdout:1333: done (3 left) 2024-09-21T09:35:54.001 INFO:tasks.rados.rados.0.smithi022.stdout:1337: done (2 left) 2024-09-21T09:35:54.001 INFO:tasks.rados.rados.0.smithi022.stdout:1338: copy_from oid 24 from oid 26 current snap is 134 2024-09-21T09:35:54.001 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 40 current snap is 134 2024-09-21T09:35:54.001 INFO:tasks.rados.rados.0.smithi022.stdout:1339: seq_num 441 ranges {5849088=188416} 2024-09-21T09:35:54.035 INFO:tasks.rados.rados.0.smithi022.stdout:1339: writing smithi02284960-40 from 5849088 to 6037504 tid 1 2024-09-21T09:35:54.035 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 37 current snap is 134 2024-09-21T09:35:54.035 INFO:tasks.rados.rados.0.smithi022.stdout:1336: finishing copy_from racing read to smithi02284960-49 2024-09-21T09:35:54.035 INFO:tasks.rados.rados.0.smithi022.stdout:1340: seq_num 442 ranges {2088960=204800} 2024-09-21T09:35:54.047 INFO:tasks.rados.rados.0.smithi022.stdout:1340: writing smithi02284960-37 from 2088960 to 2293760 tid 1 2024-09-21T09:35:54.047 INFO:tasks.rados.rados.0.smithi022.stdout:1336: finishing copy_from to smithi02284960-49 2024-09-21T09:35:54.047 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 49 v 758 (ObjNum 177 snap 0 seq_num 1) dirty exists 2024-09-21T09:35:54.048 INFO:tasks.rados.rados.0.smithi022.stdout:1335: expect (ObjNum 360 snap 114 seq_num 360) 2024-09-21T09:35:54.057 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:53 smithi022 ceph-mon[96736]: pgmap v100: 121 pgs: 15 stale+active+clean, 106 active+clean; 230 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 451 KiB/s rd, 2.2 MiB/s wr, 6 op/s 2024-09-21T09:35:54.057 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:53 smithi022 ceph-mon[96736]: osdmap e332: 8 total, 7 up, 8 in 2024-09-21T09:35:54.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:53 smithi022 ceph-mon[98615]: pgmap v100: 121 pgs: 15 stale+active+clean, 106 active+clean; 230 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 451 KiB/s rd, 2.2 MiB/s wr, 6 op/s 2024-09-21T09:35:54.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:53 smithi022 ceph-mon[98615]: osdmap e332: 8 total, 7 up, 8 in 2024-09-21T09:35:54.058 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 21 09:35:53 smithi022 podman[103498]: 2024-09-21 09:35:53.758358963 +0000 UTC m=+0.607369233 container remove 820b1f3bfdf63fd9b4dd379c9af9b6820ff08f182e5a9e8433e5ebd5e61f38da (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-0-deactivate, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-62da54b, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD) 2024-09-21T09:35:54.058 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 21 09:35:53 smithi022 systemd[1]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490@osd.0.service: Deactivated successfully. 2024-09-21T09:35:54.058 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 21 09:35:53 smithi022 systemd[1]: Stopped Ceph osd.0 for d65e801e-77fa-11ef-baf6-efdc52797490. 2024-09-21T09:35:54.058 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 21 09:35:53 smithi022 systemd[1]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490@osd.0.service: Consumed 12.796s CPU time. 2024-09-21T09:35:54.058 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 21 09:35:53 smithi022 systemd[1]: Starting Ceph osd.0 for d65e801e-77fa-11ef-baf6-efdc52797490... 2024-09-21T09:35:54.155 INFO:tasks.rados.rados.0.smithi022.stdout:1338: finishing copy_from racing read to smithi02284960-24 2024-09-21T09:35:54.155 INFO:tasks.rados.rados.0.smithi022.stdout:1338: finishing copy_from to smithi02284960-24 2024-09-21T09:35:54.155 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 24 v 591 (ObjNum 411 snap 128 seq_num 411) dirty exists 2024-09-21T09:35:54.155 INFO:tasks.rados.rados.0.smithi022.stdout:1339: finishing write tid 1 to smithi02284960-40 2024-09-21T09:35:54.155 INFO:tasks.rados.rados.0.smithi022.stdout:1339: finishing write tid 2 to smithi02284960-40 2024-09-21T09:35:54.155 INFO:tasks.rados.rados.0.smithi022.stdout:1335: done (4 left) 2024-09-21T09:35:54.155 INFO:tasks.rados.rados.0.smithi022.stdout:1336: done (3 left) 2024-09-21T09:35:54.156 INFO:tasks.rados.rados.0.smithi022.stdout:1338: done (2 left) 2024-09-21T09:35:54.156 INFO:tasks.rados.rados.0.smithi022.stdout:1341: read oid 2 snap -1 2024-09-21T09:35:54.156 INFO:tasks.rados.rados.0.smithi022.stdout:1341: expect deleted 2024-09-21T09:35:54.156 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 24 current snap is 134 2024-09-21T09:35:54.156 INFO:tasks.rados.rados.0.smithi022.stdout:1339: finishing write tid 3 to smithi02284960-40 2024-09-21T09:35:54.156 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 40 v 772 (ObjNum 441 snap 134 seq_num 441) dirty exists 2024-09-21T09:35:54.156 INFO:tasks.rados.rados.0.smithi022.stdout:1339: left oid 40 (ObjNum 441 snap 134 seq_num 441) 2024-09-21T09:35:54.156 INFO:tasks.rados.rados.0.smithi022.stdout:1340: finishing write tid 1 to smithi02284960-37 2024-09-21T09:35:54.156 INFO:tasks.rados.rados.0.smithi022.stdout:1340: finishing write tid 2 to smithi02284960-37 2024-09-21T09:35:54.156 INFO:tasks.rados.rados.0.smithi022.stdout:1340: finishing write tid 3 to smithi02284960-37 2024-09-21T09:35:54.156 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 37 v 898 (ObjNum 442 snap 134 seq_num 442) dirty exists 2024-09-21T09:35:54.156 INFO:tasks.rados.rados.0.smithi022.stdout:1340: left oid 37 (ObjNum 442 snap 134 seq_num 442) 2024-09-21T09:35:54.156 INFO:tasks.rados.rados.0.smithi022.stdout:1342: seq_num 443 ranges {2097152=221184} 2024-09-21T09:35:54.167 INFO:tasks.rados.rados.0.smithi022.stdout:1342: writing smithi02284960-24 from 2097152 to 2318336 tid 1 2024-09-21T09:35:54.167 INFO:tasks.rados.rados.0.smithi022.stdout:1339: done (3 left) 2024-09-21T09:35:54.167 INFO:tasks.rados.rados.0.smithi022.stdout:1340: done (2 left) 2024-09-21T09:35:54.168 INFO:tasks.rados.rados.0.smithi022.stdout:1343: rollback oid 41 current snap is 134 2024-09-21T09:35:54.168 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 41 to 100 2024-09-21T09:35:54.168 INFO:tasks.rados.rados.0.smithi022.stdout:1341: done (2 left) 2024-09-21T09:35:54.168 INFO:tasks.rados.rados.0.smithi022.stdout:1344: copy_from oid 44 from oid 14 current snap is 134 2024-09-21T09:35:54.168 INFO:tasks.rados.rados.0.smithi022.stdout:1345: snap_remove snap 132 2024-09-21T09:35:54.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:53 smithi064 ceph-mon[86709]: pgmap v100: 121 pgs: 15 stale+active+clean, 106 active+clean; 230 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 451 KiB/s rd, 2.2 MiB/s wr, 6 op/s 2024-09-21T09:35:54.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:53 smithi064 ceph-mon[86709]: osdmap e332: 8 total, 7 up, 8 in 2024-09-21T09:35:54.442 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:35:54 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:35:54.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:35:54.489 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 21 09:35:54 smithi022 podman[103603]: 2024-09-21 09:35:54.116769911 +0000 UTC m=+0.018584177 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:35:54.489 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 21 09:35:54 smithi022 podman[103603]: 2024-09-21 09:35:54.268463115 +0000 UTC m=+0.170277352 container create 5b070439b79dd730ce92cf194d1d706275f8fa4bdd377b705373e4078c09f5a1 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-0-activate, RELEASE=main-62da54b, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD) 2024-09-21T09:35:54.808 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 21 09:35:54 smithi022 podman[103603]: 2024-09-21 09:35:54.487778804 +0000 UTC m=+0.389593056 container init 5b070439b79dd730ce92cf194d1d706275f8fa4bdd377b705373e4078c09f5a1 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-0-activate, GIT_CLEAN=True, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, RELEASE=main-62da54b, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T09:35:54.808 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 21 09:35:54 smithi022 podman[103603]: 2024-09-21 09:35:54.493422343 +0000 UTC m=+0.395236581 container start 5b070439b79dd730ce92cf194d1d706275f8fa4bdd377b705373e4078c09f5a1 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-0-activate, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=main-62da54b, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T09:35:54.808 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 21 09:35:54 smithi022 podman[103603]: 2024-09-21 09:35:54.579251189 +0000 UTC m=+0.481065447 container attach 5b070439b79dd730ce92cf194d1d706275f8fa4bdd377b705373e4078c09f5a1 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-0-activate, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-62da54b) 2024-09-21T09:35:54.808 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 21 09:35:54 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-0-activate[103618]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T09:35:54.808 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 21 09:35:54 smithi022 bash[103603]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T09:35:54.808 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 21 09:35:54 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-0-activate[103618]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T09:35:54.808 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 21 09:35:54 smithi022 bash[103603]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T09:35:55.002 INFO:tasks.rados.rados.0.smithi022.stdout:1345: done (3 left) 2024-09-21T09:35:55.002 INFO:tasks.rados.rados.0.smithi022.stdout:1346: copy_from oid 23 from oid 33 current snap is 134 2024-09-21T09:35:55.002 INFO:tasks.rados.rados.0.smithi022.stdout:1342: finishing write tid 1 to smithi02284960-24 2024-09-21T09:35:55.002 INFO:tasks.rados.rados.0.smithi022.stdout:1342: finishing write tid 2 to smithi02284960-24 2024-09-21T09:35:55.002 INFO:tasks.rados.rados.0.smithi022.stdout:1347: snap_create 2024-09-21T09:35:55.002 INFO:tasks.rados.rados.0.smithi022.stdout:1343: finishing rollback tid 0 to smithi02284960-41 2024-09-21T09:35:55.002 INFO:tasks.rados.rados.0.smithi022.stdout:1343: finishing rollback tid 1 to smithi02284960-41 2024-09-21T09:35:55.002 INFO:tasks.rados.rados.0.smithi022.stdout:1343: finishing rollback tid 2 to smithi02284960-41 2024-09-21T09:35:55.002 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 41 v 776 (ObjNum 247 snap 67 seq_num 247) dirty exists 2024-09-21T09:35:55.003 INFO:tasks.rados.rados.0.smithi022.stdout:1342: finishing write tid 3 to smithi02284960-24 2024-09-21T09:35:55.003 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 24 v 900 (ObjNum 443 snap 134 seq_num 443) dirty exists 2024-09-21T09:35:55.003 INFO:tasks.rados.rados.0.smithi022.stdout:1342: left oid 24 (ObjNum 443 snap 134 seq_num 443) 2024-09-21T09:35:55.003 INFO:tasks.rados.rados.0.smithi022.stdout:1344: finishing copy_from racing read to smithi02284960-44 2024-09-21T09:35:55.003 INFO:tasks.rados.rados.0.smithi022.stdout:1344: finishing copy_from to smithi02284960-44 2024-09-21T09:35:55.003 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 44 v 587 (ObjNum 436 snap 134 seq_num 436) dirty exists 2024-09-21T09:35:55.012 INFO:tasks.rados.rados.0.smithi022.stdout:1346: finishing copy_from racing read to smithi02284960-23 2024-09-21T09:35:55.013 INFO:tasks.rados.rados.0.smithi022.stdout:1346: finishing copy_from to smithi02284960-23 2024-09-21T09:35:55.013 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 23 v 830 (ObjNum 309 snap 91 seq_num 309) dirty exists 2024-09-21T09:35:55.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:54 smithi022 ceph-mon[96736]: osdmap e333: 8 total, 7 up, 8 in 2024-09-21T09:35:55.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:54 smithi022 ceph-mon[98615]: osdmap e333: 8 total, 7 up, 8 in 2024-09-21T09:35:55.309 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 21 09:35:55 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-0-activate[103618]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-21T09:35:55.309 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 21 09:35:55 smithi022 bash[103603]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-21T09:35:55.309 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 21 09:35:55 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-0-activate[103618]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-21T09:35:55.309 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 21 09:35:55 smithi022 bash[103603]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-21T09:35:55.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:54 smithi064 ceph-mon[86709]: osdmap e333: 8 total, 7 up, 8 in 2024-09-21T09:35:55.717 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 21 09:35:55 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-0-activate[103618]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-21T09:35:55.717 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 21 09:35:55 smithi022 bash[103603]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-21T09:35:55.717 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 21 09:35:55 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-0-activate[103618]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-21T09:35:55.717 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 21 09:35:55 smithi022 bash[103603]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-21T09:35:55.717 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 21 09:35:55 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-0-activate[103618]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-09-21T09:35:55.717 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 21 09:35:55 smithi022 bash[103603]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-09-21T09:35:55.717 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 21 09:35:55 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-0-activate[103618]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-21T09:35:55.718 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 21 09:35:55 smithi022 bash[103603]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-21T09:35:55.718 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 21 09:35:55 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-0-activate[103618]: --> ceph-volume raw activate successful for osd ID: 0 2024-09-21T09:35:55.718 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 21 09:35:55 smithi022 bash[103603]: --> ceph-volume raw activate successful for osd ID: 0 2024-09-21T09:35:55.718 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 21 09:35:55 smithi022 podman[103603]: 2024-09-21 09:35:55.42300632 +0000 UTC m=+1.324820566 container died 5b070439b79dd730ce92cf194d1d706275f8fa4bdd377b705373e4078c09f5a1 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-0-activate, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, RELEASE=main-62da54b, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T09:35:55.998 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:35:55 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:35:55] "GET /metrics HTTP/1.1" 200 36112 "" "Prometheus/2.51.0" 2024-09-21T09:35:55.999 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:35:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:35:55.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:35:55.999 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:35:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:35:55.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:35:56.006 INFO:tasks.rados.rados.0.smithi022.stdout:1342: done (4 left) 2024-09-21T09:35:56.006 INFO:tasks.rados.rados.0.smithi022.stdout:1343: done (3 left) 2024-09-21T09:35:56.006 INFO:tasks.rados.rados.0.smithi022.stdout:1344: done (2 left) 2024-09-21T09:35:56.006 INFO:tasks.rados.rados.0.smithi022.stdout:1346: done (1 left) 2024-09-21T09:35:56.006 INFO:tasks.rados.rados.0.smithi022.stdout:1347: done (0 left) 2024-09-21T09:35:56.006 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 11 current snap is 135 2024-09-21T09:35:56.006 INFO:tasks.rados.rados.0.smithi022.stdout:1348: seq_num 444 ranges {0=237568} 2024-09-21T09:35:56.007 INFO:tasks.rados.rados.0.smithi022.stdout:1348: writing smithi02284960-11 from 0 to 237568 tid 1 2024-09-21T09:35:56.007 INFO:tasks.rados.rados.0.smithi022.stdout:1349: rmattr oid 30 current snap is 135 2024-09-21T09:35:56.007 INFO:tasks.rados.rados.0.smithi022.stdout:1349: done (1 left) 2024-09-21T09:35:56.007 INFO:tasks.rados.rados.0.smithi022.stdout:1350: delete oid 19 current snap is 135 2024-09-21T09:35:56.013 INFO:tasks.rados.rados.0.smithi022.stdout:1350: done (1 left) 2024-09-21T09:35:56.013 INFO:tasks.rados.rados.0.smithi022.stdout:1351: rollback oid 30 current snap is 135 2024-09-21T09:35:56.013 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 30 to 133 2024-09-21T09:35:56.013 INFO:tasks.rados.rados.0.smithi022.stdout:1352: copy_from oid 23 from oid 17 current snap is 135 2024-09-21T09:35:56.013 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 24 current snap is 135 2024-09-21T09:35:56.013 INFO:tasks.rados.rados.0.smithi022.stdout:1353: seq_num 446 ranges {2318336=278528} 2024-09-21T09:35:56.028 INFO:tasks.rados.rados.0.smithi022.stdout:1353: writing smithi02284960-24 from 2318336 to 2596864 tid 1 2024-09-21T09:35:56.028 INFO:tasks.rados.rados.0.smithi022.stdout:1354: read oid 34 snap 134 2024-09-21T09:35:56.028 INFO:tasks.rados.rados.0.smithi022.stdout:1354: expect (ObjNum 327 snap 99 seq_num 327) 2024-09-21T09:35:56.028 INFO:tasks.rados.rados.0.smithi022.stdout:1355: snap_create 2024-09-21T09:35:56.028 INFO:tasks.rados.rados.0.smithi022.stdout:1348: finishing write tid 1 to smithi02284960-11 2024-09-21T09:35:56.028 INFO:tasks.rados.rados.0.smithi022.stdout:1348: finishing write tid 2 to smithi02284960-11 2024-09-21T09:35:56.028 INFO:tasks.rados.rados.0.smithi022.stdout:1348: finishing write tid 3 to smithi02284960-11 2024-09-21T09:35:56.028 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 11 v 842 (ObjNum 444 snap 135 seq_num 444) dirty exists 2024-09-21T09:35:56.028 INFO:tasks.rados.rados.0.smithi022.stdout:1348: left oid 11 (ObjNum 444 snap 135 seq_num 444) 2024-09-21T09:35:56.028 INFO:tasks.rados.rados.0.smithi022.stdout:1352: finishing copy_from to smithi02284960-23 2024-09-21T09:35:56.028 INFO:tasks.rados.rados.0.smithi022.stdout:1352: got expected ENOENT (src dne) 2024-09-21T09:35:56.028 INFO:tasks.rados.rados.0.smithi022.stdout:1352: finishing copy_from racing read to smithi02284960-23 2024-09-21T09:35:56.029 INFO:tasks.rados.rados.0.smithi022.stdout:1351: finishing rollback tid 1 to smithi02284960-30 2024-09-21T09:35:56.029 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 30 v 475 (ObjNum 257 snap 0 seq_num 3570559888) dirty dne 2024-09-21T09:35:56.031 INFO:tasks.rados.rados.0.smithi022.stdout:1353: finishing write tid 1 to smithi02284960-24 2024-09-21T09:35:56.032 INFO:tasks.rados.rados.0.smithi022.stdout:1353: finishing write tid 2 to smithi02284960-24 2024-09-21T09:35:56.032 INFO:tasks.rados.rados.0.smithi022.stdout:1354: expect (ObjNum 327 snap 99 seq_num 327) 2024-09-21T09:35:56.050 INFO:tasks.rados.rados.0.smithi022.stdout:1353: finishing write tid 3 to smithi02284960-24 2024-09-21T09:35:56.050 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 24 v 911 (ObjNum 446 snap 135 seq_num 446) dirty exists 2024-09-21T09:35:56.050 INFO:tasks.rados.rados.0.smithi022.stdout:1353: left oid 24 (ObjNum 446 snap 135 seq_num 446) 2024-09-21T09:35:56.308 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 21 09:35:56 smithi022 podman[103765]: 2024-09-21 09:35:56.108670011 +0000 UTC m=+0.679054723 container remove 5b070439b79dd730ce92cf194d1d706275f8fa4bdd377b705373e4078c09f5a1 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-0-activate, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=main-62da54b, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T09:35:56.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:55 smithi022 ceph-mon[96736]: pgmap v103: 121 pgs: 37 active+undersized, 15 active+undersized+degraded, 69 active+clean; 238 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 2.4 MiB/s wr, 22 op/s; 97/558 objects degraded (17.384%) 2024-09-21T09:35:56.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:55 smithi022 ceph-mon[96736]: Health check failed: Degraded data redundancy: 97/558 objects degraded (17.384%), 15 pgs degraded (PG_DEGRADED) 2024-09-21T09:35:56.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:55 smithi022 ceph-mon[96736]: osdmap e334: 8 total, 7 up, 8 in 2024-09-21T09:35:56.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:55 smithi022 ceph-mon[98615]: pgmap v103: 121 pgs: 37 active+undersized, 15 active+undersized+degraded, 69 active+clean; 238 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 2.4 MiB/s wr, 22 op/s; 97/558 objects degraded (17.384%) 2024-09-21T09:35:56.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:55 smithi022 ceph-mon[98615]: Health check failed: Degraded data redundancy: 97/558 objects degraded (17.384%), 15 pgs degraded (PG_DEGRADED) 2024-09-21T09:35:56.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:55 smithi022 ceph-mon[98615]: osdmap e334: 8 total, 7 up, 8 in 2024-09-21T09:35:56.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:55 smithi064 ceph-mon[86709]: pgmap v103: 121 pgs: 37 active+undersized, 15 active+undersized+degraded, 69 active+clean; 238 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 2.4 MiB/s wr, 22 op/s; 97/558 objects degraded (17.384%) 2024-09-21T09:35:56.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:55 smithi064 ceph-mon[86709]: Health check failed: Degraded data redundancy: 97/558 objects degraded (17.384%), 15 pgs degraded (PG_DEGRADED) 2024-09-21T09:35:56.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:55 smithi064 ceph-mon[86709]: osdmap e334: 8 total, 7 up, 8 in 2024-09-21T09:35:56.580 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 21 09:35:56 smithi022 podman[103828]: 2024-09-21 09:35:56.227451345 +0000 UTC m=+0.016944284 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:35:56.580 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 21 09:35:56 smithi022 podman[103828]: 2024-09-21 09:35:56.376385192 +0000 UTC m=+0.165878121 container create 40ce23a2b5692f7801d559f179ec2889d38d95b7dc65fc444d6d3438210e26e3 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True, ceph=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=main-62da54b) 2024-09-21T09:35:57.008 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 21 09:35:56 smithi022 podman[103828]: 2024-09-21 09:35:56.578720877 +0000 UTC m=+0.368213814 container init 40ce23a2b5692f7801d559f179ec2889d38d95b7dc65fc444d6d3438210e26e3 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-0, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, ceph=True, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, RELEASE=main-62da54b, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T09:35:57.008 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 21 09:35:56 smithi022 podman[103828]: 2024-09-21 09:35:56.584054523 +0000 UTC m=+0.373547452 container start 40ce23a2b5692f7801d559f179ec2889d38d95b7dc65fc444d6d3438210e26e3 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-0, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, ceph=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=main-62da54b, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2) 2024-09-21T09:35:57.009 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 21 09:35:56 smithi022 ceph-osd[103866]: -- 172.21.15.22:0/176622333 <== mon.2 v2:172.21.15.22:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55ec5ccfda40 con 0x55ec5dad0400 2024-09-21T09:35:57.009 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 21 09:35:56 smithi022 bash[103828]: 40ce23a2b5692f7801d559f179ec2889d38d95b7dc65fc444d6d3438210e26e3 2024-09-21T09:35:57.009 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 21 09:35:56 smithi022 systemd[1]: Started Ceph osd.0 for d65e801e-77fa-11ef-baf6-efdc52797490. 2024-09-21T09:35:57.009 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:57 smithi022 ceph-mon[96736]: osdmap e335: 8 total, 7 up, 8 in 2024-09-21T09:35:57.009 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:57 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:57.009 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:57 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:57.009 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:57 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:35:57.012 INFO:tasks.rados.rados.0.smithi022.stdout:1348: done (5 left) 2024-09-21T09:35:57.012 INFO:tasks.rados.rados.0.smithi022.stdout:1351: done (4 left) 2024-09-21T09:35:57.012 INFO:tasks.rados.rados.0.smithi022.stdout:1352: done (3 left) 2024-09-21T09:35:57.012 INFO:tasks.rados.rados.0.smithi022.stdout:1353: done (2 left) 2024-09-21T09:35:57.012 INFO:tasks.rados.rados.0.smithi022.stdout:1354: done (1 left) 2024-09-21T09:35:57.012 INFO:tasks.rados.rados.0.smithi022.stdout:1355: done (0 left) 2024-09-21T09:35:57.012 INFO:tasks.rados.rados.0.smithi022.stdout:1356: delete oid 26 current snap is 136 2024-09-21T09:35:57.019 INFO:tasks.rados.rados.0.smithi022.stdout:1356: done (0 left) 2024-09-21T09:35:57.019 INFO:tasks.rados.rados.0.smithi022.stdout:1357: delete oid 5 current snap is 136 2024-09-21T09:35:57.023 INFO:tasks.rados.rados.0.smithi022.stdout:1357: done (0 left) 2024-09-21T09:35:57.023 INFO:tasks.rados.rados.0.smithi022.stdout:1358: copy_from oid 21 from oid 29 current snap is 136 2024-09-21T09:35:57.023 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 3 current snap is 136 2024-09-21T09:35:57.023 INFO:tasks.rados.rados.0.smithi022.stdout:1359: seq_num 449 ranges {2670592=327680} 2024-09-21T09:35:57.040 INFO:tasks.rados.rados.0.smithi022.stdout:1359: writing smithi02284960-3 from 2670592 to 2998272 tid 1 2024-09-21T09:35:57.041 INFO:tasks.rados.rados.0.smithi022.stdout:1360: copy_from oid 35 from oid 34 current snap is 136 2024-09-21T09:35:57.041 INFO:tasks.rados.rados.0.smithi022.stdout:1358: finishing copy_from racing read to smithi02284960-21 2024-09-21T09:35:57.041 INFO:tasks.rados.rados.0.smithi022.stdout:1358: finishing copy_from to smithi02284960-21 2024-09-21T09:35:57.041 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 21 v 494 (ObjNum 434 snap 134 seq_num 434) dirty exists 2024-09-21T09:35:57.041 INFO:tasks.rados.rados.0.smithi022.stdout:1358: done (2 left) 2024-09-21T09:35:57.041 INFO:tasks.rados.rados.0.smithi022.stdout:1361: read oid 38 snap -1 2024-09-21T09:35:57.041 INFO:tasks.rados.rados.0.smithi022.stdout:1361: expect deleted 2024-09-21T09:35:57.041 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 26 current snap is 136 2024-09-21T09:35:57.041 INFO:tasks.rados.rados.0.smithi022.stdout:1362: seq_num 450 ranges {0=344064} 2024-09-21T09:35:57.043 INFO:tasks.rados.rados.0.smithi022.stdout:1362: writing smithi02284960-26 from 0 to 344064 tid 1 2024-09-21T09:35:57.043 INFO:tasks.rados.rados.0.smithi022.stdout:1363: copy_from oid 30 from oid 31 current snap is 136 2024-09-21T09:35:57.043 INFO:tasks.rados.rados.0.smithi022.stdout:1361: done (4 left) 2024-09-21T09:35:57.043 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 6 current snap is 136 2024-09-21T09:35:57.043 INFO:tasks.rados.rados.0.smithi022.stdout:1364: seq_num 451 ranges {3063808=360448} 2024-09-21T09:35:57.062 INFO:tasks.rados.rados.0.smithi022.stdout:1364: writing smithi02284960-6 from 3063808 to 3424256 tid 1 2024-09-21T09:35:57.062 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 41 current snap is 136 2024-09-21T09:35:57.062 INFO:tasks.rados.rados.0.smithi022.stdout:1359: finishing write tid 1 to smithi02284960-3 2024-09-21T09:35:57.062 INFO:tasks.rados.rados.0.smithi022.stdout:1365: seq_num 452 ranges {3293184=376832} 2024-09-21T09:35:57.082 INFO:tasks.rados.rados.0.smithi022.stdout:1365: writing smithi02284960-41 from 3293184 to 3670016 tid 1 2024-09-21T09:35:57.083 INFO:tasks.rados.rados.0.smithi022.stdout:1366: read oid 18 snap -1 2024-09-21T09:35:57.083 INFO:tasks.rados.rados.0.smithi022.stdout:1366: expect (ObjNum 427 snap 133 seq_num 427) 2024-09-21T09:35:57.083 INFO:tasks.rados.rados.0.smithi022.stdout:1367: snap_create 2024-09-21T09:35:57.083 INFO:tasks.rados.rados.0.smithi022.stdout:1359: finishing write tid 2 to smithi02284960-3 2024-09-21T09:35:57.083 INFO:tasks.rados.rados.0.smithi022.stdout:1362: finishing write tid 1 to smithi02284960-26 2024-09-21T09:35:57.083 INFO:tasks.rados.rados.0.smithi022.stdout:1362: finishing write tid 2 to smithi02284960-26 2024-09-21T09:35:57.083 INFO:tasks.rados.rados.0.smithi022.stdout:1362: finishing write tid 3 to smithi02284960-26 2024-09-21T09:35:57.083 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 26 v 628 (ObjNum 450 snap 136 seq_num 450) dirty exists 2024-09-21T09:35:57.083 INFO:tasks.rados.rados.0.smithi022.stdout:1362: left oid 26 (ObjNum 450 snap 136 seq_num 450) 2024-09-21T09:35:57.083 INFO:tasks.rados.rados.0.smithi022.stdout:1359: finishing write tid 3 to smithi02284960-3 2024-09-21T09:35:57.083 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 3 v 471 (ObjNum 449 snap 136 seq_num 449) dirty exists 2024-09-21T09:35:57.083 INFO:tasks.rados.rados.0.smithi022.stdout:1359: left oid 3 (ObjNum 449 snap 136 seq_num 449) 2024-09-21T09:35:57.083 INFO:tasks.rados.rados.0.smithi022.stdout:1363: finishing copy_from racing read to smithi02284960-30 2024-09-21T09:35:57.083 INFO:tasks.rados.rados.0.smithi022.stdout:1360: finishing copy_from racing read to smithi02284960-35 2024-09-21T09:35:57.083 INFO:tasks.rados.rados.0.smithi022.stdout:1363: finishing copy_from to smithi02284960-30 2024-09-21T09:35:57.084 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 30 v 891 (ObjNum 439 snap 134 seq_num 439) dirty exists 2024-09-21T09:35:57.084 INFO:tasks.rados.rados.0.smithi022.stdout:1360: finishing copy_from to smithi02284960-35 2024-09-21T09:35:57.084 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 35 v 262 (ObjNum 327 snap 99 seq_num 327) dirty exists 2024-09-21T09:35:57.084 INFO:tasks.rados.rados.0.smithi022.stdout:1364: finishing write tid 1 to smithi02284960-6 2024-09-21T09:35:57.084 INFO:tasks.rados.rados.0.smithi022.stdout:1364: finishing write tid 2 to smithi02284960-6 2024-09-21T09:35:57.084 INFO:tasks.rados.rados.0.smithi022.stdout:1364: finishing write tid 3 to smithi02284960-6 2024-09-21T09:35:57.084 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 6 v 833 (ObjNum 451 snap 136 seq_num 451) dirty exists 2024-09-21T09:35:57.084 INFO:tasks.rados.rados.0.smithi022.stdout:1364: left oid 6 (ObjNum 451 snap 136 seq_num 451) 2024-09-21T09:35:57.086 INFO:tasks.rados.rados.0.smithi022.stdout:1365: finishing write tid 1 to smithi02284960-41 2024-09-21T09:35:57.086 INFO:tasks.rados.rados.0.smithi022.stdout:1365: finishing write tid 2 to smithi02284960-41 2024-09-21T09:35:57.088 INFO:tasks.rados.rados.0.smithi022.stdout:1365: finishing write tid 3 to smithi02284960-41 2024-09-21T09:35:57.088 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 41 v 779 (ObjNum 452 snap 136 seq_num 452) dirty exists 2024-09-21T09:35:57.088 INFO:tasks.rados.rados.0.smithi022.stdout:1365: left oid 41 (ObjNum 452 snap 136 seq_num 452) 2024-09-21T09:35:57.097 INFO:tasks.rados.rados.0.smithi022.stdout:1366: expect (ObjNum 427 snap 133 seq_num 427) 2024-09-21T09:35:57.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:57 smithi022 ceph-mon[98615]: osdmap e335: 8 total, 7 up, 8 in 2024-09-21T09:35:57.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:57 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:57.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:57 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:57.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:57 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:35:57.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:57 smithi064 ceph-mon[86709]: osdmap e335: 8 total, 7 up, 8 in 2024-09-21T09:35:57.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:57 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:57.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:57 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:57.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:57 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:35:57.442 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:35:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:35:56.950Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi022\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:35:58.017 INFO:tasks.rados.rados.0.smithi022.stdout:1359: done (7 left) 2024-09-21T09:35:58.017 INFO:tasks.rados.rados.0.smithi022.stdout:1360: done (6 left) 2024-09-21T09:35:58.017 INFO:tasks.rados.rados.0.smithi022.stdout:1362: done (5 left) 2024-09-21T09:35:58.017 INFO:tasks.rados.rados.0.smithi022.stdout:1363: done (4 left) 2024-09-21T09:35:58.017 INFO:tasks.rados.rados.0.smithi022.stdout:1364: done (3 left) 2024-09-21T09:35:58.017 INFO:tasks.rados.rados.0.smithi022.stdout:1365: done (2 left) 2024-09-21T09:35:58.017 INFO:tasks.rados.rados.0.smithi022.stdout:1366: done (1 left) 2024-09-21T09:35:58.018 INFO:tasks.rados.rados.0.smithi022.stdout:1367: done (0 left) 2024-09-21T09:35:58.018 INFO:tasks.rados.rados.0.smithi022.stdout:1368: setattr oid 23 current snap is 137 2024-09-21T09:35:58.018 INFO:tasks.rados.rados.0.smithi022.stdout:1369: snap_remove snap 133 2024-09-21T09:35:58.262 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:58 smithi022 ceph-mon[96736]: pgmap v106: 121 pgs: 37 active+undersized, 15 active+undersized+degraded, 69 active+clean; 238 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 2.4 MiB/s wr, 22 op/s; 97/558 objects degraded (17.384%) 2024-09-21T09:35:58.262 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:58 smithi022 ceph-mon[96736]: osdmap e336: 8 total, 7 up, 8 in 2024-09-21T09:35:58.262 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:58 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:58.262 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:58 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:35:58.263 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:58 smithi022 ceph-mon[98615]: pgmap v106: 121 pgs: 37 active+undersized, 15 active+undersized+degraded, 69 active+clean; 238 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 2.4 MiB/s wr, 22 op/s; 97/558 objects degraded (17.384%) 2024-09-21T09:35:58.263 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:58 smithi022 ceph-mon[98615]: osdmap e336: 8 total, 7 up, 8 in 2024-09-21T09:35:58.263 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:58 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:58.263 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:58 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:35:58.263 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 21 09:35:58 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-0[103843]: 2024-09-21T09:35:58.241+0000 7f9f645e3640 -1 Falling back to public interface 2024-09-21T09:35:58.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:58 smithi064 ceph-mon[86709]: pgmap v106: 121 pgs: 37 active+undersized, 15 active+undersized+degraded, 69 active+clean; 238 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 2.4 MiB/s wr, 22 op/s; 97/558 objects degraded (17.384%) 2024-09-21T09:35:58.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:58 smithi064 ceph-mon[86709]: osdmap e336: 8 total, 7 up, 8 in 2024-09-21T09:35:58.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:58 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:35:58.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:58 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:35:58.950 INFO:tasks.rados.rados.0.smithi022.stdout:1369: done (1 left) 2024-09-21T09:35:58.950 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 45 current snap is 137 2024-09-21T09:35:58.950 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 23 v 831 (ObjNum 309 snap 91 seq_num 309) dirty exists 2024-09-21T09:35:58.951 INFO:tasks.rados.rados.0.smithi022.stdout:1370: seq_num 453 ranges {2342912=393216} 2024-09-21T09:35:58.965 INFO:tasks.rados.rados.0.smithi022.stdout:1370: writing smithi02284960-45 from 2342912 to 2736128 tid 1 2024-09-21T09:35:58.966 INFO:tasks.rados.rados.0.smithi022.stdout:1368: done (1 left) 2024-09-21T09:35:58.966 INFO:tasks.rados.rados.0.smithi022.stdout:1371: snap_remove snap 136 2024-09-21T09:35:59.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:59 smithi022 ceph-mon[98615]: osdmap e337: 8 total, 7 up, 8 in 2024-09-21T09:35:59.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:35:59 smithi022 ceph-mon[98615]: osdmap e338: 8 total, 7 up, 8 in 2024-09-21T09:35:59.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:59 smithi022 ceph-mon[96736]: osdmap e337: 8 total, 7 up, 8 in 2024-09-21T09:35:59.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:35:59 smithi022 ceph-mon[96736]: osdmap e338: 8 total, 7 up, 8 in 2024-09-21T09:35:59.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:59 smithi064 ceph-mon[86709]: osdmap e337: 8 total, 7 up, 8 in 2024-09-21T09:35:59.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:35:59 smithi064 ceph-mon[86709]: osdmap e338: 8 total, 7 up, 8 in 2024-09-21T09:35:59.952 INFO:tasks.rados.rados.0.smithi022.stdout:1371: done (1 left) 2024-09-21T09:35:59.953 INFO:tasks.rados.rados.0.smithi022.stdout:1372: delete oid 14 current snap is 137 2024-09-21T09:35:59.953 INFO:tasks.rados.rados.0.smithi022.stdout:1370: finishing write tid 1 to smithi02284960-45 2024-09-21T09:35:59.953 INFO:tasks.rados.rados.0.smithi022.stdout:1370: finishing write tid 2 to smithi02284960-45 2024-09-21T09:35:59.953 INFO:tasks.rados.rados.0.smithi022.stdout:1370: finishing write tid 3 to smithi02284960-45 2024-09-21T09:35:59.953 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 45 v 666 (ObjNum 453 snap 137 seq_num 453) dirty exists 2024-09-21T09:35:59.953 INFO:tasks.rados.rados.0.smithi022.stdout:1370: left oid 45 (ObjNum 453 snap 137 seq_num 453) 2024-09-21T09:35:59.958 INFO:tasks.rados.rados.0.smithi022.stdout:1370: done (1 left) 2024-09-21T09:35:59.958 INFO:tasks.rados.rados.0.smithi022.stdout:1372: done (0 left) 2024-09-21T09:35:59.958 INFO:tasks.rados.rados.0.smithi022.stdout:1373: rollback oid 50 current snap is 137 2024-09-21T09:35:59.958 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 50 to 96 2024-09-21T09:35:59.959 INFO:tasks.rados.rados.0.smithi022.stdout:1374: copy_from oid 39 from oid 10 current snap is 137 2024-09-21T09:35:59.959 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 2 current snap is 137 2024-09-21T09:35:59.959 INFO:tasks.rados.rados.0.smithi022.stdout:1375: seq_num 455 ranges {0=425984} 2024-09-21T09:35:59.961 INFO:tasks.rados.rados.0.smithi022.stdout:1375: writing smithi02284960-2 from 0 to 425984 tid 1 2024-09-21T09:35:59.961 INFO:tasks.rados.rados.0.smithi022.stdout:1376: read oid 36 snap -1 2024-09-21T09:35:59.961 INFO:tasks.rados.rados.0.smithi022.stdout:1376: expect deleted 2024-09-21T09:35:59.962 INFO:tasks.rados.rados.0.smithi022.stdout:1377: rollback oid 32 current snap is 137 2024-09-21T09:35:59.962 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 32 to 127 2024-09-21T09:35:59.962 INFO:tasks.rados.rados.0.smithi022.stdout:1373: finishing rollback tid 1 to smithi02284960-50 2024-09-21T09:35:59.962 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 50 v 658 (ObjNum 145 snap 0 seq_num 3570562928) dirty dne 2024-09-21T09:35:59.962 INFO:tasks.rados.rados.0.smithi022.stdout:1373: done (4 left) 2024-09-21T09:35:59.962 INFO:tasks.rados.rados.0.smithi022.stdout:1378: rollback oid 25 current snap is 137 2024-09-21T09:35:59.962 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 25 to 135 2024-09-21T09:35:59.962 INFO:tasks.rados.rados.0.smithi022.stdout:1379: copy_from oid 34 from oid 3 current snap is 137 2024-09-21T09:35:59.962 INFO:tasks.rados.rados.0.smithi022.stdout:1380: snap_create 2024-09-21T09:35:59.966 INFO:tasks.rados.rados.0.smithi022.stdout:1375: finishing write tid 1 to smithi02284960-2 2024-09-21T09:35:59.966 INFO:tasks.rados.rados.0.smithi022.stdout:1375: finishing write tid 2 to smithi02284960-2 2024-09-21T09:35:59.967 INFO:tasks.rados.rados.0.smithi022.stdout:1378: finishing rollback tid 0 to smithi02284960-25 2024-09-21T09:35:59.967 INFO:tasks.rados.rados.0.smithi022.stdout:1377: finishing rollback tid 0 to smithi02284960-32 2024-09-21T09:35:59.967 INFO:tasks.rados.rados.0.smithi022.stdout:1378: finishing rollback tid 1 to smithi02284960-25 2024-09-21T09:35:59.968 INFO:tasks.rados.rados.0.smithi022.stdout:1378: finishing rollback tid 2 to smithi02284960-25 2024-09-21T09:35:59.968 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 25 v 783 (ObjNum 428 snap 133 seq_num 428) dirty exists 2024-09-21T09:35:59.968 INFO:tasks.rados.rados.0.smithi022.stdout:1377: finishing rollback tid 1 to smithi02284960-32 2024-09-21T09:35:59.968 INFO:tasks.rados.rados.0.smithi022.stdout:1377: finishing rollback tid 2 to smithi02284960-32 2024-09-21T09:35:59.968 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 32 v 837 (ObjNum 382 snap 119 seq_num 382) dirty exists 2024-09-21T09:35:59.969 INFO:tasks.rados.rados.0.smithi022.stdout:1375: finishing write tid 3 to smithi02284960-2 2024-09-21T09:35:59.969 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 2 v 496 (ObjNum 455 snap 137 seq_num 455) dirty exists 2024-09-21T09:35:59.969 INFO:tasks.rados.rados.0.smithi022.stdout:1375: left oid 2 (ObjNum 455 snap 137 seq_num 455) 2024-09-21T09:35:59.970 INFO:tasks.rados.rados.0.smithi022.stdout:1374: finishing copy_from racing read to smithi02284960-39 2024-09-21T09:35:59.970 INFO:tasks.rados.rados.0.smithi022.stdout:1374: finishing copy_from to smithi02284960-39 2024-09-21T09:35:59.971 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 39 v 535 (ObjNum 311 snap 94 seq_num 311) dirty exists 2024-09-21T09:35:59.978 INFO:tasks.rados.rados.0.smithi022.stdout:1379: finishing copy_from racing read to smithi02284960-34 2024-09-21T09:35:59.980 INFO:tasks.rados.rados.0.smithi022.stdout:1379: finishing copy_from to smithi02284960-34 2024-09-21T09:35:59.980 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 34 v 471 (ObjNum 449 snap 136 seq_num 449) dirty exists 2024-09-21T09:36:00.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:00 smithi022 ceph-mon[98615]: pgmap v109: 121 pgs: 37 active+undersized, 15 active+undersized+degraded, 69 active+clean; 245 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 3.1 MiB/s rd, 1.8 MiB/s wr, 11 op/s; 101/585 objects degraded (17.265%) 2024-09-21T09:36:00.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:00 smithi022 ceph-mon[98615]: osdmap e339: 8 total, 7 up, 8 in 2024-09-21T09:36:00.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:00 smithi022 ceph-mon[96736]: pgmap v109: 121 pgs: 37 active+undersized, 15 active+undersized+degraded, 69 active+clean; 245 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 3.1 MiB/s rd, 1.8 MiB/s wr, 11 op/s; 101/585 objects degraded (17.265%) 2024-09-21T09:36:00.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:00 smithi022 ceph-mon[96736]: osdmap e339: 8 total, 7 up, 8 in 2024-09-21T09:36:00.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:00 smithi064 ceph-mon[86709]: pgmap v109: 121 pgs: 37 active+undersized, 15 active+undersized+degraded, 69 active+clean; 245 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 3.1 MiB/s rd, 1.8 MiB/s wr, 11 op/s; 101/585 objects degraded (17.265%) 2024-09-21T09:36:00.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:00 smithi064 ceph-mon[86709]: osdmap e339: 8 total, 7 up, 8 in 2024-09-21T09:36:00.442 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:36:00 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:36:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:36:00.959 INFO:tasks.rados.rados.0.smithi022.stdout:1374: done (6 left) 2024-09-21T09:36:00.959 INFO:tasks.rados.rados.0.smithi022.stdout:1375: done (5 left) 2024-09-21T09:36:00.959 INFO:tasks.rados.rados.0.smithi022.stdout:1376: done (4 left) 2024-09-21T09:36:00.960 INFO:tasks.rados.rados.0.smithi022.stdout:1377: done (3 left) 2024-09-21T09:36:00.960 INFO:tasks.rados.rados.0.smithi022.stdout:1378: done (2 left) 2024-09-21T09:36:00.960 INFO:tasks.rados.rados.0.smithi022.stdout:1379: done (1 left) 2024-09-21T09:36:00.960 INFO:tasks.rados.rados.0.smithi022.stdout:1380: done (0 left) 2024-09-21T09:36:00.960 INFO:tasks.rados.rados.0.smithi022.stdout:1381: read oid 31 snap 100 2024-09-21T09:36:00.960 INFO:tasks.rados.rados.0.smithi022.stdout:1381: expect deleted 2024-09-21T09:36:00.960 INFO:tasks.rados.rados.0.smithi022.stdout:1382: delete oid 4 current snap is 138 2024-09-21T09:36:00.962 INFO:tasks.rados.rados.0.smithi022.stdout:1381: done (1 left) 2024-09-21T09:36:00.963 INFO:tasks.rados.rados.0.smithi022.stdout:1382: done (0 left) 2024-09-21T09:36:00.963 INFO:tasks.rados.rados.0.smithi022.stdout:1383: snap_remove snap 135 2024-09-21T09:36:01.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:01 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:36:01.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:01 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:36:01.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:01 smithi022 ceph-mon[98615]: pgmap v112: 121 pgs: 37 active+undersized, 15 active+undersized+degraded, 69 active+clean; 252 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.5 MiB/s rd, 2.8 MiB/s wr, 21 op/s; 103/609 objects degraded (16.913%) 2024-09-21T09:36:01.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:01 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:36:01.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:01 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:36:01.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:01 smithi022 ceph-mon[98615]: osdmap e340: 8 total, 7 up, 8 in 2024-09-21T09:36:01.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:01 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:36:01.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:01 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:36:01.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:01 smithi022 ceph-mon[96736]: pgmap v112: 121 pgs: 37 active+undersized, 15 active+undersized+degraded, 69 active+clean; 252 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.5 MiB/s rd, 2.8 MiB/s wr, 21 op/s; 103/609 objects degraded (16.913%) 2024-09-21T09:36:01.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:01 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:36:01.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:01 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:36:01.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:01 smithi022 ceph-mon[96736]: osdmap e340: 8 total, 7 up, 8 in 2024-09-21T09:36:01.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:01 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:36:01.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:01 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:36:01.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:01 smithi064 ceph-mon[86709]: pgmap v112: 121 pgs: 37 active+undersized, 15 active+undersized+degraded, 69 active+clean; 252 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.5 MiB/s rd, 2.8 MiB/s wr, 21 op/s; 103/609 objects degraded (16.913%) 2024-09-21T09:36:01.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:01 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:36:01.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:01 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:36:01.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:01 smithi064 ceph-mon[86709]: osdmap e340: 8 total, 7 up, 8 in 2024-09-21T09:36:01.960 INFO:tasks.rados.rados.0.smithi022.stdout:1383: done (0 left) 2024-09-21T09:36:01.960 INFO:tasks.rados.rados.0.smithi022.stdout:1384: read oid 34 snap -1 2024-09-21T09:36:01.960 INFO:tasks.rados.rados.0.smithi022.stdout:1384: expect (ObjNum 449 snap 136 seq_num 449) 2024-09-21T09:36:01.960 INFO:tasks.rados.rados.0.smithi022.stdout:1385: delete oid 33 current snap is 138 2024-09-21T09:36:01.966 INFO:tasks.rados.rados.0.smithi022.stdout:1385: done (1 left) 2024-09-21T09:36:01.966 INFO:tasks.rados.rados.0.smithi022.stdout:1386: snap_remove snap 137 2024-09-21T09:36:02.308 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 21 09:36:02 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-0[103843]: 2024-09-21T09:36:02.013+0000 7f9f645e3640 -1 osd.0 329 log_to_monitors true 2024-09-21T09:36:02.646 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:02 smithi022 ceph-mon[96736]: Health check update: Degraded data redundancy: 103/609 objects degraded (16.913%), 15 pgs degraded (PG_DEGRADED) 2024-09-21T09:36:02.646 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:02 smithi022 ceph-mon[96736]: osdmap e341: 8 total, 7 up, 8 in 2024-09-21T09:36:02.646 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:02 smithi022 ceph-mon[96736]: from='osd.0 [v2:172.21.15.22:6802/1264061432,v1:172.21.15.22:6803/1264061432]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-21T09:36:02.646 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:02 smithi022 ceph-mon[98615]: Health check update: Degraded data redundancy: 103/609 objects degraded (16.913%), 15 pgs degraded (PG_DEGRADED) 2024-09-21T09:36:02.646 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:02 smithi022 ceph-mon[98615]: osdmap e341: 8 total, 7 up, 8 in 2024-09-21T09:36:02.647 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:02 smithi022 ceph-mon[98615]: from='osd.0 [v2:172.21.15.22:6802/1264061432,v1:172.21.15.22:6803/1264061432]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-21T09:36:02.693 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:02 smithi064 ceph-mon[86709]: Health check update: Degraded data redundancy: 103/609 objects degraded (16.913%), 15 pgs degraded (PG_DEGRADED) 2024-09-21T09:36:02.693 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:02 smithi064 ceph-mon[86709]: osdmap e341: 8 total, 7 up, 8 in 2024-09-21T09:36:02.694 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:02 smithi064 ceph-mon[86709]: from='osd.0 [v2:172.21.15.22:6802/1264061432,v1:172.21.15.22:6803/1264061432]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-21T09:36:02.961 INFO:tasks.rados.rados.0.smithi022.stdout:1386: done (1 left) 2024-09-21T09:36:02.962 INFO:tasks.rados.rados.0.smithi022.stdout:1387: read oid 4 snap 89 2024-09-21T09:36:02.962 INFO:tasks.rados.rados.0.smithi022.stdout:1387: expect (ObjNum 253 snap 71 seq_num 253) 2024-09-21T09:36:02.962 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 35 current snap is 138 2024-09-21T09:36:02.962 INFO:tasks.rados.rados.0.smithi022.stdout:1388: seq_num 458 ranges {679936=475136} 2024-09-21T09:36:02.969 INFO:tasks.rados.rados.0.smithi022.stdout:1388: writing smithi02284960-35 from 679936 to 1155072 tid 1 2024-09-21T09:36:02.969 INFO:tasks.rados.rados.0.smithi022.stdout:1389: snap_create 2024-09-21T09:36:02.969 INFO:tasks.rados.rados.0.smithi022.stdout:1384: expect (ObjNum 449 snap 136 seq_num 449) 2024-09-21T09:36:03.155 INFO:tasks.rados.rados.0.smithi022.stdout:1388: finishing write tid 1 to smithi02284960-35 2024-09-21T09:36:03.155 INFO:tasks.rados.rados.0.smithi022.stdout:1388: finishing write tid 2 to smithi02284960-35 2024-09-21T09:36:03.155 INFO:tasks.rados.rados.0.smithi022.stdout:1387: expect (ObjNum 253 snap 71 seq_num 253) 2024-09-21T09:36:03.323 INFO:tasks.rados.rados.0.smithi022.stdout:1388: finishing write tid 3 to smithi02284960-35 2024-09-21T09:36:03.323 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 35 v 542 (ObjNum 458 snap 138 seq_num 458) dirty exists 2024-09-21T09:36:03.324 INFO:tasks.rados.rados.0.smithi022.stdout:1388: left oid 35 (ObjNum 458 snap 138 seq_num 458) 2024-09-21T09:36:03.558 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 21 09:36:03 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-0[103843]: 2024-09-21T09:36:03.272+0000 7f9f58385640 -1 osd.0 329 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T09:36:03.970 INFO:tasks.rados.rados.0.smithi022.stdout:1384: done (3 left) 2024-09-21T09:36:03.970 INFO:tasks.rados.rados.0.smithi022.stdout:1387: done (2 left) 2024-09-21T09:36:03.970 INFO:tasks.rados.rados.0.smithi022.stdout:1388: done (1 left) 2024-09-21T09:36:03.970 INFO:tasks.rados.rados.0.smithi022.stdout:1389: done (0 left) 2024-09-21T09:36:03.970 INFO:tasks.rados.rados.0.smithi022.stdout:1390: read oid 39 snap -1 2024-09-21T09:36:03.971 INFO:tasks.rados.rados.0.smithi022.stdout:1390: expect (ObjNum 311 snap 94 seq_num 311) 2024-09-21T09:36:03.971 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 31 current snap is 139 2024-09-21T09:36:03.971 INFO:tasks.rados.rados.0.smithi022.stdout:1391: seq_num 459 ranges {155648=491520} 2024-09-21T09:36:03.974 INFO:tasks.rados.rados.0.smithi022.stdout:1391: writing smithi02284960-31 from 155648 to 647168 tid 1 2024-09-21T09:36:03.975 INFO:tasks.rados.rados.0.smithi022.stdout:1392: read oid 20 snap -1 2024-09-21T09:36:03.975 INFO:tasks.rados.rados.0.smithi022.stdout:1392: expect (ObjNum 132 snap 27 seq_num 132) 2024-09-21T09:36:03.975 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 13 current snap is 139 2024-09-21T09:36:03.975 INFO:tasks.rados.rados.0.smithi022.stdout:1393: seq_num 460 ranges {5234688=507904} 2024-09-21T09:36:04.008 INFO:tasks.rados.rados.0.smithi022.stdout:1393: writing smithi02284960-13 from 5234688 to 5742592 tid 1 2024-09-21T09:36:04.008 INFO:tasks.rados.rados.0.smithi022.stdout:1394: snap_create 2024-09-21T09:36:04.008 INFO:tasks.rados.rados.0.smithi022.stdout:1391: finishing write tid 1 to smithi02284960-31 2024-09-21T09:36:04.008 INFO:tasks.rados.rados.0.smithi022.stdout:1391: finishing write tid 2 to smithi02284960-31 2024-09-21T09:36:04.008 INFO:tasks.rados.rados.0.smithi022.stdout:1391: finishing write tid 3 to smithi02284960-31 2024-09-21T09:36:04.008 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 31 v 922 (ObjNum 459 snap 139 seq_num 459) dirty exists 2024-09-21T09:36:04.008 INFO:tasks.rados.rados.0.smithi022.stdout:1391: left oid 31 (ObjNum 459 snap 139 seq_num 459) 2024-09-21T09:36:04.011 INFO:tasks.rados.rados.0.smithi022.stdout:1393: finishing write tid 1 to smithi02284960-13 2024-09-21T09:36:04.012 INFO:tasks.rados.rados.0.smithi022.stdout:1393: finishing write tid 2 to smithi02284960-13 2024-09-21T09:36:04.013 INFO:tasks.rados.rados.0.smithi022.stdout:1393: finishing write tid 3 to smithi02284960-13 2024-09-21T09:36:04.013 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 13 v 635 (ObjNum 460 snap 139 seq_num 460) dirty exists 2024-09-21T09:36:04.013 INFO:tasks.rados.rados.0.smithi022.stdout:1393: left oid 13 (ObjNum 460 snap 139 seq_num 460) 2024-09-21T09:36:04.103 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:03 smithi022 ceph-mon[98615]: pgmap v115: 121 pgs: 37 active+undersized, 15 active+undersized+degraded, 69 active+clean; 252 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 2.4 MiB/s rd, 1.0 MiB/s wr, 10 op/s; 103/609 objects degraded (16.913%) 2024-09-21T09:36:04.103 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:03 smithi022 ceph-mon[98615]: from='osd.0 [v2:172.21.15.22:6802/1264061432,v1:172.21.15.22:6803/1264061432]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-21T09:36:04.103 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:03 smithi022 ceph-mon[98615]: osdmap e342: 8 total, 7 up, 8 in 2024-09-21T09:36:04.103 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:03 smithi022 ceph-mon[98615]: from='osd.0 [v2:172.21.15.22:6802/1264061432,v1:172.21.15.22:6803/1264061432]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi022", "root=default"]} : dispatch 2024-09-21T09:36:04.104 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:03 smithi022 ceph-mon[98615]: from='osd.0 [v2:172.21.15.22:6802/1264061432,v1:172.21.15.22:6803/1264061432]' entity='osd.0' 2024-09-21T09:36:04.104 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:03 smithi022 ceph-mon[96736]: pgmap v115: 121 pgs: 37 active+undersized, 15 active+undersized+degraded, 69 active+clean; 252 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 2.4 MiB/s rd, 1.0 MiB/s wr, 10 op/s; 103/609 objects degraded (16.913%) 2024-09-21T09:36:04.104 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:03 smithi022 ceph-mon[96736]: from='osd.0 [v2:172.21.15.22:6802/1264061432,v1:172.21.15.22:6803/1264061432]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-21T09:36:04.104 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:03 smithi022 ceph-mon[96736]: osdmap e342: 8 total, 7 up, 8 in 2024-09-21T09:36:04.104 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:03 smithi022 ceph-mon[96736]: from='osd.0 [v2:172.21.15.22:6802/1264061432,v1:172.21.15.22:6803/1264061432]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi022", "root=default"]} : dispatch 2024-09-21T09:36:04.104 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:03 smithi022 ceph-mon[96736]: from='osd.0 [v2:172.21.15.22:6802/1264061432,v1:172.21.15.22:6803/1264061432]' entity='osd.0' 2024-09-21T09:36:04.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:03 smithi064 ceph-mon[86709]: pgmap v115: 121 pgs: 37 active+undersized, 15 active+undersized+degraded, 69 active+clean; 252 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 2.4 MiB/s rd, 1.0 MiB/s wr, 10 op/s; 103/609 objects degraded (16.913%) 2024-09-21T09:36:04.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:03 smithi064 ceph-mon[86709]: from='osd.0 [v2:172.21.15.22:6802/1264061432,v1:172.21.15.22:6803/1264061432]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-21T09:36:04.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:03 smithi064 ceph-mon[86709]: osdmap e342: 8 total, 7 up, 8 in 2024-09-21T09:36:04.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:03 smithi064 ceph-mon[86709]: from='osd.0 [v2:172.21.15.22:6802/1264061432,v1:172.21.15.22:6803/1264061432]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi022", "root=default"]} : dispatch 2024-09-21T09:36:04.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:03 smithi064 ceph-mon[86709]: from='osd.0 [v2:172.21.15.22:6802/1264061432,v1:172.21.15.22:6803/1264061432]' entity='osd.0' 2024-09-21T09:36:04.442 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:36:04 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:36:04.148Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:36:04.558 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:36:04 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: 2024-09-21T09:36:04.232+0000 7feec1b50640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (19 PGs are or would become offline) 2024-09-21T09:36:04.971 INFO:tasks.rados.rados.0.smithi022.stdout:1391: done (4 left) 2024-09-21T09:36:04.972 INFO:tasks.rados.rados.0.smithi022.stdout:1393: done (3 left) 2024-09-21T09:36:04.972 INFO:tasks.rados.rados.0.smithi022.stdout:1394: done (2 left) 2024-09-21T09:36:04.972 INFO:tasks.rados.rados.0.smithi022.stdout:1395: snap_remove snap 123 2024-09-21T09:36:05.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:04 smithi064 ceph-mon[86709]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T09:36:05.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:04 smithi064 ceph-mon[86709]: osd.0 [v2:172.21.15.22:6802/1264061432,v1:172.21.15.22:6803/1264061432] boot 2024-09-21T09:36:05.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:04 smithi064 ceph-mon[86709]: osdmap e343: 8 total, 8 up, 8 in 2024-09-21T09:36:05.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:04 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-21T09:36:05.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:04 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:36:05.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:04 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:36:05.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:04 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:36:05.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:04 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:36:05.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:04 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:36:05.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:04 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:36:05.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:04 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:36:05.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:04 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:36:05.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:04 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:36:05.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:04 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-21T09:36:05.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:04 smithi022 ceph-mon[98615]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T09:36:05.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:04 smithi022 ceph-mon[98615]: osd.0 [v2:172.21.15.22:6802/1264061432,v1:172.21.15.22:6803/1264061432] boot 2024-09-21T09:36:05.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:04 smithi022 ceph-mon[98615]: osdmap e343: 8 total, 8 up, 8 in 2024-09-21T09:36:05.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:04 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-21T09:36:05.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:04 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:36:05.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:04 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:36:05.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:04 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:36:05.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:04 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:36:05.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:04 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:36:05.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:04 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:36:05.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:04 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:36:05.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:04 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:36:05.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:04 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:36:05.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:04 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-21T09:36:05.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:04 smithi022 ceph-mon[96736]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T09:36:05.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:04 smithi022 ceph-mon[96736]: osd.0 [v2:172.21.15.22:6802/1264061432,v1:172.21.15.22:6803/1264061432] boot 2024-09-21T09:36:05.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:04 smithi022 ceph-mon[96736]: osdmap e343: 8 total, 8 up, 8 in 2024-09-21T09:36:05.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:04 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-21T09:36:05.310 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:04 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:36:05.310 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:04 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:36:05.310 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:04 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:36:05.310 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:04 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:36:05.310 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:04 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:36:05.310 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:04 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:36:05.310 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:04 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:36:05.310 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:04 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:36:05.310 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:04 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:36:05.310 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:04 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-21T09:36:05.972 INFO:tasks.rados.rados.0.smithi022.stdout:1395: done (2 left) 2024-09-21T09:36:05.972 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 17 current snap is 140 2024-09-21T09:36:05.972 INFO:tasks.rados.rados.0.smithi022.stdout:1396: seq_num 461 ranges {0=524288} 2024-09-21T09:36:05.975 INFO:tasks.rados.rados.0.smithi022.stdout:1396: writing smithi02284960-17 from 0 to 524288 tid 1 2024-09-21T09:36:05.975 INFO:tasks.rados.rados.0.smithi022.stdout:1397: rollback oid 7 current snap is 140 2024-09-21T09:36:05.975 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 7 to 134 2024-09-21T09:36:05.975 INFO:tasks.rados.rados.0.smithi022.stdout:1390: expect (ObjNum 311 snap 94 seq_num 311) 2024-09-21T09:36:05.976 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:36:05 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:36:05] "GET /metrics HTTP/1.1" 200 36241 "" "Prometheus/2.51.0" 2024-09-21T09:36:06.005 INFO:tasks.rados.rados.0.smithi022.stdout:1392: expect (ObjNum 132 snap 27 seq_num 132) 2024-09-21T09:36:06.172 INFO:tasks.rados.rados.0.smithi022.stdout:1396: finishing write tid 1 to smithi02284960-17 2024-09-21T09:36:06.172 INFO:tasks.rados.rados.0.smithi022.stdout:1396: finishing write tid 2 to smithi02284960-17 2024-09-21T09:36:06.172 INFO:tasks.rados.rados.0.smithi022.stdout:1396: finishing write tid 3 to smithi02284960-17 2024-09-21T09:36:06.172 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 17 v 893 (ObjNum 461 snap 140 seq_num 461) dirty exists 2024-09-21T09:36:06.172 INFO:tasks.rados.rados.0.smithi022.stdout:1396: left oid 17 (ObjNum 461 snap 140 seq_num 461) 2024-09-21T09:36:06.172 INFO:tasks.rados.rados.0.smithi022.stdout:1390: done (3 left) 2024-09-21T09:36:06.172 INFO:tasks.rados.rados.0.smithi022.stdout:1392: done (2 left) 2024-09-21T09:36:06.172 INFO:tasks.rados.rados.0.smithi022.stdout:1396: done (1 left) 2024-09-21T09:36:06.172 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 6 current snap is 140 2024-09-21T09:36:06.172 INFO:tasks.rados.rados.0.smithi022.stdout:1397: finishing rollback tid 1 to smithi02284960-7 2024-09-21T09:36:06.172 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 7 v 813 (ObjNum 161 snap 0 seq_num 1) dirty dne 2024-09-21T09:36:06.173 INFO:tasks.rados.rados.0.smithi022.stdout:1398: seq_num 462 ranges {3424256=540672} 2024-09-21T09:36:06.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:05 smithi064 ceph-mon[86709]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-21T09:36:06.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:05 smithi064 ceph-mon[86709]: Upgrade: unsafe to stop osd(s) at this time (19 PGs are or would become offline) 2024-09-21T09:36:06.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:05 smithi064 ceph-mon[86709]: pgmap v118: 121 pgs: 37 active+undersized, 15 active+undersized+degraded, 69 active+clean; 253 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4.6 MiB/s rd, 1.2 MiB/s wr, 6 op/s; 110/624 objects degraded (17.628%) 2024-09-21T09:36:06.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:05 smithi064 ceph-mon[86709]: osdmap e344: 8 total, 8 up, 8 in 2024-09-21T09:36:06.192 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:36:05 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:36:05.739Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:36:06.192 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:36:05 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:36:05.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:36:06.193 INFO:tasks.rados.rados.0.smithi022.stdout:1398: writing smithi02284960-6 from 3424256 to 3964928 tid 1 2024-09-21T09:36:06.193 INFO:tasks.rados.rados.0.smithi022.stdout:1397: done (1 left) 2024-09-21T09:36:06.193 INFO:tasks.rados.rados.0.smithi022.stdout:1399: read oid 39 snap -1 2024-09-21T09:36:06.193 INFO:tasks.rados.rados.0.smithi022.stdout:1399: expect (ObjNum 311 snap 94 seq_num 311) 2024-09-21T09:36:06.193 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 22 current snap is 140 2024-09-21T09:36:06.193 INFO:tasks.rados.rados.0.smithi022.stdout:1400: seq_num 463 ranges {0=557056} 2024-09-21T09:36:06.196 INFO:tasks.rados.rados.0.smithi022.stdout:1400: writing smithi02284960-22 from 0 to 557056 tid 1 2024-09-21T09:36:06.196 INFO:tasks.rados.rados.0.smithi022.stdout:1399: expect (ObjNum 311 snap 94 seq_num 311) 2024-09-21T09:36:06.217 INFO:tasks.rados.rados.0.smithi022.stdout:1398: finishing write tid 1 to smithi02284960-6 2024-09-21T09:36:06.217 INFO:tasks.rados.rados.0.smithi022.stdout:1398: finishing write tid 2 to smithi02284960-6 2024-09-21T09:36:06.217 INFO:tasks.rados.rados.0.smithi022.stdout:1398: finishing write tid 3 to smithi02284960-6 2024-09-21T09:36:06.217 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 6 v 879 (ObjNum 462 snap 140 seq_num 462) dirty exists 2024-09-21T09:36:06.217 INFO:tasks.rados.rados.0.smithi022.stdout:1398: left oid 6 (ObjNum 462 snap 140 seq_num 462) 2024-09-21T09:36:06.217 INFO:tasks.rados.rados.0.smithi022.stdout:1400: finishing write tid 1 to smithi02284960-22 2024-09-21T09:36:06.218 INFO:tasks.rados.rados.0.smithi022.stdout:1398: done (2 left) 2024-09-21T09:36:06.218 INFO:tasks.rados.rados.0.smithi022.stdout:1399: done (1 left) 2024-09-21T09:36:06.218 INFO:tasks.rados.rados.0.smithi022.stdout:1401: rollback oid 12 current snap is 140 2024-09-21T09:36:06.218 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 12 to 138 2024-09-21T09:36:06.218 INFO:tasks.rados.rados.0.smithi022.stdout:1400: finishing write tid 2 to smithi02284960-22 2024-09-21T09:36:06.218 INFO:tasks.rados.rados.0.smithi022.stdout:1400: finishing write tid 3 to smithi02284960-22 2024-09-21T09:36:06.218 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 22 v 637 (ObjNum 463 snap 140 seq_num 463) dirty exists 2024-09-21T09:36:06.218 INFO:tasks.rados.rados.0.smithi022.stdout:1400: left oid 22 (ObjNum 463 snap 140 seq_num 463) 2024-09-21T09:36:06.218 INFO:tasks.rados.rados.0.smithi022.stdout:1400: done (1 left) 2024-09-21T09:36:06.218 INFO:tasks.rados.rados.0.smithi022.stdout:1402: read oid 46 snap -1 2024-09-21T09:36:06.218 INFO:tasks.rados.rados.0.smithi022.stdout:1402: expect (ObjNum 365 snap 115 seq_num 365) 2024-09-21T09:36:06.218 INFO:tasks.rados.rados.0.smithi022.stdout:1403: rmattr oid 16 current snap is 140 2024-09-21T09:36:06.218 INFO:tasks.rados.rados.0.smithi022.stdout:1404: delete oid 21 current snap is 140 2024-09-21T09:36:06.219 INFO:tasks.rados.rados.0.smithi022.stdout:1404: done (3 left) 2024-09-21T09:36:06.219 INFO:tasks.rados.rados.0.smithi022.stdout:1405: rmattr oid 18 current snap is 140 2024-09-21T09:36:06.219 INFO:tasks.rados.rados.0.smithi022.stdout:1405: done (3 left) 2024-09-21T09:36:06.219 INFO:tasks.rados.rados.0.smithi022.stdout:1406: rmattr oid 18 current snap is 140 2024-09-21T09:36:06.219 INFO:tasks.rados.rados.0.smithi022.stdout:1406: done (3 left) 2024-09-21T09:36:06.219 INFO:tasks.rados.rados.0.smithi022.stdout:1407: snap_create 2024-09-21T09:36:06.221 INFO:tasks.rados.rados.0.smithi022.stdout:1402: expect (ObjNum 365 snap 115 seq_num 365) 2024-09-21T09:36:06.256 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 16 v 376 (ObjNum 65 snap 0 seq_num 3570072672) dirty exists 2024-09-21T09:36:06.256 INFO:tasks.rados.rados.0.smithi022.stdout:1401: finishing rollback tid 0 to smithi02284960-12 2024-09-21T09:36:06.256 INFO:tasks.rados.rados.0.smithi022.stdout:1401: finishing rollback tid 1 to smithi02284960-12 2024-09-21T09:36:06.257 INFO:tasks.rados.rados.0.smithi022.stdout:1401: finishing rollback tid 2 to smithi02284960-12 2024-09-21T09:36:06.257 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 12 v 934 (ObjNum 433 snap 134 seq_num 433) dirty exists 2024-09-21T09:36:06.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:05 smithi022 ceph-mon[98615]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-21T09:36:06.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:05 smithi022 ceph-mon[98615]: Upgrade: unsafe to stop osd(s) at this time (19 PGs are or would become offline) 2024-09-21T09:36:06.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:05 smithi022 ceph-mon[98615]: pgmap v118: 121 pgs: 37 active+undersized, 15 active+undersized+degraded, 69 active+clean; 253 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4.6 MiB/s rd, 1.2 MiB/s wr, 6 op/s; 110/624 objects degraded (17.628%) 2024-09-21T09:36:06.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:05 smithi022 ceph-mon[98615]: osdmap e344: 8 total, 8 up, 8 in 2024-09-21T09:36:06.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:05 smithi022 ceph-mon[96736]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-21T09:36:06.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:05 smithi022 ceph-mon[96736]: Upgrade: unsafe to stop osd(s) at this time (19 PGs are or would become offline) 2024-09-21T09:36:06.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:05 smithi022 ceph-mon[96736]: pgmap v118: 121 pgs: 37 active+undersized, 15 active+undersized+degraded, 69 active+clean; 253 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4.6 MiB/s rd, 1.2 MiB/s wr, 6 op/s; 110/624 objects degraded (17.628%) 2024-09-21T09:36:06.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:05 smithi022 ceph-mon[96736]: osdmap e344: 8 total, 8 up, 8 in 2024-09-21T09:36:06.984 INFO:tasks.rados.rados.0.smithi022.stdout:1401: done (3 left) 2024-09-21T09:36:06.984 INFO:tasks.rados.rados.0.smithi022.stdout:1402: done (2 left) 2024-09-21T09:36:06.984 INFO:tasks.rados.rados.0.smithi022.stdout:1403: done (1 left) 2024-09-21T09:36:06.984 INFO:tasks.rados.rados.0.smithi022.stdout:1407: done (0 left) 2024-09-21T09:36:06.985 INFO:tasks.rados.rados.0.smithi022.stdout:1408: rollback oid 39 current snap is 141 2024-09-21T09:36:06.985 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 39 to 131 2024-09-21T09:36:06.985 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 44 current snap is 141 2024-09-21T09:36:06.985 INFO:tasks.rados.rados.0.smithi022.stdout:1409: seq_num 465 ranges {4161536=598016} 2024-09-21T09:36:07.010 INFO:tasks.rados.rados.0.smithi022.stdout:1409: writing smithi02284960-44 from 4161536 to 4759552 tid 1 2024-09-21T09:36:07.011 INFO:tasks.rados.rados.0.smithi022.stdout:1410: read oid 16 snap 138 2024-09-21T09:36:07.011 INFO:tasks.rados.rados.0.smithi022.stdout:1410: expect (ObjNum -724464640 snap 22094 seq_num 3288338304) 2024-09-21T09:36:07.011 INFO:tasks.rados.rados.0.smithi022.stdout:1411: snap_remove snap 89 2024-09-21T09:36:07.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:06 smithi022 ceph-mon[98615]: osdmap e345: 8 total, 8 up, 8 in 2024-09-21T09:36:07.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:06 smithi022 ceph-mon[96736]: osdmap e345: 8 total, 8 up, 8 in 2024-09-21T09:36:07.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:06 smithi064 ceph-mon[86709]: osdmap e345: 8 total, 8 up, 8 in 2024-09-21T09:36:07.441 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:36:06 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:36:06.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi022\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:36:07.960 INFO:teuthology.orchestra.run.smithi022.stdout:true 2024-09-21T09:36:07.985 INFO:tasks.rados.rados.0.smithi022.stdout:1411: done (3 left) 2024-09-21T09:36:07.985 INFO:tasks.rados.rados.0.smithi022.stdout:1412: snap_remove snap 127 2024-09-21T09:36:08.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:07 smithi022 ceph-mon[98615]: pgmap v121: 121 pgs: 37 active+undersized, 15 active+undersized+degraded, 69 active+clean; 253 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4.6 MiB/s rd, 1.2 MiB/s wr, 6 op/s; 110/624 objects degraded (17.628%) 2024-09-21T09:36:08.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:07 smithi022 ceph-mon[98615]: osdmap e346: 8 total, 8 up, 8 in 2024-09-21T09:36:08.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:07 smithi022 ceph-mon[96736]: pgmap v121: 121 pgs: 37 active+undersized, 15 active+undersized+degraded, 69 active+clean; 253 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4.6 MiB/s rd, 1.2 MiB/s wr, 6 op/s; 110/624 objects degraded (17.628%) 2024-09-21T09:36:08.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:07 smithi022 ceph-mon[96736]: osdmap e346: 8 total, 8 up, 8 in 2024-09-21T09:36:08.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:07 smithi064 ceph-mon[86709]: pgmap v121: 121 pgs: 37 active+undersized, 15 active+undersized+degraded, 69 active+clean; 253 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4.6 MiB/s rd, 1.2 MiB/s wr, 6 op/s; 110/624 objects degraded (17.628%) 2024-09-21T09:36:08.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:07 smithi064 ceph-mon[86709]: osdmap e346: 8 total, 8 up, 8 in 2024-09-21T09:36:08.564 INFO:teuthology.orchestra.run.smithi022.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T09:36:08.565 INFO:teuthology.orchestra.run.smithi022.stdout:alertmanager.a smithi022 *:9093,9094 running (2m) 8s ago 7m 18.4M - 0.27.0 11f11916f8cd caced80ba544 2024-09-21T09:36:08.565 INFO:teuthology.orchestra.run.smithi022.stdout:grafana.a smithi064 *:3000 running (2m) 48s ago 7m 77.9M - 9.4.7 954c08fa6188 527bb657d70d 2024-09-21T09:36:08.565 INFO:teuthology.orchestra.run.smithi022.stdout:mds.a smithi022 running (8m) 8s ago 8m 17.7M - 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 34e4221f3a27 2024-09-21T09:36:08.565 INFO:teuthology.orchestra.run.smithi022.stdout:mds.b smithi064 running (8m) 48s ago 8m 20.9M - 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 0f846b3ec694 2024-09-21T09:36:08.565 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.x smithi064 *:8443,9283,8765 running (108s) 48s ago 12m 453M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 61abc61fea8d 2024-09-21T09:36:08.565 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.y smithi022 *:8443,9283,8765 running (2m) 8s ago 13m 510M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 6c5bee78f09b 2024-09-21T09:36:08.565 INFO:teuthology.orchestra.run.smithi022.stdout:mon.a smithi022 running (95s) 8s ago 14m 48.3M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 0513395c784d 2024-09-21T09:36:08.565 INFO:teuthology.orchestra.run.smithi022.stdout:mon.b smithi064 running (53s) 48s ago 12m 31.5M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 9dab26b6a9e7 2024-09-21T09:36:08.565 INFO:teuthology.orchestra.run.smithi022.stdout:mon.c smithi022 running (71s) 8s ago 12m 38.8M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 c74b58b67bd6 2024-09-21T09:36:08.565 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.a smithi022 *:9100 running (2m) 8s ago 7m 10.4M - 1.7.0 72c9c2088986 593c6158f524 2024-09-21T09:36:08.565 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.b smithi064 *:9100 running (2m) 48s ago 6m 9.94M - 1.7.0 72c9c2088986 a533420b0139 2024-09-21T09:36:08.565 INFO:teuthology.orchestra.run.smithi022.stdout:osd.0 smithi022 running (12s) 8s ago 11m 11.3M 2481M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 40ce23a2b569 2024-09-21T09:36:08.566 INFO:teuthology.orchestra.run.smithi022.stdout:osd.1 smithi022 running (11m) 8s ago 11m 173M 2481M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 740462b5c304 2024-09-21T09:36:08.566 INFO:teuthology.orchestra.run.smithi022.stdout:osd.2 smithi022 running (11m) 8s ago 11m 211M 2481M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be fefc7c770c32 2024-09-21T09:36:08.566 INFO:teuthology.orchestra.run.smithi022.stdout:osd.3 smithi022 running (10m) 8s ago 10m 104M 2481M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be a56193b67a92 2024-09-21T09:36:08.566 INFO:teuthology.orchestra.run.smithi022.stdout:osd.4 smithi064 running (10m) 48s ago 10m 141M 2480M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 2a2937374fc4 2024-09-21T09:36:08.566 INFO:teuthology.orchestra.run.smithi022.stdout:osd.5 smithi064 running (9m) 48s ago 9m 192M 2480M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 1900c4af5d28 2024-09-21T09:36:08.566 INFO:teuthology.orchestra.run.smithi022.stdout:osd.6 smithi064 running (9m) 48s ago 9m 126M 2480M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 3cc4ec816e5b 2024-09-21T09:36:08.566 INFO:teuthology.orchestra.run.smithi022.stdout:osd.7 smithi064 running (9m) 48s ago 9m 148M 2480M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 18ef0be94104 2024-09-21T09:36:08.566 INFO:teuthology.orchestra.run.smithi022.stdout:prometheus.a smithi064 *:9095 running (113s) 48s ago 7m 44.0M - 2.51.0 1d3b7f56885b 80629ffa2179 2024-09-21T09:36:08.922 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-09-21T09:36:08.923 INFO:teuthology.orchestra.run.smithi022.stdout: "mon": { 2024-09-21T09:36:08.923 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 3 2024-09-21T09:36:08.923 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-21T09:36:08.923 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr": { 2024-09-21T09:36:08.923 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 2 2024-09-21T09:36:08.923 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-21T09:36:08.923 INFO:teuthology.orchestra.run.smithi022.stdout: "osd": { 2024-09-21T09:36:08.923 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 7, 2024-09-21T09:36:08.923 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 1 2024-09-21T09:36:08.923 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-21T09:36:08.923 INFO:teuthology.orchestra.run.smithi022.stdout: "mds": { 2024-09-21T09:36:08.923 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 2 2024-09-21T09:36:08.923 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-21T09:36:08.924 INFO:teuthology.orchestra.run.smithi022.stdout: "overall": { 2024-09-21T09:36:08.924 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 9, 2024-09-21T09:36:08.924 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 6 2024-09-21T09:36:08.924 INFO:teuthology.orchestra.run.smithi022.stdout: } 2024-09-21T09:36:08.924 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-09-21T09:36:08.994 INFO:tasks.rados.rados.0.smithi022.stdout:1412: done (3 left) 2024-09-21T09:36:08.994 INFO:tasks.rados.rados.0.smithi022.stdout:1413: snap_remove snap 96 2024-09-21T09:36:09.236 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-09-21T09:36:09.236 INFO:teuthology.orchestra.run.smithi022.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47", 2024-09-21T09:36:09.236 INFO:teuthology.orchestra.run.smithi022.stdout: "in_progress": true, 2024-09-21T09:36:09.236 INFO:teuthology.orchestra.run.smithi022.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T09:36:09.236 INFO:teuthology.orchestra.run.smithi022.stdout: "services_complete": [ 2024-09-21T09:36:09.236 INFO:teuthology.orchestra.run.smithi022.stdout: "mon", 2024-09-21T09:36:09.237 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr" 2024-09-21T09:36:09.237 INFO:teuthology.orchestra.run.smithi022.stdout: ], 2024-09-21T09:36:09.237 INFO:teuthology.orchestra.run.smithi022.stdout: "progress": "6/20 daemons upgraded", 2024-09-21T09:36:09.237 INFO:teuthology.orchestra.run.smithi022.stdout: "message": "Currently upgrading osd daemons", 2024-09-21T09:36:09.237 INFO:teuthology.orchestra.run.smithi022.stdout: "is_paused": false 2024-09-21T09:36:09.237 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-09-21T09:36:09.237 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:08 smithi022 ceph-mon[98615]: from='client.34212 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:36:09.237 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:08 smithi022 ceph-mon[98615]: osdmap e347: 8 total, 8 up, 8 in 2024-09-21T09:36:09.237 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:08 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/3374048716' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:36:09.237 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:08 smithi022 ceph-mon[98615]: Health check update: Degraded data redundancy: 110/624 objects degraded (17.628%), 15 pgs degraded (PG_DEGRADED) 2024-09-21T09:36:09.238 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:08 smithi022 ceph-mon[96736]: from='client.34212 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:36:09.238 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:08 smithi022 ceph-mon[96736]: osdmap e347: 8 total, 8 up, 8 in 2024-09-21T09:36:09.238 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:08 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/3374048716' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:36:09.238 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:08 smithi022 ceph-mon[96736]: Health check update: Degraded data redundancy: 110/624 objects degraded (17.628%), 15 pgs degraded (PG_DEGRADED) 2024-09-21T09:36:09.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:08 smithi064 ceph-mon[86709]: from='client.34212 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:36:09.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:08 smithi064 ceph-mon[86709]: osdmap e347: 8 total, 8 up, 8 in 2024-09-21T09:36:09.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:08 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/3374048716' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:36:09.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:08 smithi064 ceph-mon[86709]: Health check update: Degraded data redundancy: 110/624 objects degraded (17.628%), 15 pgs degraded (PG_DEGRADED) 2024-09-21T09:36:10.005 INFO:tasks.rados.rados.0.smithi022.stdout:1413: done (3 left) 2024-09-21T09:36:10.005 INFO:tasks.rados.rados.0.smithi022.stdout:1414: delete oid 45 current snap is 141 2024-09-21T09:36:10.005 INFO:tasks.rados.rados.0.smithi022.stdout:1408: finishing rollback tid 0 to smithi02284960-39 2024-09-21T09:36:10.005 INFO:tasks.rados.rados.0.smithi022.stdout:1408: finishing rollback tid 1 to smithi02284960-39 2024-09-21T09:36:10.005 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 39 v 895 (ObjNum 289 snap 0 seq_num 0) dirty dne 2024-09-21T09:36:10.005 INFO:tasks.rados.rados.0.smithi022.stdout:1410: expect (ObjNum -724464640 snap 22094 seq_num 3288338304) 2024-09-21T09:36:10.012 INFO:tasks.rados.rados.0.smithi022.stdout:1409: finishing write tid 1 to smithi02284960-44 2024-09-21T09:36:10.012 INFO:tasks.rados.rados.0.smithi022.stdout:1409: finishing write tid 2 to smithi02284960-44 2024-09-21T09:36:10.012 INFO:tasks.rados.rados.0.smithi022.stdout:1409: finishing write tid 3 to smithi02284960-44 2024-09-21T09:36:10.012 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 44 v 853 (ObjNum 465 snap 141 seq_num 465) dirty exists 2024-09-21T09:36:10.012 INFO:tasks.rados.rados.0.smithi022.stdout:1409: left oid 44 (ObjNum 465 snap 141 seq_num 465) 2024-09-21T09:36:10.016 INFO:tasks.rados.rados.0.smithi022.stdout:1408: done (3 left) 2024-09-21T09:36:10.016 INFO:tasks.rados.rados.0.smithi022.stdout:1409: done (2 left) 2024-09-21T09:36:10.016 INFO:tasks.rados.rados.0.smithi022.stdout:1410: done (1 left) 2024-09-21T09:36:10.016 INFO:tasks.rados.rados.0.smithi022.stdout:1414: done (0 left) 2024-09-21T09:36:10.016 INFO:tasks.rados.rados.0.smithi022.stdout:1415: delete oid 12 current snap is 141 2024-09-21T09:36:10.025 INFO:tasks.rados.rados.0.smithi022.stdout:1415: done (0 left) 2024-09-21T09:36:10.025 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 15 current snap is 141 2024-09-21T09:36:10.025 INFO:tasks.rados.rados.0.smithi022.stdout:1416: seq_num 468 ranges {0=647168} 2024-09-21T09:36:10.029 INFO:tasks.rados.rados.0.smithi022.stdout:1416: writing smithi02284960-15 from 0 to 647168 tid 1 2024-09-21T09:36:10.030 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 22 current snap is 141 2024-09-21T09:36:10.030 INFO:tasks.rados.rados.0.smithi022.stdout:1417: seq_num 469 ranges {557056=663552} 2024-09-21T09:36:10.038 INFO:tasks.rados.rados.0.smithi022.stdout:1417: writing smithi02284960-22 from 557056 to 1220608 tid 1 2024-09-21T09:36:10.038 INFO:tasks.rados.rados.0.smithi022.stdout:1418: read oid 30 snap -1 2024-09-21T09:36:10.038 INFO:tasks.rados.rados.0.smithi022.stdout:1418: expect (ObjNum 439 snap 134 seq_num 439) 2024-09-21T09:36:10.038 INFO:tasks.rados.rados.0.smithi022.stdout:1419: read oid 50 snap -1 2024-09-21T09:36:10.038 INFO:tasks.rados.rados.0.smithi022.stdout:1419: expect deleted 2024-09-21T09:36:10.038 INFO:tasks.rados.rados.0.smithi022.stdout:1420: snap_create 2024-09-21T09:36:10.038 INFO:tasks.rados.rados.0.smithi022.stdout:1416: finishing write tid 1 to smithi02284960-15 2024-09-21T09:36:10.038 INFO:tasks.rados.rados.0.smithi022.stdout:1416: finishing write tid 2 to smithi02284960-15 2024-09-21T09:36:10.038 INFO:tasks.rados.rados.0.smithi022.stdout:1416: finishing write tid 3 to smithi02284960-15 2024-09-21T09:36:10.039 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 15 v 964 (ObjNum 468 snap 141 seq_num 468) dirty exists 2024-09-21T09:36:10.039 INFO:tasks.rados.rados.0.smithi022.stdout:1416: left oid 15 (ObjNum 468 snap 141 seq_num 468) 2024-09-21T09:36:10.040 INFO:tasks.rados.rados.0.smithi022.stdout:1418: expect (ObjNum 439 snap 134 seq_num 439) 2024-09-21T09:36:10.044 INFO:tasks.rados.rados.0.smithi022.stdout:1417: finishing write tid 1 to smithi02284960-22 2024-09-21T09:36:10.044 INFO:tasks.rados.rados.0.smithi022.stdout:1417: finishing write tid 2 to smithi02284960-22 2024-09-21T09:36:10.044 INFO:tasks.rados.rados.0.smithi022.stdout:1417: finishing write tid 3 to smithi02284960-22 2024-09-21T09:36:10.045 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 22 v 648 (ObjNum 469 snap 141 seq_num 469) dirty exists 2024-09-21T09:36:10.045 INFO:tasks.rados.rados.0.smithi022.stdout:1417: left oid 22 (ObjNum 469 snap 141 seq_num 469) 2024-09-21T09:36:10.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:09 smithi022 ceph-mon[98615]: from='client.34218 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:36:10.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:09 smithi022 ceph-mon[98615]: pgmap v124: 121 pgs: 2 active+undersized, 1 active+undersized+degraded, 118 active+clean; 230 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 5.1 MiB/s rd, 786 KiB/s wr, 11 op/s; 9/543 objects degraded (1.657%); 15 MiB/s, 6 objects/s recovering 2024-09-21T09:36:10.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:09 smithi022 ceph-mon[98615]: from='client.44150 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:36:10.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:09 smithi022 ceph-mon[98615]: osdmap e348: 8 total, 8 up, 8 in 2024-09-21T09:36:10.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:09 smithi022 ceph-mon[96736]: from='client.34218 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:36:10.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:09 smithi022 ceph-mon[96736]: pgmap v124: 121 pgs: 2 active+undersized, 1 active+undersized+degraded, 118 active+clean; 230 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 5.1 MiB/s rd, 786 KiB/s wr, 11 op/s; 9/543 objects degraded (1.657%); 15 MiB/s, 6 objects/s recovering 2024-09-21T09:36:10.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:09 smithi022 ceph-mon[96736]: from='client.44150 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:36:10.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:09 smithi022 ceph-mon[96736]: osdmap e348: 8 total, 8 up, 8 in 2024-09-21T09:36:10.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:09 smithi064 ceph-mon[86709]: from='client.34218 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:36:10.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:09 smithi064 ceph-mon[86709]: pgmap v124: 121 pgs: 2 active+undersized, 1 active+undersized+degraded, 118 active+clean; 230 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 5.1 MiB/s rd, 786 KiB/s wr, 11 op/s; 9/543 objects degraded (1.657%); 15 MiB/s, 6 objects/s recovering 2024-09-21T09:36:10.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:09 smithi064 ceph-mon[86709]: from='client.44150 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:36:10.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:09 smithi064 ceph-mon[86709]: osdmap e348: 8 total, 8 up, 8 in 2024-09-21T09:36:10.442 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:36:10 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:36:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:36:11.010 INFO:tasks.rados.rados.0.smithi022.stdout:1416: done (4 left) 2024-09-21T09:36:11.010 INFO:tasks.rados.rados.0.smithi022.stdout:1417: done (3 left) 2024-09-21T09:36:11.010 INFO:tasks.rados.rados.0.smithi022.stdout:1418: done (2 left) 2024-09-21T09:36:11.010 INFO:tasks.rados.rados.0.smithi022.stdout:1419: done (1 left) 2024-09-21T09:36:11.011 INFO:tasks.rados.rados.0.smithi022.stdout:1420: done (0 left) 2024-09-21T09:36:11.011 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 17 current snap is 142 2024-09-21T09:36:11.011 INFO:tasks.rados.rados.0.smithi022.stdout:1421: seq_num 470 ranges {524288=679936} 2024-09-21T09:36:11.017 INFO:tasks.rados.rados.0.smithi022.stdout:1421: writing smithi02284960-17 from 524288 to 1204224 tid 1 2024-09-21T09:36:11.017 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 35 current snap is 142 2024-09-21T09:36:11.017 INFO:tasks.rados.rados.0.smithi022.stdout:1422: seq_num 471 ranges {1155072=696320} 2024-09-21T09:36:11.027 INFO:tasks.rados.rados.0.smithi022.stdout:1422: writing smithi02284960-35 from 1155072 to 1851392 tid 1 2024-09-21T09:36:11.027 INFO:tasks.rados.rados.0.smithi022.stdout:1423: snap_create 2024-09-21T09:36:11.028 INFO:tasks.rados.rados.0.smithi022.stdout:1421: finishing write tid 1 to smithi02284960-17 2024-09-21T09:36:11.028 INFO:tasks.rados.rados.0.smithi022.stdout:1421: finishing write tid 2 to smithi02284960-17 2024-09-21T09:36:11.028 INFO:tasks.rados.rados.0.smithi022.stdout:1421: finishing write tid 3 to smithi02284960-17 2024-09-21T09:36:11.028 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 17 v 897 (ObjNum 470 snap 142 seq_num 470) dirty exists 2024-09-21T09:36:11.028 INFO:tasks.rados.rados.0.smithi022.stdout:1421: left oid 17 (ObjNum 470 snap 142 seq_num 470) 2024-09-21T09:36:11.032 INFO:tasks.rados.rados.0.smithi022.stdout:1422: finishing write tid 1 to smithi02284960-35 2024-09-21T09:36:11.032 INFO:tasks.rados.rados.0.smithi022.stdout:1422: finishing write tid 2 to smithi02284960-35 2024-09-21T09:36:11.033 INFO:tasks.rados.rados.0.smithi022.stdout:1422: finishing write tid 3 to smithi02284960-35 2024-09-21T09:36:11.033 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 35 v 544 (ObjNum 471 snap 142 seq_num 471) dirty exists 2024-09-21T09:36:11.033 INFO:tasks.rados.rados.0.smithi022.stdout:1422: left oid 35 (ObjNum 471 snap 142 seq_num 471) 2024-09-21T09:36:11.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:11 smithi022 ceph-mon[98615]: from='client.44162 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:36:11.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:11 smithi022 ceph-mon[98615]: osdmap e349: 8 total, 8 up, 8 in 2024-09-21T09:36:11.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:11 smithi022 ceph-mon[96736]: from='client.44162 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:36:11.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:11 smithi022 ceph-mon[96736]: osdmap e349: 8 total, 8 up, 8 in 2024-09-21T09:36:11.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:11 smithi064 ceph-mon[86709]: from='client.44162 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:36:11.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:11 smithi064 ceph-mon[86709]: osdmap e349: 8 total, 8 up, 8 in 2024-09-21T09:36:12.011 INFO:tasks.rados.rados.0.smithi022.stdout:1421: done (2 left) 2024-09-21T09:36:12.011 INFO:tasks.rados.rados.0.smithi022.stdout:1422: done (1 left) 2024-09-21T09:36:12.011 INFO:tasks.rados.rados.0.smithi022.stdout:1423: done (0 left) 2024-09-21T09:36:12.011 INFO:tasks.rados.rados.0.smithi022.stdout:1424: delete oid 44 current snap is 143 2024-09-21T09:36:12.017 INFO:tasks.rados.rados.0.smithi022.stdout:1424: done (0 left) 2024-09-21T09:36:12.017 INFO:tasks.rados.rados.0.smithi022.stdout:1425: rmattr oid 21 current snap is 143 2024-09-21T09:36:12.017 INFO:tasks.rados.rados.0.smithi022.stdout:1425: done (0 left) 2024-09-21T09:36:12.017 INFO:tasks.rados.rados.0.smithi022.stdout:1426: read oid 50 snap -1 2024-09-21T09:36:12.017 INFO:tasks.rados.rados.0.smithi022.stdout:1426: expect deleted 2024-09-21T09:36:12.017 INFO:tasks.rados.rados.0.smithi022.stdout:1427: read oid 4 snap -1 2024-09-21T09:36:12.017 INFO:tasks.rados.rados.0.smithi022.stdout:1427: expect deleted 2024-09-21T09:36:12.017 INFO:tasks.rados.rados.0.smithi022.stdout:1428: snap_create 2024-09-21T09:36:12.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:12 smithi022 ceph-mon[98615]: pgmap v127: 121 pgs: 121 active+clean; 230 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 5.1 MiB/s rd, 786 KiB/s wr, 11 op/s; 16 MiB/s, 7 objects/s recovering 2024-09-21T09:36:12.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:12 smithi022 ceph-mon[98615]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 9/543 objects degraded (1.657%), 1 pg degraded) 2024-09-21T09:36:12.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:12 smithi022 ceph-mon[98615]: Cluster is now healthy 2024-09-21T09:36:12.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:12 smithi022 ceph-mon[98615]: osdmap e350: 8 total, 8 up, 8 in 2024-09-21T09:36:12.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:12 smithi022 ceph-mon[96736]: pgmap v127: 121 pgs: 121 active+clean; 230 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 5.1 MiB/s rd, 786 KiB/s wr, 11 op/s; 16 MiB/s, 7 objects/s recovering 2024-09-21T09:36:12.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:12 smithi022 ceph-mon[96736]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 9/543 objects degraded (1.657%), 1 pg degraded) 2024-09-21T09:36:12.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:12 smithi022 ceph-mon[96736]: Cluster is now healthy 2024-09-21T09:36:12.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:12 smithi022 ceph-mon[96736]: osdmap e350: 8 total, 8 up, 8 in 2024-09-21T09:36:12.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:12 smithi064 ceph-mon[86709]: pgmap v127: 121 pgs: 121 active+clean; 230 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 5.1 MiB/s rd, 786 KiB/s wr, 11 op/s; 16 MiB/s, 7 objects/s recovering 2024-09-21T09:36:12.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:12 smithi064 ceph-mon[86709]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 9/543 objects degraded (1.657%), 1 pg degraded) 2024-09-21T09:36:12.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:12 smithi064 ceph-mon[86709]: Cluster is now healthy 2024-09-21T09:36:12.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:12 smithi064 ceph-mon[86709]: osdmap e350: 8 total, 8 up, 8 in 2024-09-21T09:36:13.021 INFO:tasks.rados.rados.0.smithi022.stdout:1426: done (2 left) 2024-09-21T09:36:13.022 INFO:tasks.rados.rados.0.smithi022.stdout:1427: done (1 left) 2024-09-21T09:36:13.022 INFO:tasks.rados.rados.0.smithi022.stdout:1428: done (0 left) 2024-09-21T09:36:13.022 INFO:tasks.rados.rados.0.smithi022.stdout:1429: read oid 40 snap 100 2024-09-21T09:36:13.022 INFO:tasks.rados.rados.0.smithi022.stdout:1429: expect (ObjNum 214 snap 61 seq_num 214) 2024-09-21T09:36:13.022 INFO:tasks.rados.rados.0.smithi022.stdout:1430: delete oid 47 current snap is 144 2024-09-21T09:36:13.025 INFO:tasks.rados.rados.0.smithi022.stdout:1430: done (1 left) 2024-09-21T09:36:13.025 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 31 current snap is 144 2024-09-21T09:36:13.025 INFO:tasks.rados.rados.0.smithi022.stdout:1431: seq_num 474 ranges {647168=745472} 2024-09-21T09:36:13.033 INFO:tasks.rados.rados.0.smithi022.stdout:1431: writing smithi02284960-31 from 647168 to 1392640 tid 1 2024-09-21T09:36:13.033 INFO:tasks.rados.rados.0.smithi022.stdout:1432: delete oid 43 current snap is 144 2024-09-21T09:36:13.034 INFO:tasks.rados.rados.0.smithi022.stdout:1429: expect (ObjNum 214 snap 61 seq_num 214) 2024-09-21T09:36:13.066 INFO:tasks.rados.rados.0.smithi022.stdout:1431: finishing write tid 1 to smithi02284960-31 2024-09-21T09:36:13.066 INFO:tasks.rados.rados.0.smithi022.stdout:1431: finishing write tid 2 to smithi02284960-31 2024-09-21T09:36:13.066 INFO:tasks.rados.rados.0.smithi022.stdout:1431: finishing write tid 3 to smithi02284960-31 2024-09-21T09:36:13.066 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 31 v 967 (ObjNum 474 snap 144 seq_num 474) dirty exists 2024-09-21T09:36:13.066 INFO:tasks.rados.rados.0.smithi022.stdout:1431: left oid 31 (ObjNum 474 snap 144 seq_num 474) 2024-09-21T09:36:13.067 INFO:tasks.rados.rados.0.smithi022.stdout:1429: done (2 left) 2024-09-21T09:36:13.067 INFO:tasks.rados.rados.0.smithi022.stdout:1431: done (1 left) 2024-09-21T09:36:13.067 INFO:tasks.rados.rados.0.smithi022.stdout:1432: done (0 left) 2024-09-21T09:36:13.067 INFO:tasks.rados.rados.0.smithi022.stdout:1433: rollback oid 19 current snap is 144 2024-09-21T09:36:13.067 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 19 to 138 2024-09-21T09:36:13.067 INFO:tasks.rados.rados.0.smithi022.stdout:1434: read oid 23 snap -1 2024-09-21T09:36:13.067 INFO:tasks.rados.rados.0.smithi022.stdout:1434: expect (ObjNum 309 snap 91 seq_num 309) 2024-09-21T09:36:13.067 INFO:tasks.rados.rados.0.smithi022.stdout:1435: copy_from oid 45 from oid 4 current snap is 144 2024-09-21T09:36:13.067 INFO:tasks.rados.rados.0.smithi022.stdout:1436: read oid 39 snap -1 2024-09-21T09:36:13.067 INFO:tasks.rados.rados.0.smithi022.stdout:1436: expect deleted 2024-09-21T09:36:13.067 INFO:tasks.rados.rados.0.smithi022.stdout:1437: snap_remove snap 100 2024-09-21T09:36:13.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:13 smithi022 ceph-mon[98615]: osdmap e351: 8 total, 8 up, 8 in 2024-09-21T09:36:13.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:13 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:36:13.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:13 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:36:13.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:13 smithi022 ceph-mon[96736]: osdmap e351: 8 total, 8 up, 8 in 2024-09-21T09:36:13.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:13 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:36:13.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:13 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:36:13.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:13 smithi064 ceph-mon[86709]: osdmap e351: 8 total, 8 up, 8 in 2024-09-21T09:36:13.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:13 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:36:13.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:13 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:36:13.955 INFO:tasks.rados.rados.0.smithi022.stdout:1437: done (4 left) 2024-09-21T09:36:13.955 INFO:tasks.rados.rados.0.smithi022.stdout:1438: snap_remove snap 131 2024-09-21T09:36:14.307 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:14 smithi022 ceph-mon[96736]: pgmap v130: 121 pgs: 121 active+clean; 230 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s; 1020 KiB/s, 0 objects/s recovering 2024-09-21T09:36:14.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:14 smithi022 ceph-mon[96736]: osdmap e352: 8 total, 8 up, 8 in 2024-09-21T09:36:14.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:14 smithi022 ceph-mon[96736]: osdmap e353: 8 total, 8 up, 8 in 2024-09-21T09:36:14.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:14 smithi022 ceph-mon[98615]: pgmap v130: 121 pgs: 121 active+clean; 230 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s; 1020 KiB/s, 0 objects/s recovering 2024-09-21T09:36:14.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:14 smithi022 ceph-mon[98615]: osdmap e352: 8 total, 8 up, 8 in 2024-09-21T09:36:14.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:14 smithi022 ceph-mon[98615]: osdmap e353: 8 total, 8 up, 8 in 2024-09-21T09:36:14.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:14 smithi064 ceph-mon[86709]: pgmap v130: 121 pgs: 121 active+clean; 230 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s; 1020 KiB/s, 0 objects/s recovering 2024-09-21T09:36:14.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:14 smithi064 ceph-mon[86709]: osdmap e352: 8 total, 8 up, 8 in 2024-09-21T09:36:14.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:14 smithi064 ceph-mon[86709]: osdmap e353: 8 total, 8 up, 8 in 2024-09-21T09:36:14.442 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:36:14 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:36:14.147Z caller=alerting.go:391 level=warn component="rule manager" alert="unsupported value type" msg="Expanding alert template failed" err="error executing template __alert_CephOSDDown: template: __alert_CephOSDDown:1:358: executing \"__alert_CephOSDDown\" at : error calling query: found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}];many-to-many matching not allowed: matching labels must be unique on one side" data="unsupported value type" 2024-09-21T09:36:14.442 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:36:14 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:36:14.148Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:36:14.957 INFO:tasks.rados.rados.0.smithi022.stdout:1438: done (4 left) 2024-09-21T09:36:14.957 INFO:tasks.rados.rados.0.smithi022.stdout:1439: snap_create 2024-09-21T09:36:14.958 INFO:tasks.rados.rados.0.smithi022.stdout:1433: finishing rollback tid 1 to smithi02284960-19 2024-09-21T09:36:14.958 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 19 v 908 (ObjNum 65 snap 0 seq_num 2955206643) dirty dne 2024-09-21T09:36:14.958 INFO:tasks.rados.rados.0.smithi022.stdout:1435: finishing copy_from to smithi02284960-45 2024-09-21T09:36:14.958 INFO:tasks.rados.rados.0.smithi022.stdout:1435: got expected ENOENT (src dne) 2024-09-21T09:36:14.958 INFO:tasks.rados.rados.0.smithi022.stdout:1435: finishing copy_from racing read to smithi02284960-45 2024-09-21T09:36:14.958 INFO:tasks.rados.rados.0.smithi022.stdout:1434: expect (ObjNum 309 snap 91 seq_num 309) 2024-09-21T09:36:15.959 INFO:tasks.rados.rados.0.smithi022.stdout:1433: done (4 left) 2024-09-21T09:36:15.959 INFO:tasks.rados.rados.0.smithi022.stdout:1434: done (3 left) 2024-09-21T09:36:15.959 INFO:tasks.rados.rados.0.smithi022.stdout:1435: done (2 left) 2024-09-21T09:36:15.959 INFO:tasks.rados.rados.0.smithi022.stdout:1436: done (1 left) 2024-09-21T09:36:15.959 INFO:tasks.rados.rados.0.smithi022.stdout:1439: done (0 left) 2024-09-21T09:36:15.959 INFO:tasks.rados.rados.0.smithi022.stdout:1440: rollback oid 14 current snap is 145 2024-09-21T09:36:15.959 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 14 to 143 2024-09-21T09:36:15.959 INFO:tasks.rados.rados.0.smithi022.stdout:1441: snap_remove snap 138 2024-09-21T09:36:16.058 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:36:15 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:36:15] "GET /metrics HTTP/1.1" 200 36179 "" "Prometheus/2.51.0" 2024-09-21T09:36:16.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:15 smithi022 ceph-mon[98615]: pgmap v133: 121 pgs: 121 active+clean; 226 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 2.3 MiB/s rd, 839 KiB/s wr, 9 op/s 2024-09-21T09:36:16.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:15 smithi022 ceph-mon[98615]: osdmap e354: 8 total, 8 up, 8 in 2024-09-21T09:36:16.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:15 smithi022 ceph-mon[98615]: mgrmap e40: y(active, since 92s), standbys: x 2024-09-21T09:36:16.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:15 smithi022 ceph-mon[96736]: pgmap v133: 121 pgs: 121 active+clean; 226 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 2.3 MiB/s rd, 839 KiB/s wr, 9 op/s 2024-09-21T09:36:16.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:15 smithi022 ceph-mon[96736]: osdmap e354: 8 total, 8 up, 8 in 2024-09-21T09:36:16.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:15 smithi022 ceph-mon[96736]: mgrmap e40: y(active, since 92s), standbys: x 2024-09-21T09:36:16.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:15 smithi064 ceph-mon[86709]: pgmap v133: 121 pgs: 121 active+clean; 226 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 2.3 MiB/s rd, 839 KiB/s wr, 9 op/s 2024-09-21T09:36:16.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:15 smithi064 ceph-mon[86709]: osdmap e354: 8 total, 8 up, 8 in 2024-09-21T09:36:16.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:15 smithi064 ceph-mon[86709]: mgrmap e40: y(active, since 92s), standbys: x 2024-09-21T09:36:16.192 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:36:15 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:36:15.741Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:36:16.192 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:36:15 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:36:15.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:36:16.968 INFO:tasks.rados.rados.0.smithi022.stdout:1441: done (1 left) 2024-09-21T09:36:16.968 INFO:tasks.rados.rados.0.smithi022.stdout:1442: snap_remove snap 125 2024-09-21T09:36:17.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:16 smithi022 ceph-mon[98615]: osdmap e355: 8 total, 8 up, 8 in 2024-09-21T09:36:17.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:16 smithi022 ceph-mon[96736]: osdmap e355: 8 total, 8 up, 8 in 2024-09-21T09:36:17.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:16 smithi064 ceph-mon[86709]: osdmap e355: 8 total, 8 up, 8 in 2024-09-21T09:36:17.442 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:36:16 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:36:16.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi022\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:36:17.973 INFO:tasks.rados.rados.0.smithi022.stdout:1442: done (1 left) 2024-09-21T09:36:17.973 INFO:tasks.rados.rados.0.smithi022.stdout:1443: rollback oid 38 current snap is 145 2024-09-21T09:36:17.974 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 38 to 142 2024-09-21T09:36:17.974 INFO:tasks.rados.rados.0.smithi022.stdout:1440: finishing rollback tid 1 to smithi02284960-14 2024-09-21T09:36:17.974 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 14 v 759 (ObjNum 65 snap 0 seq_num 3570437968) dirty dne 2024-09-21T09:36:17.974 INFO:tasks.rados.rados.0.smithi022.stdout:1440: done (1 left) 2024-09-21T09:36:17.974 INFO:tasks.rados.rados.0.smithi022.stdout:1444: read oid 30 snap -1 2024-09-21T09:36:17.974 INFO:tasks.rados.rados.0.smithi022.stdout:1444: expect (ObjNum 439 snap 134 seq_num 439) 2024-09-21T09:36:17.974 INFO:tasks.rados.rados.0.smithi022.stdout:1445: read oid 6 snap -1 2024-09-21T09:36:17.974 INFO:tasks.rados.rados.0.smithi022.stdout:1445: expect (ObjNum 462 snap 140 seq_num 462) 2024-09-21T09:36:17.974 INFO:tasks.rados.rados.0.smithi022.stdout:1446: rollback oid 22 current snap is 145 2024-09-21T09:36:17.974 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 22 to 143 2024-09-21T09:36:17.974 INFO:tasks.rados.rados.0.smithi022.stdout:1447: read oid 50 snap -1 2024-09-21T09:36:17.974 INFO:tasks.rados.rados.0.smithi022.stdout:1447: expect deleted 2024-09-21T09:36:17.974 INFO:tasks.rados.rados.0.smithi022.stdout:1448: rmattr oid 2 current snap is 145 2024-09-21T09:36:17.975 INFO:tasks.rados.rados.0.smithi022.stdout:1448: done (5 left) 2024-09-21T09:36:17.975 INFO:tasks.rados.rados.0.smithi022.stdout:1449: copy_from oid 1 from oid 19 current snap is 145 2024-09-21T09:36:17.975 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 4 current snap is 145 2024-09-21T09:36:17.975 INFO:tasks.rados.rados.0.smithi022.stdout:1450: seq_num 476 ranges {0=778240} 2024-09-21T09:36:17.978 INFO:tasks.rados.rados.0.smithi022.stdout:1450: writing smithi02284960-4 from 0 to 778240 tid 1 2024-09-21T09:36:17.979 INFO:tasks.rados.rados.0.smithi022.stdout:1451: copy_from oid 40 from oid 34 current snap is 145 2024-09-21T09:36:17.979 INFO:tasks.rados.rados.0.smithi022.stdout:1452: read oid 21 snap -1 2024-09-21T09:36:17.979 INFO:tasks.rados.rados.0.smithi022.stdout:1452: expect deleted 2024-09-21T09:36:17.980 INFO:tasks.rados.rados.0.smithi022.stdout:1453: rollback oid 7 current snap is 145 2024-09-21T09:36:17.980 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 7 to 142 2024-09-21T09:36:17.980 INFO:tasks.rados.rados.0.smithi022.stdout:1454: rollback oid 16 current snap is 145 2024-09-21T09:36:17.980 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 16 to 139 2024-09-21T09:36:17.980 INFO:tasks.rados.rados.0.smithi022.stdout:1455: snap_create 2024-09-21T09:36:17.983 INFO:tasks.rados.rados.0.smithi022.stdout:1449: finishing copy_from to smithi02284960-1 2024-09-21T09:36:17.983 INFO:tasks.rados.rados.0.smithi022.stdout:1449: got expected ENOENT (src dne) 2024-09-21T09:36:17.985 INFO:tasks.rados.rados.0.smithi022.stdout:1444: expect (ObjNum 439 snap 134 seq_num 439) 2024-09-21T09:36:17.992 INFO:tasks.rados.rados.0.smithi022.stdout:1449: finishing copy_from racing read to smithi02284960-1 2024-09-21T09:36:17.993 INFO:tasks.rados.rados.0.smithi022.stdout:1446: finishing rollback tid 0 to smithi02284960-22 2024-09-21T09:36:17.993 INFO:tasks.rados.rados.0.smithi022.stdout:1446: finishing rollback tid 1 to smithi02284960-22 2024-09-21T09:36:17.993 INFO:tasks.rados.rados.0.smithi022.stdout:1446: finishing rollback tid 2 to smithi02284960-22 2024-09-21T09:36:17.993 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 22 v 662 (ObjNum 469 snap 141 seq_num 469) dirty exists 2024-09-21T09:36:17.993 INFO:tasks.rados.rados.0.smithi022.stdout:1443: finishing rollback tid 1 to smithi02284960-38 2024-09-21T09:36:17.993 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 38 v 495 (ObjNum 129 snap 0 seq_num 3571207280) dirty dne 2024-09-21T09:36:17.995 INFO:tasks.rados.rados.0.smithi022.stdout:1450: finishing write tid 1 to smithi02284960-4 2024-09-21T09:36:17.995 INFO:tasks.rados.rados.0.smithi022.stdout:1450: finishing write tid 2 to smithi02284960-4 2024-09-21T09:36:17.995 INFO:tasks.rados.rados.0.smithi022.stdout:1454: finishing rollback tid 0 to smithi02284960-16 2024-09-21T09:36:17.996 INFO:tasks.rados.rados.0.smithi022.stdout:1454: finishing rollback tid 1 to smithi02284960-16 2024-09-21T09:36:17.996 INFO:tasks.rados.rados.0.smithi022.stdout:1454: finishing rollback tid 2 to smithi02284960-16 2024-09-21T09:36:17.997 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 16 v 396 (ObjNum 129 snap 0 seq_num 3570284272) dirty exists 2024-09-21T09:36:17.997 INFO:tasks.rados.rados.0.smithi022.stdout:1450: finishing write tid 3 to smithi02284960-4 2024-09-21T09:36:17.997 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 4 v 553 (ObjNum 476 snap 145 seq_num 476) dirty exists 2024-09-21T09:36:17.997 INFO:tasks.rados.rados.0.smithi022.stdout:1450: left oid 4 (ObjNum 476 snap 145 seq_num 476) 2024-09-21T09:36:18.002 INFO:tasks.rados.rados.0.smithi022.stdout:1451: finishing copy_from racing read to smithi02284960-40 2024-09-21T09:36:18.004 INFO:tasks.rados.rados.0.smithi022.stdout:1451: finishing copy_from to smithi02284960-40 2024-09-21T09:36:18.004 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 40 v 471 (ObjNum 449 snap 136 seq_num 449) dirty exists 2024-09-21T09:36:18.005 INFO:tasks.rados.rados.0.smithi022.stdout:1453: finishing rollback tid 1 to smithi02284960-7 2024-09-21T09:36:18.005 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 7 v 813 (ObjNum 129 snap 0 seq_num 3571197768) dirty dne 2024-09-21T09:36:18.008 INFO:tasks.rados.rados.0.smithi022.stdout:1445: expect (ObjNum 462 snap 140 seq_num 462) 2024-09-21T09:36:18.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:17 smithi022 ceph-mon[98615]: pgmap v136: 121 pgs: 121 active+clean; 226 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 2.3 MiB/s rd, 839 KiB/s wr, 9 op/s 2024-09-21T09:36:18.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:17 smithi022 ceph-mon[98615]: osdmap e356: 8 total, 8 up, 8 in 2024-09-21T09:36:18.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:17 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:36:18.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:17 smithi022 ceph-mon[96736]: pgmap v136: 121 pgs: 121 active+clean; 226 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 2.3 MiB/s rd, 839 KiB/s wr, 9 op/s 2024-09-21T09:36:18.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:17 smithi022 ceph-mon[96736]: osdmap e356: 8 total, 8 up, 8 in 2024-09-21T09:36:18.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:17 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:36:18.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:17 smithi064 ceph-mon[86709]: pgmap v136: 121 pgs: 121 active+clean; 226 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 2.3 MiB/s rd, 839 KiB/s wr, 9 op/s 2024-09-21T09:36:18.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:17 smithi064 ceph-mon[86709]: osdmap e356: 8 total, 8 up, 8 in 2024-09-21T09:36:18.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:17 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:36:18.978 INFO:tasks.rados.rados.0.smithi022.stdout:1443: done (11 left) 2024-09-21T09:36:18.978 INFO:tasks.rados.rados.0.smithi022.stdout:1444: done (10 left) 2024-09-21T09:36:18.978 INFO:tasks.rados.rados.0.smithi022.stdout:1445: done (9 left) 2024-09-21T09:36:18.978 INFO:tasks.rados.rados.0.smithi022.stdout:1446: done (8 left) 2024-09-21T09:36:18.979 INFO:tasks.rados.rados.0.smithi022.stdout:1447: done (7 left) 2024-09-21T09:36:18.979 INFO:tasks.rados.rados.0.smithi022.stdout:1449: done (6 left) 2024-09-21T09:36:18.979 INFO:tasks.rados.rados.0.smithi022.stdout:1450: done (5 left) 2024-09-21T09:36:18.979 INFO:tasks.rados.rados.0.smithi022.stdout:1451: done (4 left) 2024-09-21T09:36:18.979 INFO:tasks.rados.rados.0.smithi022.stdout:1452: done (3 left) 2024-09-21T09:36:18.979 INFO:tasks.rados.rados.0.smithi022.stdout:1453: done (2 left) 2024-09-21T09:36:18.979 INFO:tasks.rados.rados.0.smithi022.stdout:1454: done (1 left) 2024-09-21T09:36:18.979 INFO:tasks.rados.rados.0.smithi022.stdout:1455: done (0 left) 2024-09-21T09:36:18.980 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 5 current snap is 146 2024-09-21T09:36:18.980 INFO:tasks.rados.rados.0.smithi022.stdout:1456: seq_num 477 ranges {0=794624} 2024-09-21T09:36:18.982 INFO:tasks.rados.rados.0.smithi022.stdout:1456: writing smithi02284960-5 from 0 to 794624 tid 1 2024-09-21T09:36:18.982 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 10 current snap is 146 2024-09-21T09:36:18.982 INFO:tasks.rados.rados.0.smithi022.stdout:1457: seq_num 478 ranges {614400=409600,1024000=401408} 2024-09-21T09:36:18.988 INFO:tasks.rados.rados.0.smithi022.stdout:1457: writing smithi02284960-10 from 614400 to 1024000 tid 1 2024-09-21T09:36:18.991 INFO:tasks.rados.rados.0.smithi022.stdout:1457: writing smithi02284960-10 from 1024000 to 1425408 tid 2 2024-09-21T09:36:18.991 INFO:tasks.rados.rados.0.smithi022.stdout:1458: rmattr oid 30 current snap is 146 2024-09-21T09:36:18.991 INFO:tasks.rados.rados.0.smithi022.stdout:1458: done (2 left) 2024-09-21T09:36:18.991 INFO:tasks.rados.rados.0.smithi022.stdout:1459: copy_from oid 33 from oid 50 current snap is 146 2024-09-21T09:36:18.991 INFO:tasks.rados.rados.0.smithi022.stdout:1456: finishing write tid 1 to smithi02284960-5 2024-09-21T09:36:18.991 INFO:tasks.rados.rados.0.smithi022.stdout:1456: finishing write tid 2 to smithi02284960-5 2024-09-21T09:36:18.992 INFO:tasks.rados.rados.0.smithi022.stdout:1456: finishing write tid 3 to smithi02284960-5 2024-09-21T09:36:18.992 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 5 v 557 (ObjNum 477 snap 146 seq_num 477) dirty exists 2024-09-21T09:36:18.992 INFO:tasks.rados.rados.0.smithi022.stdout:1456: left oid 5 (ObjNum 477 snap 146 seq_num 477) 2024-09-21T09:36:18.992 INFO:tasks.rados.rados.0.smithi022.stdout:1456: done (2 left) 2024-09-21T09:36:18.992 INFO:tasks.rados.rados.0.smithi022.stdout:1460: read oid 28 snap -1 2024-09-21T09:36:18.992 INFO:tasks.rados.rados.0.smithi022.stdout:1460: expect (ObjNum 392 snap 122 seq_num 392) 2024-09-21T09:36:18.992 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 20 current snap is 146 2024-09-21T09:36:18.992 INFO:tasks.rados.rados.0.smithi022.stdout:1461: seq_num 479 ranges {4562944=425984,4988928=401408} 2024-09-21T09:36:19.019 INFO:tasks.rados.rados.0.smithi022.stdout:1461: writing smithi02284960-20 from 4562944 to 4988928 tid 1 2024-09-21T09:36:19.022 INFO:tasks.rados.rados.0.smithi022.stdout:1461: writing smithi02284960-20 from 4988928 to 5390336 tid 2 2024-09-21T09:36:19.022 INFO:tasks.rados.rados.0.smithi022.stdout:1462: copy_from oid 36 from oid 38 current snap is 146 2024-09-21T09:36:19.022 INFO:tasks.rados.rados.0.smithi022.stdout:1459: finishing copy_from to smithi02284960-33 2024-09-21T09:36:19.022 INFO:tasks.rados.rados.0.smithi022.stdout:1459: got expected ENOENT (src dne) 2024-09-21T09:36:19.023 INFO:tasks.rados.rados.0.smithi022.stdout:1459: finishing copy_from racing read to smithi02284960-33 2024-09-21T09:36:19.023 INFO:tasks.rados.rados.0.smithi022.stdout:1457: finishing write tid 1 to smithi02284960-10 2024-09-21T09:36:19.023 INFO:tasks.rados.rados.0.smithi022.stdout:1457: finishing write tid 2 to smithi02284960-10 2024-09-21T09:36:19.023 INFO:tasks.rados.rados.0.smithi022.stdout:1457: finishing write tid 3 to smithi02284960-10 2024-09-21T09:36:19.023 INFO:tasks.rados.rados.0.smithi022.stdout:1457: finishing write tid 4 to smithi02284960-10 2024-09-21T09:36:19.023 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 10 v 855 (ObjNum 478 snap 146 seq_num 478) dirty exists 2024-09-21T09:36:19.024 INFO:tasks.rados.rados.0.smithi022.stdout:1457: left oid 10 (ObjNum 478 snap 146 seq_num 478) 2024-09-21T09:36:19.024 INFO:tasks.rados.rados.0.smithi022.stdout:1460: expect (ObjNum 392 snap 122 seq_num 392) 2024-09-21T09:36:19.070 INFO:tasks.rados.rados.0.smithi022.stdout:1462: finishing copy_from to smithi02284960-36 2024-09-21T09:36:19.070 INFO:tasks.rados.rados.0.smithi022.stdout:1462: got expected ENOENT (src dne) 2024-09-21T09:36:19.070 INFO:tasks.rados.rados.0.smithi022.stdout:1462: finishing copy_from racing read to smithi02284960-36 2024-09-21T09:36:19.070 INFO:tasks.rados.rados.0.smithi022.stdout:1461: finishing write tid 1 to smithi02284960-20 2024-09-21T09:36:19.070 INFO:tasks.rados.rados.0.smithi022.stdout:1461: finishing write tid 2 to smithi02284960-20 2024-09-21T09:36:19.070 INFO:tasks.rados.rados.0.smithi022.stdout:1461: finishing write tid 3 to smithi02284960-20 2024-09-21T09:36:19.070 INFO:tasks.rados.rados.0.smithi022.stdout:1461: finishing write tid 4 to smithi02284960-20 2024-09-21T09:36:19.070 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 20 v 940 (ObjNum 479 snap 146 seq_num 479) dirty exists 2024-09-21T09:36:19.070 INFO:tasks.rados.rados.0.smithi022.stdout:1461: left oid 20 (ObjNum 479 snap 146 seq_num 479) 2024-09-21T09:36:19.071 INFO:tasks.rados.rados.0.smithi022.stdout:1457: done (4 left) 2024-09-21T09:36:19.071 INFO:tasks.rados.rados.0.smithi022.stdout:1459: done (3 left) 2024-09-21T09:36:19.071 INFO:tasks.rados.rados.0.smithi022.stdout:1460: done (2 left) 2024-09-21T09:36:19.071 INFO:tasks.rados.rados.0.smithi022.stdout:1461: done (1 left) 2024-09-21T09:36:19.071 INFO:tasks.rados.rados.0.smithi022.stdout:1462: done (0 left) 2024-09-21T09:36:19.071 INFO:tasks.rados.rados.0.smithi022.stdout:1463: snap_remove snap 145 2024-09-21T09:36:19.307 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:18 smithi022 ceph-mon[98615]: osdmap e357: 8 total, 8 up, 8 in 2024-09-21T09:36:19.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:18 smithi022 ceph-mon[96736]: osdmap e357: 8 total, 8 up, 8 in 2024-09-21T09:36:19.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:18 smithi064 ceph-mon[86709]: osdmap e357: 8 total, 8 up, 8 in 2024-09-21T09:36:19.983 INFO:tasks.rados.rados.0.smithi022.stdout:1463: done (0 left) 2024-09-21T09:36:19.984 INFO:tasks.rados.rados.0.smithi022.stdout:1464: snap_remove snap 140 2024-09-21T09:36:20.302 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:19 smithi022 ceph-mon[98615]: pgmap v139: 121 pgs: 121 active+clean; 145 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.9 MiB/s rd, 922 KiB/s wr, 4 op/s 2024-09-21T09:36:20.302 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:19 smithi022 ceph-mon[98615]: osdmap e358: 8 total, 8 up, 8 in 2024-09-21T09:36:20.302 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:19 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-21T09:36:20.302 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:19 smithi022 ceph-mon[96736]: pgmap v139: 121 pgs: 121 active+clean; 145 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.9 MiB/s rd, 922 KiB/s wr, 4 op/s 2024-09-21T09:36:20.302 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:19 smithi022 ceph-mon[96736]: osdmap e358: 8 total, 8 up, 8 in 2024-09-21T09:36:20.302 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:19 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-21T09:36:20.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:19 smithi064 ceph-mon[86709]: pgmap v139: 121 pgs: 121 active+clean; 145 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.9 MiB/s rd, 922 KiB/s wr, 4 op/s 2024-09-21T09:36:20.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:19 smithi064 ceph-mon[86709]: osdmap e358: 8 total, 8 up, 8 in 2024-09-21T09:36:20.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:19 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-21T09:36:20.442 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:36:20 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:36:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:36:20.988 INFO:tasks.rados.rados.0.smithi022.stdout:1464: done (0 left) 2024-09-21T09:36:20.988 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 12 current snap is 146 2024-09-21T09:36:20.989 INFO:tasks.rados.rados.0.smithi022.stdout:1465: seq_num 480 ranges {0=442368,442368=401408} 2024-09-21T09:36:20.991 INFO:tasks.rados.rados.0.smithi022.stdout:1465: writing smithi02284960-12 from 0 to 442368 tid 1 2024-09-21T09:36:20.994 INFO:tasks.rados.rados.0.smithi022.stdout:1465: writing smithi02284960-12 from 442368 to 843776 tid 2 2024-09-21T09:36:20.995 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 30 current snap is 146 2024-09-21T09:36:20.995 INFO:tasks.rados.rados.0.smithi022.stdout:1466: seq_num 481 ranges {155648=458752,614400=401408} 2024-09-21T09:36:20.999 INFO:tasks.rados.rados.0.smithi022.stdout:1466: writing smithi02284960-30 from 155648 to 614400 tid 1 2024-09-21T09:36:21.003 INFO:tasks.rados.rados.0.smithi022.stdout:1466: writing smithi02284960-30 from 614400 to 1015808 tid 2 2024-09-21T09:36:21.003 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 48 current snap is 146 2024-09-21T09:36:21.003 INFO:tasks.rados.rados.0.smithi022.stdout:1467: seq_num 482 ranges {0=475136,475136=401408} 2024-09-21T09:36:21.007 INFO:tasks.rados.rados.0.smithi022.stdout:1467: writing smithi02284960-48 from 0 to 475136 tid 1 2024-09-21T09:36:21.011 INFO:tasks.rados.rados.0.smithi022.stdout:1467: writing smithi02284960-48 from 475136 to 876544 tid 2 2024-09-21T09:36:21.011 INFO:tasks.rados.rados.0.smithi022.stdout:1465: finishing write tid 1 to smithi02284960-12 2024-09-21T09:36:21.011 INFO:tasks.rados.rados.0.smithi022.stdout:1465: finishing write tid 2 to smithi02284960-12 2024-09-21T09:36:21.011 INFO:tasks.rados.rados.0.smithi022.stdout:1465: finishing write tid 3 to smithi02284960-12 2024-09-21T09:36:21.011 INFO:tasks.rados.rados.0.smithi022.stdout:1466: finishing write tid 1 to smithi02284960-30 2024-09-21T09:36:21.011 INFO:tasks.rados.rados.0.smithi022.stdout:1466: finishing write tid 2 to smithi02284960-30 2024-09-21T09:36:21.011 INFO:tasks.rados.rados.0.smithi022.stdout:1465: finishing write tid 4 to smithi02284960-12 2024-09-21T09:36:21.011 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 12 v 999 (ObjNum 480 snap 146 seq_num 480) dirty exists 2024-09-21T09:36:21.011 INFO:tasks.rados.rados.0.smithi022.stdout:1465: left oid 12 (ObjNum 480 snap 146 seq_num 480) 2024-09-21T09:36:21.011 INFO:tasks.rados.rados.0.smithi022.stdout:1466: finishing write tid 3 to smithi02284960-30 2024-09-21T09:36:21.011 INFO:tasks.rados.rados.0.smithi022.stdout:1466: finishing write tid 4 to smithi02284960-30 2024-09-21T09:36:21.011 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 30 v 900 (ObjNum 481 snap 146 seq_num 481) dirty exists 2024-09-21T09:36:21.011 INFO:tasks.rados.rados.0.smithi022.stdout:1466: left oid 30 (ObjNum 481 snap 146 seq_num 481) 2024-09-21T09:36:21.012 INFO:tasks.rados.rados.0.smithi022.stdout:1465: done (2 left) 2024-09-21T09:36:21.012 INFO:tasks.rados.rados.0.smithi022.stdout:1466: done (1 left) 2024-09-21T09:36:21.012 INFO:tasks.rados.rados.0.smithi022.stdout:1468: copy_from oid 47 from oid 32 current snap is 146 2024-09-21T09:36:21.012 INFO:tasks.rados.rados.0.smithi022.stdout:1469: copy_from oid 45 from oid 37 current snap is 146 2024-09-21T09:36:21.012 INFO:tasks.rados.rados.0.smithi022.stdout:1470: rollback oid 35 current snap is 146 2024-09-21T09:36:21.012 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 35 to 142 2024-09-21T09:36:21.012 INFO:tasks.rados.rados.0.smithi022.stdout:1471: read oid 40 snap -1 2024-09-21T09:36:21.012 INFO:tasks.rados.rados.0.smithi022.stdout:1471: expect (ObjNum 449 snap 136 seq_num 449) 2024-09-21T09:36:21.012 INFO:tasks.rados.rados.0.smithi022.stdout:1472: snap_remove snap 144 2024-09-21T09:36:21.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:20 smithi022 ceph-mon[98615]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-21T09:36:21.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:20 smithi022 ceph-mon[98615]: Upgrade: osd.1 is safe to restart 2024-09-21T09:36:21.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:20 smithi022 ceph-mon[98615]: osdmap e359: 8 total, 8 up, 8 in 2024-09-21T09:36:21.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:20 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:36:21.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:20 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-21T09:36:21.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:20 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:36:21.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:20 smithi022 ceph-mon[96736]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-21T09:36:21.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:20 smithi022 ceph-mon[96736]: Upgrade: osd.1 is safe to restart 2024-09-21T09:36:21.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:20 smithi022 ceph-mon[96736]: osdmap e359: 8 total, 8 up, 8 in 2024-09-21T09:36:21.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:20 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:36:21.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:20 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-21T09:36:21.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:20 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:36:21.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:20 smithi064 ceph-mon[86709]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-21T09:36:21.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:20 smithi064 ceph-mon[86709]: Upgrade: osd.1 is safe to restart 2024-09-21T09:36:21.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:20 smithi064 ceph-mon[86709]: osdmap e359: 8 total, 8 up, 8 in 2024-09-21T09:36:21.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:20 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:36:21.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:20 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-21T09:36:21.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:20 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:36:21.992 INFO:tasks.rados.rados.0.smithi022.stdout:1472: done (5 left) 2024-09-21T09:36:21.993 INFO:tasks.rados.rados.0.smithi022.stdout:1473: copy_from oid 11 from oid 17 current snap is 146 2024-09-21T09:36:21.993 INFO:tasks.rados.rados.0.smithi022.stdout:1467: finishing write tid 1 to smithi02284960-48 2024-09-21T09:36:21.993 INFO:tasks.rados.rados.0.smithi022.stdout:1470: finishing rollback tid 0 to smithi02284960-35 2024-09-21T09:36:21.993 INFO:tasks.rados.rados.0.smithi022.stdout:1467: finishing write tid 2 to smithi02284960-48 2024-09-21T09:36:21.993 INFO:tasks.rados.rados.0.smithi022.stdout:1467: finishing write tid 3 to smithi02284960-48 2024-09-21T09:36:21.993 INFO:tasks.rados.rados.0.smithi022.stdout:1467: finishing write tid 4 to smithi02284960-48 2024-09-21T09:36:21.993 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 48 v 947 (ObjNum 482 snap 146 seq_num 482) dirty exists 2024-09-21T09:36:21.993 INFO:tasks.rados.rados.0.smithi022.stdout:1467: left oid 48 (ObjNum 482 snap 146 seq_num 482) 2024-09-21T09:36:21.993 INFO:tasks.rados.rados.0.smithi022.stdout:1470: finishing rollback tid 1 to smithi02284960-35 2024-09-21T09:36:21.993 INFO:tasks.rados.rados.0.smithi022.stdout:1470: finishing rollback tid 2 to smithi02284960-35 2024-09-21T09:36:21.993 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 35 v 547 (ObjNum 471 snap 142 seq_num 471) dirty exists 2024-09-21T09:36:21.993 INFO:tasks.rados.rados.0.smithi022.stdout:1468: finishing copy_from racing read to smithi02284960-47 2024-09-21T09:36:21.993 INFO:tasks.rados.rados.0.smithi022.stdout:1468: finishing copy_from to smithi02284960-47 2024-09-21T09:36:21.993 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 47 v 837 (ObjNum 382 snap 119 seq_num 382) dirty exists 2024-09-21T09:36:21.993 INFO:tasks.rados.rados.0.smithi022.stdout:1469: finishing copy_from racing read to smithi02284960-45 2024-09-21T09:36:21.994 INFO:tasks.rados.rados.0.smithi022.stdout:1469: finishing copy_from to smithi02284960-45 2024-09-21T09:36:21.994 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 45 v 898 (ObjNum 442 snap 134 seq_num 442) dirty exists 2024-09-21T09:36:21.994 INFO:tasks.rados.rados.0.smithi022.stdout:1471: expect (ObjNum 449 snap 136 seq_num 449) 2024-09-21T09:36:22.151 INFO:tasks.rados.rados.0.smithi022.stdout:1473: finishing copy_from racing read to smithi02284960-11 2024-09-21T09:36:22.151 INFO:tasks.rados.rados.0.smithi022.stdout:1473: finishing copy_from to smithi02284960-11 2024-09-21T09:36:22.151 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 11 v 897 (ObjNum 470 snap 142 seq_num 470) dirty exists 2024-09-21T09:36:22.152 INFO:tasks.rados.rados.0.smithi022.stdout:1467: done (5 left) 2024-09-21T09:36:22.152 INFO:tasks.rados.rados.0.smithi022.stdout:1468: done (4 left) 2024-09-21T09:36:22.152 INFO:tasks.rados.rados.0.smithi022.stdout:1469: done (3 left) 2024-09-21T09:36:22.152 INFO:tasks.rados.rados.0.smithi022.stdout:1470: done (2 left) 2024-09-21T09:36:22.152 INFO:tasks.rados.rados.0.smithi022.stdout:1471: done (1 left) 2024-09-21T09:36:22.152 INFO:tasks.rados.rados.0.smithi022.stdout:1473: done (0 left) 2024-09-21T09:36:22.152 INFO:tasks.rados.rados.0.smithi022.stdout:1474: read oid 19 snap 142 2024-09-21T09:36:22.152 INFO:tasks.rados.rados.0.smithi022.stdout:1474: expect deleted 2024-09-21T09:36:22.152 INFO:tasks.rados.rados.0.smithi022.stdout:1475: rollback oid 14 current snap is 146 2024-09-21T09:36:22.152 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 14 to 142 2024-09-21T09:36:22.152 INFO:tasks.rados.rados.0.smithi022.stdout:1476: read oid 23 snap -1 2024-09-21T09:36:22.152 INFO:tasks.rados.rados.0.smithi022.stdout:1476: expect (ObjNum 309 snap 91 seq_num 309) 2024-09-21T09:36:22.152 INFO:tasks.rados.rados.0.smithi022.stdout:1477: snap_remove snap 139 2024-09-21T09:36:22.277 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:21 smithi022 ceph-mon[98615]: pgmap v142: 121 pgs: 121 active+clean; 145 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.9 MiB/s rd, 922 KiB/s wr, 4 op/s 2024-09-21T09:36:22.277 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:21 smithi022 ceph-mon[98615]: Upgrade: Updating osd.1 2024-09-21T09:36:22.277 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:21 smithi022 ceph-mon[98615]: Deploying daemon osd.1 on smithi022 2024-09-21T09:36:22.277 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:21 smithi022 ceph-mon[98615]: osdmap e360: 8 total, 8 up, 8 in 2024-09-21T09:36:22.277 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:21 smithi022 ceph-mon[96736]: pgmap v142: 121 pgs: 121 active+clean; 145 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.9 MiB/s rd, 922 KiB/s wr, 4 op/s 2024-09-21T09:36:22.277 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:21 smithi022 ceph-mon[96736]: Upgrade: Updating osd.1 2024-09-21T09:36:22.277 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:21 smithi022 ceph-mon[96736]: Deploying daemon osd.1 on smithi022 2024-09-21T09:36:22.278 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:21 smithi022 ceph-mon[96736]: osdmap e360: 8 total, 8 up, 8 in 2024-09-21T09:36:22.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:21 smithi064 ceph-mon[86709]: pgmap v142: 121 pgs: 121 active+clean; 145 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.9 MiB/s rd, 922 KiB/s wr, 4 op/s 2024-09-21T09:36:22.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:21 smithi064 ceph-mon[86709]: Upgrade: Updating osd.1 2024-09-21T09:36:22.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:21 smithi064 ceph-mon[86709]: Deploying daemon osd.1 on smithi022 2024-09-21T09:36:22.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:21 smithi064 ceph-mon[86709]: osdmap e360: 8 total, 8 up, 8 in 2024-09-21T09:36:22.999 INFO:tasks.rados.rados.0.smithi022.stdout:1477: done (3 left) 2024-09-21T09:36:22.999 INFO:tasks.rados.rados.0.smithi022.stdout:1478: delete oid 17 current snap is 146 2024-09-21T09:36:22.999 INFO:tasks.rados.rados.0.smithi022.stdout:1475: finishing rollback tid 1 to smithi02284960-14 2024-09-21T09:36:22.999 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 14 v 759 (ObjNum 177 snap 0 seq_num 1) dirty dne 2024-09-21T09:36:22.999 INFO:tasks.rados.rados.0.smithi022.stdout:1476: expect (ObjNum 309 snap 91 seq_num 309) 2024-09-21T09:36:23.125 INFO:tasks.rados.rados.0.smithi022.stdout:1474: done (3 left) 2024-09-21T09:36:23.125 INFO:tasks.rados.rados.0.smithi022.stdout:1475: done (2 left) 2024-09-21T09:36:23.125 INFO:tasks.rados.rados.0.smithi022.stdout:1476: done (1 left) 2024-09-21T09:36:23.125 INFO:tasks.rados.rados.0.smithi022.stdout:1478: done (0 left) 2024-09-21T09:36:23.125 INFO:tasks.rados.rados.0.smithi022.stdout:1479: copy_from oid 23 from oid 1 current snap is 146 2024-09-21T09:36:23.125 INFO:tasks.rados.rados.0.smithi022.stdout:1480: snap_remove snap 142 2024-09-21T09:36:23.218 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:22 smithi022 ceph-mon[96736]: osdmap e361: 8 total, 8 up, 8 in 2024-09-21T09:36:23.218 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:22 smithi022 ceph-mon[98615]: osdmap e361: 8 total, 8 up, 8 in 2024-09-21T09:36:23.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:22 smithi064 ceph-mon[86709]: osdmap e361: 8 total, 8 up, 8 in 2024-09-21T09:36:23.958 INFO:tasks.rados.rados.0.smithi022.stdout:1480: done (1 left) 2024-09-21T09:36:23.958 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 39 current snap is 146 2024-09-21T09:36:23.958 INFO:tasks.rados.rados.0.smithi022.stdout:1481: seq_num 484 ranges {0=507904,507904=401408} 2024-09-21T09:36:23.960 INFO:tasks.rados.rados.0.smithi022.stdout:1481: writing smithi02284960-39 from 0 to 507904 tid 1 2024-09-21T09:36:23.963 INFO:tasks.rados.rados.0.smithi022.stdout:1481: writing smithi02284960-39 from 507904 to 909312 tid 2 2024-09-21T09:36:23.964 INFO:tasks.rados.rados.0.smithi022.stdout:1482: copy_from oid 34 from oid 31 current snap is 146 2024-09-21T09:36:23.964 INFO:tasks.rados.rados.0.smithi022.stdout:1479: finishing copy_from racing read to smithi02284960-23 2024-09-21T09:36:23.964 INFO:tasks.rados.rados.0.smithi022.stdout:1479: finishing copy_from to smithi02284960-23 2024-09-21T09:36:23.964 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 23 v 802 (ObjNum 423 snap 132 seq_num 423) dirty exists 2024-09-21T09:36:23.964 INFO:tasks.rados.rados.0.smithi022.stdout:1479: done (2 left) 2024-09-21T09:36:23.965 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 38 current snap is 146 2024-09-21T09:36:23.965 INFO:tasks.rados.rados.0.smithi022.stdout:1483: seq_num 485 ranges {0=532480,532480=401408} 2024-09-21T09:36:23.968 INFO:tasks.rados.rados.0.smithi022.stdout:1483: writing smithi02284960-38 from 0 to 532480 tid 1 2024-09-21T09:36:23.971 INFO:tasks.rados.rados.0.smithi022.stdout:1483: writing smithi02284960-38 from 532480 to 933888 tid 2 2024-09-21T09:36:23.973 INFO:tasks.rados.rados.0.smithi022.stdout:1481: finishing write tid 1 to smithi02284960-39 2024-09-21T09:36:23.973 INFO:tasks.rados.rados.0.smithi022.stdout:1481: finishing write tid 2 to smithi02284960-39 2024-09-21T09:36:23.973 INFO:tasks.rados.rados.0.smithi022.stdout:1481: finishing write tid 3 to smithi02284960-39 2024-09-21T09:36:23.973 INFO:tasks.rados.rados.0.smithi022.stdout:1484: read oid 17 snap -1 2024-09-21T09:36:23.973 INFO:tasks.rados.rados.0.smithi022.stdout:1484: expect deleted 2024-09-21T09:36:23.973 INFO:tasks.rados.rados.0.smithi022.stdout:1485: read oid 44 snap -1 2024-09-21T09:36:23.973 INFO:tasks.rados.rados.0.smithi022.stdout:1485: expect deleted 2024-09-21T09:36:23.973 INFO:tasks.rados.rados.0.smithi022.stdout:1486: read oid 10 snap -1 2024-09-21T09:36:23.973 INFO:tasks.rados.rados.0.smithi022.stdout:1486: expect (ObjNum 478 snap 146 seq_num 478) 2024-09-21T09:36:23.973 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 15 current snap is 146 2024-09-21T09:36:23.973 INFO:tasks.rados.rados.0.smithi022.stdout:1487: seq_num 486 ranges {647168=548864,1196032=401408} 2024-09-21T09:36:23.981 INFO:tasks.rados.rados.0.smithi022.stdout:1487: writing smithi02284960-15 from 647168 to 1196032 tid 1 2024-09-21T09:36:23.984 INFO:tasks.rados.rados.0.smithi022.stdout:1487: writing smithi02284960-15 from 1196032 to 1597440 tid 2 2024-09-21T09:36:23.984 INFO:tasks.rados.rados.0.smithi022.stdout:1488: rollback oid 4 current snap is 146 2024-09-21T09:36:23.985 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 4 to 134 2024-09-21T09:36:23.985 INFO:tasks.rados.rados.0.smithi022.stdout:1481: finishing write tid 4 to smithi02284960-39 2024-09-21T09:36:23.985 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 39 v 904 (ObjNum 484 snap 146 seq_num 484) dirty exists 2024-09-21T09:36:23.985 INFO:tasks.rados.rados.0.smithi022.stdout:1481: left oid 39 (ObjNum 484 snap 146 seq_num 484) 2024-09-21T09:36:23.985 INFO:tasks.rados.rados.0.smithi022.stdout:1481: done (7 left) 2024-09-21T09:36:23.985 INFO:tasks.rados.rados.0.smithi022.stdout:1489: read oid 6 snap -1 2024-09-21T09:36:23.985 INFO:tasks.rados.rados.0.smithi022.stdout:1489: expect (ObjNum 462 snap 140 seq_num 462) 2024-09-21T09:36:23.985 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 1 current snap is 146 2024-09-21T09:36:23.985 INFO:tasks.rados.rados.0.smithi022.stdout:1483: finishing write tid 1 to smithi02284960-38 2024-09-21T09:36:23.985 INFO:tasks.rados.rados.0.smithi022.stdout:1482: finishing copy_from racing read to smithi02284960-34 2024-09-21T09:36:23.985 INFO:tasks.rados.rados.0.smithi022.stdout:1483: finishing write tid 2 to smithi02284960-38 2024-09-21T09:36:23.985 INFO:tasks.rados.rados.0.smithi022.stdout:1483: finishing write tid 3 to smithi02284960-38 2024-09-21T09:36:23.985 INFO:tasks.rados.rados.0.smithi022.stdout:1490: seq_num 487 ranges {2908160=565248,3473408=401408} 2024-09-21T09:36:24.005 INFO:tasks.rados.rados.0.smithi022.stdout:1490: writing smithi02284960-1 from 2908160 to 3473408 tid 1 2024-09-21T09:36:24.009 INFO:tasks.rados.rados.0.smithi022.stdout:1490: writing smithi02284960-1 from 3473408 to 3874816 tid 2 2024-09-21T09:36:24.009 INFO:tasks.rados.rados.0.smithi022.stdout:1484: done (8 left) 2024-09-21T09:36:24.009 INFO:tasks.rados.rados.0.smithi022.stdout:1485: done (7 left) 2024-09-21T09:36:24.009 INFO:tasks.rados.rados.0.smithi022.stdout:1491: snap_remove snap 141 2024-09-21T09:36:24.245 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:23 smithi022 ceph-mon[98615]: pgmap v145: 121 pgs: 121 active+clean; 145 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:36:24.245 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:23 smithi022 ceph-mon[98615]: osdmap e362: 8 total, 8 up, 8 in 2024-09-21T09:36:24.245 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:23 smithi022 ceph-mon[98615]: osdmap e363: 8 total, 8 up, 8 in 2024-09-21T09:36:24.245 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:23 smithi022 ceph-mon[96736]: pgmap v145: 121 pgs: 121 active+clean; 145 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:36:24.246 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:23 smithi022 ceph-mon[96736]: osdmap e362: 8 total, 8 up, 8 in 2024-09-21T09:36:24.246 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:23 smithi022 ceph-mon[96736]: osdmap e363: 8 total, 8 up, 8 in 2024-09-21T09:36:24.246 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 21 09:36:23 smithi022 systemd[1]: Stopping Ceph osd.1 for d65e801e-77fa-11ef-baf6-efdc52797490... 2024-09-21T09:36:24.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:23 smithi064 ceph-mon[86709]: pgmap v145: 121 pgs: 121 active+clean; 145 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:36:24.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:23 smithi064 ceph-mon[86709]: osdmap e362: 8 total, 8 up, 8 in 2024-09-21T09:36:24.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:23 smithi064 ceph-mon[86709]: osdmap e363: 8 total, 8 up, 8 in 2024-09-21T09:36:24.442 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:36:24 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:36:24.148Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.1\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.1\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.1\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:36:24.558 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 21 09:36:24 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-1[50027]: 2024-09-21T09:36:24.242+0000 7f3a7de86640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T09:36:24.558 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 21 09:36:24 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-1[50027]: 2024-09-21T09:36:24.242+0000 7f3a7de86640 -1 osd.1 363 *** Got signal Terminated *** 2024-09-21T09:36:24.558 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 21 09:36:24 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-1[50027]: 2024-09-21T09:36:24.242+0000 7f3a7de86640 -1 osd.1 363 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T09:36:25.014 INFO:tasks.rados.rados.0.smithi022.stdout:1483: finishing write tid 4 to smithi02284960-38 2024-09-21T09:36:25.014 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 38 v 834 (ObjNum 485 snap 146 seq_num 485) dirty exists 2024-09-21T09:36:25.014 INFO:tasks.rados.rados.0.smithi022.stdout:1483: left oid 38 (ObjNum 485 snap 146 seq_num 485) 2024-09-21T09:36:25.015 INFO:tasks.rados.rados.0.smithi022.stdout:1486: expect (ObjNum 478 snap 146 seq_num 478) 2024-09-21T09:36:25.063 INFO:tasks.rados.rados.0.smithi022.stdout:1482: finishing copy_from to smithi02284960-34 2024-09-21T09:36:25.063 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 34 v 967 (ObjNum 474 snap 144 seq_num 474) dirty exists 2024-09-21T09:36:25.063 INFO:tasks.rados.rados.0.smithi022.stdout:1488: finishing rollback tid 0 to smithi02284960-4 2024-09-21T09:36:25.063 INFO:tasks.rados.rados.0.smithi022.stdout:1488: finishing rollback tid 1 to smithi02284960-4 2024-09-21T09:36:25.063 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 4 v 559 (ObjNum 145 snap 0 seq_num 3570210040) dirty dne 2024-09-21T09:36:25.063 INFO:tasks.rados.rados.0.smithi022.stdout:1487: finishing write tid 1 to smithi02284960-15 2024-09-21T09:36:25.063 INFO:tasks.rados.rados.0.smithi022.stdout:1487: finishing write tid 2 to smithi02284960-15 2024-09-21T09:36:25.063 INFO:tasks.rados.rados.0.smithi022.stdout:1487: finishing write tid 3 to smithi02284960-15 2024-09-21T09:36:25.063 INFO:tasks.rados.rados.0.smithi022.stdout:1487: finishing write tid 4 to smithi02284960-15 2024-09-21T09:36:25.063 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 15 v 1013 (ObjNum 486 snap 146 seq_num 486) dirty exists 2024-09-21T09:36:25.063 INFO:tasks.rados.rados.0.smithi022.stdout:1487: left oid 15 (ObjNum 486 snap 146 seq_num 486) 2024-09-21T09:36:25.064 INFO:tasks.rados.rados.0.smithi022.stdout:1489: expect (ObjNum 462 snap 140 seq_num 462) 2024-09-21T09:36:25.269 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:25 smithi022 ceph-mon[98615]: osd.1 marked itself down and dead 2024-09-21T09:36:25.269 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:24 smithi022 ceph-mon[96736]: osd.1 marked itself down and dead 2024-09-21T09:36:25.289 INFO:tasks.rados.rados.0.smithi022.stdout:1490: finishing write tid 1 to smithi02284960-1 2024-09-21T09:36:25.289 INFO:tasks.rados.rados.0.smithi022.stdout:1490: finishing write tid 2 to smithi02284960-1 2024-09-21T09:36:25.289 INFO:tasks.rados.rados.0.smithi022.stdout:1490: finishing write tid 3 to smithi02284960-1 2024-09-21T09:36:25.289 INFO:tasks.rados.rados.0.smithi022.stdout:1490: finishing write tid 4 to smithi02284960-1 2024-09-21T09:36:25.289 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 1 v 928 (ObjNum 487 snap 146 seq_num 487) dirty exists 2024-09-21T09:36:25.289 INFO:tasks.rados.rados.0.smithi022.stdout:1490: left oid 1 (ObjNum 487 snap 146 seq_num 487) 2024-09-21T09:36:25.290 INFO:tasks.rados.rados.0.smithi022.stdout:1482: done (7 left) 2024-09-21T09:36:25.290 INFO:tasks.rados.rados.0.smithi022.stdout:1483: done (6 left) 2024-09-21T09:36:25.290 INFO:tasks.rados.rados.0.smithi022.stdout:1486: done (5 left) 2024-09-21T09:36:25.290 INFO:tasks.rados.rados.0.smithi022.stdout:1487: done (4 left) 2024-09-21T09:36:25.290 INFO:tasks.rados.rados.0.smithi022.stdout:1488: done (3 left) 2024-09-21T09:36:25.290 INFO:tasks.rados.rados.0.smithi022.stdout:1489: done (2 left) 2024-09-21T09:36:25.290 INFO:tasks.rados.rados.0.smithi022.stdout:1490: done (1 left) 2024-09-21T09:36:25.290 INFO:tasks.rados.rados.0.smithi022.stdout:1491: done (0 left) 2024-09-21T09:36:25.290 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 5 current snap is 146 2024-09-21T09:36:25.290 INFO:tasks.rados.rados.0.smithi022.stdout:1492: seq_num 488 ranges {794624=581632,1376256=401408} 2024-09-21T09:36:25.296 INFO:tasks.rados.rados.0.smithi022.stdout:1492: writing smithi02284960-5 from 794624 to 1376256 tid 1 2024-09-21T09:36:25.299 INFO:tasks.rados.rados.0.smithi022.stdout:1492: writing smithi02284960-5 from 1376256 to 1777664 tid 2 2024-09-21T09:36:25.299 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 39 current snap is 146 2024-09-21T09:36:25.299 INFO:tasks.rados.rados.0.smithi022.stdout:1493: seq_num 489 ranges {909312=598016,1507328=401408} 2024-09-21T09:36:25.307 INFO:tasks.rados.rados.0.smithi022.stdout:1493: writing smithi02284960-39 from 909312 to 1507328 tid 1 2024-09-21T09:36:25.310 INFO:tasks.rados.rados.0.smithi022.stdout:1493: writing smithi02284960-39 from 1507328 to 1908736 tid 2 2024-09-21T09:36:25.310 INFO:tasks.rados.rados.0.smithi022.stdout:1494: copy_from oid 18 from oid 48 current snap is 146 2024-09-21T09:36:25.310 INFO:tasks.rados.rados.0.smithi022.stdout:1495: read oid 6 snap 128 2024-09-21T09:36:25.310 INFO:tasks.rados.rados.0.smithi022.stdout:1495: expect (ObjNum 408 snap 128 seq_num 408) 2024-09-21T09:36:25.310 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 13 current snap is 146 2024-09-21T09:36:25.310 INFO:tasks.rados.rados.0.smithi022.stdout:1496: seq_num 490 ranges {5742592=614400,6356992=401408} 2024-09-21T09:36:25.344 INFO:tasks.rados.rados.0.smithi022.stdout:1496: writing smithi02284960-13 from 5742592 to 6356992 tid 1 2024-09-21T09:36:25.347 INFO:tasks.rados.rados.0.smithi022.stdout:1496: writing smithi02284960-13 from 6356992 to 6758400 tid 2 2024-09-21T09:36:25.347 INFO:tasks.rados.rados.0.smithi022.stdout:1497: setattr oid 50 current snap is 146 2024-09-21T09:36:25.347 INFO:tasks.rados.rados.0.smithi022.stdout:1498: snap_create 2024-09-21T09:36:25.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:25 smithi064 ceph-mon[86709]: osd.1 marked itself down and dead 2024-09-21T09:36:25.808 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:36:25 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:36:25] "GET /metrics HTTP/1.1" 200 36179 "" "Prometheus/2.51.0" 2024-09-21T09:36:25.808 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 21 09:36:25 smithi022 podman[107950]: 2024-09-21 09:36:25.556010587 +0000 UTC m=+1.418566197 container died 740462b5c304f749f4685a5816a42391bb2a99768c05ca153fb2b4092627e900 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-1, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=quincy-fbd1a6a, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2) 2024-09-21T09:36:26.002 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:36:25 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:36:25.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:36:26.002 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:36:25 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:36:25.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:36:26.008 INFO:tasks.rados.rados.0.smithi022.stdout:1498: done (6 left) 2024-09-21T09:36:26.008 INFO:tasks.rados.rados.0.smithi022.stdout:1499: rmattr oid 47 current snap is 147 2024-09-21T09:36:26.008 INFO:tasks.rados.rados.0.smithi022.stdout:1499: done (6 left) 2024-09-21T09:36:26.008 INFO:tasks.rados.rados.0.smithi022.stdout:1500: snap_remove snap 146 2024-09-21T09:36:26.126 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:26 smithi022 ceph-mon[98615]: pgmap v148: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 157 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4.8 MiB/s rd, 3.1 MiB/s wr, 11 op/s 2024-09-21T09:36:26.126 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:26 smithi022 ceph-mon[98615]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T09:36:26.126 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:26 smithi022 ceph-mon[98615]: osdmap e364: 8 total, 7 up, 8 in 2024-09-21T09:36:26.127 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:26 smithi022 ceph-mon[96736]: pgmap v148: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 157 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4.8 MiB/s rd, 3.1 MiB/s wr, 11 op/s 2024-09-21T09:36:26.127 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:26 smithi022 ceph-mon[96736]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T09:36:26.127 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:26 smithi022 ceph-mon[96736]: osdmap e364: 8 total, 7 up, 8 in 2024-09-21T09:36:26.127 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 21 09:36:25 smithi022 podman[107950]: 2024-09-21 09:36:25.932120116 +0000 UTC m=+1.794675732 container cleanup 740462b5c304f749f4685a5816a42391bb2a99768c05ca153fb2b4092627e900 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-1, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=quincy-fbd1a6a, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2) 2024-09-21T09:36:26.127 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 21 09:36:25 smithi022 bash[107950]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-1 2024-09-21T09:36:26.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:26 smithi064 ceph-mon[86709]: pgmap v148: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 157 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4.8 MiB/s rd, 3.1 MiB/s wr, 11 op/s 2024-09-21T09:36:26.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:26 smithi064 ceph-mon[86709]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T09:36:26.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:26 smithi064 ceph-mon[86709]: osdmap e364: 8 total, 7 up, 8 in 2024-09-21T09:36:26.552 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 21 09:36:26 smithi022 podman[107983]: 2024-09-21 09:36:26.125071978 +0000 UTC m=+0.565946337 container remove 740462b5c304f749f4685a5816a42391bb2a99768c05ca153fb2b4092627e900 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-1, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=quincy-fbd1a6a, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2) 2024-09-21T09:36:26.808 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 21 09:36:26 smithi022 podman[108076]: 2024-09-21 09:36:26.551161207 +0000 UTC m=+0.109651061 container create a1b792b6ea9ef7466f44af743ed522f3ede89145f62be42c22ecd89e77bbc55b (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-1-deactivate, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux , RELEASE=main-62da54b, org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T09:36:26.808 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 21 09:36:26 smithi022 podman[108076]: 2024-09-21 09:36:26.458002224 +0000 UTC m=+0.016492038 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:36:26.808 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 21 09:36:26 smithi022 podman[108076]: 2024-09-21 09:36:26.770380835 +0000 UTC m=+0.328870656 container init a1b792b6ea9ef7466f44af743ed522f3ede89145f62be42c22ecd89e77bbc55b (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-1-deactivate, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=main-62da54b, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_CLEAN=True, maintainer=Guillaume Abrioux ) 2024-09-21T09:36:26.808 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 21 09:36:26 smithi022 podman[108076]: 2024-09-21 09:36:26.775138275 +0000 UTC m=+0.333628079 container start a1b792b6ea9ef7466f44af743ed522f3ede89145f62be42c22ecd89e77bbc55b (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-1-deactivate, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, RELEASE=main-62da54b, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=) 2024-09-21T09:36:27.013 INFO:tasks.rados.rados.0.smithi022.stdout:1500: done (6 left) 2024-09-21T09:36:27.013 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 17 current snap is 147 2024-09-21T09:36:27.013 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 50 v 899 (ObjNum 161 snap 0 seq_num 3412041072) dirty exists 2024-09-21T09:36:27.013 INFO:tasks.rados.rados.0.smithi022.stdout:1501: seq_num 491 ranges {0=630784,630784=401408} 2024-09-21T09:36:27.016 INFO:tasks.rados.rados.0.smithi022.stdout:1501: writing smithi02284960-17 from 0 to 630784 tid 1 2024-09-21T09:36:27.018 INFO:tasks.rados.rados.0.smithi022.stdout:1501: writing smithi02284960-17 from 630784 to 1032192 tid 2 2024-09-21T09:36:27.019 INFO:tasks.rados.rados.0.smithi022.stdout:1497: done (6 left) 2024-09-21T09:36:27.019 INFO:tasks.rados.rados.0.smithi022.stdout:1502: delete oid 24 current snap is 147 2024-09-21T09:36:27.019 INFO:tasks.rados.rados.0.smithi022.stdout:1493: finishing write tid 1 to smithi02284960-39 2024-09-21T09:36:27.019 INFO:tasks.rados.rados.0.smithi022.stdout:1493: finishing write tid 2 to smithi02284960-39 2024-09-21T09:36:27.019 INFO:tasks.rados.rados.0.smithi022.stdout:1493: finishing write tid 3 to smithi02284960-39 2024-09-21T09:36:27.019 INFO:tasks.rados.rados.0.smithi022.stdout:1496: finishing write tid 1 to smithi02284960-13 2024-09-21T09:36:27.019 INFO:tasks.rados.rados.0.smithi022.stdout:1496: finishing write tid 2 to smithi02284960-13 2024-09-21T09:36:27.019 INFO:tasks.rados.rados.0.smithi022.stdout:1492: finishing write tid 1 to smithi02284960-5 2024-09-21T09:36:27.019 INFO:tasks.rados.rados.0.smithi022.stdout:1493: finishing write tid 4 to smithi02284960-39 2024-09-21T09:36:27.019 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 39 v 907 (ObjNum 489 snap 146 seq_num 489) dirty exists 2024-09-21T09:36:27.019 INFO:tasks.rados.rados.0.smithi022.stdout:1493: left oid 39 (ObjNum 489 snap 146 seq_num 489) 2024-09-21T09:36:27.019 INFO:tasks.rados.rados.0.smithi022.stdout:1496: finishing write tid 3 to smithi02284960-13 2024-09-21T09:36:27.019 INFO:tasks.rados.rados.0.smithi022.stdout:1492: finishing write tid 2 to smithi02284960-5 2024-09-21T09:36:27.020 INFO:tasks.rados.rados.0.smithi022.stdout:1495: expect (ObjNum 408 snap 128 seq_num 408) 2024-09-21T09:36:27.117 INFO:tasks.rados.rados.0.smithi022.stdout:1492: finishing write tid 3 to smithi02284960-5 2024-09-21T09:36:27.117 INFO:tasks.rados.rados.0.smithi022.stdout:1496: finishing write tid 4 to smithi02284960-13 2024-09-21T09:36:27.117 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 13 v 672 (ObjNum 490 snap 146 seq_num 490) dirty exists 2024-09-21T09:36:27.117 INFO:tasks.rados.rados.0.smithi022.stdout:1496: left oid 13 (ObjNum 490 snap 146 seq_num 490) 2024-09-21T09:36:27.117 INFO:tasks.rados.rados.0.smithi022.stdout:1492: finishing write tid 4 to smithi02284960-5 2024-09-21T09:36:27.117 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 5 v 562 (ObjNum 488 snap 146 seq_num 488) dirty exists 2024-09-21T09:36:27.117 INFO:tasks.rados.rados.0.smithi022.stdout:1492: left oid 5 (ObjNum 488 snap 146 seq_num 488) 2024-09-21T09:36:27.118 INFO:tasks.rados.rados.0.smithi022.stdout:1494: finishing copy_from racing read to smithi02284960-18 2024-09-21T09:36:27.118 INFO:tasks.rados.rados.0.smithi022.stdout:1494: finishing copy_from to smithi02284960-18 2024-09-21T09:36:27.118 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 18 v 947 (ObjNum 482 snap 146 seq_num 482) dirty exists 2024-09-21T09:36:27.118 INFO:tasks.rados.rados.0.smithi022.stdout:1501: finishing write tid 1 to smithi02284960-17 2024-09-21T09:36:27.118 INFO:tasks.rados.rados.0.smithi022.stdout:1501: finishing write tid 2 to smithi02284960-17 2024-09-21T09:36:27.118 INFO:tasks.rados.rados.0.smithi022.stdout:1492: done (6 left) 2024-09-21T09:36:27.118 INFO:tasks.rados.rados.0.smithi022.stdout:1493: done (5 left) 2024-09-21T09:36:27.118 INFO:tasks.rados.rados.0.smithi022.stdout:1494: done (4 left) 2024-09-21T09:36:27.118 INFO:tasks.rados.rados.0.smithi022.stdout:1495: done (3 left) 2024-09-21T09:36:27.118 INFO:tasks.rados.rados.0.smithi022.stdout:1496: done (2 left) 2024-09-21T09:36:27.118 INFO:tasks.rados.rados.0.smithi022.stdout:1502: done (1 left) 2024-09-21T09:36:27.118 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 19 current snap is 147 2024-09-21T09:36:27.119 INFO:tasks.rados.rados.0.smithi022.stdout:1501: finishing write tid 3 to smithi02284960-17 2024-09-21T09:36:27.119 INFO:tasks.rados.rados.0.smithi022.stdout:1503: seq_num 493 ranges {0=663552,663552=401408} 2024-09-21T09:36:27.120 INFO:tasks.rados.rados.0.smithi022.stdout:1503: writing smithi02284960-19 from 0 to 663552 tid 1 2024-09-21T09:36:27.123 INFO:tasks.rados.rados.0.smithi022.stdout:1503: writing smithi02284960-19 from 663552 to 1064960 tid 2 2024-09-21T09:36:27.123 INFO:tasks.rados.rados.0.smithi022.stdout:1504: delete oid 1 current snap is 147 2024-09-21T09:36:27.123 INFO:tasks.rados.rados.0.smithi022.stdout:1501: finishing write tid 4 to smithi02284960-17 2024-09-21T09:36:27.123 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 17 v 910 (ObjNum 491 snap 147 seq_num 491) dirty exists 2024-09-21T09:36:27.123 INFO:tasks.rados.rados.0.smithi022.stdout:1501: left oid 17 (ObjNum 491 snap 147 seq_num 491) 2024-09-21T09:36:27.125 INFO:tasks.rados.rados.0.smithi022.stdout:1501: done (2 left) 2024-09-21T09:36:27.125 INFO:tasks.rados.rados.0.smithi022.stdout:1504: done (1 left) 2024-09-21T09:36:27.125 INFO:tasks.rados.rados.0.smithi022.stdout:1505: copy_from oid 4 from oid 1 current snap is 147 2024-09-21T09:36:27.125 INFO:tasks.rados.rados.0.smithi022.stdout:1506: snap_create 2024-09-21T09:36:27.126 INFO:tasks.rados.rados.0.smithi022.stdout:1505: finishing copy_from to smithi02284960-4 2024-09-21T09:36:27.126 INFO:tasks.rados.rados.0.smithi022.stdout:1505: got expected ENOENT (src dne) 2024-09-21T09:36:27.126 INFO:tasks.rados.rados.0.smithi022.stdout:1505: finishing copy_from racing read to smithi02284960-4 2024-09-21T09:36:27.127 INFO:tasks.rados.rados.0.smithi022.stdout:1503: finishing write tid 1 to smithi02284960-19 2024-09-21T09:36:27.127 INFO:tasks.rados.rados.0.smithi022.stdout:1503: finishing write tid 2 to smithi02284960-19 2024-09-21T09:36:27.127 INFO:tasks.rados.rados.0.smithi022.stdout:1503: finishing write tid 3 to smithi02284960-19 2024-09-21T09:36:27.128 INFO:tasks.rados.rados.0.smithi022.stdout:1503: finishing write tid 4 to smithi02284960-19 2024-09-21T09:36:27.128 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 19 v 1022 (ObjNum 493 snap 147 seq_num 493) dirty exists 2024-09-21T09:36:27.128 INFO:tasks.rados.rados.0.smithi022.stdout:1503: left oid 19 (ObjNum 493 snap 147 seq_num 493) 2024-09-21T09:36:27.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:27 smithi022 ceph-mon[96736]: osdmap e365: 8 total, 7 up, 8 in 2024-09-21T09:36:27.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:27 smithi022 ceph-mon[98615]: osdmap e365: 8 total, 7 up, 8 in 2024-09-21T09:36:27.308 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 21 09:36:26 smithi022 podman[108076]: 2024-09-21 09:36:26.861605557 +0000 UTC m=+0.420095379 container attach a1b792b6ea9ef7466f44af743ed522f3ede89145f62be42c22ecd89e77bbc55b (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-1-deactivate, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=main-62da54b, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731) 2024-09-21T09:36:27.308 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 21 09:36:26 smithi022 podman[108076]: 2024-09-21 09:36:26.966935487 +0000 UTC m=+0.525425298 container died a1b792b6ea9ef7466f44af743ed522f3ede89145f62be42c22ecd89e77bbc55b (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-1-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, RELEASE=main-62da54b, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, ceph=True) 2024-09-21T09:36:27.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:27 smithi064 ceph-mon[86709]: osdmap e365: 8 total, 7 up, 8 in 2024-09-21T09:36:27.441 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:36:26 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:36:26.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi022\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:36:27.567 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 21 09:36:27 smithi022 podman[108119]: 2024-09-21 09:36:27.549051513 +0000 UTC m=+0.576459412 container remove a1b792b6ea9ef7466f44af743ed522f3ede89145f62be42c22ecd89e77bbc55b (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-1-deactivate, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, org.label-schema.schema-version=1.0, RELEASE=main-62da54b, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, ceph=True, GIT_BRANCH=HEAD) 2024-09-21T09:36:27.568 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 21 09:36:27 smithi022 systemd[1]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490@osd.1.service: Deactivated successfully. 2024-09-21T09:36:27.568 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 21 09:36:27 smithi022 systemd[1]: Stopped Ceph osd.1 for d65e801e-77fa-11ef-baf6-efdc52797490. 2024-09-21T09:36:27.993 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 21 09:36:27 smithi022 systemd[1]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490@osd.1.service: Consumed 13.201s CPU time. 2024-09-21T09:36:27.993 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 21 09:36:27 smithi022 systemd[1]: Starting Ceph osd.1 for d65e801e-77fa-11ef-baf6-efdc52797490... 2024-09-21T09:36:28.017 INFO:tasks.rados.rados.0.smithi022.stdout:1503: done (2 left) 2024-09-21T09:36:28.017 INFO:tasks.rados.rados.0.smithi022.stdout:1505: done (1 left) 2024-09-21T09:36:28.017 INFO:tasks.rados.rados.0.smithi022.stdout:1506: done (0 left) 2024-09-21T09:36:28.018 INFO:tasks.rados.rados.0.smithi022.stdout:1507: setattr oid 25 current snap is 148 2024-09-21T09:36:28.019 INFO:tasks.rados.rados.0.smithi022.stdout:1508: setattr oid 47 current snap is 148 2024-09-21T09:36:28.021 INFO:tasks.rados.rados.0.smithi022.stdout:1509: snap_create 2024-09-21T09:36:28.023 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 25 v 887 (ObjNum 428 snap 133 seq_num 428) dirty exists 2024-09-21T09:36:28.027 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 47 v 939 (ObjNum 382 snap 119 seq_num 382) dirty exists 2024-09-21T09:36:28.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:28 smithi022 ceph-mon[98615]: pgmap v151: 121 pgs: 16 stale+active+clean, 2 active+clean+snaptrim, 103 active+clean; 157 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4.8 MiB/s rd, 3.1 MiB/s wr, 11 op/s 2024-09-21T09:36:28.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:28 smithi022 ceph-mon[98615]: osdmap e366: 8 total, 7 up, 8 in 2024-09-21T09:36:28.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:28 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:36:28.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:28 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:36:28.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:28 smithi022 ceph-mon[96736]: pgmap v151: 121 pgs: 16 stale+active+clean, 2 active+clean+snaptrim, 103 active+clean; 157 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4.8 MiB/s rd, 3.1 MiB/s wr, 11 op/s 2024-09-21T09:36:28.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:28 smithi022 ceph-mon[96736]: osdmap e366: 8 total, 7 up, 8 in 2024-09-21T09:36:28.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:28 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:36:28.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:28 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:36:28.309 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 21 09:36:27 smithi022 podman[108221]: 2024-09-21 09:36:27.9917858 +0000 UTC m=+0.108836008 container create 04e9e42815b1c7c4ccfbdf65d921b39473a679b2b6de00d5e724b4630c635600 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-1-activate, ceph=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=main-62da54b, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True) 2024-09-21T09:36:28.309 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 21 09:36:27 smithi022 podman[108221]: 2024-09-21 09:36:27.899751599 +0000 UTC m=+0.016801820 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:36:28.309 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 21 09:36:28 smithi022 podman[108221]: 2024-09-21 09:36:28.219952019 +0000 UTC m=+0.337002245 container init 04e9e42815b1c7c4ccfbdf65d921b39473a679b2b6de00d5e724b4630c635600 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-1-activate, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, RELEASE=main-62da54b, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731) 2024-09-21T09:36:28.309 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 21 09:36:28 smithi022 podman[108221]: 2024-09-21 09:36:28.225577009 +0000 UTC m=+0.342627213 container start 04e9e42815b1c7c4ccfbdf65d921b39473a679b2b6de00d5e724b4630c635600 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-1-activate, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, RELEASE=main-62da54b, org.label-schema.build-date=20240731, ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD) 2024-09-21T09:36:28.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:28 smithi064 ceph-mon[86709]: pgmap v151: 121 pgs: 16 stale+active+clean, 2 active+clean+snaptrim, 103 active+clean; 157 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4.8 MiB/s rd, 3.1 MiB/s wr, 11 op/s 2024-09-21T09:36:28.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:28 smithi064 ceph-mon[86709]: osdmap e366: 8 total, 7 up, 8 in 2024-09-21T09:36:28.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:28 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:36:28.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:28 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:36:28.808 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 21 09:36:28 smithi022 podman[108221]: 2024-09-21 09:36:28.336093528 +0000 UTC m=+0.453143738 container attach 04e9e42815b1c7c4ccfbdf65d921b39473a679b2b6de00d5e724b4630c635600 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-1-activate, RELEASE=main-62da54b, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=) 2024-09-21T09:36:28.808 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 21 09:36:28 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-1-activate[108236]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T09:36:28.808 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 21 09:36:28 smithi022 bash[108221]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T09:36:28.808 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 21 09:36:28 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-1-activate[108236]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T09:36:28.808 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 21 09:36:28 smithi022 bash[108221]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T09:36:29.024 INFO:tasks.rados.rados.0.smithi022.stdout:1507: done (2 left) 2024-09-21T09:36:29.024 INFO:tasks.rados.rados.0.smithi022.stdout:1508: done (1 left) 2024-09-21T09:36:29.024 INFO:tasks.rados.rados.0.smithi022.stdout:1509: done (0 left) 2024-09-21T09:36:29.024 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 7 current snap is 149 2024-09-21T09:36:29.024 INFO:tasks.rados.rados.0.smithi022.stdout:1510: seq_num 495 ranges {0=696320,696320=401408} 2024-09-21T09:36:29.027 INFO:tasks.rados.rados.0.smithi022.stdout:1510: writing smithi02284960-7 from 0 to 696320 tid 1 2024-09-21T09:36:29.030 INFO:tasks.rados.rados.0.smithi022.stdout:1510: writing smithi02284960-7 from 696320 to 1097728 tid 2 2024-09-21T09:36:29.030 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 32 current snap is 149 2024-09-21T09:36:29.030 INFO:tasks.rados.rados.0.smithi022.stdout:1511: seq_num 496 ranges {1605632=712704,2318336=401408} 2024-09-21T09:36:29.042 INFO:tasks.rados.rados.0.smithi022.stdout:1511: writing smithi02284960-32 from 1605632 to 2318336 tid 1 2024-09-21T09:36:29.045 INFO:tasks.rados.rados.0.smithi022.stdout:1511: writing smithi02284960-32 from 2318336 to 2719744 tid 2 2024-09-21T09:36:29.045 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 34 current snap is 149 2024-09-21T09:36:29.045 INFO:tasks.rados.rados.0.smithi022.stdout:1510: finishing write tid 1 to smithi02284960-7 2024-09-21T09:36:29.045 INFO:tasks.rados.rados.0.smithi022.stdout:1510: finishing write tid 2 to smithi02284960-7 2024-09-21T09:36:29.045 INFO:tasks.rados.rados.0.smithi022.stdout:1510: finishing write tid 3 to smithi02284960-7 2024-09-21T09:36:29.045 INFO:tasks.rados.rados.0.smithi022.stdout:1510: finishing write tid 4 to smithi02284960-7 2024-09-21T09:36:29.046 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 7 v 958 (ObjNum 495 snap 149 seq_num 495) dirty exists 2024-09-21T09:36:29.046 INFO:tasks.rados.rados.0.smithi022.stdout:1510: left oid 7 (ObjNum 495 snap 149 seq_num 495) 2024-09-21T09:36:29.046 INFO:tasks.rados.rados.0.smithi022.stdout:1512: seq_num 497 ranges {1392640=729088,2121728=401408} 2024-09-21T09:36:29.056 INFO:tasks.rados.rados.0.smithi022.stdout:1512: writing smithi02284960-34 from 1392640 to 2121728 tid 1 2024-09-21T09:36:29.059 INFO:tasks.rados.rados.0.smithi022.stdout:1512: writing smithi02284960-34 from 2121728 to 2523136 tid 2 2024-09-21T09:36:29.059 INFO:tasks.rados.rados.0.smithi022.stdout:1510: done (2 left) 2024-09-21T09:36:29.059 INFO:tasks.rados.rados.0.smithi022.stdout:1513: snap_create 2024-09-21T09:36:29.059 INFO:tasks.rados.rados.0.smithi022.stdout:1511: finishing write tid 1 to smithi02284960-32 2024-09-21T09:36:29.059 INFO:tasks.rados.rados.0.smithi022.stdout:1511: finishing write tid 2 to smithi02284960-32 2024-09-21T09:36:29.059 INFO:tasks.rados.rados.0.smithi022.stdout:1511: finishing write tid 3 to smithi02284960-32 2024-09-21T09:36:29.059 INFO:tasks.rados.rados.0.smithi022.stdout:1511: finishing write tid 4 to smithi02284960-32 2024-09-21T09:36:29.059 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 32 v 962 (ObjNum 496 snap 149 seq_num 496) dirty exists 2024-09-21T09:36:29.059 INFO:tasks.rados.rados.0.smithi022.stdout:1511: left oid 32 (ObjNum 496 snap 149 seq_num 496) 2024-09-21T09:36:29.062 INFO:tasks.rados.rados.0.smithi022.stdout:1512: finishing write tid 1 to smithi02284960-34 2024-09-21T09:36:29.062 INFO:tasks.rados.rados.0.smithi022.stdout:1512: finishing write tid 2 to smithi02284960-34 2024-09-21T09:36:29.062 INFO:tasks.rados.rados.0.smithi022.stdout:1512: finishing write tid 3 to smithi02284960-34 2024-09-21T09:36:29.063 INFO:tasks.rados.rados.0.smithi022.stdout:1512: finishing write tid 4 to smithi02284960-34 2024-09-21T09:36:29.064 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 34 v 970 (ObjNum 497 snap 149 seq_num 497) dirty exists 2024-09-21T09:36:29.064 INFO:tasks.rados.rados.0.smithi022.stdout:1512: left oid 34 (ObjNum 497 snap 149 seq_num 497) 2024-09-21T09:36:29.132 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:29 smithi022 ceph-mon[98615]: osdmap e367: 8 total, 7 up, 8 in 2024-09-21T09:36:29.133 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:29 smithi022 ceph-mon[96736]: osdmap e367: 8 total, 7 up, 8 in 2024-09-21T09:36:29.133 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 21 09:36:28 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-1-activate[108236]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-21T09:36:29.133 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 21 09:36:28 smithi022 bash[108221]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-21T09:36:29.133 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 21 09:36:28 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-1-activate[108236]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-21T09:36:29.133 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 21 09:36:28 smithi022 bash[108221]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-21T09:36:29.133 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 21 09:36:29 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-1-activate[108236]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-21T09:36:29.133 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 21 09:36:29 smithi022 bash[108221]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-21T09:36:29.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:29 smithi064 ceph-mon[86709]: osdmap e367: 8 total, 7 up, 8 in 2024-09-21T09:36:29.483 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 21 09:36:29 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-1-activate[108236]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-21T09:36:29.484 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 21 09:36:29 smithi022 bash[108221]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-21T09:36:29.484 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 21 09:36:29 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-1-activate[108236]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-09-21T09:36:29.484 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 21 09:36:29 smithi022 bash[108221]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-09-21T09:36:29.484 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 21 09:36:29 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-1-activate[108236]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-21T09:36:29.484 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 21 09:36:29 smithi022 bash[108221]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-21T09:36:29.484 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 21 09:36:29 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-1-activate[108236]: --> ceph-volume raw activate successful for osd ID: 1 2024-09-21T09:36:29.484 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 21 09:36:29 smithi022 bash[108221]: --> ceph-volume raw activate successful for osd ID: 1 2024-09-21T09:36:29.484 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 21 09:36:29 smithi022 podman[108221]: 2024-09-21 09:36:29.159749987 +0000 UTC m=+1.276800194 container died 04e9e42815b1c7c4ccfbdf65d921b39473a679b2b6de00d5e724b4630c635600 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-1-activate, ceph=True, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, RELEASE=main-62da54b) 2024-09-21T09:36:30.028 INFO:tasks.rados.rados.0.smithi022.stdout:1511: done (2 left) 2024-09-21T09:36:30.028 INFO:tasks.rados.rados.0.smithi022.stdout:1512: done (1 left) 2024-09-21T09:36:30.028 INFO:tasks.rados.rados.0.smithi022.stdout:1513: done (0 left) 2024-09-21T09:36:30.028 INFO:tasks.rados.rados.0.smithi022.stdout:1514: delete oid 40 current snap is 150 2024-09-21T09:36:30.030 INFO:tasks.rados.rados.0.smithi022.stdout:1514: done (0 left) 2024-09-21T09:36:30.030 INFO:tasks.rados.rados.0.smithi022.stdout:1515: snap_remove snap 147 2024-09-21T09:36:30.248 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:30 smithi022 ceph-mon[98615]: pgmap v154: 121 pgs: 33 active+undersized, 12 active+undersized+degraded, 76 active+clean; 165 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 3.2 MiB/s rd, 2.7 MiB/s wr, 27 op/s; 63/423 objects degraded (14.894%) 2024-09-21T09:36:30.249 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:30 smithi022 ceph-mon[98615]: Health check failed: Degraded data redundancy: 63/423 objects degraded (14.894%), 12 pgs degraded (PG_DEGRADED) 2024-09-21T09:36:30.249 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:30 smithi022 ceph-mon[98615]: osdmap e368: 8 total, 7 up, 8 in 2024-09-21T09:36:30.249 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:30 smithi022 ceph-mon[96736]: pgmap v154: 121 pgs: 33 active+undersized, 12 active+undersized+degraded, 76 active+clean; 165 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 3.2 MiB/s rd, 2.7 MiB/s wr, 27 op/s; 63/423 objects degraded (14.894%) 2024-09-21T09:36:30.249 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:30 smithi022 ceph-mon[96736]: Health check failed: Degraded data redundancy: 63/423 objects degraded (14.894%), 12 pgs degraded (PG_DEGRADED) 2024-09-21T09:36:30.249 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:30 smithi022 ceph-mon[96736]: osdmap e368: 8 total, 7 up, 8 in 2024-09-21T09:36:30.249 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 21 09:36:29 smithi022 podman[108381]: 2024-09-21 09:36:29.890271943 +0000 UTC m=+0.724289816 container remove 04e9e42815b1c7c4ccfbdf65d921b39473a679b2b6de00d5e724b4630c635600 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-1-activate, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=main-62da54b, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T09:36:30.249 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 21 09:36:30 smithi022 podman[108445]: 2024-09-21 09:36:30.00497974 +0000 UTC m=+0.016980614 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:36:30.249 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 21 09:36:30 smithi022 podman[108445]: 2024-09-21 09:36:30.12466916 +0000 UTC m=+0.136670056 container create 7cd0c18d224c55427944767fc60453ec5d64c875b6a7ca6defa3978dd350bd30 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-1, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-62da54b, org.label-schema.schema-version=1.0) 2024-09-21T09:36:30.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:30 smithi064 ceph-mon[86709]: pgmap v154: 121 pgs: 33 active+undersized, 12 active+undersized+degraded, 76 active+clean; 165 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 3.2 MiB/s rd, 2.7 MiB/s wr, 27 op/s; 63/423 objects degraded (14.894%) 2024-09-21T09:36:30.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:30 smithi064 ceph-mon[86709]: Health check failed: Degraded data redundancy: 63/423 objects degraded (14.894%), 12 pgs degraded (PG_DEGRADED) 2024-09-21T09:36:30.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:30 smithi064 ceph-mon[86709]: osdmap e368: 8 total, 7 up, 8 in 2024-09-21T09:36:30.442 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:36:30 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:36:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:36:30.808 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 21 09:36:30 smithi022 podman[108445]: 2024-09-21 09:36:30.360359673 +0000 UTC m=+0.372360542 container init 7cd0c18d224c55427944767fc60453ec5d64c875b6a7ca6defa3978dd350bd30 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-1, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux , ceph=True, CEPH_POINT_RELEASE=, RELEASE=main-62da54b) 2024-09-21T09:36:30.808 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 21 09:36:30 smithi022 podman[108445]: 2024-09-21 09:36:30.365563743 +0000 UTC m=+0.377564600 container start 7cd0c18d224c55427944767fc60453ec5d64c875b6a7ca6defa3978dd350bd30 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-1, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , RELEASE=main-62da54b, io.buildah.version=1.37.2) 2024-09-21T09:36:30.808 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 21 09:36:30 smithi022 ceph-osd[108482]: -- 172.21.15.22:0/2449647109 <== mon.0 v2:172.21.15.22:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55fc347fda40 con 0x55fc355cc400 2024-09-21T09:36:30.808 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 21 09:36:30 smithi022 bash[108445]: 7cd0c18d224c55427944767fc60453ec5d64c875b6a7ca6defa3978dd350bd30 2024-09-21T09:36:30.808 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 21 09:36:30 smithi022 systemd[1]: Started Ceph osd.1 for d65e801e-77fa-11ef-baf6-efdc52797490. 2024-09-21T09:36:31.035 INFO:tasks.rados.rados.0.smithi022.stdout:1515: done (0 left) 2024-09-21T09:36:31.035 INFO:tasks.rados.rados.0.smithi022.stdout:1516: snap_remove snap 148 2024-09-21T09:36:31.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:31 smithi022 ceph-mon[96736]: osdmap e369: 8 total, 7 up, 8 in 2024-09-21T09:36:31.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:31 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:36:31.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:31 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:36:31.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:31 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:36:31.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:31 smithi022 ceph-mon[98615]: osdmap e369: 8 total, 7 up, 8 in 2024-09-21T09:36:31.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:31 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:36:31.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:31 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:36:31.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:31 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:36:31.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:31 smithi064 ceph-mon[86709]: osdmap e369: 8 total, 7 up, 8 in 2024-09-21T09:36:31.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:31 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:36:31.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:31 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:36:31.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:31 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:36:32.038 INFO:tasks.rados.rados.0.smithi022.stdout:1516: done (0 left) 2024-09-21T09:36:32.038 INFO:tasks.rados.rados.0.smithi022.stdout:1517: snap_remove snap 128 2024-09-21T09:36:32.038 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 21 09:36:31 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-1[108459]: 2024-09-21T09:36:31.756+0000 7f8cd63cf640 -1 Falling back to public interface 2024-09-21T09:36:32.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:32 smithi022 ceph-mon[98615]: pgmap v157: 121 pgs: 33 active+undersized, 12 active+undersized+degraded, 76 active+clean; 165 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 3.2 MiB/s rd, 2.7 MiB/s wr, 27 op/s; 63/423 objects degraded (14.894%) 2024-09-21T09:36:32.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:32 smithi022 ceph-mon[98615]: osdmap e370: 8 total, 7 up, 8 in 2024-09-21T09:36:32.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:32 smithi022 ceph-mon[96736]: pgmap v157: 121 pgs: 33 active+undersized, 12 active+undersized+degraded, 76 active+clean; 165 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 3.2 MiB/s rd, 2.7 MiB/s wr, 27 op/s; 63/423 objects degraded (14.894%) 2024-09-21T09:36:32.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:32 smithi022 ceph-mon[96736]: osdmap e370: 8 total, 7 up, 8 in 2024-09-21T09:36:32.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:32 smithi064 ceph-mon[86709]: pgmap v157: 121 pgs: 33 active+undersized, 12 active+undersized+degraded, 76 active+clean; 165 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 3.2 MiB/s rd, 2.7 MiB/s wr, 27 op/s; 63/423 objects degraded (14.894%) 2024-09-21T09:36:32.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:32 smithi064 ceph-mon[86709]: osdmap e370: 8 total, 7 up, 8 in 2024-09-21T09:36:33.045 INFO:tasks.rados.rados.0.smithi022.stdout:1517: done (0 left) 2024-09-21T09:36:33.045 INFO:tasks.rados.rados.0.smithi022.stdout:1518: delete oid 34 current snap is 150 2024-09-21T09:36:33.049 INFO:tasks.rados.rados.0.smithi022.stdout:1518: done (0 left) 2024-09-21T09:36:33.049 INFO:tasks.rados.rados.0.smithi022.stdout:1519: read oid 33 snap -1 2024-09-21T09:36:33.049 INFO:tasks.rados.rados.0.smithi022.stdout:1519: expect deleted 2024-09-21T09:36:33.049 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 48 current snap is 150 2024-09-21T09:36:33.049 INFO:tasks.rados.rados.0.smithi022.stdout:1520: seq_num 500 ranges {876544=778240,1654784=401408} 2024-09-21T09:36:33.058 INFO:tasks.rados.rados.0.smithi022.stdout:1520: writing smithi02284960-48 from 876544 to 1654784 tid 1 2024-09-21T09:36:33.061 INFO:tasks.rados.rados.0.smithi022.stdout:1520: writing smithi02284960-48 from 1654784 to 2056192 tid 2 2024-09-21T09:36:33.061 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 5 current snap is 150 2024-09-21T09:36:33.061 INFO:tasks.rados.rados.0.smithi022.stdout:1521: seq_num 501 ranges {1777664=794624,2572288=401408} 2024-09-21T09:36:33.075 INFO:tasks.rados.rados.0.smithi022.stdout:1521: writing smithi02284960-5 from 1777664 to 2572288 tid 1 2024-09-21T09:36:33.078 INFO:tasks.rados.rados.0.smithi022.stdout:1521: writing smithi02284960-5 from 2572288 to 2973696 tid 2 2024-09-21T09:36:33.078 INFO:tasks.rados.rados.0.smithi022.stdout:1519: done (2 left) 2024-09-21T09:36:33.078 INFO:tasks.rados.rados.0.smithi022.stdout:1522: read oid 8 snap -1 2024-09-21T09:36:33.078 INFO:tasks.rados.rados.0.smithi022.stdout:1522: expect (ObjNum 437 snap 134 seq_num 437) 2024-09-21T09:36:33.078 INFO:tasks.rados.rados.0.smithi022.stdout:1523: delete oid 40 current snap is 150 2024-09-21T09:36:33.078 INFO:tasks.rados.rados.0.smithi022.stdout:1520: finishing write tid 1 to smithi02284960-48 2024-09-21T09:36:33.079 INFO:tasks.rados.rados.0.smithi022.stdout:1520: finishing write tid 2 to smithi02284960-48 2024-09-21T09:36:33.079 INFO:tasks.rados.rados.0.smithi022.stdout:1520: finishing write tid 3 to smithi02284960-48 2024-09-21T09:36:33.079 INFO:tasks.rados.rados.0.smithi022.stdout:1520: finishing write tid 4 to smithi02284960-48 2024-09-21T09:36:33.079 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 48 v 966 (ObjNum 500 snap 150 seq_num 500) dirty exists 2024-09-21T09:36:33.079 INFO:tasks.rados.rados.0.smithi022.stdout:1520: left oid 48 (ObjNum 500 snap 150 seq_num 500) 2024-09-21T09:36:33.080 INFO:tasks.rados.rados.0.smithi022.stdout:1520: done (3 left) 2024-09-21T09:36:33.080 INFO:tasks.rados.rados.0.smithi022.stdout:1523: done (2 left) 2024-09-21T09:36:33.080 INFO:tasks.rados.rados.0.smithi022.stdout:1524: rollback oid 46 current snap is 150 2024-09-21T09:36:33.080 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 46 to 134 2024-09-21T09:36:33.080 INFO:tasks.rados.rados.0.smithi022.stdout:1525: read oid 39 snap 149 2024-09-21T09:36:33.080 INFO:tasks.rados.rados.0.smithi022.stdout:1525: expect (ObjNum 489 snap 146 seq_num 489) 2024-09-21T09:36:33.080 INFO:tasks.rados.rados.0.smithi022.stdout:1526: rmattr oid 35 current snap is 150 2024-09-21T09:36:33.080 INFO:tasks.rados.rados.0.smithi022.stdout:1526: done (4 left) 2024-09-21T09:36:33.080 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 1 current snap is 150 2024-09-21T09:36:33.080 INFO:tasks.rados.rados.0.smithi022.stdout:1527: seq_num 503 ranges {0=425984,425984=704512,1130496=98304} 2024-09-21T09:36:33.083 INFO:tasks.rados.rados.0.smithi022.stdout:1527: writing smithi02284960-1 from 0 to 425984 tid 1 2024-09-21T09:36:33.087 INFO:tasks.rados.rados.0.smithi022.stdout:1527: writing smithi02284960-1 from 425984 to 1130496 tid 2 2024-09-21T09:36:33.088 INFO:tasks.rados.rados.0.smithi022.stdout:1527: writing smithi02284960-1 from 1130496 to 1228800 tid 3 2024-09-21T09:36:33.088 INFO:tasks.rados.rados.0.smithi022.stdout:1521: finishing write tid 1 to smithi02284960-5 2024-09-21T09:36:33.088 INFO:tasks.rados.rados.0.smithi022.stdout:1521: finishing write tid 2 to smithi02284960-5 2024-09-21T09:36:33.088 INFO:tasks.rados.rados.0.smithi022.stdout:1521: finishing write tid 3 to smithi02284960-5 2024-09-21T09:36:33.088 INFO:tasks.rados.rados.0.smithi022.stdout:1528: snap_remove snap 143 2024-09-21T09:36:33.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:33 smithi022 ceph-mon[96736]: osdmap e371: 8 total, 7 up, 8 in 2024-09-21T09:36:33.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:33 smithi022 ceph-mon[98615]: osdmap e371: 8 total, 7 up, 8 in 2024-09-21T09:36:33.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:33 smithi064 ceph-mon[86709]: osdmap e371: 8 total, 7 up, 8 in 2024-09-21T09:36:33.959 INFO:tasks.rados.rados.0.smithi022.stdout:1528: done (5 left) 2024-09-21T09:36:33.959 INFO:tasks.rados.rados.0.smithi022.stdout:1529: read oid 18 snap -1 2024-09-21T09:36:33.960 INFO:tasks.rados.rados.0.smithi022.stdout:1529: expect (ObjNum 482 snap 146 seq_num 482) 2024-09-21T09:36:33.960 INFO:tasks.rados.rados.0.smithi022.stdout:1530: snap_create 2024-09-21T09:36:33.960 INFO:tasks.rados.rados.0.smithi022.stdout:1521: finishing write tid 4 to smithi02284960-5 2024-09-21T09:36:33.960 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 5 v 566 (ObjNum 501 snap 150 seq_num 501) dirty exists 2024-09-21T09:36:33.960 INFO:tasks.rados.rados.0.smithi022.stdout:1521: left oid 5 (ObjNum 501 snap 150 seq_num 501) 2024-09-21T09:36:33.960 INFO:tasks.rados.rados.0.smithi022.stdout:1524: finishing rollback tid 0 to smithi02284960-46 2024-09-21T09:36:33.960 INFO:tasks.rados.rados.0.smithi022.stdout:1524: finishing rollback tid 1 to smithi02284960-46 2024-09-21T09:36:33.960 INFO:tasks.rados.rados.0.smithi022.stdout:1524: finishing rollback tid 2 to smithi02284960-46 2024-09-21T09:36:33.960 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 46 v 953 (ObjNum 365 snap 115 seq_num 365) dirty exists 2024-09-21T09:36:33.960 INFO:tasks.rados.rados.0.smithi022.stdout:1522: expect (ObjNum 437 snap 134 seq_num 437) 2024-09-21T09:36:34.069 INFO:tasks.rados.rados.0.smithi022.stdout:1527: finishing write tid 1 to smithi02284960-1 2024-09-21T09:36:34.069 INFO:tasks.rados.rados.0.smithi022.stdout:1525: expect (ObjNum 489 snap 146 seq_num 489) 2024-09-21T09:36:34.134 INFO:tasks.rados.rados.0.smithi022.stdout:1527: finishing write tid 2 to smithi02284960-1 2024-09-21T09:36:34.134 INFO:tasks.rados.rados.0.smithi022.stdout:1527: finishing write tid 3 to smithi02284960-1 2024-09-21T09:36:34.134 INFO:tasks.rados.rados.0.smithi022.stdout:1527: finishing write tid 4 to smithi02284960-1 2024-09-21T09:36:34.134 INFO:tasks.rados.rados.0.smithi022.stdout:1527: finishing write tid 5 to smithi02284960-1 2024-09-21T09:36:34.135 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 1 v 957 (ObjNum 503 snap 150 seq_num 503) dirty exists 2024-09-21T09:36:34.135 INFO:tasks.rados.rados.0.smithi022.stdout:1527: left oid 1 (ObjNum 503 snap 150 seq_num 503) 2024-09-21T09:36:34.135 INFO:tasks.rados.rados.0.smithi022.stdout:1529: expect (ObjNum 482 snap 146 seq_num 482) 2024-09-21T09:36:34.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:34 smithi022 ceph-mon[98615]: pgmap v160: 121 pgs: 33 active+undersized, 12 active+undersized+degraded, 76 active+clean; 165 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 63/423 objects degraded (14.894%) 2024-09-21T09:36:34.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:34 smithi022 ceph-mon[98615]: osdmap e372: 8 total, 7 up, 8 in 2024-09-21T09:36:34.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:34 smithi022 ceph-mon[98615]: osdmap e373: 8 total, 7 up, 8 in 2024-09-21T09:36:34.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:34 smithi022 ceph-mon[96736]: pgmap v160: 121 pgs: 33 active+undersized, 12 active+undersized+degraded, 76 active+clean; 165 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 63/423 objects degraded (14.894%) 2024-09-21T09:36:34.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:34 smithi022 ceph-mon[96736]: osdmap e372: 8 total, 7 up, 8 in 2024-09-21T09:36:34.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:34 smithi022 ceph-mon[96736]: osdmap e373: 8 total, 7 up, 8 in 2024-09-21T09:36:34.343 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:34 smithi064 ceph-mon[86709]: pgmap v160: 121 pgs: 33 active+undersized, 12 active+undersized+degraded, 76 active+clean; 165 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 63/423 objects degraded (14.894%) 2024-09-21T09:36:34.343 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:34 smithi064 ceph-mon[86709]: osdmap e372: 8 total, 7 up, 8 in 2024-09-21T09:36:34.343 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:34 smithi064 ceph-mon[86709]: osdmap e373: 8 total, 7 up, 8 in 2024-09-21T09:36:34.343 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:36:34 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:36:34.148Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.1\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.1\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.1\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:36:34.961 INFO:tasks.rados.rados.0.smithi022.stdout:1521: done (6 left) 2024-09-21T09:36:34.961 INFO:tasks.rados.rados.0.smithi022.stdout:1522: done (5 left) 2024-09-21T09:36:34.962 INFO:tasks.rados.rados.0.smithi022.stdout:1524: done (4 left) 2024-09-21T09:36:34.962 INFO:tasks.rados.rados.0.smithi022.stdout:1525: done (3 left) 2024-09-21T09:36:34.962 INFO:tasks.rados.rados.0.smithi022.stdout:1527: done (2 left) 2024-09-21T09:36:34.962 INFO:tasks.rados.rados.0.smithi022.stdout:1529: done (1 left) 2024-09-21T09:36:34.962 INFO:tasks.rados.rados.0.smithi022.stdout:1530: done (0 left) 2024-09-21T09:36:34.962 INFO:tasks.rados.rados.0.smithi022.stdout:1531: rollback oid 42 current snap is 151 2024-09-21T09:36:34.963 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 42 to 134 2024-09-21T09:36:34.963 INFO:tasks.rados.rados.0.smithi022.stdout:1532: read oid 22 snap -1 2024-09-21T09:36:34.963 INFO:tasks.rados.rados.0.smithi022.stdout:1532: expect (ObjNum 469 snap 141 seq_num 469) 2024-09-21T09:36:34.963 INFO:tasks.rados.rados.0.smithi022.stdout:1533: snap_remove snap 150 2024-09-21T09:36:35.482 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 21 09:36:35 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-1[108459]: 2024-09-21T09:36:35.211+0000 7f8cd63cf640 -1 osd.1 363 log_to_monitors true 2024-09-21T09:36:35.741 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:35 smithi064 ceph-mon[86709]: pgmap v163: 121 pgs: 33 active+undersized, 12 active+undersized+degraded, 76 active+clean; 160 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 2.9 MiB/s rd, 1.7 MiB/s wr, 9 op/s; 68/405 objects degraded (16.790%) 2024-09-21T09:36:35.742 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:35 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:36:35.742 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:35 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:36:35.742 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:35 smithi064 ceph-mon[86709]: osdmap e374: 8 total, 7 up, 8 in 2024-09-21T09:36:35.742 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:35 smithi064 ceph-mon[86709]: Health check update: Degraded data redundancy: 68/405 objects degraded (16.790%), 12 pgs degraded (PG_DEGRADED) 2024-09-21T09:36:35.742 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:35 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:36:35.742 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:35 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:36:35.742 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:35 smithi064 ceph-mon[86709]: from='osd.1 [v2:172.21.15.22:6810/2947123190,v1:172.21.15.22:6811/2947123190]' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-09-21T09:36:35.742 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:35 smithi064 ceph-mon[86709]: from='osd.1 ' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-09-21T09:36:35.782 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:36:35 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:36:35] "GET /metrics HTTP/1.1" 200 36238 "" "Prometheus/2.51.0" 2024-09-21T09:36:35.782 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:35 smithi022 ceph-mon[96736]: pgmap v163: 121 pgs: 33 active+undersized, 12 active+undersized+degraded, 76 active+clean; 160 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 2.9 MiB/s rd, 1.7 MiB/s wr, 9 op/s; 68/405 objects degraded (16.790%) 2024-09-21T09:36:35.782 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:35 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:36:35.782 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:35 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:36:35.782 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:35 smithi022 ceph-mon[96736]: osdmap e374: 8 total, 7 up, 8 in 2024-09-21T09:36:35.782 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:35 smithi022 ceph-mon[96736]: Health check update: Degraded data redundancy: 68/405 objects degraded (16.790%), 12 pgs degraded (PG_DEGRADED) 2024-09-21T09:36:35.782 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:35 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:36:35.782 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:35 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:36:35.783 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:35 smithi022 ceph-mon[96736]: from='osd.1 [v2:172.21.15.22:6810/2947123190,v1:172.21.15.22:6811/2947123190]' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-09-21T09:36:35.783 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:35 smithi022 ceph-mon[96736]: from='osd.1 ' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-09-21T09:36:35.783 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:35 smithi022 ceph-mon[98615]: pgmap v163: 121 pgs: 33 active+undersized, 12 active+undersized+degraded, 76 active+clean; 160 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 2.9 MiB/s rd, 1.7 MiB/s wr, 9 op/s; 68/405 objects degraded (16.790%) 2024-09-21T09:36:35.783 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:35 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:36:35.783 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:35 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:36:35.783 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:35 smithi022 ceph-mon[98615]: osdmap e374: 8 total, 7 up, 8 in 2024-09-21T09:36:35.783 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:35 smithi022 ceph-mon[98615]: Health check update: Degraded data redundancy: 68/405 objects degraded (16.790%), 12 pgs degraded (PG_DEGRADED) 2024-09-21T09:36:35.783 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:35 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:36:35.783 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:35 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:36:35.783 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:35 smithi022 ceph-mon[98615]: from='osd.1 [v2:172.21.15.22:6810/2947123190,v1:172.21.15.22:6811/2947123190]' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-09-21T09:36:35.784 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:35 smithi022 ceph-mon[98615]: from='osd.1 ' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-09-21T09:36:35.963 INFO:tasks.rados.rados.0.smithi022.stdout:1533: done (2 left) 2024-09-21T09:36:35.963 INFO:tasks.rados.rados.0.smithi022.stdout:1534: setattr oid 33 current snap is 151 2024-09-21T09:36:35.963 INFO:tasks.rados.rados.0.smithi022.stdout:1531: finishing rollback tid 0 to smithi02284960-42 2024-09-21T09:36:35.963 INFO:tasks.rados.rados.0.smithi022.stdout:1531: finishing rollback tid 1 to smithi02284960-42 2024-09-21T09:36:35.963 INFO:tasks.rados.rados.0.smithi022.stdout:1531: finishing rollback tid 2 to smithi02284960-42 2024-09-21T09:36:35.963 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 42 v 676 (ObjNum 430 snap 134 seq_num 430) dirty exists 2024-09-21T09:36:35.963 INFO:tasks.rados.rados.0.smithi022.stdout:1532: expect (ObjNum 469 snap 141 seq_num 469) 2024-09-21T09:36:36.008 INFO:tasks.rados.rados.0.smithi022.stdout:1531: done (2 left) 2024-09-21T09:36:36.008 INFO:tasks.rados.rados.0.smithi022.stdout:1532: done (1 left) 2024-09-21T09:36:36.008 INFO:tasks.rados.rados.0.smithi022.stdout:1535: copy_from oid 12 from oid 18 current snap is 151 2024-09-21T09:36:36.008 INFO:tasks.rados.rados.0.smithi022.stdout:1536: read oid 24 snap -1 2024-09-21T09:36:36.008 INFO:tasks.rados.rados.0.smithi022.stdout:1536: expect deleted 2024-09-21T09:36:36.008 INFO:tasks.rados.rados.0.smithi022.stdout:1537: delete oid 7 current snap is 151 2024-09-21T09:36:36.052 INFO:tasks.rados.rados.0.smithi022.stdout:1535: finishing copy_from racing read to smithi02284960-12 2024-09-21T09:36:36.053 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 33 v 967 (ObjNum 81 snap 0 seq_num 1) dirty exists 2024-09-21T09:36:36.055 INFO:tasks.rados.rados.0.smithi022.stdout:1534: done (3 left) 2024-09-21T09:36:36.055 INFO:tasks.rados.rados.0.smithi022.stdout:1536: done (2 left) 2024-09-21T09:36:36.055 INFO:tasks.rados.rados.0.smithi022.stdout:1537: done (1 left) 2024-09-21T09:36:36.055 INFO:tasks.rados.rados.0.smithi022.stdout:1538: snap_remove snap 134 2024-09-21T09:36:36.191 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:36:35 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:36:35.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:36:36.192 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:36:35 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:36:35.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:36:36.535 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 21 09:36:36 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-1[108459]: 2024-09-21T09:36:36.247+0000 7f8cca171640 -1 osd.1 363 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T09:36:36.972 INFO:tasks.rados.rados.0.smithi022.stdout:1538: done (1 left) 2024-09-21T09:36:36.972 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 15 current snap is 151 2024-09-21T09:36:36.972 INFO:tasks.rados.rados.0.smithi022.stdout:1539: seq_num 505 ranges {1597440=466944,2064384=475136,2539520=327680} 2024-09-21T09:36:36.993 INFO:tasks.rados.rados.0.smithi022.stdout:1539: writing smithi02284960-15 from 1597440 to 2064384 tid 1 2024-09-21T09:36:36.997 INFO:tasks.rados.rados.0.smithi022.stdout:1539: writing smithi02284960-15 from 2064384 to 2539520 tid 2 2024-09-21T09:36:37.000 INFO:tasks.rados.rados.0.smithi022.stdout:1539: writing smithi02284960-15 from 2539520 to 2867200 tid 3 2024-09-21T09:36:37.000 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 29 current snap is 151 2024-09-21T09:36:37.000 INFO:tasks.rados.rados.0.smithi022.stdout:1540: seq_num 506 ranges {73728=483328,557056=761856,1318912=40960} 2024-09-21T09:36:37.004 INFO:tasks.rados.rados.0.smithi022.stdout:1540: writing smithi02284960-29 from 73728 to 557056 tid 1 2024-09-21T09:36:37.009 INFO:tasks.rados.rados.0.smithi022.stdout:1540: writing smithi02284960-29 from 557056 to 1318912 tid 2 2024-09-21T09:36:37.010 INFO:tasks.rados.rados.0.smithi022.stdout:1540: writing smithi02284960-29 from 1318912 to 1359872 tid 3 2024-09-21T09:36:37.010 INFO:tasks.rados.rados.0.smithi022.stdout:1541: rollback oid 27 current snap is 151 2024-09-21T09:36:37.010 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 27 to 149 2024-09-21T09:36:37.010 INFO:tasks.rados.rados.0.smithi022.stdout:1535: finishing copy_from to smithi02284960-12 2024-09-21T09:36:37.010 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 12 v 947 (ObjNum 482 snap 146 seq_num 482) dirty exists 2024-09-21T09:36:37.011 INFO:tasks.rados.rados.0.smithi022.stdout:1539: finishing write tid 1 to smithi02284960-15 2024-09-21T09:36:37.011 INFO:tasks.rados.rados.0.smithi022.stdout:1539: finishing write tid 2 to smithi02284960-15 2024-09-21T09:36:37.011 INFO:tasks.rados.rados.0.smithi022.stdout:1535: done (3 left) 2024-09-21T09:36:37.011 INFO:tasks.rados.rados.0.smithi022.stdout:1542: setattr oid 47 current snap is 151 2024-09-21T09:36:37.011 INFO:tasks.rados.rados.0.smithi022.stdout:1539: finishing write tid 3 to smithi02284960-15 2024-09-21T09:36:37.011 INFO:tasks.rados.rados.0.smithi022.stdout:1539: finishing write tid 4 to smithi02284960-15 2024-09-21T09:36:37.011 INFO:tasks.rados.rados.0.smithi022.stdout:1539: finishing write tid 5 to smithi02284960-15 2024-09-21T09:36:37.011 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 15 v 1051 (ObjNum 505 snap 151 seq_num 505) dirty exists 2024-09-21T09:36:37.011 INFO:tasks.rados.rados.0.smithi022.stdout:1539: left oid 15 (ObjNum 505 snap 151 seq_num 505) 2024-09-21T09:36:37.012 INFO:tasks.rados.rados.0.smithi022.stdout:1539: done (3 left) 2024-09-21T09:36:37.012 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 19 current snap is 151 2024-09-21T09:36:37.012 INFO:tasks.rados.rados.0.smithi022.stdout:1543: seq_num 507 ranges {1064960=499712,1564672=638976,2203648=163840} 2024-09-21T09:36:37.022 INFO:tasks.rados.rados.0.smithi022.stdout:1543: writing smithi02284960-19 from 1064960 to 1564672 tid 1 2024-09-21T09:36:37.026 INFO:tasks.rados.rados.0.smithi022.stdout:1543: writing smithi02284960-19 from 1564672 to 2203648 tid 2 2024-09-21T09:36:37.028 INFO:tasks.rados.rados.0.smithi022.stdout:1543: writing smithi02284960-19 from 2203648 to 2367488 tid 3 2024-09-21T09:36:37.028 INFO:tasks.rados.rados.0.smithi022.stdout:1544: copy_from oid 10 from oid 36 current snap is 151 2024-09-21T09:36:37.028 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 47 v 977 (ObjNum 382 snap 119 seq_num 382) dirty exists 2024-09-21T09:36:37.029 INFO:tasks.rados.rados.0.smithi022.stdout:1543: finishing write tid 1 to smithi02284960-19 2024-09-21T09:36:37.029 INFO:tasks.rados.rados.0.smithi022.stdout:1542: done (4 left) 2024-09-21T09:36:37.029 INFO:tasks.rados.rados.0.smithi022.stdout:1545: read oid 30 snap -1 2024-09-21T09:36:37.029 INFO:tasks.rados.rados.0.smithi022.stdout:1545: expect (ObjNum 481 snap 146 seq_num 481) 2024-09-21T09:36:37.029 INFO:tasks.rados.rados.0.smithi022.stdout:1546: delete oid 46 current snap is 151 2024-09-21T09:36:37.031 INFO:tasks.rados.rados.0.smithi022.stdout:1546: done (5 left) 2024-09-21T09:36:37.031 INFO:tasks.rados.rados.0.smithi022.stdout:1547: copy_from oid 2 from oid 41 current snap is 151 2024-09-21T09:36:37.031 INFO:tasks.rados.rados.0.smithi022.stdout:1548: copy_from oid 31 from oid 33 current snap is 151 2024-09-21T09:36:37.031 INFO:tasks.rados.rados.0.smithi022.stdout:1549: copy_from oid 16 from oid 4 current snap is 151 2024-09-21T09:36:37.031 INFO:tasks.rados.rados.0.smithi022.stdout:1550: read oid 44 snap -1 2024-09-21T09:36:37.031 INFO:tasks.rados.rados.0.smithi022.stdout:1550: expect deleted 2024-09-21T09:36:37.031 INFO:tasks.rados.rados.0.smithi022.stdout:1551: read oid 45 snap -1 2024-09-21T09:36:37.031 INFO:tasks.rados.rados.0.smithi022.stdout:1551: expect (ObjNum 442 snap 134 seq_num 442) 2024-09-21T09:36:37.031 INFO:tasks.rados.rados.0.smithi022.stdout:1552: read oid 12 snap -1 2024-09-21T09:36:37.031 INFO:tasks.rados.rados.0.smithi022.stdout:1552: expect (ObjNum 482 snap 146 seq_num 482) 2024-09-21T09:36:37.031 INFO:tasks.rados.rados.0.smithi022.stdout:1553: setattr oid 42 current snap is 151 2024-09-21T09:36:37.032 INFO:tasks.rados.rados.0.smithi022.stdout:1543: finishing write tid 2 to smithi02284960-19 2024-09-21T09:36:37.032 INFO:tasks.rados.rados.0.smithi022.stdout:1543: finishing write tid 3 to smithi02284960-19 2024-09-21T09:36:37.032 INFO:tasks.rados.rados.0.smithi022.stdout:1550: done (11 left) 2024-09-21T09:36:37.032 INFO:tasks.rados.rados.0.smithi022.stdout:1554: read oid 25 snap -1 2024-09-21T09:36:37.032 INFO:tasks.rados.rados.0.smithi022.stdout:1554: expect (ObjNum 428 snap 133 seq_num 428) 2024-09-21T09:36:37.032 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 3 current snap is 151 2024-09-21T09:36:37.033 INFO:tasks.rados.rados.0.smithi022.stdout:1555: seq_num 509 ranges {2998272=532480,3530752=409600,3940352=393216} 2024-09-21T09:36:37.052 INFO:tasks.rados.rados.0.smithi022.stdout:1555: writing smithi02284960-3 from 2998272 to 3530752 tid 1 2024-09-21T09:36:37.055 INFO:tasks.rados.rados.0.smithi022.stdout:1555: writing smithi02284960-3 from 3530752 to 3940352 tid 2 2024-09-21T09:36:37.057 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:36 smithi022 ceph-mon[98615]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-21T09:36:37.057 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:36 smithi022 ceph-mon[98615]: osdmap e375: 8 total, 7 up, 8 in 2024-09-21T09:36:37.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:36 smithi022 ceph-mon[98615]: from='osd.1 [v2:172.21.15.22:6810/2947123190,v1:172.21.15.22:6811/2947123190]' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi022", "root=default"]} : dispatch 2024-09-21T09:36:37.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:36 smithi022 ceph-mon[98615]: from='osd.1 ' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi022", "root=default"]} : dispatch 2024-09-21T09:36:37.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:36 smithi022 ceph-mon[98615]: from='osd.1 ' entity='osd.1' 2024-09-21T09:36:37.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:36 smithi022 ceph-mon[96736]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-21T09:36:37.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:36 smithi022 ceph-mon[96736]: osdmap e375: 8 total, 7 up, 8 in 2024-09-21T09:36:37.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:36 smithi022 ceph-mon[96736]: from='osd.1 [v2:172.21.15.22:6810/2947123190,v1:172.21.15.22:6811/2947123190]' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi022", "root=default"]} : dispatch 2024-09-21T09:36:37.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:36 smithi022 ceph-mon[96736]: from='osd.1 ' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi022", "root=default"]} : dispatch 2024-09-21T09:36:37.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:36 smithi022 ceph-mon[96736]: from='osd.1 ' entity='osd.1' 2024-09-21T09:36:37.059 INFO:tasks.rados.rados.0.smithi022.stdout:1555: writing smithi02284960-3 from 3940352 to 4333568 tid 3 2024-09-21T09:36:37.059 INFO:tasks.rados.rados.0.smithi022.stdout:1556: read oid 18 snap 149 2024-09-21T09:36:37.059 INFO:tasks.rados.rados.0.smithi022.stdout:1556: expect (ObjNum 482 snap 146 seq_num 482) 2024-09-21T09:36:37.059 INFO:tasks.rados.rados.0.smithi022.stdout:1557: snap_create 2024-09-21T09:36:37.059 INFO:tasks.rados.rados.0.smithi022.stdout:1543: finishing write tid 4 to smithi02284960-19 2024-09-21T09:36:37.059 INFO:tasks.rados.rados.0.smithi022.stdout:1543: finishing write tid 5 to smithi02284960-19 2024-09-21T09:36:37.059 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 19 v 1056 (ObjNum 507 snap 151 seq_num 507) dirty exists 2024-09-21T09:36:37.059 INFO:tasks.rados.rados.0.smithi022.stdout:1543: left oid 19 (ObjNum 507 snap 151 seq_num 507) 2024-09-21T09:36:37.060 INFO:tasks.rados.rados.0.smithi022.stdout:1552: expect (ObjNum 482 snap 146 seq_num 482) 2024-09-21T09:36:37.082 INFO:tasks.rados.rados.0.smithi022.stdout:1554: expect (ObjNum 428 snap 133 seq_num 428) 2024-09-21T09:36:37.189 INFO:tasks.rados.rados.0.smithi022.stdout:1555: finishing write tid 1 to smithi02284960-3 2024-09-21T09:36:37.189 INFO:tasks.rados.rados.0.smithi022.stdout:1555: finishing write tid 2 to smithi02284960-3 2024-09-21T09:36:37.190 INFO:tasks.rados.rados.0.smithi022.stdout:1555: finishing write tid 3 to smithi02284960-3 2024-09-21T09:36:37.190 INFO:tasks.rados.rados.0.smithi022.stdout:1555: finishing write tid 4 to smithi02284960-3 2024-09-21T09:36:37.190 INFO:tasks.rados.rados.0.smithi022.stdout:1555: finishing write tid 5 to smithi02284960-3 2024-09-21T09:36:37.190 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 3 v 475 (ObjNum 509 snap 151 seq_num 509) dirty exists 2024-09-21T09:36:37.190 INFO:tasks.rados.rados.0.smithi022.stdout:1555: left oid 3 (ObjNum 509 snap 151 seq_num 509) 2024-09-21T09:36:37.190 INFO:tasks.rados.rados.0.smithi022.stdout:1556: expect (ObjNum 482 snap 146 seq_num 482) 2024-09-21T09:36:37.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:36 smithi064 ceph-mon[86709]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-21T09:36:37.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:36 smithi064 ceph-mon[86709]: osdmap e375: 8 total, 7 up, 8 in 2024-09-21T09:36:37.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:36 smithi064 ceph-mon[86709]: from='osd.1 [v2:172.21.15.22:6810/2947123190,v1:172.21.15.22:6811/2947123190]' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi022", "root=default"]} : dispatch 2024-09-21T09:36:37.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:36 smithi064 ceph-mon[86709]: from='osd.1 ' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi022", "root=default"]} : dispatch 2024-09-21T09:36:37.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:36 smithi064 ceph-mon[86709]: from='osd.1 ' entity='osd.1' 2024-09-21T09:36:37.442 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:36:36 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:36:36.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi022\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:36:37.974 INFO:tasks.rados.rados.0.smithi022.stdout:1543: done (14 left) 2024-09-21T09:36:37.974 INFO:tasks.rados.rados.0.smithi022.stdout:1552: done (13 left) 2024-09-21T09:36:37.974 INFO:tasks.rados.rados.0.smithi022.stdout:1554: done (12 left) 2024-09-21T09:36:37.974 INFO:tasks.rados.rados.0.smithi022.stdout:1555: done (11 left) 2024-09-21T09:36:37.974 INFO:tasks.rados.rados.0.smithi022.stdout:1556: done (10 left) 2024-09-21T09:36:37.974 INFO:tasks.rados.rados.0.smithi022.stdout:1557: done (9 left) 2024-09-21T09:36:37.974 INFO:tasks.rados.rados.0.smithi022.stdout:1558: rmattr oid 17 current snap is 152 2024-09-21T09:36:37.974 INFO:tasks.rados.rados.0.smithi022.stdout:1558: done (9 left) 2024-09-21T09:36:37.974 INFO:tasks.rados.rados.0.smithi022.stdout:1559: snap_remove snap 151 2024-09-21T09:36:38.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:37 smithi022 ceph-mon[96736]: pgmap v166: 121 pgs: 33 active+undersized, 12 active+undersized+degraded, 76 active+clean; 160 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.9 MiB/s rd, 1.7 MiB/s wr, 9 op/s; 68/405 objects degraded (16.790%) 2024-09-21T09:36:38.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:37 smithi022 ceph-mon[96736]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T09:36:38.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:37 smithi022 ceph-mon[96736]: osd.1 [v2:172.21.15.22:6810/2947123190,v1:172.21.15.22:6811/2947123190] boot 2024-09-21T09:36:38.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:37 smithi022 ceph-mon[96736]: osdmap e376: 8 total, 8 up, 8 in 2024-09-21T09:36:38.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:37 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-21T09:36:38.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:37 smithi022 ceph-mon[98615]: pgmap v166: 121 pgs: 33 active+undersized, 12 active+undersized+degraded, 76 active+clean; 160 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.9 MiB/s rd, 1.7 MiB/s wr, 9 op/s; 68/405 objects degraded (16.790%) 2024-09-21T09:36:38.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:37 smithi022 ceph-mon[98615]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T09:36:38.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:37 smithi022 ceph-mon[98615]: osd.1 [v2:172.21.15.22:6810/2947123190,v1:172.21.15.22:6811/2947123190] boot 2024-09-21T09:36:38.310 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:37 smithi022 ceph-mon[98615]: osdmap e376: 8 total, 8 up, 8 in 2024-09-21T09:36:38.310 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:37 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-21T09:36:38.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:37 smithi064 ceph-mon[86709]: pgmap v166: 121 pgs: 33 active+undersized, 12 active+undersized+degraded, 76 active+clean; 160 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.9 MiB/s rd, 1.7 MiB/s wr, 9 op/s; 68/405 objects degraded (16.790%) 2024-09-21T09:36:38.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:37 smithi064 ceph-mon[86709]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T09:36:38.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:37 smithi064 ceph-mon[86709]: osd.1 [v2:172.21.15.22:6810/2947123190,v1:172.21.15.22:6811/2947123190] boot 2024-09-21T09:36:38.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:37 smithi064 ceph-mon[86709]: osdmap e376: 8 total, 8 up, 8 in 2024-09-21T09:36:38.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:37 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-21T09:36:38.808 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:36:38 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: 2024-09-21T09:36:38.558+0000 7feec1b50640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (4 PGs are or would become offline) 2024-09-21T09:36:38.977 INFO:tasks.rados.rados.0.smithi022.stdout:1559: done (9 left) 2024-09-21T09:36:38.977 INFO:tasks.rados.rados.0.smithi022.stdout:1560: read oid 40 snap -1 2024-09-21T09:36:38.977 INFO:tasks.rados.rados.0.smithi022.stdout:1560: expect deleted 2024-09-21T09:36:38.977 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 9 current snap is 152 2024-09-21T09:36:38.977 INFO:tasks.rados.rados.0.smithi022.stdout:1549: finishing copy_from to smithi02284960-16 2024-09-21T09:36:38.978 INFO:tasks.rados.rados.0.smithi022.stdout:1549: got expected ENOENT (src dne) 2024-09-21T09:36:38.978 INFO:tasks.rados.rados.0.smithi022.stdout:1561: seq_num 510 ranges {0=548864,548864=745472,1294336=57344} 2024-09-21T09:36:38.980 INFO:tasks.rados.rados.0.smithi022.stdout:1561: writing smithi02284960-9 from 0 to 548864 tid 1 2024-09-21T09:36:38.984 INFO:tasks.rados.rados.0.smithi022.stdout:1561: writing smithi02284960-9 from 548864 to 1294336 tid 2 2024-09-21T09:36:38.985 INFO:tasks.rados.rados.0.smithi022.stdout:1561: writing smithi02284960-9 from 1294336 to 1351680 tid 3 2024-09-21T09:36:38.986 INFO:tasks.rados.rados.0.smithi022.stdout:1562: read oid 3 snap -1 2024-09-21T09:36:38.986 INFO:tasks.rados.rados.0.smithi022.stdout:1562: expect (ObjNum 509 snap 151 seq_num 509) 2024-09-21T09:36:38.986 INFO:tasks.rados.rados.0.smithi022.stdout:1563: snap_create 2024-09-21T09:36:38.986 INFO:tasks.rados.rados.0.smithi022.stdout:1549: finishing copy_from racing read to smithi02284960-16 2024-09-21T09:36:38.986 INFO:tasks.rados.rados.0.smithi022.stdout:1540: finishing write tid 1 to smithi02284960-29 2024-09-21T09:36:38.986 INFO:tasks.rados.rados.0.smithi022.stdout:1540: finishing write tid 2 to smithi02284960-29 2024-09-21T09:36:38.986 INFO:tasks.rados.rados.0.smithi022.stdout:1540: finishing write tid 3 to smithi02284960-29 2024-09-21T09:36:38.986 INFO:tasks.rados.rados.0.smithi022.stdout:1540: finishing write tid 4 to smithi02284960-29 2024-09-21T09:36:38.986 INFO:tasks.rados.rados.0.smithi022.stdout:1548: finishing copy_from racing read to smithi02284960-31 2024-09-21T09:36:38.986 INFO:tasks.rados.rados.0.smithi022.stdout:1548: finishing copy_from to smithi02284960-31 2024-09-21T09:36:38.986 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 31 v 967 (ObjNum 65 snap 0 seq_num 1952206943) dirty exists 2024-09-21T09:36:38.986 INFO:tasks.rados.rados.0.smithi022.stdout:1540: finishing write tid 5 to smithi02284960-29 2024-09-21T09:36:38.986 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 29 v 838 (ObjNum 506 snap 151 seq_num 506) dirty exists 2024-09-21T09:36:38.986 INFO:tasks.rados.rados.0.smithi022.stdout:1540: left oid 29 (ObjNum 506 snap 151 seq_num 506) 2024-09-21T09:36:38.987 INFO:tasks.rados.rados.0.smithi022.stdout:1541: finishing rollback tid 1 to smithi02284960-27 2024-09-21T09:36:38.987 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 27 v 0 (ObjNum 81 snap 0 seq_num 1) dirty dne 2024-09-21T09:36:38.987 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 42 v 678 (ObjNum 430 snap 134 seq_num 430) dirty exists 2024-09-21T09:36:38.987 INFO:tasks.rados.rados.0.smithi022.stdout:1544: finishing copy_from to smithi02284960-10 2024-09-21T09:36:38.987 INFO:tasks.rados.rados.0.smithi022.stdout:1544: got expected ENOENT (src dne) 2024-09-21T09:36:38.987 INFO:tasks.rados.rados.0.smithi022.stdout:1544: finishing copy_from racing read to smithi02284960-10 2024-09-21T09:36:38.987 INFO:tasks.rados.rados.0.smithi022.stdout:1545: expect (ObjNum 481 snap 146 seq_num 481) 2024-09-21T09:36:39.018 INFO:tasks.rados.rados.0.smithi022.stdout:1547: finishing copy_from racing read to smithi02284960-2 2024-09-21T09:36:39.018 INFO:tasks.rados.rados.0.smithi022.stdout:1551: expect (ObjNum 442 snap 134 seq_num 442) 2024-09-21T09:36:39.132 INFO:tasks.rados.rados.0.smithi022.stdout:1547: finishing copy_from to smithi02284960-2 2024-09-21T09:36:39.132 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 2 v 779 (ObjNum 452 snap 136 seq_num 452) dirty exists 2024-09-21T09:36:39.132 INFO:tasks.rados.rados.0.smithi022.stdout:1561: finishing write tid 1 to smithi02284960-9 2024-09-21T09:36:39.132 INFO:tasks.rados.rados.0.smithi022.stdout:1561: finishing write tid 2 to smithi02284960-9 2024-09-21T09:36:39.132 INFO:tasks.rados.rados.0.smithi022.stdout:1561: finishing write tid 3 to smithi02284960-9 2024-09-21T09:36:39.132 INFO:tasks.rados.rados.0.smithi022.stdout:1561: finishing write tid 4 to smithi02284960-9 2024-09-21T09:36:39.132 INFO:tasks.rados.rados.0.smithi022.stdout:1561: finishing write tid 5 to smithi02284960-9 2024-09-21T09:36:39.132 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 9 v 576 (ObjNum 510 snap 152 seq_num 510) dirty exists 2024-09-21T09:36:39.132 INFO:tasks.rados.rados.0.smithi022.stdout:1561: left oid 9 (ObjNum 510 snap 152 seq_num 510) 2024-09-21T09:36:39.132 INFO:tasks.rados.rados.0.smithi022.stdout:1562: expect (ObjNum 509 snap 151 seq_num 509) 2024-09-21T09:36:39.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:38 smithi022 ceph-mon[98615]: osdmap e377: 8 total, 8 up, 8 in 2024-09-21T09:36:39.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:38 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:36:39.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:38 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:36:39.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:38 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:36:39.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:38 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:36:39.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:38 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:36:39.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:38 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:36:39.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:38 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:36:39.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:38 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:36:39.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:38 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:36:39.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:38 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-21T09:36:39.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:38 smithi022 ceph-mon[96736]: osdmap e377: 8 total, 8 up, 8 in 2024-09-21T09:36:39.310 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:38 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:36:39.310 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:38 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:36:39.310 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:38 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:36:39.310 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:38 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:36:39.310 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:38 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:36:39.310 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:38 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:36:39.310 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:38 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:36:39.311 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:38 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:36:39.311 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:38 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:36:39.311 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:38 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-21T09:36:39.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:38 smithi064 ceph-mon[86709]: osdmap e377: 8 total, 8 up, 8 in 2024-09-21T09:36:39.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:38 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:36:39.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:38 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:36:39.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:38 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:36:39.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:38 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:36:39.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:38 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:36:39.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:38 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:36:39.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:38 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:36:39.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:38 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:36:39.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:38 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:36:39.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:38 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-21T09:36:39.549 INFO:teuthology.orchestra.run.smithi022.stdout:true 2024-09-21T09:36:39.978 INFO:tasks.rados.rados.0.smithi022.stdout:1540: done (12 left) 2024-09-21T09:36:39.978 INFO:tasks.rados.rados.0.smithi022.stdout:1541: done (11 left) 2024-09-21T09:36:39.978 INFO:tasks.rados.rados.0.smithi022.stdout:1544: done (10 left) 2024-09-21T09:36:39.978 INFO:tasks.rados.rados.0.smithi022.stdout:1545: done (9 left) 2024-09-21T09:36:39.978 INFO:tasks.rados.rados.0.smithi022.stdout:1547: done (8 left) 2024-09-21T09:36:39.978 INFO:tasks.rados.rados.0.smithi022.stdout:1548: done (7 left) 2024-09-21T09:36:39.978 INFO:tasks.rados.rados.0.smithi022.stdout:1549: done (6 left) 2024-09-21T09:36:39.978 INFO:tasks.rados.rados.0.smithi022.stdout:1551: done (5 left) 2024-09-21T09:36:39.979 INFO:tasks.rados.rados.0.smithi022.stdout:1553: done (4 left) 2024-09-21T09:36:39.979 INFO:tasks.rados.rados.0.smithi022.stdout:1560: done (3 left) 2024-09-21T09:36:39.979 INFO:tasks.rados.rados.0.smithi022.stdout:1561: done (2 left) 2024-09-21T09:36:39.979 INFO:tasks.rados.rados.0.smithi022.stdout:1562: done (1 left) 2024-09-21T09:36:39.979 INFO:tasks.rados.rados.0.smithi022.stdout:1563: done (0 left) 2024-09-21T09:36:39.979 INFO:tasks.rados.rados.0.smithi022.stdout:1564: snap_remove snap 149 2024-09-21T09:36:40.136 INFO:teuthology.orchestra.run.smithi022.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T09:36:40.136 INFO:teuthology.orchestra.run.smithi022.stdout:alertmanager.a smithi022 *:9093,9094 running (3m) 5s ago 7m 17.1M - 0.27.0 11f11916f8cd caced80ba544 2024-09-21T09:36:40.136 INFO:teuthology.orchestra.run.smithi022.stdout:grafana.a smithi064 *:3000 running (2m) 80s ago 7m 77.9M - 9.4.7 954c08fa6188 527bb657d70d 2024-09-21T09:36:40.136 INFO:teuthology.orchestra.run.smithi022.stdout:mds.a smithi022 running (9m) 5s ago 9m 17.9M - 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 34e4221f3a27 2024-09-21T09:36:40.136 INFO:teuthology.orchestra.run.smithi022.stdout:mds.b smithi064 running (9m) 80s ago 9m 20.9M - 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 0f846b3ec694 2024-09-21T09:36:40.136 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.x smithi064 *:8443,9283,8765 running (2m) 80s ago 12m 453M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 61abc61fea8d 2024-09-21T09:36:40.136 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.y smithi022 *:8443,9283,8765 running (2m) 5s ago 14m 512M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 6c5bee78f09b 2024-09-21T09:36:40.136 INFO:teuthology.orchestra.run.smithi022.stdout:mon.a smithi022 running (2m) 5s ago 14m 52.6M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 0513395c784d 2024-09-21T09:36:40.136 INFO:teuthology.orchestra.run.smithi022.stdout:mon.b smithi064 running (85s) 80s ago 13m 31.5M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 9dab26b6a9e7 2024-09-21T09:36:40.136 INFO:teuthology.orchestra.run.smithi022.stdout:mon.c smithi022 running (103s) 5s ago 13m 43.1M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 c74b58b67bd6 2024-09-21T09:36:40.137 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.a smithi022 *:9100 running (3m) 5s ago 7m 10.1M - 1.7.0 72c9c2088986 593c6158f524 2024-09-21T09:36:40.137 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.b smithi064 *:9100 running (2m) 80s ago 7m 9.94M - 1.7.0 72c9c2088986 a533420b0139 2024-09-21T09:36:40.137 INFO:teuthology.orchestra.run.smithi022.stdout:osd.0 smithi022 running (43s) 5s ago 12m 110M 2481M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 40ce23a2b569 2024-09-21T09:36:40.137 INFO:teuthology.orchestra.run.smithi022.stdout:osd.1 smithi022 running (10s) 5s ago 12m 14.5M 2481M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 7cd0c18d224c 2024-09-21T09:36:40.137 INFO:teuthology.orchestra.run.smithi022.stdout:osd.2 smithi022 running (11m) 5s ago 11m 225M 2481M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be fefc7c770c32 2024-09-21T09:36:40.137 INFO:teuthology.orchestra.run.smithi022.stdout:osd.3 smithi022 running (11m) 5s ago 11m 109M 2481M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be a56193b67a92 2024-09-21T09:36:40.137 INFO:teuthology.orchestra.run.smithi022.stdout:osd.4 smithi064 running (10m) 80s ago 10m 141M 2480M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 2a2937374fc4 2024-09-21T09:36:40.137 INFO:teuthology.orchestra.run.smithi022.stdout:osd.5 smithi064 running (10m) 80s ago 10m 192M 2480M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 1900c4af5d28 2024-09-21T09:36:40.137 INFO:teuthology.orchestra.run.smithi022.stdout:osd.6 smithi064 running (10m) 80s ago 10m 126M 2480M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 3cc4ec816e5b 2024-09-21T09:36:40.137 INFO:teuthology.orchestra.run.smithi022.stdout:osd.7 smithi064 running (9m) 80s ago 9m 148M 2480M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 18ef0be94104 2024-09-21T09:36:40.137 INFO:teuthology.orchestra.run.smithi022.stdout:prometheus.a smithi064 *:9095 running (2m) 80s ago 8m 44.0M - 2.51.0 1d3b7f56885b 80629ffa2179 2024-09-21T09:36:40.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:39 smithi022 ceph-mon[96736]: pgmap v169: 121 pgs: 1 active+recovery_wait+degraded, 11 active+undersized, 5 active+undersized+degraded, 104 active+clean; 93 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 5.2 MiB/s rd, 2.3 MiB/s wr, 28 op/s; 14/273 objects degraded (5.128%); 7.0 MiB/s, 3 objects/s recovering 2024-09-21T09:36:40.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:39 smithi022 ceph-mon[96736]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-21T09:36:40.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:39 smithi022 ceph-mon[96736]: Upgrade: unsafe to stop osd(s) at this time (4 PGs are or would become offline) 2024-09-21T09:36:40.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:39 smithi022 ceph-mon[96736]: osdmap e378: 8 total, 8 up, 8 in 2024-09-21T09:36:40.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:39 smithi022 ceph-mon[98615]: pgmap v169: 121 pgs: 1 active+recovery_wait+degraded, 11 active+undersized, 5 active+undersized+degraded, 104 active+clean; 93 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 5.2 MiB/s rd, 2.3 MiB/s wr, 28 op/s; 14/273 objects degraded (5.128%); 7.0 MiB/s, 3 objects/s recovering 2024-09-21T09:36:40.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:39 smithi022 ceph-mon[98615]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-21T09:36:40.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:39 smithi022 ceph-mon[98615]: Upgrade: unsafe to stop osd(s) at this time (4 PGs are or would become offline) 2024-09-21T09:36:40.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:39 smithi022 ceph-mon[98615]: osdmap e378: 8 total, 8 up, 8 in 2024-09-21T09:36:40.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:39 smithi064 ceph-mon[86709]: pgmap v169: 121 pgs: 1 active+recovery_wait+degraded, 11 active+undersized, 5 active+undersized+degraded, 104 active+clean; 93 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 5.2 MiB/s rd, 2.3 MiB/s wr, 28 op/s; 14/273 objects degraded (5.128%); 7.0 MiB/s, 3 objects/s recovering 2024-09-21T09:36:40.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:39 smithi064 ceph-mon[86709]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-21T09:36:40.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:39 smithi064 ceph-mon[86709]: Upgrade: unsafe to stop osd(s) at this time (4 PGs are or would become offline) 2024-09-21T09:36:40.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:39 smithi064 ceph-mon[86709]: osdmap e378: 8 total, 8 up, 8 in 2024-09-21T09:36:40.442 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:36:40 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:36:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:36:40.490 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-09-21T09:36:40.490 INFO:teuthology.orchestra.run.smithi022.stdout: "mon": { 2024-09-21T09:36:40.490 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 3 2024-09-21T09:36:40.490 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-21T09:36:40.491 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr": { 2024-09-21T09:36:40.491 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 2 2024-09-21T09:36:40.491 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-21T09:36:40.491 INFO:teuthology.orchestra.run.smithi022.stdout: "osd": { 2024-09-21T09:36:40.491 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 6, 2024-09-21T09:36:40.491 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 2 2024-09-21T09:36:40.491 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-21T09:36:40.491 INFO:teuthology.orchestra.run.smithi022.stdout: "mds": { 2024-09-21T09:36:40.491 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 2 2024-09-21T09:36:40.491 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-21T09:36:40.491 INFO:teuthology.orchestra.run.smithi022.stdout: "overall": { 2024-09-21T09:36:40.491 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 8, 2024-09-21T09:36:40.491 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 7 2024-09-21T09:36:40.492 INFO:teuthology.orchestra.run.smithi022.stdout: } 2024-09-21T09:36:40.492 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-09-21T09:36:40.789 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-09-21T09:36:40.789 INFO:teuthology.orchestra.run.smithi022.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47", 2024-09-21T09:36:40.789 INFO:teuthology.orchestra.run.smithi022.stdout: "in_progress": true, 2024-09-21T09:36:40.789 INFO:teuthology.orchestra.run.smithi022.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T09:36:40.789 INFO:teuthology.orchestra.run.smithi022.stdout: "services_complete": [ 2024-09-21T09:36:40.789 INFO:teuthology.orchestra.run.smithi022.stdout: "mon", 2024-09-21T09:36:40.789 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr" 2024-09-21T09:36:40.789 INFO:teuthology.orchestra.run.smithi022.stdout: ], 2024-09-21T09:36:40.789 INFO:teuthology.orchestra.run.smithi022.stdout: "progress": "7/20 daemons upgraded", 2024-09-21T09:36:40.789 INFO:teuthology.orchestra.run.smithi022.stdout: "message": "Currently upgrading osd daemons", 2024-09-21T09:36:40.789 INFO:teuthology.orchestra.run.smithi022.stdout: "is_paused": false 2024-09-21T09:36:40.790 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-09-21T09:36:40.987 INFO:tasks.rados.rados.0.smithi022.stdout:1564: done (0 left) 2024-09-21T09:36:40.988 INFO:tasks.rados.rados.0.smithi022.stdout:1565: rollback oid 10 current snap is 153 2024-09-21T09:36:40.988 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 10 to 152 2024-09-21T09:36:40.988 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 5 current snap is 153 2024-09-21T09:36:40.988 INFO:tasks.rados.rados.0.smithi022.stdout:1566: seq_num 511 ranges {2973696=565248,3538944=630784,4169728=172032} 2024-09-21T09:36:41.007 INFO:tasks.rados.rados.0.smithi022.stdout:1566: writing smithi02284960-5 from 2973696 to 3538944 tid 1 2024-09-21T09:36:41.011 INFO:tasks.rados.rados.0.smithi022.stdout:1566: writing smithi02284960-5 from 3538944 to 4169728 tid 2 2024-09-21T09:36:41.013 INFO:tasks.rados.rados.0.smithi022.stdout:1566: writing smithi02284960-5 from 4169728 to 4341760 tid 3 2024-09-21T09:36:41.013 INFO:tasks.rados.rados.0.smithi022.stdout:1567: copy_from oid 39 from oid 21 current snap is 153 2024-09-21T09:36:41.013 INFO:tasks.rados.rados.0.smithi022.stdout:1568: rollback oid 35 current snap is 153 2024-09-21T09:36:41.013 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 35 to 152 2024-09-21T09:36:41.013 INFO:tasks.rados.rados.0.smithi022.stdout:1565: finishing rollback tid 0 to smithi02284960-10 2024-09-21T09:36:41.013 INFO:tasks.rados.rados.0.smithi022.stdout:1565: finishing rollback tid 1 to smithi02284960-10 2024-09-21T09:36:41.013 INFO:tasks.rados.rados.0.smithi022.stdout:1565: finishing rollback tid 2 to smithi02284960-10 2024-09-21T09:36:41.013 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 10 v 928 (ObjNum 478 snap 146 seq_num 478) dirty exists 2024-09-21T09:36:41.013 INFO:tasks.rados.rados.0.smithi022.stdout:1565: done (3 left) 2024-09-21T09:36:41.014 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 4 current snap is 153 2024-09-21T09:36:41.014 INFO:tasks.rados.rados.0.smithi022.stdout:1569: seq_num 512 ranges {0=581632,581632=516096,1097728=286720} 2024-09-21T09:36:41.017 INFO:tasks.rados.rados.0.smithi022.stdout:1569: writing smithi02284960-4 from 0 to 581632 tid 1 2024-09-21T09:36:41.020 INFO:tasks.rados.rados.0.smithi022.stdout:1569: writing smithi02284960-4 from 581632 to 1097728 tid 2 2024-09-21T09:36:41.023 INFO:tasks.rados.rados.0.smithi022.stdout:1569: writing smithi02284960-4 from 1097728 to 1384448 tid 3 2024-09-21T09:36:41.023 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 48 current snap is 153 2024-09-21T09:36:41.023 INFO:tasks.rados.rados.0.smithi022.stdout:1568: finishing rollback tid 0 to smithi02284960-35 2024-09-21T09:36:41.023 INFO:tasks.rados.rados.0.smithi022.stdout:1567: finishing copy_from to smithi02284960-39 2024-09-21T09:36:41.023 INFO:tasks.rados.rados.0.smithi022.stdout:1567: got expected ENOENT (src dne) 2024-09-21T09:36:41.023 INFO:tasks.rados.rados.0.smithi022.stdout:1570: seq_num 513 ranges {2056192=598016,2654208=794624,3448832=8192} 2024-09-21T09:36:41.037 INFO:tasks.rados.rados.0.smithi022.stdout:1570: writing smithi02284960-48 from 2056192 to 2654208 tid 1 2024-09-21T09:36:41.043 INFO:tasks.rados.rados.0.smithi022.stdout:1570: writing smithi02284960-48 from 2654208 to 3448832 tid 2 2024-09-21T09:36:41.044 INFO:tasks.rados.rados.0.smithi022.stdout:1570: writing smithi02284960-48 from 3448832 to 3457024 tid 3 2024-09-21T09:36:41.044 INFO:tasks.rados.rados.0.smithi022.stdout:1571: setattr oid 8 current snap is 153 2024-09-21T09:36:41.044 INFO:tasks.rados.rados.0.smithi022.stdout:1567: finishing copy_from racing read to smithi02284960-39 2024-09-21T09:36:41.044 INFO:tasks.rados.rados.0.smithi022.stdout:1566: finishing write tid 1 to smithi02284960-5 2024-09-21T09:36:41.044 INFO:tasks.rados.rados.0.smithi022.stdout:1568: finishing rollback tid 1 to smithi02284960-35 2024-09-21T09:36:41.044 INFO:tasks.rados.rados.0.smithi022.stdout:1568: finishing rollback tid 2 to smithi02284960-35 2024-09-21T09:36:41.044 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 35 v 550 (ObjNum 471 snap 142 seq_num 471) dirty exists 2024-09-21T09:36:41.045 INFO:tasks.rados.rados.0.smithi022.stdout:1566: finishing write tid 2 to smithi02284960-5 2024-09-21T09:36:41.045 INFO:tasks.rados.rados.0.smithi022.stdout:1566: finishing write tid 3 to smithi02284960-5 2024-09-21T09:36:41.045 INFO:tasks.rados.rados.0.smithi022.stdout:1566: finishing write tid 4 to smithi02284960-5 2024-09-21T09:36:41.045 INFO:tasks.rados.rados.0.smithi022.stdout:1569: finishing write tid 1 to smithi02284960-4 2024-09-21T09:36:41.045 INFO:tasks.rados.rados.0.smithi022.stdout:1566: finishing write tid 5 to smithi02284960-5 2024-09-21T09:36:41.045 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 5 v 581 (ObjNum 511 snap 153 seq_num 511) dirty exists 2024-09-21T09:36:41.045 INFO:tasks.rados.rados.0.smithi022.stdout:1566: left oid 5 (ObjNum 511 snap 153 seq_num 511) 2024-09-21T09:36:41.045 INFO:tasks.rados.rados.0.smithi022.stdout:1569: finishing write tid 2 to smithi02284960-4 2024-09-21T09:36:41.045 INFO:tasks.rados.rados.0.smithi022.stdout:1569: finishing write tid 3 to smithi02284960-4 2024-09-21T09:36:41.046 INFO:tasks.rados.rados.0.smithi022.stdout:1569: finishing write tid 4 to smithi02284960-4 2024-09-21T09:36:41.046 INFO:tasks.rados.rados.0.smithi022.stdout:1569: finishing write tid 5 to smithi02284960-4 2024-09-21T09:36:41.046 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 4 v 585 (ObjNum 512 snap 153 seq_num 512) dirty exists 2024-09-21T09:36:41.046 INFO:tasks.rados.rados.0.smithi022.stdout:1569: left oid 4 (ObjNum 512 snap 153 seq_num 512) 2024-09-21T09:36:41.046 INFO:tasks.rados.rados.0.smithi022.stdout:1570: finishing write tid 1 to smithi02284960-48 2024-09-21T09:36:41.046 INFO:tasks.rados.rados.0.smithi022.stdout:1566: done (5 left) 2024-09-21T09:36:41.047 INFO:tasks.rados.rados.0.smithi022.stdout:1567: done (4 left) 2024-09-21T09:36:41.047 INFO:tasks.rados.rados.0.smithi022.stdout:1568: done (3 left) 2024-09-21T09:36:41.047 INFO:tasks.rados.rados.0.smithi022.stdout:1569: done (2 left) 2024-09-21T09:36:41.047 INFO:tasks.rados.rados.0.smithi022.stdout:1572: snap_remove snap 152 2024-09-21T09:36:41.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:40 smithi022 ceph-mon[98615]: from='client.34248 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:36:41.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:40 smithi022 ceph-mon[98615]: from='client.44180 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:36:41.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:40 smithi022 ceph-mon[98615]: osdmap e379: 8 total, 8 up, 8 in 2024-09-21T09:36:41.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:40 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/1495681736' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:36:41.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:40 smithi022 ceph-mon[96736]: from='client.34248 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:36:41.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:40 smithi022 ceph-mon[96736]: from='client.44180 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:36:41.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:40 smithi022 ceph-mon[96736]: osdmap e379: 8 total, 8 up, 8 in 2024-09-21T09:36:41.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:40 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/1495681736' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:36:41.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:40 smithi064 ceph-mon[86709]: from='client.34248 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:36:41.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:40 smithi064 ceph-mon[86709]: from='client.44180 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:36:41.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:40 smithi064 ceph-mon[86709]: osdmap e379: 8 total, 8 up, 8 in 2024-09-21T09:36:41.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:40 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/1495681736' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:36:41.992 INFO:tasks.rados.rados.0.smithi022.stdout:1572: done (2 left) 2024-09-21T09:36:41.992 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 37 current snap is 153 2024-09-21T09:36:41.992 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 8 v 589 (ObjNum 437 snap 134 seq_num 437) dirty exists 2024-09-21T09:36:41.992 INFO:tasks.rados.rados.0.smithi022.stdout:1573: seq_num 514 ranges {2293760=614400,2908160=679936,3588096=122880} 2024-09-21T09:36:42.008 INFO:tasks.rados.rados.0.smithi022.stdout:1573: writing smithi02284960-37 from 2293760 to 2908160 tid 1 2024-09-21T09:36:42.011 INFO:tasks.rados.rados.0.smithi022.stdout:1573: writing smithi02284960-37 from 2908160 to 3588096 tid 2 2024-09-21T09:36:42.013 INFO:tasks.rados.rados.0.smithi022.stdout:1573: writing smithi02284960-37 from 3588096 to 3710976 tid 3 2024-09-21T09:36:42.013 INFO:tasks.rados.rados.0.smithi022.stdout:1571: done (2 left) 2024-09-21T09:36:42.013 INFO:tasks.rados.rados.0.smithi022.stdout:1574: setattr oid 24 current snap is 153 2024-09-21T09:36:42.013 INFO:tasks.rados.rados.0.smithi022.stdout:1570: finishing write tid 2 to smithi02284960-48 2024-09-21T09:36:42.013 INFO:tasks.rados.rados.0.smithi022.stdout:1570: finishing write tid 3 to smithi02284960-48 2024-09-21T09:36:42.013 INFO:tasks.rados.rados.0.smithi022.stdout:1570: finishing write tid 4 to smithi02284960-48 2024-09-21T09:36:42.013 INFO:tasks.rados.rados.0.smithi022.stdout:1570: finishing write tid 5 to smithi02284960-48 2024-09-21T09:36:42.013 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 48 v 996 (ObjNum 513 snap 153 seq_num 513) dirty exists 2024-09-21T09:36:42.013 INFO:tasks.rados.rados.0.smithi022.stdout:1570: left oid 48 (ObjNum 513 snap 153 seq_num 513) 2024-09-21T09:36:42.014 INFO:tasks.rados.rados.0.smithi022.stdout:1570: done (2 left) 2024-09-21T09:36:42.014 INFO:tasks.rados.rados.0.smithi022.stdout:1575: snap_create 2024-09-21T09:36:42.018 INFO:tasks.rados.rados.0.smithi022.stdout:1573: finishing write tid 1 to smithi02284960-37 2024-09-21T09:36:42.020 INFO:tasks.rados.rados.0.smithi022.stdout:1573: finishing write tid 2 to smithi02284960-37 2024-09-21T09:36:42.021 INFO:tasks.rados.rados.0.smithi022.stdout:1573: finishing write tid 3 to smithi02284960-37 2024-09-21T09:36:42.021 INFO:tasks.rados.rados.0.smithi022.stdout:1573: finishing write tid 4 to smithi02284960-37 2024-09-21T09:36:42.022 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 24 v 1071 (ObjNum 33 snap 0 seq_num 3570189952) dirty exists 2024-09-21T09:36:42.023 INFO:tasks.rados.rados.0.smithi022.stdout:1573: finishing write tid 5 to smithi02284960-37 2024-09-21T09:36:42.023 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 37 v 1070 (ObjNum 514 snap 153 seq_num 514) dirty exists 2024-09-21T09:36:42.023 INFO:tasks.rados.rados.0.smithi022.stdout:1573: left oid 37 (ObjNum 514 snap 153 seq_num 514) 2024-09-21T09:36:42.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:41 smithi022 ceph-mon[98615]: from='client.34260 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:36:42.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:41 smithi022 ceph-mon[98615]: pgmap v172: 121 pgs: 1 active+recovery_wait+degraded, 11 active+undersized, 5 active+undersized+degraded, 104 active+clean; 93 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 5.2 MiB/s rd, 2.3 MiB/s wr, 28 op/s; 14/273 objects degraded (5.128%); 7.0 MiB/s, 3 objects/s recovering 2024-09-21T09:36:42.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:41 smithi022 ceph-mon[98615]: from='client.44198 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:36:42.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:41 smithi022 ceph-mon[98615]: osdmap e380: 8 total, 8 up, 8 in 2024-09-21T09:36:42.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:41 smithi022 ceph-mon[96736]: from='client.34260 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:36:42.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:41 smithi022 ceph-mon[96736]: pgmap v172: 121 pgs: 1 active+recovery_wait+degraded, 11 active+undersized, 5 active+undersized+degraded, 104 active+clean; 93 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 5.2 MiB/s rd, 2.3 MiB/s wr, 28 op/s; 14/273 objects degraded (5.128%); 7.0 MiB/s, 3 objects/s recovering 2024-09-21T09:36:42.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:41 smithi022 ceph-mon[96736]: from='client.44198 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:36:42.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:41 smithi022 ceph-mon[96736]: osdmap e380: 8 total, 8 up, 8 in 2024-09-21T09:36:42.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:41 smithi064 ceph-mon[86709]: from='client.34260 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:36:42.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:41 smithi064 ceph-mon[86709]: pgmap v172: 121 pgs: 1 active+recovery_wait+degraded, 11 active+undersized, 5 active+undersized+degraded, 104 active+clean; 93 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 5.2 MiB/s rd, 2.3 MiB/s wr, 28 op/s; 14/273 objects degraded (5.128%); 7.0 MiB/s, 3 objects/s recovering 2024-09-21T09:36:42.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:41 smithi064 ceph-mon[86709]: from='client.44198 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:36:42.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:41 smithi064 ceph-mon[86709]: osdmap e380: 8 total, 8 up, 8 in 2024-09-21T09:36:43.001 INFO:tasks.rados.rados.0.smithi022.stdout:1573: done (2 left) 2024-09-21T09:36:43.001 INFO:tasks.rados.rados.0.smithi022.stdout:1574: done (1 left) 2024-09-21T09:36:43.001 INFO:tasks.rados.rados.0.smithi022.stdout:1575: done (0 left) 2024-09-21T09:36:43.001 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 29 current snap is 154 2024-09-21T09:36:43.002 INFO:tasks.rados.rados.0.smithi022.stdout:1576: seq_num 515 ranges {1359872=630784,1990656=565248,2555904=237568} 2024-09-21T09:36:43.011 INFO:tasks.rados.rados.0.smithi022.stdout:1576: writing smithi02284960-29 from 1359872 to 1990656 tid 1 2024-09-21T09:36:43.015 INFO:tasks.rados.rados.0.smithi022.stdout:1576: writing smithi02284960-29 from 1990656 to 2555904 tid 2 2024-09-21T09:36:43.017 INFO:tasks.rados.rados.0.smithi022.stdout:1576: writing smithi02284960-29 from 2555904 to 2793472 tid 3 2024-09-21T09:36:43.017 INFO:tasks.rados.rados.0.smithi022.stdout:1577: copy_from oid 14 from oid 8 current snap is 154 2024-09-21T09:36:43.017 INFO:tasks.rados.rados.0.smithi022.stdout:1578: setattr oid 25 current snap is 154 2024-09-21T09:36:43.017 INFO:tasks.rados.rados.0.smithi022.stdout:1576: finishing write tid 1 to smithi02284960-29 2024-09-21T09:36:43.018 INFO:tasks.rados.rados.0.smithi022.stdout:1579: rollback oid 45 current snap is 154 2024-09-21T09:36:43.018 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 45 to 153 2024-09-21T09:36:43.018 INFO:tasks.rados.rados.0.smithi022.stdout:1580: rollback oid 50 current snap is 154 2024-09-21T09:36:43.019 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 50 to 153 2024-09-21T09:36:43.019 INFO:tasks.rados.rados.0.smithi022.stdout:1581: delete oid 6 current snap is 154 2024-09-21T09:36:43.021 INFO:tasks.rados.rados.0.smithi022.stdout:1576: finishing write tid 2 to smithi02284960-29 2024-09-21T09:36:43.022 INFO:tasks.rados.rados.0.smithi022.stdout:1576: finishing write tid 3 to smithi02284960-29 2024-09-21T09:36:43.022 INFO:tasks.rados.rados.0.smithi022.stdout:1576: finishing write tid 4 to smithi02284960-29 2024-09-21T09:36:43.025 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 25 v 934 (ObjNum 428 snap 133 seq_num 428) dirty exists 2024-09-21T09:36:43.027 INFO:tasks.rados.rados.0.smithi022.stdout:1576: finishing write tid 5 to smithi02284960-29 2024-09-21T09:36:43.027 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 29 v 842 (ObjNum 515 snap 154 seq_num 515) dirty exists 2024-09-21T09:36:43.027 INFO:tasks.rados.rados.0.smithi022.stdout:1576: left oid 29 (ObjNum 515 snap 154 seq_num 515) 2024-09-21T09:36:43.027 INFO:tasks.rados.rados.0.smithi022.stdout:1576: done (5 left) 2024-09-21T09:36:43.027 INFO:tasks.rados.rados.0.smithi022.stdout:1578: done (4 left) 2024-09-21T09:36:43.028 INFO:tasks.rados.rados.0.smithi022.stdout:1581: done (3 left) 2024-09-21T09:36:43.028 INFO:tasks.rados.rados.0.smithi022.stdout:1582: read oid 29 snap -1 2024-09-21T09:36:43.028 INFO:tasks.rados.rados.0.smithi022.stdout:1582: expect (ObjNum 515 snap 154 seq_num 515) 2024-09-21T09:36:43.028 INFO:tasks.rados.rados.0.smithi022.stdout:1583: read oid 48 snap 153 2024-09-21T09:36:43.028 INFO:tasks.rados.rados.0.smithi022.stdout:1583: expect (ObjNum 513 snap 153 seq_num 513) 2024-09-21T09:36:43.028 INFO:tasks.rados.rados.0.smithi022.stdout:1584: read oid 22 snap 153 2024-09-21T09:36:43.028 INFO:tasks.rados.rados.0.smithi022.stdout:1584: expect (ObjNum 469 snap 141 seq_num 469) 2024-09-21T09:36:43.028 INFO:tasks.rados.rados.0.smithi022.stdout:1585: copy_from oid 36 from oid 24 current snap is 154 2024-09-21T09:36:43.028 INFO:tasks.rados.rados.0.smithi022.stdout:1586: copy_from oid 11 from oid 23 current snap is 154 2024-09-21T09:36:43.028 INFO:tasks.rados.rados.0.smithi022.stdout:1587: delete oid 41 current snap is 154 2024-09-21T09:36:43.030 INFO:tasks.rados.rados.0.smithi022.stdout:1579: finishing rollback tid 0 to smithi02284960-45 2024-09-21T09:36:43.030 INFO:tasks.rados.rados.0.smithi022.stdout:1579: finishing rollback tid 1 to smithi02284960-45 2024-09-21T09:36:43.031 INFO:tasks.rados.rados.0.smithi022.stdout:1579: finishing rollback tid 2 to smithi02284960-45 2024-09-21T09:36:43.031 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 45 v 902 (ObjNum 442 snap 134 seq_num 442) dirty exists 2024-09-21T09:36:43.032 INFO:tasks.rados.rados.0.smithi022.stdout:1579: done (8 left) 2024-09-21T09:36:43.032 INFO:tasks.rados.rados.0.smithi022.stdout:1587: done (7 left) 2024-09-21T09:36:43.033 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 12 current snap is 154 2024-09-21T09:36:43.033 INFO:tasks.rados.rados.0.smithi022.stdout:1588: seq_num 518 ranges {876544=679936,1556480=671744,2228224=131072} 2024-09-21T09:36:43.042 INFO:tasks.rados.rados.0.smithi022.stdout:1588: writing smithi02284960-12 from 876544 to 1556480 tid 1 2024-09-21T09:36:43.047 INFO:tasks.rados.rados.0.smithi022.stdout:1588: writing smithi02284960-12 from 1556480 to 2228224 tid 2 2024-09-21T09:36:43.048 INFO:tasks.rados.rados.0.smithi022.stdout:1588: writing smithi02284960-12 from 2228224 to 2359296 tid 3 2024-09-21T09:36:43.048 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 7 current snap is 154 2024-09-21T09:36:43.048 INFO:tasks.rados.rados.0.smithi022.stdout:1580: finishing rollback tid 0 to smithi02284960-50 2024-09-21T09:36:43.048 INFO:tasks.rados.rados.0.smithi022.stdout:1577: finishing copy_from racing read to smithi02284960-14 2024-09-21T09:36:43.048 INFO:tasks.rados.rados.0.smithi022.stdout:1585: finishing copy_from racing read to smithi02284960-36 2024-09-21T09:36:43.049 INFO:tasks.rados.rados.0.smithi022.stdout:1580: finishing rollback tid 1 to smithi02284960-50 2024-09-21T09:36:43.049 INFO:tasks.rados.rados.0.smithi022.stdout:1580: finishing rollback tid 2 to smithi02284960-50 2024-09-21T09:36:43.049 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 50 v 905 (ObjNum 65 snap 0 seq_num 1768384096) dirty exists 2024-09-21T09:36:43.049 INFO:tasks.rados.rados.0.smithi022.stdout:1584: expect (ObjNum 469 snap 141 seq_num 469) 2024-09-21T09:36:43.091 INFO:tasks.rados.rados.0.smithi022.stdout:1585: finishing copy_from to smithi02284960-36 2024-09-21T09:36:43.091 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 36 v 1071 (ObjNum 113 snap 0 seq_num 3288354848) dirty exists 2024-09-21T09:36:43.091 INFO:tasks.rados.rados.0.smithi022.stdout:1577: finishing copy_from to smithi02284960-14 2024-09-21T09:36:43.091 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 14 v 589 (ObjNum 437 snap 134 seq_num 437) dirty exists 2024-09-21T09:36:43.092 INFO:tasks.rados.rados.0.smithi022.stdout:1583: expect (ObjNum 513 snap 153 seq_num 513) 2024-09-21T09:36:43.230 INFO:tasks.rados.rados.0.smithi022.stdout:1582: expect (ObjNum 515 snap 154 seq_num 515) 2024-09-21T09:36:43.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:42 smithi022 ceph-mon[98615]: osdmap e381: 8 total, 8 up, 8 in 2024-09-21T09:36:43.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:42 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:36:43.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:42 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:36:43.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:42 smithi022 ceph-mon[96736]: osdmap e381: 8 total, 8 up, 8 in 2024-09-21T09:36:43.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:42 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:36:43.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:42 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:36:43.326 INFO:tasks.rados.rados.0.smithi022.stdout:1588: finishing write tid 1 to smithi02284960-12 2024-09-21T09:36:43.326 INFO:tasks.rados.rados.0.smithi022.stdout:1588: finishing write tid 2 to smithi02284960-12 2024-09-21T09:36:43.326 INFO:tasks.rados.rados.0.smithi022.stdout:1588: finishing write tid 3 to smithi02284960-12 2024-09-21T09:36:43.326 INFO:tasks.rados.rados.0.smithi022.stdout:1586: finishing copy_from racing read to smithi02284960-11 2024-09-21T09:36:43.326 INFO:tasks.rados.rados.0.smithi022.stdout:1588: finishing write tid 4 to smithi02284960-12 2024-09-21T09:36:43.326 INFO:tasks.rados.rados.0.smithi022.stdout:1588: finishing write tid 5 to smithi02284960-12 2024-09-21T09:36:43.326 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 12 v 1076 (ObjNum 518 snap 154 seq_num 518) dirty exists 2024-09-21T09:36:43.326 INFO:tasks.rados.rados.0.smithi022.stdout:1588: left oid 12 (ObjNum 518 snap 154 seq_num 518) 2024-09-21T09:36:43.326 INFO:tasks.rados.rados.0.smithi022.stdout:1586: finishing copy_from to smithi02284960-11 2024-09-21T09:36:43.327 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 11 v 802 (ObjNum 423 snap 132 seq_num 423) dirty exists 2024-09-21T09:36:43.327 INFO:tasks.rados.rados.0.smithi022.stdout:1589: seq_num 519 ranges {0=696320,696320=548864,1245184=253952} 2024-09-21T09:36:43.329 INFO:tasks.rados.rados.0.smithi022.stdout:1589: writing smithi02284960-7 from 0 to 696320 tid 1 2024-09-21T09:36:43.333 INFO:tasks.rados.rados.0.smithi022.stdout:1589: writing smithi02284960-7 from 696320 to 1245184 tid 2 2024-09-21T09:36:43.335 INFO:tasks.rados.rados.0.smithi022.stdout:1589: writing smithi02284960-7 from 1245184 to 1499136 tid 3 2024-09-21T09:36:43.336 INFO:tasks.rados.rados.0.smithi022.stdout:1577: done (8 left) 2024-09-21T09:36:43.336 INFO:tasks.rados.rados.0.smithi022.stdout:1580: done (7 left) 2024-09-21T09:36:43.336 INFO:tasks.rados.rados.0.smithi022.stdout:1582: done (6 left) 2024-09-21T09:36:43.336 INFO:tasks.rados.rados.0.smithi022.stdout:1583: done (5 left) 2024-09-21T09:36:43.336 INFO:tasks.rados.rados.0.smithi022.stdout:1584: done (4 left) 2024-09-21T09:36:43.336 INFO:tasks.rados.rados.0.smithi022.stdout:1585: done (3 left) 2024-09-21T09:36:43.336 INFO:tasks.rados.rados.0.smithi022.stdout:1586: done (2 left) 2024-09-21T09:36:43.336 INFO:tasks.rados.rados.0.smithi022.stdout:1588: done (1 left) 2024-09-21T09:36:43.336 INFO:tasks.rados.rados.0.smithi022.stdout:1590: copy_from oid 42 from oid 25 current snap is 154 2024-09-21T09:36:43.336 INFO:tasks.rados.rados.0.smithi022.stdout:1589: finishing write tid 1 to smithi02284960-7 2024-09-21T09:36:43.336 INFO:tasks.rados.rados.0.smithi022.stdout:1591: read oid 14 snap -1 2024-09-21T09:36:43.336 INFO:tasks.rados.rados.0.smithi022.stdout:1591: expect (ObjNum 437 snap 134 seq_num 437) 2024-09-21T09:36:43.337 INFO:tasks.rados.rados.0.smithi022.stdout:1592: read oid 18 snap -1 2024-09-21T09:36:43.337 INFO:tasks.rados.rados.0.smithi022.stdout:1592: expect (ObjNum 482 snap 146 seq_num 482) 2024-09-21T09:36:43.337 INFO:tasks.rados.rados.0.smithi022.stdout:1593: delete oid 10 current snap is 154 2024-09-21T09:36:43.337 INFO:tasks.rados.rados.0.smithi022.stdout:1589: finishing write tid 2 to smithi02284960-7 2024-09-21T09:36:43.338 INFO:tasks.rados.rados.0.smithi022.stdout:1589: finishing write tid 3 to smithi02284960-7 2024-09-21T09:36:43.338 INFO:tasks.rados.rados.0.smithi022.stdout:1589: finishing write tid 4 to smithi02284960-7 2024-09-21T09:36:43.338 INFO:tasks.rados.rados.0.smithi022.stdout:1592: expect (ObjNum 482 snap 146 seq_num 482) 2024-09-21T09:36:43.364 INFO:tasks.rados.rados.0.smithi022.stdout:1589: finishing write tid 5 to smithi02284960-7 2024-09-21T09:36:43.364 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 7 v 1004 (ObjNum 519 snap 154 seq_num 519) dirty exists 2024-09-21T09:36:43.364 INFO:tasks.rados.rados.0.smithi022.stdout:1589: left oid 7 (ObjNum 519 snap 154 seq_num 519) 2024-09-21T09:36:43.365 INFO:tasks.rados.rados.0.smithi022.stdout:1591: expect (ObjNum 437 snap 134 seq_num 437) 2024-09-21T09:36:43.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:43 smithi064 ceph-mon[86709]: osdmap e381: 8 total, 8 up, 8 in 2024-09-21T09:36:43.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:43 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:36:43.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:43 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:36:43.457 INFO:tasks.rados.rados.0.smithi022.stdout:1590: finishing copy_from racing read to smithi02284960-42 2024-09-21T09:36:43.458 INFO:tasks.rados.rados.0.smithi022.stdout:1590: finishing copy_from to smithi02284960-42 2024-09-21T09:36:43.458 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 42 v 934 (ObjNum 428 snap 133 seq_num 428) dirty exists 2024-09-21T09:36:43.458 INFO:tasks.rados.rados.0.smithi022.stdout:1589: done (4 left) 2024-09-21T09:36:43.458 INFO:tasks.rados.rados.0.smithi022.stdout:1590: done (3 left) 2024-09-21T09:36:43.458 INFO:tasks.rados.rados.0.smithi022.stdout:1591: done (2 left) 2024-09-21T09:36:43.458 INFO:tasks.rados.rados.0.smithi022.stdout:1592: done (1 left) 2024-09-21T09:36:43.458 INFO:tasks.rados.rados.0.smithi022.stdout:1593: done (0 left) 2024-09-21T09:36:43.458 INFO:tasks.rados.rados.0.smithi022.stdout:1594: read oid 30 snap -1 2024-09-21T09:36:43.458 INFO:tasks.rados.rados.0.smithi022.stdout:1594: expect (ObjNum 481 snap 146 seq_num 481) 2024-09-21T09:36:43.458 INFO:tasks.rados.rados.0.smithi022.stdout:1595: read oid 43 snap -1 2024-09-21T09:36:43.458 INFO:tasks.rados.rados.0.smithi022.stdout:1595: expect deleted 2024-09-21T09:36:43.458 INFO:tasks.rados.rados.0.smithi022.stdout:1596: delete oid 29 current snap is 154 2024-09-21T09:36:43.459 INFO:tasks.rados.rados.0.smithi022.stdout:1595: done (2 left) 2024-09-21T09:36:43.459 INFO:tasks.rados.rados.0.smithi022.stdout:1596: done (1 left) 2024-09-21T09:36:43.460 INFO:tasks.rados.rados.0.smithi022.stdout:1597: read oid 36 snap 153 2024-09-21T09:36:43.460 INFO:tasks.rados.rados.0.smithi022.stdout:1597: expect deleted 2024-09-21T09:36:43.460 INFO:tasks.rados.rados.0.smithi022.stdout:1598: rollback oid 28 current snap is 154 2024-09-21T09:36:43.460 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 28 to 153 2024-09-21T09:36:43.460 INFO:tasks.rados.rados.0.smithi022.stdout:1599: snap_create 2024-09-21T09:36:43.461 INFO:tasks.rados.rados.0.smithi022.stdout:1594: expect (ObjNum 481 snap 146 seq_num 481) 2024-09-21T09:36:43.491 INFO:tasks.rados.rados.0.smithi022.stdout:1598: finishing rollback tid 0 to smithi02284960-28 2024-09-21T09:36:43.491 INFO:tasks.rados.rados.0.smithi022.stdout:1598: finishing rollback tid 1 to smithi02284960-28 2024-09-21T09:36:43.491 INFO:tasks.rados.rados.0.smithi022.stdout:1598: finishing rollback tid 2 to smithi02284960-28 2024-09-21T09:36:43.492 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 28 v 553 (ObjNum 392 snap 122 seq_num 392) dirty exists 2024-09-21T09:36:44.005 INFO:tasks.rados.rados.0.smithi022.stdout:1594: done (3 left) 2024-09-21T09:36:44.005 INFO:tasks.rados.rados.0.smithi022.stdout:1597: done (2 left) 2024-09-21T09:36:44.005 INFO:tasks.rados.rados.0.smithi022.stdout:1598: done (1 left) 2024-09-21T09:36:44.005 INFO:tasks.rados.rados.0.smithi022.stdout:1599: done (0 left) 2024-09-21T09:36:44.005 INFO:tasks.rados.rados.0.smithi022.stdout:1600: rollback oid 29 current snap is 155 2024-09-21T09:36:44.005 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 29 to 154 2024-09-21T09:36:44.005 INFO:tasks.rados.rados.0.smithi022.stdout:1601: read oid 47 snap -1 2024-09-21T09:36:44.005 INFO:tasks.rados.rados.0.smithi022.stdout:1601: expect (ObjNum 382 snap 119 seq_num 382) 2024-09-21T09:36:44.005 INFO:tasks.rados.rados.0.smithi022.stdout:1602: delete oid 13 current snap is 155 2024-09-21T09:36:44.009 INFO:tasks.rados.rados.0.smithi022.stdout:1600: finishing rollback tid 1 to smithi02284960-29 2024-09-21T09:36:44.009 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 29 v 843 (ObjNum 289 snap 0 seq_num 0) dirty dne 2024-09-21T09:36:44.011 INFO:tasks.rados.rados.0.smithi022.stdout:1600: done (2 left) 2024-09-21T09:36:44.011 INFO:tasks.rados.rados.0.smithi022.stdout:1602: done (1 left) 2024-09-21T09:36:44.012 INFO:tasks.rados.rados.0.smithi022.stdout:1603: delete oid 22 current snap is 155 2024-09-21T09:36:44.014 INFO:tasks.rados.rados.0.smithi022.stdout:1601: expect (ObjNum 382 snap 119 seq_num 382) 2024-09-21T09:36:44.061 INFO:tasks.rados.rados.0.smithi022.stdout:1601: done (1 left) 2024-09-21T09:36:44.061 INFO:tasks.rados.rados.0.smithi022.stdout:1603: done (0 left) 2024-09-21T09:36:44.061 INFO:tasks.rados.rados.0.smithi022.stdout:1604: read oid 32 snap -1 2024-09-21T09:36:44.061 INFO:tasks.rados.rados.0.smithi022.stdout:1604: expect (ObjNum 496 snap 149 seq_num 496) 2024-09-21T09:36:44.061 INFO:tasks.rados.rados.0.smithi022.stdout:1605: setattr oid 19 current snap is 155 2024-09-21T09:36:44.064 INFO:tasks.rados.rados.0.smithi022.stdout:1606: snap_remove snap 154 2024-09-21T09:36:44.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:43 smithi022 ceph-mon[98615]: pgmap v175: 121 pgs: 1 active+recovery_wait+degraded, 120 active+clean; 86 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 1/258 objects degraded (0.388%); 718 KiB/s, 0 objects/s recovering 2024-09-21T09:36:44.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:43 smithi022 ceph-mon[98615]: Health check update: Degraded data redundancy: 1/258 objects degraded (0.388%), 1 pg degraded (PG_DEGRADED) 2024-09-21T09:36:44.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:43 smithi022 ceph-mon[98615]: osdmap e382: 8 total, 8 up, 8 in 2024-09-21T09:36:44.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:43 smithi022 ceph-mon[96736]: pgmap v175: 121 pgs: 1 active+recovery_wait+degraded, 120 active+clean; 86 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 1/258 objects degraded (0.388%); 718 KiB/s, 0 objects/s recovering 2024-09-21T09:36:44.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:43 smithi022 ceph-mon[96736]: Health check update: Degraded data redundancy: 1/258 objects degraded (0.388%), 1 pg degraded (PG_DEGRADED) 2024-09-21T09:36:44.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:43 smithi022 ceph-mon[96736]: osdmap e382: 8 total, 8 up, 8 in 2024-09-21T09:36:44.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:43 smithi064 ceph-mon[86709]: pgmap v175: 121 pgs: 1 active+recovery_wait+degraded, 120 active+clean; 86 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 1/258 objects degraded (0.388%); 718 KiB/s, 0 objects/s recovering 2024-09-21T09:36:44.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:43 smithi064 ceph-mon[86709]: Health check update: Degraded data redundancy: 1/258 objects degraded (0.388%), 1 pg degraded (PG_DEGRADED) 2024-09-21T09:36:44.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:43 smithi064 ceph-mon[86709]: osdmap e382: 8 total, 8 up, 8 in 2024-09-21T09:36:44.442 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:36:44 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:36:44.147Z caller=alerting.go:391 level=warn component="rule manager" alert="unsupported value type" msg="Expanding alert template failed" err="error executing template __alert_CephOSDDown: template: __alert_CephOSDDown:1:358: executing \"__alert_CephOSDDown\" at : error calling query: found duplicate series for the match group {ceph_daemon=\"osd.1\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.1\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.1\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}];many-to-many matching not allowed: matching labels must be unique on one side" data="unsupported value type" 2024-09-21T09:36:44.442 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:36:44 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:36:44.148Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.1\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.1\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.1\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:36:45.013 INFO:tasks.rados.rados.0.smithi022.stdout:1606: done (2 left) 2024-09-21T09:36:45.013 INFO:tasks.rados.rados.0.smithi022.stdout:1607: read oid 10 snap -1 2024-09-21T09:36:45.013 INFO:tasks.rados.rados.0.smithi022.stdout:1607: expect deleted 2024-09-21T09:36:45.014 INFO:tasks.rados.rados.0.smithi022.stdout:1608: rmattr oid 50 current snap is 155 2024-09-21T09:36:45.014 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 19 v 1078 (ObjNum 507 snap 151 seq_num 507) dirty exists 2024-09-21T09:36:45.014 INFO:tasks.rados.rados.0.smithi022.stdout:1604: expect (ObjNum 496 snap 149 seq_num 496) 2024-09-21T09:36:45.113 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 50 v 1072 (ObjNum 289 snap 0 seq_num 1) dirty exists 2024-09-21T09:36:45.114 INFO:tasks.rados.rados.0.smithi022.stdout:1604: done (3 left) 2024-09-21T09:36:45.114 INFO:tasks.rados.rados.0.smithi022.stdout:1605: done (2 left) 2024-09-21T09:36:45.114 INFO:tasks.rados.rados.0.smithi022.stdout:1607: done (1 left) 2024-09-21T09:36:45.114 INFO:tasks.rados.rados.0.smithi022.stdout:1608: done (0 left) 2024-09-21T09:36:45.114 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 17 current snap is 155 2024-09-21T09:36:45.114 INFO:tasks.rados.rados.0.smithi022.stdout:1609: seq_num 524 ranges {1032192=786432,1818624=778240,2596864=24576} 2024-09-21T09:36:45.123 INFO:tasks.rados.rados.0.smithi022.stdout:1609: writing smithi02284960-17 from 1032192 to 1818624 tid 1 2024-09-21T09:36:45.128 INFO:tasks.rados.rados.0.smithi022.stdout:1609: writing smithi02284960-17 from 1818624 to 2596864 tid 2 2024-09-21T09:36:45.128 INFO:tasks.rados.rados.0.smithi022.stdout:1609: writing smithi02284960-17 from 2596864 to 2621440 tid 3 2024-09-21T09:36:45.128 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 11 current snap is 155 2024-09-21T09:36:45.128 INFO:tasks.rados.rados.0.smithi022.stdout:1609: finishing write tid 1 to smithi02284960-17 2024-09-21T09:36:45.128 INFO:tasks.rados.rados.0.smithi022.stdout:1610: seq_num 525 ranges {2908160=802816,3710976=712704,4423680=90112} 2024-09-21T09:36:45.148 INFO:tasks.rados.rados.0.smithi022.stdout:1610: writing smithi02284960-11 from 2908160 to 3710976 tid 1 2024-09-21T09:36:45.153 INFO:tasks.rados.rados.0.smithi022.stdout:1610: writing smithi02284960-11 from 3710976 to 4423680 tid 2 2024-09-21T09:36:45.154 INFO:tasks.rados.rados.0.smithi022.stdout:1610: writing smithi02284960-11 from 4423680 to 4513792 tid 3 2024-09-21T09:36:45.154 INFO:tasks.rados.rados.0.smithi022.stdout:1611: snap_create 2024-09-21T09:36:45.154 INFO:tasks.rados.rados.0.smithi022.stdout:1609: finishing write tid 2 to smithi02284960-17 2024-09-21T09:36:45.154 INFO:tasks.rados.rados.0.smithi022.stdout:1609: finishing write tid 3 to smithi02284960-17 2024-09-21T09:36:45.154 INFO:tasks.rados.rados.0.smithi022.stdout:1609: finishing write tid 4 to smithi02284960-17 2024-09-21T09:36:45.154 INFO:tasks.rados.rados.0.smithi022.stdout:1609: finishing write tid 5 to smithi02284960-17 2024-09-21T09:36:45.154 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 17 v 914 (ObjNum 524 snap 155 seq_num 524) dirty exists 2024-09-21T09:36:45.154 INFO:tasks.rados.rados.0.smithi022.stdout:1609: left oid 17 (ObjNum 524 snap 155 seq_num 524) 2024-09-21T09:36:45.154 INFO:tasks.rados.rados.0.smithi022.stdout:1610: finishing write tid 1 to smithi02284960-11 2024-09-21T09:36:45.157 INFO:tasks.rados.rados.0.smithi022.stdout:1610: finishing write tid 2 to smithi02284960-11 2024-09-21T09:36:45.157 INFO:tasks.rados.rados.0.smithi022.stdout:1610: finishing write tid 3 to smithi02284960-11 2024-09-21T09:36:45.157 INFO:tasks.rados.rados.0.smithi022.stdout:1610: finishing write tid 4 to smithi02284960-11 2024-09-21T09:36:45.158 INFO:tasks.rados.rados.0.smithi022.stdout:1610: finishing write tid 5 to smithi02284960-11 2024-09-21T09:36:45.158 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 11 v 988 (ObjNum 525 snap 155 seq_num 525) dirty exists 2024-09-21T09:36:45.158 INFO:tasks.rados.rados.0.smithi022.stdout:1610: left oid 11 (ObjNum 525 snap 155 seq_num 525) 2024-09-21T09:36:45.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:45 smithi022 ceph-mon[98615]: osdmap e383: 8 total, 8 up, 8 in 2024-09-21T09:36:45.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:45 smithi022 ceph-mon[96736]: osdmap e383: 8 total, 8 up, 8 in 2024-09-21T09:36:45.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:45 smithi064 ceph-mon[86709]: osdmap e383: 8 total, 8 up, 8 in 2024-09-21T09:36:46.010 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:36:45 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:36:45] "GET /metrics HTTP/1.1" 200 36241 "" "Prometheus/2.51.0" 2024-09-21T09:36:46.010 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:36:45 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:36:45.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:36:46.010 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:36:45 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:36:45.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:36:46.017 INFO:tasks.rados.rados.0.smithi022.stdout:1609: done (2 left) 2024-09-21T09:36:46.017 INFO:tasks.rados.rados.0.smithi022.stdout:1610: done (1 left) 2024-09-21T09:36:46.017 INFO:tasks.rados.rados.0.smithi022.stdout:1611: done (0 left) 2024-09-21T09:36:46.017 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 9 current snap is 156 2024-09-21T09:36:46.018 INFO:tasks.rados.rados.0.smithi022.stdout:1612: seq_num 526 ranges {1351680=417792,1769472=589824,2359296=614400} 2024-09-21T09:36:46.027 INFO:tasks.rados.rados.0.smithi022.stdout:1612: writing smithi02284960-9 from 1351680 to 1769472 tid 1 2024-09-21T09:36:46.030 INFO:tasks.rados.rados.0.smithi022.stdout:1612: writing smithi02284960-9 from 1769472 to 2359296 tid 2 2024-09-21T09:36:46.034 INFO:tasks.rados.rados.0.smithi022.stdout:1612: writing smithi02284960-9 from 2359296 to 2973696 tid 3 2024-09-21T09:36:46.034 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 10 current snap is 156 2024-09-21T09:36:46.035 INFO:tasks.rados.rados.0.smithi022.stdout:1612: finishing write tid 1 to smithi02284960-9 2024-09-21T09:36:46.035 INFO:tasks.rados.rados.0.smithi022.stdout:1612: finishing write tid 2 to smithi02284960-9 2024-09-21T09:36:46.035 INFO:tasks.rados.rados.0.smithi022.stdout:1613: seq_num 527 ranges {0=434176,434176=475136,909312=565248,1474560=163840} 2024-09-21T09:36:46.037 INFO:tasks.rados.rados.0.smithi022.stdout:1613: writing smithi02284960-10 from 0 to 434176 tid 1 2024-09-21T09:36:46.040 INFO:tasks.rados.rados.0.smithi022.stdout:1613: writing smithi02284960-10 from 434176 to 909312 tid 2 2024-09-21T09:36:46.043 INFO:tasks.rados.rados.0.smithi022.stdout:1613: writing smithi02284960-10 from 909312 to 1474560 tid 3 2024-09-21T09:36:46.045 INFO:tasks.rados.rados.0.smithi022.stdout:1613: writing smithi02284960-10 from 1474560 to 1638400 tid 4 2024-09-21T09:36:46.045 INFO:tasks.rados.rados.0.smithi022.stdout:1614: copy_from oid 6 from oid 7 current snap is 156 2024-09-21T09:36:46.045 INFO:tasks.rados.rados.0.smithi022.stdout:1612: finishing write tid 3 to smithi02284960-9 2024-09-21T09:36:46.045 INFO:tasks.rados.rados.0.smithi022.stdout:1612: finishing write tid 4 to smithi02284960-9 2024-09-21T09:36:46.045 INFO:tasks.rados.rados.0.smithi022.stdout:1612: finishing write tid 5 to smithi02284960-9 2024-09-21T09:36:46.045 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 9 v 598 (ObjNum 526 snap 156 seq_num 526) dirty exists 2024-09-21T09:36:46.045 INFO:tasks.rados.rados.0.smithi022.stdout:1612: left oid 9 (ObjNum 526 snap 156 seq_num 526) 2024-09-21T09:36:46.045 INFO:tasks.rados.rados.0.smithi022.stdout:1612: done (2 left) 2024-09-21T09:36:46.045 INFO:tasks.rados.rados.0.smithi022.stdout:1615: rmattr oid 50 current snap is 156 2024-09-21T09:36:46.045 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 4 current snap is 156 2024-09-21T09:36:46.046 INFO:tasks.rados.rados.0.smithi022.stdout:1616: seq_num 528 ranges {1384448=450560,1835008=761856,2596864=442368} 2024-09-21T09:36:46.055 INFO:tasks.rados.rados.0.smithi022.stdout:1616: writing smithi02284960-4 from 1384448 to 1835008 tid 1 2024-09-21T09:36:46.059 INFO:tasks.rados.rados.0.smithi022.stdout:1616: writing smithi02284960-4 from 1835008 to 2596864 tid 2 2024-09-21T09:36:46.062 INFO:tasks.rados.rados.0.smithi022.stdout:1616: writing smithi02284960-4 from 2596864 to 3039232 tid 3 2024-09-21T09:36:46.062 INFO:tasks.rados.rados.0.smithi022.stdout:1617: delete oid 15 current snap is 156 2024-09-21T09:36:46.063 INFO:tasks.rados.rados.0.smithi022.stdout:1613: finishing write tid 1 to smithi02284960-10 2024-09-21T09:36:46.063 INFO:tasks.rados.rados.0.smithi022.stdout:1613: finishing write tid 2 to smithi02284960-10 2024-09-21T09:36:46.063 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 50 v 1073 (ObjNum 65 snap 0 seq_num 1768384096) dirty exists 2024-09-21T09:36:46.063 INFO:tasks.rados.rados.0.smithi022.stdout:1613: finishing write tid 3 to smithi02284960-10 2024-09-21T09:36:46.063 INFO:tasks.rados.rados.0.smithi022.stdout:1613: finishing write tid 4 to smithi02284960-10 2024-09-21T09:36:46.063 INFO:tasks.rados.rados.0.smithi022.stdout:1613: finishing write tid 5 to smithi02284960-10 2024-09-21T09:36:46.063 INFO:tasks.rados.rados.0.smithi022.stdout:1613: finishing write tid 6 to smithi02284960-10 2024-09-21T09:36:46.063 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 10 v 943 (ObjNum 527 snap 156 seq_num 527) dirty exists 2024-09-21T09:36:46.063 INFO:tasks.rados.rados.0.smithi022.stdout:1613: left oid 10 (ObjNum 527 snap 156 seq_num 527) 2024-09-21T09:36:46.063 INFO:tasks.rados.rados.0.smithi022.stdout:1614: finishing copy_from racing read to smithi02284960-6 2024-09-21T09:36:46.063 INFO:tasks.rados.rados.0.smithi022.stdout:1614: finishing copy_from to smithi02284960-6 2024-09-21T09:36:46.063 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 6 v 1004 (ObjNum 519 snap 154 seq_num 519) dirty exists 2024-09-21T09:36:46.064 INFO:tasks.rados.rados.0.smithi022.stdout:1616: finishing write tid 1 to smithi02284960-4 2024-09-21T09:36:46.064 INFO:tasks.rados.rados.0.smithi022.stdout:1616: finishing write tid 2 to smithi02284960-4 2024-09-21T09:36:46.064 INFO:tasks.rados.rados.0.smithi022.stdout:1616: finishing write tid 3 to smithi02284960-4 2024-09-21T09:36:46.065 INFO:tasks.rados.rados.0.smithi022.stdout:1616: finishing write tid 4 to smithi02284960-4 2024-09-21T09:36:46.067 INFO:tasks.rados.rados.0.smithi022.stdout:1616: finishing write tid 5 to smithi02284960-4 2024-09-21T09:36:46.067 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 4 v 603 (ObjNum 528 snap 156 seq_num 528) dirty exists 2024-09-21T09:36:46.067 INFO:tasks.rados.rados.0.smithi022.stdout:1616: left oid 4 (ObjNum 528 snap 156 seq_num 528) 2024-09-21T09:36:46.067 INFO:tasks.rados.rados.0.smithi022.stdout:1613: done (4 left) 2024-09-21T09:36:46.067 INFO:tasks.rados.rados.0.smithi022.stdout:1614: done (3 left) 2024-09-21T09:36:46.067 INFO:tasks.rados.rados.0.smithi022.stdout:1615: done (2 left) 2024-09-21T09:36:46.067 INFO:tasks.rados.rados.0.smithi022.stdout:1616: done (1 left) 2024-09-21T09:36:46.067 INFO:tasks.rados.rados.0.smithi022.stdout:1617: done (0 left) 2024-09-21T09:36:46.067 INFO:tasks.rados.rados.0.smithi022.stdout:1618: read oid 38 snap -1 2024-09-21T09:36:46.067 INFO:tasks.rados.rados.0.smithi022.stdout:1618: expect (ObjNum 485 snap 146 seq_num 485) 2024-09-21T09:36:46.067 INFO:tasks.rados.rados.0.smithi022.stdout:1619: snap_create 2024-09-21T09:36:46.072 INFO:tasks.rados.rados.0.smithi022.stdout:1618: expect (ObjNum 485 snap 146 seq_num 485) 2024-09-21T09:36:46.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:46 smithi022 ceph-mon[98615]: pgmap v178: 121 pgs: 121 active+clean; 89 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 4.7 MiB/s wr, 35 op/s; 718 KiB/s, 0 objects/s recovering 2024-09-21T09:36:46.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:46 smithi022 ceph-mon[98615]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/258 objects degraded (0.388%), 1 pg degraded) 2024-09-21T09:36:46.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:46 smithi022 ceph-mon[98615]: Cluster is now healthy 2024-09-21T09:36:46.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:46 smithi022 ceph-mon[98615]: osdmap e384: 8 total, 8 up, 8 in 2024-09-21T09:36:46.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:46 smithi022 ceph-mon[96736]: pgmap v178: 121 pgs: 121 active+clean; 89 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 4.7 MiB/s wr, 35 op/s; 718 KiB/s, 0 objects/s recovering 2024-09-21T09:36:46.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:46 smithi022 ceph-mon[96736]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/258 objects degraded (0.388%), 1 pg degraded) 2024-09-21T09:36:46.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:46 smithi022 ceph-mon[96736]: Cluster is now healthy 2024-09-21T09:36:46.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:46 smithi022 ceph-mon[96736]: osdmap e384: 8 total, 8 up, 8 in 2024-09-21T09:36:46.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:46 smithi064 ceph-mon[86709]: pgmap v178: 121 pgs: 121 active+clean; 89 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 4.7 MiB/s wr, 35 op/s; 718 KiB/s, 0 objects/s recovering 2024-09-21T09:36:46.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:46 smithi064 ceph-mon[86709]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/258 objects degraded (0.388%), 1 pg degraded) 2024-09-21T09:36:46.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:46 smithi064 ceph-mon[86709]: Cluster is now healthy 2024-09-21T09:36:46.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:46 smithi064 ceph-mon[86709]: osdmap e384: 8 total, 8 up, 8 in 2024-09-21T09:36:47.024 INFO:tasks.rados.rados.0.smithi022.stdout:1618: done (1 left) 2024-09-21T09:36:47.024 INFO:tasks.rados.rados.0.smithi022.stdout:1619: done (0 left) 2024-09-21T09:36:47.024 INFO:tasks.rados.rados.0.smithi022.stdout:1620: delete oid 24 current snap is 157 2024-09-21T09:36:47.031 INFO:tasks.rados.rados.0.smithi022.stdout:1620: done (0 left) 2024-09-21T09:36:47.031 INFO:tasks.rados.rados.0.smithi022.stdout:1621: snap_create 2024-09-21T09:36:47.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:47 smithi022 ceph-mon[98615]: osdmap e385: 8 total, 8 up, 8 in 2024-09-21T09:36:47.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:47 smithi022 ceph-mon[96736]: osdmap e385: 8 total, 8 up, 8 in 2024-09-21T09:36:47.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:47 smithi064 ceph-mon[86709]: osdmap e385: 8 total, 8 up, 8 in 2024-09-21T09:36:47.442 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:36:46 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:36:46.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi022\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:36:48.025 INFO:tasks.rados.rados.0.smithi022.stdout:1621: done (0 left) 2024-09-21T09:36:48.025 INFO:tasks.rados.rados.0.smithi022.stdout:1622: snap_remove snap 153 2024-09-21T09:36:48.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:48 smithi022 ceph-mon[98615]: pgmap v181: 121 pgs: 121 active+clean; 90 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 5.1 MiB/s wr, 38 op/s; 0 B/s, 0 objects/s recovering 2024-09-21T09:36:48.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:48 smithi022 ceph-mon[98615]: osdmap e386: 8 total, 8 up, 8 in 2024-09-21T09:36:48.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:48 smithi022 ceph-mon[96736]: pgmap v181: 121 pgs: 121 active+clean; 90 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 5.1 MiB/s wr, 38 op/s; 0 B/s, 0 objects/s recovering 2024-09-21T09:36:48.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:48 smithi022 ceph-mon[96736]: osdmap e386: 8 total, 8 up, 8 in 2024-09-21T09:36:48.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:48 smithi064 ceph-mon[86709]: pgmap v181: 121 pgs: 121 active+clean; 90 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 5.1 MiB/s wr, 38 op/s; 0 B/s, 0 objects/s recovering 2024-09-21T09:36:48.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:48 smithi064 ceph-mon[86709]: osdmap e386: 8 total, 8 up, 8 in 2024-09-21T09:36:48.964 INFO:tasks.rados.rados.0.smithi022.stdout:1622: done (0 left) 2024-09-21T09:36:48.964 INFO:tasks.rados.rados.0.smithi022.stdout:1623: snap_create 2024-09-21T09:36:49.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:49 smithi022 ceph-mon[98615]: osdmap e387: 8 total, 8 up, 8 in 2024-09-21T09:36:49.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:49 smithi022 ceph-mon[98615]: osdmap e388: 8 total, 8 up, 8 in 2024-09-21T09:36:49.310 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:49 smithi022 ceph-mon[96736]: osdmap e387: 8 total, 8 up, 8 in 2024-09-21T09:36:49.310 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:49 smithi022 ceph-mon[96736]: osdmap e388: 8 total, 8 up, 8 in 2024-09-21T09:36:49.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:49 smithi064 ceph-mon[86709]: osdmap e387: 8 total, 8 up, 8 in 2024-09-21T09:36:49.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:49 smithi064 ceph-mon[86709]: osdmap e388: 8 total, 8 up, 8 in 2024-09-21T09:36:49.967 INFO:tasks.rados.rados.0.smithi022.stdout:1623: done (0 left) 2024-09-21T09:36:49.967 INFO:tasks.rados.rados.0.smithi022.stdout:1624: read oid 11 snap -1 2024-09-21T09:36:49.967 INFO:tasks.rados.rados.0.smithi022.stdout:1624: expect (ObjNum 525 snap 155 seq_num 525) 2024-09-21T09:36:49.967 INFO:tasks.rados.rados.0.smithi022.stdout:1625: rollback oid 4 current snap is 159 2024-09-21T09:36:49.967 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 4 to 158 2024-09-21T09:36:49.967 INFO:tasks.rados.rados.0.smithi022.stdout:1626: rollback oid 26 current snap is 159 2024-09-21T09:36:49.967 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 26 to 155 2024-09-21T09:36:49.967 INFO:tasks.rados.rados.0.smithi022.stdout:1627: copy_from oid 12 from oid 6 current snap is 159 2024-09-21T09:36:49.967 INFO:tasks.rados.rados.0.smithi022.stdout:1628: rmattr oid 31 current snap is 159 2024-09-21T09:36:49.968 INFO:tasks.rados.rados.0.smithi022.stdout:1629: setattr oid 32 current snap is 159 2024-09-21T09:36:49.968 INFO:tasks.rados.rados.0.smithi022.stdout:1630: rollback oid 37 current snap is 159 2024-09-21T09:36:49.968 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 37 to 156 2024-09-21T09:36:49.968 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 41 current snap is 159 2024-09-21T09:36:49.968 INFO:tasks.rados.rados.0.smithi022.stdout:1631: seq_num 531 ranges {0=499712,499712=417792,917504=548864,1466368=237568} 2024-09-21T09:36:49.974 INFO:tasks.rados.rados.0.smithi022.stdout:1631: writing smithi02284960-41 from 0 to 499712 tid 1 2024-09-21T09:36:49.975 INFO:tasks.rados.rados.0.smithi022.stdout:1631: writing smithi02284960-41 from 499712 to 917504 tid 2 2024-09-21T09:36:49.978 INFO:tasks.rados.rados.0.smithi022.stdout:1631: writing smithi02284960-41 from 917504 to 1466368 tid 3 2024-09-21T09:36:49.980 INFO:tasks.rados.rados.0.smithi022.stdout:1631: writing smithi02284960-41 from 1466368 to 1703936 tid 4 2024-09-21T09:36:49.980 INFO:tasks.rados.rados.0.smithi022.stdout:1632: snap_remove snap 157 2024-09-21T09:36:50.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:50 smithi022 ceph-mon[96736]: pgmap v184: 121 pgs: 121 active+clean; 96 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 4.5 MiB/s rd, 2.3 MiB/s wr, 22 op/s 2024-09-21T09:36:50.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:50 smithi022 ceph-mon[96736]: osdmap e389: 8 total, 8 up, 8 in 2024-09-21T09:36:50.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:50 smithi022 ceph-mon[98615]: pgmap v184: 121 pgs: 121 active+clean; 96 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 4.5 MiB/s rd, 2.3 MiB/s wr, 22 op/s 2024-09-21T09:36:50.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:50 smithi022 ceph-mon[98615]: osdmap e389: 8 total, 8 up, 8 in 2024-09-21T09:36:50.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:50 smithi064 ceph-mon[86709]: pgmap v184: 121 pgs: 121 active+clean; 96 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 4.5 MiB/s rd, 2.3 MiB/s wr, 22 op/s 2024-09-21T09:36:50.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:50 smithi064 ceph-mon[86709]: osdmap e389: 8 total, 8 up, 8 in 2024-09-21T09:36:50.442 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:36:50 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:36:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:36:50.967 INFO:tasks.rados.rados.0.smithi022.stdout:1632: done (8 left) 2024-09-21T09:36:50.968 INFO:tasks.rados.rados.0.smithi022.stdout:1633: snap_create 2024-09-21T09:36:50.968 INFO:tasks.rados.rados.0.smithi022.stdout:1625: finishing rollback tid 0 to smithi02284960-4 2024-09-21T09:36:50.968 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 32 v 1009 (ObjNum 496 snap 149 seq_num 496) dirty exists 2024-09-21T09:36:50.968 INFO:tasks.rados.rados.0.smithi022.stdout:1626: finishing rollback tid 0 to smithi02284960-26 2024-09-21T09:36:50.968 INFO:tasks.rados.rados.0.smithi022.stdout:1626: finishing rollback tid 1 to smithi02284960-26 2024-09-21T09:36:50.968 INFO:tasks.rados.rados.0.smithi022.stdout:1626: finishing rollback tid 2 to smithi02284960-26 2024-09-21T09:36:50.968 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 26 v 939 (ObjNum 450 snap 136 seq_num 450) dirty exists 2024-09-21T09:36:50.968 INFO:tasks.rados.rados.0.smithi022.stdout:1625: finishing rollback tid 1 to smithi02284960-4 2024-09-21T09:36:50.968 INFO:tasks.rados.rados.0.smithi022.stdout:1625: finishing rollback tid 2 to smithi02284960-4 2024-09-21T09:36:50.968 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 4 v 611 (ObjNum 528 snap 156 seq_num 528) dirty exists 2024-09-21T09:36:50.968 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 31 v 1094 (ObjNum 289 snap 0 seq_num 0) dirty exists 2024-09-21T09:36:50.968 INFO:tasks.rados.rados.0.smithi022.stdout:1631: finishing write tid 1 to smithi02284960-41 2024-09-21T09:36:50.968 INFO:tasks.rados.rados.0.smithi022.stdout:1631: finishing write tid 2 to smithi02284960-41 2024-09-21T09:36:50.968 INFO:tasks.rados.rados.0.smithi022.stdout:1631: finishing write tid 3 to smithi02284960-41 2024-09-21T09:36:50.968 INFO:tasks.rados.rados.0.smithi022.stdout:1631: finishing write tid 4 to smithi02284960-41 2024-09-21T09:36:50.969 INFO:tasks.rados.rados.0.smithi022.stdout:1627: finishing copy_from racing read to smithi02284960-12 2024-09-21T09:36:50.969 INFO:tasks.rados.rados.0.smithi022.stdout:1630: finishing rollback tid 0 to smithi02284960-37 2024-09-21T09:36:50.969 INFO:tasks.rados.rados.0.smithi022.stdout:1631: finishing write tid 5 to smithi02284960-41 2024-09-21T09:36:50.969 INFO:tasks.rados.rados.0.smithi022.stdout:1630: finishing rollback tid 1 to smithi02284960-37 2024-09-21T09:36:50.969 INFO:tasks.rados.rados.0.smithi022.stdout:1631: finishing write tid 6 to smithi02284960-41 2024-09-21T09:36:50.969 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 41 v 954 (ObjNum 531 snap 159 seq_num 531) dirty exists 2024-09-21T09:36:50.969 INFO:tasks.rados.rados.0.smithi022.stdout:1631: left oid 41 (ObjNum 531 snap 159 seq_num 531) 2024-09-21T09:36:50.969 INFO:tasks.rados.rados.0.smithi022.stdout:1630: finishing rollback tid 2 to smithi02284960-37 2024-09-21T09:36:50.969 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 37 v 1098 (ObjNum 514 snap 153 seq_num 514) dirty exists 2024-09-21T09:36:50.969 INFO:tasks.rados.rados.0.smithi022.stdout:1627: finishing copy_from to smithi02284960-12 2024-09-21T09:36:50.969 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 12 v 1004 (ObjNum 519 snap 154 seq_num 519) dirty exists 2024-09-21T09:36:50.969 INFO:tasks.rados.rados.0.smithi022.stdout:1624: expect (ObjNum 525 snap 155 seq_num 525) 2024-09-21T09:36:51.970 INFO:tasks.rados.rados.0.smithi022.stdout:1624: done (8 left) 2024-09-21T09:36:51.970 INFO:tasks.rados.rados.0.smithi022.stdout:1625: done (7 left) 2024-09-21T09:36:51.970 INFO:tasks.rados.rados.0.smithi022.stdout:1626: done (6 left) 2024-09-21T09:36:51.971 INFO:tasks.rados.rados.0.smithi022.stdout:1627: done (5 left) 2024-09-21T09:36:51.971 INFO:tasks.rados.rados.0.smithi022.stdout:1628: done (4 left) 2024-09-21T09:36:51.971 INFO:tasks.rados.rados.0.smithi022.stdout:1629: done (3 left) 2024-09-21T09:36:51.971 INFO:tasks.rados.rados.0.smithi022.stdout:1630: done (2 left) 2024-09-21T09:36:51.971 INFO:tasks.rados.rados.0.smithi022.stdout:1631: done (1 left) 2024-09-21T09:36:51.971 INFO:tasks.rados.rados.0.smithi022.stdout:1633: done (0 left) 2024-09-21T09:36:51.971 INFO:tasks.rados.rados.0.smithi022.stdout:1634: delete oid 13 current snap is 160 2024-09-21T09:36:51.977 INFO:tasks.rados.rados.0.smithi022.stdout:1634: done (0 left) 2024-09-21T09:36:51.977 INFO:tasks.rados.rados.0.smithi022.stdout:1635: setattr oid 18 current snap is 160 2024-09-21T09:36:51.980 INFO:tasks.rados.rados.0.smithi022.stdout:1636: delete oid 22 current snap is 160 2024-09-21T09:36:51.981 INFO:tasks.rados.rados.0.smithi022.stdout:1636: done (1 left) 2024-09-21T09:36:51.981 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 6 current snap is 160 2024-09-21T09:36:51.982 INFO:tasks.rados.rados.0.smithi022.stdout:1637: seq_num 534 ranges {1499136=548864,2048000=516096,2564096=688128} 2024-09-21T09:36:51.993 INFO:tasks.rados.rados.0.smithi022.stdout:1637: writing smithi02284960-6 from 1499136 to 2048000 tid 1 2024-09-21T09:36:51.996 INFO:tasks.rados.rados.0.smithi022.stdout:1637: writing smithi02284960-6 from 2048000 to 2564096 tid 2 2024-09-21T09:36:52.000 INFO:tasks.rados.rados.0.smithi022.stdout:1637: writing smithi02284960-6 from 2564096 to 3252224 tid 3 2024-09-21T09:36:52.000 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 20 current snap is 160 2024-09-21T09:36:52.000 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 18 v 948 (ObjNum 482 snap 146 seq_num 482) dirty exists 2024-09-21T09:36:52.000 INFO:tasks.rados.rados.0.smithi022.stdout:1638: seq_num 535 ranges {5390336=565248,5955584=802816,6758400=401408} 2024-09-21T09:36:52.033 INFO:tasks.rados.rados.0.smithi022.stdout:1638: writing smithi02284960-20 from 5390336 to 5955584 tid 1 2024-09-21T09:36:52.038 INFO:tasks.rados.rados.0.smithi022.stdout:1638: writing smithi02284960-20 from 5955584 to 6758400 tid 2 2024-09-21T09:36:52.040 INFO:tasks.rados.rados.0.smithi022.stdout:1638: writing smithi02284960-20 from 6758400 to 7159808 tid 3 2024-09-21T09:36:52.040 INFO:tasks.rados.rados.0.smithi022.stdout:1635: done (2 left) 2024-09-21T09:36:52.040 INFO:tasks.rados.rados.0.smithi022.stdout:1639: snap_remove snap 159 2024-09-21T09:36:52.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:51 smithi022 ceph-mon[96736]: pgmap v187: 121 pgs: 121 active+clean; 96 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3.8 MiB/s rd, 1.9 MiB/s wr, 20 op/s 2024-09-21T09:36:52.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:51 smithi022 ceph-mon[96736]: osdmap e390: 8 total, 8 up, 8 in 2024-09-21T09:36:52.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:51 smithi022 ceph-mon[98615]: pgmap v187: 121 pgs: 121 active+clean; 96 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3.8 MiB/s rd, 1.9 MiB/s wr, 20 op/s 2024-09-21T09:36:52.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:51 smithi022 ceph-mon[98615]: osdmap e390: 8 total, 8 up, 8 in 2024-09-21T09:36:52.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:51 smithi064 ceph-mon[86709]: pgmap v187: 121 pgs: 121 active+clean; 96 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3.8 MiB/s rd, 1.9 MiB/s wr, 20 op/s 2024-09-21T09:36:52.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:51 smithi064 ceph-mon[86709]: osdmap e390: 8 total, 8 up, 8 in 2024-09-21T09:36:52.982 INFO:tasks.rados.rados.0.smithi022.stdout:1639: done (2 left) 2024-09-21T09:36:52.982 INFO:tasks.rados.rados.0.smithi022.stdout:1640: snap_remove snap 155 2024-09-21T09:36:53.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:52 smithi022 ceph-mon[96736]: osdmap e391: 8 total, 8 up, 8 in 2024-09-21T09:36:53.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:52 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:36:53.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:52 smithi022 ceph-mon[98615]: osdmap e391: 8 total, 8 up, 8 in 2024-09-21T09:36:53.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:52 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:36:53.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:52 smithi064 ceph-mon[86709]: osdmap e391: 8 total, 8 up, 8 in 2024-09-21T09:36:53.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:52 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:36:53.965 INFO:tasks.rados.rados.0.smithi022.stdout:1640: done (2 left) 2024-09-21T09:36:53.965 INFO:tasks.rados.rados.0.smithi022.stdout:1641: snap_create 2024-09-21T09:36:53.965 INFO:tasks.rados.rados.0.smithi022.stdout:1637: finishing write tid 1 to smithi02284960-6 2024-09-21T09:36:53.965 INFO:tasks.rados.rados.0.smithi022.stdout:1637: finishing write tid 2 to smithi02284960-6 2024-09-21T09:36:53.965 INFO:tasks.rados.rados.0.smithi022.stdout:1637: finishing write tid 3 to smithi02284960-6 2024-09-21T09:36:53.965 INFO:tasks.rados.rados.0.smithi022.stdout:1637: finishing write tid 4 to smithi02284960-6 2024-09-21T09:36:53.965 INFO:tasks.rados.rados.0.smithi022.stdout:1637: finishing write tid 5 to smithi02284960-6 2024-09-21T09:36:53.965 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 6 v 1016 (ObjNum 534 snap 160 seq_num 534) dirty exists 2024-09-21T09:36:53.965 INFO:tasks.rados.rados.0.smithi022.stdout:1637: left oid 6 (ObjNum 534 snap 160 seq_num 534) 2024-09-21T09:36:53.965 INFO:tasks.rados.rados.0.smithi022.stdout:1638: finishing write tid 1 to smithi02284960-20 2024-09-21T09:36:53.965 INFO:tasks.rados.rados.0.smithi022.stdout:1638: finishing write tid 2 to smithi02284960-20 2024-09-21T09:36:53.965 INFO:tasks.rados.rados.0.smithi022.stdout:1638: finishing write tid 3 to smithi02284960-20 2024-09-21T09:36:53.965 INFO:tasks.rados.rados.0.smithi022.stdout:1638: finishing write tid 4 to smithi02284960-20 2024-09-21T09:36:53.965 INFO:tasks.rados.rados.0.smithi022.stdout:1638: finishing write tid 5 to smithi02284960-20 2024-09-21T09:36:53.966 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 20 v 1021 (ObjNum 535 snap 160 seq_num 535) dirty exists 2024-09-21T09:36:53.966 INFO:tasks.rados.rados.0.smithi022.stdout:1638: left oid 20 (ObjNum 535 snap 160 seq_num 535) 2024-09-21T09:36:54.196 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:53 smithi022 ceph-mon[98615]: pgmap v190: 121 pgs: 121 active+clean; 96 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:36:54.196 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:53 smithi022 ceph-mon[98615]: osdmap e392: 8 total, 8 up, 8 in 2024-09-21T09:36:54.197 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:53 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-21T09:36:54.197 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:53 smithi022 ceph-mon[98615]: osdmap e393: 8 total, 8 up, 8 in 2024-09-21T09:36:54.197 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:53 smithi022 ceph-mon[96736]: pgmap v190: 121 pgs: 121 active+clean; 96 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:36:54.197 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:53 smithi022 ceph-mon[96736]: osdmap e392: 8 total, 8 up, 8 in 2024-09-21T09:36:54.197 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:53 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-21T09:36:54.197 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:53 smithi022 ceph-mon[96736]: osdmap e393: 8 total, 8 up, 8 in 2024-09-21T09:36:54.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:53 smithi064 ceph-mon[86709]: pgmap v190: 121 pgs: 121 active+clean; 96 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:36:54.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:53 smithi064 ceph-mon[86709]: osdmap e392: 8 total, 8 up, 8 in 2024-09-21T09:36:54.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:53 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-21T09:36:54.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:53 smithi064 ceph-mon[86709]: osdmap e393: 8 total, 8 up, 8 in 2024-09-21T09:36:54.442 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:36:54 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:36:54.148Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.2\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.2\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.2\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:36:54.966 INFO:tasks.rados.rados.0.smithi022.stdout:1637: done (2 left) 2024-09-21T09:36:54.966 INFO:tasks.rados.rados.0.smithi022.stdout:1638: done (1 left) 2024-09-21T09:36:54.966 INFO:tasks.rados.rados.0.smithi022.stdout:1641: done (0 left) 2024-09-21T09:36:54.966 INFO:tasks.rados.rados.0.smithi022.stdout:1642: read oid 50 snap -1 2024-09-21T09:36:54.966 INFO:tasks.rados.rados.0.smithi022.stdout:1642: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-21T09:36:54.966 INFO:tasks.rados.rados.0.smithi022.stdout:1643: read oid 30 snap -1 2024-09-21T09:36:54.966 INFO:tasks.rados.rados.0.smithi022.stdout:1643: expect (ObjNum 481 snap 146 seq_num 481) 2024-09-21T09:36:54.966 INFO:tasks.rados.rados.0.smithi022.stdout:1644: rmattr oid 23 current snap is 161 2024-09-21T09:36:54.966 INFO:tasks.rados.rados.0.smithi022.stdout:1645: read oid 11 snap -1 2024-09-21T09:36:54.966 INFO:tasks.rados.rados.0.smithi022.stdout:1645: expect (ObjNum 525 snap 155 seq_num 525) 2024-09-21T09:36:54.966 INFO:tasks.rados.rados.0.smithi022.stdout:1646: rmattr oid 15 current snap is 161 2024-09-21T09:36:54.967 INFO:tasks.rados.rados.0.smithi022.stdout:1646: done (4 left) 2024-09-21T09:36:54.967 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 46 current snap is 161 2024-09-21T09:36:54.967 INFO:tasks.rados.rados.0.smithi022.stdout:1647: seq_num 536 ranges {0=581632,581632=688128,1269760=516096} 2024-09-21T09:36:54.969 INFO:tasks.rados.rados.0.smithi022.stdout:1647: writing smithi02284960-46 from 0 to 581632 tid 1 2024-09-21T09:36:54.974 INFO:tasks.rados.rados.0.smithi022.stdout:1647: writing smithi02284960-46 from 581632 to 1269760 tid 2 2024-09-21T09:36:54.978 INFO:tasks.rados.rados.0.smithi022.stdout:1647: writing smithi02284960-46 from 1269760 to 1785856 tid 3 2024-09-21T09:36:54.978 INFO:tasks.rados.rados.0.smithi022.stdout:1648: read oid 43 snap -1 2024-09-21T09:36:54.978 INFO:tasks.rados.rados.0.smithi022.stdout:1648: expect deleted 2024-09-21T09:36:54.978 INFO:tasks.rados.rados.0.smithi022.stdout:1649: rollback oid 29 current snap is 161 2024-09-21T09:36:54.978 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 29 to 160 2024-09-21T09:36:54.978 INFO:tasks.rados.rados.0.smithi022.stdout:1650: snap_remove snap 156 2024-09-21T09:36:55.268 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:54 smithi022 ceph-mon[98615]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-21T09:36:55.268 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:54 smithi022 ceph-mon[98615]: Upgrade: osd.2 is safe to restart 2024-09-21T09:36:55.268 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:54 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:36:55.268 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:54 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-21T09:36:55.268 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:54 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:36:55.268 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:54 smithi022 ceph-mon[98615]: osdmap e394: 8 total, 8 up, 8 in 2024-09-21T09:36:55.268 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:54 smithi022 ceph-mon[96736]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-21T09:36:55.268 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:54 smithi022 ceph-mon[96736]: Upgrade: osd.2 is safe to restart 2024-09-21T09:36:55.268 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:54 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:36:55.268 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:54 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-21T09:36:55.269 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:54 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:36:55.269 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:54 smithi022 ceph-mon[96736]: osdmap e394: 8 total, 8 up, 8 in 2024-09-21T09:36:55.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:54 smithi064 ceph-mon[86709]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-21T09:36:55.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:54 smithi064 ceph-mon[86709]: Upgrade: osd.2 is safe to restart 2024-09-21T09:36:55.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:54 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:36:55.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:54 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-21T09:36:55.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:54 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:36:55.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:54 smithi064 ceph-mon[86709]: osdmap e394: 8 total, 8 up, 8 in 2024-09-21T09:36:55.892 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:36:55 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:36:55] "GET /metrics HTTP/1.1" 200 36241 "" "Prometheus/2.51.0" 2024-09-21T09:36:55.969 INFO:tasks.rados.rados.0.smithi022.stdout:1650: done (7 left) 2024-09-21T09:36:55.969 INFO:tasks.rados.rados.0.smithi022.stdout:1651: read oid 7 snap -1 2024-09-21T09:36:55.969 INFO:tasks.rados.rados.0.smithi022.stdout:1651: expect (ObjNum 519 snap 154 seq_num 519) 2024-09-21T09:36:55.969 INFO:tasks.rados.rados.0.smithi022.stdout:1652: delete oid 44 current snap is 161 2024-09-21T09:36:55.969 INFO:tasks.rados.rados.0.smithi022.stdout:1642: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-21T09:36:55.970 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 23 v 844 (ObjNum 423 snap 132 seq_num 423) dirty exists 2024-09-21T09:36:55.970 INFO:tasks.rados.rados.0.smithi022.stdout:1643: expect (ObjNum 481 snap 146 seq_num 481) 2024-09-21T09:36:56.006 INFO:tasks.rados.rados.0.smithi022.stdout:1649: finishing rollback tid 1 to smithi02284960-29 2024-09-21T09:36:56.006 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 29 v 0 (ObjNum 289 snap 0 seq_num 1) dirty dne 2024-09-21T09:36:56.006 INFO:tasks.rados.rados.0.smithi022.stdout:1647: finishing write tid 1 to smithi02284960-46 2024-09-21T09:36:56.006 INFO:tasks.rados.rados.0.smithi022.stdout:1642: done (8 left) 2024-09-21T09:36:56.006 INFO:tasks.rados.rados.0.smithi022.stdout:1643: done (7 left) 2024-09-21T09:36:56.006 INFO:tasks.rados.rados.0.smithi022.stdout:1644: done (6 left) 2024-09-21T09:36:56.006 INFO:tasks.rados.rados.0.smithi022.stdout:1648: done (5 left) 2024-09-21T09:36:56.007 INFO:tasks.rados.rados.0.smithi022.stdout:1649: done (4 left) 2024-09-21T09:36:56.007 INFO:tasks.rados.rados.0.smithi022.stdout:1652: done (3 left) 2024-09-21T09:36:56.007 INFO:tasks.rados.rados.0.smithi022.stdout:1653: snap_create 2024-09-21T09:36:56.007 INFO:tasks.rados.rados.0.smithi022.stdout:1645: expect (ObjNum 525 snap 155 seq_num 525) 2024-09-21T09:36:56.187 INFO:tasks.rados.rados.0.smithi022.stdout:1647: finishing write tid 2 to smithi02284960-46 2024-09-21T09:36:56.188 INFO:tasks.rados.rados.0.smithi022.stdout:1647: finishing write tid 3 to smithi02284960-46 2024-09-21T09:36:56.188 INFO:tasks.rados.rados.0.smithi022.stdout:1647: finishing write tid 4 to smithi02284960-46 2024-09-21T09:36:56.188 INFO:tasks.rados.rados.0.smithi022.stdout:1647: finishing write tid 5 to smithi02284960-46 2024-09-21T09:36:56.188 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 46 v 994 (ObjNum 536 snap 161 seq_num 536) dirty exists 2024-09-21T09:36:56.188 INFO:tasks.rados.rados.0.smithi022.stdout:1647: left oid 46 (ObjNum 536 snap 161 seq_num 536) 2024-09-21T09:36:56.188 INFO:tasks.rados.rados.0.smithi022.stdout:1651: expect (ObjNum 519 snap 154 seq_num 519) 2024-09-21T09:36:56.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:55 smithi064 ceph-mon[86709]: pgmap v193: 121 pgs: 121 active+clean; 84 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3.2 MiB/s rd, 1.6 MiB/s wr, 17 op/s 2024-09-21T09:36:56.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:55 smithi064 ceph-mon[86709]: Upgrade: Updating osd.2 2024-09-21T09:36:56.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:55 smithi064 ceph-mon[86709]: Deploying daemon osd.2 on smithi022 2024-09-21T09:36:56.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:55 smithi064 ceph-mon[86709]: osdmap e395: 8 total, 8 up, 8 in 2024-09-21T09:36:56.192 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:36:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:36:55.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:36:56.192 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:36:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:36:55.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:36:56.199 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:55 smithi022 ceph-mon[96736]: pgmap v193: 121 pgs: 121 active+clean; 84 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3.2 MiB/s rd, 1.6 MiB/s wr, 17 op/s 2024-09-21T09:36:56.199 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:55 smithi022 ceph-mon[96736]: Upgrade: Updating osd.2 2024-09-21T09:36:56.199 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:55 smithi022 ceph-mon[96736]: Deploying daemon osd.2 on smithi022 2024-09-21T09:36:56.200 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:55 smithi022 ceph-mon[96736]: osdmap e395: 8 total, 8 up, 8 in 2024-09-21T09:36:56.200 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:55 smithi022 ceph-mon[98615]: pgmap v193: 121 pgs: 121 active+clean; 84 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3.2 MiB/s rd, 1.6 MiB/s wr, 17 op/s 2024-09-21T09:36:56.200 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:55 smithi022 ceph-mon[98615]: Upgrade: Updating osd.2 2024-09-21T09:36:56.200 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:55 smithi022 ceph-mon[98615]: Deploying daemon osd.2 on smithi022 2024-09-21T09:36:56.200 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:55 smithi022 ceph-mon[98615]: osdmap e395: 8 total, 8 up, 8 in 2024-09-21T09:36:56.994 INFO:tasks.rados.rados.0.smithi022.stdout:1645: done (3 left) 2024-09-21T09:36:56.994 INFO:tasks.rados.rados.0.smithi022.stdout:1647: done (2 left) 2024-09-21T09:36:56.994 INFO:tasks.rados.rados.0.smithi022.stdout:1651: done (1 left) 2024-09-21T09:36:56.994 INFO:tasks.rados.rados.0.smithi022.stdout:1653: done (0 left) 2024-09-21T09:36:56.994 INFO:tasks.rados.rados.0.smithi022.stdout:1654: read oid 13 snap -1 2024-09-21T09:36:56.995 INFO:tasks.rados.rados.0.smithi022.stdout:1654: expect deleted 2024-09-21T09:36:56.995 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 32 current snap is 162 2024-09-21T09:36:56.995 INFO:tasks.rados.rados.0.smithi022.stdout:1655: seq_num 538 ranges {2719744=614400,3334144=450560,3784704=704512,4489216=49152} 2024-09-21T09:36:57.011 INFO:tasks.rados.rados.0.smithi022.stdout:1655: writing smithi02284960-32 from 2719744 to 3334144 tid 1 2024-09-21T09:36:57.014 INFO:tasks.rados.rados.0.smithi022.stdout:1655: writing smithi02284960-32 from 3334144 to 3784704 tid 2 2024-09-21T09:36:57.019 INFO:tasks.rados.rados.0.smithi022.stdout:1655: writing smithi02284960-32 from 3784704 to 4489216 tid 3 2024-09-21T09:36:57.020 INFO:tasks.rados.rados.0.smithi022.stdout:1655: writing smithi02284960-32 from 4489216 to 4538368 tid 4 2024-09-21T09:36:57.020 INFO:tasks.rados.rados.0.smithi022.stdout:1656: copy_from oid 3 from oid 14 current snap is 162 2024-09-21T09:36:57.020 INFO:tasks.rados.rados.0.smithi022.stdout:1655: finishing write tid 1 to smithi02284960-32 2024-09-21T09:36:57.020 INFO:tasks.rados.rados.0.smithi022.stdout:1655: finishing write tid 2 to smithi02284960-32 2024-09-21T09:36:57.020 INFO:tasks.rados.rados.0.smithi022.stdout:1654: done (2 left) 2024-09-21T09:36:57.020 INFO:tasks.rados.rados.0.smithi022.stdout:1657: snap_create 2024-09-21T09:36:57.022 INFO:tasks.rados.rados.0.smithi022.stdout:1655: finishing write tid 3 to smithi02284960-32 2024-09-21T09:36:57.023 INFO:tasks.rados.rados.0.smithi022.stdout:1655: finishing write tid 4 to smithi02284960-32 2024-09-21T09:36:57.024 INFO:tasks.rados.rados.0.smithi022.stdout:1655: finishing write tid 5 to smithi02284960-32 2024-09-21T09:36:57.025 INFO:tasks.rados.rados.0.smithi022.stdout:1655: finishing write tid 6 to smithi02284960-32 2024-09-21T09:36:57.025 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 32 v 1041 (ObjNum 538 snap 162 seq_num 538) dirty exists 2024-09-21T09:36:57.025 INFO:tasks.rados.rados.0.smithi022.stdout:1655: left oid 32 (ObjNum 538 snap 162 seq_num 538) 2024-09-21T09:36:57.033 INFO:tasks.rados.rados.0.smithi022.stdout:1656: finishing copy_from racing read to smithi02284960-3 2024-09-21T09:36:57.034 INFO:tasks.rados.rados.0.smithi022.stdout:1656: finishing copy_from to smithi02284960-3 2024-09-21T09:36:57.034 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 3 v 589 (ObjNum 437 snap 134 seq_num 437) dirty exists 2024-09-21T09:36:57.441 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:36:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:36:56.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi022\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:36:57.998 INFO:tasks.rados.rados.0.smithi022.stdout:1655: done (2 left) 2024-09-21T09:36:57.999 INFO:tasks.rados.rados.0.smithi022.stdout:1656: done (1 left) 2024-09-21T09:36:57.999 INFO:tasks.rados.rados.0.smithi022.stdout:1657: done (0 left) 2024-09-21T09:36:57.999 INFO:tasks.rados.rados.0.smithi022.stdout:1658: delete oid 33 current snap is 163 2024-09-21T09:36:58.003 INFO:tasks.rados.rados.0.smithi022.stdout:1658: done (0 left) 2024-09-21T09:36:58.003 INFO:tasks.rados.rados.0.smithi022.stdout:1659: read oid 12 snap -1 2024-09-21T09:36:58.003 INFO:tasks.rados.rados.0.smithi022.stdout:1659: expect (ObjNum 519 snap 154 seq_num 519) 2024-09-21T09:36:58.003 INFO:tasks.rados.rados.0.smithi022.stdout:1660: rmattr oid 26 current snap is 163 2024-09-21T09:36:58.003 INFO:tasks.rados.rados.0.smithi022.stdout:1660: done (1 left) 2024-09-21T09:36:58.003 INFO:tasks.rados.rados.0.smithi022.stdout:1661: copy_from oid 6 from oid 23 current snap is 163 2024-09-21T09:36:58.004 INFO:tasks.rados.rados.0.smithi022.stdout:1662: rollback oid 17 current snap is 163 2024-09-21T09:36:58.004 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 17 to 161 2024-09-21T09:36:58.004 INFO:tasks.rados.rados.0.smithi022.stdout:1663: snap_create 2024-09-21T09:36:58.006 INFO:tasks.rados.rados.0.smithi022.stdout:1662: finishing rollback tid 0 to smithi02284960-17 2024-09-21T09:36:58.007 INFO:tasks.rados.rados.0.smithi022.stdout:1662: finishing rollback tid 1 to smithi02284960-17 2024-09-21T09:36:58.007 INFO:tasks.rados.rados.0.smithi022.stdout:1662: finishing rollback tid 2 to smithi02284960-17 2024-09-21T09:36:58.007 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 17 v 917 (ObjNum 524 snap 155 seq_num 524) dirty exists 2024-09-21T09:36:58.013 INFO:tasks.rados.rados.0.smithi022.stdout:1659: expect (ObjNum 519 snap 154 seq_num 519) 2024-09-21T09:36:58.016 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:57 smithi022 ceph-mon[98615]: pgmap v196: 121 pgs: 121 active+clean; 84 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 4.0 MiB/s rd, 1.6 MiB/s wr, 18 op/s 2024-09-21T09:36:58.017 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:57 smithi022 ceph-mon[98615]: osdmap e396: 8 total, 8 up, 8 in 2024-09-21T09:36:58.017 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:57 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:36:58.017 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:57 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:36:58.017 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:57 smithi022 ceph-mon[96736]: pgmap v196: 121 pgs: 121 active+clean; 84 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 4.0 MiB/s rd, 1.6 MiB/s wr, 18 op/s 2024-09-21T09:36:58.017 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:57 smithi022 ceph-mon[96736]: osdmap e396: 8 total, 8 up, 8 in 2024-09-21T09:36:58.018 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:57 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:36:58.018 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:57 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:36:58.018 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 21 09:36:57 smithi022 systemd[1]: Stopping Ceph osd.2 for d65e801e-77fa-11ef-baf6-efdc52797490... 2024-09-21T09:36:58.054 INFO:tasks.rados.rados.0.smithi022.stdout:1661: finishing copy_from racing read to smithi02284960-6 2024-09-21T09:36:58.054 INFO:tasks.rados.rados.0.smithi022.stdout:1661: finishing copy_from to smithi02284960-6 2024-09-21T09:36:58.054 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 6 v 844 (ObjNum 423 snap 132 seq_num 423) dirty exists 2024-09-21T09:36:58.308 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 21 09:36:58 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-2[56454]: 2024-09-21T09:36:58.013+0000 7f0831258640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T09:36:58.308 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 21 09:36:58 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-2[56454]: 2024-09-21T09:36:58.013+0000 7f0831258640 -1 osd.2 397 *** Got signal Terminated *** 2024-09-21T09:36:58.308 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 21 09:36:58 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-2[56454]: 2024-09-21T09:36:58.013+0000 7f0831258640 -1 osd.2 397 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T09:36:58.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:57 smithi064 ceph-mon[86709]: pgmap v196: 121 pgs: 121 active+clean; 84 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 4.0 MiB/s rd, 1.6 MiB/s wr, 18 op/s 2024-09-21T09:36:58.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:57 smithi064 ceph-mon[86709]: osdmap e396: 8 total, 8 up, 8 in 2024-09-21T09:36:58.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:57 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:36:58.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:57 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:36:58.969 INFO:tasks.rados.rados.0.smithi022.stdout:1659: done (3 left) 2024-09-21T09:36:58.969 INFO:tasks.rados.rados.0.smithi022.stdout:1661: done (2 left) 2024-09-21T09:36:58.970 INFO:tasks.rados.rados.0.smithi022.stdout:1662: done (1 left) 2024-09-21T09:36:58.970 INFO:tasks.rados.rados.0.smithi022.stdout:1663: done (0 left) 2024-09-21T09:36:58.970 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 25 current snap is 164 2024-09-21T09:36:58.970 INFO:tasks.rados.rados.0.smithi022.stdout:1664: seq_num 540 ranges {3366912=647168,4014080=671744,4685824=532480} 2024-09-21T09:36:58.992 INFO:tasks.rados.rados.0.smithi022.stdout:1664: writing smithi02284960-25 from 3366912 to 4014080 tid 1 2024-09-21T09:36:58.995 INFO:tasks.rados.rados.0.smithi022.stdout:1664: writing smithi02284960-25 from 4014080 to 4685824 tid 2 2024-09-21T09:36:58.999 INFO:tasks.rados.rados.0.smithi022.stdout:1664: writing smithi02284960-25 from 4685824 to 5218304 tid 3 2024-09-21T09:36:58.999 INFO:tasks.rados.rados.0.smithi022.stdout:1665: delete oid 33 current snap is 164 2024-09-21T09:36:58.999 INFO:tasks.rados.rados.0.smithi022.stdout:1664: finishing write tid 1 to smithi02284960-25 2024-09-21T09:36:59.003 INFO:tasks.rados.rados.0.smithi022.stdout:1664: finishing write tid 2 to smithi02284960-25 2024-09-21T09:36:59.004 INFO:tasks.rados.rados.0.smithi022.stdout:1664: finishing write tid 3 to smithi02284960-25 2024-09-21T09:36:59.004 INFO:tasks.rados.rados.0.smithi022.stdout:1664: finishing write tid 4 to smithi02284960-25 2024-09-21T09:36:59.005 INFO:tasks.rados.rados.0.smithi022.stdout:1664: finishing write tid 5 to smithi02284960-25 2024-09-21T09:36:59.005 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 25 v 959 (ObjNum 540 snap 164 seq_num 540) dirty exists 2024-09-21T09:36:59.006 INFO:tasks.rados.rados.0.smithi022.stdout:1664: left oid 25 (ObjNum 540 snap 164 seq_num 540) 2024-09-21T09:36:59.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:59 smithi022 ceph-mon[98615]: osdmap e397: 8 total, 8 up, 8 in 2024-09-21T09:36:59.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:59 smithi022 ceph-mon[98615]: osd.2 marked itself down and dead 2024-09-21T09:36:59.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:59 smithi022 ceph-mon[98615]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T09:36:59.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:36:59 smithi022 ceph-mon[98615]: osdmap e398: 8 total, 7 up, 8 in 2024-09-21T09:36:59.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:59 smithi022 ceph-mon[96736]: osdmap e397: 8 total, 8 up, 8 in 2024-09-21T09:36:59.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:59 smithi022 ceph-mon[96736]: osd.2 marked itself down and dead 2024-09-21T09:36:59.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:59 smithi022 ceph-mon[96736]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T09:36:59.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:36:59 smithi022 ceph-mon[96736]: osdmap e398: 8 total, 7 up, 8 in 2024-09-21T09:36:59.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:59 smithi064 ceph-mon[86709]: osdmap e397: 8 total, 8 up, 8 in 2024-09-21T09:36:59.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:59 smithi064 ceph-mon[86709]: osd.2 marked itself down and dead 2024-09-21T09:36:59.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:59 smithi064 ceph-mon[86709]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T09:36:59.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:36:59 smithi064 ceph-mon[86709]: osdmap e398: 8 total, 7 up, 8 in 2024-09-21T09:36:59.721 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 21 09:36:59 smithi022 podman[112852]: 2024-09-21 09:36:59.428677042 +0000 UTC m=+1.528766113 container died fefc7c770c325dbc7145f119a86072c5503cd2c6fd61db72f287288836d4299c (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , ceph=True, io.buildah.version=1.37.2, RELEASE=quincy-fbd1a6a, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T09:36:59.979 INFO:tasks.rados.rados.0.smithi022.stdout:1664: done (1 left) 2024-09-21T09:36:59.979 INFO:tasks.rados.rados.0.smithi022.stdout:1665: done (0 left) 2024-09-21T09:36:59.979 INFO:tasks.rados.rados.0.smithi022.stdout:1666: snap_create 2024-09-21T09:37:00.004 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 21 09:36:59 smithi022 podman[112852]: 2024-09-21 09:36:59.810088333 +0000 UTC m=+1.910177403 container cleanup fefc7c770c325dbc7145f119a86072c5503cd2c6fd61db72f287288836d4299c (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-2, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=quincy-fbd1a6a, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2) 2024-09-21T09:37:00.004 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 21 09:36:59 smithi022 bash[112852]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-2 2024-09-21T09:37:00.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:00 smithi022 ceph-mon[98615]: pgmap v199: 121 pgs: 121 active+clean; 90 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6.1 MiB/s rd, 2.1 MiB/s wr, 9 op/s 2024-09-21T09:37:00.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:00 smithi022 ceph-mon[98615]: osdmap e399: 8 total, 7 up, 8 in 2024-09-21T09:37:00.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:00 smithi022 ceph-mon[96736]: pgmap v199: 121 pgs: 121 active+clean; 90 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6.1 MiB/s rd, 2.1 MiB/s wr, 9 op/s 2024-09-21T09:37:00.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:00 smithi022 ceph-mon[96736]: osdmap e399: 8 total, 7 up, 8 in 2024-09-21T09:37:00.308 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 21 09:37:00 smithi022 podman[112872]: 2024-09-21 09:37:00.078198708 +0000 UTC m=+0.646095618 container remove fefc7c770c325dbc7145f119a86072c5503cd2c6fd61db72f287288836d4299c (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, RELEASE=quincy-fbd1a6a, org.label-schema.license=GPLv2) 2024-09-21T09:37:00.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:00 smithi064 ceph-mon[86709]: pgmap v199: 121 pgs: 121 active+clean; 90 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6.1 MiB/s rd, 2.1 MiB/s wr, 9 op/s 2024-09-21T09:37:00.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:00 smithi064 ceph-mon[86709]: osdmap e399: 8 total, 7 up, 8 in 2024-09-21T09:37:00.442 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:37:00 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:37:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:37:00.716 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 21 09:37:00 smithi022 podman[112964]: 2024-09-21 09:37:00.296735965 +0000 UTC m=+0.016284170 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:37:00.716 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 21 09:37:00 smithi022 podman[112964]: 2024-09-21 09:37:00.421041871 +0000 UTC m=+0.140590062 container create 0daa283a9f55af87ebb2140d499be1c3e4c63756287bcb3ad47c39edf036cfda (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-2-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=main-62da54b, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_CLEAN=True) 2024-09-21T09:37:00.716 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 21 09:37:00 smithi022 podman[112964]: 2024-09-21 09:37:00.623331156 +0000 UTC m=+0.342879347 container init 0daa283a9f55af87ebb2140d499be1c3e4c63756287bcb3ad47c39edf036cfda (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-2-deactivate, GIT_CLEAN=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, RELEASE=main-62da54b, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2) 2024-09-21T09:37:00.716 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 21 09:37:00 smithi022 podman[112964]: 2024-09-21 09:37:00.628296546 +0000 UTC m=+0.347844738 container start 0daa283a9f55af87ebb2140d499be1c3e4c63756287bcb3ad47c39edf036cfda (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-2-deactivate, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, RELEASE=main-62da54b, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=) 2024-09-21T09:37:00.716 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 21 09:37:00 smithi022 podman[112964]: 2024-09-21 09:37:00.714818811 +0000 UTC m=+0.434367020 container attach 0daa283a9f55af87ebb2140d499be1c3e4c63756287bcb3ad47c39edf036cfda (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-2-deactivate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, RELEASE=main-62da54b, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T09:37:00.970 INFO:tasks.rados.rados.0.smithi022.stdout:1666: done (0 left) 2024-09-21T09:37:00.970 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 5 current snap is 165 2024-09-21T09:37:00.970 INFO:tasks.rados.rados.0.smithi022.stdout:1667: seq_num 542 ranges {4341760=679936,5021696=442368,5464064=679936,6144000=81920} 2024-09-21T09:37:00.997 INFO:tasks.rados.rados.0.smithi022.stdout:1667: writing smithi02284960-5 from 4341760 to 5021696 tid 1 2024-09-21T09:37:00.999 INFO:tasks.rados.rados.0.smithi022.stdout:1667: writing smithi02284960-5 from 5021696 to 5464064 tid 2 2024-09-21T09:37:01.004 INFO:tasks.rados.rados.0.smithi022.stdout:1667: writing smithi02284960-5 from 5464064 to 6144000 tid 3 2024-09-21T09:37:01.005 INFO:tasks.rados.rados.0.smithi022.stdout:1667: writing smithi02284960-5 from 6144000 to 6225920 tid 4 2024-09-21T09:37:01.005 INFO:tasks.rados.rados.0.smithi022.stdout:1668: rollback oid 11 current snap is 165 2024-09-21T09:37:01.005 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 11 to 161 2024-09-21T09:37:01.005 INFO:tasks.rados.rados.0.smithi022.stdout:1667: finishing write tid 1 to smithi02284960-5 2024-09-21T09:37:01.005 INFO:tasks.rados.rados.0.smithi022.stdout:1667: finishing write tid 2 to smithi02284960-5 2024-09-21T09:37:01.006 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 4 current snap is 165 2024-09-21T09:37:01.006 INFO:tasks.rados.rados.0.smithi022.stdout:1669: seq_num 543 ranges {3039232=704512,3743744=729088,4472832=475136} 2024-09-21T09:37:01.025 INFO:tasks.rados.rados.0.smithi022.stdout:1669: writing smithi02284960-4 from 3039232 to 3743744 tid 1 2024-09-21T09:37:01.030 INFO:tasks.rados.rados.0.smithi022.stdout:1669: writing smithi02284960-4 from 3743744 to 4472832 tid 2 2024-09-21T09:37:01.033 INFO:tasks.rados.rados.0.smithi022.stdout:1669: writing smithi02284960-4 from 4472832 to 4947968 tid 3 2024-09-21T09:37:01.033 INFO:tasks.rados.rados.0.smithi022.stdout:1670: delete oid 30 current snap is 165 2024-09-21T09:37:01.034 INFO:tasks.rados.rados.0.smithi022.stdout:1667: finishing write tid 3 to smithi02284960-5 2024-09-21T09:37:01.034 INFO:tasks.rados.rados.0.smithi022.stdout:1667: finishing write tid 4 to smithi02284960-5 2024-09-21T09:37:01.034 INFO:tasks.rados.rados.0.smithi022.stdout:1667: finishing write tid 5 to smithi02284960-5 2024-09-21T09:37:01.034 INFO:tasks.rados.rados.0.smithi022.stdout:1667: finishing write tid 6 to smithi02284960-5 2024-09-21T09:37:01.034 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 5 v 625 (ObjNum 542 snap 165 seq_num 542) dirty exists 2024-09-21T09:37:01.034 INFO:tasks.rados.rados.0.smithi022.stdout:1667: left oid 5 (ObjNum 542 snap 165 seq_num 542) 2024-09-21T09:37:01.034 INFO:tasks.rados.rados.0.smithi022.stdout:1668: finishing rollback tid 0 to smithi02284960-11 2024-09-21T09:37:01.034 INFO:tasks.rados.rados.0.smithi022.stdout:1668: finishing rollback tid 1 to smithi02284960-11 2024-09-21T09:37:01.035 INFO:tasks.rados.rados.0.smithi022.stdout:1668: finishing rollback tid 2 to smithi02284960-11 2024-09-21T09:37:01.035 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 11 v 999 (ObjNum 525 snap 155 seq_num 525) dirty exists 2024-09-21T09:37:01.035 INFO:tasks.rados.rados.0.smithi022.stdout:1669: finishing write tid 1 to smithi02284960-4 2024-09-21T09:37:01.035 INFO:tasks.rados.rados.0.smithi022.stdout:1669: finishing write tid 2 to smithi02284960-4 2024-09-21T09:37:01.037 INFO:tasks.rados.rados.0.smithi022.stdout:1667: done (3 left) 2024-09-21T09:37:01.037 INFO:tasks.rados.rados.0.smithi022.stdout:1668: done (2 left) 2024-09-21T09:37:01.037 INFO:tasks.rados.rados.0.smithi022.stdout:1670: done (1 left) 2024-09-21T09:37:01.037 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 15 current snap is 165 2024-09-21T09:37:01.037 INFO:tasks.rados.rados.0.smithi022.stdout:1671: seq_num 545 ranges {0=737280,737280=491520,1228800=450560,1679360=262144} 2024-09-21T09:37:01.040 INFO:tasks.rados.rados.0.smithi022.stdout:1671: writing smithi02284960-15 from 0 to 737280 tid 1 2024-09-21T09:37:01.043 INFO:tasks.rados.rados.0.smithi022.stdout:1671: writing smithi02284960-15 from 737280 to 1228800 tid 2 2024-09-21T09:37:01.045 INFO:tasks.rados.rados.0.smithi022.stdout:1671: writing smithi02284960-15 from 1228800 to 1679360 tid 3 2024-09-21T09:37:01.047 INFO:tasks.rados.rados.0.smithi022.stdout:1671: writing smithi02284960-15 from 1679360 to 1941504 tid 4 2024-09-21T09:37:01.047 INFO:tasks.rados.rados.0.smithi022.stdout:1672: delete oid 34 current snap is 165 2024-09-21T09:37:01.047 INFO:tasks.rados.rados.0.smithi022.stdout:1669: finishing write tid 3 to smithi02284960-4 2024-09-21T09:37:01.047 INFO:tasks.rados.rados.0.smithi022.stdout:1669: finishing write tid 4 to smithi02284960-4 2024-09-21T09:37:01.048 INFO:tasks.rados.rados.0.smithi022.stdout:1669: finishing write tid 5 to smithi02284960-4 2024-09-21T09:37:01.048 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 4 v 630 (ObjNum 543 snap 165 seq_num 543) dirty exists 2024-09-21T09:37:01.048 INFO:tasks.rados.rados.0.smithi022.stdout:1669: left oid 4 (ObjNum 543 snap 165 seq_num 543) 2024-09-21T09:37:01.048 INFO:tasks.rados.rados.0.smithi022.stdout:1671: finishing write tid 1 to smithi02284960-15 2024-09-21T09:37:01.048 INFO:tasks.rados.rados.0.smithi022.stdout:1671: finishing write tid 2 to smithi02284960-15 2024-09-21T09:37:01.048 INFO:tasks.rados.rados.0.smithi022.stdout:1669: done (2 left) 2024-09-21T09:37:01.049 INFO:tasks.rados.rados.0.smithi022.stdout:1672: done (1 left) 2024-09-21T09:37:01.049 INFO:tasks.rados.rados.0.smithi022.stdout:1673: rollback oid 13 current snap is 165 2024-09-21T09:37:01.049 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 13 to 164 2024-09-21T09:37:01.049 INFO:tasks.rados.rados.0.smithi022.stdout:1674: snap_create 2024-09-21T09:37:01.049 INFO:tasks.rados.rados.0.smithi022.stdout:1673: finishing rollback tid 1 to smithi02284960-13 2024-09-21T09:37:01.049 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 13 v 0 (ObjNum 161 snap 0 seq_num 3569956512) dirty dne 2024-09-21T09:37:01.049 INFO:tasks.rados.rados.0.smithi022.stdout:1671: finishing write tid 3 to smithi02284960-15 2024-09-21T09:37:01.050 INFO:tasks.rados.rados.0.smithi022.stdout:1671: finishing write tid 4 to smithi02284960-15 2024-09-21T09:37:01.050 INFO:tasks.rados.rados.0.smithi022.stdout:1671: finishing write tid 5 to smithi02284960-15 2024-09-21T09:37:01.053 INFO:tasks.rados.rados.0.smithi022.stdout:1671: finishing write tid 6 to smithi02284960-15 2024-09-21T09:37:01.053 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 15 v 1129 (ObjNum 545 snap 165 seq_num 545) dirty exists 2024-09-21T09:37:01.053 INFO:tasks.rados.rados.0.smithi022.stdout:1671: left oid 15 (ObjNum 545 snap 165 seq_num 545) 2024-09-21T09:37:01.058 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 21 09:37:00 smithi022 podman[112964]: 2024-09-21 09:37:00.824185131 +0000 UTC m=+0.543733329 container died 0daa283a9f55af87ebb2140d499be1c3e4c63756287bcb3ad47c39edf036cfda (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-2-deactivate, RELEASE=main-62da54b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T09:37:01.808 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 21 09:37:01 smithi022 podman[113006]: 2024-09-21 09:37:01.485990071 +0000 UTC m=+0.651965288 container remove 0daa283a9f55af87ebb2140d499be1c3e4c63756287bcb3ad47c39edf036cfda (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-2-deactivate, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, ceph=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-62da54b, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2) 2024-09-21T09:37:01.808 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 21 09:37:01 smithi022 systemd[1]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490@osd.2.service: Deactivated successfully. 2024-09-21T09:37:01.809 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 21 09:37:01 smithi022 systemd[1]: Stopped Ceph osd.2 for d65e801e-77fa-11ef-baf6-efdc52797490. 2024-09-21T09:37:01.809 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 21 09:37:01 smithi022 systemd[1]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490@osd.2.service: Consumed 16.660s CPU time. 2024-09-21T09:37:01.809 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 21 09:37:01 smithi022 systemd[1]: Starting Ceph osd.2 for d65e801e-77fa-11ef-baf6-efdc52797490... 2024-09-21T09:37:02.010 INFO:tasks.rados.rados.0.smithi022.stdout:1671: done (2 left) 2024-09-21T09:37:02.010 INFO:tasks.rados.rados.0.smithi022.stdout:1673: done (1 left) 2024-09-21T09:37:02.010 INFO:tasks.rados.rados.0.smithi022.stdout:1674: done (0 left) 2024-09-21T09:37:02.010 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 38 current snap is 166 2024-09-21T09:37:02.010 INFO:tasks.rados.rados.0.smithi022.stdout:1675: seq_num 547 ranges {933888=770048,1703936=663552,2367488=466944,2834432=73728} 2024-09-21T09:37:02.020 INFO:tasks.rados.rados.0.smithi022.stdout:1675: writing smithi02284960-38 from 933888 to 1703936 tid 1 2024-09-21T09:37:02.024 INFO:tasks.rados.rados.0.smithi022.stdout:1675: writing smithi02284960-38 from 1703936 to 2367488 tid 2 2024-09-21T09:37:02.027 INFO:tasks.rados.rados.0.smithi022.stdout:1675: writing smithi02284960-38 from 2367488 to 2834432 tid 3 2024-09-21T09:37:02.028 INFO:tasks.rados.rados.0.smithi022.stdout:1675: writing smithi02284960-38 from 2834432 to 2908160 tid 4 2024-09-21T09:37:02.028 INFO:tasks.rados.rados.0.smithi022.stdout:1676: rollback oid 50 current snap is 166 2024-09-21T09:37:02.028 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 50 to 158 2024-09-21T09:37:02.028 INFO:tasks.rados.rados.0.smithi022.stdout:1675: finishing write tid 1 to smithi02284960-38 2024-09-21T09:37:02.028 INFO:tasks.rados.rados.0.smithi022.stdout:1675: finishing write tid 2 to smithi02284960-38 2024-09-21T09:37:02.029 INFO:tasks.rados.rados.0.smithi022.stdout:1677: setattr oid 41 current snap is 166 2024-09-21T09:37:02.031 INFO:tasks.rados.rados.0.smithi022.stdout:1678: rollback oid 7 current snap is 166 2024-09-21T09:37:02.031 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 7 to 158 2024-09-21T09:37:02.031 INFO:tasks.rados.rados.0.smithi022.stdout:1679: snap_remove snap 164 2024-09-21T09:37:02.248 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:01 smithi022 ceph-mon[98615]: pgmap v202: 121 pgs: 12 stale+active+clean, 109 active+clean; 90 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 5.4 MiB/s rd, 2.1 MiB/s wr, 8 op/s 2024-09-21T09:37:02.249 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:01 smithi022 ceph-mon[98615]: osdmap e400: 8 total, 7 up, 8 in 2024-09-21T09:37:02.249 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:01 smithi022 ceph-mon[96736]: pgmap v202: 121 pgs: 12 stale+active+clean, 109 active+clean; 90 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 5.4 MiB/s rd, 2.1 MiB/s wr, 8 op/s 2024-09-21T09:37:02.249 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:01 smithi022 ceph-mon[96736]: osdmap e400: 8 total, 7 up, 8 in 2024-09-21T09:37:02.249 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 21 09:37:01 smithi022 podman[113112]: 2024-09-21 09:37:01.831952043 +0000 UTC m=+0.015974053 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:37:02.249 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 21 09:37:01 smithi022 podman[113112]: 2024-09-21 09:37:01.936988422 +0000 UTC m=+0.121010437 container create f0bc5bfdbbf86cc5a6867c9d66401043b1f1a2a228c8e8520a23036f5a7e3898 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-2-activate, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, RELEASE=main-62da54b, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_CLEAN=True) 2024-09-21T09:37:02.249 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 21 09:37:02 smithi022 podman[113112]: 2024-09-21 09:37:02.156148077 +0000 UTC m=+0.340170085 container init f0bc5bfdbbf86cc5a6867c9d66401043b1f1a2a228c8e8520a23036f5a7e3898 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-2-activate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=main-62da54b, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T09:37:02.250 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 21 09:37:02 smithi022 podman[113112]: 2024-09-21 09:37:02.161252197 +0000 UTC m=+0.345274207 container start f0bc5bfdbbf86cc5a6867c9d66401043b1f1a2a228c8e8520a23036f5a7e3898 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-2-activate, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-62da54b, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-09-21T09:37:02.250 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 21 09:37:02 smithi022 podman[113112]: 2024-09-21 09:37:02.247725431 +0000 UTC m=+0.431747448 container attach f0bc5bfdbbf86cc5a6867c9d66401043b1f1a2a228c8e8520a23036f5a7e3898 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-2-activate, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=main-62da54b, CEPH_POINT_RELEASE=, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T09:37:02.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:01 smithi064 ceph-mon[86709]: pgmap v202: 121 pgs: 12 stale+active+clean, 109 active+clean; 90 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 5.4 MiB/s rd, 2.1 MiB/s wr, 8 op/s 2024-09-21T09:37:02.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:01 smithi064 ceph-mon[86709]: osdmap e400: 8 total, 7 up, 8 in 2024-09-21T09:37:02.558 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 21 09:37:02 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-2-activate[113127]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T09:37:02.558 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 21 09:37:02 smithi022 bash[113112]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T09:37:02.558 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 21 09:37:02 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-2-activate[113127]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T09:37:02.558 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 21 09:37:02 smithi022 bash[113112]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T09:37:03.014 INFO:tasks.rados.rados.0.smithi022.stdout:1679: done (4 left) 2024-09-21T09:37:03.015 INFO:tasks.rados.rados.0.smithi022.stdout:1680: rollback oid 14 current snap is 166 2024-09-21T09:37:03.015 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 14 to 160 2024-09-21T09:37:03.015 INFO:tasks.rados.rados.0.smithi022.stdout:1675: finishing write tid 3 to smithi02284960-38 2024-09-21T09:37:03.015 INFO:tasks.rados.rados.0.smithi022.stdout:1675: finishing write tid 4 to smithi02284960-38 2024-09-21T09:37:03.015 INFO:tasks.rados.rados.0.smithi022.stdout:1675: finishing write tid 5 to smithi02284960-38 2024-09-21T09:37:03.015 INFO:tasks.rados.rados.0.smithi022.stdout:1676: finishing rollback tid 0 to smithi02284960-50 2024-09-21T09:37:03.015 INFO:tasks.rados.rados.0.smithi022.stdout:1675: finishing write tid 6 to smithi02284960-38 2024-09-21T09:37:03.015 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 38 v 849 (ObjNum 547 snap 166 seq_num 547) dirty exists 2024-09-21T09:37:03.015 INFO:tasks.rados.rados.0.smithi022.stdout:1675: left oid 38 (ObjNum 547 snap 166 seq_num 547) 2024-09-21T09:37:03.015 INFO:tasks.rados.rados.0.smithi022.stdout:1676: finishing rollback tid 1 to smithi02284960-50 2024-09-21T09:37:03.015 INFO:tasks.rados.rados.0.smithi022.stdout:1678: finishing rollback tid 0 to smithi02284960-7 2024-09-21T09:37:03.015 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 41 v 961 (ObjNum 531 snap 159 seq_num 531) dirty exists 2024-09-21T09:37:03.015 INFO:tasks.rados.rados.0.smithi022.stdout:1676: finishing rollback tid 2 to smithi02284960-50 2024-09-21T09:37:03.016 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 50 v 1076 (ObjNum 65 snap 0 seq_num 1499165800) dirty exists 2024-09-21T09:37:03.016 INFO:tasks.rados.rados.0.smithi022.stdout:1678: finishing rollback tid 1 to smithi02284960-7 2024-09-21T09:37:03.016 INFO:tasks.rados.rados.0.smithi022.stdout:1678: finishing rollback tid 2 to smithi02284960-7 2024-09-21T09:37:03.016 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 7 v 1050 (ObjNum 519 snap 154 seq_num 519) dirty exists 2024-09-21T09:37:03.016 INFO:tasks.rados.rados.0.smithi022.stdout:1675: done (4 left) 2024-09-21T09:37:03.016 INFO:tasks.rados.rados.0.smithi022.stdout:1676: done (3 left) 2024-09-21T09:37:03.016 INFO:tasks.rados.rados.0.smithi022.stdout:1677: done (2 left) 2024-09-21T09:37:03.016 INFO:tasks.rados.rados.0.smithi022.stdout:1678: done (1 left) 2024-09-21T09:37:03.016 INFO:tasks.rados.rados.0.smithi022.stdout:1681: delete oid 31 current snap is 166 2024-09-21T09:37:03.023 INFO:tasks.rados.rados.0.smithi022.stdout:1680: finishing rollback tid 0 to smithi02284960-14 2024-09-21T09:37:03.023 INFO:tasks.rados.rados.0.smithi022.stdout:1680: finishing rollback tid 1 to smithi02284960-14 2024-09-21T09:37:03.023 INFO:tasks.rados.rados.0.smithi022.stdout:1681: done (1 left) 2024-09-21T09:37:03.023 INFO:tasks.rados.rados.0.smithi022.stdout:1682: snap_remove snap 161 2024-09-21T09:37:03.057 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:03 smithi022 ceph-mon[98615]: osdmap e401: 8 total, 7 up, 8 in 2024-09-21T09:37:03.057 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:03 smithi022 ceph-mon[96736]: osdmap e401: 8 total, 7 up, 8 in 2024-09-21T09:37:03.057 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 21 09:37:02 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-2-activate[113127]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-21T09:37:03.058 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 21 09:37:02 smithi022 bash[113112]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-21T09:37:03.058 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 21 09:37:02 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-2-activate[113127]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-21T09:37:03.058 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 21 09:37:02 smithi022 bash[113112]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-21T09:37:03.058 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 21 09:37:03 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-2-activate[113127]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-21T09:37:03.058 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 21 09:37:03 smithi022 bash[113112]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-21T09:37:03.308 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 21 09:37:03 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-2-activate[113127]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-21T09:37:03.308 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 21 09:37:03 smithi022 bash[113112]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-21T09:37:03.308 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 21 09:37:03 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-2-activate[113127]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-09-21T09:37:03.308 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 21 09:37:03 smithi022 bash[113112]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-09-21T09:37:03.308 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 21 09:37:03 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-2-activate[113127]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-21T09:37:03.308 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 21 09:37:03 smithi022 bash[113112]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-21T09:37:03.308 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 21 09:37:03 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-2-activate[113127]: --> ceph-volume raw activate successful for osd ID: 2 2024-09-21T09:37:03.308 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 21 09:37:03 smithi022 bash[113112]: --> ceph-volume raw activate successful for osd ID: 2 2024-09-21T09:37:03.308 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 21 09:37:03 smithi022 podman[113112]: 2024-09-21 09:37:03.084743465 +0000 UTC m=+1.268765480 container died f0bc5bfdbbf86cc5a6867c9d66401043b1f1a2a228c8e8520a23036f5a7e3898 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-2-activate, GIT_BRANCH=HEAD, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-62da54b, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-09-21T09:37:03.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:03 smithi064 ceph-mon[86709]: osdmap e401: 8 total, 7 up, 8 in 2024-09-21T09:37:03.968 INFO:tasks.rados.rados.0.smithi022.stdout:1682: done (1 left) 2024-09-21T09:37:03.968 INFO:tasks.rados.rados.0.smithi022.stdout:1683: read oid 23 snap 162 2024-09-21T09:37:03.968 INFO:tasks.rados.rados.0.smithi022.stdout:1683: expect (ObjNum 423 snap 132 seq_num 423) 2024-09-21T09:37:03.968 INFO:tasks.rados.rados.0.smithi022.stdout:1684: read oid 18 snap 165 2024-09-21T09:37:03.968 INFO:tasks.rados.rados.0.smithi022.stdout:1684: expect (ObjNum 482 snap 146 seq_num 482) 2024-09-21T09:37:03.968 INFO:tasks.rados.rados.0.smithi022.stdout:1685: rollback oid 3 current snap is 166 2024-09-21T09:37:03.968 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 3 to 158 2024-09-21T09:37:03.968 INFO:tasks.rados.rados.0.smithi022.stdout:1680: finishing rollback tid 2 to smithi02284960-14 2024-09-21T09:37:03.968 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 14 v 762 (ObjNum 437 snap 134 seq_num 437) dirty exists 2024-09-21T09:37:03.968 INFO:tasks.rados.rados.0.smithi022.stdout:1680: done (3 left) 2024-09-21T09:37:03.968 INFO:tasks.rados.rados.0.smithi022.stdout:1686: read oid 27 snap -1 2024-09-21T09:37:03.968 INFO:tasks.rados.rados.0.smithi022.stdout:1686: expect deleted 2024-09-21T09:37:03.969 INFO:tasks.rados.rados.0.smithi022.stdout:1687: read oid 44 snap -1 2024-09-21T09:37:03.969 INFO:tasks.rados.rados.0.smithi022.stdout:1687: expect deleted 2024-09-21T09:37:03.969 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 48 current snap is 166 2024-09-21T09:37:03.969 INFO:tasks.rados.rados.0.smithi022.stdout:1688: seq_num 549 ranges {3457024=802816,4259840=483328,4743168=434176,5177344=286720} 2024-09-21T09:37:03.991 INFO:tasks.rados.rados.0.smithi022.stdout:1688: writing smithi02284960-48 from 3457024 to 4259840 tid 1 2024-09-21T09:37:03.994 INFO:tasks.rados.rados.0.smithi022.stdout:1688: writing smithi02284960-48 from 4259840 to 4743168 tid 2 2024-09-21T09:37:03.996 INFO:tasks.rados.rados.0.smithi022.stdout:1688: writing smithi02284960-48 from 4743168 to 5177344 tid 3 2024-09-21T09:37:03.999 INFO:tasks.rados.rados.0.smithi022.stdout:1688: writing smithi02284960-48 from 5177344 to 5464064 tid 4 2024-09-21T09:37:03.999 INFO:tasks.rados.rados.0.smithi022.stdout:1689: rmattr oid 31 current snap is 166 2024-09-21T09:37:03.999 INFO:tasks.rados.rados.0.smithi022.stdout:1689: done (6 left) 2024-09-21T09:37:03.999 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 33 current snap is 166 2024-09-21T09:37:03.999 INFO:tasks.rados.rados.0.smithi022.stdout:1690: seq_num 550 ranges {0=417792,417792=770048,1187840=638976,1826816=196608} 2024-09-21T09:37:04.001 INFO:tasks.rados.rados.0.smithi022.stdout:1690: writing smithi02284960-33 from 0 to 417792 tid 1 2024-09-21T09:37:04.005 INFO:tasks.rados.rados.0.smithi022.stdout:1690: writing smithi02284960-33 from 417792 to 1187840 tid 2 2024-09-21T09:37:04.009 INFO:tasks.rados.rados.0.smithi022.stdout:1690: writing smithi02284960-33 from 1187840 to 1826816 tid 3 2024-09-21T09:37:04.011 INFO:tasks.rados.rados.0.smithi022.stdout:1690: writing smithi02284960-33 from 1826816 to 2023424 tid 4 2024-09-21T09:37:04.011 INFO:tasks.rados.rados.0.smithi022.stdout:1691: delete oid 29 current snap is 166 2024-09-21T09:37:04.011 INFO:tasks.rados.rados.0.smithi022.stdout:1685: finishing rollback tid 0 to smithi02284960-3 2024-09-21T09:37:04.011 INFO:tasks.rados.rados.0.smithi022.stdout:1685: finishing rollback tid 1 to smithi02284960-3 2024-09-21T09:37:04.011 INFO:tasks.rados.rados.0.smithi022.stdout:1685: finishing rollback tid 2 to smithi02284960-3 2024-09-21T09:37:04.011 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 3 v 592 (ObjNum 509 snap 151 seq_num 509) dirty exists 2024-09-21T09:37:04.012 INFO:tasks.rados.rados.0.smithi022.stdout:1683: expect (ObjNum 423 snap 132 seq_num 423) 2024-09-21T09:37:04.105 INFO:tasks.rados.rados.0.smithi022.stdout:1684: expect (ObjNum 482 snap 146 seq_num 482) 2024-09-21T09:37:04.131 INFO:tasks.rados.rados.0.smithi022.stdout:1688: finishing write tid 1 to smithi02284960-48 2024-09-21T09:37:04.132 INFO:tasks.rados.rados.0.smithi022.stdout:1688: finishing write tid 2 to smithi02284960-48 2024-09-21T09:37:04.132 INFO:tasks.rados.rados.0.smithi022.stdout:1688: finishing write tid 3 to smithi02284960-48 2024-09-21T09:37:04.132 INFO:tasks.rados.rados.0.smithi022.stdout:1688: finishing write tid 4 to smithi02284960-48 2024-09-21T09:37:04.132 INFO:tasks.rados.rados.0.smithi022.stdout:1688: finishing write tid 5 to smithi02284960-48 2024-09-21T09:37:04.132 INFO:tasks.rados.rados.0.smithi022.stdout:1688: finishing write tid 6 to smithi02284960-48 2024-09-21T09:37:04.132 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 48 v 1056 (ObjNum 549 snap 166 seq_num 549) dirty exists 2024-09-21T09:37:04.132 INFO:tasks.rados.rados.0.smithi022.stdout:1688: left oid 48 (ObjNum 549 snap 166 seq_num 549) 2024-09-21T09:37:04.132 INFO:tasks.rados.rados.0.smithi022.stdout:1690: finishing write tid 1 to smithi02284960-33 2024-09-21T09:37:04.132 INFO:tasks.rados.rados.0.smithi022.stdout:1690: finishing write tid 2 to smithi02284960-33 2024-09-21T09:37:04.132 INFO:tasks.rados.rados.0.smithi022.stdout:1683: done (7 left) 2024-09-21T09:37:04.132 INFO:tasks.rados.rados.0.smithi022.stdout:1684: done (6 left) 2024-09-21T09:37:04.132 INFO:tasks.rados.rados.0.smithi022.stdout:1685: done (5 left) 2024-09-21T09:37:04.132 INFO:tasks.rados.rados.0.smithi022.stdout:1686: done (4 left) 2024-09-21T09:37:04.133 INFO:tasks.rados.rados.0.smithi022.stdout:1687: done (3 left) 2024-09-21T09:37:04.133 INFO:tasks.rados.rados.0.smithi022.stdout:1688: done (2 left) 2024-09-21T09:37:04.133 INFO:tasks.rados.rados.0.smithi022.stdout:1691: done (1 left) 2024-09-21T09:37:04.133 INFO:tasks.rados.rados.0.smithi022.stdout:1692: snap_remove snap 160 2024-09-21T09:37:04.154 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:04 smithi022 ceph-mon[98615]: pgmap v205: 121 pgs: 4 active+undersized, 11 stale+active+clean, 1 active+undersized+degraded, 105 active+clean; 90 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s; 4/240 objects degraded (1.667%) 2024-09-21T09:37:04.155 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:04 smithi022 ceph-mon[98615]: Health check failed: Degraded data redundancy: 4/240 objects degraded (1.667%), 1 pg degraded (PG_DEGRADED) 2024-09-21T09:37:04.155 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:04 smithi022 ceph-mon[98615]: osdmap e402: 8 total, 7 up, 8 in 2024-09-21T09:37:04.155 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:04 smithi022 ceph-mon[98615]: osdmap e403: 8 total, 7 up, 8 in 2024-09-21T09:37:04.155 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:04 smithi022 ceph-mon[96736]: pgmap v205: 121 pgs: 4 active+undersized, 11 stale+active+clean, 1 active+undersized+degraded, 105 active+clean; 90 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s; 4/240 objects degraded (1.667%) 2024-09-21T09:37:04.155 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:04 smithi022 ceph-mon[96736]: Health check failed: Degraded data redundancy: 4/240 objects degraded (1.667%), 1 pg degraded (PG_DEGRADED) 2024-09-21T09:37:04.155 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:04 smithi022 ceph-mon[96736]: osdmap e402: 8 total, 7 up, 8 in 2024-09-21T09:37:04.155 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:04 smithi022 ceph-mon[96736]: osdmap e403: 8 total, 7 up, 8 in 2024-09-21T09:37:04.156 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 21 09:37:03 smithi022 podman[113285]: 2024-09-21 09:37:03.835211961 +0000 UTC m=+0.741432413 container remove f0bc5bfdbbf86cc5a6867c9d66401043b1f1a2a228c8e8520a23036f5a7e3898 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-2-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=main-62da54b, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0) 2024-09-21T09:37:04.156 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 21 09:37:04 smithi022 podman[113352]: 2024-09-21 09:37:03.946464046 +0000 UTC m=+0.016900289 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:37:04.156 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 21 09:37:04 smithi022 podman[113352]: 2024-09-21 09:37:04.061346564 +0000 UTC m=+0.131782798 container create b8c9443066987ecda42255c21a370c89df1058a3fac9c29a920df395bc6c26da (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-2, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-62da54b, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS) 2024-09-21T09:37:04.343 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:37:04 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:37:04.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.2\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.2\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.2\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:37:04.343 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:04 smithi064 ceph-mon[86709]: pgmap v205: 121 pgs: 4 active+undersized, 11 stale+active+clean, 1 active+undersized+degraded, 105 active+clean; 90 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s; 4/240 objects degraded (1.667%) 2024-09-21T09:37:04.343 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:04 smithi064 ceph-mon[86709]: Health check failed: Degraded data redundancy: 4/240 objects degraded (1.667%), 1 pg degraded (PG_DEGRADED) 2024-09-21T09:37:04.343 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:04 smithi064 ceph-mon[86709]: osdmap e402: 8 total, 7 up, 8 in 2024-09-21T09:37:04.343 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:04 smithi064 ceph-mon[86709]: osdmap e403: 8 total, 7 up, 8 in 2024-09-21T09:37:04.558 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 21 09:37:04 smithi022 podman[113352]: 2024-09-21 09:37:04.255451397 +0000 UTC m=+0.325887630 container init b8c9443066987ecda42255c21a370c89df1058a3fac9c29a920df395bc6c26da (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , ceph=True, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, RELEASE=main-62da54b, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T09:37:04.558 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 21 09:37:04 smithi022 podman[113352]: 2024-09-21 09:37:04.260450628 +0000 UTC m=+0.330886862 container start b8c9443066987ecda42255c21a370c89df1058a3fac9c29a920df395bc6c26da (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, RELEASE=main-62da54b, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0) 2024-09-21T09:37:04.558 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 21 09:37:04 smithi022 ceph-osd[113390]: -- 172.21.15.22:0/1065286342 <== mon.0 v2:172.21.15.22:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5647f4a2fa40 con 0x5647f4a0dc00 2024-09-21T09:37:04.558 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 21 09:37:04 smithi022 bash[113352]: b8c9443066987ecda42255c21a370c89df1058a3fac9c29a920df395bc6c26da 2024-09-21T09:37:04.558 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 21 09:37:04 smithi022 systemd[1]: Started Ceph osd.2 for d65e801e-77fa-11ef-baf6-efdc52797490. 2024-09-21T09:37:05.021 INFO:tasks.rados.rados.0.smithi022.stdout:1692: done (1 left) 2024-09-21T09:37:05.021 INFO:tasks.rados.rados.0.smithi022.stdout:1693: setattr oid 23 current snap is 166 2024-09-21T09:37:05.021 INFO:tasks.rados.rados.0.smithi022.stdout:1690: finishing write tid 3 to smithi02284960-33 2024-09-21T09:37:05.021 INFO:tasks.rados.rados.0.smithi022.stdout:1690: finishing write tid 4 to smithi02284960-33 2024-09-21T09:37:05.021 INFO:tasks.rados.rados.0.smithi022.stdout:1690: finishing write tid 5 to smithi02284960-33 2024-09-21T09:37:05.021 INFO:tasks.rados.rados.0.smithi022.stdout:1690: finishing write tid 6 to smithi02284960-33 2024-09-21T09:37:05.021 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 33 v 1061 (ObjNum 550 snap 166 seq_num 550) dirty exists 2024-09-21T09:37:05.021 INFO:tasks.rados.rados.0.smithi022.stdout:1690: left oid 33 (ObjNum 550 snap 166 seq_num 550) 2024-09-21T09:37:05.025 INFO:tasks.rados.rados.0.smithi022.stdout:1690: done (1 left) 2024-09-21T09:37:05.025 INFO:tasks.rados.rados.0.smithi022.stdout:1694: copy_from oid 42 from oid 28 current snap is 166 2024-09-21T09:37:05.026 INFO:tasks.rados.rados.0.smithi022.stdout:1695: read oid 26 snap -1 2024-09-21T09:37:05.026 INFO:tasks.rados.rados.0.smithi022.stdout:1695: expect (ObjNum 450 snap 136 seq_num 450) 2024-09-21T09:37:05.026 INFO:tasks.rados.rados.0.smithi022.stdout:1696: read oid 35 snap -1 2024-09-21T09:37:05.026 INFO:tasks.rados.rados.0.smithi022.stdout:1696: expect (ObjNum 471 snap 142 seq_num 471) 2024-09-21T09:37:05.026 INFO:tasks.rados.rados.0.smithi022.stdout:1697: rollback oid 3 current snap is 166 2024-09-21T09:37:05.026 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 3 to 158 2024-09-21T09:37:05.026 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 1 current snap is 166 2024-09-21T09:37:05.026 INFO:tasks.rados.rados.0.smithi022.stdout:1698: seq_num 552 ranges {1228800=450560,1679360=532480,2211840=655360,2867200=417792} 2024-09-21T09:37:05.037 INFO:tasks.rados.rados.0.smithi022.stdout:1698: writing smithi02284960-1 from 1228800 to 1679360 tid 1 2024-09-21T09:37:05.040 INFO:tasks.rados.rados.0.smithi022.stdout:1698: writing smithi02284960-1 from 1679360 to 2211840 tid 2 2024-09-21T09:37:05.044 INFO:tasks.rados.rados.0.smithi022.stdout:1698: writing smithi02284960-1 from 2211840 to 2867200 tid 3 2024-09-21T09:37:05.047 INFO:tasks.rados.rados.0.smithi022.stdout:1698: writing smithi02284960-1 from 2867200 to 3284992 tid 4 2024-09-21T09:37:05.047 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 11 current snap is 166 2024-09-21T09:37:05.047 INFO:tasks.rados.rados.0.smithi022.stdout:1699: seq_num 553 ranges {4513792=466944,4980736=417792,5398528=417792,5816320=516096,6332416=253952} 2024-09-21T09:37:05.074 INFO:tasks.rados.rados.0.smithi022.stdout:1699: writing smithi02284960-11 from 4513792 to 4980736 tid 1 2024-09-21T09:37:05.077 INFO:tasks.rados.rados.0.smithi022.stdout:1699: writing smithi02284960-11 from 4980736 to 5398528 tid 2 2024-09-21T09:37:05.079 INFO:tasks.rados.rados.0.smithi022.stdout:1699: writing smithi02284960-11 from 5398528 to 5816320 tid 3 2024-09-21T09:37:05.082 INFO:tasks.rados.rados.0.smithi022.stdout:1699: writing smithi02284960-11 from 5816320 to 6332416 tid 4 2024-09-21T09:37:05.084 INFO:tasks.rados.rados.0.smithi022.stdout:1699: writing smithi02284960-11 from 6332416 to 6586368 tid 5 2024-09-21T09:37:05.084 INFO:tasks.rados.rados.0.smithi022.stdout:1700: copy_from oid 19 from oid 21 current snap is 166 2024-09-21T09:37:05.084 INFO:tasks.rados.rados.0.smithi022.stdout:1697: finishing rollback tid 0 to smithi02284960-3 2024-09-21T09:37:05.084 INFO:tasks.rados.rados.0.smithi022.stdout:1697: finishing rollback tid 1 to smithi02284960-3 2024-09-21T09:37:05.084 INFO:tasks.rados.rados.0.smithi022.stdout:1697: finishing rollback tid 2 to smithi02284960-3 2024-09-21T09:37:05.084 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 3 v 595 (ObjNum 509 snap 151 seq_num 509) dirty exists 2024-09-21T09:37:05.084 INFO:tasks.rados.rados.0.smithi022.stdout:1696: expect (ObjNum 471 snap 142 seq_num 471) 2024-09-21T09:37:05.171 INFO:tasks.rados.rados.0.smithi022.stdout:1695: expect (ObjNum 450 snap 136 seq_num 450) 2024-09-21T09:37:05.181 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 23 v 850 (ObjNum 423 snap 132 seq_num 423) dirty exists 2024-09-21T09:37:05.181 INFO:tasks.rados.rados.0.smithi022.stdout:1698: finishing write tid 1 to smithi02284960-1 2024-09-21T09:37:05.181 INFO:tasks.rados.rados.0.smithi022.stdout:1694: finishing copy_from racing read to smithi02284960-42 2024-09-21T09:37:05.181 INFO:tasks.rados.rados.0.smithi022.stdout:1698: finishing write tid 2 to smithi02284960-1 2024-09-21T09:37:05.181 INFO:tasks.rados.rados.0.smithi022.stdout:1694: finishing copy_from to smithi02284960-42 2024-09-21T09:37:05.181 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 42 v 553 (ObjNum 392 snap 122 seq_num 392) dirty exists 2024-09-21T09:37:05.181 INFO:tasks.rados.rados.0.smithi022.stdout:1693: done (7 left) 2024-09-21T09:37:05.181 INFO:tasks.rados.rados.0.smithi022.stdout:1694: done (6 left) 2024-09-21T09:37:05.181 INFO:tasks.rados.rados.0.smithi022.stdout:1695: done (5 left) 2024-09-21T09:37:05.181 INFO:tasks.rados.rados.0.smithi022.stdout:1696: done (4 left) 2024-09-21T09:37:05.181 INFO:tasks.rados.rados.0.smithi022.stdout:1697: done (3 left) 2024-09-21T09:37:05.181 INFO:tasks.rados.rados.0.smithi022.stdout:1701: snap_remove snap 158 2024-09-21T09:37:05.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:05 smithi064 ceph-mon[86709]: pgmap v208: 121 pgs: 27 active+undersized, 9 active+undersized+degraded, 85 active+clean; 99 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2.3 MiB/s wr, 13 op/s; 44/273 objects degraded (16.117%) 2024-09-21T09:37:05.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:05 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:37:05.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:05 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:37:05.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:05 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:37:05.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:05 smithi064 ceph-mon[86709]: osdmap e404: 8 total, 7 up, 8 in 2024-09-21T09:37:05.717 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:05 smithi022 ceph-mon[98615]: pgmap v208: 121 pgs: 27 active+undersized, 9 active+undersized+degraded, 85 active+clean; 99 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2.3 MiB/s wr, 13 op/s; 44/273 objects degraded (16.117%) 2024-09-21T09:37:05.717 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:05 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:37:05.717 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:05 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:37:05.717 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:05 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:37:05.717 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:05 smithi022 ceph-mon[98615]: osdmap e404: 8 total, 7 up, 8 in 2024-09-21T09:37:05.718 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:05 smithi022 ceph-mon[96736]: pgmap v208: 121 pgs: 27 active+undersized, 9 active+undersized+degraded, 85 active+clean; 99 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2.3 MiB/s wr, 13 op/s; 44/273 objects degraded (16.117%) 2024-09-21T09:37:05.718 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:05 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:37:05.718 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:05 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:37:05.718 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:05 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:37:05.718 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:05 smithi022 ceph-mon[96736]: osdmap e404: 8 total, 7 up, 8 in 2024-09-21T09:37:06.004 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:37:05 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:37:05] "GET /metrics HTTP/1.1" 200 36242 "" "Prometheus/2.51.0" 2024-09-21T09:37:06.005 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 21 09:37:05 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-2[113367]: 2024-09-21T09:37:05.892+0000 7f02fde00640 -1 Falling back to public interface 2024-09-21T09:37:06.021 INFO:tasks.rados.rados.0.smithi022.stdout:1701: done (3 left) 2024-09-21T09:37:06.021 INFO:tasks.rados.rados.0.smithi022.stdout:1702: snap_create 2024-09-21T09:37:06.021 INFO:tasks.rados.rados.0.smithi022.stdout:1698: finishing write tid 3 to smithi02284960-1 2024-09-21T09:37:06.021 INFO:tasks.rados.rados.0.smithi022.stdout:1698: finishing write tid 4 to smithi02284960-1 2024-09-21T09:37:06.021 INFO:tasks.rados.rados.0.smithi022.stdout:1698: finishing write tid 5 to smithi02284960-1 2024-09-21T09:37:06.021 INFO:tasks.rados.rados.0.smithi022.stdout:1698: finishing write tid 6 to smithi02284960-1 2024-09-21T09:37:06.021 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 1 v 1005 (ObjNum 552 snap 166 seq_num 552) dirty exists 2024-09-21T09:37:06.022 INFO:tasks.rados.rados.0.smithi022.stdout:1698: left oid 1 (ObjNum 552 snap 166 seq_num 552) 2024-09-21T09:37:06.022 INFO:tasks.rados.rados.0.smithi022.stdout:1699: finishing write tid 1 to smithi02284960-11 2024-09-21T09:37:06.022 INFO:tasks.rados.rados.0.smithi022.stdout:1699: finishing write tid 2 to smithi02284960-11 2024-09-21T09:37:06.022 INFO:tasks.rados.rados.0.smithi022.stdout:1699: finishing write tid 3 to smithi02284960-11 2024-09-21T09:37:06.022 INFO:tasks.rados.rados.0.smithi022.stdout:1700: finishing copy_from to smithi02284960-19 2024-09-21T09:37:06.022 INFO:tasks.rados.rados.0.smithi022.stdout:1700: got expected ENOENT (src dne) 2024-09-21T09:37:06.022 INFO:tasks.rados.rados.0.smithi022.stdout:1700: finishing copy_from racing read to smithi02284960-19 2024-09-21T09:37:06.022 INFO:tasks.rados.rados.0.smithi022.stdout:1699: finishing write tid 4 to smithi02284960-11 2024-09-21T09:37:06.022 INFO:tasks.rados.rados.0.smithi022.stdout:1699: finishing write tid 5 to smithi02284960-11 2024-09-21T09:37:06.022 INFO:tasks.rados.rados.0.smithi022.stdout:1699: finishing write tid 6 to smithi02284960-11 2024-09-21T09:37:06.022 INFO:tasks.rados.rados.0.smithi022.stdout:1699: finishing write tid 7 to smithi02284960-11 2024-09-21T09:37:06.022 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 11 v 1012 (ObjNum 553 snap 166 seq_num 553) dirty exists 2024-09-21T09:37:06.022 INFO:tasks.rados.rados.0.smithi022.stdout:1699: left oid 11 (ObjNum 553 snap 166 seq_num 553) 2024-09-21T09:37:06.191 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:37:05 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:37:05.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:37:06.192 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:37:05 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:37:05.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:37:07.027 INFO:tasks.rados.rados.0.smithi022.stdout:1698: done (3 left) 2024-09-21T09:37:07.027 INFO:tasks.rados.rados.0.smithi022.stdout:1699: done (2 left) 2024-09-21T09:37:07.028 INFO:tasks.rados.rados.0.smithi022.stdout:1700: done (1 left) 2024-09-21T09:37:07.028 INFO:tasks.rados.rados.0.smithi022.stdout:1702: done (0 left) 2024-09-21T09:37:07.028 INFO:tasks.rados.rados.0.smithi022.stdout:1703: delete oid 39 current snap is 167 2024-09-21T09:37:07.031 INFO:tasks.rados.rados.0.smithi022.stdout:1703: done (0 left) 2024-09-21T09:37:07.032 INFO:tasks.rados.rados.0.smithi022.stdout:1704: snap_remove snap 163 2024-09-21T09:37:07.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:07 smithi022 ceph-mon[98615]: osdmap e405: 8 total, 7 up, 8 in 2024-09-21T09:37:07.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:07 smithi022 ceph-mon[96736]: osdmap e405: 8 total, 7 up, 8 in 2024-09-21T09:37:07.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:07 smithi064 ceph-mon[86709]: osdmap e405: 8 total, 7 up, 8 in 2024-09-21T09:37:07.442 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:37:06 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:37:06.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi022\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:37:08.034 INFO:tasks.rados.rados.0.smithi022.stdout:1704: done (0 left) 2024-09-21T09:37:08.034 INFO:tasks.rados.rados.0.smithi022.stdout:1705: snap_remove snap 165 2024-09-21T09:37:08.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:08 smithi022 ceph-mon[98615]: pgmap v211: 121 pgs: 27 active+undersized, 9 active+undersized+degraded, 85 active+clean; 99 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 0 B/s rd, 2.3 MiB/s wr, 13 op/s; 44/276 objects degraded (15.942%) 2024-09-21T09:37:08.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:08 smithi022 ceph-mon[98615]: osdmap e406: 8 total, 7 up, 8 in 2024-09-21T09:37:08.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:08 smithi022 ceph-mon[96736]: pgmap v211: 121 pgs: 27 active+undersized, 9 active+undersized+degraded, 85 active+clean; 99 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 0 B/s rd, 2.3 MiB/s wr, 13 op/s; 44/276 objects degraded (15.942%) 2024-09-21T09:37:08.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:08 smithi022 ceph-mon[96736]: osdmap e406: 8 total, 7 up, 8 in 2024-09-21T09:37:08.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:08 smithi064 ceph-mon[86709]: pgmap v211: 121 pgs: 27 active+undersized, 9 active+undersized+degraded, 85 active+clean; 99 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 0 B/s rd, 2.3 MiB/s wr, 13 op/s; 44/276 objects degraded (15.942%) 2024-09-21T09:37:08.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:08 smithi064 ceph-mon[86709]: osdmap e406: 8 total, 7 up, 8 in 2024-09-21T09:37:09.039 INFO:tasks.rados.rados.0.smithi022.stdout:1705: done (0 left) 2024-09-21T09:37:09.040 INFO:tasks.rados.rados.0.smithi022.stdout:1706: rollback oid 19 current snap is 167 2024-09-21T09:37:09.040 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 19 to 166 2024-09-21T09:37:09.040 INFO:tasks.rados.rados.0.smithi022.stdout:1707: rmattr oid 33 current snap is 167 2024-09-21T09:37:09.040 INFO:tasks.rados.rados.0.smithi022.stdout:1707: done (1 left) 2024-09-21T09:37:09.040 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 23 current snap is 167 2024-09-21T09:37:09.040 INFO:tasks.rados.rados.0.smithi022.stdout:1708: seq_num 555 ranges {2908160=499712,3407872=638976,4046848=425984,4472832=540672} 2024-09-21T09:37:09.058 INFO:tasks.rados.rados.0.smithi022.stdout:1708: writing smithi02284960-23 from 2908160 to 3407872 tid 1 2024-09-21T09:37:09.062 INFO:tasks.rados.rados.0.smithi022.stdout:1708: writing smithi02284960-23 from 3407872 to 4046848 tid 2 2024-09-21T09:37:09.065 INFO:tasks.rados.rados.0.smithi022.stdout:1708: writing smithi02284960-23 from 4046848 to 4472832 tid 3 2024-09-21T09:37:09.069 INFO:tasks.rados.rados.0.smithi022.stdout:1708: writing smithi02284960-23 from 4472832 to 5013504 tid 4 2024-09-21T09:37:09.069 INFO:tasks.rados.rados.0.smithi022.stdout:1709: rmattr oid 37 current snap is 167 2024-09-21T09:37:09.069 INFO:tasks.rados.rados.0.smithi022.stdout:1706: finishing rollback tid 0 to smithi02284960-19 2024-09-21T09:37:09.069 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 30 current snap is 167 2024-09-21T09:37:09.069 INFO:tasks.rados.rados.0.smithi022.stdout:1706: finishing rollback tid 1 to smithi02284960-19 2024-09-21T09:37:09.069 INFO:tasks.rados.rados.0.smithi022.stdout:1706: finishing rollback tid 2 to smithi02284960-19 2024-09-21T09:37:09.069 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 19 v 1135 (ObjNum 507 snap 151 seq_num 507) dirty exists 2024-09-21T09:37:09.069 INFO:tasks.rados.rados.0.smithi022.stdout:1708: finishing write tid 1 to smithi02284960-23 2024-09-21T09:37:09.069 INFO:tasks.rados.rados.0.smithi022.stdout:1708: finishing write tid 2 to smithi02284960-23 2024-09-21T09:37:09.069 INFO:tasks.rados.rados.0.smithi022.stdout:1710: seq_num 556 ranges {0=516096,516096=524288,1040384=638976,1679360=442368} 2024-09-21T09:37:09.072 INFO:tasks.rados.rados.0.smithi022.stdout:1710: writing smithi02284960-30 from 0 to 516096 tid 1 2024-09-21T09:37:09.075 INFO:tasks.rados.rados.0.smithi022.stdout:1710: writing smithi02284960-30 from 516096 to 1040384 tid 2 2024-09-21T09:37:09.079 INFO:tasks.rados.rados.0.smithi022.stdout:1710: writing smithi02284960-30 from 1040384 to 1679360 tid 3 2024-09-21T09:37:09.081 INFO:tasks.rados.rados.0.smithi022.stdout:1710: writing smithi02284960-30 from 1679360 to 2121728 tid 4 2024-09-21T09:37:09.082 INFO:tasks.rados.rados.0.smithi022.stdout:1706: done (3 left) 2024-09-21T09:37:09.082 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 18 current snap is 167 2024-09-21T09:37:09.082 INFO:tasks.rados.rados.0.smithi022.stdout:1708: finishing write tid 3 to smithi02284960-23 2024-09-21T09:37:09.082 INFO:tasks.rados.rados.0.smithi022.stdout:1708: finishing write tid 4 to smithi02284960-23 2024-09-21T09:37:09.082 INFO:tasks.rados.rados.0.smithi022.stdout:1711: seq_num 557 ranges {876544=532480,1409024=802816,2211840=794624,3006464=8192} 2024-09-21T09:37:09.089 INFO:tasks.rados.rados.0.smithi022.stdout:1711: writing smithi02284960-18 from 876544 to 1409024 tid 1 2024-09-21T09:37:09.094 INFO:tasks.rados.rados.0.smithi022.stdout:1711: writing smithi02284960-18 from 1409024 to 2211840 tid 2 2024-09-21T09:37:09.099 INFO:tasks.rados.rados.0.smithi022.stdout:1711: writing smithi02284960-18 from 2211840 to 3006464 tid 3 2024-09-21T09:37:09.099 INFO:tasks.rados.rados.0.smithi022.stdout:1711: writing smithi02284960-18 from 3006464 to 3014656 tid 4 2024-09-21T09:37:09.099 INFO:tasks.rados.rados.0.smithi022.stdout:1712: read oid 26 snap -1 2024-09-21T09:37:09.099 INFO:tasks.rados.rados.0.smithi022.stdout:1712: expect (ObjNum 450 snap 136 seq_num 450) 2024-09-21T09:37:09.099 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 5 current snap is 167 2024-09-21T09:37:09.099 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 37 v 1137 (ObjNum 514 snap 153 seq_num 514) dirty exists 2024-09-21T09:37:09.099 INFO:tasks.rados.rados.0.smithi022.stdout:1708: finishing write tid 5 to smithi02284960-23 2024-09-21T09:37:09.099 INFO:tasks.rados.rados.0.smithi022.stdout:1713: seq_num 558 ranges {6225920=548864,6774784=688128,7462912=606208,8069120=311296} 2024-09-21T09:37:09.137 INFO:tasks.rados.rados.0.smithi022.stdout:1713: writing smithi02284960-5 from 6225920 to 6774784 tid 1 2024-09-21T09:37:09.141 INFO:tasks.rados.rados.0.smithi022.stdout:1713: writing smithi02284960-5 from 6774784 to 7462912 tid 2 2024-09-21T09:37:09.144 INFO:tasks.rados.rados.0.smithi022.stdout:1713: writing smithi02284960-5 from 7462912 to 8069120 tid 3 2024-09-21T09:37:09.146 INFO:tasks.rados.rados.0.smithi022.stdout:1713: writing smithi02284960-5 from 8069120 to 8380416 tid 4 2024-09-21T09:37:09.146 INFO:tasks.rados.rados.0.smithi022.stdout:1709: done (5 left) 2024-09-21T09:37:09.146 INFO:tasks.rados.rados.0.smithi022.stdout:1714: snap_remove snap 162 2024-09-21T09:37:09.300 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:09 smithi022 ceph-mon[98615]: osdmap e407: 8 total, 7 up, 8 in 2024-09-21T09:37:09.300 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:09 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:37:09.300 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:09 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:37:09.300 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:09 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:37:09.300 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:09 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:37:09.300 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:09 smithi022 ceph-mon[98615]: Health check update: Degraded data redundancy: 44/276 objects degraded (15.942%), 9 pgs degraded (PG_DEGRADED) 2024-09-21T09:37:09.301 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:09 smithi022 ceph-mon[96736]: osdmap e407: 8 total, 7 up, 8 in 2024-09-21T09:37:09.301 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:09 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:37:09.301 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:09 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:37:09.301 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:09 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:37:09.301 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:09 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:37:09.301 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:09 smithi022 ceph-mon[96736]: Health check update: Degraded data redundancy: 44/276 objects degraded (15.942%), 9 pgs degraded (PG_DEGRADED) 2024-09-21T09:37:09.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:09 smithi064 ceph-mon[86709]: osdmap e407: 8 total, 7 up, 8 in 2024-09-21T09:37:09.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:09 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:37:09.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:09 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:37:09.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:09 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:37:09.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:09 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:37:09.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:09 smithi064 ceph-mon[86709]: Health check update: Degraded data redundancy: 44/276 objects degraded (15.942%), 9 pgs degraded (PG_DEGRADED) 2024-09-21T09:37:10.098 INFO:tasks.rados.rados.0.smithi022.stdout:1714: done (5 left) 2024-09-21T09:37:10.098 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 39 current snap is 167 2024-09-21T09:37:10.098 INFO:tasks.rados.rados.0.smithi022.stdout:1715: seq_num 559 ranges {0=565248,565248=573440,1138688=409600,1548288=622592} 2024-09-21T09:37:10.103 INFO:tasks.rados.rados.0.smithi022.stdout:1715: writing smithi02284960-39 from 0 to 565248 tid 1 2024-09-21T09:37:10.106 INFO:tasks.rados.rados.0.smithi022.stdout:1715: writing smithi02284960-39 from 565248 to 1138688 tid 2 2024-09-21T09:37:10.109 INFO:tasks.rados.rados.0.smithi022.stdout:1715: writing smithi02284960-39 from 1138688 to 1548288 tid 3 2024-09-21T09:37:10.113 INFO:tasks.rados.rados.0.smithi022.stdout:1715: writing smithi02284960-39 from 1548288 to 2170880 tid 4 2024-09-21T09:37:10.113 INFO:tasks.rados.rados.0.smithi022.stdout:1716: snap_create 2024-09-21T09:37:10.113 INFO:tasks.rados.rados.0.smithi022.stdout:1708: finishing write tid 6 to smithi02284960-23 2024-09-21T09:37:10.113 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 23 v 855 (ObjNum 555 snap 167 seq_num 555) dirty exists 2024-09-21T09:37:10.114 INFO:tasks.rados.rados.0.smithi022.stdout:1708: left oid 23 (ObjNum 555 snap 167 seq_num 555) 2024-09-21T09:37:10.114 INFO:tasks.rados.rados.0.smithi022.stdout:1710: finishing write tid 1 to smithi02284960-30 2024-09-21T09:37:10.114 INFO:tasks.rados.rados.0.smithi022.stdout:1710: finishing write tid 2 to smithi02284960-30 2024-09-21T09:37:10.114 INFO:tasks.rados.rados.0.smithi022.stdout:1710: finishing write tid 3 to smithi02284960-30 2024-09-21T09:37:10.114 INFO:tasks.rados.rados.0.smithi022.stdout:1710: finishing write tid 4 to smithi02284960-30 2024-09-21T09:37:10.114 INFO:tasks.rados.rados.0.smithi022.stdout:1710: finishing write tid 5 to smithi02284960-30 2024-09-21T09:37:10.114 INFO:tasks.rados.rados.0.smithi022.stdout:1710: finishing write tid 6 to smithi02284960-30 2024-09-21T09:37:10.114 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 30 v 924 (ObjNum 556 snap 167 seq_num 556) dirty exists 2024-09-21T09:37:10.114 INFO:tasks.rados.rados.0.smithi022.stdout:1710: left oid 30 (ObjNum 556 snap 167 seq_num 556) 2024-09-21T09:37:10.114 INFO:tasks.rados.rados.0.smithi022.stdout:1711: finishing write tid 1 to smithi02284960-18 2024-09-21T09:37:10.114 INFO:tasks.rados.rados.0.smithi022.stdout:1711: finishing write tid 2 to smithi02284960-18 2024-09-21T09:37:10.114 INFO:tasks.rados.rados.0.smithi022.stdout:1712: expect (ObjNum 450 snap 136 seq_num 450) 2024-09-21T09:37:10.125 INFO:tasks.rados.rados.0.smithi022.stdout:1711: finishing write tid 3 to smithi02284960-18 2024-09-21T09:37:10.125 INFO:tasks.rados.rados.0.smithi022.stdout:1711: finishing write tid 4 to smithi02284960-18 2024-09-21T09:37:10.125 INFO:tasks.rados.rados.0.smithi022.stdout:1711: finishing write tid 5 to smithi02284960-18 2024-09-21T09:37:10.125 INFO:tasks.rados.rados.0.smithi022.stdout:1711: finishing write tid 6 to smithi02284960-18 2024-09-21T09:37:10.125 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 18 v 953 (ObjNum 557 snap 167 seq_num 557) dirty exists 2024-09-21T09:37:10.125 INFO:tasks.rados.rados.0.smithi022.stdout:1711: left oid 18 (ObjNum 557 snap 167 seq_num 557) 2024-09-21T09:37:10.125 INFO:tasks.rados.rados.0.smithi022.stdout:1713: finishing write tid 1 to smithi02284960-5 2024-09-21T09:37:10.125 INFO:tasks.rados.rados.0.smithi022.stdout:1713: finishing write tid 2 to smithi02284960-5 2024-09-21T09:37:10.125 INFO:tasks.rados.rados.0.smithi022.stdout:1713: finishing write tid 3 to smithi02284960-5 2024-09-21T09:37:10.126 INFO:tasks.rados.rados.0.smithi022.stdout:1713: finishing write tid 4 to smithi02284960-5 2024-09-21T09:37:10.126 INFO:tasks.rados.rados.0.smithi022.stdout:1713: finishing write tid 5 to smithi02284960-5 2024-09-21T09:37:10.126 INFO:tasks.rados.rados.0.smithi022.stdout:1713: finishing write tid 6 to smithi02284960-5 2024-09-21T09:37:10.126 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 5 v 636 (ObjNum 558 snap 167 seq_num 558) dirty exists 2024-09-21T09:37:10.126 INFO:tasks.rados.rados.0.smithi022.stdout:1713: left oid 5 (ObjNum 558 snap 167 seq_num 558) 2024-09-21T09:37:10.126 INFO:tasks.rados.rados.0.smithi022.stdout:1715: finishing write tid 1 to smithi02284960-39 2024-09-21T09:37:10.126 INFO:tasks.rados.rados.0.smithi022.stdout:1715: finishing write tid 2 to smithi02284960-39 2024-09-21T09:37:10.126 INFO:tasks.rados.rados.0.smithi022.stdout:1715: finishing write tid 3 to smithi02284960-39 2024-09-21T09:37:10.126 INFO:tasks.rados.rados.0.smithi022.stdout:1715: finishing write tid 4 to smithi02284960-39 2024-09-21T09:37:10.126 INFO:tasks.rados.rados.0.smithi022.stdout:1715: finishing write tid 5 to smithi02284960-39 2024-09-21T09:37:10.126 INFO:tasks.rados.rados.0.smithi022.stdout:1715: finishing write tid 6 to smithi02284960-39 2024-09-21T09:37:10.126 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 39 v 929 (ObjNum 559 snap 167 seq_num 559) dirty exists 2024-09-21T09:37:10.127 INFO:tasks.rados.rados.0.smithi022.stdout:1715: left oid 39 (ObjNum 559 snap 167 seq_num 559) 2024-09-21T09:37:10.189 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:10 smithi022 ceph-mon[98615]: pgmap v214: 121 pgs: 27 active+undersized, 9 active+undersized+degraded, 85 active+clean; 108 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 4.3 MiB/s rd, 2.4 MiB/s wr, 17 op/s; 47/285 objects degraded (16.491%) 2024-09-21T09:37:10.189 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:10 smithi022 ceph-mon[98615]: osdmap e408: 8 total, 7 up, 8 in 2024-09-21T09:37:10.189 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:10 smithi022 ceph-mon[98615]: from='osd.2 [v2:172.21.15.22:6818/4029603729,v1:172.21.15.22:6819/4029603729]' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-21T09:37:10.189 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:10 smithi022 ceph-mon[98615]: from='osd.2 ' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-21T09:37:10.189 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:10 smithi022 ceph-mon[96736]: pgmap v214: 121 pgs: 27 active+undersized, 9 active+undersized+degraded, 85 active+clean; 108 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 4.3 MiB/s rd, 2.4 MiB/s wr, 17 op/s; 47/285 objects degraded (16.491%) 2024-09-21T09:37:10.190 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:10 smithi022 ceph-mon[96736]: osdmap e408: 8 total, 7 up, 8 in 2024-09-21T09:37:10.190 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:10 smithi022 ceph-mon[96736]: from='osd.2 [v2:172.21.15.22:6818/4029603729,v1:172.21.15.22:6819/4029603729]' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-21T09:37:10.190 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:10 smithi022 ceph-mon[96736]: from='osd.2 ' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-21T09:37:10.190 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 21 09:37:09 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-2[113367]: 2024-09-21T09:37:09.879+0000 7f02fde00640 -1 osd.2 397 log_to_monitors true 2024-09-21T09:37:10.190 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 21 09:37:10 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-2[113367]: 2024-09-21T09:37:10.120+0000 7f02f5baa640 -1 osd.2 397 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T09:37:10.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:10 smithi064 ceph-mon[86709]: pgmap v214: 121 pgs: 27 active+undersized, 9 active+undersized+degraded, 85 active+clean; 108 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 4.3 MiB/s rd, 2.4 MiB/s wr, 17 op/s; 47/285 objects degraded (16.491%) 2024-09-21T09:37:10.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:10 smithi064 ceph-mon[86709]: osdmap e408: 8 total, 7 up, 8 in 2024-09-21T09:37:10.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:10 smithi064 ceph-mon[86709]: from='osd.2 [v2:172.21.15.22:6818/4029603729,v1:172.21.15.22:6819/4029603729]' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-21T09:37:10.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:10 smithi064 ceph-mon[86709]: from='osd.2 ' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-21T09:37:10.442 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:37:10 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:37:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:37:11.107 INFO:tasks.rados.rados.0.smithi022.stdout:1708: done (6 left) 2024-09-21T09:37:11.107 INFO:tasks.rados.rados.0.smithi022.stdout:1710: done (5 left) 2024-09-21T09:37:11.107 INFO:tasks.rados.rados.0.smithi022.stdout:1711: done (4 left) 2024-09-21T09:37:11.107 INFO:tasks.rados.rados.0.smithi022.stdout:1712: done (3 left) 2024-09-21T09:37:11.107 INFO:tasks.rados.rados.0.smithi022.stdout:1713: done (2 left) 2024-09-21T09:37:11.107 INFO:tasks.rados.rados.0.smithi022.stdout:1715: done (1 left) 2024-09-21T09:37:11.107 INFO:tasks.rados.rados.0.smithi022.stdout:1716: done (0 left) 2024-09-21T09:37:11.107 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 44 current snap is 168 2024-09-21T09:37:11.107 INFO:tasks.rados.rados.0.smithi022.stdout:1717: seq_num 560 ranges {0=581632,581632=458752,1040384=614400,1654784=532480} 2024-09-21T09:37:11.111 INFO:tasks.rados.rados.0.smithi022.stdout:1717: writing smithi02284960-44 from 0 to 581632 tid 1 2024-09-21T09:37:11.114 INFO:tasks.rados.rados.0.smithi022.stdout:1717: writing smithi02284960-44 from 581632 to 1040384 tid 2 2024-09-21T09:37:11.116 INFO:teuthology.orchestra.run.smithi022.stdout:true 2024-09-21T09:37:11.119 INFO:tasks.rados.rados.0.smithi022.stdout:1717: writing smithi02284960-44 from 1040384 to 1654784 tid 3 2024-09-21T09:37:11.123 INFO:tasks.rados.rados.0.smithi022.stdout:1717: writing smithi02284960-44 from 1654784 to 2187264 tid 4 2024-09-21T09:37:11.123 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 48 current snap is 168 2024-09-21T09:37:11.123 INFO:tasks.rados.rados.0.smithi022.stdout:1718: seq_num 561 ranges {5464064=598016,6062080=745472,6807552=778240,7585792=81920} 2024-09-21T09:37:11.157 INFO:tasks.rados.rados.0.smithi022.stdout:1718: writing smithi02284960-48 from 5464064 to 6062080 tid 1 2024-09-21T09:37:11.161 INFO:tasks.rados.rados.0.smithi022.stdout:1718: writing smithi02284960-48 from 6062080 to 6807552 tid 2 2024-09-21T09:37:11.166 INFO:tasks.rados.rados.0.smithi022.stdout:1718: writing smithi02284960-48 from 6807552 to 7585792 tid 3 2024-09-21T09:37:11.167 INFO:tasks.rados.rados.0.smithi022.stdout:1718: writing smithi02284960-48 from 7585792 to 7667712 tid 4 2024-09-21T09:37:11.167 INFO:tasks.rados.rados.0.smithi022.stdout:1719: delete oid 25 current snap is 168 2024-09-21T09:37:11.170 INFO:tasks.rados.rados.0.smithi022.stdout:1719: done (2 left) 2024-09-21T09:37:11.170 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 17 current snap is 168 2024-09-21T09:37:11.170 INFO:tasks.rados.rados.0.smithi022.stdout:1720: seq_num 563 ranges {2621440=638976,3260416=565248,3825664=794624,4620288=245760} 2024-09-21T09:37:11.188 INFO:tasks.rados.rados.0.smithi022.stdout:1720: writing smithi02284960-17 from 2621440 to 3260416 tid 1 2024-09-21T09:37:11.192 INFO:tasks.rados.rados.0.smithi022.stdout:1720: writing smithi02284960-17 from 3260416 to 3825664 tid 2 2024-09-21T09:37:11.197 INFO:tasks.rados.rados.0.smithi022.stdout:1720: writing smithi02284960-17 from 3825664 to 4620288 tid 3 2024-09-21T09:37:11.199 INFO:tasks.rados.rados.0.smithi022.stdout:1720: writing smithi02284960-17 from 4620288 to 4866048 tid 4 2024-09-21T09:37:11.199 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 30 current snap is 168 2024-09-21T09:37:11.199 INFO:tasks.rados.rados.0.smithi022.stdout:1721: seq_num 564 ranges {2121728=655360,2777088=442368,3219456=598016,3817472=565248} 2024-09-21T09:37:11.215 INFO:tasks.rados.rados.0.smithi022.stdout:1721: writing smithi02284960-30 from 2121728 to 2777088 tid 1 2024-09-21T09:37:11.218 INFO:tasks.rados.rados.0.smithi022.stdout:1721: writing smithi02284960-30 from 2777088 to 3219456 tid 2 2024-09-21T09:37:11.223 INFO:tasks.rados.rados.0.smithi022.stdout:1721: writing smithi02284960-30 from 3219456 to 3817472 tid 3 2024-09-21T09:37:11.227 INFO:tasks.rados.rados.0.smithi022.stdout:1721: writing smithi02284960-30 from 3817472 to 4382720 tid 4 2024-09-21T09:37:11.227 INFO:tasks.rados.rados.0.smithi022.stdout:1722: read oid 49 snap 166 2024-09-21T09:37:11.227 INFO:tasks.rados.rados.0.smithi022.stdout:1722: expect (ObjNum -725376656 snap 22094 seq_num 3569772432) 2024-09-21T09:37:11.227 INFO:tasks.rados.rados.0.smithi022.stdout:1723: snap_create 2024-09-21T09:37:11.227 INFO:tasks.rados.rados.0.smithi022.stdout:1720: finishing write tid 1 to smithi02284960-17 2024-09-21T09:37:11.227 INFO:tasks.rados.rados.0.smithi022.stdout:1720: finishing write tid 2 to smithi02284960-17 2024-09-21T09:37:11.227 INFO:tasks.rados.rados.0.smithi022.stdout:1720: finishing write tid 3 to smithi02284960-17 2024-09-21T09:37:11.228 INFO:tasks.rados.rados.0.smithi022.stdout:1720: finishing write tid 4 to smithi02284960-17 2024-09-21T09:37:11.228 INFO:tasks.rados.rados.0.smithi022.stdout:1720: finishing write tid 5 to smithi02284960-17 2024-09-21T09:37:11.228 INFO:tasks.rados.rados.0.smithi022.stdout:1720: finishing write tid 6 to smithi02284960-17 2024-09-21T09:37:11.228 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 17 v 934 (ObjNum 563 snap 168 seq_num 563) dirty exists 2024-09-21T09:37:11.228 INFO:tasks.rados.rados.0.smithi022.stdout:1720: left oid 17 (ObjNum 563 snap 168 seq_num 563) 2024-09-21T09:37:11.228 INFO:tasks.rados.rados.0.smithi022.stdout:1721: finishing write tid 1 to smithi02284960-30 2024-09-21T09:37:11.228 INFO:tasks.rados.rados.0.smithi022.stdout:1721: finishing write tid 2 to smithi02284960-30 2024-09-21T09:37:11.228 INFO:tasks.rados.rados.0.smithi022.stdout:1721: finishing write tid 3 to smithi02284960-30 2024-09-21T09:37:11.229 INFO:tasks.rados.rados.0.smithi022.stdout:1722: expect (ObjNum -725376656 snap 22094 seq_num 3569772432) 2024-09-21T09:37:11.231 INFO:tasks.rados.rados.0.smithi022.stdout:1721: finishing write tid 4 to smithi02284960-30 2024-09-21T09:37:11.231 INFO:tasks.rados.rados.0.smithi022.stdout:1721: finishing write tid 5 to smithi02284960-30 2024-09-21T09:37:11.231 INFO:tasks.rados.rados.0.smithi022.stdout:1721: finishing write tid 6 to smithi02284960-30 2024-09-21T09:37:11.231 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 30 v 939 (ObjNum 564 snap 168 seq_num 564) dirty exists 2024-09-21T09:37:11.231 INFO:tasks.rados.rados.0.smithi022.stdout:1721: left oid 30 (ObjNum 564 snap 168 seq_num 564) 2024-09-21T09:37:11.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:11 smithi064 ceph-mon[86709]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-21T09:37:11.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:11 smithi064 ceph-mon[86709]: osdmap e409: 8 total, 7 up, 8 in 2024-09-21T09:37:11.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:11 smithi064 ceph-mon[86709]: from='osd.2 [v2:172.21.15.22:6818/4029603729,v1:172.21.15.22:6819/4029603729]' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi022", "root=default"]} : dispatch 2024-09-21T09:37:11.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:11 smithi064 ceph-mon[86709]: from='osd.2 ' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi022", "root=default"]} : dispatch 2024-09-21T09:37:11.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:11 smithi064 ceph-mon[86709]: pgmap v217: 121 pgs: 27 active+undersized, 9 active+undersized+degraded, 85 active+clean; 108 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4.3 MiB/s rd, 2.4 MiB/s wr, 16 op/s; 47/285 objects degraded (16.491%) 2024-09-21T09:37:11.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:11 smithi022 ceph-mon[98615]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-21T09:37:11.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:11 smithi022 ceph-mon[98615]: osdmap e409: 8 total, 7 up, 8 in 2024-09-21T09:37:11.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:11 smithi022 ceph-mon[98615]: from='osd.2 [v2:172.21.15.22:6818/4029603729,v1:172.21.15.22:6819/4029603729]' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi022", "root=default"]} : dispatch 2024-09-21T09:37:11.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:11 smithi022 ceph-mon[98615]: from='osd.2 ' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi022", "root=default"]} : dispatch 2024-09-21T09:37:11.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:11 smithi022 ceph-mon[98615]: pgmap v217: 121 pgs: 27 active+undersized, 9 active+undersized+degraded, 85 active+clean; 108 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4.3 MiB/s rd, 2.4 MiB/s wr, 16 op/s; 47/285 objects degraded (16.491%) 2024-09-21T09:37:11.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:11 smithi022 ceph-mon[96736]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-21T09:37:11.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:11 smithi022 ceph-mon[96736]: osdmap e409: 8 total, 7 up, 8 in 2024-09-21T09:37:11.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:11 smithi022 ceph-mon[96736]: from='osd.2 [v2:172.21.15.22:6818/4029603729,v1:172.21.15.22:6819/4029603729]' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi022", "root=default"]} : dispatch 2024-09-21T09:37:11.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:11 smithi022 ceph-mon[96736]: from='osd.2 ' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi022", "root=default"]} : dispatch 2024-09-21T09:37:11.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:11 smithi022 ceph-mon[96736]: pgmap v217: 121 pgs: 27 active+undersized, 9 active+undersized+degraded, 85 active+clean; 108 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4.3 MiB/s rd, 2.4 MiB/s wr, 16 op/s; 47/285 objects degraded (16.491%) 2024-09-21T09:37:11.736 INFO:teuthology.orchestra.run.smithi022.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T09:37:11.736 INFO:teuthology.orchestra.run.smithi022.stdout:alertmanager.a smithi022 *:9093,9094 running (3m) 3s ago 8m 22.0M - 0.27.0 11f11916f8cd caced80ba544 2024-09-21T09:37:11.736 INFO:teuthology.orchestra.run.smithi022.stdout:grafana.a smithi064 *:3000 running (3m) 111s ago 8m 77.9M - 9.4.7 954c08fa6188 527bb657d70d 2024-09-21T09:37:11.736 INFO:teuthology.orchestra.run.smithi022.stdout:mds.a smithi022 running (9m) 3s ago 9m 18.0M - 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 34e4221f3a27 2024-09-21T09:37:11.736 INFO:teuthology.orchestra.run.smithi022.stdout:mds.b smithi064 running (9m) 111s ago 9m 20.9M - 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 0f846b3ec694 2024-09-21T09:37:11.736 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.x smithi064 *:8443,9283,8765 running (2m) 111s ago 13m 453M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 61abc61fea8d 2024-09-21T09:37:11.736 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.y smithi022 *:8443,9283,8765 running (3m) 3s ago 14m 513M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 6c5bee78f09b 2024-09-21T09:37:11.736 INFO:teuthology.orchestra.run.smithi022.stdout:mon.a smithi022 running (2m) 3s ago 15m 55.2M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 0513395c784d 2024-09-21T09:37:11.736 INFO:teuthology.orchestra.run.smithi022.stdout:mon.b smithi064 running (116s) 111s ago 13m 31.5M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 9dab26b6a9e7 2024-09-21T09:37:11.736 INFO:teuthology.orchestra.run.smithi022.stdout:mon.c smithi022 running (2m) 3s ago 13m 46.5M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 c74b58b67bd6 2024-09-21T09:37:11.736 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.a smithi022 *:9100 running (3m) 3s ago 8m 11.9M - 1.7.0 72c9c2088986 593c6158f524 2024-09-21T09:37:11.737 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.b smithi064 *:9100 running (3m) 111s ago 8m 9.94M - 1.7.0 72c9c2088986 a533420b0139 2024-09-21T09:37:11.737 INFO:teuthology.orchestra.run.smithi022.stdout:osd.0 smithi022 running (75s) 3s ago 13m 120M 2481M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 40ce23a2b569 2024-09-21T09:37:11.737 INFO:teuthology.orchestra.run.smithi022.stdout:osd.1 smithi022 running (41s) 3s ago 12m 82.7M 2481M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 7cd0c18d224c 2024-09-21T09:37:11.737 INFO:teuthology.orchestra.run.smithi022.stdout:osd.2 smithi022 running (7s) 3s ago 12m 12.1M 2481M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 b8c944306698 2024-09-21T09:37:11.737 INFO:teuthology.orchestra.run.smithi022.stdout:osd.3 smithi022 running (11m) 3s ago 11m 110M 2481M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be a56193b67a92 2024-09-21T09:37:11.737 INFO:teuthology.orchestra.run.smithi022.stdout:osd.4 smithi064 running (11m) 111s ago 11m 141M 2480M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 2a2937374fc4 2024-09-21T09:37:11.737 INFO:teuthology.orchestra.run.smithi022.stdout:osd.5 smithi064 running (10m) 111s ago 10m 192M 2480M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 1900c4af5d28 2024-09-21T09:37:11.737 INFO:teuthology.orchestra.run.smithi022.stdout:osd.6 smithi064 running (10m) 111s ago 10m 126M 2480M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 3cc4ec816e5b 2024-09-21T09:37:11.737 INFO:teuthology.orchestra.run.smithi022.stdout:osd.7 smithi064 running (10m) 111s ago 10m 148M 2480M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 18ef0be94104 2024-09-21T09:37:11.737 INFO:teuthology.orchestra.run.smithi022.stdout:prometheus.a smithi064 *:9095 running (2m) 111s ago 8m 44.0M - 2.51.0 1d3b7f56885b 80629ffa2179 2024-09-21T09:37:12.100 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-09-21T09:37:12.100 INFO:teuthology.orchestra.run.smithi022.stdout: "mon": { 2024-09-21T09:37:12.100 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 3 2024-09-21T09:37:12.100 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-21T09:37:12.100 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr": { 2024-09-21T09:37:12.100 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 2 2024-09-21T09:37:12.100 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-21T09:37:12.100 INFO:teuthology.orchestra.run.smithi022.stdout: "osd": { 2024-09-21T09:37:12.100 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 5, 2024-09-21T09:37:12.100 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 3 2024-09-21T09:37:12.100 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-21T09:37:12.100 INFO:teuthology.orchestra.run.smithi022.stdout: "mds": { 2024-09-21T09:37:12.100 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 2 2024-09-21T09:37:12.101 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-21T09:37:12.101 INFO:teuthology.orchestra.run.smithi022.stdout: "overall": { 2024-09-21T09:37:12.101 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 7, 2024-09-21T09:37:12.101 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 8 2024-09-21T09:37:12.101 INFO:teuthology.orchestra.run.smithi022.stdout: } 2024-09-21T09:37:12.101 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-09-21T09:37:12.108 INFO:tasks.rados.rados.0.smithi022.stdout:1720: done (5 left) 2024-09-21T09:37:12.108 INFO:tasks.rados.rados.0.smithi022.stdout:1721: done (4 left) 2024-09-21T09:37:12.108 INFO:tasks.rados.rados.0.smithi022.stdout:1722: done (3 left) 2024-09-21T09:37:12.108 INFO:tasks.rados.rados.0.smithi022.stdout:1723: done (2 left) 2024-09-21T09:37:12.108 INFO:tasks.rados.rados.0.smithi022.stdout:1724: snap_create 2024-09-21T09:37:12.134 INFO:tasks.rados.rados.0.smithi022.stdout:1717: finishing write tid 1 to smithi02284960-44 2024-09-21T09:37:12.136 INFO:tasks.rados.rados.0.smithi022.stdout:1717: finishing write tid 2 to smithi02284960-44 2024-09-21T09:37:12.137 INFO:tasks.rados.rados.0.smithi022.stdout:1717: finishing write tid 3 to smithi02284960-44 2024-09-21T09:37:12.138 INFO:tasks.rados.rados.0.smithi022.stdout:1717: finishing write tid 4 to smithi02284960-44 2024-09-21T09:37:12.139 INFO:tasks.rados.rados.0.smithi022.stdout:1717: finishing write tid 5 to smithi02284960-44 2024-09-21T09:37:12.140 INFO:tasks.rados.rados.0.smithi022.stdout:1717: finishing write tid 6 to smithi02284960-44 2024-09-21T09:37:12.140 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 44 v 1017 (ObjNum 560 snap 168 seq_num 560) dirty exists 2024-09-21T09:37:12.141 INFO:tasks.rados.rados.0.smithi022.stdout:1717: left oid 44 (ObjNum 560 snap 168 seq_num 560) 2024-09-21T09:37:12.156 INFO:tasks.rados.rados.0.smithi022.stdout:1718: finishing write tid 1 to smithi02284960-48 2024-09-21T09:37:12.156 INFO:tasks.rados.rados.0.smithi022.stdout:1718: finishing write tid 2 to smithi02284960-48 2024-09-21T09:37:12.159 INFO:tasks.rados.rados.0.smithi022.stdout:1718: finishing write tid 3 to smithi02284960-48 2024-09-21T09:37:12.159 INFO:tasks.rados.rados.0.smithi022.stdout:1718: finishing write tid 4 to smithi02284960-48 2024-09-21T09:37:12.159 INFO:tasks.rados.rados.0.smithi022.stdout:1718: finishing write tid 5 to smithi02284960-48 2024-09-21T09:37:12.162 INFO:tasks.rados.rados.0.smithi022.stdout:1718: finishing write tid 6 to smithi02284960-48 2024-09-21T09:37:12.162 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 48 v 1067 (ObjNum 561 snap 168 seq_num 561) dirty exists 2024-09-21T09:37:12.162 INFO:tasks.rados.rados.0.smithi022.stdout:1718: left oid 48 (ObjNum 561 snap 168 seq_num 561) 2024-09-21T09:37:12.308 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:37:12 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: 2024-09-21T09:37:12.068+0000 7feec1b50640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-09-21T09:37:12.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:12 smithi022 ceph-mon[98615]: from='client.44210 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:37:12.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:12 smithi022 ceph-mon[98615]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T09:37:12.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:12 smithi022 ceph-mon[98615]: osd.2 [v2:172.21.15.22:6818/4029603729,v1:172.21.15.22:6819/4029603729] boot 2024-09-21T09:37:12.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:12 smithi022 ceph-mon[98615]: osdmap e410: 8 total, 8 up, 8 in 2024-09-21T09:37:12.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:12 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-21T09:37:12.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:12 smithi022 ceph-mon[98615]: from='client.44216 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:37:12.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:12 smithi022 ceph-mon[98615]: from='client.44222 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:37:12.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:12 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:37:12.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:12 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:37:12.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:12 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:37:12.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:12 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:37:12.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:12 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:37:12.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:12 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:37:12.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:12 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:37:12.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:12 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:37:12.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:12 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:37:12.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:12 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-21T09:37:12.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:12 smithi022 ceph-mon[96736]: from='client.44210 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:37:12.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:12 smithi022 ceph-mon[96736]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T09:37:12.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:12 smithi022 ceph-mon[96736]: osd.2 [v2:172.21.15.22:6818/4029603729,v1:172.21.15.22:6819/4029603729] boot 2024-09-21T09:37:12.310 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:12 smithi022 ceph-mon[96736]: osdmap e410: 8 total, 8 up, 8 in 2024-09-21T09:37:12.310 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:12 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-21T09:37:12.310 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:12 smithi022 ceph-mon[96736]: from='client.44216 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:37:12.310 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:12 smithi022 ceph-mon[96736]: from='client.44222 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:37:12.310 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:12 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:37:12.310 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:12 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:37:12.310 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:12 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:37:12.310 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:12 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:37:12.310 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:12 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:37:12.310 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:12 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:37:12.310 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:12 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:37:12.310 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:12 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:37:12.310 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:12 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:37:12.310 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:12 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-21T09:37:12.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:12 smithi064 ceph-mon[86709]: from='client.44210 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:37:12.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:12 smithi064 ceph-mon[86709]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T09:37:12.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:12 smithi064 ceph-mon[86709]: osd.2 [v2:172.21.15.22:6818/4029603729,v1:172.21.15.22:6819/4029603729] boot 2024-09-21T09:37:12.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:12 smithi064 ceph-mon[86709]: osdmap e410: 8 total, 8 up, 8 in 2024-09-21T09:37:12.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:12 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-21T09:37:12.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:12 smithi064 ceph-mon[86709]: from='client.44216 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:37:12.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:12 smithi064 ceph-mon[86709]: from='client.44222 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:37:12.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:12 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:37:12.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:12 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:37:12.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:12 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:37:12.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:12 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:37:12.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:12 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:37:12.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:12 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:37:12.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:12 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:37:12.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:12 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:37:12.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:12 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:37:12.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:12 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-21T09:37:12.466 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-09-21T09:37:12.466 INFO:teuthology.orchestra.run.smithi022.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47", 2024-09-21T09:37:12.466 INFO:teuthology.orchestra.run.smithi022.stdout: "in_progress": true, 2024-09-21T09:37:12.467 INFO:teuthology.orchestra.run.smithi022.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T09:37:12.467 INFO:teuthology.orchestra.run.smithi022.stdout: "services_complete": [ 2024-09-21T09:37:12.467 INFO:teuthology.orchestra.run.smithi022.stdout: "mon", 2024-09-21T09:37:12.467 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr" 2024-09-21T09:37:12.467 INFO:teuthology.orchestra.run.smithi022.stdout: ], 2024-09-21T09:37:12.467 INFO:teuthology.orchestra.run.smithi022.stdout: "progress": "8/20 daemons upgraded", 2024-09-21T09:37:12.467 INFO:teuthology.orchestra.run.smithi022.stdout: "message": "Currently upgrading osd daemons", 2024-09-21T09:37:12.467 INFO:teuthology.orchestra.run.smithi022.stdout: "is_paused": false 2024-09-21T09:37:12.467 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-09-21T09:37:13.118 INFO:tasks.rados.rados.0.smithi022.stdout:1717: done (2 left) 2024-09-21T09:37:13.118 INFO:tasks.rados.rados.0.smithi022.stdout:1718: done (1 left) 2024-09-21T09:37:13.118 INFO:tasks.rados.rados.0.smithi022.stdout:1724: done (0 left) 2024-09-21T09:37:13.118 INFO:tasks.rados.rados.0.smithi022.stdout:1725: rollback oid 43 current snap is 170 2024-09-21T09:37:13.118 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 43 to 169 2024-09-21T09:37:13.119 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 4 current snap is 170 2024-09-21T09:37:13.119 INFO:tasks.rados.rados.0.smithi022.stdout:1726: seq_num 565 ranges {4947968=671744,5619712=729088,6348800=753664,7102464=122880} 2024-09-21T09:37:13.148 INFO:tasks.rados.rados.0.smithi022.stdout:1726: writing smithi02284960-4 from 4947968 to 5619712 tid 1 2024-09-21T09:37:13.153 INFO:tasks.rados.rados.0.smithi022.stdout:1726: writing smithi02284960-4 from 5619712 to 6348800 tid 2 2024-09-21T09:37:13.157 INFO:tasks.rados.rados.0.smithi022.stdout:1726: writing smithi02284960-4 from 6348800 to 7102464 tid 3 2024-09-21T09:37:13.159 INFO:tasks.rados.rados.0.smithi022.stdout:1726: writing smithi02284960-4 from 7102464 to 7225344 tid 4 2024-09-21T09:37:13.159 INFO:tasks.rados.rados.0.smithi022.stdout:1725: finishing rollback tid 1 to smithi02284960-43 2024-09-21T09:37:13.159 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 43 v 0 (ObjNum 129 snap 0 seq_num 3570201872) dirty dne 2024-09-21T09:37:13.159 INFO:tasks.rados.rados.0.smithi022.stdout:1727: read oid 18 snap -1 2024-09-21T09:37:13.159 INFO:tasks.rados.rados.0.smithi022.stdout:1727: expect (ObjNum 557 snap 167 seq_num 557) 2024-09-21T09:37:13.159 INFO:tasks.rados.rados.0.smithi022.stdout:1725: done (2 left) 2024-09-21T09:37:13.159 INFO:tasks.rados.rados.0.smithi022.stdout:1728: read oid 36 snap -1 2024-09-21T09:37:13.159 INFO:tasks.rados.rados.0.smithi022.stdout:1728: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-21T09:37:13.160 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 19 current snap is 170 2024-09-21T09:37:13.160 INFO:tasks.rados.rados.0.smithi022.stdout:1726: finishing write tid 1 to smithi02284960-4 2024-09-21T09:37:13.160 INFO:tasks.rados.rados.0.smithi022.stdout:1726: finishing write tid 2 to smithi02284960-4 2024-09-21T09:37:13.160 INFO:tasks.rados.rados.0.smithi022.stdout:1729: seq_num 566 ranges {2367488=688128,3055616=614400,3670016=565248,4235264=425984} 2024-09-21T09:37:13.176 INFO:tasks.rados.rados.0.smithi022.stdout:1729: writing smithi02284960-19 from 2367488 to 3055616 tid 1 2024-09-21T09:37:13.179 INFO:tasks.rados.rados.0.smithi022.stdout:1729: writing smithi02284960-19 from 3055616 to 3670016 tid 2 2024-09-21T09:37:13.182 INFO:tasks.rados.rados.0.smithi022.stdout:1729: writing smithi02284960-19 from 3670016 to 4235264 tid 3 2024-09-21T09:37:13.185 INFO:tasks.rados.rados.0.smithi022.stdout:1729: writing smithi02284960-19 from 4235264 to 4661248 tid 4 2024-09-21T09:37:13.185 INFO:tasks.rados.rados.0.smithi022.stdout:1730: copy_from oid 20 from oid 31 current snap is 170 2024-09-21T09:37:13.185 INFO:tasks.rados.rados.0.smithi022.stdout:1726: finishing write tid 3 to smithi02284960-4 2024-09-21T09:37:13.185 INFO:tasks.rados.rados.0.smithi022.stdout:1728: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-21T09:37:13.187 INFO:tasks.rados.rados.0.smithi022.stdout:1726: finishing write tid 4 to smithi02284960-4 2024-09-21T09:37:13.187 INFO:tasks.rados.rados.0.smithi022.stdout:1726: finishing write tid 5 to smithi02284960-4 2024-09-21T09:37:13.188 INFO:tasks.rados.rados.0.smithi022.stdout:1728: done (4 left) 2024-09-21T09:37:13.188 INFO:tasks.rados.rados.0.smithi022.stdout:1731: rollback oid 7 current snap is 170 2024-09-21T09:37:13.188 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 7 to 168 2024-09-21T09:37:13.188 INFO:tasks.rados.rados.0.smithi022.stdout:1726: finishing write tid 6 to smithi02284960-4 2024-09-21T09:37:13.188 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 4 v 648 (ObjNum 565 snap 170 seq_num 565) dirty exists 2024-09-21T09:37:13.188 INFO:tasks.rados.rados.0.smithi022.stdout:1726: left oid 4 (ObjNum 565 snap 170 seq_num 565) 2024-09-21T09:37:13.188 INFO:tasks.rados.rados.0.smithi022.stdout:1727: expect (ObjNum 557 snap 167 seq_num 557) 2024-09-21T09:37:13.285 INFO:tasks.rados.rados.0.smithi022.stdout:1729: finishing write tid 1 to smithi02284960-19 2024-09-21T09:37:13.286 INFO:tasks.rados.rados.0.smithi022.stdout:1729: finishing write tid 2 to smithi02284960-19 2024-09-21T09:37:13.286 INFO:tasks.rados.rados.0.smithi022.stdout:1729: finishing write tid 3 to smithi02284960-19 2024-09-21T09:37:13.286 INFO:tasks.rados.rados.0.smithi022.stdout:1729: finishing write tid 4 to smithi02284960-19 2024-09-21T09:37:13.286 INFO:tasks.rados.rados.0.smithi022.stdout:1729: finishing write tid 5 to smithi02284960-19 2024-09-21T09:37:13.286 INFO:tasks.rados.rados.0.smithi022.stdout:1730: finishing copy_from to smithi02284960-20 2024-09-21T09:37:13.286 INFO:tasks.rados.rados.0.smithi022.stdout:1730: got expected ENOENT (src dne) 2024-09-21T09:37:13.286 INFO:tasks.rados.rados.0.smithi022.stdout:1730: finishing copy_from racing read to smithi02284960-20 2024-09-21T09:37:13.286 INFO:tasks.rados.rados.0.smithi022.stdout:1729: finishing write tid 6 to smithi02284960-19 2024-09-21T09:37:13.286 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 19 v 1155 (ObjNum 566 snap 170 seq_num 566) dirty exists 2024-09-21T09:37:13.286 INFO:tasks.rados.rados.0.smithi022.stdout:1729: left oid 19 (ObjNum 566 snap 170 seq_num 566) 2024-09-21T09:37:13.286 INFO:tasks.rados.rados.0.smithi022.stdout:1731: finishing rollback tid 0 to smithi02284960-7 2024-09-21T09:37:13.286 INFO:tasks.rados.rados.0.smithi022.stdout:1731: finishing rollback tid 1 to smithi02284960-7 2024-09-21T09:37:13.287 INFO:tasks.rados.rados.0.smithi022.stdout:1731: finishing rollback tid 2 to smithi02284960-7 2024-09-21T09:37:13.287 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 7 v 1087 (ObjNum 519 snap 154 seq_num 519) dirty exists 2024-09-21T09:37:13.287 INFO:tasks.rados.rados.0.smithi022.stdout:1726: done (4 left) 2024-09-21T09:37:13.287 INFO:tasks.rados.rados.0.smithi022.stdout:1727: done (3 left) 2024-09-21T09:37:13.287 INFO:tasks.rados.rados.0.smithi022.stdout:1729: done (2 left) 2024-09-21T09:37:13.287 INFO:tasks.rados.rados.0.smithi022.stdout:1730: done (1 left) 2024-09-21T09:37:13.287 INFO:tasks.rados.rados.0.smithi022.stdout:1731: done (0 left) 2024-09-21T09:37:13.287 INFO:tasks.rados.rados.0.smithi022.stdout:1732: rollback oid 45 current snap is 170 2024-09-21T09:37:13.287 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 45 to 166 2024-09-21T09:37:13.287 INFO:tasks.rados.rados.0.smithi022.stdout:1733: snap_remove snap 169 2024-09-21T09:37:13.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:13 smithi064 ceph-mon[86709]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-21T09:37:13.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:13 smithi064 ceph-mon[86709]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-09-21T09:37:13.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:13 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/1443830627' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:37:13.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:13 smithi064 ceph-mon[86709]: osdmap e411: 8 total, 8 up, 8 in 2024-09-21T09:37:13.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:13 smithi064 ceph-mon[86709]: pgmap v220: 121 pgs: 27 active+undersized, 9 active+undersized+degraded, 85 active+clean; 111 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 1.0 MiB/s wr, 3 op/s; 47/282 objects degraded (16.667%) 2024-09-21T09:37:13.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:13 smithi064 ceph-mon[86709]: from='client.34308 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:37:13.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:13 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:37:13.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:13 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:37:13.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:13 smithi022 ceph-mon[96736]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-21T09:37:13.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:13 smithi022 ceph-mon[96736]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-09-21T09:37:13.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:13 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/1443830627' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:37:13.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:13 smithi022 ceph-mon[96736]: osdmap e411: 8 total, 8 up, 8 in 2024-09-21T09:37:13.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:13 smithi022 ceph-mon[96736]: pgmap v220: 121 pgs: 27 active+undersized, 9 active+undersized+degraded, 85 active+clean; 111 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 1.0 MiB/s wr, 3 op/s; 47/282 objects degraded (16.667%) 2024-09-21T09:37:13.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:13 smithi022 ceph-mon[96736]: from='client.34308 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:37:13.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:13 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:37:13.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:13 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:37:13.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:13 smithi022 ceph-mon[98615]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-21T09:37:13.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:13 smithi022 ceph-mon[98615]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-09-21T09:37:13.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:13 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/1443830627' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:37:13.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:13 smithi022 ceph-mon[98615]: osdmap e411: 8 total, 8 up, 8 in 2024-09-21T09:37:13.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:13 smithi022 ceph-mon[98615]: pgmap v220: 121 pgs: 27 active+undersized, 9 active+undersized+degraded, 85 active+clean; 111 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 1.0 MiB/s wr, 3 op/s; 47/282 objects degraded (16.667%) 2024-09-21T09:37:13.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:13 smithi022 ceph-mon[98615]: from='client.34308 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:37:13.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:13 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:37:13.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:13 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:37:14.124 INFO:tasks.rados.rados.0.smithi022.stdout:1733: done (1 left) 2024-09-21T09:37:14.124 INFO:tasks.rados.rados.0.smithi022.stdout:1734: snap_remove snap 167 2024-09-21T09:37:14.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:14 smithi064 ceph-mon[86709]: osdmap e412: 8 total, 8 up, 8 in 2024-09-21T09:37:14.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:14 smithi064 ceph-mon[86709]: Health check update: Degraded data redundancy: 47/282 objects degraded (16.667%), 9 pgs degraded (PG_DEGRADED) 2024-09-21T09:37:14.442 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:37:14 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:37:14.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.2\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.2\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.2\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:37:14.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:14 smithi022 ceph-mon[98615]: osdmap e412: 8 total, 8 up, 8 in 2024-09-21T09:37:14.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:14 smithi022 ceph-mon[98615]: Health check update: Degraded data redundancy: 47/282 objects degraded (16.667%), 9 pgs degraded (PG_DEGRADED) 2024-09-21T09:37:14.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:14 smithi022 ceph-mon[96736]: osdmap e412: 8 total, 8 up, 8 in 2024-09-21T09:37:14.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:14 smithi022 ceph-mon[96736]: Health check update: Degraded data redundancy: 47/282 objects degraded (16.667%), 9 pgs degraded (PG_DEGRADED) 2024-09-21T09:37:15.126 INFO:tasks.rados.rados.0.smithi022.stdout:1734: done (1 left) 2024-09-21T09:37:15.126 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 49 current snap is 170 2024-09-21T09:37:15.127 INFO:tasks.rados.rados.0.smithi022.stdout:1732: finishing rollback tid 0 to smithi02284960-45 2024-09-21T09:37:15.127 INFO:tasks.rados.rados.0.smithi022.stdout:1732: finishing rollback tid 1 to smithi02284960-45 2024-09-21T09:37:15.127 INFO:tasks.rados.rados.0.smithi022.stdout:1732: finishing rollback tid 2 to smithi02284960-45 2024-09-21T09:37:15.127 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 45 v 1079 (ObjNum 442 snap 134 seq_num 442) dirty exists 2024-09-21T09:37:15.127 INFO:tasks.rados.rados.0.smithi022.stdout:1735: seq_num 567 ranges {0=704512,704512=499712,1204224=770048,1974272=335872} 2024-09-21T09:37:15.130 INFO:tasks.rados.rados.0.smithi022.stdout:1735: writing smithi02284960-49 from 0 to 704512 tid 1 2024-09-21T09:37:15.133 INFO:tasks.rados.rados.0.smithi022.stdout:1735: writing smithi02284960-49 from 704512 to 1204224 tid 2 2024-09-21T09:37:15.137 INFO:tasks.rados.rados.0.smithi022.stdout:1735: writing smithi02284960-49 from 1204224 to 1974272 tid 3 2024-09-21T09:37:15.140 INFO:tasks.rados.rados.0.smithi022.stdout:1735: writing smithi02284960-49 from 1974272 to 2310144 tid 4 2024-09-21T09:37:15.140 INFO:tasks.rados.rados.0.smithi022.stdout:1732: done (1 left) 2024-09-21T09:37:15.140 INFO:tasks.rados.rados.0.smithi022.stdout:1736: read oid 5 snap -1 2024-09-21T09:37:15.140 INFO:tasks.rados.rados.0.smithi022.stdout:1736: expect (ObjNum 558 snap 167 seq_num 558) 2024-09-21T09:37:15.140 INFO:tasks.rados.rados.0.smithi022.stdout:1737: read oid 32 snap 166 2024-09-21T09:37:15.140 INFO:tasks.rados.rados.0.smithi022.stdout:1737: expect (ObjNum 538 snap 162 seq_num 538) 2024-09-21T09:37:15.140 INFO:tasks.rados.rados.0.smithi022.stdout:1738: delete oid 24 current snap is 170 2024-09-21T09:37:15.140 INFO:tasks.rados.rados.0.smithi022.stdout:1735: finishing write tid 1 to smithi02284960-49 2024-09-21T09:37:15.141 INFO:tasks.rados.rados.0.smithi022.stdout:1735: finishing write tid 2 to smithi02284960-49 2024-09-21T09:37:15.143 INFO:tasks.rados.rados.0.smithi022.stdout:1735: finishing write tid 3 to smithi02284960-49 2024-09-21T09:37:15.143 INFO:tasks.rados.rados.0.smithi022.stdout:1735: finishing write tid 4 to smithi02284960-49 2024-09-21T09:37:15.144 INFO:tasks.rados.rados.0.smithi022.stdout:1735: finishing write tid 5 to smithi02284960-49 2024-09-21T09:37:15.148 INFO:tasks.rados.rados.0.smithi022.stdout:1735: finishing write tid 6 to smithi02284960-49 2024-09-21T09:37:15.148 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 49 v 767 (ObjNum 567 snap 170 seq_num 567) dirty exists 2024-09-21T09:37:15.148 INFO:tasks.rados.rados.0.smithi022.stdout:1735: left oid 49 (ObjNum 567 snap 170 seq_num 567) 2024-09-21T09:37:15.148 INFO:tasks.rados.rados.0.smithi022.stdout:1735: done (3 left) 2024-09-21T09:37:15.148 INFO:tasks.rados.rados.0.smithi022.stdout:1738: done (2 left) 2024-09-21T09:37:15.148 INFO:tasks.rados.rados.0.smithi022.stdout:1739: read oid 35 snap -1 2024-09-21T09:37:15.148 INFO:tasks.rados.rados.0.smithi022.stdout:1739: expect (ObjNum 471 snap 142 seq_num 471) 2024-09-21T09:37:15.148 INFO:tasks.rados.rados.0.smithi022.stdout:1740: copy_from oid 25 from oid 29 current snap is 170 2024-09-21T09:37:15.148 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 11 current snap is 170 2024-09-21T09:37:15.149 INFO:tasks.rados.rados.0.smithi022.stdout:1741: seq_num 569 ranges {6586368=737280,7323648=671744,7995392=737280,8732672=196608} 2024-09-21T09:37:15.191 INFO:tasks.rados.rados.0.smithi022.stdout:1741: writing smithi02284960-11 from 6586368 to 7323648 tid 1 2024-09-21T09:37:15.195 INFO:tasks.rados.rados.0.smithi022.stdout:1741: writing smithi02284960-11 from 7323648 to 7995392 tid 2 2024-09-21T09:37:15.199 INFO:tasks.rados.rados.0.smithi022.stdout:1741: writing smithi02284960-11 from 7995392 to 8732672 tid 3 2024-09-21T09:37:15.201 INFO:tasks.rados.rados.0.smithi022.stdout:1741: writing smithi02284960-11 from 8732672 to 8929280 tid 4 2024-09-21T09:37:15.201 INFO:tasks.rados.rados.0.smithi022.stdout:1742: copy_from oid 12 from oid 45 current snap is 170 2024-09-21T09:37:15.201 INFO:tasks.rados.rados.0.smithi022.stdout:1740: finishing copy_from to smithi02284960-25 2024-09-21T09:37:15.201 INFO:tasks.rados.rados.0.smithi022.stdout:1740: got expected ENOENT (src dne) 2024-09-21T09:37:15.201 INFO:tasks.rados.rados.0.smithi022.stdout:1743: read oid 7 snap -1 2024-09-21T09:37:15.201 INFO:tasks.rados.rados.0.smithi022.stdout:1743: expect (ObjNum 519 snap 154 seq_num 519) 2024-09-21T09:37:15.201 INFO:tasks.rados.rados.0.smithi022.stdout:1744: snap_remove snap 168 2024-09-21T09:37:15.201 INFO:tasks.rados.rados.0.smithi022.stdout:1740: finishing copy_from racing read to smithi02284960-25 2024-09-21T09:37:15.201 INFO:tasks.rados.rados.0.smithi022.stdout:1739: expect (ObjNum 471 snap 142 seq_num 471) 2024-09-21T09:37:15.281 INFO:tasks.rados.rados.0.smithi022.stdout:1737: expect (ObjNum 538 snap 162 seq_num 538) 2024-09-21T09:37:15.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:15 smithi064 ceph-mon[86709]: osdmap e413: 8 total, 8 up, 8 in 2024-09-21T09:37:15.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:15 smithi064 ceph-mon[86709]: pgmap v223: 121 pgs: 10 active+undersized, 6 active+undersized+degraded, 105 active+clean; 108 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 265 KiB/s rd, 2.1 MiB/s wr, 10 op/s; 36/258 objects degraded (13.953%); 3.4 MiB/s, 1 objects/s recovering 2024-09-21T09:37:15.472 INFO:tasks.rados.rados.0.smithi022.stdout:1736: expect (ObjNum 558 snap 167 seq_num 558) 2024-09-21T09:37:15.557 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:15 smithi022 ceph-mon[98615]: osdmap e413: 8 total, 8 up, 8 in 2024-09-21T09:37:15.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:15 smithi022 ceph-mon[98615]: pgmap v223: 121 pgs: 10 active+undersized, 6 active+undersized+degraded, 105 active+clean; 108 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 265 KiB/s rd, 2.1 MiB/s wr, 10 op/s; 36/258 objects degraded (13.953%); 3.4 MiB/s, 1 objects/s recovering 2024-09-21T09:37:15.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:15 smithi022 ceph-mon[96736]: osdmap e413: 8 total, 8 up, 8 in 2024-09-21T09:37:15.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:15 smithi022 ceph-mon[96736]: pgmap v223: 121 pgs: 10 active+undersized, 6 active+undersized+degraded, 105 active+clean; 108 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 265 KiB/s rd, 2.1 MiB/s wr, 10 op/s; 36/258 objects degraded (13.953%); 3.4 MiB/s, 1 objects/s recovering 2024-09-21T09:37:15.955 INFO:tasks.rados.rados.0.smithi022.stdout:1741: finishing write tid 1 to smithi02284960-11 2024-09-21T09:37:15.955 INFO:tasks.rados.rados.0.smithi022.stdout:1741: finishing write tid 2 to smithi02284960-11 2024-09-21T09:37:15.955 INFO:tasks.rados.rados.0.smithi022.stdout:1741: finishing write tid 3 to smithi02284960-11 2024-09-21T09:37:15.955 INFO:tasks.rados.rados.0.smithi022.stdout:1741: finishing write tid 4 to smithi02284960-11 2024-09-21T09:37:15.955 INFO:tasks.rados.rados.0.smithi022.stdout:1741: finishing write tid 5 to smithi02284960-11 2024-09-21T09:37:15.955 INFO:tasks.rados.rados.0.smithi022.stdout:1743: expect (ObjNum 519 snap 154 seq_num 519) 2024-09-21T09:37:15.994 INFO:tasks.rados.rados.0.smithi022.stdout:1741: finishing write tid 6 to smithi02284960-11 2024-09-21T09:37:15.994 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 11 v 1029 (ObjNum 569 snap 170 seq_num 569) dirty exists 2024-09-21T09:37:15.994 INFO:tasks.rados.rados.0.smithi022.stdout:1741: left oid 11 (ObjNum 569 snap 170 seq_num 569) 2024-09-21T09:37:15.994 INFO:tasks.rados.rados.0.smithi022.stdout:1742: finishing copy_from racing read to smithi02284960-12 2024-09-21T09:37:15.995 INFO:tasks.rados.rados.0.smithi022.stdout:1742: finishing copy_from to smithi02284960-12 2024-09-21T09:37:15.995 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 12 v 1079 (ObjNum 442 snap 134 seq_num 442) dirty exists 2024-09-21T09:37:16.057 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:37:15 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:37:15] "GET /metrics HTTP/1.1" 200 36094 "" "Prometheus/2.51.0" 2024-09-21T09:37:16.133 INFO:tasks.rados.rados.0.smithi022.stdout:1736: done (7 left) 2024-09-21T09:37:16.133 INFO:tasks.rados.rados.0.smithi022.stdout:1737: done (6 left) 2024-09-21T09:37:16.133 INFO:tasks.rados.rados.0.smithi022.stdout:1739: done (5 left) 2024-09-21T09:37:16.133 INFO:tasks.rados.rados.0.smithi022.stdout:1740: done (4 left) 2024-09-21T09:37:16.133 INFO:tasks.rados.rados.0.smithi022.stdout:1741: done (3 left) 2024-09-21T09:37:16.133 INFO:tasks.rados.rados.0.smithi022.stdout:1742: done (2 left) 2024-09-21T09:37:16.133 INFO:tasks.rados.rados.0.smithi022.stdout:1743: done (1 left) 2024-09-21T09:37:16.133 INFO:tasks.rados.rados.0.smithi022.stdout:1744: done (0 left) 2024-09-21T09:37:16.133 INFO:tasks.rados.rados.0.smithi022.stdout:1745: snap_create 2024-09-21T09:37:16.136 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:37:15 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:37:15.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:37:16.136 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:37:15 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:37:15.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:37:16.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:16 smithi064 ceph-mon[86709]: osdmap e414: 8 total, 8 up, 8 in 2024-09-21T09:37:16.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:16 smithi022 ceph-mon[98615]: osdmap e414: 8 total, 8 up, 8 in 2024-09-21T09:37:16.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:16 smithi022 ceph-mon[96736]: osdmap e414: 8 total, 8 up, 8 in 2024-09-21T09:37:17.144 INFO:tasks.rados.rados.0.smithi022.stdout:1745: done (0 left) 2024-09-21T09:37:17.144 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 29 current snap is 171 2024-09-21T09:37:17.144 INFO:tasks.rados.rados.0.smithi022.stdout:1746: seq_num 570 ranges {0=753664,753664=548864,1302528=540672,1843200=516096} 2024-09-21T09:37:17.148 INFO:tasks.rados.rados.0.smithi022.stdout:1746: writing smithi02284960-29 from 0 to 753664 tid 1 2024-09-21T09:37:17.151 INFO:tasks.rados.rados.0.smithi022.stdout:1746: writing smithi02284960-29 from 753664 to 1302528 tid 2 2024-09-21T09:37:17.155 INFO:tasks.rados.rados.0.smithi022.stdout:1746: writing smithi02284960-29 from 1302528 to 1843200 tid 3 2024-09-21T09:37:17.158 INFO:tasks.rados.rados.0.smithi022.stdout:1746: writing smithi02284960-29 from 1843200 to 2359296 tid 4 2024-09-21T09:37:17.158 INFO:tasks.rados.rados.0.smithi022.stdout:1747: snap_remove snap 170 2024-09-21T09:37:17.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:17 smithi064 ceph-mon[86709]: osdmap e415: 8 total, 8 up, 8 in 2024-09-21T09:37:17.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:17 smithi064 ceph-mon[86709]: pgmap v226: 121 pgs: 121 active+clean; 120 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2.4 MiB/s rd, 4.7 MiB/s wr, 19 op/s; 19 MiB/s, 5 objects/s recovering 2024-09-21T09:37:17.442 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:37:16 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:37:16.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi022\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:37:17.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:17 smithi022 ceph-mon[96736]: osdmap e415: 8 total, 8 up, 8 in 2024-09-21T09:37:17.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:17 smithi022 ceph-mon[96736]: pgmap v226: 121 pgs: 121 active+clean; 120 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2.4 MiB/s rd, 4.7 MiB/s wr, 19 op/s; 19 MiB/s, 5 objects/s recovering 2024-09-21T09:37:17.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:17 smithi022 ceph-mon[98615]: osdmap e415: 8 total, 8 up, 8 in 2024-09-21T09:37:17.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:17 smithi022 ceph-mon[98615]: pgmap v226: 121 pgs: 121 active+clean; 120 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2.4 MiB/s rd, 4.7 MiB/s wr, 19 op/s; 19 MiB/s, 5 objects/s recovering 2024-09-21T09:37:18.148 INFO:tasks.rados.rados.0.smithi022.stdout:1747: done (1 left) 2024-09-21T09:37:18.148 INFO:tasks.rados.rados.0.smithi022.stdout:1748: rmattr oid 40 current snap is 171 2024-09-21T09:37:18.148 INFO:tasks.rados.rados.0.smithi022.stdout:1748: done (1 left) 2024-09-21T09:37:18.148 INFO:tasks.rados.rados.0.smithi022.stdout:1749: delete oid 21 current snap is 171 2024-09-21T09:37:18.149 INFO:tasks.rados.rados.0.smithi022.stdout:1746: finishing write tid 1 to smithi02284960-29 2024-09-21T09:37:18.149 INFO:tasks.rados.rados.0.smithi022.stdout:1746: finishing write tid 2 to smithi02284960-29 2024-09-21T09:37:18.149 INFO:tasks.rados.rados.0.smithi022.stdout:1746: finishing write tid 3 to smithi02284960-29 2024-09-21T09:37:18.149 INFO:tasks.rados.rados.0.smithi022.stdout:1746: finishing write tid 4 to smithi02284960-29 2024-09-21T09:37:18.149 INFO:tasks.rados.rados.0.smithi022.stdout:1746: finishing write tid 5 to smithi02284960-29 2024-09-21T09:37:18.149 INFO:tasks.rados.rados.0.smithi022.stdout:1746: finishing write tid 6 to smithi02284960-29 2024-09-21T09:37:18.149 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 29 v 860 (ObjNum 570 snap 171 seq_num 570) dirty exists 2024-09-21T09:37:18.149 INFO:tasks.rados.rados.0.smithi022.stdout:1746: left oid 29 (ObjNum 570 snap 171 seq_num 570) 2024-09-21T09:37:18.152 INFO:tasks.rados.rados.0.smithi022.stdout:1746: done (1 left) 2024-09-21T09:37:18.153 INFO:tasks.rados.rados.0.smithi022.stdout:1749: done (0 left) 2024-09-21T09:37:18.153 INFO:tasks.rados.rados.0.smithi022.stdout:1750: read oid 10 snap -1 2024-09-21T09:37:18.153 INFO:tasks.rados.rados.0.smithi022.stdout:1750: expect (ObjNum 527 snap 156 seq_num 527) 2024-09-21T09:37:18.153 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 24 current snap is 171 2024-09-21T09:37:18.153 INFO:tasks.rados.rados.0.smithi022.stdout:1751: seq_num 572 ranges {0=786432,786432=770048,1556480=557056,2113536=278528} 2024-09-21T09:37:18.157 INFO:tasks.rados.rados.0.smithi022.stdout:1751: writing smithi02284960-24 from 0 to 786432 tid 1 2024-09-21T09:37:18.162 INFO:tasks.rados.rados.0.smithi022.stdout:1751: writing smithi02284960-24 from 786432 to 1556480 tid 2 2024-09-21T09:37:18.166 INFO:tasks.rados.rados.0.smithi022.stdout:1751: writing smithi02284960-24 from 1556480 to 2113536 tid 3 2024-09-21T09:37:18.168 INFO:tasks.rados.rados.0.smithi022.stdout:1751: writing smithi02284960-24 from 2113536 to 2392064 tid 4 2024-09-21T09:37:18.168 INFO:tasks.rados.rados.0.smithi022.stdout:1752: copy_from oid 50 from oid 18 current snap is 171 2024-09-21T09:37:18.168 INFO:tasks.rados.rados.0.smithi022.stdout:1750: expect (ObjNum 527 snap 156 seq_num 527) 2024-09-21T09:37:18.217 INFO:tasks.rados.rados.0.smithi022.stdout:1751: finishing write tid 1 to smithi02284960-24 2024-09-21T09:37:18.218 INFO:tasks.rados.rados.0.smithi022.stdout:1751: finishing write tid 2 to smithi02284960-24 2024-09-21T09:37:18.218 INFO:tasks.rados.rados.0.smithi022.stdout:1751: finishing write tid 3 to smithi02284960-24 2024-09-21T09:37:18.218 INFO:tasks.rados.rados.0.smithi022.stdout:1751: finishing write tid 4 to smithi02284960-24 2024-09-21T09:37:18.218 INFO:tasks.rados.rados.0.smithi022.stdout:1751: finishing write tid 5 to smithi02284960-24 2024-09-21T09:37:18.218 INFO:tasks.rados.rados.0.smithi022.stdout:1751: finishing write tid 6 to smithi02284960-24 2024-09-21T09:37:18.218 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 24 v 1171 (ObjNum 572 snap 171 seq_num 572) dirty exists 2024-09-21T09:37:18.218 INFO:tasks.rados.rados.0.smithi022.stdout:1751: left oid 24 (ObjNum 572 snap 171 seq_num 572) 2024-09-21T09:37:18.218 INFO:tasks.rados.rados.0.smithi022.stdout:1752: finishing copy_from racing read to smithi02284960-50 2024-09-21T09:37:18.218 INFO:tasks.rados.rados.0.smithi022.stdout:1752: finishing copy_from to smithi02284960-50 2024-09-21T09:37:18.218 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 50 v 953 (ObjNum 557 snap 167 seq_num 557) dirty exists 2024-09-21T09:37:18.218 INFO:tasks.rados.rados.0.smithi022.stdout:1750: done (2 left) 2024-09-21T09:37:18.219 INFO:tasks.rados.rados.0.smithi022.stdout:1751: done (1 left) 2024-09-21T09:37:18.219 INFO:tasks.rados.rados.0.smithi022.stdout:1752: done (0 left) 2024-09-21T09:37:18.219 INFO:tasks.rados.rados.0.smithi022.stdout:1753: rollback oid 13 current snap is 171 2024-09-21T09:37:18.219 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 13 to 166 2024-09-21T09:37:18.219 INFO:tasks.rados.rados.0.smithi022.stdout:1754: rollback oid 30 current snap is 171 2024-09-21T09:37:18.219 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 30 to 166 2024-09-21T09:37:18.219 INFO:tasks.rados.rados.0.smithi022.stdout:1755: read oid 14 snap -1 2024-09-21T09:37:18.219 INFO:tasks.rados.rados.0.smithi022.stdout:1755: expect (ObjNum 437 snap 134 seq_num 437) 2024-09-21T09:37:18.219 INFO:tasks.rados.rados.0.smithi022.stdout:1756: setattr oid 5 current snap is 171 2024-09-21T09:37:18.219 INFO:tasks.rados.rados.0.smithi022.stdout:1757: read oid 12 snap 166 2024-09-21T09:37:18.219 INFO:tasks.rados.rados.0.smithi022.stdout:1757: expect (ObjNum 519 snap 154 seq_num 519) 2024-09-21T09:37:18.219 INFO:tasks.rados.rados.0.smithi022.stdout:1758: setattr oid 41 current snap is 171 2024-09-21T09:37:18.220 INFO:tasks.rados.rados.0.smithi022.stdout:1759: read oid 8 snap 166 2024-09-21T09:37:18.220 INFO:tasks.rados.rados.0.smithi022.stdout:1759: expect (ObjNum 437 snap 134 seq_num 437) 2024-09-21T09:37:18.220 INFO:tasks.rados.rados.0.smithi022.stdout:1760: delete oid 16 current snap is 171 2024-09-21T09:37:18.220 INFO:tasks.rados.rados.0.smithi022.stdout:1753: finishing rollback tid 1 to smithi02284960-13 2024-09-21T09:37:18.220 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 13 v 0 (ObjNum 65 snap 0 seq_num 1399614567) dirty dne 2024-09-21T09:37:18.220 INFO:tasks.rados.rados.0.smithi022.stdout:1754: finishing rollback tid 0 to smithi02284960-30 2024-09-21T09:37:18.220 INFO:tasks.rados.rados.0.smithi022.stdout:1754: finishing rollback tid 1 to smithi02284960-30 2024-09-21T09:37:18.220 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 30 v 941 (ObjNum 65 snap 0 seq_num 1415733618) dirty dne 2024-09-21T09:37:18.225 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 41 v 989 (ObjNum 531 snap 159 seq_num 531) dirty exists 2024-09-21T09:37:18.225 INFO:tasks.rados.rados.0.smithi022.stdout:1753: done (7 left) 2024-09-21T09:37:18.225 INFO:tasks.rados.rados.0.smithi022.stdout:1754: done (6 left) 2024-09-21T09:37:18.225 INFO:tasks.rados.rados.0.smithi022.stdout:1758: done (5 left) 2024-09-21T09:37:18.225 INFO:tasks.rados.rados.0.smithi022.stdout:1760: done (4 left) 2024-09-21T09:37:18.225 INFO:tasks.rados.rados.0.smithi022.stdout:1761: snap_create 2024-09-21T09:37:18.231 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 5 v 655 (ObjNum 558 snap 167 seq_num 558) dirty exists 2024-09-21T09:37:18.233 INFO:tasks.rados.rados.0.smithi022.stdout:1759: expect (ObjNum 437 snap 134 seq_num 437) 2024-09-21T09:37:18.326 INFO:tasks.rados.rados.0.smithi022.stdout:1757: expect (ObjNum 519 snap 154 seq_num 519) 2024-09-21T09:37:18.366 INFO:tasks.rados.rados.0.smithi022.stdout:1755: expect (ObjNum 437 snap 134 seq_num 437) 2024-09-21T09:37:18.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:18 smithi064 ceph-mon[86709]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 36/258 objects degraded (13.953%), 6 pgs degraded) 2024-09-21T09:37:18.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:18 smithi064 ceph-mon[86709]: Cluster is now healthy 2024-09-21T09:37:18.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:18 smithi064 ceph-mon[86709]: osdmap e416: 8 total, 8 up, 8 in 2024-09-21T09:37:18.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:18 smithi022 ceph-mon[96736]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 36/258 objects degraded (13.953%), 6 pgs degraded) 2024-09-21T09:37:18.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:18 smithi022 ceph-mon[96736]: Cluster is now healthy 2024-09-21T09:37:18.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:18 smithi022 ceph-mon[96736]: osdmap e416: 8 total, 8 up, 8 in 2024-09-21T09:37:18.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:18 smithi022 ceph-mon[98615]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 36/258 objects degraded (13.953%), 6 pgs degraded) 2024-09-21T09:37:18.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:18 smithi022 ceph-mon[98615]: Cluster is now healthy 2024-09-21T09:37:18.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:18 smithi022 ceph-mon[98615]: osdmap e416: 8 total, 8 up, 8 in 2024-09-21T09:37:18.972 INFO:tasks.rados.rados.0.smithi022.stdout:1755: done (4 left) 2024-09-21T09:37:18.972 INFO:tasks.rados.rados.0.smithi022.stdout:1756: done (3 left) 2024-09-21T09:37:18.972 INFO:tasks.rados.rados.0.smithi022.stdout:1757: done (2 left) 2024-09-21T09:37:18.972 INFO:tasks.rados.rados.0.smithi022.stdout:1759: done (1 left) 2024-09-21T09:37:18.972 INFO:tasks.rados.rados.0.smithi022.stdout:1761: done (0 left) 2024-09-21T09:37:18.972 INFO:tasks.rados.rados.0.smithi022.stdout:1762: copy_from oid 49 from oid 48 current snap is 172 2024-09-21T09:37:18.973 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 15 current snap is 172 2024-09-21T09:37:18.973 INFO:tasks.rados.rados.0.smithi022.stdout:1763: seq_num 574 ranges {1941504=16384} 2024-09-21T09:37:18.983 INFO:tasks.rados.rados.0.smithi022.stdout:1763: writing smithi02284960-15 from 1941504 to 1957888 tid 1 2024-09-21T09:37:18.983 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 17 current snap is 172 2024-09-21T09:37:18.983 INFO:tasks.rados.rados.0.smithi022.stdout:1764: seq_num 575 ranges {4866048=32768} 2024-09-21T09:37:19.011 INFO:tasks.rados.rados.0.smithi022.stdout:1764: writing smithi02284960-17 from 4866048 to 4898816 tid 1 2024-09-21T09:37:19.011 INFO:tasks.rados.rados.0.smithi022.stdout:1765: rollback oid 28 current snap is 172 2024-09-21T09:37:19.011 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 28 to 171 2024-09-21T09:37:19.011 INFO:tasks.rados.rados.0.smithi022.stdout:1763: finishing write tid 1 to smithi02284960-15 2024-09-21T09:37:19.011 INFO:tasks.rados.rados.0.smithi022.stdout:1763: finishing write tid 2 to smithi02284960-15 2024-09-21T09:37:19.011 INFO:tasks.rados.rados.0.smithi022.stdout:1763: finishing write tid 3 to smithi02284960-15 2024-09-21T09:37:19.011 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 15 v 1174 (ObjNum 574 snap 172 seq_num 574) dirty exists 2024-09-21T09:37:19.011 INFO:tasks.rados.rados.0.smithi022.stdout:1763: left oid 15 (ObjNum 574 snap 172 seq_num 574) 2024-09-21T09:37:19.011 INFO:tasks.rados.rados.0.smithi022.stdout:1763: done (3 left) 2024-09-21T09:37:19.011 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 37 current snap is 172 2024-09-21T09:37:19.011 INFO:tasks.rados.rados.0.smithi022.stdout:1766: seq_num 576 ranges {3710976=49152} 2024-09-21T09:37:19.031 INFO:tasks.rados.rados.0.smithi022.stdout:1766: writing smithi02284960-37 from 3710976 to 3760128 tid 1 2024-09-21T09:37:19.032 INFO:tasks.rados.rados.0.smithi022.stdout:1767: delete oid 50 current snap is 172 2024-09-21T09:37:19.032 INFO:tasks.rados.rados.0.smithi022.stdout:1762: finishing copy_from racing read to smithi02284960-49 2024-09-21T09:37:19.032 INFO:tasks.rados.rados.0.smithi022.stdout:1765: finishing rollback tid 0 to smithi02284960-28 2024-09-21T09:37:19.032 INFO:tasks.rados.rados.0.smithi022.stdout:1764: finishing write tid 1 to smithi02284960-17 2024-09-21T09:37:19.032 INFO:tasks.rados.rados.0.smithi022.stdout:1764: finishing write tid 2 to smithi02284960-17 2024-09-21T09:37:19.032 INFO:tasks.rados.rados.0.smithi022.stdout:1765: finishing rollback tid 1 to smithi02284960-28 2024-09-21T09:37:19.032 INFO:tasks.rados.rados.0.smithi022.stdout:1765: finishing rollback tid 2 to smithi02284960-28 2024-09-21T09:37:19.032 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 28 v 556 (ObjNum 392 snap 122 seq_num 392) dirty exists 2024-09-21T09:37:19.032 INFO:tasks.rados.rados.0.smithi022.stdout:1762: finishing copy_from to smithi02284960-49 2024-09-21T09:37:19.032 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 49 v 1067 (ObjNum 561 snap 168 seq_num 561) dirty exists 2024-09-21T09:37:19.032 INFO:tasks.rados.rados.0.smithi022.stdout:1764: finishing write tid 3 to smithi02284960-17 2024-09-21T09:37:19.032 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 17 v 943 (ObjNum 575 snap 172 seq_num 575) dirty exists 2024-09-21T09:37:19.032 INFO:tasks.rados.rados.0.smithi022.stdout:1764: left oid 17 (ObjNum 575 snap 172 seq_num 575) 2024-09-21T09:37:19.033 INFO:tasks.rados.rados.0.smithi022.stdout:1762: done (4 left) 2024-09-21T09:37:19.033 INFO:tasks.rados.rados.0.smithi022.stdout:1764: done (3 left) 2024-09-21T09:37:19.034 INFO:tasks.rados.rados.0.smithi022.stdout:1765: done (2 left) 2024-09-21T09:37:19.034 INFO:tasks.rados.rados.0.smithi022.stdout:1767: done (1 left) 2024-09-21T09:37:19.034 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 44 current snap is 172 2024-09-21T09:37:19.034 INFO:tasks.rados.rados.0.smithi022.stdout:1768: seq_num 578 ranges {2187264=81920} 2024-09-21T09:37:19.047 INFO:tasks.rados.rados.0.smithi022.stdout:1768: writing smithi02284960-44 from 2187264 to 2269184 tid 1 2024-09-21T09:37:19.047 INFO:tasks.rados.rados.0.smithi022.stdout:1769: read oid 2 snap -1 2024-09-21T09:37:19.047 INFO:tasks.rados.rados.0.smithi022.stdout:1769: expect (ObjNum 452 snap 136 seq_num 452) 2024-09-21T09:37:19.047 INFO:tasks.rados.rados.0.smithi022.stdout:1770: copy_from oid 49 from oid 16 current snap is 172 2024-09-21T09:37:19.047 INFO:tasks.rados.rados.0.smithi022.stdout:1766: finishing write tid 1 to smithi02284960-37 2024-09-21T09:37:19.047 INFO:tasks.rados.rados.0.smithi022.stdout:1766: finishing write tid 2 to smithi02284960-37 2024-09-21T09:37:19.048 INFO:tasks.rados.rados.0.smithi022.stdout:1766: finishing write tid 3 to smithi02284960-37 2024-09-21T09:37:19.048 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 37 v 1177 (ObjNum 576 snap 172 seq_num 576) dirty exists 2024-09-21T09:37:19.048 INFO:tasks.rados.rados.0.smithi022.stdout:1766: left oid 37 (ObjNum 576 snap 172 seq_num 576) 2024-09-21T09:37:19.048 INFO:tasks.rados.rados.0.smithi022.stdout:1766: done (3 left) 2024-09-21T09:37:19.048 INFO:tasks.rados.rados.0.smithi022.stdout:1771: setattr oid 23 current snap is 172 2024-09-21T09:37:19.049 INFO:tasks.rados.rados.0.smithi022.stdout:1770: finishing copy_from to smithi02284960-49 2024-09-21T09:37:19.049 INFO:tasks.rados.rados.0.smithi022.stdout:1770: got expected ENOENT (src dne) 2024-09-21T09:37:19.049 INFO:tasks.rados.rados.0.smithi022.stdout:1770: finishing copy_from racing read to smithi02284960-49 2024-09-21T09:37:19.049 INFO:tasks.rados.rados.0.smithi022.stdout:1770: done (3 left) 2024-09-21T09:37:19.050 INFO:tasks.rados.rados.0.smithi022.stdout:1772: delete oid 34 current snap is 172 2024-09-21T09:37:19.050 INFO:tasks.rados.rados.0.smithi022.stdout:1768: finishing write tid 1 to smithi02284960-44 2024-09-21T09:37:19.051 INFO:tasks.rados.rados.0.smithi022.stdout:1768: finishing write tid 2 to smithi02284960-44 2024-09-21T09:37:19.051 INFO:tasks.rados.rados.0.smithi022.stdout:1768: finishing write tid 3 to smithi02284960-44 2024-09-21T09:37:19.051 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 44 v 1034 (ObjNum 578 snap 172 seq_num 578) dirty exists 2024-09-21T09:37:19.051 INFO:tasks.rados.rados.0.smithi022.stdout:1768: left oid 44 (ObjNum 578 snap 172 seq_num 578) 2024-09-21T09:37:19.052 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 23 v 861 (ObjNum 555 snap 167 seq_num 555) dirty exists 2024-09-21T09:37:19.064 INFO:tasks.rados.rados.0.smithi022.stdout:1769: expect (ObjNum 452 snap 136 seq_num 452) 2024-09-21T09:37:19.242 INFO:tasks.rados.rados.0.smithi022.stdout:1768: done (3 left) 2024-09-21T09:37:19.242 INFO:tasks.rados.rados.0.smithi022.stdout:1769: done (2 left) 2024-09-21T09:37:19.242 INFO:tasks.rados.rados.0.smithi022.stdout:1771: done (1 left) 2024-09-21T09:37:19.243 INFO:tasks.rados.rados.0.smithi022.stdout:1772: done (0 left) 2024-09-21T09:37:19.243 INFO:tasks.rados.rados.0.smithi022.stdout:1773: read oid 14 snap 166 2024-09-21T09:37:19.243 INFO:tasks.rados.rados.0.smithi022.stdout:1773: expect (ObjNum 437 snap 134 seq_num 437) 2024-09-21T09:37:19.243 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 36 current snap is 172 2024-09-21T09:37:19.243 INFO:tasks.rados.rados.0.smithi022.stdout:1774: seq_num 580 ranges {0=114688} 2024-09-21T09:37:19.243 INFO:tasks.rados.rados.0.smithi022.stdout:1774: writing smithi02284960-36 from 0 to 114688 tid 1 2024-09-21T09:37:19.243 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 48 current snap is 172 2024-09-21T09:37:19.243 INFO:tasks.rados.rados.0.smithi022.stdout:1775: seq_num 581 ranges {7667712=139264} 2024-09-21T09:37:19.285 INFO:tasks.rados.rados.0.smithi022.stdout:1775: writing smithi02284960-48 from 7667712 to 7806976 tid 1 2024-09-21T09:37:19.286 INFO:tasks.rados.rados.0.smithi022.stdout:1776: snap_remove snap 171 2024-09-21T09:37:19.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:19 smithi064 ceph-mon[86709]: osdmap e417: 8 total, 8 up, 8 in 2024-09-21T09:37:19.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:19 smithi064 ceph-mon[86709]: pgmap v229: 121 pgs: 121 active+clean; 129 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 7.3 MiB/s rd, 5.9 MiB/s wr, 26 op/s; 15 MiB/s, 4 objects/s recovering 2024-09-21T09:37:19.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:19 smithi064 ceph-mon[86709]: osdmap e418: 8 total, 8 up, 8 in 2024-09-21T09:37:19.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:19 smithi022 ceph-mon[98615]: osdmap e417: 8 total, 8 up, 8 in 2024-09-21T09:37:19.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:19 smithi022 ceph-mon[98615]: pgmap v229: 121 pgs: 121 active+clean; 129 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 7.3 MiB/s rd, 5.9 MiB/s wr, 26 op/s; 15 MiB/s, 4 objects/s recovering 2024-09-21T09:37:19.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:19 smithi022 ceph-mon[98615]: osdmap e418: 8 total, 8 up, 8 in 2024-09-21T09:37:19.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:19 smithi022 ceph-mon[96736]: osdmap e417: 8 total, 8 up, 8 in 2024-09-21T09:37:19.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:19 smithi022 ceph-mon[96736]: pgmap v229: 121 pgs: 121 active+clean; 129 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 7.3 MiB/s rd, 5.9 MiB/s wr, 26 op/s; 15 MiB/s, 4 objects/s recovering 2024-09-21T09:37:19.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:19 smithi022 ceph-mon[96736]: osdmap e418: 8 total, 8 up, 8 in 2024-09-21T09:37:20.156 INFO:tasks.rados.rados.0.smithi022.stdout:1776: done (3 left) 2024-09-21T09:37:20.156 INFO:tasks.rados.rados.0.smithi022.stdout:1777: read oid 45 snap -1 2024-09-21T09:37:20.156 INFO:tasks.rados.rados.0.smithi022.stdout:1777: expect (ObjNum 442 snap 134 seq_num 442) 2024-09-21T09:37:20.157 INFO:tasks.rados.rados.0.smithi022.stdout:1778: copy_from oid 21 from oid 34 current snap is 172 2024-09-21T09:37:20.157 INFO:tasks.rados.rados.0.smithi022.stdout:1779: delete oid 3 current snap is 172 2024-09-21T09:37:20.157 INFO:tasks.rados.rados.0.smithi022.stdout:1774: finishing write tid 1 to smithi02284960-36 2024-09-21T09:37:20.157 INFO:tasks.rados.rados.0.smithi022.stdout:1774: finishing write tid 2 to smithi02284960-36 2024-09-21T09:37:20.157 INFO:tasks.rados.rados.0.smithi022.stdout:1774: finishing write tid 3 to smithi02284960-36 2024-09-21T09:37:20.157 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 36 v 1082 (ObjNum 580 snap 172 seq_num 580) dirty exists 2024-09-21T09:37:20.157 INFO:tasks.rados.rados.0.smithi022.stdout:1774: left oid 36 (ObjNum 580 snap 172 seq_num 580) 2024-09-21T09:37:20.157 INFO:tasks.rados.rados.0.smithi022.stdout:1773: expect (ObjNum 437 snap 134 seq_num 437) 2024-09-21T09:37:20.255 INFO:tasks.rados.rados.0.smithi022.stdout:1775: finishing write tid 1 to smithi02284960-48 2024-09-21T09:37:20.255 INFO:tasks.rados.rados.0.smithi022.stdout:1775: finishing write tid 2 to smithi02284960-48 2024-09-21T09:37:20.255 INFO:tasks.rados.rados.0.smithi022.stdout:1775: finishing write tid 3 to smithi02284960-48 2024-09-21T09:37:20.255 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 48 v 1098 (ObjNum 581 snap 172 seq_num 581) dirty exists 2024-09-21T09:37:20.255 INFO:tasks.rados.rados.0.smithi022.stdout:1775: left oid 48 (ObjNum 581 snap 172 seq_num 581) 2024-09-21T09:37:20.255 INFO:tasks.rados.rados.0.smithi022.stdout:1773: done (5 left) 2024-09-21T09:37:20.255 INFO:tasks.rados.rados.0.smithi022.stdout:1774: done (4 left) 2024-09-21T09:37:20.255 INFO:tasks.rados.rados.0.smithi022.stdout:1775: done (3 left) 2024-09-21T09:37:20.255 INFO:tasks.rados.rados.0.smithi022.stdout:1779: done (2 left) 2024-09-21T09:37:20.255 INFO:tasks.rados.rados.0.smithi022.stdout:1780: snap_remove snap 166 2024-09-21T09:37:20.691 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:37:20 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:37:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:37:21.162 INFO:tasks.rados.rados.0.smithi022.stdout:1780: done (2 left) 2024-09-21T09:37:21.163 INFO:tasks.rados.rados.0.smithi022.stdout:1781: read oid 1 snap -1 2024-09-21T09:37:21.163 INFO:tasks.rados.rados.0.smithi022.stdout:1781: expect (ObjNum 552 snap 166 seq_num 552) 2024-09-21T09:37:21.163 INFO:tasks.rados.rados.0.smithi022.stdout:1782: rmattr oid 38 current snap is 172 2024-09-21T09:37:21.163 INFO:tasks.rados.rados.0.smithi022.stdout:1778: finishing copy_from to smithi02284960-21 2024-09-21T09:37:21.163 INFO:tasks.rados.rados.0.smithi022.stdout:1778: got expected ENOENT (src dne) 2024-09-21T09:37:21.163 INFO:tasks.rados.rados.0.smithi022.stdout:1778: finishing copy_from racing read to smithi02284960-21 2024-09-21T09:37:21.163 INFO:tasks.rados.rados.0.smithi022.stdout:1777: expect (ObjNum 442 snap 134 seq_num 442) 2024-09-21T09:37:21.283 INFO:tasks.rados.rados.0.smithi022.stdout:1781: expect (ObjNum 552 snap 166 seq_num 552) 2024-09-21T09:37:21.390 INFO:tasks.rados.rados.0.smithi022.stdout:1777: done (3 left) 2024-09-21T09:37:21.390 INFO:tasks.rados.rados.0.smithi022.stdout:1778: done (2 left) 2024-09-21T09:37:21.390 INFO:tasks.rados.rados.0.smithi022.stdout:1781: done (1 left) 2024-09-21T09:37:21.390 INFO:tasks.rados.rados.0.smithi022.stdout:1782: done (0 left) 2024-09-21T09:37:21.390 INFO:tasks.rados.rados.0.smithi022.stdout:1783: read oid 10 snap -1 2024-09-21T09:37:21.390 INFO:tasks.rados.rados.0.smithi022.stdout:1783: expect (ObjNum 527 snap 156 seq_num 527) 2024-09-21T09:37:21.390 INFO:tasks.rados.rados.0.smithi022.stdout:1784: copy_from oid 47 from oid 9 current snap is 172 2024-09-21T09:37:21.390 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 37 current snap is 172 2024-09-21T09:37:21.390 INFO:tasks.rados.rados.0.smithi022.stdout:1785: seq_num 583 ranges {3760128=172032} 2024-09-21T09:37:21.412 INFO:tasks.rados.rados.0.smithi022.stdout:1785: writing smithi02284960-37 from 3760128 to 3932160 tid 1 2024-09-21T09:37:21.412 INFO:tasks.rados.rados.0.smithi022.stdout:1786: setattr oid 46 current snap is 172 2024-09-21T09:37:21.412 INFO:tasks.rados.rados.0.smithi022.stdout:1783: expect (ObjNum 527 snap 156 seq_num 527) 2024-09-21T09:37:21.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:21 smithi064 ceph-mon[86709]: osdmap e419: 8 total, 8 up, 8 in 2024-09-21T09:37:21.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:21 smithi064 ceph-mon[86709]: pgmap v232: 121 pgs: 121 active+clean; 131 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 2.8 MiB/s wr, 27 op/s 2024-09-21T09:37:21.456 INFO:tasks.rados.rados.0.smithi022.stdout:1784: finishing copy_from racing read to smithi02284960-47 2024-09-21T09:37:21.456 INFO:tasks.rados.rados.0.smithi022.stdout:1784: finishing copy_from to smithi02284960-47 2024-09-21T09:37:21.456 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 47 v 598 (ObjNum 526 snap 156 seq_num 526) dirty exists 2024-09-21T09:37:21.456 INFO:tasks.rados.rados.0.smithi022.stdout:1785: finishing write tid 1 to smithi02284960-37 2024-09-21T09:37:21.456 INFO:tasks.rados.rados.0.smithi022.stdout:1785: finishing write tid 2 to smithi02284960-37 2024-09-21T09:37:21.456 INFO:tasks.rados.rados.0.smithi022.stdout:1785: finishing write tid 3 to smithi02284960-37 2024-09-21T09:37:21.456 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 37 v 1191 (ObjNum 583 snap 172 seq_num 583) dirty exists 2024-09-21T09:37:21.456 INFO:tasks.rados.rados.0.smithi022.stdout:1785: left oid 37 (ObjNum 583 snap 172 seq_num 583) 2024-09-21T09:37:21.456 INFO:tasks.rados.rados.0.smithi022.stdout:1783: done (3 left) 2024-09-21T09:37:21.456 INFO:tasks.rados.rados.0.smithi022.stdout:1784: done (2 left) 2024-09-21T09:37:21.456 INFO:tasks.rados.rados.0.smithi022.stdout:1785: done (1 left) 2024-09-21T09:37:21.457 INFO:tasks.rados.rados.0.smithi022.stdout:1787: read oid 49 snap -1 2024-09-21T09:37:21.457 INFO:tasks.rados.rados.0.smithi022.stdout:1787: expect (ObjNum 561 snap 168 seq_num 561) 2024-09-21T09:37:21.457 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 13 current snap is 172 2024-09-21T09:37:21.457 INFO:tasks.rados.rados.0.smithi022.stdout:1788: seq_num 584 ranges {0=188416} 2024-09-21T09:37:21.457 INFO:tasks.rados.rados.0.smithi022.stdout:1788: writing smithi02284960-13 from 0 to 188416 tid 1 2024-09-21T09:37:21.458 INFO:tasks.rados.rados.0.smithi022.stdout:1789: read oid 26 snap -1 2024-09-21T09:37:21.458 INFO:tasks.rados.rados.0.smithi022.stdout:1789: expect (ObjNum 450 snap 136 seq_num 450) 2024-09-21T09:37:21.458 INFO:tasks.rados.rados.0.smithi022.stdout:1790: read oid 7 snap -1 2024-09-21T09:37:21.458 INFO:tasks.rados.rados.0.smithi022.stdout:1790: expect (ObjNum 519 snap 154 seq_num 519) 2024-09-21T09:37:21.458 INFO:tasks.rados.rados.0.smithi022.stdout:1791: delete oid 8 current snap is 172 2024-09-21T09:37:21.460 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 46 v 1040 (ObjNum 536 snap 161 seq_num 536) dirty exists 2024-09-21T09:37:21.460 INFO:tasks.rados.rados.0.smithi022.stdout:1790: expect (ObjNum 519 snap 154 seq_num 519) 2024-09-21T09:37:21.500 INFO:tasks.rados.rados.0.smithi022.stdout:1788: finishing write tid 1 to smithi02284960-13 2024-09-21T09:37:21.500 INFO:tasks.rados.rados.0.smithi022.stdout:1788: finishing write tid 2 to smithi02284960-13 2024-09-21T09:37:21.501 INFO:tasks.rados.rados.0.smithi022.stdout:1788: finishing write tid 3 to smithi02284960-13 2024-09-21T09:37:21.501 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 13 v 941 (ObjNum 584 snap 172 seq_num 584) dirty exists 2024-09-21T09:37:21.501 INFO:tasks.rados.rados.0.smithi022.stdout:1788: left oid 13 (ObjNum 584 snap 172 seq_num 584) 2024-09-21T09:37:21.501 INFO:tasks.rados.rados.0.smithi022.stdout:1789: expect (ObjNum 450 snap 136 seq_num 450) 2024-09-21T09:37:21.509 INFO:tasks.rados.rados.0.smithi022.stdout:1787: expect (ObjNum 561 snap 168 seq_num 561) 2024-09-21T09:37:21.557 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:21 smithi022 ceph-mon[96736]: osdmap e419: 8 total, 8 up, 8 in 2024-09-21T09:37:21.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:21 smithi022 ceph-mon[96736]: pgmap v232: 121 pgs: 121 active+clean; 131 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 2.8 MiB/s wr, 27 op/s 2024-09-21T09:37:21.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:21 smithi022 ceph-mon[98615]: osdmap e419: 8 total, 8 up, 8 in 2024-09-21T09:37:21.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:21 smithi022 ceph-mon[98615]: pgmap v232: 121 pgs: 121 active+clean; 131 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 2.8 MiB/s wr, 27 op/s 2024-09-21T09:37:21.904 INFO:tasks.rados.rados.0.smithi022.stdout:1786: done (5 left) 2024-09-21T09:37:21.904 INFO:tasks.rados.rados.0.smithi022.stdout:1787: done (4 left) 2024-09-21T09:37:21.904 INFO:tasks.rados.rados.0.smithi022.stdout:1788: done (3 left) 2024-09-21T09:37:21.904 INFO:tasks.rados.rados.0.smithi022.stdout:1789: done (2 left) 2024-09-21T09:37:21.905 INFO:tasks.rados.rados.0.smithi022.stdout:1790: done (1 left) 2024-09-21T09:37:21.905 INFO:tasks.rados.rados.0.smithi022.stdout:1791: done (0 left) 2024-09-21T09:37:21.905 INFO:tasks.rados.rados.0.smithi022.stdout:1792: read oid 15 snap -1 2024-09-21T09:37:21.905 INFO:tasks.rados.rados.0.smithi022.stdout:1792: expect (ObjNum 574 snap 172 seq_num 574) 2024-09-21T09:37:21.905 INFO:tasks.rados.rados.0.smithi022.stdout:1793: snap_create 2024-09-21T09:37:21.913 INFO:tasks.rados.rados.0.smithi022.stdout:1792: expect (ObjNum 574 snap 172 seq_num 574) 2024-09-21T09:37:22.160 INFO:tasks.rados.rados.0.smithi022.stdout:1792: done (1 left) 2024-09-21T09:37:22.160 INFO:tasks.rados.rados.0.smithi022.stdout:1793: done (0 left) 2024-09-21T09:37:22.161 INFO:tasks.rados.rados.0.smithi022.stdout:1794: delete oid 13 current snap is 173 2024-09-21T09:37:22.166 INFO:tasks.rados.rados.0.smithi022.stdout:1794: done (0 left) 2024-09-21T09:37:22.166 INFO:tasks.rados.rados.0.smithi022.stdout:1795: delete oid 34 current snap is 173 2024-09-21T09:37:22.168 INFO:tasks.rados.rados.0.smithi022.stdout:1795: done (0 left) 2024-09-21T09:37:22.168 INFO:tasks.rados.rados.0.smithi022.stdout:1796: snap_create 2024-09-21T09:37:22.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:22 smithi064 ceph-mon[86709]: osdmap e420: 8 total, 8 up, 8 in 2024-09-21T09:37:22.459 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:22 smithi022 ceph-mon[98615]: osdmap e420: 8 total, 8 up, 8 in 2024-09-21T09:37:22.459 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:22 smithi022 ceph-mon[96736]: osdmap e420: 8 total, 8 up, 8 in 2024-09-21T09:37:23.170 INFO:tasks.rados.rados.0.smithi022.stdout:1796: done (0 left) 2024-09-21T09:37:23.170 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 49 current snap is 174 2024-09-21T09:37:23.170 INFO:tasks.rados.rados.0.smithi022.stdout:1797: seq_num 588 ranges {7667712=253952} 2024-09-21T09:37:23.214 INFO:tasks.rados.rados.0.smithi022.stdout:1797: writing smithi02284960-49 from 7667712 to 7921664 tid 1 2024-09-21T09:37:23.214 INFO:tasks.rados.rados.0.smithi022.stdout:1798: setattr oid 31 current snap is 174 2024-09-21T09:37:23.215 INFO:tasks.rados.rados.0.smithi022.stdout:1799: read oid 35 snap -1 2024-09-21T09:37:23.215 INFO:tasks.rados.rados.0.smithi022.stdout:1799: expect (ObjNum 471 snap 142 seq_num 471) 2024-09-21T09:37:23.215 INFO:tasks.rados.rados.0.smithi022.stdout:1800: snap_remove snap 173 2024-09-21T09:37:23.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:23 smithi064 ceph-mon[86709]: osdmap e421: 8 total, 8 up, 8 in 2024-09-21T09:37:23.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:23 smithi064 ceph-mon[86709]: pgmap v235: 121 pgs: 121 active+clean; 127 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 634 KiB/s wr, 13 op/s 2024-09-21T09:37:23.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:23 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:37:23.557 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:23 smithi022 ceph-mon[98615]: osdmap e421: 8 total, 8 up, 8 in 2024-09-21T09:37:23.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:23 smithi022 ceph-mon[98615]: pgmap v235: 121 pgs: 121 active+clean; 127 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 634 KiB/s wr, 13 op/s 2024-09-21T09:37:23.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:23 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:37:23.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:23 smithi022 ceph-mon[96736]: osdmap e421: 8 total, 8 up, 8 in 2024-09-21T09:37:23.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:23 smithi022 ceph-mon[96736]: pgmap v235: 121 pgs: 121 active+clean; 127 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 634 KiB/s wr, 13 op/s 2024-09-21T09:37:23.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:23 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:37:23.972 INFO:tasks.rados.rados.0.smithi022.stdout:1800: done (3 left) 2024-09-21T09:37:23.972 INFO:tasks.rados.rados.0.smithi022.stdout:1801: setattr oid 18 current snap is 174 2024-09-21T09:37:23.972 INFO:tasks.rados.rados.0.smithi022.stdout:1797: finishing write tid 1 to smithi02284960-49 2024-09-21T09:37:23.973 INFO:tasks.rados.rados.0.smithi022.stdout:1797: finishing write tid 2 to smithi02284960-49 2024-09-21T09:37:23.973 INFO:tasks.rados.rados.0.smithi022.stdout:1797: finishing write tid 3 to smithi02284960-49 2024-09-21T09:37:23.973 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 49 v 1069 (ObjNum 588 snap 174 seq_num 588) dirty exists 2024-09-21T09:37:23.973 INFO:tasks.rados.rados.0.smithi022.stdout:1797: left oid 49 (ObjNum 588 snap 174 seq_num 588) 2024-09-21T09:37:23.973 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 31 v 1192 (ObjNum 33 snap 0 seq_num 3569628880) dirty exists 2024-09-21T09:37:23.973 INFO:tasks.rados.rados.0.smithi022.stdout:1799: expect (ObjNum 471 snap 142 seq_num 471) 2024-09-21T09:37:24.052 INFO:tasks.rados.rados.0.smithi022.stdout:1797: done (3 left) 2024-09-21T09:37:24.052 INFO:tasks.rados.rados.0.smithi022.stdout:1798: done (2 left) 2024-09-21T09:37:24.052 INFO:tasks.rados.rados.0.smithi022.stdout:1799: done (1 left) 2024-09-21T09:37:24.052 INFO:tasks.rados.rados.0.smithi022.stdout:1802: snap_create 2024-09-21T09:37:24.058 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 18 v 955 (ObjNum 557 snap 167 seq_num 557) dirty exists 2024-09-21T09:37:24.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:24 smithi064 ceph-mon[86709]: osdmap e422: 8 total, 8 up, 8 in 2024-09-21T09:37:24.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:24 smithi064 ceph-mon[86709]: osdmap e423: 8 total, 8 up, 8 in 2024-09-21T09:37:24.442 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:37:24 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:37:24.148Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.3\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.3\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.3\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:37:24.557 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:24 smithi022 ceph-mon[96736]: osdmap e422: 8 total, 8 up, 8 in 2024-09-21T09:37:24.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:24 smithi022 ceph-mon[96736]: osdmap e423: 8 total, 8 up, 8 in 2024-09-21T09:37:24.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:24 smithi022 ceph-mon[98615]: osdmap e422: 8 total, 8 up, 8 in 2024-09-21T09:37:24.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:24 smithi022 ceph-mon[98615]: osdmap e423: 8 total, 8 up, 8 in 2024-09-21T09:37:24.975 INFO:tasks.rados.rados.0.smithi022.stdout:1801: done (1 left) 2024-09-21T09:37:24.975 INFO:tasks.rados.rados.0.smithi022.stdout:1802: done (0 left) 2024-09-21T09:37:24.975 INFO:tasks.rados.rados.0.smithi022.stdout:1803: snap_create 2024-09-21T09:37:25.976 INFO:tasks.rados.rados.0.smithi022.stdout:1803: done (0 left) 2024-09-21T09:37:25.976 INFO:tasks.rados.rados.0.smithi022.stdout:1804: snap_remove snap 174 2024-09-21T09:37:25.977 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:37:25 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:37:25] "GET /metrics HTTP/1.1" 200 36094 "" "Prometheus/2.51.0" 2024-09-21T09:37:26.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:25 smithi064 ceph-mon[86709]: pgmap v238: 121 pgs: 121 active+clean; 122 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 2.7 MiB/s wr, 25 op/s 2024-09-21T09:37:26.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:25 smithi064 ceph-mon[86709]: osdmap e424: 8 total, 8 up, 8 in 2024-09-21T09:37:26.192 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:37:25 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:37:25.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:37:26.192 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:37:25 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:37:25.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:37:26.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:25 smithi022 ceph-mon[96736]: pgmap v238: 121 pgs: 121 active+clean; 122 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 2.7 MiB/s wr, 25 op/s 2024-09-21T09:37:26.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:25 smithi022 ceph-mon[96736]: osdmap e424: 8 total, 8 up, 8 in 2024-09-21T09:37:26.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:25 smithi022 ceph-mon[98615]: pgmap v238: 121 pgs: 121 active+clean; 122 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 2.7 MiB/s wr, 25 op/s 2024-09-21T09:37:26.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:25 smithi022 ceph-mon[98615]: osdmap e424: 8 total, 8 up, 8 in 2024-09-21T09:37:26.988 INFO:tasks.rados.rados.0.smithi022.stdout:1804: done (0 left) 2024-09-21T09:37:26.988 INFO:tasks.rados.rados.0.smithi022.stdout:1805: rollback oid 7 current snap is 176 2024-09-21T09:37:26.989 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 7 to 172 2024-09-21T09:37:26.989 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 31 current snap is 176 2024-09-21T09:37:26.989 INFO:tasks.rados.rados.0.smithi022.stdout:1806: seq_num 589 ranges {0=270336} 2024-09-21T09:37:26.990 INFO:tasks.rados.rados.0.smithi022.stdout:1806: writing smithi02284960-31 from 0 to 270336 tid 1 2024-09-21T09:37:26.990 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 50 current snap is 176 2024-09-21T09:37:26.990 INFO:tasks.rados.rados.0.smithi022.stdout:1807: seq_num 590 ranges {0=286720} 2024-09-21T09:37:26.993 INFO:tasks.rados.rados.0.smithi022.stdout:1807: writing smithi02284960-50 from 0 to 286720 tid 1 2024-09-21T09:37:26.993 INFO:tasks.rados.rados.0.smithi022.stdout:1808: read oid 9 snap -1 2024-09-21T09:37:26.993 INFO:tasks.rados.rados.0.smithi022.stdout:1808: expect (ObjNum 526 snap 156 seq_num 526) 2024-09-21T09:37:26.993 INFO:tasks.rados.rados.0.smithi022.stdout:1809: snap_remove snap 175 2024-09-21T09:37:27.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:26 smithi022 ceph-mon[98615]: osdmap e425: 8 total, 8 up, 8 in 2024-09-21T09:37:27.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:26 smithi022 ceph-mon[96736]: osdmap e425: 8 total, 8 up, 8 in 2024-09-21T09:37:27.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:26 smithi064 ceph-mon[86709]: osdmap e425: 8 total, 8 up, 8 in 2024-09-21T09:37:27.442 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:37:26 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:37:26.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi022\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:37:27.992 INFO:tasks.rados.rados.0.smithi022.stdout:1809: done (4 left) 2024-09-21T09:37:27.992 INFO:tasks.rados.rados.0.smithi022.stdout:1810: rollback oid 22 current snap is 176 2024-09-21T09:37:27.992 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 22 to 172 2024-09-21T09:37:27.992 INFO:tasks.rados.rados.0.smithi022.stdout:1805: finishing rollback tid 0 to smithi02284960-7 2024-09-21T09:37:27.992 INFO:tasks.rados.rados.0.smithi022.stdout:1805: finishing rollback tid 1 to smithi02284960-7 2024-09-21T09:37:27.992 INFO:tasks.rados.rados.0.smithi022.stdout:1805: finishing rollback tid 2 to smithi02284960-7 2024-09-21T09:37:27.992 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 7 v 1108 (ObjNum 519 snap 154 seq_num 519) dirty exists 2024-09-21T09:37:27.992 INFO:tasks.rados.rados.0.smithi022.stdout:1807: finishing write tid 1 to smithi02284960-50 2024-09-21T09:37:27.992 INFO:tasks.rados.rados.0.smithi022.stdout:1806: finishing write tid 1 to smithi02284960-31 2024-09-21T09:37:27.992 INFO:tasks.rados.rados.0.smithi022.stdout:1807: finishing write tid 2 to smithi02284960-50 2024-09-21T09:37:27.992 INFO:tasks.rados.rados.0.smithi022.stdout:1806: finishing write tid 2 to smithi02284960-31 2024-09-21T09:37:27.993 INFO:tasks.rados.rados.0.smithi022.stdout:1807: finishing write tid 3 to smithi02284960-50 2024-09-21T09:37:27.993 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 50 v 1084 (ObjNum 590 snap 176 seq_num 590) dirty exists 2024-09-21T09:37:27.993 INFO:tasks.rados.rados.0.smithi022.stdout:1807: left oid 50 (ObjNum 590 snap 176 seq_num 590) 2024-09-21T09:37:27.993 INFO:tasks.rados.rados.0.smithi022.stdout:1806: finishing write tid 3 to smithi02284960-31 2024-09-21T09:37:27.993 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 31 v 1195 (ObjNum 589 snap 176 seq_num 589) dirty exists 2024-09-21T09:37:27.993 INFO:tasks.rados.rados.0.smithi022.stdout:1806: left oid 31 (ObjNum 589 snap 176 seq_num 589) 2024-09-21T09:37:27.993 INFO:tasks.rados.rados.0.smithi022.stdout:1808: expect (ObjNum 526 snap 156 seq_num 526) 2024-09-21T09:37:28.096 INFO:tasks.rados.rados.0.smithi022.stdout:1810: finishing rollback tid 1 to smithi02284960-22 2024-09-21T09:37:28.096 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 22 v 0 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-09-21T09:37:28.096 INFO:tasks.rados.rados.0.smithi022.stdout:1805: done (4 left) 2024-09-21T09:37:28.096 INFO:tasks.rados.rados.0.smithi022.stdout:1806: done (3 left) 2024-09-21T09:37:28.096 INFO:tasks.rados.rados.0.smithi022.stdout:1807: done (2 left) 2024-09-21T09:37:28.096 INFO:tasks.rados.rados.0.smithi022.stdout:1808: done (1 left) 2024-09-21T09:37:28.096 INFO:tasks.rados.rados.0.smithi022.stdout:1810: done (0 left) 2024-09-21T09:37:28.096 INFO:tasks.rados.rados.0.smithi022.stdout:1811: rollback oid 14 current snap is 176 2024-09-21T09:37:28.096 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 14 to 172 2024-09-21T09:37:28.096 INFO:tasks.rados.rados.0.smithi022.stdout:1812: setattr oid 50 current snap is 176 2024-09-21T09:37:28.097 INFO:tasks.rados.rados.0.smithi022.stdout:1813: read oid 45 snap -1 2024-09-21T09:37:28.097 INFO:tasks.rados.rados.0.smithi022.stdout:1813: expect (ObjNum 442 snap 134 seq_num 442) 2024-09-21T09:37:28.098 INFO:tasks.rados.rados.0.smithi022.stdout:1814: rollback oid 34 current snap is 176 2024-09-21T09:37:28.098 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 34 to 172 2024-09-21T09:37:28.098 INFO:tasks.rados.rados.0.smithi022.stdout:1815: snap_create 2024-09-21T09:37:28.098 INFO:tasks.rados.rados.0.smithi022.stdout:1814: finishing rollback tid 1 to smithi02284960-34 2024-09-21T09:37:28.098 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 34 v 0 (ObjNum 65 snap 0 seq_num 3562300368) dirty dne 2024-09-21T09:37:28.099 INFO:tasks.rados.rados.0.smithi022.stdout:1811: finishing rollback tid 0 to smithi02284960-14 2024-09-21T09:37:28.099 INFO:tasks.rados.rados.0.smithi022.stdout:1811: finishing rollback tid 1 to smithi02284960-14 2024-09-21T09:37:28.100 INFO:tasks.rados.rados.0.smithi022.stdout:1811: finishing rollback tid 2 to smithi02284960-14 2024-09-21T09:37:28.100 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 14 v 1072 (ObjNum 437 snap 134 seq_num 437) dirty exists 2024-09-21T09:37:28.103 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 50 v 1085 (ObjNum 590 snap 176 seq_num 590) dirty exists 2024-09-21T09:37:28.105 INFO:tasks.rados.rados.0.smithi022.stdout:1813: expect (ObjNum 442 snap 134 seq_num 442) 2024-09-21T09:37:28.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:27 smithi022 ceph-mon[98615]: pgmap v241: 121 pgs: 121 active+clean; 120 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 3.4 MiB/s wr, 37 op/s 2024-09-21T09:37:28.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:27 smithi022 ceph-mon[98615]: osdmap e426: 8 total, 8 up, 8 in 2024-09-21T09:37:28.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:27 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-21T09:37:28.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:27 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:37:28.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:27 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:37:28.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:27 smithi022 ceph-mon[96736]: pgmap v241: 121 pgs: 121 active+clean; 120 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 3.4 MiB/s wr, 37 op/s 2024-09-21T09:37:28.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:27 smithi022 ceph-mon[96736]: osdmap e426: 8 total, 8 up, 8 in 2024-09-21T09:37:28.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:27 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-21T09:37:28.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:27 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:37:28.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:27 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:37:28.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:27 smithi064 ceph-mon[86709]: pgmap v241: 121 pgs: 121 active+clean; 120 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 3.4 MiB/s wr, 37 op/s 2024-09-21T09:37:28.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:27 smithi064 ceph-mon[86709]: osdmap e426: 8 total, 8 up, 8 in 2024-09-21T09:37:28.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:27 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-21T09:37:28.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:27 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:37:28.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:27 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:37:28.974 INFO:tasks.rados.rados.0.smithi022.stdout:1811: done (4 left) 2024-09-21T09:37:28.974 INFO:tasks.rados.rados.0.smithi022.stdout:1812: done (3 left) 2024-09-21T09:37:28.974 INFO:tasks.rados.rados.0.smithi022.stdout:1813: done (2 left) 2024-09-21T09:37:28.974 INFO:tasks.rados.rados.0.smithi022.stdout:1814: done (1 left) 2024-09-21T09:37:28.974 INFO:tasks.rados.rados.0.smithi022.stdout:1815: done (0 left) 2024-09-21T09:37:28.975 INFO:tasks.rados.rados.0.smithi022.stdout:1816: snap_remove snap 172 2024-09-21T09:37:29.148 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:28 smithi022 ceph-mon[98615]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-21T09:37:29.149 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:28 smithi022 ceph-mon[98615]: Upgrade: osd.3 is safe to restart 2024-09-21T09:37:29.149 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:28 smithi022 ceph-mon[98615]: osdmap e427: 8 total, 8 up, 8 in 2024-09-21T09:37:29.149 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:28 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:37:29.149 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:28 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-21T09:37:29.149 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:28 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:37:29.149 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:28 smithi022 ceph-mon[98615]: osdmap e428: 8 total, 8 up, 8 in 2024-09-21T09:37:29.149 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:28 smithi022 ceph-mon[96736]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-21T09:37:29.149 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:28 smithi022 ceph-mon[96736]: Upgrade: osd.3 is safe to restart 2024-09-21T09:37:29.149 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:28 smithi022 ceph-mon[96736]: osdmap e427: 8 total, 8 up, 8 in 2024-09-21T09:37:29.150 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:28 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:37:29.150 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:28 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-21T09:37:29.150 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:28 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:37:29.150 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:28 smithi022 ceph-mon[96736]: osdmap e428: 8 total, 8 up, 8 in 2024-09-21T09:37:29.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:28 smithi064 ceph-mon[86709]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-21T09:37:29.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:28 smithi064 ceph-mon[86709]: Upgrade: osd.3 is safe to restart 2024-09-21T09:37:29.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:28 smithi064 ceph-mon[86709]: osdmap e427: 8 total, 8 up, 8 in 2024-09-21T09:37:29.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:28 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:37:29.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:28 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-21T09:37:29.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:28 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:37:29.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:28 smithi064 ceph-mon[86709]: osdmap e428: 8 total, 8 up, 8 in 2024-09-21T09:37:29.977 INFO:tasks.rados.rados.0.smithi022.stdout:1816: done (0 left) 2024-09-21T09:37:29.977 INFO:tasks.rados.rados.0.smithi022.stdout:1817: rmattr oid 46 current snap is 177 2024-09-21T09:37:29.978 INFO:tasks.rados.rados.0.smithi022.stdout:1818: rollback oid 50 current snap is 177 2024-09-21T09:37:29.978 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 50 to 176 2024-09-21T09:37:29.978 INFO:tasks.rados.rados.0.smithi022.stdout:1819: read oid 25 snap -1 2024-09-21T09:37:29.978 INFO:tasks.rados.rados.0.smithi022.stdout:1819: expect deleted 2024-09-21T09:37:29.978 INFO:tasks.rados.rados.0.smithi022.stdout:1820: snap_create 2024-09-21T09:37:29.989 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 46 v 1042 (ObjNum 536 snap 161 seq_num 536) dirty exists 2024-09-21T09:37:29.990 INFO:tasks.rados.rados.0.smithi022.stdout:1818: finishing rollback tid 0 to smithi02284960-50 2024-09-21T09:37:29.990 INFO:tasks.rados.rados.0.smithi022.stdout:1818: finishing rollback tid 1 to smithi02284960-50 2024-09-21T09:37:29.991 INFO:tasks.rados.rados.0.smithi022.stdout:1818: finishing rollback tid 2 to smithi02284960-50 2024-09-21T09:37:29.991 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 50 v 1088 (ObjNum 590 snap 176 seq_num 590) dirty exists 2024-09-21T09:37:30.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:29 smithi022 ceph-mon[98615]: pgmap v244: 121 pgs: 121 active+clean; 120 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4.7 MiB/s rd, 862 KiB/s wr, 15 op/s 2024-09-21T09:37:30.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:29 smithi022 ceph-mon[98615]: Upgrade: Updating osd.3 2024-09-21T09:37:30.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:29 smithi022 ceph-mon[98615]: Deploying daemon osd.3 on smithi022 2024-09-21T09:37:30.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:29 smithi022 ceph-mon[98615]: osdmap e429: 8 total, 8 up, 8 in 2024-09-21T09:37:30.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:29 smithi022 ceph-mon[96736]: pgmap v244: 121 pgs: 121 active+clean; 120 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4.7 MiB/s rd, 862 KiB/s wr, 15 op/s 2024-09-21T09:37:30.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:29 smithi022 ceph-mon[96736]: Upgrade: Updating osd.3 2024-09-21T09:37:30.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:29 smithi022 ceph-mon[96736]: Deploying daemon osd.3 on smithi022 2024-09-21T09:37:30.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:29 smithi022 ceph-mon[96736]: osdmap e429: 8 total, 8 up, 8 in 2024-09-21T09:37:30.441 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:37:30 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:37:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:37:30.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:29 smithi064 ceph-mon[86709]: pgmap v244: 121 pgs: 121 active+clean; 120 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4.7 MiB/s rd, 862 KiB/s wr, 15 op/s 2024-09-21T09:37:30.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:29 smithi064 ceph-mon[86709]: Upgrade: Updating osd.3 2024-09-21T09:37:30.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:29 smithi064 ceph-mon[86709]: Deploying daemon osd.3 on smithi022 2024-09-21T09:37:30.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:29 smithi064 ceph-mon[86709]: osdmap e429: 8 total, 8 up, 8 in 2024-09-21T09:37:30.977 INFO:tasks.rados.rados.0.smithi022.stdout:1817: done (3 left) 2024-09-21T09:37:30.977 INFO:tasks.rados.rados.0.smithi022.stdout:1818: done (2 left) 2024-09-21T09:37:30.977 INFO:tasks.rados.rados.0.smithi022.stdout:1819: done (1 left) 2024-09-21T09:37:30.977 INFO:tasks.rados.rados.0.smithi022.stdout:1820: done (0 left) 2024-09-21T09:37:30.977 INFO:tasks.rados.rados.0.smithi022.stdout:1821: read oid 1 snap -1 2024-09-21T09:37:30.977 INFO:tasks.rados.rados.0.smithi022.stdout:1821: expect (ObjNum 552 snap 166 seq_num 552) 2024-09-21T09:37:30.977 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 14 current snap is 178 2024-09-21T09:37:30.977 INFO:tasks.rados.rados.0.smithi022.stdout:1822: seq_num 591 ranges {2187264=303104} 2024-09-21T09:37:30.992 INFO:tasks.rados.rados.0.smithi022.stdout:1822: writing smithi02284960-14 from 2187264 to 2490368 tid 1 2024-09-21T09:37:30.992 INFO:tasks.rados.rados.0.smithi022.stdout:1823: snap_remove snap 177 2024-09-21T09:37:31.983 INFO:tasks.rados.rados.0.smithi022.stdout:1823: done (2 left) 2024-09-21T09:37:31.983 INFO:tasks.rados.rados.0.smithi022.stdout:1824: rollback oid 30 current snap is 178 2024-09-21T09:37:31.983 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 30 to 176 2024-09-21T09:37:31.983 INFO:tasks.rados.rados.0.smithi022.stdout:1821: expect (ObjNum 552 snap 166 seq_num 552) 2024-09-21T09:37:32.094 INFO:tasks.rados.rados.0.smithi022.stdout:1822: finishing write tid 1 to smithi02284960-14 2024-09-21T09:37:32.094 INFO:tasks.rados.rados.0.smithi022.stdout:1822: finishing write tid 2 to smithi02284960-14 2024-09-21T09:37:32.094 INFO:tasks.rados.rados.0.smithi022.stdout:1822: finishing write tid 3 to smithi02284960-14 2024-09-21T09:37:32.094 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 14 v 1074 (ObjNum 591 snap 178 seq_num 591) dirty exists 2024-09-21T09:37:32.094 INFO:tasks.rados.rados.0.smithi022.stdout:1822: left oid 14 (ObjNum 591 snap 178 seq_num 591) 2024-09-21T09:37:32.095 INFO:tasks.rados.rados.0.smithi022.stdout:1824: finishing rollback tid 1 to smithi02284960-30 2024-09-21T09:37:32.095 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 30 v 0 (ObjNum 161 snap 0 seq_num 1) dirty dne 2024-09-21T09:37:32.095 INFO:tasks.rados.rados.0.smithi022.stdout:1821: done (2 left) 2024-09-21T09:37:32.095 INFO:tasks.rados.rados.0.smithi022.stdout:1822: done (1 left) 2024-09-21T09:37:32.095 INFO:tasks.rados.rados.0.smithi022.stdout:1824: done (0 left) 2024-09-21T09:37:32.095 INFO:tasks.rados.rados.0.smithi022.stdout:1825: copy_from oid 28 from oid 30 current snap is 178 2024-09-21T09:37:32.095 INFO:tasks.rados.rados.0.smithi022.stdout:1826: setattr oid 11 current snap is 178 2024-09-21T09:37:32.096 INFO:tasks.rados.rados.0.smithi022.stdout:1827: copy_from oid 39 from oid 32 current snap is 178 2024-09-21T09:37:32.096 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 23 current snap is 178 2024-09-21T09:37:32.096 INFO:tasks.rados.rados.0.smithi022.stdout:1828: seq_num 592 ranges {5013504=319488} 2024-09-21T09:37:32.124 INFO:tasks.rados.rados.0.smithi022.stdout:1828: writing smithi02284960-23 from 5013504 to 5332992 tid 1 2024-09-21T09:37:32.125 INFO:tasks.rados.rados.0.smithi022.stdout:1829: snap_remove snap 176 2024-09-21T09:37:32.141 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:31 smithi022 ceph-mon[98615]: pgmap v247: 121 pgs: 121 active+clean; 120 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3.5 MiB/s rd, 136 KiB/s wr, 5 op/s 2024-09-21T09:37:32.142 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:31 smithi022 ceph-mon[98615]: osdmap e430: 8 total, 8 up, 8 in 2024-09-21T09:37:32.142 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:31 smithi022 ceph-mon[96736]: pgmap v247: 121 pgs: 121 active+clean; 120 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3.5 MiB/s rd, 136 KiB/s wr, 5 op/s 2024-09-21T09:37:32.142 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:31 smithi022 ceph-mon[96736]: osdmap e430: 8 total, 8 up, 8 in 2024-09-21T09:37:32.142 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 21 09:37:31 smithi022 systemd[1]: Stopping Ceph osd.3 for d65e801e-77fa-11ef-baf6-efdc52797490... 2024-09-21T09:37:32.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:31 smithi064 ceph-mon[86709]: pgmap v247: 121 pgs: 121 active+clean; 120 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3.5 MiB/s rd, 136 KiB/s wr, 5 op/s 2024-09-21T09:37:32.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:31 smithi064 ceph-mon[86709]: osdmap e430: 8 total, 8 up, 8 in 2024-09-21T09:37:32.558 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 21 09:37:32 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-3[62753]: 2024-09-21T09:37:32.139+0000 7fda0aa90640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T09:37:32.558 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 21 09:37:32 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-3[62753]: 2024-09-21T09:37:32.139+0000 7fda0aa90640 -1 osd.3 431 *** Got signal Terminated *** 2024-09-21T09:37:32.558 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 21 09:37:32 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-3[62753]: 2024-09-21T09:37:32.139+0000 7fda0aa90640 -1 osd.3 431 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T09:37:32.991 INFO:tasks.rados.rados.0.smithi022.stdout:1825: finishing copy_from to smithi02284960-28 2024-09-21T09:37:32.991 INFO:tasks.rados.rados.0.smithi022.stdout:1825: got expected ENOENT (src dne) 2024-09-21T09:37:32.991 INFO:tasks.rados.rados.0.smithi022.stdout:1825: finishing copy_from racing read to smithi02284960-28 2024-09-21T09:37:32.992 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 11 v 1044 (ObjNum 569 snap 170 seq_num 569) dirty exists 2024-09-21T09:37:32.992 INFO:tasks.rados.rados.0.smithi022.stdout:1825: done (4 left) 2024-09-21T09:37:32.992 INFO:tasks.rados.rados.0.smithi022.stdout:1826: done (3 left) 2024-09-21T09:37:32.992 INFO:tasks.rados.rados.0.smithi022.stdout:1829: done (2 left) 2024-09-21T09:37:32.992 INFO:tasks.rados.rados.0.smithi022.stdout:1830: copy_from oid 17 from oid 27 current snap is 178 2024-09-21T09:37:32.992 INFO:tasks.rados.rados.0.smithi022.stdout:1827: finishing copy_from racing read to smithi02284960-39 2024-09-21T09:37:32.992 INFO:tasks.rados.rados.0.smithi022.stdout:1827: finishing copy_from to smithi02284960-39 2024-09-21T09:37:32.992 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 39 v 1041 (ObjNum 538 snap 162 seq_num 538) dirty exists 2024-09-21T09:37:32.992 INFO:tasks.rados.rados.0.smithi022.stdout:1828: finishing write tid 1 to smithi02284960-23 2024-09-21T09:37:32.992 INFO:tasks.rados.rados.0.smithi022.stdout:1828: finishing write tid 2 to smithi02284960-23 2024-09-21T09:37:32.992 INFO:tasks.rados.rados.0.smithi022.stdout:1828: finishing write tid 3 to smithi02284960-23 2024-09-21T09:37:32.992 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 23 v 863 (ObjNum 592 snap 178 seq_num 592) dirty exists 2024-09-21T09:37:32.992 INFO:tasks.rados.rados.0.smithi022.stdout:1828: left oid 23 (ObjNum 592 snap 178 seq_num 592) 2024-09-21T09:37:32.992 INFO:tasks.rados.rados.0.smithi022.stdout:1827: done (2 left) 2024-09-21T09:37:32.992 INFO:tasks.rados.rados.0.smithi022.stdout:1828: done (1 left) 2024-09-21T09:37:32.993 INFO:tasks.rados.rados.0.smithi022.stdout:1831: snap_create 2024-09-21T09:37:33.007 INFO:tasks.rados.rados.0.smithi022.stdout:1830: finishing copy_from to smithi02284960-17 2024-09-21T09:37:33.007 INFO:tasks.rados.rados.0.smithi022.stdout:1830: got expected ENOENT (src dne) 2024-09-21T09:37:33.007 INFO:tasks.rados.rados.0.smithi022.stdout:1830: finishing copy_from racing read to smithi02284960-17 2024-09-21T09:37:33.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:32 smithi022 ceph-mon[98615]: osdmap e431: 8 total, 8 up, 8 in 2024-09-21T09:37:33.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:32 smithi022 ceph-mon[98615]: osd.3 marked itself down and dead 2024-09-21T09:37:33.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:32 smithi022 ceph-mon[96736]: osdmap e431: 8 total, 8 up, 8 in 2024-09-21T09:37:33.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:32 smithi022 ceph-mon[96736]: osd.3 marked itself down and dead 2024-09-21T09:37:33.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:32 smithi064 ceph-mon[86709]: osdmap e431: 8 total, 8 up, 8 in 2024-09-21T09:37:33.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:32 smithi064 ceph-mon[86709]: osd.3 marked itself down and dead 2024-09-21T09:37:33.808 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 21 09:37:33 smithi022 podman[117480]: 2024-09-21 09:37:33.529921656 +0000 UTC m=+1.495661765 container died a56193b67a92a6fd8784828a25d60a0e6a9d2711db13d7dc6182c0610510f3a8 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-3, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-fbd1a6a, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.37.2, GIT_BRANCH=HEAD) 2024-09-21T09:37:33.975 INFO:tasks.rados.rados.0.smithi022.stdout:1830: done (1 left) 2024-09-21T09:37:33.976 INFO:tasks.rados.rados.0.smithi022.stdout:1831: done (0 left) 2024-09-21T09:37:33.976 INFO:tasks.rados.rados.0.smithi022.stdout:1832: delete oid 30 current snap is 179 2024-09-21T09:37:33.983 INFO:tasks.rados.rados.0.smithi022.stdout:1832: done (0 left) 2024-09-21T09:37:33.983 INFO:tasks.rados.rados.0.smithi022.stdout:1833: rollback oid 41 current snap is 179 2024-09-21T09:37:33.983 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 41 to 178 2024-09-21T09:37:33.984 INFO:tasks.rados.rados.0.smithi022.stdout:1834: read oid 42 snap -1 2024-09-21T09:37:33.984 INFO:tasks.rados.rados.0.smithi022.stdout:1834: expect (ObjNum 392 snap 122 seq_num 392) 2024-09-21T09:37:33.984 INFO:tasks.rados.rados.0.smithi022.stdout:1835: copy_from oid 31 from oid 50 current snap is 179 2024-09-21T09:37:33.984 INFO:tasks.rados.rados.0.smithi022.stdout:1836: snap_create 2024-09-21T09:37:33.993 INFO:tasks.rados.rados.0.smithi022.stdout:1835: finishing copy_from racing read to smithi02284960-31 2024-09-21T09:37:33.994 INFO:tasks.rados.rados.0.smithi022.stdout:1833: finishing rollback tid 0 to smithi02284960-41 2024-09-21T09:37:33.994 INFO:tasks.rados.rados.0.smithi022.stdout:1833: finishing rollback tid 1 to smithi02284960-41 2024-09-21T09:37:33.994 INFO:tasks.rados.rados.0.smithi022.stdout:1833: finishing rollback tid 2 to smithi02284960-41 2024-09-21T09:37:33.995 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 41 v 997 (ObjNum 531 snap 159 seq_num 531) dirty exists 2024-09-21T09:37:33.997 INFO:tasks.rados.rados.0.smithi022.stdout:1835: finishing copy_from to smithi02284960-31 2024-09-21T09:37:33.997 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 31 v 1088 (ObjNum 590 snap 176 seq_num 590) dirty exists 2024-09-21T09:37:34.003 INFO:tasks.rados.rados.0.smithi022.stdout:1834: expect (ObjNum 392 snap 122 seq_num 392) 2024-09-21T09:37:34.256 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:33 smithi022 ceph-mon[98615]: pgmap v250: 121 pgs: 121 active+clean; 120 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2.1 MiB/s rd, 0 B/s wr, 2 op/s 2024-09-21T09:37:34.257 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:33 smithi022 ceph-mon[98615]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T09:37:34.257 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:33 smithi022 ceph-mon[98615]: osdmap e432: 8 total, 7 up, 8 in 2024-09-21T09:37:34.257 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:33 smithi022 ceph-mon[98615]: osdmap e433: 8 total, 7 up, 8 in 2024-09-21T09:37:34.257 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 21 09:37:33 smithi022 podman[117480]: 2024-09-21 09:37:33.99580512 +0000 UTC m=+1.961545228 container cleanup a56193b67a92a6fd8784828a25d60a0e6a9d2711db13d7dc6182c0610510f3a8 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-3, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , RELEASE=quincy-fbd1a6a, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-09-21T09:37:34.257 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 21 09:37:33 smithi022 bash[117480]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-3 2024-09-21T09:37:34.257 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:33 smithi022 ceph-mon[96736]: pgmap v250: 121 pgs: 121 active+clean; 120 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2.1 MiB/s rd, 0 B/s wr, 2 op/s 2024-09-21T09:37:34.257 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:33 smithi022 ceph-mon[96736]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T09:37:34.258 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:33 smithi022 ceph-mon[96736]: osdmap e432: 8 total, 7 up, 8 in 2024-09-21T09:37:34.258 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:33 smithi022 ceph-mon[96736]: osdmap e433: 8 total, 7 up, 8 in 2024-09-21T09:37:34.343 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:33 smithi064 ceph-mon[86709]: pgmap v250: 121 pgs: 121 active+clean; 120 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2.1 MiB/s rd, 0 B/s wr, 2 op/s 2024-09-21T09:37:34.343 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:33 smithi064 ceph-mon[86709]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T09:37:34.343 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:33 smithi064 ceph-mon[86709]: osdmap e432: 8 total, 7 up, 8 in 2024-09-21T09:37:34.343 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:33 smithi064 ceph-mon[86709]: osdmap e433: 8 total, 7 up, 8 in 2024-09-21T09:37:34.343 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:37:34 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:37:34.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.3\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.3\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.3\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:37:34.557 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 21 09:37:34 smithi022 podman[117501]: 2024-09-21 09:37:34.255608049 +0000 UTC m=+0.718647454 container remove a56193b67a92a6fd8784828a25d60a0e6a9d2711db13d7dc6182c0610510f3a8 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-3, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, RELEASE=quincy-fbd1a6a, org.label-schema.build-date=20240731, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS) 2024-09-21T09:37:34.910 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 21 09:37:34 smithi022 podman[117595]: 2024-09-21 09:37:34.528740154 +0000 UTC m=+0.016855137 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:37:34.910 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 21 09:37:34 smithi022 podman[117595]: 2024-09-21 09:37:34.631732303 +0000 UTC m=+0.119847274 container create be6fb80065cb6cea1ce4702acdaff4775cfbf03e550810a242f4cf43bc161545 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-3-deactivate, ceph=True, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, RELEASE=main-62da54b) 2024-09-21T09:37:34.910 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 21 09:37:34 smithi022 podman[117595]: 2024-09-21 09:37:34.817431571 +0000 UTC m=+0.305546552 container init be6fb80065cb6cea1ce4702acdaff4775cfbf03e550810a242f4cf43bc161545 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-3-deactivate, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-62da54b, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, org.label-schema.build-date=20240731) 2024-09-21T09:37:34.910 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 21 09:37:34 smithi022 podman[117595]: 2024-09-21 09:37:34.822442696 +0000 UTC m=+0.310557668 container start be6fb80065cb6cea1ce4702acdaff4775cfbf03e550810a242f4cf43bc161545 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-3-deactivate, org.label-schema.vendor=CentOS, RELEASE=main-62da54b, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=) 2024-09-21T09:37:34.911 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 21 09:37:34 smithi022 podman[117595]: 2024-09-21 09:37:34.908927018 +0000 UTC m=+0.397042004 container attach be6fb80065cb6cea1ce4702acdaff4775cfbf03e550810a242f4cf43bc161545 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-3-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-62da54b, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T09:37:34.977 INFO:tasks.rados.rados.0.smithi022.stdout:1833: done (3 left) 2024-09-21T09:37:34.977 INFO:tasks.rados.rados.0.smithi022.stdout:1834: done (2 left) 2024-09-21T09:37:34.977 INFO:tasks.rados.rados.0.smithi022.stdout:1835: done (1 left) 2024-09-21T09:37:34.977 INFO:tasks.rados.rados.0.smithi022.stdout:1836: done (0 left) 2024-09-21T09:37:34.977 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 47 current snap is 180 2024-09-21T09:37:34.977 INFO:tasks.rados.rados.0.smithi022.stdout:1837: seq_num 594 ranges {2973696=352256} 2024-09-21T09:37:34.995 INFO:tasks.rados.rados.0.smithi022.stdout:1837: writing smithi02284960-47 from 2973696 to 3325952 tid 1 2024-09-21T09:37:34.996 INFO:tasks.rados.rados.0.smithi022.stdout:1838: read oid 37 snap -1 2024-09-21T09:37:34.996 INFO:tasks.rados.rados.0.smithi022.stdout:1838: expect (ObjNum 583 snap 172 seq_num 583) 2024-09-21T09:37:34.996 INFO:tasks.rados.rados.0.smithi022.stdout:1839: snap_remove snap 178 2024-09-21T09:37:35.308 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 21 09:37:35 smithi022 podman[117595]: 2024-09-21 09:37:35.024210567 +0000 UTC m=+0.512325548 container died be6fb80065cb6cea1ce4702acdaff4775cfbf03e550810a242f4cf43bc161545 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-3-deactivate, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, RELEASE=main-62da54b, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True) 2024-09-21T09:37:35.664 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 21 09:37:35 smithi022 podman[117638]: 2024-09-21 09:37:35.646005323 +0000 UTC m=+0.615548346 container remove be6fb80065cb6cea1ce4702acdaff4775cfbf03e550810a242f4cf43bc161545 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-3-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux , RELEASE=main-62da54b, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS) 2024-09-21T09:37:35.975 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:37:35 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:37:35] "GET /metrics HTTP/1.1" 200 36237 "" "Prometheus/2.51.0" 2024-09-21T09:37:35.975 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 21 09:37:35 smithi022 systemd[1]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490@osd.3.service: Deactivated successfully. 2024-09-21T09:37:35.976 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 21 09:37:35 smithi022 systemd[1]: Stopped Ceph osd.3 for d65e801e-77fa-11ef-baf6-efdc52797490. 2024-09-21T09:37:35.976 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 21 09:37:35 smithi022 systemd[1]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490@osd.3.service: Consumed 10.342s CPU time. 2024-09-21T09:37:35.976 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 21 09:37:35 smithi022 systemd[1]: Starting Ceph osd.3 for d65e801e-77fa-11ef-baf6-efdc52797490... 2024-09-21T09:37:35.998 INFO:tasks.rados.rados.0.smithi022.stdout:1839: done (2 left) 2024-09-21T09:37:35.999 INFO:tasks.rados.rados.0.smithi022.stdout:1840: read oid 29 snap 179 2024-09-21T09:37:35.999 INFO:tasks.rados.rados.0.smithi022.stdout:1840: expect (ObjNum 570 snap 171 seq_num 570) 2024-09-21T09:37:35.999 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 23 current snap is 180 2024-09-21T09:37:35.999 INFO:tasks.rados.rados.0.smithi022.stdout:1841: seq_num 595 ranges {5332992=368640} 2024-09-21T09:37:36.030 INFO:tasks.rados.rados.0.smithi022.stdout:1841: writing smithi02284960-23 from 5332992 to 5701632 tid 1 2024-09-21T09:37:36.030 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 49 current snap is 180 2024-09-21T09:37:36.030 INFO:tasks.rados.rados.0.smithi022.stdout:1842: seq_num 596 ranges {7921664=385024} 2024-09-21T09:37:36.075 INFO:tasks.rados.rados.0.smithi022.stdout:1842: writing smithi02284960-49 from 7921664 to 8306688 tid 1 2024-09-21T09:37:36.075 INFO:tasks.rados.rados.0.smithi022.stdout:1843: read oid 12 snap -1 2024-09-21T09:37:36.075 INFO:tasks.rados.rados.0.smithi022.stdout:1843: expect (ObjNum 442 snap 134 seq_num 442) 2024-09-21T09:37:36.076 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 45 current snap is 180 2024-09-21T09:37:36.076 INFO:tasks.rados.rados.0.smithi022.stdout:1837: finishing write tid 1 to smithi02284960-47 2024-09-21T09:37:36.076 INFO:tasks.rados.rados.0.smithi022.stdout:1837: finishing write tid 2 to smithi02284960-47 2024-09-21T09:37:36.076 INFO:tasks.rados.rados.0.smithi022.stdout:1837: finishing write tid 3 to smithi02284960-47 2024-09-21T09:37:36.076 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 47 v 1051 (ObjNum 594 snap 180 seq_num 594) dirty exists 2024-09-21T09:37:36.076 INFO:tasks.rados.rados.0.smithi022.stdout:1837: left oid 47 (ObjNum 594 snap 180 seq_num 594) 2024-09-21T09:37:36.076 INFO:tasks.rados.rados.0.smithi022.stdout:1838: expect (ObjNum 583 snap 172 seq_num 583) 2024-09-21T09:37:36.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:35 smithi064 ceph-mon[86709]: pgmap v253: 121 pgs: 27 peering, 4 stale+active+clean, 90 active+clean; 120 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1.7 MiB/s rd, 74 KiB/s wr, 7 op/s 2024-09-21T09:37:36.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:35 smithi064 ceph-mon[86709]: osdmap e434: 8 total, 7 up, 8 in 2024-09-21T09:37:36.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:35 smithi064 ceph-mon[86709]: Health check failed: Reduced data availability: 7 pgs inactive, 7 pgs peering (PG_AVAILABILITY) 2024-09-21T09:37:36.192 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:37:35 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:37:35.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:37:36.192 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:37:35 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:37:35.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:37:36.252 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:35 smithi022 ceph-mon[98615]: pgmap v253: 121 pgs: 27 peering, 4 stale+active+clean, 90 active+clean; 120 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1.7 MiB/s rd, 74 KiB/s wr, 7 op/s 2024-09-21T09:37:36.252 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:35 smithi022 ceph-mon[98615]: osdmap e434: 8 total, 7 up, 8 in 2024-09-21T09:37:36.252 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:35 smithi022 ceph-mon[98615]: Health check failed: Reduced data availability: 7 pgs inactive, 7 pgs peering (PG_AVAILABILITY) 2024-09-21T09:37:36.252 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 21 09:37:36 smithi022 podman[117741]: 2024-09-21 09:37:36.008166493 +0000 UTC m=+0.022413638 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:37:36.252 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 21 09:37:36 smithi022 podman[117741]: 2024-09-21 09:37:36.139164716 +0000 UTC m=+0.153411863 container create 7e74f752cc9c27f9db8bebb04ca9e50e0414ae0ed0fc004d56b26a497fabcfa9 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-3-activate, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=main-62da54b, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T09:37:36.253 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:35 smithi022 ceph-mon[96736]: pgmap v253: 121 pgs: 27 peering, 4 stale+active+clean, 90 active+clean; 120 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1.7 MiB/s rd, 74 KiB/s wr, 7 op/s 2024-09-21T09:37:36.253 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:35 smithi022 ceph-mon[96736]: osdmap e434: 8 total, 7 up, 8 in 2024-09-21T09:37:36.253 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:35 smithi022 ceph-mon[96736]: Health check failed: Reduced data availability: 7 pgs inactive, 7 pgs peering (PG_AVAILABILITY) 2024-09-21T09:37:36.394 INFO:tasks.rados.rados.0.smithi022.stdout:1840: expect (ObjNum 570 snap 171 seq_num 570) 2024-09-21T09:37:36.457 INFO:tasks.rados.rados.0.smithi022.stdout:1841: finishing write tid 1 to smithi02284960-23 2024-09-21T09:37:36.457 INFO:tasks.rados.rados.0.smithi022.stdout:1841: finishing write tid 2 to smithi02284960-23 2024-09-21T09:37:36.457 INFO:tasks.rados.rados.0.smithi022.stdout:1841: finishing write tid 3 to smithi02284960-23 2024-09-21T09:37:36.457 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 23 v 865 (ObjNum 595 snap 180 seq_num 595) dirty exists 2024-09-21T09:37:36.457 INFO:tasks.rados.rados.0.smithi022.stdout:1841: left oid 23 (ObjNum 595 snap 180 seq_num 595) 2024-09-21T09:37:36.457 INFO:tasks.rados.rados.0.smithi022.stdout:1842: finishing write tid 1 to smithi02284960-49 2024-09-21T09:37:36.457 INFO:tasks.rados.rados.0.smithi022.stdout:1844: seq_num 597 ranges {2293760=401408} 2024-09-21T09:37:36.473 INFO:tasks.rados.rados.0.smithi022.stdout:1844: writing smithi02284960-45 from 2293760 to 2695168 tid 1 2024-09-21T09:37:36.473 INFO:tasks.rados.rados.0.smithi022.stdout:1837: done (6 left) 2024-09-21T09:37:36.473 INFO:tasks.rados.rados.0.smithi022.stdout:1838: done (5 left) 2024-09-21T09:37:36.473 INFO:tasks.rados.rados.0.smithi022.stdout:1840: done (4 left) 2024-09-21T09:37:36.473 INFO:tasks.rados.rados.0.smithi022.stdout:1841: done (3 left) 2024-09-21T09:37:36.474 INFO:tasks.rados.rados.0.smithi022.stdout:1845: snap_create 2024-09-21T09:37:36.474 INFO:tasks.rados.rados.0.smithi022.stdout:1842: finishing write tid 2 to smithi02284960-49 2024-09-21T09:37:36.474 INFO:tasks.rados.rados.0.smithi022.stdout:1842: finishing write tid 3 to smithi02284960-49 2024-09-21T09:37:36.474 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 49 v 1076 (ObjNum 596 snap 180 seq_num 596) dirty exists 2024-09-21T09:37:36.474 INFO:tasks.rados.rados.0.smithi022.stdout:1842: left oid 49 (ObjNum 596 snap 180 seq_num 596) 2024-09-21T09:37:36.474 INFO:tasks.rados.rados.0.smithi022.stdout:1843: expect (ObjNum 442 snap 134 seq_num 442) 2024-09-21T09:37:36.506 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 21 09:37:36 smithi022 podman[117741]: 2024-09-21 09:37:36.341720507 +0000 UTC m=+0.355967659 container init 7e74f752cc9c27f9db8bebb04ca9e50e0414ae0ed0fc004d56b26a497fabcfa9 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-3-activate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, org.label-schema.build-date=20240731, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, RELEASE=main-62da54b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-09-21T09:37:36.506 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 21 09:37:36 smithi022 podman[117741]: 2024-09-21 09:37:36.346834724 +0000 UTC m=+0.361081872 container start 7e74f752cc9c27f9db8bebb04ca9e50e0414ae0ed0fc004d56b26a497fabcfa9 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-3-activate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_CLEAN=True, RELEASE=main-62da54b, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T09:37:36.506 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 21 09:37:36 smithi022 podman[117741]: 2024-09-21 09:37:36.450272781 +0000 UTC m=+0.464519932 container attach 7e74f752cc9c27f9db8bebb04ca9e50e0414ae0ed0fc004d56b26a497fabcfa9 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-3-activate, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_CLEAN=True, RELEASE=main-62da54b, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=) 2024-09-21T09:37:36.506 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 21 09:37:36 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-3-activate[117756]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T09:37:36.506 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 21 09:37:36 smithi022 bash[117741]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T09:37:36.506 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 21 09:37:36 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-3-activate[117756]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T09:37:36.506 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 21 09:37:36 smithi022 bash[117741]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T09:37:36.587 INFO:tasks.rados.rados.0.smithi022.stdout:1844: finishing write tid 1 to smithi02284960-45 2024-09-21T09:37:36.587 INFO:tasks.rados.rados.0.smithi022.stdout:1844: finishing write tid 2 to smithi02284960-45 2024-09-21T09:37:36.587 INFO:tasks.rados.rados.0.smithi022.stdout:1844: finishing write tid 3 to smithi02284960-45 2024-09-21T09:37:36.587 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 45 v 1090 (ObjNum 597 snap 180 seq_num 597) dirty exists 2024-09-21T09:37:36.588 INFO:tasks.rados.rados.0.smithi022.stdout:1844: left oid 45 (ObjNum 597 snap 180 seq_num 597) 2024-09-21T09:37:37.002 INFO:tasks.rados.rados.0.smithi022.stdout:1842: done (3 left) 2024-09-21T09:37:37.003 INFO:tasks.rados.rados.0.smithi022.stdout:1843: done (2 left) 2024-09-21T09:37:37.003 INFO:tasks.rados.rados.0.smithi022.stdout:1844: done (1 left) 2024-09-21T09:37:37.003 INFO:tasks.rados.rados.0.smithi022.stdout:1845: done (0 left) 2024-09-21T09:37:37.003 INFO:tasks.rados.rados.0.smithi022.stdout:1846: rollback oid 13 current snap is 181 2024-09-21T09:37:37.003 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 13 to 180 2024-09-21T09:37:37.003 INFO:tasks.rados.rados.0.smithi022.stdout:1847: read oid 46 snap -1 2024-09-21T09:37:37.003 INFO:tasks.rados.rados.0.smithi022.stdout:1847: expect (ObjNum 536 snap 161 seq_num 536) 2024-09-21T09:37:37.003 INFO:tasks.rados.rados.0.smithi022.stdout:1848: read oid 48 snap -1 2024-09-21T09:37:37.003 INFO:tasks.rados.rados.0.smithi022.stdout:1848: expect (ObjNum 581 snap 172 seq_num 581) 2024-09-21T09:37:37.003 INFO:tasks.rados.rados.0.smithi022.stdout:1849: read oid 28 snap -1 2024-09-21T09:37:37.003 INFO:tasks.rados.rados.0.smithi022.stdout:1849: expect (ObjNum 392 snap 122 seq_num 392) 2024-09-21T09:37:37.003 INFO:tasks.rados.rados.0.smithi022.stdout:1850: delete oid 37 current snap is 181 2024-09-21T09:37:37.003 INFO:tasks.rados.rados.0.smithi022.stdout:1846: finishing rollback tid 1 to smithi02284960-13 2024-09-21T09:37:37.003 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 13 v 0 (ObjNum 177 snap 0 seq_num 3570065648) dirty dne 2024-09-21T09:37:37.006 INFO:tasks.rados.rados.0.smithi022.stdout:1846: done (4 left) 2024-09-21T09:37:37.006 INFO:tasks.rados.rados.0.smithi022.stdout:1850: done (3 left) 2024-09-21T09:37:37.006 INFO:tasks.rados.rados.0.smithi022.stdout:1851: read oid 5 snap 179 2024-09-21T09:37:37.006 INFO:tasks.rados.rados.0.smithi022.stdout:1851: expect (ObjNum 558 snap 167 seq_num 558) 2024-09-21T09:37:37.006 INFO:tasks.rados.rados.0.smithi022.stdout:1852: read oid 15 snap -1 2024-09-21T09:37:37.006 INFO:tasks.rados.rados.0.smithi022.stdout:1852: expect (ObjNum 574 snap 172 seq_num 574) 2024-09-21T09:37:37.007 INFO:tasks.rados.rados.0.smithi022.stdout:1853: copy_from oid 27 from oid 31 current snap is 181 2024-09-21T09:37:37.007 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 6 current snap is 181 2024-09-21T09:37:37.007 INFO:tasks.rados.rados.0.smithi022.stdout:1854: seq_num 599 ranges {2908160=434176} 2024-09-21T09:37:37.026 INFO:tasks.rados.rados.0.smithi022.stdout:1854: writing smithi02284960-6 from 2908160 to 3342336 tid 1 2024-09-21T09:37:37.026 INFO:tasks.rados.rados.0.smithi022.stdout:1855: snap_create 2024-09-21T09:37:37.026 INFO:tasks.rados.rados.0.smithi022.stdout:1847: expect (ObjNum 536 snap 161 seq_num 536) 2024-09-21T09:37:37.076 INFO:tasks.rados.rados.0.smithi022.stdout:1852: expect (ObjNum 574 snap 172 seq_num 574) 2024-09-21T09:37:37.159 INFO:tasks.rados.rados.0.smithi022.stdout:1853: finishing copy_from racing read to smithi02284960-27 2024-09-21T09:37:37.159 INFO:tasks.rados.rados.0.smithi022.stdout:1853: finishing copy_from to smithi02284960-27 2024-09-21T09:37:37.159 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 27 v 1088 (ObjNum 590 snap 176 seq_num 590) dirty exists 2024-09-21T09:37:37.159 INFO:tasks.rados.rados.0.smithi022.stdout:1849: expect (ObjNum 392 snap 122 seq_num 392) 2024-09-21T09:37:37.205 INFO:tasks.rados.rados.0.smithi022.stdout:1848: expect (ObjNum 581 snap 172 seq_num 581) 2024-09-21T09:37:37.262 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:36 smithi022 ceph-mon[98615]: osdmap e435: 8 total, 7 up, 8 in 2024-09-21T09:37:37.263 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:36 smithi022 ceph-mon[96736]: osdmap e435: 8 total, 7 up, 8 in 2024-09-21T09:37:37.263 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 21 09:37:36 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-3-activate[117756]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-21T09:37:37.263 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 21 09:37:36 smithi022 bash[117741]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-21T09:37:37.263 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 21 09:37:36 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-3-activate[117756]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-21T09:37:37.263 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 21 09:37:36 smithi022 bash[117741]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-21T09:37:37.263 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 21 09:37:37 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-3-activate[117756]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-21T09:37:37.263 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 21 09:37:37 smithi022 bash[117741]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-21T09:37:37.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:36 smithi064 ceph-mon[86709]: osdmap e435: 8 total, 7 up, 8 in 2024-09-21T09:37:37.442 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:37:36 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:37:36.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi022\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:37:37.558 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 21 09:37:37 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-3-activate[117756]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-21T09:37:37.558 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 21 09:37:37 smithi022 bash[117741]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-21T09:37:37.558 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 21 09:37:37 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-3-activate[117756]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-09-21T09:37:37.558 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 21 09:37:37 smithi022 bash[117741]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-09-21T09:37:37.558 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 21 09:37:37 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-3-activate[117756]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-21T09:37:37.558 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 21 09:37:37 smithi022 bash[117741]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-21T09:37:37.558 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 21 09:37:37 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-3-activate[117756]: --> ceph-volume raw activate successful for osd ID: 3 2024-09-21T09:37:37.559 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 21 09:37:37 smithi022 bash[117741]: --> ceph-volume raw activate successful for osd ID: 3 2024-09-21T09:37:37.559 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 21 09:37:37 smithi022 podman[117741]: 2024-09-21 09:37:37.29041318 +0000 UTC m=+1.304660330 container died 7e74f752cc9c27f9db8bebb04ca9e50e0414ae0ed0fc004d56b26a497fabcfa9 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-3-activate, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, RELEASE=main-62da54b, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T09:37:37.732 INFO:tasks.rados.rados.0.smithi022.stdout:1854: finishing write tid 1 to smithi02284960-6 2024-09-21T09:37:37.732 INFO:tasks.rados.rados.0.smithi022.stdout:1854: finishing write tid 2 to smithi02284960-6 2024-09-21T09:37:37.732 INFO:tasks.rados.rados.0.smithi022.stdout:1854: finishing write tid 3 to smithi02284960-6 2024-09-21T09:37:37.732 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 6 v 1115 (ObjNum 599 snap 181 seq_num 599) dirty exists 2024-09-21T09:37:37.732 INFO:tasks.rados.rados.0.smithi022.stdout:1854: left oid 6 (ObjNum 599 snap 181 seq_num 599) 2024-09-21T09:37:37.732 INFO:tasks.rados.rados.0.smithi022.stdout:1851: expect (ObjNum 558 snap 167 seq_num 558) 2024-09-21T09:37:38.222 INFO:tasks.rados.rados.0.smithi022.stdout:1847: done (7 left) 2024-09-21T09:37:38.222 INFO:tasks.rados.rados.0.smithi022.stdout:1848: done (6 left) 2024-09-21T09:37:38.222 INFO:tasks.rados.rados.0.smithi022.stdout:1849: done (5 left) 2024-09-21T09:37:38.222 INFO:tasks.rados.rados.0.smithi022.stdout:1851: done (4 left) 2024-09-21T09:37:38.222 INFO:tasks.rados.rados.0.smithi022.stdout:1852: done (3 left) 2024-09-21T09:37:38.222 INFO:tasks.rados.rados.0.smithi022.stdout:1853: done (2 left) 2024-09-21T09:37:38.223 INFO:tasks.rados.rados.0.smithi022.stdout:1854: done (1 left) 2024-09-21T09:37:38.223 INFO:tasks.rados.rados.0.smithi022.stdout:1855: done (0 left) 2024-09-21T09:37:38.223 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 47 current snap is 182 2024-09-21T09:37:38.223 INFO:tasks.rados.rados.0.smithi022.stdout:1856: seq_num 600 ranges {3325952=450560} 2024-09-21T09:37:38.242 INFO:tasks.rados.rados.0.smithi022.stdout:1856: writing smithi02284960-47 from 3325952 to 3776512 tid 1 2024-09-21T09:37:38.242 INFO:tasks.rados.rados.0.smithi022.stdout:1857: setattr oid 48 current snap is 182 2024-09-21T09:37:38.242 INFO:tasks.rados.rados.0.smithi022.stdout:1858: read oid 17 snap 180 2024-09-21T09:37:38.243 INFO:tasks.rados.rados.0.smithi022.stdout:1858: expect (ObjNum 575 snap 172 seq_num 575) 2024-09-21T09:37:38.243 INFO:tasks.rados.rados.0.smithi022.stdout:1859: snap_create 2024-09-21T09:37:38.247 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 48 v 1117 (ObjNum 581 snap 172 seq_num 581) dirty exists 2024-09-21T09:37:38.248 INFO:tasks.rados.rados.0.smithi022.stdout:1856: finishing write tid 1 to smithi02284960-47 2024-09-21T09:37:38.249 INFO:tasks.rados.rados.0.smithi022.stdout:1856: finishing write tid 2 to smithi02284960-47 2024-09-21T09:37:38.249 INFO:tasks.rados.rados.0.smithi022.stdout:1856: finishing write tid 3 to smithi02284960-47 2024-09-21T09:37:38.250 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 47 v 1056 (ObjNum 600 snap 182 seq_num 600) dirty exists 2024-09-21T09:37:38.250 INFO:tasks.rados.rados.0.smithi022.stdout:1856: left oid 47 (ObjNum 600 snap 182 seq_num 600) 2024-09-21T09:37:38.262 INFO:tasks.rados.rados.0.smithi022.stdout:1858: expect (ObjNum 575 snap 172 seq_num 575) 2024-09-21T09:37:38.283 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:37 smithi022 ceph-mon[98615]: pgmap v256: 121 pgs: 10 active+undersized, 27 peering, 4 active+undersized+degraded, 80 active+clean; 123 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4.0 MiB/s rd, 1.3 MiB/s wr, 15 op/s; 7/198 objects degraded (3.535%) 2024-09-21T09:37:38.283 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:37 smithi022 ceph-mon[98615]: Health check failed: Degraded data redundancy: 7/198 objects degraded (3.535%), 4 pgs degraded (PG_DEGRADED) 2024-09-21T09:37:38.284 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:37 smithi022 ceph-mon[98615]: osdmap e436: 8 total, 7 up, 8 in 2024-09-21T09:37:38.284 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:37 smithi022 ceph-mon[96736]: pgmap v256: 121 pgs: 10 active+undersized, 27 peering, 4 active+undersized+degraded, 80 active+clean; 123 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4.0 MiB/s rd, 1.3 MiB/s wr, 15 op/s; 7/198 objects degraded (3.535%) 2024-09-21T09:37:38.284 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:37 smithi022 ceph-mon[96736]: Health check failed: Degraded data redundancy: 7/198 objects degraded (3.535%), 4 pgs degraded (PG_DEGRADED) 2024-09-21T09:37:38.284 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:37 smithi022 ceph-mon[96736]: osdmap e436: 8 total, 7 up, 8 in 2024-09-21T09:37:38.284 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 21 09:37:37 smithi022 podman[117907]: 2024-09-21 09:37:37.937299846 +0000 UTC m=+0.640628602 container remove 7e74f752cc9c27f9db8bebb04ca9e50e0414ae0ed0fc004d56b26a497fabcfa9 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-3-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, ceph=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-62da54b, io.buildah.version=1.37.2, org.label-schema.license=GPLv2) 2024-09-21T09:37:38.284 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 21 09:37:38 smithi022 podman[117972]: 2024-09-21 09:37:38.146453045 +0000 UTC m=+0.112828554 container create c90b27f61bb68eef229931b9a71c344cd61b309e989fc3f74ee43afab03eebe0 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-3, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, RELEASE=main-62da54b) 2024-09-21T09:37:38.284 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 21 09:37:38 smithi022 podman[117972]: 2024-09-21 09:37:38.050196438 +0000 UTC m=+0.016571966 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:37:38.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:37 smithi064 ceph-mon[86709]: pgmap v256: 121 pgs: 10 active+undersized, 27 peering, 4 active+undersized+degraded, 80 active+clean; 123 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4.0 MiB/s rd, 1.3 MiB/s wr, 15 op/s; 7/198 objects degraded (3.535%) 2024-09-21T09:37:38.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:37 smithi064 ceph-mon[86709]: Health check failed: Degraded data redundancy: 7/198 objects degraded (3.535%), 4 pgs degraded (PG_DEGRADED) 2024-09-21T09:37:38.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:37 smithi064 ceph-mon[86709]: osdmap e436: 8 total, 7 up, 8 in 2024-09-21T09:37:38.808 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 21 09:37:38 smithi022 podman[117972]: 2024-09-21 09:37:38.391138627 +0000 UTC m=+0.357514151 container init c90b27f61bb68eef229931b9a71c344cd61b309e989fc3f74ee43afab03eebe0 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-3, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, RELEASE=main-62da54b, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, ceph=True, org.label-schema.schema-version=1.0) 2024-09-21T09:37:38.808 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 21 09:37:38 smithi022 podman[117972]: 2024-09-21 09:37:38.395996677 +0000 UTC m=+0.362372189 container start c90b27f61bb68eef229931b9a71c344cd61b309e989fc3f74ee43afab03eebe0 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-3, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, RELEASE=main-62da54b, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux ) 2024-09-21T09:37:38.808 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 21 09:37:38 smithi022 ceph-osd[118011]: -- 172.21.15.22:0/3486190411 <== mon.2 v2:172.21.15.22:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x563758d57a40 con 0x563759b26000 2024-09-21T09:37:38.808 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 21 09:37:38 smithi022 bash[117972]: c90b27f61bb68eef229931b9a71c344cd61b309e989fc3f74ee43afab03eebe0 2024-09-21T09:37:38.808 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 21 09:37:38 smithi022 systemd[1]: Started Ceph osd.3 for d65e801e-77fa-11ef-baf6-efdc52797490. 2024-09-21T09:37:38.975 INFO:tasks.rados.rados.0.smithi022.stdout:1856: done (3 left) 2024-09-21T09:37:38.976 INFO:tasks.rados.rados.0.smithi022.stdout:1857: done (2 left) 2024-09-21T09:37:38.976 INFO:tasks.rados.rados.0.smithi022.stdout:1858: done (1 left) 2024-09-21T09:37:38.976 INFO:tasks.rados.rados.0.smithi022.stdout:1859: done (0 left) 2024-09-21T09:37:38.976 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 5 current snap is 183 2024-09-21T09:37:38.976 INFO:tasks.rados.rados.0.smithi022.stdout:1860: seq_num 601 ranges {8380416=475136} 2024-09-21T09:37:39.025 INFO:tasks.rados.rados.0.smithi022.stdout:1860: writing smithi02284960-5 from 8380416 to 8855552 tid 1 2024-09-21T09:37:39.025 INFO:tasks.rados.rados.0.smithi022.stdout:1861: rollback oid 45 current snap is 183 2024-09-21T09:37:39.025 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 45 to 179 2024-09-21T09:37:39.025 INFO:tasks.rados.rados.0.smithi022.stdout:1862: setattr oid 38 current snap is 183 2024-09-21T09:37:39.027 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 7 current snap is 183 2024-09-21T09:37:39.027 INFO:tasks.rados.rados.0.smithi022.stdout:1863: seq_num 602 ranges {1499136=491520} 2024-09-21T09:37:39.040 INFO:tasks.rados.rados.0.smithi022.stdout:1863: writing smithi02284960-7 from 1499136 to 1990656 tid 1 2024-09-21T09:37:39.040 INFO:tasks.rados.rados.0.smithi022.stdout:1864: read oid 11 snap -1 2024-09-21T09:37:39.040 INFO:tasks.rados.rados.0.smithi022.stdout:1864: expect (ObjNum 569 snap 170 seq_num 569) 2024-09-21T09:37:39.040 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 25 current snap is 183 2024-09-21T09:37:39.040 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 38 v 866 (ObjNum 547 snap 166 seq_num 547) dirty exists 2024-09-21T09:37:39.040 INFO:tasks.rados.rados.0.smithi022.stdout:1860: finishing write tid 1 to smithi02284960-5 2024-09-21T09:37:39.040 INFO:tasks.rados.rados.0.smithi022.stdout:1860: finishing write tid 2 to smithi02284960-5 2024-09-21T09:37:39.040 INFO:tasks.rados.rados.0.smithi022.stdout:1865: seq_num 603 ranges {0=507904} 2024-09-21T09:37:39.044 INFO:tasks.rados.rados.0.smithi022.stdout:1865: writing smithi02284960-25 from 0 to 507904 tid 1 2024-09-21T09:37:39.044 INFO:tasks.rados.rados.0.smithi022.stdout:1862: done (5 left) 2024-09-21T09:37:39.044 INFO:tasks.rados.rados.0.smithi022.stdout:1866: rmattr oid 43 current snap is 183 2024-09-21T09:37:39.044 INFO:tasks.rados.rados.0.smithi022.stdout:1866: done (5 left) 2024-09-21T09:37:39.044 INFO:tasks.rados.rados.0.smithi022.stdout:1867: read oid 37 snap -1 2024-09-21T09:37:39.045 INFO:tasks.rados.rados.0.smithi022.stdout:1867: expect deleted 2024-09-21T09:37:39.045 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 47 current snap is 183 2024-09-21T09:37:39.045 INFO:tasks.rados.rados.0.smithi022.stdout:1868: seq_num 604 ranges {3776512=524288} 2024-09-21T09:37:39.070 INFO:tasks.rados.rados.0.smithi022.stdout:1868: writing smithi02284960-47 from 3776512 to 4300800 tid 1 2024-09-21T09:37:39.117 INFO:tasks.rados.rados.0.smithi022.stdout:1869: delete oid 23 current snap is 183 2024-09-21T09:37:39.117 INFO:tasks.rados.rados.0.smithi022.stdout:1861: finishing rollback tid 0 to smithi02284960-45 2024-09-21T09:37:39.117 INFO:tasks.rados.rados.0.smithi022.stdout:1861: finishing rollback tid 1 to smithi02284960-45 2024-09-21T09:37:39.117 INFO:tasks.rados.rados.0.smithi022.stdout:1861: finishing rollback tid 2 to smithi02284960-45 2024-09-21T09:37:39.117 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 45 v 1093 (ObjNum 442 snap 134 seq_num 442) dirty exists 2024-09-21T09:37:39.117 INFO:tasks.rados.rados.0.smithi022.stdout:1860: finishing write tid 3 to smithi02284960-5 2024-09-21T09:37:39.117 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 5 v 663 (ObjNum 601 snap 183 seq_num 601) dirty exists 2024-09-21T09:37:39.117 INFO:tasks.rados.rados.0.smithi022.stdout:1860: left oid 5 (ObjNum 601 snap 183 seq_num 601) 2024-09-21T09:37:39.117 INFO:tasks.rados.rados.0.smithi022.stdout:1863: finishing write tid 1 to smithi02284960-7 2024-09-21T09:37:39.117 INFO:tasks.rados.rados.0.smithi022.stdout:1863: finishing write tid 2 to smithi02284960-7 2024-09-21T09:37:39.117 INFO:tasks.rados.rados.0.smithi022.stdout:1863: finishing write tid 3 to smithi02284960-7 2024-09-21T09:37:39.117 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 7 v 1120 (ObjNum 602 snap 183 seq_num 602) dirty exists 2024-09-21T09:37:39.118 INFO:tasks.rados.rados.0.smithi022.stdout:1863: left oid 7 (ObjNum 602 snap 183 seq_num 602) 2024-09-21T09:37:39.118 INFO:tasks.rados.rados.0.smithi022.stdout:1865: finishing write tid 1 to smithi02284960-25 2024-09-21T09:37:39.118 INFO:tasks.rados.rados.0.smithi022.stdout:1865: finishing write tid 2 to smithi02284960-25 2024-09-21T09:37:39.118 INFO:tasks.rados.rados.0.smithi022.stdout:1865: finishing write tid 3 to smithi02284960-25 2024-09-21T09:37:39.118 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 25 v 1001 (ObjNum 603 snap 183 seq_num 603) dirty exists 2024-09-21T09:37:39.118 INFO:tasks.rados.rados.0.smithi022.stdout:1865: left oid 25 (ObjNum 603 snap 183 seq_num 603) 2024-09-21T09:37:39.118 INFO:tasks.rados.rados.0.smithi022.stdout:1864: expect (ObjNum 569 snap 170 seq_num 569) 2024-09-21T09:37:39.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:39 smithi022 ceph-mon[98615]: osdmap e437: 8 total, 7 up, 8 in 2024-09-21T09:37:39.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:39 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:37:39.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:39 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:37:39.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:39 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:37:39.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:39 smithi022 ceph-mon[98615]: osdmap e438: 8 total, 7 up, 8 in 2024-09-21T09:37:39.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:39 smithi022 ceph-mon[96736]: osdmap e437: 8 total, 7 up, 8 in 2024-09-21T09:37:39.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:39 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:37:39.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:39 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:37:39.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:39 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:37:39.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:39 smithi022 ceph-mon[96736]: osdmap e438: 8 total, 7 up, 8 in 2024-09-21T09:37:39.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:39 smithi064 ceph-mon[86709]: osdmap e437: 8 total, 7 up, 8 in 2024-09-21T09:37:39.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:39 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:37:39.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:39 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:37:39.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:39 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:37:39.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:39 smithi064 ceph-mon[86709]: osdmap e438: 8 total, 7 up, 8 in 2024-09-21T09:37:39.555 INFO:tasks.rados.rados.0.smithi022.stdout:1868: finishing write tid 1 to smithi02284960-47 2024-09-21T09:37:39.556 INFO:tasks.rados.rados.0.smithi022.stdout:1868: finishing write tid 2 to smithi02284960-47 2024-09-21T09:37:39.556 INFO:tasks.rados.rados.0.smithi022.stdout:1860: done (7 left) 2024-09-21T09:37:39.556 INFO:tasks.rados.rados.0.smithi022.stdout:1861: done (6 left) 2024-09-21T09:37:39.556 INFO:tasks.rados.rados.0.smithi022.stdout:1863: done (5 left) 2024-09-21T09:37:39.556 INFO:tasks.rados.rados.0.smithi022.stdout:1864: done (4 left) 2024-09-21T09:37:39.556 INFO:tasks.rados.rados.0.smithi022.stdout:1865: done (3 left) 2024-09-21T09:37:39.556 INFO:tasks.rados.rados.0.smithi022.stdout:1867: done (2 left) 2024-09-21T09:37:39.556 INFO:tasks.rados.rados.0.smithi022.stdout:1869: done (1 left) 2024-09-21T09:37:39.556 INFO:tasks.rados.rados.0.smithi022.stdout:1870: snap_remove snap 179 2024-09-21T09:37:39.807 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 21 09:37:39 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-3[117988]: 2024-09-21T09:37:39.538+0000 7fbfdd678640 -1 Falling back to public interface 2024-09-21T09:37:40.019 INFO:tasks.rados.rados.0.smithi022.stdout:1870: done (1 left) 2024-09-21T09:37:40.019 INFO:tasks.rados.rados.0.smithi022.stdout:1871: delete oid 50 current snap is 183 2024-09-21T09:37:40.019 INFO:tasks.rados.rados.0.smithi022.stdout:1868: finishing write tid 3 to smithi02284960-47 2024-09-21T09:37:40.019 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 47 v 1059 (ObjNum 604 snap 183 seq_num 604) dirty exists 2024-09-21T09:37:40.019 INFO:tasks.rados.rados.0.smithi022.stdout:1868: left oid 47 (ObjNum 604 snap 183 seq_num 604) 2024-09-21T09:37:40.026 INFO:tasks.rados.rados.0.smithi022.stdout:1868: done (1 left) 2024-09-21T09:37:40.026 INFO:tasks.rados.rados.0.smithi022.stdout:1871: done (0 left) 2024-09-21T09:37:40.026 INFO:tasks.rados.rados.0.smithi022.stdout:1872: delete oid 23 current snap is 183 2024-09-21T09:37:40.028 INFO:tasks.rados.rados.0.smithi022.stdout:1872: done (0 left) 2024-09-21T09:37:40.028 INFO:tasks.rados.rados.0.smithi022.stdout:1873: copy_from oid 6 from oid 30 current snap is 183 2024-09-21T09:37:40.028 INFO:tasks.rados.rados.0.smithi022.stdout:1874: snap_remove snap 182 2024-09-21T09:37:40.274 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:40 smithi022 ceph-mon[96736]: pgmap v259: 121 pgs: 34 active+undersized, 7 active+undersized+degraded, 80 active+clean; 124 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 1.6 MiB/s wr, 16 op/s; 15/210 objects degraded (7.143%) 2024-09-21T09:37:40.275 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:40 smithi022 ceph-mon[96736]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 7 pgs inactive, 7 pgs peering) 2024-09-21T09:37:40.275 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:40 smithi022 ceph-mon[98615]: pgmap v259: 121 pgs: 34 active+undersized, 7 active+undersized+degraded, 80 active+clean; 124 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 1.6 MiB/s wr, 16 op/s; 15/210 objects degraded (7.143%) 2024-09-21T09:37:40.275 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:40 smithi022 ceph-mon[98615]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 7 pgs inactive, 7 pgs peering) 2024-09-21T09:37:40.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:40 smithi064 ceph-mon[86709]: pgmap v259: 121 pgs: 34 active+undersized, 7 active+undersized+degraded, 80 active+clean; 124 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 1.6 MiB/s wr, 16 op/s; 15/210 objects degraded (7.143%) 2024-09-21T09:37:40.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:40 smithi064 ceph-mon[86709]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 7 pgs inactive, 7 pgs peering) 2024-09-21T09:37:40.442 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:37:40 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:37:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:37:41.025 INFO:tasks.rados.rados.0.smithi022.stdout:1874: done (1 left) 2024-09-21T09:37:41.025 INFO:tasks.rados.rados.0.smithi022.stdout:1875: snap_create 2024-09-21T09:37:41.025 INFO:tasks.rados.rados.0.smithi022.stdout:1873: finishing copy_from to smithi02284960-6 2024-09-21T09:37:41.025 INFO:tasks.rados.rados.0.smithi022.stdout:1873: got expected ENOENT (src dne) 2024-09-21T09:37:41.025 INFO:tasks.rados.rados.0.smithi022.stdout:1873: finishing copy_from racing read to smithi02284960-6 2024-09-21T09:37:41.307 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:41 smithi022 ceph-mon[98615]: osdmap e439: 8 total, 7 up, 8 in 2024-09-21T09:37:41.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:41 smithi022 ceph-mon[96736]: osdmap e439: 8 total, 7 up, 8 in 2024-09-21T09:37:41.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:41 smithi064 ceph-mon[86709]: osdmap e439: 8 total, 7 up, 8 in 2024-09-21T09:37:42.027 INFO:tasks.rados.rados.0.smithi022.stdout:1873: done (1 left) 2024-09-21T09:37:42.027 INFO:tasks.rados.rados.0.smithi022.stdout:1875: done (0 left) 2024-09-21T09:37:42.027 INFO:tasks.rados.rados.0.smithi022.stdout:1876: delete oid 46 current snap is 184 2024-09-21T09:37:42.033 INFO:tasks.rados.rados.0.smithi022.stdout:1876: done (0 left) 2024-09-21T09:37:42.033 INFO:tasks.rados.rados.0.smithi022.stdout:1877: rollback oid 6 current snap is 184 2024-09-21T09:37:42.033 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 6 to 183 2024-09-21T09:37:42.033 INFO:tasks.rados.rados.0.smithi022.stdout:1878: read oid 17 snap -1 2024-09-21T09:37:42.033 INFO:tasks.rados.rados.0.smithi022.stdout:1878: expect (ObjNum 575 snap 172 seq_num 575) 2024-09-21T09:37:42.033 INFO:tasks.rados.rados.0.smithi022.stdout:1879: snap_create 2024-09-21T09:37:42.037 INFO:tasks.rados.rados.0.smithi022.stdout:1877: finishing rollback tid 0 to smithi02284960-6 2024-09-21T09:37:42.039 INFO:tasks.rados.rados.0.smithi022.stdout:1877: finishing rollback tid 1 to smithi02284960-6 2024-09-21T09:37:42.039 INFO:tasks.rados.rados.0.smithi022.stdout:1877: finishing rollback tid 2 to smithi02284960-6 2024-09-21T09:37:42.039 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 6 v 1132 (ObjNum 599 snap 181 seq_num 599) dirty exists 2024-09-21T09:37:42.048 INFO:tasks.rados.rados.0.smithi022.stdout:1878: expect (ObjNum 575 snap 172 seq_num 575) 2024-09-21T09:37:42.307 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:42 smithi022 ceph-mon[98615]: pgmap v262: 121 pgs: 34 active+undersized, 7 active+undersized+degraded, 80 active+clean; 126 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 28 MiB/s rd, 912 KiB/s wr, 21 op/s; 15/225 objects degraded (6.667%) 2024-09-21T09:37:42.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:42 smithi022 ceph-mon[98615]: osdmap e440: 8 total, 7 up, 8 in 2024-09-21T09:37:42.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:42 smithi022 ceph-mon[96736]: pgmap v262: 121 pgs: 34 active+undersized, 7 active+undersized+degraded, 80 active+clean; 126 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 28 MiB/s rd, 912 KiB/s wr, 21 op/s; 15/225 objects degraded (6.667%) 2024-09-21T09:37:42.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:42 smithi022 ceph-mon[96736]: osdmap e440: 8 total, 7 up, 8 in 2024-09-21T09:37:42.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:42 smithi064 ceph-mon[86709]: pgmap v262: 121 pgs: 34 active+undersized, 7 active+undersized+degraded, 80 active+clean; 126 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 28 MiB/s rd, 912 KiB/s wr, 21 op/s; 15/225 objects degraded (6.667%) 2024-09-21T09:37:42.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:42 smithi064 ceph-mon[86709]: osdmap e440: 8 total, 7 up, 8 in 2024-09-21T09:37:42.859 INFO:teuthology.orchestra.run.smithi022.stdout:true 2024-09-21T09:37:43.040 INFO:tasks.rados.rados.0.smithi022.stdout:1877: done (2 left) 2024-09-21T09:37:43.040 INFO:tasks.rados.rados.0.smithi022.stdout:1878: done (1 left) 2024-09-21T09:37:43.040 INFO:tasks.rados.rados.0.smithi022.stdout:1879: done (0 left) 2024-09-21T09:37:43.040 INFO:tasks.rados.rados.0.smithi022.stdout:1880: rollback oid 5 current snap is 185 2024-09-21T09:37:43.041 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 5 to 180 2024-09-21T09:37:43.041 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 46 current snap is 185 2024-09-21T09:37:43.041 INFO:tasks.rados.rados.0.smithi022.stdout:1881: seq_num 609 ranges {0=606208} 2024-09-21T09:37:43.045 INFO:tasks.rados.rados.0.smithi022.stdout:1881: writing smithi02284960-46 from 0 to 606208 tid 1 2024-09-21T09:37:43.045 INFO:tasks.rados.rados.0.smithi022.stdout:1882: snap_create 2024-09-21T09:37:43.049 INFO:tasks.rados.rados.0.smithi022.stdout:1880: finishing rollback tid 0 to smithi02284960-5 2024-09-21T09:37:43.050 INFO:tasks.rados.rados.0.smithi022.stdout:1880: finishing rollback tid 1 to smithi02284960-5 2024-09-21T09:37:43.050 INFO:tasks.rados.rados.0.smithi022.stdout:1881: finishing write tid 1 to smithi02284960-46 2024-09-21T09:37:43.051 INFO:tasks.rados.rados.0.smithi022.stdout:1881: finishing write tid 2 to smithi02284960-46 2024-09-21T09:37:43.051 INFO:tasks.rados.rados.0.smithi022.stdout:1880: finishing rollback tid 2 to smithi02284960-5 2024-09-21T09:37:43.051 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 5 v 671 (ObjNum 558 snap 167 seq_num 558) dirty exists 2024-09-21T09:37:43.052 INFO:tasks.rados.rados.0.smithi022.stdout:1881: finishing write tid 3 to smithi02284960-46 2024-09-21T09:37:43.052 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 46 v 1067 (ObjNum 609 snap 185 seq_num 609) dirty exists 2024-09-21T09:37:43.052 INFO:tasks.rados.rados.0.smithi022.stdout:1881: left oid 46 (ObjNum 609 snap 185 seq_num 609) 2024-09-21T09:37:43.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:43 smithi022 ceph-mon[98615]: osdmap e441: 8 total, 7 up, 8 in 2024-09-21T09:37:43.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:43 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:37:43.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:43 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:37:43.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:43 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:37:43.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:43 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:37:43.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:43 smithi022 ceph-mon[98615]: from='osd.3 [v2:172.21.15.22:6826/4226372252,v1:172.21.15.22:6827/4226372252]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-21T09:37:43.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:43 smithi022 ceph-mon[98615]: from='osd.3 ' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-21T09:37:43.308 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 21 09:37:42 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-3[117988]: 2024-09-21T09:37:42.918+0000 7fbfdd678640 -1 osd.3 431 log_to_monitors true 2024-09-21T09:37:43.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:43 smithi022 ceph-mon[96736]: osdmap e441: 8 total, 7 up, 8 in 2024-09-21T09:37:43.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:43 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:37:43.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:43 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:37:43.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:43 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:37:43.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:43 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:37:43.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:43 smithi022 ceph-mon[96736]: from='osd.3 [v2:172.21.15.22:6826/4226372252,v1:172.21.15.22:6827/4226372252]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-21T09:37:43.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:43 smithi022 ceph-mon[96736]: from='osd.3 ' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-21T09:37:43.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:43 smithi064 ceph-mon[86709]: osdmap e441: 8 total, 7 up, 8 in 2024-09-21T09:37:43.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:43 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:37:43.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:43 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:37:43.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:43 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:37:43.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:43 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:37:43.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:43 smithi064 ceph-mon[86709]: from='osd.3 [v2:172.21.15.22:6826/4226372252,v1:172.21.15.22:6827/4226372252]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-21T09:37:43.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:43 smithi064 ceph-mon[86709]: from='osd.3 ' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-21T09:37:43.534 INFO:teuthology.orchestra.run.smithi022.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T09:37:43.534 INFO:teuthology.orchestra.run.smithi022.stdout:alertmanager.a smithi022 *:9093,9094 running (4m) 1s ago 9m 22.0M - 0.27.0 11f11916f8cd caced80ba544 2024-09-21T09:37:43.534 INFO:teuthology.orchestra.run.smithi022.stdout:grafana.a smithi064 *:3000 running (4m) 2m ago 8m 77.9M - 9.4.7 954c08fa6188 527bb657d70d 2024-09-21T09:37:43.534 INFO:teuthology.orchestra.run.smithi022.stdout:mds.a smithi022 running (10m) 1s ago 10m 18.1M - 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 34e4221f3a27 2024-09-21T09:37:43.534 INFO:teuthology.orchestra.run.smithi022.stdout:mds.b smithi064 running (10m) 2m ago 10m 20.9M - 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 0f846b3ec694 2024-09-21T09:37:43.534 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.x smithi064 *:8443,9283,8765 running (3m) 2m ago 13m 453M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 61abc61fea8d 2024-09-21T09:37:43.535 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.y smithi022 *:8443,9283,8765 running (3m) 1s ago 15m 517M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 6c5bee78f09b 2024-09-21T09:37:43.535 INFO:teuthology.orchestra.run.smithi022.stdout:mon.a smithi022 running (3m) 1s ago 15m 58.2M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 0513395c784d 2024-09-21T09:37:43.535 INFO:teuthology.orchestra.run.smithi022.stdout:mon.b smithi064 running (2m) 2m ago 14m 31.5M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 9dab26b6a9e7 2024-09-21T09:37:43.535 INFO:teuthology.orchestra.run.smithi022.stdout:mon.c smithi022 running (2m) 1s ago 14m 49.3M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 c74b58b67bd6 2024-09-21T09:37:43.535 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.a smithi022 *:9100 running (4m) 1s ago 8m 11.1M - 1.7.0 72c9c2088986 593c6158f524 2024-09-21T09:37:43.535 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.b smithi064 *:9100 running (3m) 2m ago 8m 9.94M - 1.7.0 72c9c2088986 a533420b0139 2024-09-21T09:37:43.535 INFO:teuthology.orchestra.run.smithi022.stdout:osd.0 smithi022 running (107s) 1s ago 13m 141M 2481M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 40ce23a2b569 2024-09-21T09:37:43.535 INFO:teuthology.orchestra.run.smithi022.stdout:osd.1 smithi022 running (73s) 1s ago 13m 115M 2481M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 7cd0c18d224c 2024-09-21T09:37:43.535 INFO:teuthology.orchestra.run.smithi022.stdout:osd.2 smithi022 running (39s) 1s ago 12m 108M 2481M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 b8c944306698 2024-09-21T09:37:43.537 INFO:teuthology.orchestra.run.smithi022.stdout:osd.3 smithi022 running (5s) 1s ago 12m 12.2M 2481M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 c90b27f61bb6 2024-09-21T09:37:43.537 INFO:teuthology.orchestra.run.smithi022.stdout:osd.4 smithi064 running (11m) 2m ago 11m 141M 2480M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 2a2937374fc4 2024-09-21T09:37:43.537 INFO:teuthology.orchestra.run.smithi022.stdout:osd.5 smithi064 running (11m) 2m ago 11m 192M 2480M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 1900c4af5d28 2024-09-21T09:37:43.537 INFO:teuthology.orchestra.run.smithi022.stdout:osd.6 smithi064 running (11m) 2m ago 11m 126M 2480M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 3cc4ec816e5b 2024-09-21T09:37:43.537 INFO:teuthology.orchestra.run.smithi022.stdout:osd.7 smithi064 running (10m) 2m ago 10m 148M 2480M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 18ef0be94104 2024-09-21T09:37:43.537 INFO:teuthology.orchestra.run.smithi022.stdout:prometheus.a smithi064 *:9095 running (3m) 2m ago 9m 44.0M - 2.51.0 1d3b7f56885b 80629ffa2179 2024-09-21T09:37:43.604 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 21 09:37:43 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-3[117988]: 2024-09-21T09:37:43.329+0000 7fbfd141a640 -1 osd.3 431 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T09:37:43.905 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-09-21T09:37:43.906 INFO:teuthology.orchestra.run.smithi022.stdout: "mon": { 2024-09-21T09:37:43.906 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 3 2024-09-21T09:37:43.906 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-21T09:37:43.906 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr": { 2024-09-21T09:37:43.906 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 2 2024-09-21T09:37:43.906 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-21T09:37:43.906 INFO:teuthology.orchestra.run.smithi022.stdout: "osd": { 2024-09-21T09:37:43.906 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 4, 2024-09-21T09:37:43.906 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 3 2024-09-21T09:37:43.906 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-21T09:37:43.906 INFO:teuthology.orchestra.run.smithi022.stdout: "mds": { 2024-09-21T09:37:43.906 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 2 2024-09-21T09:37:43.907 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-21T09:37:43.907 INFO:teuthology.orchestra.run.smithi022.stdout: "overall": { 2024-09-21T09:37:43.907 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 6, 2024-09-21T09:37:43.907 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 8 2024-09-21T09:37:43.907 INFO:teuthology.orchestra.run.smithi022.stdout: } 2024-09-21T09:37:43.907 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-09-21T09:37:44.040 INFO:tasks.rados.rados.0.smithi022.stdout:1880: done (2 left) 2024-09-21T09:37:44.040 INFO:tasks.rados.rados.0.smithi022.stdout:1881: done (1 left) 2024-09-21T09:37:44.040 INFO:tasks.rados.rados.0.smithi022.stdout:1882: done (0 left) 2024-09-21T09:37:44.040 INFO:tasks.rados.rados.0.smithi022.stdout:1883: snap_remove snap 185 2024-09-21T09:37:44.231 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-09-21T09:37:44.231 INFO:teuthology.orchestra.run.smithi022.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47", 2024-09-21T09:37:44.231 INFO:teuthology.orchestra.run.smithi022.stdout: "in_progress": true, 2024-09-21T09:37:44.231 INFO:teuthology.orchestra.run.smithi022.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T09:37:44.231 INFO:teuthology.orchestra.run.smithi022.stdout: "services_complete": [ 2024-09-21T09:37:44.231 INFO:teuthology.orchestra.run.smithi022.stdout: "mon", 2024-09-21T09:37:44.231 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr" 2024-09-21T09:37:44.231 INFO:teuthology.orchestra.run.smithi022.stdout: ], 2024-09-21T09:37:44.231 INFO:teuthology.orchestra.run.smithi022.stdout: "progress": "9/20 daemons upgraded", 2024-09-21T09:37:44.231 INFO:teuthology.orchestra.run.smithi022.stdout: "message": "Currently upgrading osd daemons", 2024-09-21T09:37:44.232 INFO:teuthology.orchestra.run.smithi022.stdout: "is_paused": false 2024-09-21T09:37:44.232 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-09-21T09:37:44.232 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:44 smithi022 ceph-mon[96736]: pgmap v265: 121 pgs: 34 active+undersized, 7 active+undersized+degraded, 80 active+clean; 127 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 670 KiB/s wr, 21 op/s; 18/234 objects degraded (7.692%) 2024-09-21T09:37:44.232 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:44 smithi022 ceph-mon[96736]: from='client.34317 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:37:44.232 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:44 smithi022 ceph-mon[96736]: Health check update: Degraded data redundancy: 18/234 objects degraded (7.692%), 7 pgs degraded (PG_DEGRADED) 2024-09-21T09:37:44.232 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:44 smithi022 ceph-mon[96736]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-21T09:37:44.232 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:44 smithi022 ceph-mon[96736]: osdmap e442: 8 total, 7 up, 8 in 2024-09-21T09:37:44.232 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:44 smithi022 ceph-mon[96736]: from='osd.3 [v2:172.21.15.22:6826/4226372252,v1:172.21.15.22:6827/4226372252]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi022", "root=default"]} : dispatch 2024-09-21T09:37:44.232 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:44 smithi022 ceph-mon[96736]: from='osd.3 ' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi022", "root=default"]} : dispatch 2024-09-21T09:37:44.232 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:44 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:37:44.233 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:44 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:37:44.233 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:44 smithi022 ceph-mon[96736]: from='osd.3 ' entity='osd.3' 2024-09-21T09:37:44.233 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:44 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/803903707' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:37:44.233 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:44 smithi022 ceph-mon[98615]: pgmap v265: 121 pgs: 34 active+undersized, 7 active+undersized+degraded, 80 active+clean; 127 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 670 KiB/s wr, 21 op/s; 18/234 objects degraded (7.692%) 2024-09-21T09:37:44.233 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:44 smithi022 ceph-mon[98615]: from='client.34317 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:37:44.233 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:44 smithi022 ceph-mon[98615]: Health check update: Degraded data redundancy: 18/234 objects degraded (7.692%), 7 pgs degraded (PG_DEGRADED) 2024-09-21T09:37:44.233 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:44 smithi022 ceph-mon[98615]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-21T09:37:44.233 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:44 smithi022 ceph-mon[98615]: osdmap e442: 8 total, 7 up, 8 in 2024-09-21T09:37:44.233 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:44 smithi022 ceph-mon[98615]: from='osd.3 [v2:172.21.15.22:6826/4226372252,v1:172.21.15.22:6827/4226372252]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi022", "root=default"]} : dispatch 2024-09-21T09:37:44.233 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:44 smithi022 ceph-mon[98615]: from='osd.3 ' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi022", "root=default"]} : dispatch 2024-09-21T09:37:44.233 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:44 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:37:44.233 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:44 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:37:44.234 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:44 smithi022 ceph-mon[98615]: from='osd.3 ' entity='osd.3' 2024-09-21T09:37:44.234 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:44 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/803903707' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:37:44.441 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:37:44 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:37:44.148Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.3\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.3\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.3\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:37:44.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:44 smithi064 ceph-mon[86709]: pgmap v265: 121 pgs: 34 active+undersized, 7 active+undersized+degraded, 80 active+clean; 127 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 670 KiB/s wr, 21 op/s; 18/234 objects degraded (7.692%) 2024-09-21T09:37:44.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:44 smithi064 ceph-mon[86709]: from='client.34317 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:37:44.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:44 smithi064 ceph-mon[86709]: Health check update: Degraded data redundancy: 18/234 objects degraded (7.692%), 7 pgs degraded (PG_DEGRADED) 2024-09-21T09:37:44.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:44 smithi064 ceph-mon[86709]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-21T09:37:44.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:44 smithi064 ceph-mon[86709]: osdmap e442: 8 total, 7 up, 8 in 2024-09-21T09:37:44.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:44 smithi064 ceph-mon[86709]: from='osd.3 [v2:172.21.15.22:6826/4226372252,v1:172.21.15.22:6827/4226372252]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi022", "root=default"]} : dispatch 2024-09-21T09:37:44.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:44 smithi064 ceph-mon[86709]: from='osd.3 ' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi022", "root=default"]} : dispatch 2024-09-21T09:37:44.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:44 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:37:44.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:44 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:37:44.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:44 smithi064 ceph-mon[86709]: from='osd.3 ' entity='osd.3' 2024-09-21T09:37:44.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:44 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/803903707' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:37:45.043 INFO:tasks.rados.rados.0.smithi022.stdout:1883: done (0 left) 2024-09-21T09:37:45.043 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 49 current snap is 186 2024-09-21T09:37:45.043 INFO:tasks.rados.rados.0.smithi022.stdout:1884: seq_num 610 ranges {8306688=622592} 2024-09-21T09:37:45.095 INFO:tasks.rados.rados.0.smithi022.stdout:1884: writing smithi02284960-49 from 8306688 to 8929280 tid 1 2024-09-21T09:37:45.096 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 25 current snap is 186 2024-09-21T09:37:45.096 INFO:tasks.rados.rados.0.smithi022.stdout:1885: seq_num 611 ranges {507904=638976} 2024-09-21T09:37:45.102 INFO:tasks.rados.rados.0.smithi022.stdout:1885: writing smithi02284960-25 from 507904 to 1146880 tid 1 2024-09-21T09:37:45.103 INFO:tasks.rados.rados.0.smithi022.stdout:1884: finishing write tid 1 to smithi02284960-49 2024-09-21T09:37:45.103 INFO:tasks.rados.rados.0.smithi022.stdout:1884: finishing write tid 2 to smithi02284960-49 2024-09-21T09:37:45.103 INFO:tasks.rados.rados.0.smithi022.stdout:1884: finishing write tid 3 to smithi02284960-49 2024-09-21T09:37:45.103 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 49 v 1078 (ObjNum 610 snap 186 seq_num 610) dirty exists 2024-09-21T09:37:45.103 INFO:tasks.rados.rados.0.smithi022.stdout:1884: left oid 49 (ObjNum 610 snap 186 seq_num 610) 2024-09-21T09:37:45.104 INFO:tasks.rados.rados.0.smithi022.stdout:1884: done (1 left) 2024-09-21T09:37:45.104 INFO:tasks.rados.rados.0.smithi022.stdout:1886: copy_from oid 14 from oid 44 current snap is 186 2024-09-21T09:37:45.104 INFO:tasks.rados.rados.0.smithi022.stdout:1887: read oid 35 snap -1 2024-09-21T09:37:45.104 INFO:tasks.rados.rados.0.smithi022.stdout:1887: expect (ObjNum 471 snap 142 seq_num 471) 2024-09-21T09:37:45.104 INFO:tasks.rados.rados.0.smithi022.stdout:1888: setattr oid 47 current snap is 186 2024-09-21T09:37:45.106 INFO:tasks.rados.rados.0.smithi022.stdout:1889: read oid 34 snap -1 2024-09-21T09:37:45.106 INFO:tasks.rados.rados.0.smithi022.stdout:1889: expect deleted 2024-09-21T09:37:45.106 INFO:tasks.rados.rados.0.smithi022.stdout:1890: delete oid 24 current snap is 186 2024-09-21T09:37:45.109 INFO:tasks.rados.rados.0.smithi022.stdout:1885: finishing write tid 1 to smithi02284960-25 2024-09-21T09:37:45.109 INFO:tasks.rados.rados.0.smithi022.stdout:1885: finishing write tid 2 to smithi02284960-25 2024-09-21T09:37:45.109 INFO:tasks.rados.rados.0.smithi022.stdout:1889: done (5 left) 2024-09-21T09:37:45.110 INFO:tasks.rados.rados.0.smithi022.stdout:1890: done (4 left) 2024-09-21T09:37:45.110 INFO:tasks.rados.rados.0.smithi022.stdout:1891: snap_remove snap 184 2024-09-21T09:37:45.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:45 smithi022 ceph-mon[96736]: from='client.34326 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:37:45.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:45 smithi022 ceph-mon[96736]: from='client.44258 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:37:45.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:45 smithi022 ceph-mon[96736]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T09:37:45.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:45 smithi022 ceph-mon[96736]: osd.3 [v2:172.21.15.22:6826/4226372252,v1:172.21.15.22:6827/4226372252] boot 2024-09-21T09:37:45.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:45 smithi022 ceph-mon[96736]: osdmap e443: 8 total, 8 up, 8 in 2024-09-21T09:37:45.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:45 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-21T09:37:45.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:45 smithi022 ceph-mon[98615]: from='client.34326 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:37:45.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:45 smithi022 ceph-mon[98615]: from='client.44258 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:37:45.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:45 smithi022 ceph-mon[98615]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T09:37:45.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:45 smithi022 ceph-mon[98615]: osd.3 [v2:172.21.15.22:6826/4226372252,v1:172.21.15.22:6827/4226372252] boot 2024-09-21T09:37:45.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:45 smithi022 ceph-mon[98615]: osdmap e443: 8 total, 8 up, 8 in 2024-09-21T09:37:45.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:45 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-21T09:37:45.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:45 smithi064 ceph-mon[86709]: from='client.34326 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:37:45.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:45 smithi064 ceph-mon[86709]: from='client.44258 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:37:45.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:45 smithi064 ceph-mon[86709]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T09:37:45.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:45 smithi064 ceph-mon[86709]: osd.3 [v2:172.21.15.22:6826/4226372252,v1:172.21.15.22:6827/4226372252] boot 2024-09-21T09:37:45.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:45 smithi064 ceph-mon[86709]: osdmap e443: 8 total, 8 up, 8 in 2024-09-21T09:37:45.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:45 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-21T09:37:46.043 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:37:45 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:37:45.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:37:46.043 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:37:45 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:37:45.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:37:46.043 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:37:45 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:37:45] "GET /metrics HTTP/1.1" 200 36235 "" "Prometheus/2.51.0" 2024-09-21T09:37:46.049 INFO:tasks.rados.rados.0.smithi022.stdout:1891: done (4 left) 2024-09-21T09:37:46.049 INFO:tasks.rados.rados.0.smithi022.stdout:1892: rollback oid 49 current snap is 186 2024-09-21T09:37:46.049 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 49 to 183 2024-09-21T09:37:46.049 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 47 v 1069 (ObjNum 604 snap 183 seq_num 604) dirty exists 2024-09-21T09:37:46.049 INFO:tasks.rados.rados.0.smithi022.stdout:1885: finishing write tid 3 to smithi02284960-25 2024-09-21T09:37:46.049 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 25 v 1004 (ObjNum 611 snap 186 seq_num 611) dirty exists 2024-09-21T09:37:46.049 INFO:tasks.rados.rados.0.smithi022.stdout:1885: left oid 25 (ObjNum 611 snap 186 seq_num 611) 2024-09-21T09:37:46.049 INFO:tasks.rados.rados.0.smithi022.stdout:1887: expect (ObjNum 471 snap 142 seq_num 471) 2024-09-21T09:37:46.129 INFO:tasks.rados.rados.0.smithi022.stdout:1886: finishing copy_from racing read to smithi02284960-14 2024-09-21T09:37:46.129 INFO:tasks.rados.rados.0.smithi022.stdout:1886: finishing copy_from to smithi02284960-14 2024-09-21T09:37:46.129 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 14 v 1034 (ObjNum 578 snap 172 seq_num 578) dirty exists 2024-09-21T09:37:46.129 INFO:tasks.rados.rados.0.smithi022.stdout:1885: done (4 left) 2024-09-21T09:37:46.129 INFO:tasks.rados.rados.0.smithi022.stdout:1886: done (3 left) 2024-09-21T09:37:46.129 INFO:tasks.rados.rados.0.smithi022.stdout:1887: done (2 left) 2024-09-21T09:37:46.129 INFO:tasks.rados.rados.0.smithi022.stdout:1888: done (1 left) 2024-09-21T09:37:46.129 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 29 current snap is 186 2024-09-21T09:37:46.129 INFO:tasks.rados.rados.0.smithi022.stdout:1892: finishing rollback tid 0 to smithi02284960-49 2024-09-21T09:37:46.129 INFO:tasks.rados.rados.0.smithi022.stdout:1892: finishing rollback tid 1 to smithi02284960-49 2024-09-21T09:37:46.130 INFO:tasks.rados.rados.0.smithi022.stdout:1893: seq_num 613 ranges {2359296=671744} 2024-09-21T09:37:46.145 INFO:tasks.rados.rados.0.smithi022.stdout:1893: writing smithi02284960-29 from 2359296 to 3031040 tid 1 2024-09-21T09:37:46.145 INFO:tasks.rados.rados.0.smithi022.stdout:1894: delete oid 30 current snap is 186 2024-09-21T09:37:46.145 INFO:tasks.rados.rados.0.smithi022.stdout:1892: finishing rollback tid 2 to smithi02284960-49 2024-09-21T09:37:46.146 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 49 v 1081 (ObjNum 596 snap 180 seq_num 596) dirty exists 2024-09-21T09:37:46.149 INFO:tasks.rados.rados.0.smithi022.stdout:1892: done (2 left) 2024-09-21T09:37:46.149 INFO:tasks.rados.rados.0.smithi022.stdout:1894: done (1 left) 2024-09-21T09:37:46.149 INFO:tasks.rados.rados.0.smithi022.stdout:1895: delete oid 19 current snap is 186 2024-09-21T09:37:46.152 INFO:tasks.rados.rados.0.smithi022.stdout:1893: finishing write tid 1 to smithi02284960-29 2024-09-21T09:37:46.152 INFO:tasks.rados.rados.0.smithi022.stdout:1893: finishing write tid 2 to smithi02284960-29 2024-09-21T09:37:46.153 INFO:tasks.rados.rados.0.smithi022.stdout:1893: finishing write tid 3 to smithi02284960-29 2024-09-21T09:37:46.153 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 29 v 869 (ObjNum 613 snap 186 seq_num 613) dirty exists 2024-09-21T09:37:46.153 INFO:tasks.rados.rados.0.smithi022.stdout:1893: left oid 29 (ObjNum 613 snap 186 seq_num 613) 2024-09-21T09:37:46.156 INFO:tasks.rados.rados.0.smithi022.stdout:1893: done (1 left) 2024-09-21T09:37:46.156 INFO:tasks.rados.rados.0.smithi022.stdout:1895: done (0 left) 2024-09-21T09:37:46.156 INFO:tasks.rados.rados.0.smithi022.stdout:1896: read oid 29 snap -1 2024-09-21T09:37:46.157 INFO:tasks.rados.rados.0.smithi022.stdout:1896: expect (ObjNum 613 snap 186 seq_num 613) 2024-09-21T09:37:46.157 INFO:tasks.rados.rados.0.smithi022.stdout:1897: rollback oid 6 current snap is 186 2024-09-21T09:37:46.157 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 6 to 180 2024-09-21T09:37:46.157 INFO:tasks.rados.rados.0.smithi022.stdout:1898: read oid 9 snap -1 2024-09-21T09:37:46.157 INFO:tasks.rados.rados.0.smithi022.stdout:1898: expect (ObjNum 526 snap 156 seq_num 526) 2024-09-21T09:37:46.157 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 26 current snap is 186 2024-09-21T09:37:46.157 INFO:tasks.rados.rados.0.smithi022.stdout:1899: seq_num 616 ranges {344064=720896} 2024-09-21T09:37:46.164 INFO:tasks.rados.rados.0.smithi022.stdout:1899: writing smithi02284960-26 from 344064 to 1064960 tid 1 2024-09-21T09:37:46.164 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 35 current snap is 186 2024-09-21T09:37:46.164 INFO:tasks.rados.rados.0.smithi022.stdout:1897: finishing rollback tid 0 to smithi02284960-6 2024-09-21T09:37:46.164 INFO:tasks.rados.rados.0.smithi022.stdout:1897: finishing rollback tid 1 to smithi02284960-6 2024-09-21T09:37:46.164 INFO:tasks.rados.rados.0.smithi022.stdout:1897: finishing rollback tid 2 to smithi02284960-6 2024-09-21T09:37:46.164 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 6 v 1135 (ObjNum 423 snap 132 seq_num 423) dirty exists 2024-09-21T09:37:46.164 INFO:tasks.rados.rados.0.smithi022.stdout:1900: seq_num 617 ranges {1851392=737280} 2024-09-21T09:37:46.181 INFO:tasks.rados.rados.0.smithi022.stdout:1900: writing smithi02284960-35 from 1851392 to 2588672 tid 1 2024-09-21T09:37:46.181 INFO:tasks.rados.rados.0.smithi022.stdout:1897: done (4 left) 2024-09-21T09:37:46.181 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 10 current snap is 186 2024-09-21T09:37:46.181 INFO:tasks.rados.rados.0.smithi022.stdout:1901: seq_num 618 ranges {1638400=753664} 2024-09-21T09:37:46.194 INFO:tasks.rados.rados.0.smithi022.stdout:1901: writing smithi02284960-10 from 1638400 to 2392064 tid 1 2024-09-21T09:37:46.195 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 22 current snap is 186 2024-09-21T09:37:46.195 INFO:tasks.rados.rados.0.smithi022.stdout:1902: seq_num 619 ranges {0=770048} 2024-09-21T09:37:46.199 INFO:tasks.rados.rados.0.smithi022.stdout:1902: writing smithi02284960-22 from 0 to 770048 tid 1 2024-09-21T09:37:46.199 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 49 current snap is 186 2024-09-21T09:37:46.199 INFO:tasks.rados.rados.0.smithi022.stdout:1903: seq_num 620 ranges {8306688=794624} 2024-09-21T09:37:46.249 INFO:tasks.rados.rados.0.smithi022.stdout:1903: writing smithi02284960-49 from 8306688 to 9101312 tid 1 2024-09-21T09:37:46.249 INFO:tasks.rados.rados.0.smithi022.stdout:1904: delete oid 44 current snap is 186 2024-09-21T09:37:46.249 INFO:tasks.rados.rados.0.smithi022.stdout:1899: finishing write tid 1 to smithi02284960-26 2024-09-21T09:37:46.249 INFO:tasks.rados.rados.0.smithi022.stdout:1899: finishing write tid 2 to smithi02284960-26 2024-09-21T09:37:46.249 INFO:tasks.rados.rados.0.smithi022.stdout:1896: expect (ObjNum 613 snap 186 seq_num 613) 2024-09-21T09:37:46.307 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:46 smithi022 ceph-mon[98615]: from='client.34344 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:37:46.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:46 smithi022 ceph-mon[98615]: pgmap v268: 121 pgs: 34 active+undersized, 7 active+undersized+degraded, 80 active+clean; 127 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 5.2 MiB/s rd, 214 KiB/s wr, 9 op/s; 18/237 objects degraded (7.595%) 2024-09-21T09:37:46.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:46 smithi022 ceph-mon[98615]: osdmap e444: 8 total, 8 up, 8 in 2024-09-21T09:37:46.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:46 smithi022 ceph-mon[96736]: from='client.34344 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:37:46.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:46 smithi022 ceph-mon[96736]: pgmap v268: 121 pgs: 34 active+undersized, 7 active+undersized+degraded, 80 active+clean; 127 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 5.2 MiB/s rd, 214 KiB/s wr, 9 op/s; 18/237 objects degraded (7.595%) 2024-09-21T09:37:46.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:46 smithi022 ceph-mon[96736]: osdmap e444: 8 total, 8 up, 8 in 2024-09-21T09:37:46.367 INFO:tasks.rados.rados.0.smithi022.stdout:1898: expect (ObjNum 526 snap 156 seq_num 526) 2024-09-21T09:37:46.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:46 smithi064 ceph-mon[86709]: from='client.34344 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:37:46.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:46 smithi064 ceph-mon[86709]: pgmap v268: 121 pgs: 34 active+undersized, 7 active+undersized+degraded, 80 active+clean; 127 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 5.2 MiB/s rd, 214 KiB/s wr, 9 op/s; 18/237 objects degraded (7.595%) 2024-09-21T09:37:46.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:46 smithi064 ceph-mon[86709]: osdmap e444: 8 total, 8 up, 8 in 2024-09-21T09:37:46.467 INFO:tasks.rados.rados.0.smithi022.stdout:1899: finishing write tid 3 to smithi02284960-26 2024-09-21T09:37:46.467 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 26 v 1090 (ObjNum 616 snap 186 seq_num 616) dirty exists 2024-09-21T09:37:46.467 INFO:tasks.rados.rados.0.smithi022.stdout:1899: left oid 26 (ObjNum 616 snap 186 seq_num 616) 2024-09-21T09:37:46.468 INFO:tasks.rados.rados.0.smithi022.stdout:1896: done (7 left) 2024-09-21T09:37:46.468 INFO:tasks.rados.rados.0.smithi022.stdout:1898: done (6 left) 2024-09-21T09:37:46.468 INFO:tasks.rados.rados.0.smithi022.stdout:1899: done (5 left) 2024-09-21T09:37:46.468 INFO:tasks.rados.rados.0.smithi022.stdout:1904: done (4 left) 2024-09-21T09:37:46.468 INFO:tasks.rados.rados.0.smithi022.stdout:1905: rollback oid 32 current snap is 186 2024-09-21T09:37:46.468 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 32 to 183 2024-09-21T09:37:46.468 INFO:tasks.rados.rados.0.smithi022.stdout:1900: finishing write tid 1 to smithi02284960-35 2024-09-21T09:37:46.468 INFO:tasks.rados.rados.0.smithi022.stdout:1900: finishing write tid 2 to smithi02284960-35 2024-09-21T09:37:46.468 INFO:tasks.rados.rados.0.smithi022.stdout:1900: finishing write tid 3 to smithi02284960-35 2024-09-21T09:37:46.468 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 35 v 558 (ObjNum 617 snap 186 seq_num 617) dirty exists 2024-09-21T09:37:46.468 INFO:tasks.rados.rados.0.smithi022.stdout:1900: left oid 35 (ObjNum 617 snap 186 seq_num 617) 2024-09-21T09:37:46.468 INFO:tasks.rados.rados.0.smithi022.stdout:1901: finishing write tid 1 to smithi02284960-10 2024-09-21T09:37:46.468 INFO:tasks.rados.rados.0.smithi022.stdout:1901: finishing write tid 2 to smithi02284960-10 2024-09-21T09:37:46.469 INFO:tasks.rados.rados.0.smithi022.stdout:1901: finishing write tid 3 to smithi02284960-10 2024-09-21T09:37:46.469 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 10 v 1009 (ObjNum 618 snap 186 seq_num 618) dirty exists 2024-09-21T09:37:46.469 INFO:tasks.rados.rados.0.smithi022.stdout:1901: left oid 10 (ObjNum 618 snap 186 seq_num 618) 2024-09-21T09:37:46.469 INFO:tasks.rados.rados.0.smithi022.stdout:1902: finishing write tid 1 to smithi02284960-22 2024-09-21T09:37:46.469 INFO:tasks.rados.rados.0.smithi022.stdout:1900: done (4 left) 2024-09-21T09:37:46.469 INFO:tasks.rados.rados.0.smithi022.stdout:1901: done (3 left) 2024-09-21T09:37:46.469 INFO:tasks.rados.rados.0.smithi022.stdout:1906: snap_remove snap 181 2024-09-21T09:37:46.808 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:37:46 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: 2024-09-21T09:37:46.619+0000 7feec1b50640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (9 PGs are or would become offline) 2024-09-21T09:37:47.057 INFO:tasks.rados.rados.0.smithi022.stdout:1906: done (3 left) 2024-09-21T09:37:47.057 INFO:tasks.rados.rados.0.smithi022.stdout:1907: copy_from oid 24 from oid 11 current snap is 186 2024-09-21T09:37:47.057 INFO:tasks.rados.rados.0.smithi022.stdout:1902: finishing write tid 2 to smithi02284960-22 2024-09-21T09:37:47.057 INFO:tasks.rados.rados.0.smithi022.stdout:1902: finishing write tid 3 to smithi02284960-22 2024-09-21T09:37:47.057 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 22 v 1092 (ObjNum 619 snap 186 seq_num 619) dirty exists 2024-09-21T09:37:47.057 INFO:tasks.rados.rados.0.smithi022.stdout:1902: left oid 22 (ObjNum 619 snap 186 seq_num 619) 2024-09-21T09:37:47.057 INFO:tasks.rados.rados.0.smithi022.stdout:1903: finishing write tid 1 to smithi02284960-49 2024-09-21T09:37:47.057 INFO:tasks.rados.rados.0.smithi022.stdout:1903: finishing write tid 2 to smithi02284960-49 2024-09-21T09:37:47.057 INFO:tasks.rados.rados.0.smithi022.stdout:1903: finishing write tid 3 to smithi02284960-49 2024-09-21T09:37:47.058 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 49 v 1083 (ObjNum 620 snap 186 seq_num 620) dirty exists 2024-09-21T09:37:47.058 INFO:tasks.rados.rados.0.smithi022.stdout:1903: left oid 49 (ObjNum 620 snap 186 seq_num 620) 2024-09-21T09:37:47.058 INFO:tasks.rados.rados.0.smithi022.stdout:1905: finishing rollback tid 0 to smithi02284960-32 2024-09-21T09:37:47.058 INFO:tasks.rados.rados.0.smithi022.stdout:1905: finishing rollback tid 1 to smithi02284960-32 2024-09-21T09:37:47.058 INFO:tasks.rados.rados.0.smithi022.stdout:1905: finishing rollback tid 2 to smithi02284960-32 2024-09-21T09:37:47.058 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 32 v 1139 (ObjNum 538 snap 162 seq_num 538) dirty exists 2024-09-21T09:37:47.058 INFO:tasks.rados.rados.0.smithi022.stdout:1902: done (3 left) 2024-09-21T09:37:47.058 INFO:tasks.rados.rados.0.smithi022.stdout:1903: done (2 left) 2024-09-21T09:37:47.058 INFO:tasks.rados.rados.0.smithi022.stdout:1905: done (1 left) 2024-09-21T09:37:47.058 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 38 current snap is 186 2024-09-21T09:37:47.058 INFO:tasks.rados.rados.0.smithi022.stdout:1908: seq_num 622 ranges {2908160=425984,3334144=401408} 2024-09-21T09:37:47.076 INFO:tasks.rados.rados.0.smithi022.stdout:1908: writing smithi02284960-38 from 2908160 to 3334144 tid 1 2024-09-21T09:37:47.079 INFO:tasks.rados.rados.0.smithi022.stdout:1908: writing smithi02284960-38 from 3334144 to 3735552 tid 2 2024-09-21T09:37:47.079 INFO:tasks.rados.rados.0.smithi022.stdout:1909: rollback oid 13 current snap is 186 2024-09-21T09:37:47.079 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 13 to 180 2024-09-21T09:37:47.079 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 48 current snap is 186 2024-09-21T09:37:47.079 INFO:tasks.rados.rados.0.smithi022.stdout:1910: seq_num 623 ranges {7806976=442368,8249344=401408} 2024-09-21T09:37:47.125 INFO:tasks.rados.rados.0.smithi022.stdout:1910: writing smithi02284960-48 from 7806976 to 8249344 tid 1 2024-09-21T09:37:47.127 INFO:tasks.rados.rados.0.smithi022.stdout:1910: writing smithi02284960-48 from 8249344 to 8650752 tid 2 2024-09-21T09:37:47.128 INFO:tasks.rados.rados.0.smithi022.stdout:1911: rollback oid 28 current snap is 186 2024-09-21T09:37:47.128 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 28 to 183 2024-09-21T09:37:47.128 INFO:tasks.rados.rados.0.smithi022.stdout:1909: finishing rollback tid 1 to smithi02284960-13 2024-09-21T09:37:47.128 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 13 v 0 (ObjNum 161 snap 0 seq_num 1) dirty dne 2024-09-21T09:37:47.128 INFO:tasks.rados.rados.0.smithi022.stdout:1908: finishing write tid 1 to smithi02284960-38 2024-09-21T09:37:47.128 INFO:tasks.rados.rados.0.smithi022.stdout:1908: finishing write tid 2 to smithi02284960-38 2024-09-21T09:37:47.128 INFO:tasks.rados.rados.0.smithi022.stdout:1908: finishing write tid 3 to smithi02284960-38 2024-09-21T09:37:47.128 INFO:tasks.rados.rados.0.smithi022.stdout:1908: finishing write tid 4 to smithi02284960-38 2024-09-21T09:37:47.128 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 38 v 872 (ObjNum 622 snap 186 seq_num 622) dirty exists 2024-09-21T09:37:47.128 INFO:tasks.rados.rados.0.smithi022.stdout:1908: left oid 38 (ObjNum 622 snap 186 seq_num 622) 2024-09-21T09:37:47.128 INFO:tasks.rados.rados.0.smithi022.stdout:1907: finishing copy_from racing read to smithi02284960-24 2024-09-21T09:37:47.128 INFO:tasks.rados.rados.0.smithi022.stdout:1907: finishing copy_from to smithi02284960-24 2024-09-21T09:37:47.128 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 24 v 1044 (ObjNum 569 snap 170 seq_num 569) dirty exists 2024-09-21T09:37:47.128 INFO:tasks.rados.rados.0.smithi022.stdout:1907: done (4 left) 2024-09-21T09:37:47.129 INFO:tasks.rados.rados.0.smithi022.stdout:1908: done (3 left) 2024-09-21T09:37:47.129 INFO:tasks.rados.rados.0.smithi022.stdout:1909: done (2 left) 2024-09-21T09:37:47.129 INFO:tasks.rados.rados.0.smithi022.stdout:1912: setattr oid 16 current snap is 186 2024-09-21T09:37:47.129 INFO:tasks.rados.rados.0.smithi022.stdout:1910: finishing write tid 1 to smithi02284960-48 2024-09-21T09:37:47.130 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 40 current snap is 186 2024-09-21T09:37:47.130 INFO:tasks.rados.rados.0.smithi022.stdout:1913: seq_num 624 ranges {0=458752,458752=401408} 2024-09-21T09:37:47.133 INFO:tasks.rados.rados.0.smithi022.stdout:1913: writing smithi02284960-40 from 0 to 458752 tid 1 2024-09-21T09:37:47.136 INFO:tasks.rados.rados.0.smithi022.stdout:1913: writing smithi02284960-40 from 458752 to 860160 tid 2 2024-09-21T09:37:47.136 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 12 current snap is 186 2024-09-21T09:37:47.136 INFO:tasks.rados.rados.0.smithi022.stdout:1911: finishing rollback tid 0 to smithi02284960-28 2024-09-21T09:37:47.136 INFO:tasks.rados.rados.0.smithi022.stdout:1911: finishing rollback tid 1 to smithi02284960-28 2024-09-21T09:37:47.136 INFO:tasks.rados.rados.0.smithi022.stdout:1910: finishing write tid 2 to smithi02284960-48 2024-09-21T09:37:47.136 INFO:tasks.rados.rados.0.smithi022.stdout:1911: finishing rollback tid 2 to smithi02284960-28 2024-09-21T09:37:47.136 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 28 v 561 (ObjNum 392 snap 122 seq_num 392) dirty exists 2024-09-21T09:37:47.136 INFO:tasks.rados.rados.0.smithi022.stdout:1910: finishing write tid 3 to smithi02284960-48 2024-09-21T09:37:47.137 INFO:tasks.rados.rados.0.smithi022.stdout:1910: finishing write tid 4 to smithi02284960-48 2024-09-21T09:37:47.137 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 48 v 1151 (ObjNum 623 snap 186 seq_num 623) dirty exists 2024-09-21T09:37:47.137 INFO:tasks.rados.rados.0.smithi022.stdout:1910: left oid 48 (ObjNum 623 snap 186 seq_num 623) 2024-09-21T09:37:47.137 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 16 v 956 (ObjNum 273 snap 0 seq_num 0) dirty exists 2024-09-21T09:37:47.137 INFO:tasks.rados.rados.0.smithi022.stdout:1914: seq_num 625 ranges {2293760=475136,2768896=401408} 2024-09-21T09:37:47.151 INFO:tasks.rados.rados.0.smithi022.stdout:1914: writing smithi02284960-12 from 2293760 to 2768896 tid 1 2024-09-21T09:37:47.153 INFO:tasks.rados.rados.0.smithi022.stdout:1914: writing smithi02284960-12 from 2768896 to 3170304 tid 2 2024-09-21T09:37:47.154 INFO:tasks.rados.rados.0.smithi022.stdout:1910: done (4 left) 2024-09-21T09:37:47.154 INFO:tasks.rados.rados.0.smithi022.stdout:1911: done (3 left) 2024-09-21T09:37:47.154 INFO:tasks.rados.rados.0.smithi022.stdout:1912: done (2 left) 2024-09-21T09:37:47.154 INFO:tasks.rados.rados.0.smithi022.stdout:1915: read oid 13 snap -1 2024-09-21T09:37:47.154 INFO:tasks.rados.rados.0.smithi022.stdout:1915: expect deleted 2024-09-21T09:37:47.154 INFO:tasks.rados.rados.0.smithi022.stdout:1916: read oid 50 snap -1 2024-09-21T09:37:47.154 INFO:tasks.rados.rados.0.smithi022.stdout:1916: expect deleted 2024-09-21T09:37:47.154 INFO:tasks.rados.rados.0.smithi022.stdout:1917: snap_create 2024-09-21T09:37:47.154 INFO:tasks.rados.rados.0.smithi022.stdout:1913: finishing write tid 1 to smithi02284960-40 2024-09-21T09:37:47.154 INFO:tasks.rados.rados.0.smithi022.stdout:1913: finishing write tid 2 to smithi02284960-40 2024-09-21T09:37:47.154 INFO:tasks.rados.rados.0.smithi022.stdout:1913: finishing write tid 3 to smithi02284960-40 2024-09-21T09:37:47.154 INFO:tasks.rados.rados.0.smithi022.stdout:1913: finishing write tid 4 to smithi02284960-40 2024-09-21T09:37:47.154 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 40 v 1014 (ObjNum 624 snap 186 seq_num 624) dirty exists 2024-09-21T09:37:47.155 INFO:tasks.rados.rados.0.smithi022.stdout:1913: left oid 40 (ObjNum 624 snap 186 seq_num 624) 2024-09-21T09:37:47.156 INFO:tasks.rados.rados.0.smithi022.stdout:1914: finishing write tid 1 to smithi02284960-12 2024-09-21T09:37:47.156 INFO:tasks.rados.rados.0.smithi022.stdout:1914: finishing write tid 2 to smithi02284960-12 2024-09-21T09:37:47.156 INFO:tasks.rados.rados.0.smithi022.stdout:1914: finishing write tid 3 to smithi02284960-12 2024-09-21T09:37:47.157 INFO:tasks.rados.rados.0.smithi022.stdout:1914: finishing write tid 4 to smithi02284960-12 2024-09-21T09:37:47.158 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 12 v 1220 (ObjNum 625 snap 186 seq_num 625) dirty exists 2024-09-21T09:37:47.158 INFO:tasks.rados.rados.0.smithi022.stdout:1914: left oid 12 (ObjNum 625 snap 186 seq_num 625) 2024-09-21T09:37:47.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:47 smithi022 ceph-mon[96736]: osdmap e445: 8 total, 8 up, 8 in 2024-09-21T09:37:47.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:47 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:37:47.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:47 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:37:47.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:47 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:37:47.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:47 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:37:47.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:47 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:37:47.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:47 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:37:47.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:47 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:37:47.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:47 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:37:47.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:47 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:37:47.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:47 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-09-21T09:37:47.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:47 smithi022 ceph-mon[98615]: osdmap e445: 8 total, 8 up, 8 in 2024-09-21T09:37:47.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:47 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:37:47.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:47 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:37:47.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:47 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:37:47.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:47 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:37:47.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:47 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:37:47.310 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:47 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:37:47.310 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:47 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:37:47.310 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:47 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:37:47.310 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:47 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:37:47.310 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:47 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-09-21T09:37:47.442 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:37:46 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:37:46.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi022\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:37:47.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:47 smithi064 ceph-mon[86709]: osdmap e445: 8 total, 8 up, 8 in 2024-09-21T09:37:47.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:47 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:37:47.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:47 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:37:47.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:47 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:37:47.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:47 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:37:47.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:47 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:37:47.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:47 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:37:47.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:47 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:37:47.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:47 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:37:47.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:47 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:37:47.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:47 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-09-21T09:37:48.058 INFO:tasks.rados.rados.0.smithi022.stdout:1913: done (4 left) 2024-09-21T09:37:48.058 INFO:tasks.rados.rados.0.smithi022.stdout:1914: done (3 left) 2024-09-21T09:37:48.058 INFO:tasks.rados.rados.0.smithi022.stdout:1915: done (2 left) 2024-09-21T09:37:48.058 INFO:tasks.rados.rados.0.smithi022.stdout:1916: done (1 left) 2024-09-21T09:37:48.058 INFO:tasks.rados.rados.0.smithi022.stdout:1917: done (0 left) 2024-09-21T09:37:48.058 INFO:tasks.rados.rados.0.smithi022.stdout:1918: read oid 24 snap -1 2024-09-21T09:37:48.059 INFO:tasks.rados.rados.0.smithi022.stdout:1918: expect (ObjNum 569 snap 170 seq_num 569) 2024-09-21T09:37:48.059 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 7 current snap is 187 2024-09-21T09:37:48.059 INFO:tasks.rados.rados.0.smithi022.stdout:1919: seq_num 626 ranges {1990656=491520,2482176=401408} 2024-09-21T09:37:48.072 INFO:tasks.rados.rados.0.smithi022.stdout:1919: writing smithi02284960-7 from 1990656 to 2482176 tid 1 2024-09-21T09:37:48.074 INFO:tasks.rados.rados.0.smithi022.stdout:1919: writing smithi02284960-7 from 2482176 to 2883584 tid 2 2024-09-21T09:37:48.074 INFO:tasks.rados.rados.0.smithi022.stdout:1920: snap_create 2024-09-21T09:37:48.079 INFO:tasks.rados.rados.0.smithi022.stdout:1919: finishing write tid 1 to smithi02284960-7 2024-09-21T09:37:48.079 INFO:tasks.rados.rados.0.smithi022.stdout:1919: finishing write tid 2 to smithi02284960-7 2024-09-21T09:37:48.079 INFO:tasks.rados.rados.0.smithi022.stdout:1919: finishing write tid 3 to smithi02284960-7 2024-09-21T09:37:48.081 INFO:tasks.rados.rados.0.smithi022.stdout:1919: finishing write tid 4 to smithi02284960-7 2024-09-21T09:37:48.081 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 7 v 1155 (ObjNum 626 snap 187 seq_num 626) dirty exists 2024-09-21T09:37:48.081 INFO:tasks.rados.rados.0.smithi022.stdout:1919: left oid 7 (ObjNum 626 snap 187 seq_num 626) 2024-09-21T09:37:48.105 INFO:tasks.rados.rados.0.smithi022.stdout:1918: expect (ObjNum 569 snap 170 seq_num 569) 2024-09-21T09:37:48.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:48 smithi064 ceph-mon[86709]: pgmap v271: 121 pgs: 27 active+undersized, 4 active+undersized+degraded, 90 active+clean; 128 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 3.5 MiB/s rd, 272 KiB/s wr, 9 op/s; 15/246 objects degraded (6.098%) 2024-09-21T09:37:48.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:48 smithi064 ceph-mon[86709]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-21T09:37:48.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:48 smithi064 ceph-mon[86709]: Upgrade: unsafe to stop osd(s) at this time (9 PGs are or would become offline) 2024-09-21T09:37:48.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:48 smithi064 ceph-mon[86709]: osdmap e446: 8 total, 8 up, 8 in 2024-09-21T09:37:48.557 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:48 smithi022 ceph-mon[98615]: pgmap v271: 121 pgs: 27 active+undersized, 4 active+undersized+degraded, 90 active+clean; 128 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 3.5 MiB/s rd, 272 KiB/s wr, 9 op/s; 15/246 objects degraded (6.098%) 2024-09-21T09:37:48.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:48 smithi022 ceph-mon[98615]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-21T09:37:48.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:48 smithi022 ceph-mon[98615]: Upgrade: unsafe to stop osd(s) at this time (9 PGs are or would become offline) 2024-09-21T09:37:48.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:48 smithi022 ceph-mon[98615]: osdmap e446: 8 total, 8 up, 8 in 2024-09-21T09:37:48.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:48 smithi022 ceph-mon[96736]: pgmap v271: 121 pgs: 27 active+undersized, 4 active+undersized+degraded, 90 active+clean; 128 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 3.5 MiB/s rd, 272 KiB/s wr, 9 op/s; 15/246 objects degraded (6.098%) 2024-09-21T09:37:48.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:48 smithi022 ceph-mon[96736]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-21T09:37:48.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:48 smithi022 ceph-mon[96736]: Upgrade: unsafe to stop osd(s) at this time (9 PGs are or would become offline) 2024-09-21T09:37:48.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:48 smithi022 ceph-mon[96736]: osdmap e446: 8 total, 8 up, 8 in 2024-09-21T09:37:49.068 INFO:tasks.rados.rados.0.smithi022.stdout:1918: done (2 left) 2024-09-21T09:37:49.069 INFO:tasks.rados.rados.0.smithi022.stdout:1919: done (1 left) 2024-09-21T09:37:49.069 INFO:tasks.rados.rados.0.smithi022.stdout:1920: done (0 left) 2024-09-21T09:37:49.069 INFO:tasks.rados.rados.0.smithi022.stdout:1921: snap_create 2024-09-21T09:37:49.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:49 smithi064 ceph-mon[86709]: osdmap e447: 8 total, 8 up, 8 in 2024-09-21T09:37:49.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:49 smithi064 ceph-mon[86709]: Health check update: Degraded data redundancy: 15/246 objects degraded (6.098%), 4 pgs degraded (PG_DEGRADED) 2024-09-21T09:37:49.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:49 smithi022 ceph-mon[98615]: osdmap e447: 8 total, 8 up, 8 in 2024-09-21T09:37:49.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:49 smithi022 ceph-mon[98615]: Health check update: Degraded data redundancy: 15/246 objects degraded (6.098%), 4 pgs degraded (PG_DEGRADED) 2024-09-21T09:37:49.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:49 smithi022 ceph-mon[96736]: osdmap e447: 8 total, 8 up, 8 in 2024-09-21T09:37:49.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:49 smithi022 ceph-mon[96736]: Health check update: Degraded data redundancy: 15/246 objects degraded (6.098%), 4 pgs degraded (PG_DEGRADED) 2024-09-21T09:37:50.071 INFO:tasks.rados.rados.0.smithi022.stdout:1921: done (0 left) 2024-09-21T09:37:50.071 INFO:tasks.rados.rados.0.smithi022.stdout:1922: read oid 36 snap -1 2024-09-21T09:37:50.071 INFO:tasks.rados.rados.0.smithi022.stdout:1922: expect (ObjNum 580 snap 172 seq_num 580) 2024-09-21T09:37:50.071 INFO:tasks.rados.rados.0.smithi022.stdout:1923: read oid 28 snap -1 2024-09-21T09:37:50.071 INFO:tasks.rados.rados.0.smithi022.stdout:1923: expect (ObjNum 392 snap 122 seq_num 392) 2024-09-21T09:37:50.071 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 1 current snap is 189 2024-09-21T09:37:50.071 INFO:tasks.rados.rados.0.smithi022.stdout:1924: seq_num 627 ranges {3284992=507904,3792896=401408} 2024-09-21T09:37:50.092 INFO:tasks.rados.rados.0.smithi022.stdout:1924: writing smithi02284960-1 from 3284992 to 3792896 tid 1 2024-09-21T09:37:50.094 INFO:tasks.rados.rados.0.smithi022.stdout:1924: writing smithi02284960-1 from 3792896 to 4194304 tid 2 2024-09-21T09:37:50.095 INFO:tasks.rados.rados.0.smithi022.stdout:1925: delete oid 26 current snap is 189 2024-09-21T09:37:50.095 INFO:tasks.rados.rados.0.smithi022.stdout:1922: expect (ObjNum 580 snap 172 seq_num 580) 2024-09-21T09:37:50.100 INFO:tasks.rados.rados.0.smithi022.stdout:1923: expect (ObjNum 392 snap 122 seq_num 392) 2024-09-21T09:37:50.148 INFO:tasks.rados.rados.0.smithi022.stdout:1924: finishing write tid 1 to smithi02284960-1 2024-09-21T09:37:50.148 INFO:tasks.rados.rados.0.smithi022.stdout:1924: finishing write tid 2 to smithi02284960-1 2024-09-21T09:37:50.148 INFO:tasks.rados.rados.0.smithi022.stdout:1924: finishing write tid 3 to smithi02284960-1 2024-09-21T09:37:50.148 INFO:tasks.rados.rados.0.smithi022.stdout:1924: finishing write tid 4 to smithi02284960-1 2024-09-21T09:37:50.148 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 1 v 1083 (ObjNum 627 snap 189 seq_num 627) dirty exists 2024-09-21T09:37:50.148 INFO:tasks.rados.rados.0.smithi022.stdout:1924: left oid 1 (ObjNum 627 snap 189 seq_num 627) 2024-09-21T09:37:50.148 INFO:tasks.rados.rados.0.smithi022.stdout:1922: done (3 left) 2024-09-21T09:37:50.148 INFO:tasks.rados.rados.0.smithi022.stdout:1923: done (2 left) 2024-09-21T09:37:50.148 INFO:tasks.rados.rados.0.smithi022.stdout:1924: done (1 left) 2024-09-21T09:37:50.148 INFO:tasks.rados.rados.0.smithi022.stdout:1925: done (0 left) 2024-09-21T09:37:50.148 INFO:tasks.rados.rados.0.smithi022.stdout:1926: rollback oid 10 current snap is 189 2024-09-21T09:37:50.148 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 10 to 186 2024-09-21T09:37:50.148 INFO:tasks.rados.rados.0.smithi022.stdout:1927: snap_remove snap 188 2024-09-21T09:37:50.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:50 smithi064 ceph-mon[86709]: pgmap v274: 121 pgs: 121 active+clean; 146 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 4.8 MiB/s wr, 22 op/s; 4.0 MiB/s, 1 objects/s recovering 2024-09-21T09:37:50.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:50 smithi064 ceph-mon[86709]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 15/246 objects degraded (6.098%), 4 pgs degraded) 2024-09-21T09:37:50.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:50 smithi064 ceph-mon[86709]: Cluster is now healthy 2024-09-21T09:37:50.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:50 smithi064 ceph-mon[86709]: osdmap e448: 8 total, 8 up, 8 in 2024-09-21T09:37:50.442 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:37:50 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:37:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:37:50.557 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:50 smithi022 ceph-mon[98615]: pgmap v274: 121 pgs: 121 active+clean; 146 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 4.8 MiB/s wr, 22 op/s; 4.0 MiB/s, 1 objects/s recovering 2024-09-21T09:37:50.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:50 smithi022 ceph-mon[98615]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 15/246 objects degraded (6.098%), 4 pgs degraded) 2024-09-21T09:37:50.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:50 smithi022 ceph-mon[98615]: Cluster is now healthy 2024-09-21T09:37:50.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:50 smithi022 ceph-mon[98615]: osdmap e448: 8 total, 8 up, 8 in 2024-09-21T09:37:50.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:50 smithi022 ceph-mon[96736]: pgmap v274: 121 pgs: 121 active+clean; 146 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 4.8 MiB/s wr, 22 op/s; 4.0 MiB/s, 1 objects/s recovering 2024-09-21T09:37:50.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:50 smithi022 ceph-mon[96736]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 15/246 objects degraded (6.098%), 4 pgs degraded) 2024-09-21T09:37:50.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:50 smithi022 ceph-mon[96736]: Cluster is now healthy 2024-09-21T09:37:50.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:50 smithi022 ceph-mon[96736]: osdmap e448: 8 total, 8 up, 8 in 2024-09-21T09:37:51.083 INFO:tasks.rados.rados.0.smithi022.stdout:1927: done (1 left) 2024-09-21T09:37:51.084 INFO:tasks.rados.rados.0.smithi022.stdout:1928: copy_from oid 28 from oid 21 current snap is 189 2024-09-21T09:37:51.084 INFO:tasks.rados.rados.0.smithi022.stdout:1926: finishing rollback tid 0 to smithi02284960-10 2024-09-21T09:37:51.084 INFO:tasks.rados.rados.0.smithi022.stdout:1926: finishing rollback tid 1 to smithi02284960-10 2024-09-21T09:37:51.084 INFO:tasks.rados.rados.0.smithi022.stdout:1926: finishing rollback tid 2 to smithi02284960-10 2024-09-21T09:37:51.084 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 10 v 1018 (ObjNum 618 snap 186 seq_num 618) dirty exists 2024-09-21T09:37:51.084 INFO:tasks.rados.rados.0.smithi022.stdout:1926: done (1 left) 2024-09-21T09:37:51.084 INFO:tasks.rados.rados.0.smithi022.stdout:1929: snap_remove snap 187 2024-09-21T09:37:51.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:51 smithi064 ceph-mon[86709]: osdmap e449: 8 total, 8 up, 8 in 2024-09-21T09:37:51.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:51 smithi064 ceph-mon[86709]: pgmap v277: 121 pgs: 121 active+clean; 148 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 5.3 MiB/s wr, 24 op/s; 4.0 MiB/s, 1 objects/s recovering 2024-09-21T09:37:51.557 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:51 smithi022 ceph-mon[98615]: osdmap e449: 8 total, 8 up, 8 in 2024-09-21T09:37:51.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:51 smithi022 ceph-mon[98615]: pgmap v277: 121 pgs: 121 active+clean; 148 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 5.3 MiB/s wr, 24 op/s; 4.0 MiB/s, 1 objects/s recovering 2024-09-21T09:37:51.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:51 smithi022 ceph-mon[96736]: osdmap e449: 8 total, 8 up, 8 in 2024-09-21T09:37:51.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:51 smithi022 ceph-mon[96736]: pgmap v277: 121 pgs: 121 active+clean; 148 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 5.3 MiB/s wr, 24 op/s; 4.0 MiB/s, 1 objects/s recovering 2024-09-21T09:37:52.085 INFO:tasks.rados.rados.0.smithi022.stdout:1929: done (1 left) 2024-09-21T09:37:52.086 INFO:tasks.rados.rados.0.smithi022.stdout:1930: read oid 30 snap -1 2024-09-21T09:37:52.086 INFO:tasks.rados.rados.0.smithi022.stdout:1930: expect deleted 2024-09-21T09:37:52.086 INFO:tasks.rados.rados.0.smithi022.stdout:1931: copy_from oid 42 from oid 37 current snap is 189 2024-09-21T09:37:52.086 INFO:tasks.rados.rados.0.smithi022.stdout:1928: finishing copy_from to smithi02284960-28 2024-09-21T09:37:52.086 INFO:tasks.rados.rados.0.smithi022.stdout:1928: got expected ENOENT (src dne) 2024-09-21T09:37:52.086 INFO:tasks.rados.rados.0.smithi022.stdout:1928: finishing copy_from racing read to smithi02284960-28 2024-09-21T09:37:52.086 INFO:tasks.rados.rados.0.smithi022.stdout:1928: done (2 left) 2024-09-21T09:37:52.086 INFO:tasks.rados.rados.0.smithi022.stdout:1932: copy_from oid 16 from oid 41 current snap is 189 2024-09-21T09:37:52.086 INFO:tasks.rados.rados.0.smithi022.stdout:1933: read oid 24 snap 186 2024-09-21T09:37:52.086 INFO:tasks.rados.rados.0.smithi022.stdout:1933: expect (ObjNum 569 snap 170 seq_num 569) 2024-09-21T09:37:52.087 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 12 current snap is 189 2024-09-21T09:37:52.087 INFO:tasks.rados.rados.0.smithi022.stdout:1934: seq_num 629 ranges {3170304=540672,3710976=401408} 2024-09-21T09:37:52.107 INFO:tasks.rados.rados.0.smithi022.stdout:1934: writing smithi02284960-12 from 3170304 to 3710976 tid 1 2024-09-21T09:37:52.110 INFO:tasks.rados.rados.0.smithi022.stdout:1934: writing smithi02284960-12 from 3710976 to 4112384 tid 2 2024-09-21T09:37:52.110 INFO:tasks.rados.rados.0.smithi022.stdout:1935: setattr oid 40 current snap is 189 2024-09-21T09:37:52.110 INFO:tasks.rados.rados.0.smithi022.stdout:1931: finishing copy_from to smithi02284960-42 2024-09-21T09:37:52.110 INFO:tasks.rados.rados.0.smithi022.stdout:1931: got expected ENOENT (src dne) 2024-09-21T09:37:52.110 INFO:tasks.rados.rados.0.smithi022.stdout:1931: finishing copy_from racing read to smithi02284960-42 2024-09-21T09:37:52.111 INFO:tasks.rados.rados.0.smithi022.stdout:1932: finishing copy_from racing read to smithi02284960-16 2024-09-21T09:37:52.111 INFO:tasks.rados.rados.0.smithi022.stdout:1932: finishing copy_from to smithi02284960-16 2024-09-21T09:37:52.111 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 16 v 997 (ObjNum 531 snap 159 seq_num 531) dirty exists 2024-09-21T09:37:52.112 INFO:tasks.rados.rados.0.smithi022.stdout:1930: done (5 left) 2024-09-21T09:37:52.112 INFO:tasks.rados.rados.0.smithi022.stdout:1931: done (4 left) 2024-09-21T09:37:52.112 INFO:tasks.rados.rados.0.smithi022.stdout:1932: done (3 left) 2024-09-21T09:37:52.112 INFO:tasks.rados.rados.0.smithi022.stdout:1936: snap_remove snap 183 2024-09-21T09:37:52.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:52 smithi064 ceph-mon[86709]: osdmap e450: 8 total, 8 up, 8 in 2024-09-21T09:37:52.459 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:52 smithi022 ceph-mon[98615]: osdmap e450: 8 total, 8 up, 8 in 2024-09-21T09:37:52.459 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:52 smithi022 ceph-mon[96736]: osdmap e450: 8 total, 8 up, 8 in 2024-09-21T09:37:53.096 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 40 v 1024 (ObjNum 624 snap 186 seq_num 624) dirty exists 2024-09-21T09:37:53.096 INFO:tasks.rados.rados.0.smithi022.stdout:1933: expect (ObjNum 569 snap 170 seq_num 569) 2024-09-21T09:37:53.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:53 smithi064 ceph-mon[86709]: osdmap e451: 8 total, 8 up, 8 in 2024-09-21T09:37:53.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:53 smithi064 ceph-mon[86709]: pgmap v280: 121 pgs: 121 active+clean; 148 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 646 KiB/s wr, 10 op/s 2024-09-21T09:37:53.472 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:53 smithi022 ceph-mon[98615]: osdmap e451: 8 total, 8 up, 8 in 2024-09-21T09:37:53.473 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:53 smithi022 ceph-mon[98615]: pgmap v280: 121 pgs: 121 active+clean; 148 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 646 KiB/s wr, 10 op/s 2024-09-21T09:37:53.473 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:53 smithi022 ceph-mon[96736]: osdmap e451: 8 total, 8 up, 8 in 2024-09-21T09:37:53.473 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:53 smithi022 ceph-mon[96736]: pgmap v280: 121 pgs: 121 active+clean; 148 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 646 KiB/s wr, 10 op/s 2024-09-21T09:37:53.580 INFO:tasks.rados.rados.0.smithi022.stdout:1934: finishing write tid 1 to smithi02284960-12 2024-09-21T09:37:53.580 INFO:tasks.rados.rados.0.smithi022.stdout:1934: finishing write tid 2 to smithi02284960-12 2024-09-21T09:37:53.580 INFO:tasks.rados.rados.0.smithi022.stdout:1934: finishing write tid 3 to smithi02284960-12 2024-09-21T09:37:53.580 INFO:tasks.rados.rados.0.smithi022.stdout:1934: finishing write tid 4 to smithi02284960-12 2024-09-21T09:37:53.580 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 12 v 1224 (ObjNum 629 snap 189 seq_num 629) dirty exists 2024-09-21T09:37:53.580 INFO:tasks.rados.rados.0.smithi022.stdout:1934: left oid 12 (ObjNum 629 snap 189 seq_num 629) 2024-09-21T09:37:53.580 INFO:tasks.rados.rados.0.smithi022.stdout:1933: done (3 left) 2024-09-21T09:37:53.580 INFO:tasks.rados.rados.0.smithi022.stdout:1934: done (2 left) 2024-09-21T09:37:53.580 INFO:tasks.rados.rados.0.smithi022.stdout:1935: done (1 left) 2024-09-21T09:37:53.580 INFO:tasks.rados.rados.0.smithi022.stdout:1936: done (0 left) 2024-09-21T09:37:53.581 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 20 current snap is 189 2024-09-21T09:37:53.581 INFO:tasks.rados.rados.0.smithi022.stdout:1937: seq_num 630 ranges {7159808=557056,7716864=401408} 2024-09-21T09:37:53.621 INFO:tasks.rados.rados.0.smithi022.stdout:1937: writing smithi02284960-20 from 7159808 to 7716864 tid 1 2024-09-21T09:37:53.624 INFO:tasks.rados.rados.0.smithi022.stdout:1937: writing smithi02284960-20 from 7716864 to 8118272 tid 2 2024-09-21T09:37:53.624 INFO:tasks.rados.rados.0.smithi022.stdout:1938: rollback oid 39 current snap is 189 2024-09-21T09:37:53.624 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 39 to 180 2024-09-21T09:37:53.624 INFO:tasks.rados.rados.0.smithi022.stdout:1939: rmattr oid 9 current snap is 189 2024-09-21T09:37:53.625 INFO:tasks.rados.rados.0.smithi022.stdout:1939: done (2 left) 2024-09-21T09:37:53.625 INFO:tasks.rados.rados.0.smithi022.stdout:1940: rollback oid 25 current snap is 189 2024-09-21T09:37:53.625 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 25 to 186 2024-09-21T09:37:53.625 INFO:tasks.rados.rados.0.smithi022.stdout:1941: setattr oid 34 current snap is 189 2024-09-21T09:37:53.626 INFO:tasks.rados.rados.0.smithi022.stdout:1942: read oid 42 snap -1 2024-09-21T09:37:53.626 INFO:tasks.rados.rados.0.smithi022.stdout:1942: expect (ObjNum 392 snap 122 seq_num 392) 2024-09-21T09:37:53.626 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 3 current snap is 189 2024-09-21T09:37:53.626 INFO:tasks.rados.rados.0.smithi022.stdout:1943: seq_num 631 ranges {0=573440,573440=401408} 2024-09-21T09:37:53.630 INFO:tasks.rados.rados.0.smithi022.stdout:1943: writing smithi02284960-3 from 0 to 573440 tid 1 2024-09-21T09:37:53.634 INFO:tasks.rados.rados.0.smithi022.stdout:1943: writing smithi02284960-3 from 573440 to 974848 tid 2 2024-09-21T09:37:53.634 INFO:tasks.rados.rados.0.smithi022.stdout:1944: copy_from oid 30 from oid 38 current snap is 189 2024-09-21T09:37:53.634 INFO:tasks.rados.rados.0.smithi022.stdout:1938: finishing rollback tid 0 to smithi02284960-39 2024-09-21T09:37:53.634 INFO:tasks.rados.rados.0.smithi022.stdout:1940: finishing rollback tid 0 to smithi02284960-25 2024-09-21T09:37:53.634 INFO:tasks.rados.rados.0.smithi022.stdout:1937: finishing write tid 1 to smithi02284960-20 2024-09-21T09:37:53.635 INFO:tasks.rados.rados.0.smithi022.stdout:1940: finishing rollback tid 1 to smithi02284960-25 2024-09-21T09:37:53.635 INFO:tasks.rados.rados.0.smithi022.stdout:1945: rollback oid 43 current snap is 189 2024-09-21T09:37:53.635 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 43 to 180 2024-09-21T09:37:53.635 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 34 v 972 (ObjNum 273 snap 0 seq_num 0) dirty exists 2024-09-21T09:37:53.635 INFO:tasks.rados.rados.0.smithi022.stdout:1937: finishing write tid 2 to smithi02284960-20 2024-09-21T09:37:53.635 INFO:tasks.rados.rados.0.smithi022.stdout:1940: finishing rollback tid 2 to smithi02284960-25 2024-09-21T09:37:53.635 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 25 v 1032 (ObjNum 611 snap 186 seq_num 611) dirty exists 2024-09-21T09:37:53.635 INFO:tasks.rados.rados.0.smithi022.stdout:1938: finishing rollback tid 1 to smithi02284960-39 2024-09-21T09:37:53.635 INFO:tasks.rados.rados.0.smithi022.stdout:1937: finishing write tid 3 to smithi02284960-20 2024-09-21T09:37:53.635 INFO:tasks.rados.rados.0.smithi022.stdout:1938: finishing rollback tid 2 to smithi02284960-39 2024-09-21T09:37:53.635 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 39 v 1044 (ObjNum 538 snap 162 seq_num 538) dirty exists 2024-09-21T09:37:53.635 INFO:tasks.rados.rados.0.smithi022.stdout:1937: finishing write tid 4 to smithi02284960-20 2024-09-21T09:37:53.635 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 20 v 1167 (ObjNum 630 snap 189 seq_num 630) dirty exists 2024-09-21T09:37:53.636 INFO:tasks.rados.rados.0.smithi022.stdout:1937: left oid 20 (ObjNum 630 snap 189 seq_num 630) 2024-09-21T09:37:53.636 INFO:tasks.rados.rados.0.smithi022.stdout:1937: done (7 left) 2024-09-21T09:37:53.636 INFO:tasks.rados.rados.0.smithi022.stdout:1938: done (6 left) 2024-09-21T09:37:53.636 INFO:tasks.rados.rados.0.smithi022.stdout:1940: done (5 left) 2024-09-21T09:37:53.636 INFO:tasks.rados.rados.0.smithi022.stdout:1941: done (4 left) 2024-09-21T09:37:53.636 INFO:tasks.rados.rados.0.smithi022.stdout:1946: copy_from oid 11 from oid 24 current snap is 189 2024-09-21T09:37:53.636 INFO:tasks.rados.rados.0.smithi022.stdout:1947: read oid 1 snap -1 2024-09-21T09:37:53.636 INFO:tasks.rados.rados.0.smithi022.stdout:1947: expect (ObjNum 627 snap 189 seq_num 627) 2024-09-21T09:37:53.636 INFO:tasks.rados.rados.0.smithi022.stdout:1948: read oid 22 snap -1 2024-09-21T09:37:53.636 INFO:tasks.rados.rados.0.smithi022.stdout:1948: expect (ObjNum 619 snap 186 seq_num 619) 2024-09-21T09:37:53.636 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 50 current snap is 189 2024-09-21T09:37:53.636 INFO:tasks.rados.rados.0.smithi022.stdout:1949: seq_num 632 ranges {0=589824,589824=401408} 2024-09-21T09:37:53.637 INFO:tasks.rados.rados.0.smithi022.stdout:1949: writing smithi02284960-50 from 0 to 589824 tid 1 2024-09-21T09:37:53.640 INFO:tasks.rados.rados.0.smithi022.stdout:1949: writing smithi02284960-50 from 589824 to 991232 tid 2 2024-09-21T09:37:53.640 INFO:tasks.rados.rados.0.smithi022.stdout:1950: rollback oid 16 current snap is 189 2024-09-21T09:37:53.640 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 16 to 180 2024-09-21T09:37:53.640 INFO:tasks.rados.rados.0.smithi022.stdout:1942: expect (ObjNum 392 snap 122 seq_num 392) 2024-09-21T09:37:53.695 INFO:tasks.rados.rados.0.smithi022.stdout:1943: finishing write tid 1 to smithi02284960-3 2024-09-21T09:37:53.695 INFO:tasks.rados.rados.0.smithi022.stdout:1945: finishing rollback tid 1 to smithi02284960-43 2024-09-21T09:37:53.695 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 43 v 0 (ObjNum 65 snap 0 seq_num 1483626848) dirty dne 2024-09-21T09:37:53.696 INFO:tasks.rados.rados.0.smithi022.stdout:1943: finishing write tid 2 to smithi02284960-3 2024-09-21T09:37:53.696 INFO:tasks.rados.rados.0.smithi022.stdout:1942: done (8 left) 2024-09-21T09:37:53.696 INFO:tasks.rados.rados.0.smithi022.stdout:1945: done (7 left) 2024-09-21T09:37:53.696 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 12 current snap is 189 2024-09-21T09:37:53.696 INFO:tasks.rados.rados.0.smithi022.stdout:1951: seq_num 633 ranges {4112384=606208,4718592=401408} 2024-09-21T09:37:53.721 INFO:tasks.rados.rados.0.smithi022.stdout:1951: writing smithi02284960-12 from 4112384 to 4718592 tid 1 2024-09-21T09:37:53.723 INFO:tasks.rados.rados.0.smithi022.stdout:1951: writing smithi02284960-12 from 4718592 to 5120000 tid 2 2024-09-21T09:37:53.723 INFO:tasks.rados.rados.0.smithi022.stdout:1952: snap_create 2024-09-21T09:37:53.723 INFO:tasks.rados.rados.0.smithi022.stdout:1943: finishing write tid 3 to smithi02284960-3 2024-09-21T09:37:53.723 INFO:tasks.rados.rados.0.smithi022.stdout:1948: expect (ObjNum 619 snap 186 seq_num 619) 2024-09-21T09:37:53.744 INFO:tasks.rados.rados.0.smithi022.stdout:1943: finishing write tid 4 to smithi02284960-3 2024-09-21T09:37:53.744 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 3 v 599 (ObjNum 631 snap 189 seq_num 631) dirty exists 2024-09-21T09:37:53.745 INFO:tasks.rados.rados.0.smithi022.stdout:1943: left oid 3 (ObjNum 631 snap 189 seq_num 631) 2024-09-21T09:37:53.745 INFO:tasks.rados.rados.0.smithi022.stdout:1949: finishing write tid 1 to smithi02284960-50 2024-09-21T09:37:53.745 INFO:tasks.rados.rados.0.smithi022.stdout:1949: finishing write tid 2 to smithi02284960-50 2024-09-21T09:37:53.745 INFO:tasks.rados.rados.0.smithi022.stdout:1950: finishing rollback tid 0 to smithi02284960-16 2024-09-21T09:37:53.745 INFO:tasks.rados.rados.0.smithi022.stdout:1950: finishing rollback tid 1 to smithi02284960-16 2024-09-21T09:37:53.745 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 16 v 999 (ObjNum 129 snap 0 seq_num 3569827840) dirty dne 2024-09-21T09:37:53.745 INFO:tasks.rados.rados.0.smithi022.stdout:1949: finishing write tid 3 to smithi02284960-50 2024-09-21T09:37:53.745 INFO:tasks.rados.rados.0.smithi022.stdout:1949: finishing write tid 4 to smithi02284960-50 2024-09-21T09:37:53.745 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 50 v 1097 (ObjNum 632 snap 189 seq_num 632) dirty exists 2024-09-21T09:37:53.745 INFO:tasks.rados.rados.0.smithi022.stdout:1949: left oid 50 (ObjNum 632 snap 189 seq_num 632) 2024-09-21T09:37:53.745 INFO:tasks.rados.rados.0.smithi022.stdout:1944: finishing copy_from racing read to smithi02284960-30 2024-09-21T09:37:53.745 INFO:tasks.rados.rados.0.smithi022.stdout:1944: finishing copy_from to smithi02284960-30 2024-09-21T09:37:53.745 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 30 v 872 (ObjNum 622 snap 186 seq_num 622) dirty exists 2024-09-21T09:37:53.746 INFO:tasks.rados.rados.0.smithi022.stdout:1947: expect (ObjNum 627 snap 189 seq_num 627) 2024-09-21T09:37:53.928 INFO:tasks.rados.rados.0.smithi022.stdout:1946: finishing copy_from racing read to smithi02284960-11 2024-09-21T09:37:53.929 INFO:tasks.rados.rados.0.smithi022.stdout:1946: finishing copy_from to smithi02284960-11 2024-09-21T09:37:53.929 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 11 v 1044 (ObjNum 569 snap 170 seq_num 569) dirty exists 2024-09-21T09:37:53.929 INFO:tasks.rados.rados.0.smithi022.stdout:1951: finishing write tid 1 to smithi02284960-12 2024-09-21T09:37:53.929 INFO:tasks.rados.rados.0.smithi022.stdout:1951: finishing write tid 2 to smithi02284960-12 2024-09-21T09:37:53.929 INFO:tasks.rados.rados.0.smithi022.stdout:1951: finishing write tid 3 to smithi02284960-12 2024-09-21T09:37:53.929 INFO:tasks.rados.rados.0.smithi022.stdout:1951: finishing write tid 4 to smithi02284960-12 2024-09-21T09:37:53.929 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 12 v 1233 (ObjNum 633 snap 189 seq_num 633) dirty exists 2024-09-21T09:37:53.929 INFO:tasks.rados.rados.0.smithi022.stdout:1951: left oid 12 (ObjNum 633 snap 189 seq_num 633) 2024-09-21T09:37:53.981 INFO:tasks.rados.rados.0.smithi022.stdout:1943: done (8 left) 2024-09-21T09:37:53.981 INFO:tasks.rados.rados.0.smithi022.stdout:1944: done (7 left) 2024-09-21T09:37:53.981 INFO:tasks.rados.rados.0.smithi022.stdout:1946: done (6 left) 2024-09-21T09:37:53.981 INFO:tasks.rados.rados.0.smithi022.stdout:1947: done (5 left) 2024-09-21T09:37:53.981 INFO:tasks.rados.rados.0.smithi022.stdout:1948: done (4 left) 2024-09-21T09:37:53.981 INFO:tasks.rados.rados.0.smithi022.stdout:1949: done (3 left) 2024-09-21T09:37:53.981 INFO:tasks.rados.rados.0.smithi022.stdout:1950: done (2 left) 2024-09-21T09:37:53.981 INFO:tasks.rados.rados.0.smithi022.stdout:1951: done (1 left) 2024-09-21T09:37:53.981 INFO:tasks.rados.rados.0.smithi022.stdout:1952: done (0 left) 2024-09-21T09:37:53.982 INFO:tasks.rados.rados.0.smithi022.stdout:1953: read oid 12 snap -1 2024-09-21T09:37:53.982 INFO:tasks.rados.rados.0.smithi022.stdout:1953: expect (ObjNum 633 snap 189 seq_num 633) 2024-09-21T09:37:53.982 INFO:tasks.rados.rados.0.smithi022.stdout:1954: read oid 44 snap -1 2024-09-21T09:37:53.982 INFO:tasks.rados.rados.0.smithi022.stdout:1954: expect deleted 2024-09-21T09:37:53.982 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 14 current snap is 190 2024-09-21T09:37:53.982 INFO:tasks.rados.rados.0.smithi022.stdout:1955: seq_num 634 ranges {2269184=622592,2891776=401408} 2024-09-21T09:37:53.996 INFO:tasks.rados.rados.0.smithi022.stdout:1955: writing smithi02284960-14 from 2269184 to 2891776 tid 1 2024-09-21T09:37:53.999 INFO:tasks.rados.rados.0.smithi022.stdout:1955: writing smithi02284960-14 from 2891776 to 3293184 tid 2 2024-09-21T09:37:53.999 INFO:tasks.rados.rados.0.smithi022.stdout:1956: rollback oid 2 current snap is 190 2024-09-21T09:37:53.999 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 2 to 180 2024-09-21T09:37:53.999 INFO:tasks.rados.rados.0.smithi022.stdout:1954: done (3 left) 2024-09-21T09:37:53.999 INFO:tasks.rados.rados.0.smithi022.stdout:1957: snap_create 2024-09-21T09:37:54.003 INFO:tasks.rados.rados.0.smithi022.stdout:1955: finishing write tid 1 to smithi02284960-14 2024-09-21T09:37:54.003 INFO:tasks.rados.rados.0.smithi022.stdout:1955: finishing write tid 2 to smithi02284960-14 2024-09-21T09:37:54.003 INFO:tasks.rados.rados.0.smithi022.stdout:1955: finishing write tid 3 to smithi02284960-14 2024-09-21T09:37:54.005 INFO:tasks.rados.rados.0.smithi022.stdout:1955: finishing write tid 4 to smithi02284960-14 2024-09-21T09:37:54.005 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 14 v 1086 (ObjNum 634 snap 190 seq_num 634) dirty exists 2024-09-21T09:37:54.005 INFO:tasks.rados.rados.0.smithi022.stdout:1955: left oid 14 (ObjNum 634 snap 190 seq_num 634) 2024-09-21T09:37:54.005 INFO:tasks.rados.rados.0.smithi022.stdout:1956: finishing rollback tid 0 to smithi02284960-2 2024-09-21T09:37:54.006 INFO:tasks.rados.rados.0.smithi022.stdout:1956: finishing rollback tid 1 to smithi02284960-2 2024-09-21T09:37:54.006 INFO:tasks.rados.rados.0.smithi022.stdout:1956: finishing rollback tid 2 to smithi02284960-2 2024-09-21T09:37:54.006 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 2 v 975 (ObjNum 452 snap 136 seq_num 452) dirty exists 2024-09-21T09:37:54.007 INFO:tasks.rados.rados.0.smithi022.stdout:1953: expect (ObjNum 633 snap 189 seq_num 633) 2024-09-21T09:37:54.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:54 smithi064 ceph-mon[86709]: osdmap e452: 8 total, 8 up, 8 in 2024-09-21T09:37:54.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:54 smithi064 ceph-mon[86709]: osdmap e453: 8 total, 8 up, 8 in 2024-09-21T09:37:54.442 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:37:54 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:37:54.147Z caller=alerting.go:391 level=warn component="rule manager" alert="unsupported value type" msg="Expanding alert template failed" err="error executing template __alert_CephOSDDown: template: __alert_CephOSDDown:1:358: executing \"__alert_CephOSDDown\" at : error calling query: found duplicate series for the match group {ceph_daemon=\"osd.3\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.3\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.3\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}];many-to-many matching not allowed: matching labels must be unique on one side" data="unsupported value type" 2024-09-21T09:37:54.442 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:37:54 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:37:54.150Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.3\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.3\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.3\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:37:54.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:54 smithi022 ceph-mon[98615]: osdmap e452: 8 total, 8 up, 8 in 2024-09-21T09:37:54.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:54 smithi022 ceph-mon[98615]: osdmap e453: 8 total, 8 up, 8 in 2024-09-21T09:37:54.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:54 smithi022 ceph-mon[96736]: osdmap e452: 8 total, 8 up, 8 in 2024-09-21T09:37:54.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:54 smithi022 ceph-mon[96736]: osdmap e453: 8 total, 8 up, 8 in 2024-09-21T09:37:54.981 INFO:tasks.rados.rados.0.smithi022.stdout:1953: done (3 left) 2024-09-21T09:37:54.982 INFO:tasks.rados.rados.0.smithi022.stdout:1955: done (2 left) 2024-09-21T09:37:54.982 INFO:tasks.rados.rados.0.smithi022.stdout:1956: done (1 left) 2024-09-21T09:37:54.982 INFO:tasks.rados.rados.0.smithi022.stdout:1957: done (0 left) 2024-09-21T09:37:54.982 INFO:tasks.rados.rados.0.smithi022.stdout:1958: rmattr oid 18 current snap is 191 2024-09-21T09:37:54.982 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 14 current snap is 191 2024-09-21T09:37:54.982 INFO:tasks.rados.rados.0.smithi022.stdout:1959: seq_num 635 ranges {3293184=638976,3932160=401408} 2024-09-21T09:37:55.003 INFO:tasks.rados.rados.0.smithi022.stdout:1959: writing smithi02284960-14 from 3293184 to 3932160 tid 1 2024-09-21T09:37:55.005 INFO:tasks.rados.rados.0.smithi022.stdout:1959: writing smithi02284960-14 from 3932160 to 4333568 tid 2 2024-09-21T09:37:55.006 INFO:tasks.rados.rados.0.smithi022.stdout:1960: snap_create 2024-09-21T09:37:55.006 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 18 v 1000 (ObjNum 557 snap 167 seq_num 557) dirty exists 2024-09-21T09:37:55.008 INFO:tasks.rados.rados.0.smithi022.stdout:1959: finishing write tid 1 to smithi02284960-14 2024-09-21T09:37:55.008 INFO:tasks.rados.rados.0.smithi022.stdout:1959: finishing write tid 2 to smithi02284960-14 2024-09-21T09:37:55.009 INFO:tasks.rados.rados.0.smithi022.stdout:1959: finishing write tid 3 to smithi02284960-14 2024-09-21T09:37:55.010 INFO:tasks.rados.rados.0.smithi022.stdout:1959: finishing write tid 4 to smithi02284960-14 2024-09-21T09:37:55.010 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 14 v 1089 (ObjNum 635 snap 191 seq_num 635) dirty exists 2024-09-21T09:37:55.010 INFO:tasks.rados.rados.0.smithi022.stdout:1959: left oid 14 (ObjNum 635 snap 191 seq_num 635) 2024-09-21T09:37:55.979 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:37:55 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:37:55] "GET /metrics HTTP/1.1" 200 36235 "" "Prometheus/2.51.0" 2024-09-21T09:37:55.988 INFO:tasks.rados.rados.0.smithi022.stdout:1958: done (2 left) 2024-09-21T09:37:55.988 INFO:tasks.rados.rados.0.smithi022.stdout:1959: done (1 left) 2024-09-21T09:37:55.988 INFO:tasks.rados.rados.0.smithi022.stdout:1960: done (0 left) 2024-09-21T09:37:55.988 INFO:tasks.rados.rados.0.smithi022.stdout:1961: copy_from oid 34 from oid 41 current snap is 192 2024-09-21T09:37:55.988 INFO:tasks.rados.rados.0.smithi022.stdout:1962: rollback oid 2 current snap is 192 2024-09-21T09:37:55.988 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 2 to 189 2024-09-21T09:37:55.988 INFO:tasks.rados.rados.0.smithi022.stdout:1963: rollback oid 26 current snap is 192 2024-09-21T09:37:55.989 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 26 to 190 2024-09-21T09:37:55.989 INFO:tasks.rados.rados.0.smithi022.stdout:1964: read oid 9 snap -1 2024-09-21T09:37:55.989 INFO:tasks.rados.rados.0.smithi022.stdout:1964: expect (ObjNum 526 snap 156 seq_num 526) 2024-09-21T09:37:55.989 INFO:tasks.rados.rados.0.smithi022.stdout:1965: read oid 16 snap -1 2024-09-21T09:37:55.989 INFO:tasks.rados.rados.0.smithi022.stdout:1965: expect deleted 2024-09-21T09:37:55.989 INFO:tasks.rados.rados.0.smithi022.stdout:1966: read oid 49 snap -1 2024-09-21T09:37:55.989 INFO:tasks.rados.rados.0.smithi022.stdout:1966: expect (ObjNum 620 snap 186 seq_num 620) 2024-09-21T09:37:55.989 INFO:tasks.rados.rados.0.smithi022.stdout:1967: read oid 31 snap -1 2024-09-21T09:37:55.989 INFO:tasks.rados.rados.0.smithi022.stdout:1967: expect (ObjNum 590 snap 176 seq_num 590) 2024-09-21T09:37:55.989 INFO:tasks.rados.rados.0.smithi022.stdout:1968: snap_create 2024-09-21T09:37:55.996 INFO:tasks.rados.rados.0.smithi022.stdout:1963: finishing rollback tid 1 to smithi02284960-26 2024-09-21T09:37:55.996 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 26 v 1093 (ObjNum 161 snap 0 seq_num 1) dirty dne 2024-09-21T09:37:55.997 INFO:tasks.rados.rados.0.smithi022.stdout:1967: expect (ObjNum 590 snap 176 seq_num 590) 2024-09-21T09:37:56.009 INFO:tasks.rados.rados.0.smithi022.stdout:1962: finishing rollback tid 0 to smithi02284960-2 2024-09-21T09:37:56.009 INFO:tasks.rados.rados.0.smithi022.stdout:1962: finishing rollback tid 1 to smithi02284960-2 2024-09-21T09:37:56.009 INFO:tasks.rados.rados.0.smithi022.stdout:1962: finishing rollback tid 2 to smithi02284960-2 2024-09-21T09:37:56.009 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 2 v 978 (ObjNum 452 snap 136 seq_num 452) dirty exists 2024-09-21T09:37:56.009 INFO:tasks.rados.rados.0.smithi022.stdout:1964: expect (ObjNum 526 snap 156 seq_num 526) 2024-09-21T09:37:56.110 INFO:tasks.rados.rados.0.smithi022.stdout:1961: finishing copy_from racing read to smithi02284960-34 2024-09-21T09:37:56.110 INFO:tasks.rados.rados.0.smithi022.stdout:1961: finishing copy_from to smithi02284960-34 2024-09-21T09:37:56.110 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 34 v 997 (ObjNum 531 snap 159 seq_num 531) dirty exists 2024-09-21T09:37:56.111 INFO:tasks.rados.rados.0.smithi022.stdout:1966: expect (ObjNum 620 snap 186 seq_num 620) 2024-09-21T09:37:56.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:55 smithi064 ceph-mon[86709]: pgmap v283: 121 pgs: 121 active+clean; 149 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 7.8 MiB/s rd, 230 KiB/s wr, 10 op/s 2024-09-21T09:37:56.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:55 smithi064 ceph-mon[86709]: osdmap e454: 8 total, 8 up, 8 in 2024-09-21T09:37:56.192 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:37:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:37:55.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:37:56.192 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:37:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:37:55.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:37:56.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:55 smithi022 ceph-mon[98615]: pgmap v283: 121 pgs: 121 active+clean; 149 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 7.8 MiB/s rd, 230 KiB/s wr, 10 op/s 2024-09-21T09:37:56.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:55 smithi022 ceph-mon[98615]: osdmap e454: 8 total, 8 up, 8 in 2024-09-21T09:37:56.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:55 smithi022 ceph-mon[96736]: pgmap v283: 121 pgs: 121 active+clean; 149 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 7.8 MiB/s rd, 230 KiB/s wr, 10 op/s 2024-09-21T09:37:56.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:55 smithi022 ceph-mon[96736]: osdmap e454: 8 total, 8 up, 8 in 2024-09-21T09:37:56.992 INFO:tasks.rados.rados.0.smithi022.stdout:1961: done (7 left) 2024-09-21T09:37:56.992 INFO:tasks.rados.rados.0.smithi022.stdout:1962: done (6 left) 2024-09-21T09:37:56.992 INFO:tasks.rados.rados.0.smithi022.stdout:1963: done (5 left) 2024-09-21T09:37:56.992 INFO:tasks.rados.rados.0.smithi022.stdout:1964: done (4 left) 2024-09-21T09:37:56.992 INFO:tasks.rados.rados.0.smithi022.stdout:1965: done (3 left) 2024-09-21T09:37:56.993 INFO:tasks.rados.rados.0.smithi022.stdout:1966: done (2 left) 2024-09-21T09:37:56.993 INFO:tasks.rados.rados.0.smithi022.stdout:1967: done (1 left) 2024-09-21T09:37:56.993 INFO:tasks.rados.rados.0.smithi022.stdout:1968: done (0 left) 2024-09-21T09:37:56.993 INFO:tasks.rados.rados.0.smithi022.stdout:1969: rollback oid 35 current snap is 193 2024-09-21T09:37:56.993 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 35 to 190 2024-09-21T09:37:56.993 INFO:tasks.rados.rados.0.smithi022.stdout:1970: snap_remove snap 180 2024-09-21T09:37:57.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:56 smithi022 ceph-mon[98615]: osdmap e455: 8 total, 8 up, 8 in 2024-09-21T09:37:57.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:56 smithi022 ceph-mon[96736]: osdmap e455: 8 total, 8 up, 8 in 2024-09-21T09:37:57.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:56 smithi064 ceph-mon[86709]: osdmap e455: 8 total, 8 up, 8 in 2024-09-21T09:37:57.442 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:37:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:37:56.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi022\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:37:57.996 INFO:tasks.rados.rados.0.smithi022.stdout:1970: done (1 left) 2024-09-21T09:37:57.996 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 18 current snap is 193 2024-09-21T09:37:57.996 INFO:tasks.rados.rados.0.smithi022.stdout:1971: seq_num 636 ranges {3014656=655360,3670016=401408} 2024-09-21T09:37:58.018 INFO:tasks.rados.rados.0.smithi022.stdout:1971: writing smithi02284960-18 from 3014656 to 3670016 tid 1 2024-09-21T09:37:58.021 INFO:tasks.rados.rados.0.smithi022.stdout:1971: writing smithi02284960-18 from 3670016 to 4071424 tid 2 2024-09-21T09:37:58.021 INFO:tasks.rados.rados.0.smithi022.stdout:1972: snap_remove snap 189 2024-09-21T09:37:58.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:58 smithi022 ceph-mon[96736]: pgmap v286: 121 pgs: 121 active+clean; 161 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 3.9 MiB/s wr, 17 op/s 2024-09-21T09:37:58.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:58 smithi022 ceph-mon[96736]: osdmap e456: 8 total, 8 up, 8 in 2024-09-21T09:37:58.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:58 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:37:58.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:58 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:37:58.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:58 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:37:58.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:58 smithi022 ceph-mon[98615]: pgmap v286: 121 pgs: 121 active+clean; 161 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 3.9 MiB/s wr, 17 op/s 2024-09-21T09:37:58.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:58 smithi022 ceph-mon[98615]: osdmap e456: 8 total, 8 up, 8 in 2024-09-21T09:37:58.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:58 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:37:58.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:58 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:37:58.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:58 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:37:58.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:58 smithi064 ceph-mon[86709]: pgmap v286: 121 pgs: 121 active+clean; 161 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 3.9 MiB/s wr, 17 op/s 2024-09-21T09:37:58.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:58 smithi064 ceph-mon[86709]: osdmap e456: 8 total, 8 up, 8 in 2024-09-21T09:37:58.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:58 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:37:58.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:58 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:37:58.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:58 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:37:58.985 INFO:tasks.rados.rados.0.smithi022.stdout:1972: done (2 left) 2024-09-21T09:37:58.985 INFO:tasks.rados.rados.0.smithi022.stdout:1973: rmattr oid 13 current snap is 193 2024-09-21T09:37:58.985 INFO:tasks.rados.rados.0.smithi022.stdout:1973: done (2 left) 2024-09-21T09:37:58.985 INFO:tasks.rados.rados.0.smithi022.stdout:1974: snap_create 2024-09-21T09:37:58.985 INFO:tasks.rados.rados.0.smithi022.stdout:1969: finishing rollback tid 0 to smithi02284960-35 2024-09-21T09:37:58.985 INFO:tasks.rados.rados.0.smithi022.stdout:1969: finishing rollback tid 1 to smithi02284960-35 2024-09-21T09:37:58.985 INFO:tasks.rados.rados.0.smithi022.stdout:1969: finishing rollback tid 2 to smithi02284960-35 2024-09-21T09:37:58.985 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 35 v 564 (ObjNum 617 snap 186 seq_num 617) dirty exists 2024-09-21T09:37:58.985 INFO:tasks.rados.rados.0.smithi022.stdout:1971: finishing write tid 1 to smithi02284960-18 2024-09-21T09:37:58.985 INFO:tasks.rados.rados.0.smithi022.stdout:1971: finishing write tid 2 to smithi02284960-18 2024-09-21T09:37:58.985 INFO:tasks.rados.rados.0.smithi022.stdout:1971: finishing write tid 3 to smithi02284960-18 2024-09-21T09:37:58.986 INFO:tasks.rados.rados.0.smithi022.stdout:1971: finishing write tid 4 to smithi02284960-18 2024-09-21T09:37:58.986 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 18 v 1003 (ObjNum 636 snap 193 seq_num 636) dirty exists 2024-09-21T09:37:58.986 INFO:tasks.rados.rados.0.smithi022.stdout:1971: left oid 18 (ObjNum 636 snap 193 seq_num 636) 2024-09-21T09:37:59.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:59 smithi022 ceph-mon[98615]: osdmap e457: 8 total, 8 up, 8 in 2024-09-21T09:37:59.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:37:59 smithi022 ceph-mon[98615]: osdmap e458: 8 total, 8 up, 8 in 2024-09-21T09:37:59.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:59 smithi022 ceph-mon[96736]: osdmap e457: 8 total, 8 up, 8 in 2024-09-21T09:37:59.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:37:59 smithi022 ceph-mon[96736]: osdmap e458: 8 total, 8 up, 8 in 2024-09-21T09:37:59.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:59 smithi064 ceph-mon[86709]: osdmap e457: 8 total, 8 up, 8 in 2024-09-21T09:37:59.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:37:59 smithi064 ceph-mon[86709]: osdmap e458: 8 total, 8 up, 8 in 2024-09-21T09:37:59.985 INFO:tasks.rados.rados.0.smithi022.stdout:1969: done (2 left) 2024-09-21T09:37:59.985 INFO:tasks.rados.rados.0.smithi022.stdout:1971: done (1 left) 2024-09-21T09:37:59.985 INFO:tasks.rados.rados.0.smithi022.stdout:1974: done (0 left) 2024-09-21T09:37:59.985 INFO:tasks.rados.rados.0.smithi022.stdout:1975: delete oid 42 current snap is 194 2024-09-21T09:37:59.989 INFO:tasks.rados.rados.0.smithi022.stdout:1975: done (0 left) 2024-09-21T09:37:59.989 INFO:tasks.rados.rados.0.smithi022.stdout:1976: copy_from oid 11 from oid 21 current snap is 194 2024-09-21T09:37:59.990 INFO:tasks.rados.rados.0.smithi022.stdout:1977: rollback oid 32 current snap is 194 2024-09-21T09:37:59.990 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 32 to 192 2024-09-21T09:37:59.990 INFO:tasks.rados.rados.0.smithi022.stdout:1978: read oid 39 snap -1 2024-09-21T09:37:59.990 INFO:tasks.rados.rados.0.smithi022.stdout:1978: expect (ObjNum 538 snap 162 seq_num 538) 2024-09-21T09:37:59.990 INFO:tasks.rados.rados.0.smithi022.stdout:1979: snap_remove snap 186 2024-09-21T09:38:00.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:00 smithi022 ceph-mon[98615]: pgmap v289: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 149 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 5.3 MiB/s wr, 24 op/s 2024-09-21T09:38:00.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:00 smithi022 ceph-mon[98615]: osdmap e459: 8 total, 8 up, 8 in 2024-09-21T09:38:00.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:00 smithi022 ceph-mon[96736]: pgmap v289: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 149 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 5.3 MiB/s wr, 24 op/s 2024-09-21T09:38:00.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:00 smithi022 ceph-mon[96736]: osdmap e459: 8 total, 8 up, 8 in 2024-09-21T09:38:00.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:00 smithi064 ceph-mon[86709]: pgmap v289: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 149 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 5.3 MiB/s wr, 24 op/s 2024-09-21T09:38:00.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:00 smithi064 ceph-mon[86709]: osdmap e459: 8 total, 8 up, 8 in 2024-09-21T09:38:00.442 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:38:00 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:38:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:38:00.988 INFO:tasks.rados.rados.0.smithi022.stdout:1979: done (3 left) 2024-09-21T09:38:00.988 INFO:tasks.rados.rados.0.smithi022.stdout:1980: read oid 41 snap -1 2024-09-21T09:38:00.988 INFO:tasks.rados.rados.0.smithi022.stdout:1980: expect (ObjNum 531 snap 159 seq_num 531) 2024-09-21T09:38:00.988 INFO:tasks.rados.rados.0.smithi022.stdout:1981: copy_from oid 29 from oid 38 current snap is 194 2024-09-21T09:38:00.988 INFO:tasks.rados.rados.0.smithi022.stdout:1982: rollback oid 50 current snap is 194 2024-09-21T09:38:00.988 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 50 to 191 2024-09-21T09:38:00.988 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 25 current snap is 194 2024-09-21T09:38:00.988 INFO:tasks.rados.rados.0.smithi022.stdout:1983: seq_num 638 ranges {1146880=688128,1835008=401408} 2024-09-21T09:38:00.999 INFO:tasks.rados.rados.0.smithi022.stdout:1983: writing smithi02284960-25 from 1146880 to 1835008 tid 1 2024-09-21T09:38:01.002 INFO:tasks.rados.rados.0.smithi022.stdout:1983: writing smithi02284960-25 from 1835008 to 2236416 tid 2 2024-09-21T09:38:01.005 INFO:tasks.rados.rados.0.smithi022.stdout:1977: finishing rollback tid 0 to smithi02284960-32 2024-09-21T09:38:01.005 INFO:tasks.rados.rados.0.smithi022.stdout:1976: finishing copy_from to smithi02284960-11 2024-09-21T09:38:01.005 INFO:tasks.rados.rados.0.smithi022.stdout:1976: got expected ENOENT (src dne) 2024-09-21T09:38:01.005 INFO:tasks.rados.rados.0.smithi022.stdout:1977: finishing rollback tid 1 to smithi02284960-32 2024-09-21T09:38:01.005 INFO:tasks.rados.rados.0.smithi022.stdout:1976: finishing copy_from racing read to smithi02284960-11 2024-09-21T09:38:01.005 INFO:tasks.rados.rados.0.smithi022.stdout:1977: finishing rollback tid 2 to smithi02284960-32 2024-09-21T09:38:01.005 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 32 v 1181 (ObjNum 538 snap 162 seq_num 538) dirty exists 2024-09-21T09:38:01.006 INFO:tasks.rados.rados.0.smithi022.stdout:1978: expect (ObjNum 538 snap 162 seq_num 538) 2024-09-21T09:38:01.200 INFO:tasks.rados.rados.0.smithi022.stdout:1980: expect (ObjNum 531 snap 159 seq_num 531) 2024-09-21T09:38:01.249 INFO:tasks.rados.rados.0.smithi022.stdout:1982: finishing rollback tid 0 to smithi02284960-50 2024-09-21T09:38:01.249 INFO:tasks.rados.rados.0.smithi022.stdout:1982: finishing rollback tid 1 to smithi02284960-50 2024-09-21T09:38:01.249 INFO:tasks.rados.rados.0.smithi022.stdout:1982: finishing rollback tid 2 to smithi02284960-50 2024-09-21T09:38:01.249 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 50 v 1100 (ObjNum 632 snap 189 seq_num 632) dirty exists 2024-09-21T09:38:01.249 INFO:tasks.rados.rados.0.smithi022.stdout:1976: done (6 left) 2024-09-21T09:38:01.250 INFO:tasks.rados.rados.0.smithi022.stdout:1977: done (5 left) 2024-09-21T09:38:01.250 INFO:tasks.rados.rados.0.smithi022.stdout:1978: done (4 left) 2024-09-21T09:38:01.250 INFO:tasks.rados.rados.0.smithi022.stdout:1980: done (3 left) 2024-09-21T09:38:01.250 INFO:tasks.rados.rados.0.smithi022.stdout:1982: done (2 left) 2024-09-21T09:38:01.250 INFO:tasks.rados.rados.0.smithi022.stdout:1984: read oid 10 snap -1 2024-09-21T09:38:01.250 INFO:tasks.rados.rados.0.smithi022.stdout:1984: expect (ObjNum 618 snap 186 seq_num 618) 2024-09-21T09:38:01.250 INFO:tasks.rados.rados.0.smithi022.stdout:1985: snap_create 2024-09-21T09:38:01.250 INFO:tasks.rados.rados.0.smithi022.stdout:1983: finishing write tid 1 to smithi02284960-25 2024-09-21T09:38:01.250 INFO:tasks.rados.rados.0.smithi022.stdout:1983: finishing write tid 2 to smithi02284960-25 2024-09-21T09:38:01.250 INFO:tasks.rados.rados.0.smithi022.stdout:1983: finishing write tid 3 to smithi02284960-25 2024-09-21T09:38:01.250 INFO:tasks.rados.rados.0.smithi022.stdout:1983: finishing write tid 4 to smithi02284960-25 2024-09-21T09:38:01.250 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 25 v 1045 (ObjNum 638 snap 194 seq_num 638) dirty exists 2024-09-21T09:38:01.250 INFO:tasks.rados.rados.0.smithi022.stdout:1983: left oid 25 (ObjNum 638 snap 194 seq_num 638) 2024-09-21T09:38:01.250 INFO:tasks.rados.rados.0.smithi022.stdout:1981: finishing copy_from racing read to smithi02284960-29 2024-09-21T09:38:01.251 INFO:tasks.rados.rados.0.smithi022.stdout:1981: finishing copy_from to smithi02284960-29 2024-09-21T09:38:01.251 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 29 v 872 (ObjNum 622 snap 186 seq_num 622) dirty exists 2024-09-21T09:38:01.260 INFO:tasks.rados.rados.0.smithi022.stdout:1984: expect (ObjNum 618 snap 186 seq_num 618) 2024-09-21T09:38:02.015 INFO:tasks.rados.rados.0.smithi022.stdout:1981: done (3 left) 2024-09-21T09:38:02.015 INFO:tasks.rados.rados.0.smithi022.stdout:1983: done (2 left) 2024-09-21T09:38:02.015 INFO:tasks.rados.rados.0.smithi022.stdout:1984: done (1 left) 2024-09-21T09:38:02.015 INFO:tasks.rados.rados.0.smithi022.stdout:1985: done (0 left) 2024-09-21T09:38:02.015 INFO:tasks.rados.rados.0.smithi022.stdout:1986: setattr oid 37 current snap is 195 2024-09-21T09:38:02.016 INFO:tasks.rados.rados.0.smithi022.stdout:1987: copy_from oid 34 from oid 33 current snap is 195 2024-09-21T09:38:02.016 INFO:tasks.rados.rados.0.smithi022.stdout:1988: copy_from oid 16 from oid 43 current snap is 195 2024-09-21T09:38:02.016 INFO:tasks.rados.rados.0.smithi022.stdout:1989: rollback oid 13 current snap is 195 2024-09-21T09:38:02.016 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 13 to 194 2024-09-21T09:38:02.016 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 24 current snap is 195 2024-09-21T09:38:02.016 INFO:tasks.rados.rados.0.smithi022.stdout:1990: seq_num 639 ranges {8929280=704512,9633792=401408} 2024-09-21T09:38:02.069 INFO:tasks.rados.rados.0.smithi022.stdout:1990: writing smithi02284960-24 from 8929280 to 9633792 tid 1 2024-09-21T09:38:02.072 INFO:tasks.rados.rados.0.smithi022.stdout:1990: writing smithi02284960-24 from 9633792 to 10035200 tid 2 2024-09-21T09:38:02.072 INFO:tasks.rados.rados.0.smithi022.stdout:1991: copy_from oid 38 from oid 11 current snap is 195 2024-09-21T09:38:02.072 INFO:tasks.rados.rados.0.smithi022.stdout:1989: finishing rollback tid 1 to smithi02284960-13 2024-09-21T09:38:02.072 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 13 v 0 (ObjNum 129 snap 0 seq_num 3570566216) dirty dne 2024-09-21T09:38:02.072 INFO:tasks.rados.rados.0.smithi022.stdout:1989: done (5 left) 2024-09-21T09:38:02.073 INFO:tasks.rados.rados.0.smithi022.stdout:1992: rollback oid 9 current snap is 195 2024-09-21T09:38:02.073 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 9 to 190 2024-09-21T09:38:02.073 INFO:tasks.rados.rados.0.smithi022.stdout:1993: snap_create 2024-09-21T09:38:02.073 INFO:tasks.rados.rados.0.smithi022.stdout:1988: finishing copy_from to smithi02284960-16 2024-09-21T09:38:02.073 INFO:tasks.rados.rados.0.smithi022.stdout:1988: got expected ENOENT (src dne) 2024-09-21T09:38:02.073 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 37 v 1244 (ObjNum 33 snap 0 seq_num 3568437904) dirty exists 2024-09-21T09:38:02.073 INFO:tasks.rados.rados.0.smithi022.stdout:1988: finishing copy_from racing read to smithi02284960-16 2024-09-21T09:38:02.073 INFO:tasks.rados.rados.0.smithi022.stdout:1987: finishing copy_from racing read to smithi02284960-34 2024-09-21T09:38:02.073 INFO:tasks.rados.rados.0.smithi022.stdout:1987: finishing copy_from to smithi02284960-34 2024-09-21T09:38:02.073 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 34 v 1061 (ObjNum 550 snap 166 seq_num 550) dirty exists 2024-09-21T09:38:02.075 INFO:tasks.rados.rados.0.smithi022.stdout:1990: finishing write tid 1 to smithi02284960-24 2024-09-21T09:38:02.076 INFO:tasks.rados.rados.0.smithi022.stdout:1990: finishing write tid 2 to smithi02284960-24 2024-09-21T09:38:02.076 INFO:tasks.rados.rados.0.smithi022.stdout:1992: finishing rollback tid 0 to smithi02284960-9 2024-09-21T09:38:02.076 INFO:tasks.rados.rados.0.smithi022.stdout:1990: finishing write tid 3 to smithi02284960-24 2024-09-21T09:38:02.076 INFO:tasks.rados.rados.0.smithi022.stdout:1992: finishing rollback tid 1 to smithi02284960-9 2024-09-21T09:38:02.077 INFO:tasks.rados.rados.0.smithi022.stdout:1992: finishing rollback tid 2 to smithi02284960-9 2024-09-21T09:38:02.077 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 9 v 683 (ObjNum 526 snap 156 seq_num 526) dirty exists 2024-09-21T09:38:02.078 INFO:tasks.rados.rados.0.smithi022.stdout:1990: finishing write tid 4 to smithi02284960-24 2024-09-21T09:38:02.078 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 24 v 1248 (ObjNum 639 snap 195 seq_num 639) dirty exists 2024-09-21T09:38:02.078 INFO:tasks.rados.rados.0.smithi022.stdout:1990: left oid 24 (ObjNum 639 snap 195 seq_num 639) 2024-09-21T09:38:02.111 INFO:tasks.rados.rados.0.smithi022.stdout:1991: finishing copy_from racing read to smithi02284960-38 2024-09-21T09:38:02.113 INFO:tasks.rados.rados.0.smithi022.stdout:1991: finishing copy_from to smithi02284960-38 2024-09-21T09:38:02.113 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 38 v 1044 (ObjNum 569 snap 170 seq_num 569) dirty exists 2024-09-21T09:38:02.284 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:01 smithi064 ceph-mon[86709]: pgmap v292: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 146 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 1.9 MiB/s wr, 20 op/s 2024-09-21T09:38:02.285 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:01 smithi064 ceph-mon[86709]: osdmap e460: 8 total, 8 up, 8 in 2024-09-21T09:38:02.285 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:01 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-09-21T09:38:02.307 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:01 smithi022 ceph-mon[96736]: pgmap v292: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 146 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 1.9 MiB/s wr, 20 op/s 2024-09-21T09:38:02.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:01 smithi022 ceph-mon[96736]: osdmap e460: 8 total, 8 up, 8 in 2024-09-21T09:38:02.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:01 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-09-21T09:38:02.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:01 smithi022 ceph-mon[98615]: pgmap v292: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 146 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 1.9 MiB/s wr, 20 op/s 2024-09-21T09:38:02.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:01 smithi022 ceph-mon[98615]: osdmap e460: 8 total, 8 up, 8 in 2024-09-21T09:38:02.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:01 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-09-21T09:38:03.020 INFO:tasks.rados.rados.0.smithi022.stdout:1986: done (6 left) 2024-09-21T09:38:03.020 INFO:tasks.rados.rados.0.smithi022.stdout:1987: done (5 left) 2024-09-21T09:38:03.021 INFO:tasks.rados.rados.0.smithi022.stdout:1988: done (4 left) 2024-09-21T09:38:03.021 INFO:tasks.rados.rados.0.smithi022.stdout:1990: done (3 left) 2024-09-21T09:38:03.021 INFO:tasks.rados.rados.0.smithi022.stdout:1991: done (2 left) 2024-09-21T09:38:03.021 INFO:tasks.rados.rados.0.smithi022.stdout:1992: done (1 left) 2024-09-21T09:38:03.021 INFO:tasks.rados.rados.0.smithi022.stdout:1993: done (0 left) 2024-09-21T09:38:03.021 INFO:tasks.rados.rados.0.smithi022.stdout:1994: rmattr oid 9 current snap is 196 2024-09-21T09:38:03.021 INFO:tasks.rados.rados.0.smithi022.stdout:1994: done (0 left) 2024-09-21T09:38:03.021 INFO:tasks.rados.rados.0.smithi022.stdout:1995: copy_from oid 8 from oid 42 current snap is 196 2024-09-21T09:38:03.021 INFO:tasks.rados.rados.0.smithi022.stdout:1996: delete oid 3 current snap is 196 2024-09-21T09:38:03.022 INFO:tasks.rados.rados.0.smithi022.stdout:1996: done (1 left) 2024-09-21T09:38:03.023 INFO:tasks.rados.rados.0.smithi022.stdout:1997: rollback oid 25 current snap is 196 2024-09-21T09:38:03.023 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 25 to 193 2024-09-21T09:38:03.023 INFO:tasks.rados.rados.0.smithi022.stdout:1998: snap_create 2024-09-21T09:38:03.026 INFO:tasks.rados.rados.0.smithi022.stdout:1995: finishing copy_from to smithi02284960-8 2024-09-21T09:38:03.026 INFO:tasks.rados.rados.0.smithi022.stdout:1995: got expected ENOENT (src dne) 2024-09-21T09:38:03.026 INFO:tasks.rados.rados.0.smithi022.stdout:1995: finishing copy_from racing read to smithi02284960-8 2024-09-21T09:38:03.027 INFO:tasks.rados.rados.0.smithi022.stdout:1997: finishing rollback tid 0 to smithi02284960-25 2024-09-21T09:38:03.027 INFO:tasks.rados.rados.0.smithi022.stdout:1997: finishing rollback tid 1 to smithi02284960-25 2024-09-21T09:38:03.027 INFO:tasks.rados.rados.0.smithi022.stdout:1997: finishing rollback tid 2 to smithi02284960-25 2024-09-21T09:38:03.028 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 25 v 1049 (ObjNum 611 snap 186 seq_num 611) dirty exists 2024-09-21T09:38:03.155 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:03 smithi064 ceph-mon[86709]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-21T09:38:03.155 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:03 smithi064 ceph-mon[86709]: Upgrade: osd.4 is safe to restart 2024-09-21T09:38:03.155 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:03 smithi064 ceph-mon[86709]: osdmap e461: 8 total, 8 up, 8 in 2024-09-21T09:38:03.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:03 smithi022 ceph-mon[98615]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-21T09:38:03.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:03 smithi022 ceph-mon[98615]: Upgrade: osd.4 is safe to restart 2024-09-21T09:38:03.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:03 smithi022 ceph-mon[98615]: osdmap e461: 8 total, 8 up, 8 in 2024-09-21T09:38:03.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:03 smithi022 ceph-mon[96736]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-21T09:38:03.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:03 smithi022 ceph-mon[96736]: Upgrade: osd.4 is safe to restart 2024-09-21T09:38:03.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:03 smithi022 ceph-mon[96736]: osdmap e461: 8 total, 8 up, 8 in 2024-09-21T09:38:03.984 INFO:tasks.rados.rados.0.smithi022.stdout:1995: done (2 left) 2024-09-21T09:38:03.985 INFO:tasks.rados.rados.0.smithi022.stdout:1997: done (1 left) 2024-09-21T09:38:03.985 INFO:tasks.rados.rados.0.smithi022.stdout:1998: done (0 left) 2024-09-21T09:38:03.985 INFO:tasks.rados.rados.0.smithi022.stdout:1999: read oid 30 snap 190 2024-09-21T09:38:03.985 INFO:tasks.rados.rados.0.smithi022.stdout:1999: expect (ObjNum 622 snap 186 seq_num 622) 2024-09-21T09:38:03.985 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 42 current snap is 197 2024-09-21T09:38:03.985 INFO:tasks.rados.rados.0.smithi022.stdout:2000: seq_num 641 ranges {0=745472,745472=401408} 2024-09-21T09:38:03.989 INFO:tasks.rados.rados.0.smithi022.stdout:2000: writing smithi02284960-42 from 0 to 745472 tid 1 2024-09-21T09:38:03.991 INFO:tasks.rados.rados.0.smithi022.stdout:2000: writing smithi02284960-42 from 745472 to 1146880 tid 2 2024-09-21T09:38:03.991 INFO:tasks.rados.rados.0.smithi022.stdout:2001: rmattr oid 38 current snap is 197 2024-09-21T09:38:03.991 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 15 current snap is 197 2024-09-21T09:38:03.991 INFO:tasks.rados.rados.0.smithi022.stdout:2002: seq_num 642 ranges {1957888=761856,2719744=401408} 2024-09-21T09:38:04.006 INFO:tasks.rados.rados.0.smithi022.stdout:2002: writing smithi02284960-15 from 1957888 to 2719744 tid 1 2024-09-21T09:38:04.010 INFO:tasks.rados.rados.0.smithi022.stdout:2002: writing smithi02284960-15 from 2719744 to 3121152 tid 2 2024-09-21T09:38:04.010 INFO:tasks.rados.rados.0.smithi022.stdout:2003: read oid 46 snap 190 2024-09-21T09:38:04.010 INFO:tasks.rados.rados.0.smithi022.stdout:2003: expect (ObjNum 609 snap 185 seq_num 609) 2024-09-21T09:38:04.010 INFO:tasks.rados.rados.0.smithi022.stdout:2004: read oid 31 snap -1 2024-09-21T09:38:04.010 INFO:tasks.rados.rados.0.smithi022.stdout:2004: expect (ObjNum 590 snap 176 seq_num 590) 2024-09-21T09:38:04.010 INFO:tasks.rados.rados.0.smithi022.stdout:2005: read oid 32 snap -1 2024-09-21T09:38:04.010 INFO:tasks.rados.rados.0.smithi022.stdout:2005: expect (ObjNum 538 snap 162 seq_num 538) 2024-09-21T09:38:04.010 INFO:tasks.rados.rados.0.smithi022.stdout:2006: rollback oid 25 current snap is 197 2024-09-21T09:38:04.010 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 25 to 194 2024-09-21T09:38:04.010 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 38 v 1045 (ObjNum 569 snap 170 seq_num 569) dirty exists 2024-09-21T09:38:04.010 INFO:tasks.rados.rados.0.smithi022.stdout:2000: finishing write tid 1 to smithi02284960-42 2024-09-21T09:38:04.010 INFO:tasks.rados.rados.0.smithi022.stdout:2000: finishing write tid 2 to smithi02284960-42 2024-09-21T09:38:04.011 INFO:tasks.rados.rados.0.smithi022.stdout:2000: finishing write tid 3 to smithi02284960-42 2024-09-21T09:38:04.011 INFO:tasks.rados.rados.0.smithi022.stdout:2001: done (7 left) 2024-09-21T09:38:04.011 INFO:tasks.rados.rados.0.smithi022.stdout:2007: copy_from oid 14 from oid 36 current snap is 197 2024-09-21T09:38:04.011 INFO:tasks.rados.rados.0.smithi022.stdout:2000: finishing write tid 4 to smithi02284960-42 2024-09-21T09:38:04.011 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 42 v 1097 (ObjNum 641 snap 197 seq_num 641) dirty exists 2024-09-21T09:38:04.011 INFO:tasks.rados.rados.0.smithi022.stdout:2000: left oid 42 (ObjNum 641 snap 197 seq_num 641) 2024-09-21T09:38:04.011 INFO:tasks.rados.rados.0.smithi022.stdout:1999: expect (ObjNum 622 snap 186 seq_num 622) 2024-09-21T09:38:04.177 INFO:tasks.rados.rados.0.smithi022.stdout:2003: expect (ObjNum 609 snap 185 seq_num 609) 2024-09-21T09:38:04.191 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:38:04 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:38:04.147Z caller=alerting.go:391 level=warn component="rule manager" alert="unsupported value type" msg="Expanding alert template failed" err="error executing template __alert_CephOSDDown: template: __alert_CephOSDDown:1:358: executing \"__alert_CephOSDDown\" at : error calling query: found duplicate series for the match group {ceph_daemon=\"osd.3\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.3\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.3\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}];many-to-many matching not allowed: matching labels must be unique on one side" data="unsupported value type" 2024-09-21T09:38:04.191 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:38:04 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:38:04.148Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.3\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.3\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.3\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster_addr=\"172.21.15.22\", device_class=\"ssd\", hostname=\"smithi022\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.22\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:38:04.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:04 smithi064 ceph-mon[86709]: pgmap v295: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 146 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 5.4 MiB/s rd, 258 KiB/s wr, 3 op/s 2024-09-21T09:38:04.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:04 smithi064 ceph-mon[86709]: osdmap e462: 8 total, 8 up, 8 in 2024-09-21T09:38:04.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:04 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:38:04.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:04 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-21T09:38:04.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:04 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:38:04.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:04 smithi064 ceph-mon[86709]: osdmap e463: 8 total, 8 up, 8 in 2024-09-21T09:38:04.193 INFO:tasks.rados.rados.0.smithi022.stdout:2006: finishing rollback tid 0 to smithi02284960-25 2024-09-21T09:38:04.193 INFO:tasks.rados.rados.0.smithi022.stdout:2006: finishing rollback tid 1 to smithi02284960-25 2024-09-21T09:38:04.193 INFO:tasks.rados.rados.0.smithi022.stdout:2006: finishing rollback tid 2 to smithi02284960-25 2024-09-21T09:38:04.194 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 25 v 1053 (ObjNum 638 snap 194 seq_num 638) dirty exists 2024-09-21T09:38:04.194 INFO:tasks.rados.rados.0.smithi022.stdout:2002: finishing write tid 1 to smithi02284960-15 2024-09-21T09:38:04.194 INFO:tasks.rados.rados.0.smithi022.stdout:2002: finishing write tid 2 to smithi02284960-15 2024-09-21T09:38:04.194 INFO:tasks.rados.rados.0.smithi022.stdout:2002: finishing write tid 3 to smithi02284960-15 2024-09-21T09:38:04.194 INFO:tasks.rados.rados.0.smithi022.stdout:2004: expect (ObjNum 590 snap 176 seq_num 590) 2024-09-21T09:38:04.203 INFO:tasks.rados.rados.0.smithi022.stdout:2002: finishing write tid 4 to smithi02284960-15 2024-09-21T09:38:04.203 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 15 v 1252 (ObjNum 642 snap 197 seq_num 642) dirty exists 2024-09-21T09:38:04.203 INFO:tasks.rados.rados.0.smithi022.stdout:2002: left oid 15 (ObjNum 642 snap 197 seq_num 642) 2024-09-21T09:38:04.204 INFO:tasks.rados.rados.0.smithi022.stdout:2007: finishing copy_from racing read to smithi02284960-14 2024-09-21T09:38:04.204 INFO:tasks.rados.rados.0.smithi022.stdout:2007: finishing copy_from to smithi02284960-14 2024-09-21T09:38:04.204 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 14 v 1082 (ObjNum 580 snap 172 seq_num 580) dirty exists 2024-09-21T09:38:04.204 INFO:tasks.rados.rados.0.smithi022.stdout:2005: expect (ObjNum 538 snap 162 seq_num 538) 2024-09-21T09:38:04.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:04 smithi022 ceph-mon[96736]: pgmap v295: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 146 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 5.4 MiB/s rd, 258 KiB/s wr, 3 op/s 2024-09-21T09:38:04.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:04 smithi022 ceph-mon[96736]: osdmap e462: 8 total, 8 up, 8 in 2024-09-21T09:38:04.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:04 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:38:04.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:04 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-21T09:38:04.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:04 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:38:04.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:04 smithi022 ceph-mon[96736]: osdmap e463: 8 total, 8 up, 8 in 2024-09-21T09:38:04.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:04 smithi022 ceph-mon[98615]: pgmap v295: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 146 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 5.4 MiB/s rd, 258 KiB/s wr, 3 op/s 2024-09-21T09:38:04.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:04 smithi022 ceph-mon[98615]: osdmap e462: 8 total, 8 up, 8 in 2024-09-21T09:38:04.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:04 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:38:04.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:04 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-21T09:38:04.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:04 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:38:04.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:04 smithi022 ceph-mon[98615]: osdmap e463: 8 total, 8 up, 8 in 2024-09-21T09:38:04.406 INFO:tasks.rados.rados.0.smithi022.stdout:1999: done (7 left) 2024-09-21T09:38:04.406 INFO:tasks.rados.rados.0.smithi022.stdout:2000: done (6 left) 2024-09-21T09:38:04.406 INFO:tasks.rados.rados.0.smithi022.stdout:2002: done (5 left) 2024-09-21T09:38:04.406 INFO:tasks.rados.rados.0.smithi022.stdout:2003: done (4 left) 2024-09-21T09:38:04.406 INFO:tasks.rados.rados.0.smithi022.stdout:2004: done (3 left) 2024-09-21T09:38:04.406 INFO:tasks.rados.rados.0.smithi022.stdout:2005: done (2 left) 2024-09-21T09:38:04.406 INFO:tasks.rados.rados.0.smithi022.stdout:2006: done (1 left) 2024-09-21T09:38:04.406 INFO:tasks.rados.rados.0.smithi022.stdout:2007: done (0 left) 2024-09-21T09:38:04.406 INFO:tasks.rados.rados.0.smithi022.stdout:2008: delete oid 9 current snap is 197 2024-09-21T09:38:04.408 INFO:tasks.rados.rados.0.smithi022.stdout:2008: done (0 left) 2024-09-21T09:38:04.408 INFO:tasks.rados.rados.0.smithi022.stdout:2009: rmattr oid 31 current snap is 197 2024-09-21T09:38:04.408 INFO:tasks.rados.rados.0.smithi022.stdout:2010: copy_from oid 12 from oid 26 current snap is 197 2024-09-21T09:38:04.408 INFO:tasks.rados.rados.0.smithi022.stdout:2011: setattr oid 13 current snap is 197 2024-09-21T09:38:04.409 INFO:tasks.rados.rados.0.smithi022.stdout:2012: snap_create 2024-09-21T09:38:04.410 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 31 v 1254 (ObjNum 590 snap 176 seq_num 590) dirty exists 2024-09-21T09:38:04.411 INFO:tasks.rados.rados.0.smithi022.stdout:2010: finishing copy_from to smithi02284960-12 2024-09-21T09:38:04.411 INFO:tasks.rados.rados.0.smithi022.stdout:2010: got expected ENOENT (src dne) 2024-09-21T09:38:04.411 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 13 v 1098 (ObjNum 33 snap 0 seq_num 3569663168) dirty exists 2024-09-21T09:38:04.411 INFO:tasks.rados.rados.0.smithi022.stdout:2010: finishing copy_from racing read to smithi02284960-12 2024-09-21T09:38:05.025 INFO:tasks.rados.rados.0.smithi022.stdout:2009: done (3 left) 2024-09-21T09:38:05.025 INFO:tasks.rados.rados.0.smithi022.stdout:2010: done (2 left) 2024-09-21T09:38:05.025 INFO:tasks.rados.rados.0.smithi022.stdout:2011: done (1 left) 2024-09-21T09:38:05.025 INFO:tasks.rados.rados.0.smithi022.stdout:2012: done (0 left) 2024-09-21T09:38:05.025 INFO:tasks.rados.rados.0.smithi022.stdout:2013: delete oid 24 current snap is 198 2024-09-21T09:38:05.031 INFO:tasks.rados.rados.0.smithi022.stdout:2013: done (0 left) 2024-09-21T09:38:05.031 INFO:tasks.rados.rados.0.smithi022.stdout:2014: rmattr oid 14 current snap is 198 2024-09-21T09:38:05.031 INFO:tasks.rados.rados.0.smithi022.stdout:2015: delete oid 7 current snap is 198 2024-09-21T09:38:05.034 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 14 v 1090 (ObjNum 580 snap 172 seq_num 580) dirty exists 2024-09-21T09:38:05.034 INFO:tasks.rados.rados.0.smithi022.stdout:2014: done (1 left) 2024-09-21T09:38:05.035 INFO:tasks.rados.rados.0.smithi022.stdout:2015: done (0 left) 2024-09-21T09:38:05.035 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 4 current snap is 198 2024-09-21T09:38:05.035 INFO:tasks.rados.rados.0.smithi022.stdout:2016: seq_num 646 ranges {7225344=425984,7651328=655360,8306688=147456} 2024-09-21T09:38:05.076 INFO:tasks.rados.rados.0.smithi022.stdout:2016: writing smithi02284960-4 from 7225344 to 7651328 tid 1 2024-09-21T09:38:05.080 INFO:tasks.rados.rados.0.smithi022.stdout:2016: writing smithi02284960-4 from 7651328 to 8306688 tid 2 2024-09-21T09:38:05.081 INFO:tasks.rados.rados.0.smithi022.stdout:2016: writing smithi02284960-4 from 8306688 to 8454144 tid 3 2024-09-21T09:38:05.081 INFO:tasks.rados.rados.0.smithi022.stdout:2017: snap_create 2024-09-21T09:38:05.081 INFO:tasks.rados.rados.0.smithi022.stdout:2016: finishing write tid 1 to smithi02284960-4 2024-09-21T09:38:05.083 INFO:tasks.rados.rados.0.smithi022.stdout:2016: finishing write tid 2 to smithi02284960-4 2024-09-21T09:38:05.083 INFO:tasks.rados.rados.0.smithi022.stdout:2016: finishing write tid 3 to smithi02284960-4 2024-09-21T09:38:05.084 INFO:tasks.rados.rados.0.smithi022.stdout:2016: finishing write tid 4 to smithi02284960-4 2024-09-21T09:38:05.084 INFO:tasks.rados.rados.0.smithi022.stdout:2016: finishing write tid 5 to smithi02284960-4 2024-09-21T09:38:05.085 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 4 v 690 (ObjNum 646 snap 198 seq_num 646) dirty exists 2024-09-21T09:38:05.085 INFO:tasks.rados.rados.0.smithi022.stdout:2016: left oid 4 (ObjNum 646 snap 198 seq_num 646) 2024-09-21T09:38:05.189 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:05 smithi064 ceph-mon[86709]: Upgrade: Updating osd.4 2024-09-21T09:38:05.189 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:05 smithi064 ceph-mon[86709]: Deploying daemon osd.4 on smithi064 2024-09-21T09:38:05.307 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:05 smithi022 ceph-mon[98615]: Upgrade: Updating osd.4 2024-09-21T09:38:05.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:05 smithi022 ceph-mon[98615]: Deploying daemon osd.4 on smithi064 2024-09-21T09:38:05.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:05 smithi022 ceph-mon[96736]: Upgrade: Updating osd.4 2024-09-21T09:38:05.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:05 smithi022 ceph-mon[96736]: Deploying daemon osd.4 on smithi064 2024-09-21T09:38:06.023 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:38:05 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:38:05] "GET /metrics HTTP/1.1" 200 36231 "" "Prometheus/2.51.0" 2024-09-21T09:38:06.023 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:38:05 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:38:05.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:38:06.023 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:38:05 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:38:05.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:38:06.031 INFO:tasks.rados.rados.0.smithi022.stdout:2016: done (1 left) 2024-09-21T09:38:06.031 INFO:tasks.rados.rados.0.smithi022.stdout:2017: done (0 left) 2024-09-21T09:38:06.031 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 45 current snap is 199 2024-09-21T09:38:06.031 INFO:tasks.rados.rados.0.smithi022.stdout:2018: seq_num 647 ranges {2293760=442368,2736128=589824,3325952=212992} 2024-09-21T09:38:06.046 INFO:tasks.rados.rados.0.smithi022.stdout:2018: writing smithi02284960-45 from 2293760 to 2736128 tid 1 2024-09-21T09:38:06.049 INFO:tasks.rados.rados.0.smithi022.stdout:2018: writing smithi02284960-45 from 2736128 to 3325952 tid 2 2024-09-21T09:38:06.051 INFO:tasks.rados.rados.0.smithi022.stdout:2018: writing smithi02284960-45 from 3325952 to 3538944 tid 3 2024-09-21T09:38:06.051 INFO:tasks.rados.rados.0.smithi022.stdout:2019: snap_remove snap 190 2024-09-21T09:38:06.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:06 smithi022 ceph-mon[98615]: pgmap v298: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 161 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 6.2 MiB/s rd, 4.0 MiB/s wr, 12 op/s 2024-09-21T09:38:06.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:06 smithi022 ceph-mon[98615]: osdmap e464: 8 total, 8 up, 8 in 2024-09-21T09:38:06.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:06 smithi022 ceph-mon[96736]: pgmap v298: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 161 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 6.2 MiB/s rd, 4.0 MiB/s wr, 12 op/s 2024-09-21T09:38:06.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:06 smithi022 ceph-mon[96736]: osdmap e464: 8 total, 8 up, 8 in 2024-09-21T09:38:06.388 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:06 smithi064 ceph-mon[86709]: pgmap v298: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 161 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 6.2 MiB/s rd, 4.0 MiB/s wr, 12 op/s 2024-09-21T09:38:06.388 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:06 smithi064 ceph-mon[86709]: osdmap e464: 8 total, 8 up, 8 in 2024-09-21T09:38:06.691 INFO:journalctl@ceph.osd.4.smithi064.stdout:Sep 21 09:38:06 smithi064 systemd[1]: Stopping Ceph osd.4 for d65e801e-77fa-11ef-baf6-efdc52797490... 2024-09-21T09:38:07.040 INFO:tasks.rados.rados.0.smithi022.stdout:2019: done (1 left) 2024-09-21T09:38:07.040 INFO:tasks.rados.rados.0.smithi022.stdout:2020: copy_from oid 25 from oid 40 current snap is 199 2024-09-21T09:38:07.040 INFO:tasks.rados.rados.0.smithi022.stdout:2018: finishing write tid 1 to smithi02284960-45 2024-09-21T09:38:07.040 INFO:tasks.rados.rados.0.smithi022.stdout:2018: finishing write tid 2 to smithi02284960-45 2024-09-21T09:38:07.040 INFO:tasks.rados.rados.0.smithi022.stdout:2018: finishing write tid 3 to smithi02284960-45 2024-09-21T09:38:07.040 INFO:tasks.rados.rados.0.smithi022.stdout:2018: finishing write tid 4 to smithi02284960-45 2024-09-21T09:38:07.041 INFO:tasks.rados.rados.0.smithi022.stdout:2018: finishing write tid 5 to smithi02284960-45 2024-09-21T09:38:07.041 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 45 v 1104 (ObjNum 647 snap 199 seq_num 647) dirty exists 2024-09-21T09:38:07.041 INFO:tasks.rados.rados.0.smithi022.stdout:2018: left oid 45 (ObjNum 647 snap 199 seq_num 647) 2024-09-21T09:38:07.041 INFO:tasks.rados.rados.0.smithi022.stdout:2018: done (1 left) 2024-09-21T09:38:07.041 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 32 current snap is 199 2024-09-21T09:38:07.041 INFO:tasks.rados.rados.0.smithi022.stdout:2021: seq_num 648 ranges {4538368=458752,4997120=475136,5472256=327680} 2024-09-21T09:38:07.068 INFO:tasks.rados.rados.0.smithi022.stdout:2021: writing smithi02284960-32 from 4538368 to 4997120 tid 1 2024-09-21T09:38:07.071 INFO:tasks.rados.rados.0.smithi022.stdout:2021: writing smithi02284960-32 from 4997120 to 5472256 tid 2 2024-09-21T09:38:07.073 INFO:tasks.rados.rados.0.smithi022.stdout:2021: writing smithi02284960-32 from 5472256 to 5799936 tid 3 2024-09-21T09:38:07.073 INFO:tasks.rados.rados.0.smithi022.stdout:2022: snap_remove snap 198 2024-09-21T09:38:07.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:07 smithi064 ceph-mon[86709]: osdmap e465: 8 total, 8 up, 8 in 2024-09-21T09:38:07.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:07 smithi064 ceph-mon[86709]: osd.4 marked itself down and dead 2024-09-21T09:38:07.192 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:38:06 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:38:06.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi022\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:38:07.192 INFO:journalctl@ceph.osd.4.smithi064.stdout:Sep 21 09:38:06 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-4[36572]: 2024-09-21T09:38:06.843+0000 7fb73faf0640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T09:38:07.192 INFO:journalctl@ceph.osd.4.smithi064.stdout:Sep 21 09:38:06 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-4[36572]: 2024-09-21T09:38:06.843+0000 7fb73faf0640 -1 osd.4 465 *** Got signal Terminated *** 2024-09-21T09:38:07.192 INFO:journalctl@ceph.osd.4.smithi064.stdout:Sep 21 09:38:06 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-4[36572]: 2024-09-21T09:38:06.843+0000 7fb73faf0640 -1 osd.4 465 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T09:38:07.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:07 smithi022 ceph-mon[98615]: osdmap e465: 8 total, 8 up, 8 in 2024-09-21T09:38:07.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:07 smithi022 ceph-mon[98615]: osd.4 marked itself down and dead 2024-09-21T09:38:07.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:07 smithi022 ceph-mon[96736]: osdmap e465: 8 total, 8 up, 8 in 2024-09-21T09:38:07.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:07 smithi022 ceph-mon[96736]: osd.4 marked itself down and dead 2024-09-21T09:38:07.691 INFO:journalctl@ceph.osd.4.smithi064.stdout:Sep 21 09:38:07 smithi064 podman[91747]: 2024-09-21 09:38:07.387866249 +0000 UTC m=+0.650302186 container died 2a2937374fc48d9e87f795c4e10df43ec98ea2cc5141f99b2093a626cf449ae0 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-4, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-fbd1a6a, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_CLEAN=True) 2024-09-21T09:38:08.038 INFO:tasks.rados.rados.0.smithi022.stdout:2022: done (2 left) 2024-09-21T09:38:08.038 INFO:tasks.rados.rados.0.smithi022.stdout:2023: delete oid 19 current snap is 199 2024-09-21T09:38:08.039 INFO:tasks.rados.rados.0.smithi022.stdout:2020: finishing copy_from racing read to smithi02284960-25 2024-09-21T09:38:08.039 INFO:tasks.rados.rados.0.smithi022.stdout:2020: finishing copy_from to smithi02284960-25 2024-09-21T09:38:08.039 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 25 v 1024 (ObjNum 624 snap 186 seq_num 624) dirty exists 2024-09-21T09:38:08.039 INFO:tasks.rados.rados.0.smithi022.stdout:2021: finishing write tid 1 to smithi02284960-32 2024-09-21T09:38:08.039 INFO:tasks.rados.rados.0.smithi022.stdout:2021: finishing write tid 2 to smithi02284960-32 2024-09-21T09:38:08.039 INFO:tasks.rados.rados.0.smithi022.stdout:2021: finishing write tid 3 to smithi02284960-32 2024-09-21T09:38:08.039 INFO:tasks.rados.rados.0.smithi022.stdout:2021: finishing write tid 4 to smithi02284960-32 2024-09-21T09:38:08.039 INFO:tasks.rados.rados.0.smithi022.stdout:2021: finishing write tid 5 to smithi02284960-32 2024-09-21T09:38:08.039 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 32 v 1198 (ObjNum 648 snap 199 seq_num 648) dirty exists 2024-09-21T09:38:08.039 INFO:tasks.rados.rados.0.smithi022.stdout:2021: left oid 32 (ObjNum 648 snap 199 seq_num 648) 2024-09-21T09:38:08.046 INFO:journalctl@ceph.osd.4.smithi064.stdout:Sep 21 09:38:07 smithi064 podman[91747]: 2024-09-21 09:38:07.866707745 +0000 UTC m=+1.129143656 container cleanup 2a2937374fc48d9e87f795c4e10df43ec98ea2cc5141f99b2093a626cf449ae0 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-4, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, RELEASE=quincy-fbd1a6a, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.37.2) 2024-09-21T09:38:08.046 INFO:journalctl@ceph.osd.4.smithi064.stdout:Sep 21 09:38:07 smithi064 bash[91747]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-4 2024-09-21T09:38:08.053 INFO:tasks.rados.rados.0.smithi022.stdout:2020: done (2 left) 2024-09-21T09:38:08.053 INFO:tasks.rados.rados.0.smithi022.stdout:2021: done (1 left) 2024-09-21T09:38:08.053 INFO:tasks.rados.rados.0.smithi022.stdout:2023: done (0 left) 2024-09-21T09:38:08.053 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 23 current snap is 199 2024-09-21T09:38:08.053 INFO:tasks.rados.rados.0.smithi022.stdout:2024: seq_num 650 ranges {0=491520,491520=647168,1138688=155648} 2024-09-21T09:38:08.056 INFO:tasks.rados.rados.0.smithi022.stdout:2024: writing smithi02284960-23 from 0 to 491520 tid 1 2024-09-21T09:38:08.061 INFO:tasks.rados.rados.0.smithi022.stdout:2024: writing smithi02284960-23 from 491520 to 1138688 tid 2 2024-09-21T09:38:08.062 INFO:tasks.rados.rados.0.smithi022.stdout:2024: writing smithi02284960-23 from 1138688 to 1294336 tid 3 2024-09-21T09:38:08.062 INFO:tasks.rados.rados.0.smithi022.stdout:2025: snap_create 2024-09-21T09:38:08.062 INFO:tasks.rados.rados.0.smithi022.stdout:2024: finishing write tid 1 to smithi02284960-23 2024-09-21T09:38:08.065 INFO:tasks.rados.rados.0.smithi022.stdout:2024: finishing write tid 2 to smithi02284960-23 2024-09-21T09:38:08.065 INFO:tasks.rados.rados.0.smithi022.stdout:2024: finishing write tid 3 to smithi02284960-23 2024-09-21T09:38:08.066 INFO:tasks.rados.rados.0.smithi022.stdout:2024: finishing write tid 4 to smithi02284960-23 2024-09-21T09:38:08.067 INFO:tasks.rados.rados.0.smithi022.stdout:2024: finishing write tid 5 to smithi02284960-23 2024-09-21T09:38:08.067 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 23 v 1049 (ObjNum 650 snap 199 seq_num 650) dirty exists 2024-09-21T09:38:08.067 INFO:tasks.rados.rados.0.smithi022.stdout:2024: left oid 23 (ObjNum 650 snap 199 seq_num 650) 2024-09-21T09:38:08.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:08 smithi022 ceph-mon[98615]: pgmap v301: 121 pgs: 121 active+clean; 154 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 9.3 MiB/s rd, 4.3 MiB/s wr, 17 op/s 2024-09-21T09:38:08.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:08 smithi022 ceph-mon[98615]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T09:38:08.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:08 smithi022 ceph-mon[98615]: osdmap e466: 8 total, 7 up, 8 in 2024-09-21T09:38:08.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:08 smithi022 ceph-mon[96736]: pgmap v301: 121 pgs: 121 active+clean; 154 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 9.3 MiB/s rd, 4.3 MiB/s wr, 17 op/s 2024-09-21T09:38:08.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:08 smithi022 ceph-mon[96736]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T09:38:08.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:08 smithi022 ceph-mon[96736]: osdmap e466: 8 total, 7 up, 8 in 2024-09-21T09:38:08.441 INFO:journalctl@ceph.osd.4.smithi064.stdout:Sep 21 09:38:08 smithi064 podman[91767]: 2024-09-21 09:38:08.117859035 +0000 UTC m=+0.728463479 container remove 2a2937374fc48d9e87f795c4e10df43ec98ea2cc5141f99b2093a626cf449ae0 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-4, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=quincy-fbd1a6a, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T09:38:08.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:08 smithi064 ceph-mon[86709]: pgmap v301: 121 pgs: 121 active+clean; 154 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 9.3 MiB/s rd, 4.3 MiB/s wr, 17 op/s 2024-09-21T09:38:08.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:08 smithi064 ceph-mon[86709]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T09:38:08.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:08 smithi064 ceph-mon[86709]: osdmap e466: 8 total, 7 up, 8 in 2024-09-21T09:38:08.692 INFO:journalctl@ceph.osd.4.smithi064.stdout:Sep 21 09:38:08 smithi064 podman[91862]: 2024-09-21 09:38:08.342588939 +0000 UTC m=+0.016717494 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:38:08.692 INFO:journalctl@ceph.osd.4.smithi064.stdout:Sep 21 09:38:08 smithi064 podman[91862]: 2024-09-21 09:38:08.444179028 +0000 UTC m=+0.118307622 container create afbc5020a55e23a039de88126d995eb27e7304d59d5861a44018e2598a810686 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-4-deactivate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, RELEASE=main-62da54b, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.license=GPLv2) 2024-09-21T09:38:08.692 INFO:journalctl@ceph.osd.4.smithi064.stdout:Sep 21 09:38:08 smithi064 podman[91862]: 2024-09-21 09:38:08.67169891 +0000 UTC m=+0.345827472 container init afbc5020a55e23a039de88126d995eb27e7304d59d5861a44018e2598a810686 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-4-deactivate, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , RELEASE=main-62da54b, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T09:38:08.692 INFO:journalctl@ceph.osd.4.smithi064.stdout:Sep 21 09:38:08 smithi064 podman[91862]: 2024-09-21 09:38:08.676698372 +0000 UTC m=+0.350826926 container start afbc5020a55e23a039de88126d995eb27e7304d59d5861a44018e2598a810686 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-4-deactivate, GIT_BRANCH=HEAD, RELEASE=main-62da54b, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T09:38:08.988 INFO:tasks.rados.rados.0.smithi022.stdout:2024: done (1 left) 2024-09-21T09:38:08.988 INFO:tasks.rados.rados.0.smithi022.stdout:2025: done (0 left) 2024-09-21T09:38:08.988 INFO:tasks.rados.rados.0.smithi022.stdout:2026: read oid 33 snap -1 2024-09-21T09:38:08.988 INFO:tasks.rados.rados.0.smithi022.stdout:2026: expect (ObjNum 550 snap 166 seq_num 550) 2024-09-21T09:38:08.989 INFO:tasks.rados.rados.0.smithi022.stdout:2027: read oid 6 snap -1 2024-09-21T09:38:08.989 INFO:tasks.rados.rados.0.smithi022.stdout:2027: expect (ObjNum 423 snap 132 seq_num 423) 2024-09-21T09:38:08.989 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 10 current snap is 200 2024-09-21T09:38:08.989 INFO:tasks.rados.rados.0.smithi022.stdout:2028: seq_num 651 ranges {2392064=507904,2899968=532480,3432448=270336} 2024-09-21T09:38:09.004 INFO:tasks.rados.rados.0.smithi022.stdout:2028: writing smithi02284960-10 from 2392064 to 2899968 tid 1 2024-09-21T09:38:09.006 INFO:tasks.rados.rados.0.smithi022.stdout:2028: writing smithi02284960-10 from 2899968 to 3432448 tid 2 2024-09-21T09:38:09.009 INFO:tasks.rados.rados.0.smithi022.stdout:2028: writing smithi02284960-10 from 3432448 to 3702784 tid 3 2024-09-21T09:38:09.009 INFO:tasks.rados.rados.0.smithi022.stdout:2029: copy_from oid 34 from oid 26 current snap is 200 2024-09-21T09:38:09.009 INFO:tasks.rados.rados.0.smithi022.stdout:2026: expect (ObjNum 550 snap 166 seq_num 550) 2024-09-21T09:38:09.050 INFO:journalctl@ceph.osd.4.smithi064.stdout:Sep 21 09:38:08 smithi064 podman[91862]: 2024-09-21 09:38:08.779862169 +0000 UTC m=+0.453990739 container attach afbc5020a55e23a039de88126d995eb27e7304d59d5861a44018e2598a810686 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-4-deactivate, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, ceph=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-62da54b, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux ) 2024-09-21T09:38:09.050 INFO:journalctl@ceph.osd.4.smithi064.stdout:Sep 21 09:38:08 smithi064 podman[91862]: 2024-09-21 09:38:08.871175644 +0000 UTC m=+0.545304196 container died afbc5020a55e23a039de88126d995eb27e7304d59d5861a44018e2598a810686 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-4-deactivate, RELEASE=main-62da54b, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, maintainer=Guillaume Abrioux ) 2024-09-21T09:38:09.064 INFO:tasks.rados.rados.0.smithi022.stdout:2027: expect (ObjNum 423 snap 132 seq_num 423) 2024-09-21T09:38:09.156 INFO:tasks.rados.rados.0.smithi022.stdout:2028: finishing write tid 1 to smithi02284960-10 2024-09-21T09:38:09.157 INFO:tasks.rados.rados.0.smithi022.stdout:2028: finishing write tid 2 to smithi02284960-10 2024-09-21T09:38:09.157 INFO:tasks.rados.rados.0.smithi022.stdout:2028: finishing write tid 3 to smithi02284960-10 2024-09-21T09:38:09.157 INFO:tasks.rados.rados.0.smithi022.stdout:2028: finishing write tid 4 to smithi02284960-10 2024-09-21T09:38:09.157 INFO:tasks.rados.rados.0.smithi022.stdout:2029: finishing copy_from to smithi02284960-34 2024-09-21T09:38:09.157 INFO:tasks.rados.rados.0.smithi022.stdout:2029: got expected ENOENT (src dne) 2024-09-21T09:38:09.157 INFO:tasks.rados.rados.0.smithi022.stdout:2029: finishing copy_from racing read to smithi02284960-34 2024-09-21T09:38:09.157 INFO:tasks.rados.rados.0.smithi022.stdout:2028: finishing write tid 5 to smithi02284960-10 2024-09-21T09:38:09.157 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 10 v 1064 (ObjNum 651 snap 200 seq_num 651) dirty exists 2024-09-21T09:38:09.157 INFO:tasks.rados.rados.0.smithi022.stdout:2028: left oid 10 (ObjNum 651 snap 200 seq_num 651) 2024-09-21T09:38:09.157 INFO:tasks.rados.rados.0.smithi022.stdout:2026: done (3 left) 2024-09-21T09:38:09.157 INFO:tasks.rados.rados.0.smithi022.stdout:2027: done (2 left) 2024-09-21T09:38:09.157 INFO:tasks.rados.rados.0.smithi022.stdout:2028: done (1 left) 2024-09-21T09:38:09.158 INFO:tasks.rados.rados.0.smithi022.stdout:2029: done (0 left) 2024-09-21T09:38:09.158 INFO:tasks.rados.rados.0.smithi022.stdout:2030: copy_from oid 16 from oid 38 current snap is 200 2024-09-21T09:38:09.158 INFO:tasks.rados.rados.0.smithi022.stdout:2031: snap_create 2024-09-21T09:38:09.185 INFO:tasks.rados.rados.0.smithi022.stdout:2030: finishing copy_from racing read to smithi02284960-16 2024-09-21T09:38:09.186 INFO:tasks.rados.rados.0.smithi022.stdout:2030: finishing copy_from to smithi02284960-16 2024-09-21T09:38:09.186 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 16 v 1045 (ObjNum 569 snap 170 seq_num 569) dirty exists 2024-09-21T09:38:09.307 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:09 smithi022 ceph-mon[98615]: osdmap e467: 8 total, 7 up, 8 in 2024-09-21T09:38:09.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:09 smithi022 ceph-mon[98615]: osdmap e468: 8 total, 7 up, 8 in 2024-09-21T09:38:09.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:09 smithi022 ceph-mon[96736]: osdmap e467: 8 total, 7 up, 8 in 2024-09-21T09:38:09.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:09 smithi022 ceph-mon[96736]: osdmap e468: 8 total, 7 up, 8 in 2024-09-21T09:38:09.409 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:09 smithi064 ceph-mon[86709]: osdmap e467: 8 total, 7 up, 8 in 2024-09-21T09:38:09.409 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:09 smithi064 ceph-mon[86709]: osdmap e468: 8 total, 7 up, 8 in 2024-09-21T09:38:09.691 INFO:journalctl@ceph.osd.4.smithi064.stdout:Sep 21 09:38:09 smithi064 podman[91905]: 2024-09-21 09:38:09.408202267 +0000 UTC m=+0.530888395 container remove afbc5020a55e23a039de88126d995eb27e7304d59d5861a44018e2598a810686 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-4-deactivate, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-62da54b, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, org.label-schema.license=GPLv2) 2024-09-21T09:38:09.691 INFO:journalctl@ceph.osd.4.smithi064.stdout:Sep 21 09:38:09 smithi064 systemd[1]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490@osd.4.service: Deactivated successfully. 2024-09-21T09:38:09.692 INFO:journalctl@ceph.osd.4.smithi064.stdout:Sep 21 09:38:09 smithi064 systemd[1]: Stopped Ceph osd.4 for d65e801e-77fa-11ef-baf6-efdc52797490. 2024-09-21T09:38:09.692 INFO:journalctl@ceph.osd.4.smithi064.stdout:Sep 21 09:38:09 smithi064 systemd[1]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490@osd.4.service: Consumed 13.329s CPU time. 2024-09-21T09:38:09.692 INFO:journalctl@ceph.osd.4.smithi064.stdout:Sep 21 09:38:09 smithi064 systemd[1]: Starting Ceph osd.4 for d65e801e-77fa-11ef-baf6-efdc52797490... 2024-09-21T09:38:10.061 INFO:tasks.rados.rados.0.smithi022.stdout:2030: done (1 left) 2024-09-21T09:38:10.061 INFO:tasks.rados.rados.0.smithi022.stdout:2031: done (0 left) 2024-09-21T09:38:10.061 INFO:tasks.rados.rados.0.smithi022.stdout:2032: read oid 37 snap -1 2024-09-21T09:38:10.061 INFO:tasks.rados.rados.0.smithi022.stdout:2032: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-21T09:38:10.062 INFO:tasks.rados.rados.0.smithi022.stdout:2033: snap_remove snap 196 2024-09-21T09:38:10.137 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:10 smithi064 ceph-mon[86709]: pgmap v304: 121 pgs: 25 active+undersized, 9 peering, 2 stale+active+clean, 4 active+undersized+degraded, 81 active+clean; 160 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 6.6 MiB/s rd, 1.7 MiB/s wr, 23 op/s; 14/294 objects degraded (4.762%) 2024-09-21T09:38:10.137 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:10 smithi064 ceph-mon[86709]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2024-09-21T09:38:10.138 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:10 smithi064 ceph-mon[86709]: Health check failed: Degraded data redundancy: 14/294 objects degraded (4.762%), 4 pgs degraded (PG_DEGRADED) 2024-09-21T09:38:10.138 INFO:journalctl@ceph.osd.4.smithi064.stdout:Sep 21 09:38:09 smithi064 podman[92008]: 2024-09-21 09:38:09.834443717 +0000 UTC m=+0.106391614 container create fc53b66cb0e79b0e0523485ca9741477e1d1b9d6187d71a3d318d363cd5359c9 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-4-activate, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-62da54b, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-09-21T09:38:10.138 INFO:journalctl@ceph.osd.4.smithi064.stdout:Sep 21 09:38:09 smithi064 podman[92008]: 2024-09-21 09:38:09.744913754 +0000 UTC m=+0.016861664 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:38:10.138 INFO:journalctl@ceph.osd.4.smithi064.stdout:Sep 21 09:38:10 smithi064 podman[92008]: 2024-09-21 09:38:10.028772735 +0000 UTC m=+0.300720641 container init fc53b66cb0e79b0e0523485ca9741477e1d1b9d6187d71a3d318d363cd5359c9 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-4-activate, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, RELEASE=main-62da54b, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-09-21T09:38:10.139 INFO:journalctl@ceph.osd.4.smithi064.stdout:Sep 21 09:38:10 smithi064 podman[92008]: 2024-09-21 09:38:10.033916473 +0000 UTC m=+0.305864370 container start fc53b66cb0e79b0e0523485ca9741477e1d1b9d6187d71a3d318d363cd5359c9 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-4-activate, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-62da54b, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T09:38:10.139 INFO:journalctl@ceph.osd.4.smithi064.stdout:Sep 21 09:38:10 smithi064 podman[92008]: 2024-09-21 09:38:10.136756352 +0000 UTC m=+0.408704264 container attach fc53b66cb0e79b0e0523485ca9741477e1d1b9d6187d71a3d318d363cd5359c9 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-4-activate, RELEASE=main-62da54b, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-09-21T09:38:10.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:10 smithi022 ceph-mon[98615]: pgmap v304: 121 pgs: 25 active+undersized, 9 peering, 2 stale+active+clean, 4 active+undersized+degraded, 81 active+clean; 160 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 6.6 MiB/s rd, 1.7 MiB/s wr, 23 op/s; 14/294 objects degraded (4.762%) 2024-09-21T09:38:10.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:10 smithi022 ceph-mon[98615]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2024-09-21T09:38:10.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:10 smithi022 ceph-mon[98615]: Health check failed: Degraded data redundancy: 14/294 objects degraded (4.762%), 4 pgs degraded (PG_DEGRADED) 2024-09-21T09:38:10.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:10 smithi022 ceph-mon[96736]: pgmap v304: 121 pgs: 25 active+undersized, 9 peering, 2 stale+active+clean, 4 active+undersized+degraded, 81 active+clean; 160 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 6.6 MiB/s rd, 1.7 MiB/s wr, 23 op/s; 14/294 objects degraded (4.762%) 2024-09-21T09:38:10.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:10 smithi022 ceph-mon[96736]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2024-09-21T09:38:10.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:10 smithi022 ceph-mon[96736]: Health check failed: Degraded data redundancy: 14/294 objects degraded (4.762%), 4 pgs degraded (PG_DEGRADED) 2024-09-21T09:38:10.441 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:38:10 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:38:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:38:10.442 INFO:journalctl@ceph.osd.4.smithi064.stdout:Sep 21 09:38:10 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-4-activate[92024]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T09:38:10.442 INFO:journalctl@ceph.osd.4.smithi064.stdout:Sep 21 09:38:10 smithi064 bash[92008]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T09:38:10.442 INFO:journalctl@ceph.osd.4.smithi064.stdout:Sep 21 09:38:10 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-4-activate[92024]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T09:38:10.442 INFO:journalctl@ceph.osd.4.smithi064.stdout:Sep 21 09:38:10 smithi064 bash[92008]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T09:38:10.936 INFO:journalctl@ceph.osd.4.smithi064.stdout:Sep 21 09:38:10 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-4-activate[92024]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-21T09:38:10.936 INFO:journalctl@ceph.osd.4.smithi064.stdout:Sep 21 09:38:10 smithi064 bash[92008]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-21T09:38:10.936 INFO:journalctl@ceph.osd.4.smithi064.stdout:Sep 21 09:38:10 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-4-activate[92024]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-21T09:38:10.936 INFO:journalctl@ceph.osd.4.smithi064.stdout:Sep 21 09:38:10 smithi064 bash[92008]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-21T09:38:10.936 INFO:journalctl@ceph.osd.4.smithi064.stdout:Sep 21 09:38:10 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-4-activate[92024]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-21T09:38:10.936 INFO:journalctl@ceph.osd.4.smithi064.stdout:Sep 21 09:38:10 smithi064 bash[92008]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-21T09:38:11.067 INFO:tasks.rados.rados.0.smithi022.stdout:2033: done (1 left) 2024-09-21T09:38:11.067 INFO:tasks.rados.rados.0.smithi022.stdout:2034: read oid 28 snap 200 2024-09-21T09:38:11.067 INFO:tasks.rados.rados.0.smithi022.stdout:2034: expect (ObjNum 392 snap 122 seq_num 392) 2024-09-21T09:38:11.067 INFO:tasks.rados.rados.0.smithi022.stdout:2035: snap_remove snap 192 2024-09-21T09:38:11.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:11 smithi064 ceph-mon[86709]: osdmap e469: 8 total, 7 up, 8 in 2024-09-21T09:38:11.191 INFO:journalctl@ceph.osd.4.smithi064.stdout:Sep 21 09:38:10 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-4-activate[92024]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-21T09:38:11.191 INFO:journalctl@ceph.osd.4.smithi064.stdout:Sep 21 09:38:10 smithi064 bash[92008]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-21T09:38:11.192 INFO:journalctl@ceph.osd.4.smithi064.stdout:Sep 21 09:38:10 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-4-activate[92024]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-09-21T09:38:11.192 INFO:journalctl@ceph.osd.4.smithi064.stdout:Sep 21 09:38:10 smithi064 bash[92008]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-09-21T09:38:11.192 INFO:journalctl@ceph.osd.4.smithi064.stdout:Sep 21 09:38:10 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-4-activate[92024]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-21T09:38:11.192 INFO:journalctl@ceph.osd.4.smithi064.stdout:Sep 21 09:38:10 smithi064 bash[92008]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-21T09:38:11.192 INFO:journalctl@ceph.osd.4.smithi064.stdout:Sep 21 09:38:10 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-4-activate[92024]: --> ceph-volume raw activate successful for osd ID: 4 2024-09-21T09:38:11.192 INFO:journalctl@ceph.osd.4.smithi064.stdout:Sep 21 09:38:10 smithi064 bash[92008]: --> ceph-volume raw activate successful for osd ID: 4 2024-09-21T09:38:11.192 INFO:journalctl@ceph.osd.4.smithi064.stdout:Sep 21 09:38:10 smithi064 podman[92008]: 2024-09-21 09:38:10.966306471 +0000 UTC m=+1.238254385 container died fc53b66cb0e79b0e0523485ca9741477e1d1b9d6187d71a3d318d363cd5359c9 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-4-activate, RELEASE=main-62da54b, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, ceph=True, GIT_CLEAN=True) 2024-09-21T09:38:11.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:11 smithi022 ceph-mon[98615]: osdmap e469: 8 total, 7 up, 8 in 2024-09-21T09:38:11.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:11 smithi022 ceph-mon[96736]: osdmap e469: 8 total, 7 up, 8 in 2024-09-21T09:38:11.692 INFO:journalctl@ceph.osd.4.smithi064.stdout:Sep 21 09:38:11 smithi064 podman[92168]: 2024-09-21 09:38:11.534757835 +0000 UTC m=+0.561583029 container remove fc53b66cb0e79b0e0523485ca9741477e1d1b9d6187d71a3d318d363cd5359c9 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-4-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, RELEASE=main-62da54b, org.label-schema.license=GPLv2, ceph=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=) 2024-09-21T09:38:12.029 INFO:journalctl@ceph.osd.4.smithi064.stdout:Sep 21 09:38:11 smithi064 podman[92234]: 2024-09-21 09:38:11.655979639 +0000 UTC m=+0.016850910 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:38:12.029 INFO:journalctl@ceph.osd.4.smithi064.stdout:Sep 21 09:38:11 smithi064 podman[92234]: 2024-09-21 09:38:11.825396128 +0000 UTC m=+0.186267384 container create 936099dbca14a58261a7050c0775f8f44deb5e8910cd89fb2501fc53b38fdd54 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-4, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, RELEASE=main-62da54b, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True) 2024-09-21T09:38:12.068 INFO:tasks.rados.rados.0.smithi022.stdout:2035: done (2 left) 2024-09-21T09:38:12.068 INFO:tasks.rados.rados.0.smithi022.stdout:2036: read oid 35 snap -1 2024-09-21T09:38:12.068 INFO:tasks.rados.rados.0.smithi022.stdout:2036: expect (ObjNum 617 snap 186 seq_num 617) 2024-09-21T09:38:12.068 INFO:tasks.rados.rados.0.smithi022.stdout:2037: copy_from oid 26 from oid 15 current snap is 201 2024-09-21T09:38:12.068 INFO:tasks.rados.rados.0.smithi022.stdout:2032: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-21T09:38:12.071 INFO:tasks.rados.rados.0.smithi022.stdout:2034: expect (ObjNum 392 snap 122 seq_num 392) 2024-09-21T09:38:12.125 INFO:tasks.rados.rados.0.smithi022.stdout:2036: expect (ObjNum 617 snap 186 seq_num 617) 2024-09-21T09:38:12.256 INFO:tasks.rados.rados.0.smithi022.stdout:2037: finishing copy_from racing read to smithi02284960-26 2024-09-21T09:38:12.257 INFO:tasks.rados.rados.0.smithi022.stdout:2037: finishing copy_from to smithi02284960-26 2024-09-21T09:38:12.257 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 26 v 1252 (ObjNum 642 snap 197 seq_num 642) dirty exists 2024-09-21T09:38:12.257 INFO:tasks.rados.rados.0.smithi022.stdout:2032: done (3 left) 2024-09-21T09:38:12.257 INFO:tasks.rados.rados.0.smithi022.stdout:2034: done (2 left) 2024-09-21T09:38:12.257 INFO:tasks.rados.rados.0.smithi022.stdout:2036: done (1 left) 2024-09-21T09:38:12.257 INFO:tasks.rados.rados.0.smithi022.stdout:2037: done (0 left) 2024-09-21T09:38:12.257 INFO:tasks.rados.rados.0.smithi022.stdout:2038: read oid 13 snap -1 2024-09-21T09:38:12.257 INFO:tasks.rados.rados.0.smithi022.stdout:2038: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-21T09:38:12.257 INFO:tasks.rados.rados.0.smithi022.stdout:2039: delete oid 35 current snap is 201 2024-09-21T09:38:12.257 INFO:tasks.rados.rados.0.smithi022.stdout:2038: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-21T09:38:12.259 INFO:tasks.rados.rados.0.smithi022.stdout:2038: done (1 left) 2024-09-21T09:38:12.259 INFO:tasks.rados.rados.0.smithi022.stdout:2039: done (0 left) 2024-09-21T09:38:12.260 INFO:tasks.rados.rados.0.smithi022.stdout:2040: setattr oid 45 current snap is 201 2024-09-21T09:38:12.260 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 5 current snap is 201 2024-09-21T09:38:12.261 INFO:tasks.rados.rados.0.smithi022.stdout:2041: seq_num 653 ranges {8380416=540672,8921088=696320,9617408=106496} 2024-09-21T09:38:12.309 INFO:tasks.rados.rados.0.smithi022.stdout:2041: writing smithi02284960-5 from 8380416 to 8921088 tid 1 2024-09-21T09:38:12.313 INFO:tasks.rados.rados.0.smithi022.stdout:2041: writing smithi02284960-5 from 8921088 to 9617408 tid 2 2024-09-21T09:38:12.314 INFO:tasks.rados.rados.0.smithi022.stdout:2041: writing smithi02284960-5 from 9617408 to 9723904 tid 3 2024-09-21T09:38:12.314 INFO:tasks.rados.rados.0.smithi022.stdout:2042: snap_create 2024-09-21T09:38:12.314 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 45 v 1105 (ObjNum 647 snap 199 seq_num 647) dirty exists 2024-09-21T09:38:12.314 INFO:tasks.rados.rados.0.smithi022.stdout:2041: finishing write tid 1 to smithi02284960-5 2024-09-21T09:38:12.316 INFO:tasks.rados.rados.0.smithi022.stdout:2041: finishing write tid 2 to smithi02284960-5 2024-09-21T09:38:12.317 INFO:tasks.rados.rados.0.smithi022.stdout:2041: finishing write tid 3 to smithi02284960-5 2024-09-21T09:38:12.317 INFO:tasks.rados.rados.0.smithi022.stdout:2041: finishing write tid 4 to smithi02284960-5 2024-09-21T09:38:12.318 INFO:tasks.rados.rados.0.smithi022.stdout:2041: finishing write tid 5 to smithi02284960-5 2024-09-21T09:38:12.318 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 5 v 707 (ObjNum 653 snap 201 seq_num 653) dirty exists 2024-09-21T09:38:12.318 INFO:tasks.rados.rados.0.smithi022.stdout:2041: left oid 5 (ObjNum 653 snap 201 seq_num 653) 2024-09-21T09:38:12.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:12 smithi064 ceph-mon[86709]: pgmap v307: 121 pgs: 26 active+undersized, 9 peering, 1 stale+active+clean, 5 active+undersized+degraded, 80 active+clean; 170 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.8 MiB/s rd, 3.8 MiB/s wr, 21 op/s; 18/300 objects degraded (6.000%) 2024-09-21T09:38:12.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:12 smithi064 ceph-mon[86709]: osdmap e470: 8 total, 7 up, 8 in 2024-09-21T09:38:12.441 INFO:journalctl@ceph.osd.4.smithi064.stdout:Sep 21 09:38:12 smithi064 podman[92234]: 2024-09-21 09:38:12.028270509 +0000 UTC m=+0.389141779 container init 936099dbca14a58261a7050c0775f8f44deb5e8910cd89fb2501fc53b38fdd54 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-4, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_CLEAN=True, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, RELEASE=main-62da54b, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T09:38:12.442 INFO:journalctl@ceph.osd.4.smithi064.stdout:Sep 21 09:38:12 smithi064 podman[92234]: 2024-09-21 09:38:12.033620726 +0000 UTC m=+0.394491982 container start 936099dbca14a58261a7050c0775f8f44deb5e8910cd89fb2501fc53b38fdd54 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-4, RELEASE=main-62da54b, org.label-schema.license=GPLv2, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-09-21T09:38:12.442 INFO:journalctl@ceph.osd.4.smithi064.stdout:Sep 21 09:38:12 smithi064 ceph-osd[92272]: -- 172.21.15.64:0/2418612425 <== mon.1 v2:172.21.15.64:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x563388419a40 con 0x5633883f7c00 2024-09-21T09:38:12.442 INFO:journalctl@ceph.osd.4.smithi064.stdout:Sep 21 09:38:12 smithi064 bash[92234]: 936099dbca14a58261a7050c0775f8f44deb5e8910cd89fb2501fc53b38fdd54 2024-09-21T09:38:12.442 INFO:journalctl@ceph.osd.4.smithi064.stdout:Sep 21 09:38:12 smithi064 systemd[1]: Started Ceph osd.4 for d65e801e-77fa-11ef-baf6-efdc52797490. 2024-09-21T09:38:12.557 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:12 smithi022 ceph-mon[98615]: pgmap v307: 121 pgs: 26 active+undersized, 9 peering, 1 stale+active+clean, 5 active+undersized+degraded, 80 active+clean; 170 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.8 MiB/s rd, 3.8 MiB/s wr, 21 op/s; 18/300 objects degraded (6.000%) 2024-09-21T09:38:12.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:12 smithi022 ceph-mon[98615]: osdmap e470: 8 total, 7 up, 8 in 2024-09-21T09:38:12.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:12 smithi022 ceph-mon[96736]: pgmap v307: 121 pgs: 26 active+undersized, 9 peering, 1 stale+active+clean, 5 active+undersized+degraded, 80 active+clean; 170 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.8 MiB/s rd, 3.8 MiB/s wr, 21 op/s; 18/300 objects degraded (6.000%) 2024-09-21T09:38:12.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:12 smithi022 ceph-mon[96736]: osdmap e470: 8 total, 7 up, 8 in 2024-09-21T09:38:13.227 INFO:tasks.rados.rados.0.smithi022.stdout:2040: done (2 left) 2024-09-21T09:38:13.227 INFO:tasks.rados.rados.0.smithi022.stdout:2041: done (1 left) 2024-09-21T09:38:13.227 INFO:tasks.rados.rados.0.smithi022.stdout:2042: done (0 left) 2024-09-21T09:38:13.227 INFO:tasks.rados.rados.0.smithi022.stdout:2043: read oid 17 snap -1 2024-09-21T09:38:13.227 INFO:tasks.rados.rados.0.smithi022.stdout:2043: expect (ObjNum 575 snap 172 seq_num 575) 2024-09-21T09:38:13.227 INFO:tasks.rados.rados.0.smithi022.stdout:2044: snap_remove snap 193 2024-09-21T09:38:13.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:13 smithi064 ceph-mon[86709]: osdmap e471: 8 total, 7 up, 8 in 2024-09-21T09:38:13.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:13 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:38:13.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:13 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:38:13.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:13 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:38:13.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:13 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:38:13.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:13 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:38:13.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:13 smithi022 ceph-mon[98615]: osdmap e471: 8 total, 7 up, 8 in 2024-09-21T09:38:13.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:13 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:38:13.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:13 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:38:13.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:13 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:38:13.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:13 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:38:13.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:13 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:38:13.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:13 smithi022 ceph-mon[96736]: osdmap e471: 8 total, 7 up, 8 in 2024-09-21T09:38:13.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:13 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:38:13.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:13 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:38:13.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:13 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:38:13.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:13 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:38:13.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:13 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:38:13.727 INFO:journalctl@ceph.osd.4.smithi064.stdout:Sep 21 09:38:13 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-4[92249]: 2024-09-21T09:38:13.442+0000 7ff33628d640 -1 Falling back to public interface 2024-09-21T09:38:14.228 INFO:tasks.rados.rados.0.smithi022.stdout:2044: done (1 left) 2024-09-21T09:38:14.228 INFO:tasks.rados.rados.0.smithi022.stdout:2045: snap_remove snap 195 2024-09-21T09:38:14.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:14 smithi064 ceph-mon[86709]: pgmap v310: 121 pgs: 29 active+undersized, 9 peering, 9 active+undersized+degraded, 74 active+clean; 170 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.5 MiB/s rd, 2.4 MiB/s wr, 3 op/s; 24/300 objects degraded (8.000%) 2024-09-21T09:38:14.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:14 smithi064 ceph-mon[86709]: osdmap e472: 8 total, 7 up, 8 in 2024-09-21T09:38:14.442 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:38:14 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:38:14.148Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.4\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.4\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.4\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:38:14.553 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:14 smithi022 ceph-mon[98615]: pgmap v310: 121 pgs: 29 active+undersized, 9 peering, 9 active+undersized+degraded, 74 active+clean; 170 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.5 MiB/s rd, 2.4 MiB/s wr, 3 op/s; 24/300 objects degraded (8.000%) 2024-09-21T09:38:14.554 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:14 smithi022 ceph-mon[98615]: osdmap e472: 8 total, 7 up, 8 in 2024-09-21T09:38:14.554 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:14 smithi022 ceph-mon[96736]: pgmap v310: 121 pgs: 29 active+undersized, 9 peering, 9 active+undersized+degraded, 74 active+clean; 170 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.5 MiB/s rd, 2.4 MiB/s wr, 3 op/s; 24/300 objects degraded (8.000%) 2024-09-21T09:38:14.554 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:14 smithi022 ceph-mon[96736]: osdmap e472: 8 total, 7 up, 8 in 2024-09-21T09:38:14.554 INFO:teuthology.orchestra.run.smithi022.stdout:true 2024-09-21T09:38:15.134 INFO:teuthology.orchestra.run.smithi022.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T09:38:15.134 INFO:teuthology.orchestra.run.smithi022.stdout:alertmanager.a smithi022 *:9093,9094 running (4m) 32s ago 9m 22.0M - 0.27.0 11f11916f8cd caced80ba544 2024-09-21T09:38:15.134 INFO:teuthology.orchestra.run.smithi022.stdout:grafana.a smithi064 *:3000 running (4m) 2m ago 9m 77.9M - 9.4.7 954c08fa6188 527bb657d70d 2024-09-21T09:38:15.134 INFO:teuthology.orchestra.run.smithi022.stdout:mds.a smithi022 running (10m) 32s ago 10m 18.1M - 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 34e4221f3a27 2024-09-21T09:38:15.134 INFO:teuthology.orchestra.run.smithi022.stdout:mds.b smithi064 running (10m) 2m ago 10m 20.9M - 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 0f846b3ec694 2024-09-21T09:38:15.134 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.x smithi064 *:8443,9283,8765 running (3m) 2m ago 14m 453M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 61abc61fea8d 2024-09-21T09:38:15.134 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.y smithi022 *:8443,9283,8765 running (4m) 32s ago 16m 517M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 6c5bee78f09b 2024-09-21T09:38:15.135 INFO:teuthology.orchestra.run.smithi022.stdout:mon.a smithi022 running (3m) 32s ago 16m 58.2M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 0513395c784d 2024-09-21T09:38:15.135 INFO:teuthology.orchestra.run.smithi022.stdout:mon.b smithi064 running (3m) 2m ago 14m 31.5M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 9dab26b6a9e7 2024-09-21T09:38:15.135 INFO:teuthology.orchestra.run.smithi022.stdout:mon.c smithi022 running (3m) 32s ago 14m 49.3M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 c74b58b67bd6 2024-09-21T09:38:15.135 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.a smithi022 *:9100 running (4m) 32s ago 9m 11.1M - 1.7.0 72c9c2088986 593c6158f524 2024-09-21T09:38:15.135 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.b smithi064 *:9100 running (4m) 2m ago 9m 9.94M - 1.7.0 72c9c2088986 a533420b0139 2024-09-21T09:38:15.135 INFO:teuthology.orchestra.run.smithi022.stdout:osd.0 smithi022 running (2m) 32s ago 14m 141M 2481M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 40ce23a2b569 2024-09-21T09:38:15.135 INFO:teuthology.orchestra.run.smithi022.stdout:osd.1 smithi022 running (105s) 32s ago 13m 115M 2481M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 7cd0c18d224c 2024-09-21T09:38:15.135 INFO:teuthology.orchestra.run.smithi022.stdout:osd.2 smithi022 running (71s) 32s ago 13m 108M 2481M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 b8c944306698 2024-09-21T09:38:15.135 INFO:teuthology.orchestra.run.smithi022.stdout:osd.3 smithi022 running (37s) 32s ago 12m 12.2M 2481M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 c90b27f61bb6 2024-09-21T09:38:15.135 INFO:teuthology.orchestra.run.smithi022.stdout:osd.4 smithi064 starting - - - 2480M 2024-09-21T09:38:15.135 INFO:teuthology.orchestra.run.smithi022.stdout:osd.5 smithi064 running (12m) 2m ago 12m 192M 2480M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 1900c4af5d28 2024-09-21T09:38:15.135 INFO:teuthology.orchestra.run.smithi022.stdout:osd.6 smithi064 running (11m) 2m ago 11m 126M 2480M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 3cc4ec816e5b 2024-09-21T09:38:15.135 INFO:teuthology.orchestra.run.smithi022.stdout:osd.7 smithi064 running (11m) 2m ago 11m 148M 2480M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 18ef0be94104 2024-09-21T09:38:15.136 INFO:teuthology.orchestra.run.smithi022.stdout:prometheus.a smithi064 *:9095 running (4m) 2m ago 9m 44.0M - 2.51.0 1d3b7f56885b 80629ffa2179 2024-09-21T09:38:15.233 INFO:tasks.rados.rados.0.smithi022.stdout:2045: done (1 left) 2024-09-21T09:38:15.234 INFO:tasks.rados.rados.0.smithi022.stdout:2046: rmattr oid 32 current snap is 202 2024-09-21T09:38:15.234 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 20 current snap is 202 2024-09-21T09:38:15.234 INFO:tasks.rados.rados.0.smithi022.stdout:2047: seq_num 654 ranges {8118272=557056,8675328=630784,9306112=172032} 2024-09-21T09:38:15.281 INFO:tasks.rados.rados.0.smithi022.stdout:2047: writing smithi02284960-20 from 8118272 to 8675328 tid 1 2024-09-21T09:38:15.285 INFO:tasks.rados.rados.0.smithi022.stdout:2047: writing smithi02284960-20 from 8675328 to 9306112 tid 2 2024-09-21T09:38:15.287 INFO:tasks.rados.rados.0.smithi022.stdout:2047: writing smithi02284960-20 from 9306112 to 9478144 tid 3 2024-09-21T09:38:15.287 INFO:tasks.rados.rados.0.smithi022.stdout:2048: read oid 25 snap -1 2024-09-21T09:38:15.287 INFO:tasks.rados.rados.0.smithi022.stdout:2048: expect (ObjNum 624 snap 186 seq_num 624) 2024-09-21T09:38:15.287 INFO:tasks.rados.rados.0.smithi022.stdout:2049: copy_from oid 35 from oid 19 current snap is 202 2024-09-21T09:38:15.287 INFO:tasks.rados.rados.0.smithi022.stdout:2050: read oid 9 snap -1 2024-09-21T09:38:15.288 INFO:tasks.rados.rados.0.smithi022.stdout:2050: expect deleted 2024-09-21T09:38:15.288 INFO:tasks.rados.rados.0.smithi022.stdout:2051: rollback oid 36 current snap is 202 2024-09-21T09:38:15.288 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 36 to 197 2024-09-21T09:38:15.288 INFO:tasks.rados.rados.0.smithi022.stdout:2043: expect (ObjNum 575 snap 172 seq_num 575) 2024-09-21T09:38:15.505 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-09-21T09:38:15.506 INFO:teuthology.orchestra.run.smithi022.stdout: "mon": { 2024-09-21T09:38:15.506 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 3 2024-09-21T09:38:15.506 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-21T09:38:15.506 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr": { 2024-09-21T09:38:15.506 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 2 2024-09-21T09:38:15.506 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-21T09:38:15.506 INFO:teuthology.orchestra.run.smithi022.stdout: "osd": { 2024-09-21T09:38:15.506 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 3, 2024-09-21T09:38:15.506 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 4 2024-09-21T09:38:15.506 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-21T09:38:15.506 INFO:teuthology.orchestra.run.smithi022.stdout: "mds": { 2024-09-21T09:38:15.506 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 2 2024-09-21T09:38:15.506 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-21T09:38:15.507 INFO:teuthology.orchestra.run.smithi022.stdout: "overall": { 2024-09-21T09:38:15.507 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 5, 2024-09-21T09:38:15.507 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 9 2024-09-21T09:38:15.507 INFO:teuthology.orchestra.run.smithi022.stdout: } 2024-09-21T09:38:15.507 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-09-21T09:38:15.507 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:15 smithi022 ceph-mon[98615]: osdmap e473: 8 total, 7 up, 8 in 2024-09-21T09:38:15.507 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:15 smithi022 ceph-mon[98615]: pgmap v313: 121 pgs: 34 active+undersized, 13 active+undersized+degraded, 74 active+clean; 174 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.1 MiB/s rd, 1.1 MiB/s wr, 11 op/s; 35/309 objects degraded (11.327%) 2024-09-21T09:38:15.507 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:15 smithi022 ceph-mon[98615]: from='client.44279 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:15.507 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:15 smithi022 ceph-mon[98615]: from='client.34359 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:15.507 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:15 smithi022 ceph-mon[96736]: osdmap e473: 8 total, 7 up, 8 in 2024-09-21T09:38:15.508 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:15 smithi022 ceph-mon[96736]: pgmap v313: 121 pgs: 34 active+undersized, 13 active+undersized+degraded, 74 active+clean; 174 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.1 MiB/s rd, 1.1 MiB/s wr, 11 op/s; 35/309 objects degraded (11.327%) 2024-09-21T09:38:15.508 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:15 smithi022 ceph-mon[96736]: from='client.44279 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:15.508 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:15 smithi022 ceph-mon[96736]: from='client.34359 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:15.558 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 32 v 1214 (ObjNum 648 snap 199 seq_num 648) dirty exists 2024-09-21T09:38:15.559 INFO:tasks.rados.rados.0.smithi022.stdout:2047: finishing write tid 1 to smithi02284960-20 2024-09-21T09:38:15.559 INFO:tasks.rados.rados.0.smithi022.stdout:2043: done (6 left) 2024-09-21T09:38:15.559 INFO:tasks.rados.rados.0.smithi022.stdout:2046: done (5 left) 2024-09-21T09:38:15.559 INFO:tasks.rados.rados.0.smithi022.stdout:2052: read oid 6 snap -1 2024-09-21T09:38:15.559 INFO:tasks.rados.rados.0.smithi022.stdout:2052: expect (ObjNum 423 snap 132 seq_num 423) 2024-09-21T09:38:15.559 INFO:tasks.rados.rados.0.smithi022.stdout:2053: snap_create 2024-09-21T09:38:15.559 INFO:tasks.rados.rados.0.smithi022.stdout:2047: finishing write tid 2 to smithi02284960-20 2024-09-21T09:38:15.559 INFO:tasks.rados.rados.0.smithi022.stdout:2051: finishing rollback tid 0 to smithi02284960-36 2024-09-21T09:38:15.559 INFO:tasks.rados.rados.0.smithi022.stdout:2049: finishing copy_from to smithi02284960-35 2024-09-21T09:38:15.559 INFO:tasks.rados.rados.0.smithi022.stdout:2049: got expected ENOENT (src dne) 2024-09-21T09:38:15.559 INFO:tasks.rados.rados.0.smithi022.stdout:2049: finishing copy_from racing read to smithi02284960-35 2024-09-21T09:38:15.559 INFO:tasks.rados.rados.0.smithi022.stdout:2047: finishing write tid 3 to smithi02284960-20 2024-09-21T09:38:15.559 INFO:tasks.rados.rados.0.smithi022.stdout:2048: expect (ObjNum 624 snap 186 seq_num 624) 2024-09-21T09:38:15.586 INFO:tasks.rados.rados.0.smithi022.stdout:2051: finishing rollback tid 1 to smithi02284960-36 2024-09-21T09:38:15.587 INFO:tasks.rados.rados.0.smithi022.stdout:2051: finishing rollback tid 2 to smithi02284960-36 2024-09-21T09:38:15.587 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 36 v 1108 (ObjNum 580 snap 172 seq_num 580) dirty exists 2024-09-21T09:38:15.587 INFO:tasks.rados.rados.0.smithi022.stdout:2047: finishing write tid 4 to smithi02284960-20 2024-09-21T09:38:15.587 INFO:tasks.rados.rados.0.smithi022.stdout:2047: finishing write tid 5 to smithi02284960-20 2024-09-21T09:38:15.587 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 20 v 1223 (ObjNum 654 snap 202 seq_num 654) dirty exists 2024-09-21T09:38:15.587 INFO:tasks.rados.rados.0.smithi022.stdout:2047: left oid 20 (ObjNum 654 snap 202 seq_num 654) 2024-09-21T09:38:15.587 INFO:tasks.rados.rados.0.smithi022.stdout:2052: expect (ObjNum 423 snap 132 seq_num 423) 2024-09-21T09:38:15.604 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:15 smithi064 ceph-mon[86709]: osdmap e473: 8 total, 7 up, 8 in 2024-09-21T09:38:15.604 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:15 smithi064 ceph-mon[86709]: pgmap v313: 121 pgs: 34 active+undersized, 13 active+undersized+degraded, 74 active+clean; 174 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.1 MiB/s rd, 1.1 MiB/s wr, 11 op/s; 35/309 objects degraded (11.327%) 2024-09-21T09:38:15.605 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:15 smithi064 ceph-mon[86709]: from='client.44279 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:15.605 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:15 smithi064 ceph-mon[86709]: from='client.34359 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:15.808 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:38:15 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:38:15] "GET /metrics HTTP/1.1" 200 36305 "" "Prometheus/2.51.0" 2024-09-21T09:38:15.808 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-09-21T09:38:15.808 INFO:teuthology.orchestra.run.smithi022.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47", 2024-09-21T09:38:15.808 INFO:teuthology.orchestra.run.smithi022.stdout: "in_progress": true, 2024-09-21T09:38:15.808 INFO:teuthology.orchestra.run.smithi022.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T09:38:15.808 INFO:teuthology.orchestra.run.smithi022.stdout: "services_complete": [ 2024-09-21T09:38:15.808 INFO:teuthology.orchestra.run.smithi022.stdout: "mon", 2024-09-21T09:38:15.808 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr" 2024-09-21T09:38:15.808 INFO:teuthology.orchestra.run.smithi022.stdout: ], 2024-09-21T09:38:15.808 INFO:teuthology.orchestra.run.smithi022.stdout: "progress": "9/20 daemons upgraded", 2024-09-21T09:38:15.808 INFO:teuthology.orchestra.run.smithi022.stdout: "message": "Currently upgrading osd daemons", 2024-09-21T09:38:15.809 INFO:teuthology.orchestra.run.smithi022.stdout: "is_paused": false 2024-09-21T09:38:15.809 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-09-21T09:38:15.941 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:38:15 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:38:15.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:38:15.942 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:38:15 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:38:15.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:38:16.238 INFO:tasks.rados.rados.0.smithi022.stdout:2047: done (6 left) 2024-09-21T09:38:16.238 INFO:tasks.rados.rados.0.smithi022.stdout:2048: done (5 left) 2024-09-21T09:38:16.238 INFO:tasks.rados.rados.0.smithi022.stdout:2049: done (4 left) 2024-09-21T09:38:16.239 INFO:tasks.rados.rados.0.smithi022.stdout:2050: done (3 left) 2024-09-21T09:38:16.239 INFO:tasks.rados.rados.0.smithi022.stdout:2051: done (2 left) 2024-09-21T09:38:16.239 INFO:tasks.rados.rados.0.smithi022.stdout:2052: done (1 left) 2024-09-21T09:38:16.239 INFO:tasks.rados.rados.0.smithi022.stdout:2053: done (0 left) 2024-09-21T09:38:16.239 INFO:tasks.rados.rados.0.smithi022.stdout:2054: delete oid 2 current snap is 203 2024-09-21T09:38:16.241 INFO:tasks.rados.rados.0.smithi022.stdout:2054: done (0 left) 2024-09-21T09:38:16.241 INFO:tasks.rados.rados.0.smithi022.stdout:2055: copy_from oid 29 from oid 30 current snap is 203 2024-09-21T09:38:16.241 INFO:tasks.rados.rados.0.smithi022.stdout:2056: read oid 1 snap 199 2024-09-21T09:38:16.242 INFO:tasks.rados.rados.0.smithi022.stdout:2056: expect (ObjNum 627 snap 189 seq_num 627) 2024-09-21T09:38:16.242 INFO:tasks.rados.rados.0.smithi022.stdout:2057: rollback oid 17 current snap is 203 2024-09-21T09:38:16.242 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 17 to 194 2024-09-21T09:38:16.242 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 49 current snap is 203 2024-09-21T09:38:16.242 INFO:tasks.rados.rados.0.smithi022.stdout:2058: seq_num 656 ranges {9101312=589824,9691136=802816} 2024-09-21T09:38:16.295 INFO:tasks.rados.rados.0.smithi022.stdout:2058: writing smithi02284960-49 from 9101312 to 9691136 tid 1 2024-09-21T09:38:16.300 INFO:tasks.rados.rados.0.smithi022.stdout:2058: writing smithi02284960-49 from 9691136 to 10493952 tid 2 2024-09-21T09:38:16.300 INFO:tasks.rados.rados.0.smithi022.stdout:2059: setattr oid 5 current snap is 203 2024-09-21T09:38:16.300 INFO:tasks.rados.rados.0.smithi022.stdout:2057: finishing rollback tid 0 to smithi02284960-17 2024-09-21T09:38:16.301 INFO:tasks.rados.rados.0.smithi022.stdout:2057: finishing rollback tid 1 to smithi02284960-17 2024-09-21T09:38:16.301 INFO:tasks.rados.rados.0.smithi022.stdout:2057: finishing rollback tid 2 to smithi02284960-17 2024-09-21T09:38:16.301 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 17 v 1047 (ObjNum 575 snap 172 seq_num 575) dirty exists 2024-09-21T09:38:16.301 INFO:tasks.rados.rados.0.smithi022.stdout:2055: finishing copy_from racing read to smithi02284960-29 2024-09-21T09:38:16.301 INFO:tasks.rados.rados.0.smithi022.stdout:2056: expect (ObjNum 627 snap 189 seq_num 627) 2024-09-21T09:38:16.485 INFO:tasks.rados.rados.0.smithi022.stdout:2055: finishing copy_from to smithi02284960-29 2024-09-21T09:38:16.485 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 29 v 872 (ObjNum 622 snap 186 seq_num 622) dirty exists 2024-09-21T09:38:16.485 INFO:tasks.rados.rados.0.smithi022.stdout:2058: finishing write tid 1 to smithi02284960-49 2024-09-21T09:38:16.485 INFO:tasks.rados.rados.0.smithi022.stdout:2058: finishing write tid 2 to smithi02284960-49 2024-09-21T09:38:16.485 INFO:tasks.rados.rados.0.smithi022.stdout:2058: finishing write tid 3 to smithi02284960-49 2024-09-21T09:38:16.486 INFO:tasks.rados.rados.0.smithi022.stdout:2058: finishing write tid 4 to smithi02284960-49 2024-09-21T09:38:16.486 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 49 v 1093 (ObjNum 656 snap 203 seq_num 656) dirty exists 2024-09-21T09:38:16.486 INFO:tasks.rados.rados.0.smithi022.stdout:2058: left oid 49 (ObjNum 656 snap 203 seq_num 656) 2024-09-21T09:38:16.486 INFO:tasks.rados.rados.0.smithi022.stdout:2055: done (4 left) 2024-09-21T09:38:16.486 INFO:tasks.rados.rados.0.smithi022.stdout:2056: done (3 left) 2024-09-21T09:38:16.486 INFO:tasks.rados.rados.0.smithi022.stdout:2057: done (2 left) 2024-09-21T09:38:16.486 INFO:tasks.rados.rados.0.smithi022.stdout:2058: done (1 left) 2024-09-21T09:38:16.486 INFO:tasks.rados.rados.0.smithi022.stdout:2060: read oid 41 snap 200 2024-09-21T09:38:16.486 INFO:tasks.rados.rados.0.smithi022.stdout:2060: expect (ObjNum 531 snap 159 seq_num 531) 2024-09-21T09:38:16.486 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 10 current snap is 203 2024-09-21T09:38:16.486 INFO:tasks.rados.rados.0.smithi022.stdout:2061: seq_num 657 ranges {3702784=606208,4308992=688128,4997120=114688} 2024-09-21T09:38:16.489 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:16 smithi064 ceph-mon[86709]: from='client.44291 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:16.489 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:16 smithi064 ceph-mon[86709]: Health check update: Degraded data redundancy: 35/309 objects degraded (11.327%), 13 pgs degraded (PG_DEGRADED) 2024-09-21T09:38:16.489 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:16 smithi064 ceph-mon[86709]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2024-09-21T09:38:16.490 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:16 smithi064 ceph-mon[86709]: osdmap e474: 8 total, 7 up, 8 in 2024-09-21T09:38:16.490 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:16 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/3543380694' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:38:16.490 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:16 smithi064 ceph-mon[86709]: from='client.44303 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:16.508 INFO:tasks.rados.rados.0.smithi022.stdout:2061: writing smithi02284960-10 from 3702784 to 4308992 tid 1 2024-09-21T09:38:16.512 INFO:tasks.rados.rados.0.smithi022.stdout:2061: writing smithi02284960-10 from 4308992 to 4997120 tid 2 2024-09-21T09:38:16.513 INFO:tasks.rados.rados.0.smithi022.stdout:2061: writing smithi02284960-10 from 4997120 to 5111808 tid 3 2024-09-21T09:38:16.513 INFO:tasks.rados.rados.0.smithi022.stdout:2062: read oid 37 snap -1 2024-09-21T09:38:16.513 INFO:tasks.rados.rados.0.smithi022.stdout:2062: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-21T09:38:16.513 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 40 current snap is 203 2024-09-21T09:38:16.513 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 5 v 721 (ObjNum 653 snap 201 seq_num 653) dirty exists 2024-09-21T09:38:16.513 INFO:tasks.rados.rados.0.smithi022.stdout:2060: expect (ObjNum 531 snap 159 seq_num 531) 2024-09-21T09:38:16.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:16 smithi022 ceph-mon[98615]: from='client.44291 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:16.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:16 smithi022 ceph-mon[98615]: Health check update: Degraded data redundancy: 35/309 objects degraded (11.327%), 13 pgs degraded (PG_DEGRADED) 2024-09-21T09:38:16.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:16 smithi022 ceph-mon[98615]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2024-09-21T09:38:16.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:16 smithi022 ceph-mon[98615]: osdmap e474: 8 total, 7 up, 8 in 2024-09-21T09:38:16.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:16 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/3543380694' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:38:16.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:16 smithi022 ceph-mon[98615]: from='client.44303 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:16.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:16 smithi022 ceph-mon[96736]: from='client.44291 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:16.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:16 smithi022 ceph-mon[96736]: Health check update: Degraded data redundancy: 35/309 objects degraded (11.327%), 13 pgs degraded (PG_DEGRADED) 2024-09-21T09:38:16.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:16 smithi022 ceph-mon[96736]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2024-09-21T09:38:16.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:16 smithi022 ceph-mon[96736]: osdmap e474: 8 total, 7 up, 8 in 2024-09-21T09:38:16.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:16 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/3543380694' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:38:16.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:16 smithi022 ceph-mon[96736]: from='client.44303 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:16.561 INFO:tasks.rados.rados.0.smithi022.stdout:2061: finishing write tid 1 to smithi02284960-10 2024-09-21T09:38:16.562 INFO:tasks.rados.rados.0.smithi022.stdout:2062: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-21T09:38:16.563 INFO:tasks.rados.rados.0.smithi022.stdout:2061: finishing write tid 2 to smithi02284960-10 2024-09-21T09:38:16.563 INFO:tasks.rados.rados.0.smithi022.stdout:2061: finishing write tid 3 to smithi02284960-10 2024-09-21T09:38:16.563 INFO:tasks.rados.rados.0.smithi022.stdout:2061: finishing write tid 4 to smithi02284960-10 2024-09-21T09:38:16.564 INFO:tasks.rados.rados.0.smithi022.stdout:2061: finishing write tid 5 to smithi02284960-10 2024-09-21T09:38:16.564 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 10 v 1085 (ObjNum 657 snap 203 seq_num 657) dirty exists 2024-09-21T09:38:16.564 INFO:tasks.rados.rados.0.smithi022.stdout:2061: left oid 10 (ObjNum 657 snap 203 seq_num 657) 2024-09-21T09:38:16.564 INFO:tasks.rados.rados.0.smithi022.stdout:2063: seq_num 658 ranges {860160=622592,1482752=565248,2048000=237568} 2024-09-21T09:38:16.571 INFO:tasks.rados.rados.0.smithi022.stdout:2063: writing smithi02284960-40 from 860160 to 1482752 tid 1 2024-09-21T09:38:16.574 INFO:tasks.rados.rados.0.smithi022.stdout:2063: writing smithi02284960-40 from 1482752 to 2048000 tid 2 2024-09-21T09:38:16.576 INFO:tasks.rados.rados.0.smithi022.stdout:2063: writing smithi02284960-40 from 2048000 to 2285568 tid 3 2024-09-21T09:38:16.576 INFO:tasks.rados.rados.0.smithi022.stdout:2059: done (4 left) 2024-09-21T09:38:16.577 INFO:tasks.rados.rados.0.smithi022.stdout:2060: done (3 left) 2024-09-21T09:38:16.577 INFO:tasks.rados.rados.0.smithi022.stdout:2061: done (2 left) 2024-09-21T09:38:16.577 INFO:tasks.rados.rados.0.smithi022.stdout:2062: done (1 left) 2024-09-21T09:38:16.577 INFO:tasks.rados.rados.0.smithi022.stdout:2064: read oid 27 snap -1 2024-09-21T09:38:16.577 INFO:tasks.rados.rados.0.smithi022.stdout:2064: expect (ObjNum 590 snap 176 seq_num 590) 2024-09-21T09:38:16.577 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 20 current snap is 203 2024-09-21T09:38:16.577 INFO:tasks.rados.rados.0.smithi022.stdout:2063: finishing write tid 1 to smithi02284960-40 2024-09-21T09:38:16.577 INFO:tasks.rados.rados.0.smithi022.stdout:2065: seq_num 659 ranges {9478144=647168,10125312=450560,10575872=352256} 2024-09-21T09:38:16.631 INFO:tasks.rados.rados.0.smithi022.stdout:2065: writing smithi02284960-20 from 9478144 to 10125312 tid 1 2024-09-21T09:38:16.634 INFO:tasks.rados.rados.0.smithi022.stdout:2065: writing smithi02284960-20 from 10125312 to 10575872 tid 2 2024-09-21T09:38:16.637 INFO:tasks.rados.rados.0.smithi022.stdout:2065: writing smithi02284960-20 from 10575872 to 10928128 tid 3 2024-09-21T09:38:16.637 INFO:tasks.rados.rados.0.smithi022.stdout:2066: read oid 31 snap -1 2024-09-21T09:38:16.637 INFO:tasks.rados.rados.0.smithi022.stdout:2066: expect (ObjNum 590 snap 176 seq_num 590) 2024-09-21T09:38:16.637 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 4 current snap is 203 2024-09-21T09:38:16.637 INFO:tasks.rados.rados.0.smithi022.stdout:2063: finishing write tid 2 to smithi02284960-40 2024-09-21T09:38:16.637 INFO:tasks.rados.rados.0.smithi022.stdout:2063: finishing write tid 3 to smithi02284960-40 2024-09-21T09:38:16.637 INFO:tasks.rados.rados.0.smithi022.stdout:2063: finishing write tid 4 to smithi02284960-40 2024-09-21T09:38:16.637 INFO:tasks.rados.rados.0.smithi022.stdout:2063: finishing write tid 5 to smithi02284960-40 2024-09-21T09:38:16.637 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 40 v 1090 (ObjNum 658 snap 203 seq_num 658) dirty exists 2024-09-21T09:38:16.637 INFO:tasks.rados.rados.0.smithi022.stdout:2063: left oid 40 (ObjNum 658 snap 203 seq_num 658) 2024-09-21T09:38:16.637 INFO:tasks.rados.rados.0.smithi022.stdout:2064: expect (ObjNum 590 snap 176 seq_num 590) 2024-09-21T09:38:16.647 INFO:tasks.rados.rados.0.smithi022.stdout:2065: finishing write tid 1 to smithi02284960-20 2024-09-21T09:38:16.648 INFO:tasks.rados.rados.0.smithi022.stdout:2065: finishing write tid 2 to smithi02284960-20 2024-09-21T09:38:16.648 INFO:tasks.rados.rados.0.smithi022.stdout:2065: finishing write tid 3 to smithi02284960-20 2024-09-21T09:38:16.648 INFO:tasks.rados.rados.0.smithi022.stdout:2066: expect (ObjNum 590 snap 176 seq_num 590) 2024-09-21T09:38:16.657 INFO:tasks.rados.rados.0.smithi022.stdout:2065: finishing write tid 4 to smithi02284960-20 2024-09-21T09:38:16.657 INFO:tasks.rados.rados.0.smithi022.stdout:2065: finishing write tid 5 to smithi02284960-20 2024-09-21T09:38:16.657 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 20 v 1228 (ObjNum 659 snap 203 seq_num 659) dirty exists 2024-09-21T09:38:16.657 INFO:tasks.rados.rados.0.smithi022.stdout:2065: left oid 20 (ObjNum 659 snap 203 seq_num 659) 2024-09-21T09:38:16.657 INFO:tasks.rados.rados.0.smithi022.stdout:2067: seq_num 660 ranges {8454144=663552,9117696=737280,9854976=65536} 2024-09-21T09:38:16.706 INFO:tasks.rados.rados.0.smithi022.stdout:2067: writing smithi02284960-4 from 8454144 to 9117696 tid 1 2024-09-21T09:38:16.710 INFO:tasks.rados.rados.0.smithi022.stdout:2067: writing smithi02284960-4 from 9117696 to 9854976 tid 2 2024-09-21T09:38:16.711 INFO:tasks.rados.rados.0.smithi022.stdout:2067: writing smithi02284960-4 from 9854976 to 9920512 tid 3 2024-09-21T09:38:16.711 INFO:tasks.rados.rados.0.smithi022.stdout:2063: done (4 left) 2024-09-21T09:38:16.711 INFO:tasks.rados.rados.0.smithi022.stdout:2064: done (3 left) 2024-09-21T09:38:16.711 INFO:tasks.rados.rados.0.smithi022.stdout:2065: done (2 left) 2024-09-21T09:38:16.711 INFO:tasks.rados.rados.0.smithi022.stdout:2066: done (1 left) 2024-09-21T09:38:16.711 INFO:tasks.rados.rados.0.smithi022.stdout:2068: snap_create 2024-09-21T09:38:16.711 INFO:tasks.rados.rados.0.smithi022.stdout:2067: finishing write tid 1 to smithi02284960-4 2024-09-21T09:38:16.714 INFO:tasks.rados.rados.0.smithi022.stdout:2067: finishing write tid 2 to smithi02284960-4 2024-09-21T09:38:16.714 INFO:tasks.rados.rados.0.smithi022.stdout:2067: finishing write tid 3 to smithi02284960-4 2024-09-21T09:38:16.714 INFO:tasks.rados.rados.0.smithi022.stdout:2067: finishing write tid 4 to smithi02284960-4 2024-09-21T09:38:16.715 INFO:tasks.rados.rados.0.smithi022.stdout:2067: finishing write tid 5 to smithi02284960-4 2024-09-21T09:38:16.715 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 4 v 726 (ObjNum 660 snap 203 seq_num 660) dirty exists 2024-09-21T09:38:16.715 INFO:tasks.rados.rados.0.smithi022.stdout:2067: left oid 4 (ObjNum 660 snap 203 seq_num 660) 2024-09-21T09:38:17.242 INFO:tasks.rados.rados.0.smithi022.stdout:2067: done (1 left) 2024-09-21T09:38:17.243 INFO:tasks.rados.rados.0.smithi022.stdout:2068: done (0 left) 2024-09-21T09:38:17.243 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 18 current snap is 204 2024-09-21T09:38:17.243 INFO:tasks.rados.rados.0.smithi022.stdout:2069: seq_num 661 ranges {4071424=679936,4751360=622592,5373952=180224} 2024-09-21T09:38:17.244 INFO:journalctl@ceph.osd.4.smithi064.stdout:Sep 21 09:38:17 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-4[92249]: 2024-09-21T09:38:17.158+0000 7ff33628d640 -1 osd.4 465 log_to_monitors true 2024-09-21T09:38:17.244 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:38:16 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:38:16.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi022\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:38:17.268 INFO:tasks.rados.rados.0.smithi022.stdout:2069: writing smithi02284960-18 from 4071424 to 4751360 tid 1 2024-09-21T09:38:17.272 INFO:tasks.rados.rados.0.smithi022.stdout:2069: writing smithi02284960-18 from 4751360 to 5373952 tid 2 2024-09-21T09:38:17.273 INFO:tasks.rados.rados.0.smithi022.stdout:2069: writing smithi02284960-18 from 5373952 to 5554176 tid 3 2024-09-21T09:38:17.273 INFO:tasks.rados.rados.0.smithi022.stdout:2070: snap_create 2024-09-21T09:38:17.276 INFO:tasks.rados.rados.0.smithi022.stdout:2069: finishing write tid 1 to smithi02284960-18 2024-09-21T09:38:17.276 INFO:tasks.rados.rados.0.smithi022.stdout:2069: finishing write tid 2 to smithi02284960-18 2024-09-21T09:38:17.277 INFO:tasks.rados.rados.0.smithi022.stdout:2069: finishing write tid 3 to smithi02284960-18 2024-09-21T09:38:17.277 INFO:tasks.rados.rados.0.smithi022.stdout:2069: finishing write tid 4 to smithi02284960-18 2024-09-21T09:38:17.278 INFO:tasks.rados.rados.0.smithi022.stdout:2069: finishing write tid 5 to smithi02284960-18 2024-09-21T09:38:17.278 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 18 v 1049 (ObjNum 661 snap 204 seq_num 661) dirty exists 2024-09-21T09:38:17.278 INFO:tasks.rados.rados.0.smithi022.stdout:2069: left oid 18 (ObjNum 661 snap 204 seq_num 661) 2024-09-21T09:38:17.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:17 smithi022 ceph-mon[98615]: osdmap e475: 8 total, 7 up, 8 in 2024-09-21T09:38:17.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:17 smithi022 ceph-mon[98615]: pgmap v316: 121 pgs: 34 active+undersized, 13 active+undersized+degraded, 74 active+clean; 176 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 6.7 MiB/s rd, 1.4 MiB/s wr, 13 op/s; 35/312 objects degraded (11.218%) 2024-09-21T09:38:17.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:17 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:38:17.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:17 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:38:17.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:17 smithi022 ceph-mon[98615]: from='osd.4 [v2:172.21.15.64:6800/2985474018,v1:172.21.15.64:6801/2985474018]' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-21T09:38:17.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:17 smithi022 ceph-mon[98615]: from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-21T09:38:17.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:17 smithi022 ceph-mon[96736]: osdmap e475: 8 total, 7 up, 8 in 2024-09-21T09:38:17.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:17 smithi022 ceph-mon[96736]: pgmap v316: 121 pgs: 34 active+undersized, 13 active+undersized+degraded, 74 active+clean; 176 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 6.7 MiB/s rd, 1.4 MiB/s wr, 13 op/s; 35/312 objects degraded (11.218%) 2024-09-21T09:38:17.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:17 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:38:17.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:17 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:38:17.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:17 smithi022 ceph-mon[96736]: from='osd.4 [v2:172.21.15.64:6800/2985474018,v1:172.21.15.64:6801/2985474018]' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-21T09:38:17.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:17 smithi022 ceph-mon[96736]: from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-21T09:38:17.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:17 smithi064 ceph-mon[86709]: osdmap e475: 8 total, 7 up, 8 in 2024-09-21T09:38:17.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:17 smithi064 ceph-mon[86709]: pgmap v316: 121 pgs: 34 active+undersized, 13 active+undersized+degraded, 74 active+clean; 176 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 6.7 MiB/s rd, 1.4 MiB/s wr, 13 op/s; 35/312 objects degraded (11.218%) 2024-09-21T09:38:17.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:17 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:38:17.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:17 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:38:17.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:17 smithi064 ceph-mon[86709]: from='osd.4 [v2:172.21.15.64:6800/2985474018,v1:172.21.15.64:6801/2985474018]' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-21T09:38:17.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:17 smithi064 ceph-mon[86709]: from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-21T09:38:17.979 INFO:journalctl@ceph.osd.4.smithi064.stdout:Sep 21 09:38:17 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-4[92249]: 2024-09-21T09:38:17.724+0000 7ff32a02f640 -1 osd.4 465 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T09:38:18.245 INFO:tasks.rados.rados.0.smithi022.stdout:2069: done (1 left) 2024-09-21T09:38:18.246 INFO:tasks.rados.rados.0.smithi022.stdout:2070: done (0 left) 2024-09-21T09:38:18.246 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 17 current snap is 205 2024-09-21T09:38:18.246 INFO:tasks.rados.rados.0.smithi022.stdout:2071: seq_num 662 ranges {4898816=696320,5595136=557056,6152192=245760} 2024-09-21T09:38:18.278 INFO:tasks.rados.rados.0.smithi022.stdout:2071: writing smithi02284960-17 from 4898816 to 5595136 tid 1 2024-09-21T09:38:18.281 INFO:tasks.rados.rados.0.smithi022.stdout:2071: writing smithi02284960-17 from 5595136 to 6152192 tid 2 2024-09-21T09:38:18.283 INFO:tasks.rados.rados.0.smithi022.stdout:2071: writing smithi02284960-17 from 6152192 to 6397952 tid 3 2024-09-21T09:38:18.283 INFO:tasks.rados.rados.0.smithi022.stdout:2072: rollback oid 22 current snap is 205 2024-09-21T09:38:18.283 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 22 to 200 2024-09-21T09:38:18.283 INFO:tasks.rados.rados.0.smithi022.stdout:2073: delete oid 39 current snap is 205 2024-09-21T09:38:18.286 INFO:tasks.rados.rados.0.smithi022.stdout:2072: finishing rollback tid 0 to smithi02284960-22 2024-09-21T09:38:18.286 INFO:tasks.rados.rados.0.smithi022.stdout:2072: finishing rollback tid 1 to smithi02284960-22 2024-09-21T09:38:18.286 INFO:tasks.rados.rados.0.smithi022.stdout:2072: finishing rollback tid 2 to smithi02284960-22 2024-09-21T09:38:18.286 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 22 v 1255 (ObjNum 619 snap 186 seq_num 619) dirty exists 2024-09-21T09:38:18.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:18 smithi022 ceph-mon[98615]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-21T09:38:18.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:18 smithi022 ceph-mon[98615]: osdmap e476: 8 total, 7 up, 8 in 2024-09-21T09:38:18.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:18 smithi022 ceph-mon[98615]: from='osd.4 [v2:172.21.15.64:6800/2985474018,v1:172.21.15.64:6801/2985474018]' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi064", "root=default"]} : dispatch 2024-09-21T09:38:18.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:18 smithi022 ceph-mon[98615]: from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi064", "root=default"]} : dispatch 2024-09-21T09:38:18.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:18 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:38:18.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:18 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:38:18.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:18 smithi022 ceph-mon[98615]: from='osd.4 ' entity='osd.4' 2024-09-21T09:38:18.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:18 smithi022 ceph-mon[96736]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-21T09:38:18.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:18 smithi022 ceph-mon[96736]: osdmap e476: 8 total, 7 up, 8 in 2024-09-21T09:38:18.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:18 smithi022 ceph-mon[96736]: from='osd.4 [v2:172.21.15.64:6800/2985474018,v1:172.21.15.64:6801/2985474018]' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi064", "root=default"]} : dispatch 2024-09-21T09:38:18.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:18 smithi022 ceph-mon[96736]: from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi064", "root=default"]} : dispatch 2024-09-21T09:38:18.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:18 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:38:18.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:18 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:38:18.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:18 smithi022 ceph-mon[96736]: from='osd.4 ' entity='osd.4' 2024-09-21T09:38:18.565 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:18 smithi064 ceph-mon[86709]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-21T09:38:18.565 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:18 smithi064 ceph-mon[86709]: osdmap e476: 8 total, 7 up, 8 in 2024-09-21T09:38:18.565 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:18 smithi064 ceph-mon[86709]: from='osd.4 [v2:172.21.15.64:6800/2985474018,v1:172.21.15.64:6801/2985474018]' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi064", "root=default"]} : dispatch 2024-09-21T09:38:18.566 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:18 smithi064 ceph-mon[86709]: from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi064", "root=default"]} : dispatch 2024-09-21T09:38:18.566 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:18 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:38:18.566 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:18 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:38:18.566 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:18 smithi064 ceph-mon[86709]: from='osd.4 ' entity='osd.4' 2024-09-21T09:38:19.278 INFO:tasks.rados.rados.0.smithi022.stdout:2072: done (2 left) 2024-09-21T09:38:19.278 INFO:tasks.rados.rados.0.smithi022.stdout:2073: done (1 left) 2024-09-21T09:38:19.278 INFO:tasks.rados.rados.0.smithi022.stdout:2074: snap_remove snap 191 2024-09-21T09:38:19.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:19 smithi022 ceph-mon[98615]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T09:38:19.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:19 smithi022 ceph-mon[98615]: osd.4 [v2:172.21.15.64:6800/2985474018,v1:172.21.15.64:6801/2985474018] boot 2024-09-21T09:38:19.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:19 smithi022 ceph-mon[98615]: osdmap e477: 8 total, 8 up, 8 in 2024-09-21T09:38:19.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:19 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-09-21T09:38:19.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:19 smithi022 ceph-mon[98615]: pgmap v319: 121 pgs: 20 peering, 16 active+undersized, 11 active+undersized+degraded, 74 active+clean; 182 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 5.8 MiB/s rd, 1.9 MiB/s wr, 11 op/s; 34/330 objects degraded (10.303%) 2024-09-21T09:38:19.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:19 smithi022 ceph-mon[96736]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T09:38:19.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:19 smithi022 ceph-mon[96736]: osd.4 [v2:172.21.15.64:6800/2985474018,v1:172.21.15.64:6801/2985474018] boot 2024-09-21T09:38:19.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:19 smithi022 ceph-mon[96736]: osdmap e477: 8 total, 8 up, 8 in 2024-09-21T09:38:19.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:19 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-09-21T09:38:19.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:19 smithi022 ceph-mon[96736]: pgmap v319: 121 pgs: 20 peering, 16 active+undersized, 11 active+undersized+degraded, 74 active+clean; 182 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 5.8 MiB/s rd, 1.9 MiB/s wr, 11 op/s; 34/330 objects degraded (10.303%) 2024-09-21T09:38:19.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:19 smithi064 ceph-mon[86709]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T09:38:19.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:19 smithi064 ceph-mon[86709]: osd.4 [v2:172.21.15.64:6800/2985474018,v1:172.21.15.64:6801/2985474018] boot 2024-09-21T09:38:19.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:19 smithi064 ceph-mon[86709]: osdmap e477: 8 total, 8 up, 8 in 2024-09-21T09:38:19.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:19 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-09-21T09:38:19.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:19 smithi064 ceph-mon[86709]: pgmap v319: 121 pgs: 20 peering, 16 active+undersized, 11 active+undersized+degraded, 74 active+clean; 182 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 5.8 MiB/s rd, 1.9 MiB/s wr, 11 op/s; 34/330 objects degraded (10.303%) 2024-09-21T09:38:20.265 INFO:tasks.rados.rados.0.smithi022.stdout:2074: done (1 left) 2024-09-21T09:38:20.266 INFO:tasks.rados.rados.0.smithi022.stdout:2075: snap_create 2024-09-21T09:38:20.266 INFO:tasks.rados.rados.0.smithi022.stdout:2071: finishing write tid 1 to smithi02284960-17 2024-09-21T09:38:20.266 INFO:tasks.rados.rados.0.smithi022.stdout:2071: finishing write tid 2 to smithi02284960-17 2024-09-21T09:38:20.266 INFO:tasks.rados.rados.0.smithi022.stdout:2071: finishing write tid 3 to smithi02284960-17 2024-09-21T09:38:20.266 INFO:tasks.rados.rados.0.smithi022.stdout:2071: finishing write tid 4 to smithi02284960-17 2024-09-21T09:38:20.266 INFO:tasks.rados.rados.0.smithi022.stdout:2071: finishing write tid 5 to smithi02284960-17 2024-09-21T09:38:20.266 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 17 v 1052 (ObjNum 662 snap 205 seq_num 662) dirty exists 2024-09-21T09:38:20.266 INFO:tasks.rados.rados.0.smithi022.stdout:2071: left oid 17 (ObjNum 662 snap 205 seq_num 662) 2024-09-21T09:38:20.540 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:20 smithi064 ceph-mon[86709]: osdmap e478: 8 total, 8 up, 8 in 2024-09-21T09:38:20.540 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:38:20 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:38:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:38:20.557 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:20 smithi022 ceph-mon[98615]: osdmap e478: 8 total, 8 up, 8 in 2024-09-21T09:38:20.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:20 smithi022 ceph-mon[96736]: osdmap e478: 8 total, 8 up, 8 in 2024-09-21T09:38:21.058 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:38:20 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: 2024-09-21T09:38:20.670+0000 7feec1b50640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (11 PGs are or would become offline) 2024-09-21T09:38:21.271 INFO:tasks.rados.rados.0.smithi022.stdout:2071: done (1 left) 2024-09-21T09:38:21.272 INFO:tasks.rados.rados.0.smithi022.stdout:2075: done (0 left) 2024-09-21T09:38:21.272 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 10 current snap is 206 2024-09-21T09:38:21.272 INFO:tasks.rados.rados.0.smithi022.stdout:2076: seq_num 664 ranges {5111808=729088,5840896=729088,6569984=73728} 2024-09-21T09:38:21.303 INFO:tasks.rados.rados.0.smithi022.stdout:2076: writing smithi02284960-10 from 5111808 to 5840896 tid 1 2024-09-21T09:38:21.307 INFO:tasks.rados.rados.0.smithi022.stdout:2076: writing smithi02284960-10 from 5840896 to 6569984 tid 2 2024-09-21T09:38:21.308 INFO:tasks.rados.rados.0.smithi022.stdout:2076: writing smithi02284960-10 from 6569984 to 6643712 tid 3 2024-09-21T09:38:21.308 INFO:tasks.rados.rados.0.smithi022.stdout:2077: snap_create 2024-09-21T09:38:21.311 INFO:tasks.rados.rados.0.smithi022.stdout:2076: finishing write tid 1 to smithi02284960-10 2024-09-21T09:38:21.311 INFO:tasks.rados.rados.0.smithi022.stdout:2076: finishing write tid 2 to smithi02284960-10 2024-09-21T09:38:21.312 INFO:tasks.rados.rados.0.smithi022.stdout:2076: finishing write tid 3 to smithi02284960-10 2024-09-21T09:38:21.313 INFO:tasks.rados.rados.0.smithi022.stdout:2076: finishing write tid 4 to smithi02284960-10 2024-09-21T09:38:21.314 INFO:tasks.rados.rados.0.smithi022.stdout:2076: finishing write tid 5 to smithi02284960-10 2024-09-21T09:38:21.314 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 10 v 1101 (ObjNum 664 snap 206 seq_num 664) dirty exists 2024-09-21T09:38:21.314 INFO:tasks.rados.rados.0.smithi022.stdout:2076: left oid 10 (ObjNum 664 snap 206 seq_num 664) 2024-09-21T09:38:21.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:21 smithi022 ceph-mon[96736]: osdmap e479: 8 total, 8 up, 8 in 2024-09-21T09:38:21.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:21 smithi022 ceph-mon[96736]: pgmap v322: 121 pgs: 20 peering, 16 active+undersized, 10 active+undersized+degraded, 75 active+clean; 187 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 7.4 MiB/s rd, 2.9 MiB/s wr, 22 op/s; 30/342 objects degraded (8.772%); 4.4 MiB/s, 0 objects/s recovering 2024-09-21T09:38:21.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:21 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:38:21.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:21 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:38:21.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:21 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:38:21.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:21 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:38:21.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:21 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:38:21.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:21 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:38:21.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:21 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:38:21.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:21 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:38:21.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:21 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:38:21.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:21 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16} : dispatch 2024-09-21T09:38:21.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:21 smithi022 ceph-mon[96736]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-21T09:38:21.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:21 smithi022 ceph-mon[96736]: Upgrade: unsafe to stop osd(s) at this time (11 PGs are or would become offline) 2024-09-21T09:38:21.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:21 smithi022 ceph-mon[98615]: osdmap e479: 8 total, 8 up, 8 in 2024-09-21T09:38:21.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:21 smithi022 ceph-mon[98615]: pgmap v322: 121 pgs: 20 peering, 16 active+undersized, 10 active+undersized+degraded, 75 active+clean; 187 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 7.4 MiB/s rd, 2.9 MiB/s wr, 22 op/s; 30/342 objects degraded (8.772%); 4.4 MiB/s, 0 objects/s recovering 2024-09-21T09:38:21.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:21 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:38:21.560 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:21 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:38:21.560 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:21 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:38:21.560 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:21 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:38:21.560 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:21 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:38:21.560 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:21 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:38:21.560 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:21 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:38:21.560 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:21 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:38:21.560 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:21 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:38:21.560 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:21 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16} : dispatch 2024-09-21T09:38:21.560 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:21 smithi022 ceph-mon[98615]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-21T09:38:21.560 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:21 smithi022 ceph-mon[98615]: Upgrade: unsafe to stop osd(s) at this time (11 PGs are or would become offline) 2024-09-21T09:38:21.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:21 smithi064 ceph-mon[86709]: osdmap e479: 8 total, 8 up, 8 in 2024-09-21T09:38:21.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:21 smithi064 ceph-mon[86709]: pgmap v322: 121 pgs: 20 peering, 16 active+undersized, 10 active+undersized+degraded, 75 active+clean; 187 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 7.4 MiB/s rd, 2.9 MiB/s wr, 22 op/s; 30/342 objects degraded (8.772%); 4.4 MiB/s, 0 objects/s recovering 2024-09-21T09:38:21.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:21 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:38:21.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:21 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:38:21.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:21 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:38:21.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:21 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:38:21.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:21 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:38:21.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:21 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:38:21.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:21 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:38:21.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:21 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:38:21.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:21 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:38:21.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:21 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16} : dispatch 2024-09-21T09:38:21.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:21 smithi064 ceph-mon[86709]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-21T09:38:21.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:21 smithi064 ceph-mon[86709]: Upgrade: unsafe to stop osd(s) at this time (11 PGs are or would become offline) 2024-09-21T09:38:22.279 INFO:tasks.rados.rados.0.smithi022.stdout:2076: done (1 left) 2024-09-21T09:38:22.279 INFO:tasks.rados.rados.0.smithi022.stdout:2077: done (0 left) 2024-09-21T09:38:22.279 INFO:tasks.rados.rados.0.smithi022.stdout:2078: rollback oid 27 current snap is 207 2024-09-21T09:38:22.279 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 27 to 205 2024-09-21T09:38:22.279 INFO:tasks.rados.rados.0.smithi022.stdout:2079: delete oid 39 current snap is 207 2024-09-21T09:38:22.282 INFO:tasks.rados.rados.0.smithi022.stdout:2079: done (1 left) 2024-09-21T09:38:22.282 INFO:tasks.rados.rados.0.smithi022.stdout:2080: rollback oid 5 current snap is 207 2024-09-21T09:38:22.283 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 5 to 204 2024-09-21T09:38:22.283 INFO:tasks.rados.rados.0.smithi022.stdout:2081: copy_from oid 6 from oid 24 current snap is 207 2024-09-21T09:38:22.283 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 17 current snap is 207 2024-09-21T09:38:22.283 INFO:tasks.rados.rados.0.smithi022.stdout:2082: seq_num 666 ranges {6397952=761856,7159808=491520,7651328=311296} 2024-09-21T09:38:22.322 INFO:tasks.rados.rados.0.smithi022.stdout:2082: writing smithi02284960-17 from 6397952 to 7159808 tid 1 2024-09-21T09:38:22.325 INFO:tasks.rados.rados.0.smithi022.stdout:2082: writing smithi02284960-17 from 7159808 to 7651328 tid 2 2024-09-21T09:38:22.328 INFO:tasks.rados.rados.0.smithi022.stdout:2082: writing smithi02284960-17 from 7651328 to 7962624 tid 3 2024-09-21T09:38:22.328 INFO:tasks.rados.rados.0.smithi022.stdout:2083: snap_create 2024-09-21T09:38:22.328 INFO:tasks.rados.rados.0.smithi022.stdout:2081: finishing copy_from to smithi02284960-6 2024-09-21T09:38:22.328 INFO:tasks.rados.rados.0.smithi022.stdout:2081: got expected ENOENT (src dne) 2024-09-21T09:38:22.328 INFO:tasks.rados.rados.0.smithi022.stdout:2078: finishing rollback tid 0 to smithi02284960-27 2024-09-21T09:38:22.328 INFO:tasks.rados.rados.0.smithi022.stdout:2081: finishing copy_from racing read to smithi02284960-6 2024-09-21T09:38:22.328 INFO:tasks.rados.rados.0.smithi022.stdout:2078: finishing rollback tid 1 to smithi02284960-27 2024-09-21T09:38:22.328 INFO:tasks.rados.rados.0.smithi022.stdout:2078: finishing rollback tid 2 to smithi02284960-27 2024-09-21T09:38:22.328 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 27 v 1258 (ObjNum 590 snap 176 seq_num 590) dirty exists 2024-09-21T09:38:22.328 INFO:tasks.rados.rados.0.smithi022.stdout:2080: finishing rollback tid 0 to smithi02284960-5 2024-09-21T09:38:22.328 INFO:tasks.rados.rados.0.smithi022.stdout:2080: finishing rollback tid 1 to smithi02284960-5 2024-09-21T09:38:22.328 INFO:tasks.rados.rados.0.smithi022.stdout:2080: finishing rollback tid 2 to smithi02284960-5 2024-09-21T09:38:22.328 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 5 v 736 (ObjNum 653 snap 201 seq_num 653) dirty exists 2024-09-21T09:38:22.328 INFO:tasks.rados.rados.0.smithi022.stdout:2082: finishing write tid 1 to smithi02284960-17 2024-09-21T09:38:22.330 INFO:tasks.rados.rados.0.smithi022.stdout:2082: finishing write tid 2 to smithi02284960-17 2024-09-21T09:38:22.330 INFO:tasks.rados.rados.0.smithi022.stdout:2082: finishing write tid 3 to smithi02284960-17 2024-09-21T09:38:22.330 INFO:tasks.rados.rados.0.smithi022.stdout:2082: finishing write tid 4 to smithi02284960-17 2024-09-21T09:38:22.331 INFO:tasks.rados.rados.0.smithi022.stdout:2082: finishing write tid 5 to smithi02284960-17 2024-09-21T09:38:22.331 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 17 v 1056 (ObjNum 666 snap 207 seq_num 666) dirty exists 2024-09-21T09:38:22.331 INFO:tasks.rados.rados.0.smithi022.stdout:2082: left oid 17 (ObjNum 666 snap 207 seq_num 666) 2024-09-21T09:38:22.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:22 smithi022 ceph-mon[98615]: Health check update: Degraded data redundancy: 30/342 objects degraded (8.772%), 10 pgs degraded (PG_DEGRADED) 2024-09-21T09:38:22.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:22 smithi022 ceph-mon[98615]: osdmap e480: 8 total, 8 up, 8 in 2024-09-21T09:38:22.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:22 smithi022 ceph-mon[96736]: Health check update: Degraded data redundancy: 30/342 objects degraded (8.772%), 10 pgs degraded (PG_DEGRADED) 2024-09-21T09:38:22.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:22 smithi022 ceph-mon[96736]: osdmap e480: 8 total, 8 up, 8 in 2024-09-21T09:38:22.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:22 smithi064 ceph-mon[86709]: Health check update: Degraded data redundancy: 30/342 objects degraded (8.772%), 10 pgs degraded (PG_DEGRADED) 2024-09-21T09:38:22.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:22 smithi064 ceph-mon[86709]: osdmap e480: 8 total, 8 up, 8 in 2024-09-21T09:38:23.285 INFO:tasks.rados.rados.0.smithi022.stdout:2078: done (4 left) 2024-09-21T09:38:23.285 INFO:tasks.rados.rados.0.smithi022.stdout:2080: done (3 left) 2024-09-21T09:38:23.285 INFO:tasks.rados.rados.0.smithi022.stdout:2081: done (2 left) 2024-09-21T09:38:23.285 INFO:tasks.rados.rados.0.smithi022.stdout:2082: done (1 left) 2024-09-21T09:38:23.285 INFO:tasks.rados.rados.0.smithi022.stdout:2083: done (0 left) 2024-09-21T09:38:23.285 INFO:tasks.rados.rados.0.smithi022.stdout:2084: snap_remove snap 199 2024-09-21T09:38:23.557 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:23 smithi022 ceph-mon[98615]: osdmap e481: 8 total, 8 up, 8 in 2024-09-21T09:38:23.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:23 smithi022 ceph-mon[98615]: pgmap v325: 121 pgs: 20 peering, 7 active+undersized, 4 active+undersized+degraded, 90 active+clean; 190 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 5.1 MiB/s rd, 2.1 MiB/s wr, 16 op/s; 18/351 objects degraded (5.128%); 11 MiB/s, 1 objects/s recovering 2024-09-21T09:38:23.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:23 smithi022 ceph-mon[96736]: osdmap e481: 8 total, 8 up, 8 in 2024-09-21T09:38:23.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:23 smithi022 ceph-mon[96736]: pgmap v325: 121 pgs: 20 peering, 7 active+undersized, 4 active+undersized+degraded, 90 active+clean; 190 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 5.1 MiB/s rd, 2.1 MiB/s wr, 16 op/s; 18/351 objects degraded (5.128%); 11 MiB/s, 1 objects/s recovering 2024-09-21T09:38:23.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:23 smithi064 ceph-mon[86709]: osdmap e481: 8 total, 8 up, 8 in 2024-09-21T09:38:23.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:23 smithi064 ceph-mon[86709]: pgmap v325: 121 pgs: 20 peering, 7 active+undersized, 4 active+undersized+degraded, 90 active+clean; 190 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 5.1 MiB/s rd, 2.1 MiB/s wr, 16 op/s; 18/351 objects degraded (5.128%); 11 MiB/s, 1 objects/s recovering 2024-09-21T09:38:24.291 INFO:tasks.rados.rados.0.smithi022.stdout:2084: done (0 left) 2024-09-21T09:38:24.291 INFO:tasks.rados.rados.0.smithi022.stdout:2085: read oid 42 snap 201 2024-09-21T09:38:24.291 INFO:tasks.rados.rados.0.smithi022.stdout:2085: expect (ObjNum 641 snap 197 seq_num 641) 2024-09-21T09:38:24.291 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 33 current snap is 208 2024-09-21T09:38:24.291 INFO:tasks.rados.rados.0.smithi022.stdout:2086: seq_num 667 ranges {2023424=778240,2801664=778240,3579904=24576} 2024-09-21T09:38:24.309 INFO:tasks.rados.rados.0.smithi022.stdout:2086: writing smithi02284960-33 from 2023424 to 2801664 tid 1 2024-09-21T09:38:24.315 INFO:tasks.rados.rados.0.smithi022.stdout:2086: writing smithi02284960-33 from 2801664 to 3579904 tid 2 2024-09-21T09:38:24.315 INFO:tasks.rados.rados.0.smithi022.stdout:2086: writing smithi02284960-33 from 3579904 to 3604480 tid 3 2024-09-21T09:38:24.315 INFO:tasks.rados.rados.0.smithi022.stdout:2087: delete oid 6 current snap is 208 2024-09-21T09:38:24.315 INFO:tasks.rados.rados.0.smithi022.stdout:2085: expect (ObjNum 641 snap 197 seq_num 641) 2024-09-21T09:38:24.347 INFO:tasks.rados.rados.0.smithi022.stdout:2086: finishing write tid 1 to smithi02284960-33 2024-09-21T09:38:24.347 INFO:tasks.rados.rados.0.smithi022.stdout:2086: finishing write tid 2 to smithi02284960-33 2024-09-21T09:38:24.347 INFO:tasks.rados.rados.0.smithi022.stdout:2085: done (2 left) 2024-09-21T09:38:24.347 INFO:tasks.rados.rados.0.smithi022.stdout:2087: done (1 left) 2024-09-21T09:38:24.347 INFO:tasks.rados.rados.0.smithi022.stdout:2088: read oid 4 snap -1 2024-09-21T09:38:24.348 INFO:tasks.rados.rados.0.smithi022.stdout:2088: expect (ObjNum 660 snap 203 seq_num 660) 2024-09-21T09:38:24.348 INFO:tasks.rados.rados.0.smithi022.stdout:2089: read oid 31 snap -1 2024-09-21T09:38:24.348 INFO:tasks.rados.rados.0.smithi022.stdout:2089: expect (ObjNum 590 snap 176 seq_num 590) 2024-09-21T09:38:24.348 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 40 current snap is 208 2024-09-21T09:38:24.348 INFO:tasks.rados.rados.0.smithi022.stdout:2090: seq_num 669 ranges {2285568=409600,2695168=548864,3244032=655360} 2024-09-21T09:38:24.365 INFO:tasks.rados.rados.0.smithi022.stdout:2090: writing smithi02284960-40 from 2285568 to 2695168 tid 1 2024-09-21T09:38:24.369 INFO:tasks.rados.rados.0.smithi022.stdout:2090: writing smithi02284960-40 from 2695168 to 3244032 tid 2 2024-09-21T09:38:24.374 INFO:tasks.rados.rados.0.smithi022.stdout:2090: writing smithi02284960-40 from 3244032 to 3899392 tid 3 2024-09-21T09:38:24.374 INFO:tasks.rados.rados.0.smithi022.stdout:2086: finishing write tid 3 to smithi02284960-33 2024-09-21T09:38:24.374 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 8 current snap is 208 2024-09-21T09:38:24.374 INFO:tasks.rados.rados.0.smithi022.stdout:2086: finishing write tid 4 to smithi02284960-33 2024-09-21T09:38:24.374 INFO:tasks.rados.rados.0.smithi022.stdout:2086: finishing write tid 5 to smithi02284960-33 2024-09-21T09:38:24.374 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 33 v 1243 (ObjNum 667 snap 208 seq_num 667) dirty exists 2024-09-21T09:38:24.374 INFO:tasks.rados.rados.0.smithi022.stdout:2086: left oid 33 (ObjNum 667 snap 208 seq_num 667) 2024-09-21T09:38:24.374 INFO:tasks.rados.rados.0.smithi022.stdout:2089: expect (ObjNum 590 snap 176 seq_num 590) 2024-09-21T09:38:24.386 INFO:tasks.rados.rados.0.smithi022.stdout:2090: finishing write tid 1 to smithi02284960-40 2024-09-21T09:38:24.386 INFO:tasks.rados.rados.0.smithi022.stdout:2090: finishing write tid 2 to smithi02284960-40 2024-09-21T09:38:24.386 INFO:tasks.rados.rados.0.smithi022.stdout:2090: finishing write tid 3 to smithi02284960-40 2024-09-21T09:38:24.386 INFO:tasks.rados.rados.0.smithi022.stdout:2090: finishing write tid 4 to smithi02284960-40 2024-09-21T09:38:24.386 INFO:tasks.rados.rados.0.smithi022.stdout:2090: finishing write tid 5 to smithi02284960-40 2024-09-21T09:38:24.387 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 40 v 1110 (ObjNum 669 snap 208 seq_num 669) dirty exists 2024-09-21T09:38:24.387 INFO:tasks.rados.rados.0.smithi022.stdout:2090: left oid 40 (ObjNum 669 snap 208 seq_num 669) 2024-09-21T09:38:24.387 INFO:tasks.rados.rados.0.smithi022.stdout:2091: seq_num 670 ranges {0=425984,425984=483328,909312=507904,1417216=212992} 2024-09-21T09:38:24.388 INFO:tasks.rados.rados.0.smithi022.stdout:2091: writing smithi02284960-8 from 0 to 425984 tid 1 2024-09-21T09:38:24.391 INFO:tasks.rados.rados.0.smithi022.stdout:2091: writing smithi02284960-8 from 425984 to 909312 tid 2 2024-09-21T09:38:24.394 INFO:tasks.rados.rados.0.smithi022.stdout:2091: writing smithi02284960-8 from 909312 to 1417216 tid 3 2024-09-21T09:38:24.396 INFO:tasks.rados.rados.0.smithi022.stdout:2091: writing smithi02284960-8 from 1417216 to 1630208 tid 4 2024-09-21T09:38:24.396 INFO:tasks.rados.rados.0.smithi022.stdout:2088: expect (ObjNum 660 snap 203 seq_num 660) 2024-09-21T09:38:24.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:24 smithi064 ceph-mon[86709]: osdmap e482: 8 total, 8 up, 8 in 2024-09-21T09:38:24.441 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:38:24 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:38:24.148Z caller=alerting.go:391 level=warn component="rule manager" alert="unsupported value type" msg="Expanding alert template failed" err="error executing template __alert_CephOSDDown: template: __alert_CephOSDDown:1:358: executing \"__alert_CephOSDDown\" at : error calling query: found duplicate series for the match group {ceph_daemon=\"osd.4\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.4\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.4\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}];many-to-many matching not allowed: matching labels must be unique on one side" data="unsupported value type" 2024-09-21T09:38:24.442 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:38:24 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:38:24.149Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.4\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.4\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.4\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:38:24.557 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:24 smithi022 ceph-mon[98615]: osdmap e482: 8 total, 8 up, 8 in 2024-09-21T09:38:24.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:24 smithi022 ceph-mon[96736]: osdmap e482: 8 total, 8 up, 8 in 2024-09-21T09:38:25.069 INFO:tasks.rados.rados.0.smithi022.stdout:2091: finishing write tid 1 to smithi02284960-8 2024-09-21T09:38:25.069 INFO:tasks.rados.rados.0.smithi022.stdout:2091: finishing write tid 2 to smithi02284960-8 2024-09-21T09:38:25.069 INFO:tasks.rados.rados.0.smithi022.stdout:2091: finishing write tid 3 to smithi02284960-8 2024-09-21T09:38:25.069 INFO:tasks.rados.rados.0.smithi022.stdout:2091: finishing write tid 4 to smithi02284960-8 2024-09-21T09:38:25.069 INFO:tasks.rados.rados.0.smithi022.stdout:2091: finishing write tid 5 to smithi02284960-8 2024-09-21T09:38:25.069 INFO:tasks.rados.rados.0.smithi022.stdout:2091: finishing write tid 6 to smithi02284960-8 2024-09-21T09:38:25.069 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 8 v 745 (ObjNum 670 snap 208 seq_num 670) dirty exists 2024-09-21T09:38:25.069 INFO:tasks.rados.rados.0.smithi022.stdout:2091: left oid 8 (ObjNum 670 snap 208 seq_num 670) 2024-09-21T09:38:25.069 INFO:tasks.rados.rados.0.smithi022.stdout:2086: done (4 left) 2024-09-21T09:38:25.069 INFO:tasks.rados.rados.0.smithi022.stdout:2088: done (3 left) 2024-09-21T09:38:25.069 INFO:tasks.rados.rados.0.smithi022.stdout:2089: done (2 left) 2024-09-21T09:38:25.069 INFO:tasks.rados.rados.0.smithi022.stdout:2090: done (1 left) 2024-09-21T09:38:25.070 INFO:tasks.rados.rados.0.smithi022.stdout:2091: done (0 left) 2024-09-21T09:38:25.070 INFO:tasks.rados.rados.0.smithi022.stdout:2092: rollback oid 37 current snap is 208 2024-09-21T09:38:25.070 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 37 to 203 2024-09-21T09:38:25.070 INFO:tasks.rados.rados.0.smithi022.stdout:2093: setattr oid 48 current snap is 208 2024-09-21T09:38:25.070 INFO:tasks.rados.rados.0.smithi022.stdout:2094: read oid 41 snap -1 2024-09-21T09:38:25.070 INFO:tasks.rados.rados.0.smithi022.stdout:2094: expect (ObjNum 531 snap 159 seq_num 531) 2024-09-21T09:38:25.070 INFO:tasks.rados.rados.0.smithi022.stdout:2095: delete oid 25 current snap is 208 2024-09-21T09:38:25.073 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 48 v 1247 (ObjNum 623 snap 186 seq_num 623) dirty exists 2024-09-21T09:38:25.073 INFO:tasks.rados.rados.0.smithi022.stdout:2092: finishing rollback tid 0 to smithi02284960-37 2024-09-21T09:38:25.074 INFO:tasks.rados.rados.0.smithi022.stdout:2092: finishing rollback tid 1 to smithi02284960-37 2024-09-21T09:38:25.075 INFO:tasks.rados.rados.0.smithi022.stdout:2092: finishing rollback tid 2 to smithi02284960-37 2024-09-21T09:38:25.075 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 37 v 1297 (ObjNum 161 snap 0 seq_num 1) dirty exists 2024-09-21T09:38:25.080 INFO:tasks.rados.rados.0.smithi022.stdout:2094: expect (ObjNum 531 snap 159 seq_num 531) 2024-09-21T09:38:25.128 INFO:tasks.rados.rados.0.smithi022.stdout:2092: done (3 left) 2024-09-21T09:38:25.128 INFO:tasks.rados.rados.0.smithi022.stdout:2093: done (2 left) 2024-09-21T09:38:25.128 INFO:tasks.rados.rados.0.smithi022.stdout:2094: done (1 left) 2024-09-21T09:38:25.128 INFO:tasks.rados.rados.0.smithi022.stdout:2095: done (0 left) 2024-09-21T09:38:25.128 INFO:tasks.rados.rados.0.smithi022.stdout:2096: rmattr oid 42 current snap is 208 2024-09-21T09:38:25.129 INFO:tasks.rados.rados.0.smithi022.stdout:2096: done (0 left) 2024-09-21T09:38:25.129 INFO:tasks.rados.rados.0.smithi022.stdout:2097: read oid 47 snap -1 2024-09-21T09:38:25.129 INFO:tasks.rados.rados.0.smithi022.stdout:2097: expect (ObjNum 604 snap 183 seq_num 604) 2024-09-21T09:38:25.129 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 37 current snap is 208 2024-09-21T09:38:25.129 INFO:tasks.rados.rados.0.smithi022.stdout:2098: seq_num 672 ranges {0=458752,458752=647168,1105920=466944,1572864=90112} 2024-09-21T09:38:25.130 INFO:tasks.rados.rados.0.smithi022.stdout:2098: writing smithi02284960-37 from 0 to 458752 tid 1 2024-09-21T09:38:25.134 INFO:tasks.rados.rados.0.smithi022.stdout:2098: writing smithi02284960-37 from 458752 to 1105920 tid 2 2024-09-21T09:38:25.137 INFO:tasks.rados.rados.0.smithi022.stdout:2098: writing smithi02284960-37 from 1105920 to 1572864 tid 3 2024-09-21T09:38:25.138 INFO:tasks.rados.rados.0.smithi022.stdout:2098: writing smithi02284960-37 from 1572864 to 1662976 tid 4 2024-09-21T09:38:25.138 INFO:tasks.rados.rados.0.smithi022.stdout:2099: read oid 20 snap -1 2024-09-21T09:38:25.138 INFO:tasks.rados.rados.0.smithi022.stdout:2099: expect (ObjNum 659 snap 203 seq_num 659) 2024-09-21T09:38:25.138 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 9 current snap is 208 2024-09-21T09:38:25.138 INFO:tasks.rados.rados.0.smithi022.stdout:2100: seq_num 673 ranges {0=475136,475136=532480,1007616=671744} 2024-09-21T09:38:25.141 INFO:tasks.rados.rados.0.smithi022.stdout:2100: writing smithi02284960-9 from 0 to 475136 tid 1 2024-09-21T09:38:25.144 INFO:tasks.rados.rados.0.smithi022.stdout:2100: writing smithi02284960-9 from 475136 to 1007616 tid 2 2024-09-21T09:38:25.149 INFO:tasks.rados.rados.0.smithi022.stdout:2100: writing smithi02284960-9 from 1007616 to 1679360 tid 3 2024-09-21T09:38:25.149 INFO:tasks.rados.rados.0.smithi022.stdout:2101: read oid 10 snap -1 2024-09-21T09:38:25.149 INFO:tasks.rados.rados.0.smithi022.stdout:2101: expect (ObjNum 664 snap 206 seq_num 664) 2024-09-21T09:38:25.149 INFO:tasks.rados.rados.0.smithi022.stdout:2102: snap_remove snap 197 2024-09-21T09:38:25.302 INFO:tasks.rados.rados.0.smithi022.stdout:2102: done (5 left) 2024-09-21T09:38:25.302 INFO:tasks.rados.rados.0.smithi022.stdout:2103: rollback oid 16 current snap is 208 2024-09-21T09:38:25.302 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 16 to 207 2024-09-21T09:38:25.302 INFO:tasks.rados.rados.0.smithi022.stdout:2098: finishing write tid 1 to smithi02284960-37 2024-09-21T09:38:25.303 INFO:tasks.rados.rados.0.smithi022.stdout:2098: finishing write tid 2 to smithi02284960-37 2024-09-21T09:38:25.303 INFO:tasks.rados.rados.0.smithi022.stdout:2098: finishing write tid 3 to smithi02284960-37 2024-09-21T09:38:25.303 INFO:tasks.rados.rados.0.smithi022.stdout:2098: finishing write tid 4 to smithi02284960-37 2024-09-21T09:38:25.303 INFO:tasks.rados.rados.0.smithi022.stdout:2097: expect (ObjNum 604 snap 183 seq_num 604) 2024-09-21T09:38:25.557 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:25 smithi022 ceph-mon[96736]: osdmap e483: 8 total, 8 up, 8 in 2024-09-21T09:38:25.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:25 smithi022 ceph-mon[96736]: pgmap v328: 121 pgs: 121 active+clean; 187 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 1.1 MiB/s wr, 4 op/s; 11 MiB/s, 2 objects/s recovering 2024-09-21T09:38:25.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:25 smithi022 ceph-mon[98615]: osdmap e483: 8 total, 8 up, 8 in 2024-09-21T09:38:25.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:25 smithi022 ceph-mon[98615]: pgmap v328: 121 pgs: 121 active+clean; 187 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 1.1 MiB/s wr, 4 op/s; 11 MiB/s, 2 objects/s recovering 2024-09-21T09:38:25.613 INFO:tasks.rados.rados.0.smithi022.stdout:2098: finishing write tid 5 to smithi02284960-37 2024-09-21T09:38:25.613 INFO:tasks.rados.rados.0.smithi022.stdout:2100: finishing write tid 1 to smithi02284960-9 2024-09-21T09:38:25.613 INFO:tasks.rados.rados.0.smithi022.stdout:2098: finishing write tid 6 to smithi02284960-37 2024-09-21T09:38:25.613 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 37 v 1302 (ObjNum 672 snap 208 seq_num 672) dirty exists 2024-09-21T09:38:25.614 INFO:tasks.rados.rados.0.smithi022.stdout:2098: left oid 37 (ObjNum 672 snap 208 seq_num 672) 2024-09-21T09:38:25.614 INFO:tasks.rados.rados.0.smithi022.stdout:2100: finishing write tid 2 to smithi02284960-9 2024-09-21T09:38:25.614 INFO:tasks.rados.rados.0.smithi022.stdout:2100: finishing write tid 3 to smithi02284960-9 2024-09-21T09:38:25.614 INFO:tasks.rados.rados.0.smithi022.stdout:2100: finishing write tid 4 to smithi02284960-9 2024-09-21T09:38:25.614 INFO:tasks.rados.rados.0.smithi022.stdout:2100: finishing write tid 5 to smithi02284960-9 2024-09-21T09:38:25.614 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 9 v 749 (ObjNum 673 snap 208 seq_num 673) dirty exists 2024-09-21T09:38:25.614 INFO:tasks.rados.rados.0.smithi022.stdout:2100: left oid 9 (ObjNum 673 snap 208 seq_num 673) 2024-09-21T09:38:25.614 INFO:tasks.rados.rados.0.smithi022.stdout:2097: done (5 left) 2024-09-21T09:38:25.614 INFO:tasks.rados.rados.0.smithi022.stdout:2098: done (4 left) 2024-09-21T09:38:25.614 INFO:tasks.rados.rados.0.smithi022.stdout:2100: done (3 left) 2024-09-21T09:38:25.614 INFO:tasks.rados.rados.0.smithi022.stdout:2104: read oid 30 snap -1 2024-09-21T09:38:25.614 INFO:tasks.rados.rados.0.smithi022.stdout:2104: expect (ObjNum 622 snap 186 seq_num 622) 2024-09-21T09:38:25.614 INFO:tasks.rados.rados.0.smithi022.stdout:2105: copy_from oid 5 from oid 35 current snap is 208 2024-09-21T09:38:25.614 INFO:tasks.rados.rados.0.smithi022.stdout:2101: expect (ObjNum 664 snap 206 seq_num 664) 2024-09-21T09:38:25.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:25 smithi064 ceph-mon[86709]: osdmap e483: 8 total, 8 up, 8 in 2024-09-21T09:38:25.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:25 smithi064 ceph-mon[86709]: pgmap v328: 121 pgs: 121 active+clean; 187 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 1.1 MiB/s wr, 4 op/s; 11 MiB/s, 2 objects/s recovering 2024-09-21T09:38:25.999 INFO:tasks.rados.rados.0.smithi022.stdout:2099: expect (ObjNum 659 snap 203 seq_num 659) 2024-09-21T09:38:26.057 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:38:25 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:38:25] "GET /metrics HTTP/1.1" 200 36305 "" "Prometheus/2.51.0" 2024-09-21T09:38:26.191 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:38:25 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:38:25.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:38:26.192 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:38:25 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:38:25.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:38:26.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:26 smithi022 ceph-mon[98615]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 18/351 objects degraded (5.128%), 4 pgs degraded) 2024-09-21T09:38:26.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:26 smithi022 ceph-mon[98615]: Cluster is now healthy 2024-09-21T09:38:26.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:26 smithi022 ceph-mon[98615]: osdmap e484: 8 total, 8 up, 8 in 2024-09-21T09:38:26.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:26 smithi022 ceph-mon[96736]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 18/351 objects degraded (5.128%), 4 pgs degraded) 2024-09-21T09:38:26.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:26 smithi022 ceph-mon[96736]: Cluster is now healthy 2024-09-21T09:38:26.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:26 smithi022 ceph-mon[96736]: osdmap e484: 8 total, 8 up, 8 in 2024-09-21T09:38:26.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:26 smithi064 ceph-mon[86709]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 18/351 objects degraded (5.128%), 4 pgs degraded) 2024-09-21T09:38:26.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:26 smithi064 ceph-mon[86709]: Cluster is now healthy 2024-09-21T09:38:26.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:26 smithi064 ceph-mon[86709]: osdmap e484: 8 total, 8 up, 8 in 2024-09-21T09:38:26.898 INFO:tasks.rados.rados.0.smithi022.stdout:2103: finishing rollback tid 0 to smithi02284960-16 2024-09-21T09:38:26.898 INFO:tasks.rados.rados.0.smithi022.stdout:2103: finishing rollback tid 1 to smithi02284960-16 2024-09-21T09:38:26.898 INFO:tasks.rados.rados.0.smithi022.stdout:2103: finishing rollback tid 2 to smithi02284960-16 2024-09-21T09:38:26.898 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 16 v 1052 (ObjNum 569 snap 170 seq_num 569) dirty exists 2024-09-21T09:38:26.899 INFO:tasks.rados.rados.0.smithi022.stdout:2105: finishing copy_from to smithi02284960-5 2024-09-21T09:38:26.899 INFO:tasks.rados.rados.0.smithi022.stdout:2105: got expected ENOENT (src dne) 2024-09-21T09:38:26.899 INFO:tasks.rados.rados.0.smithi022.stdout:2105: finishing copy_from racing read to smithi02284960-5 2024-09-21T09:38:26.899 INFO:tasks.rados.rados.0.smithi022.stdout:2104: expect (ObjNum 622 snap 186 seq_num 622) 2024-09-21T09:38:27.060 INFO:tasks.rados.rados.0.smithi022.stdout:2099: done (4 left) 2024-09-21T09:38:27.061 INFO:tasks.rados.rados.0.smithi022.stdout:2101: done (3 left) 2024-09-21T09:38:27.061 INFO:tasks.rados.rados.0.smithi022.stdout:2103: done (2 left) 2024-09-21T09:38:27.061 INFO:tasks.rados.rados.0.smithi022.stdout:2104: done (1 left) 2024-09-21T09:38:27.061 INFO:tasks.rados.rados.0.smithi022.stdout:2105: done (0 left) 2024-09-21T09:38:27.061 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 34 current snap is 208 2024-09-21T09:38:27.061 INFO:tasks.rados.rados.0.smithi022.stdout:2106: seq_num 674 ranges {2023424=491520,2514944=417792,2932736=483328,3416064=303104} 2024-09-21T09:38:27.075 INFO:tasks.rados.rados.0.smithi022.stdout:2106: writing smithi02284960-34 from 2023424 to 2514944 tid 1 2024-09-21T09:38:27.077 INFO:tasks.rados.rados.0.smithi022.stdout:2106: writing smithi02284960-34 from 2514944 to 2932736 tid 2 2024-09-21T09:38:27.080 INFO:tasks.rados.rados.0.smithi022.stdout:2106: writing smithi02284960-34 from 2932736 to 3416064 tid 3 2024-09-21T09:38:27.082 INFO:tasks.rados.rados.0.smithi022.stdout:2106: writing smithi02284960-34 from 3416064 to 3719168 tid 4 2024-09-21T09:38:27.082 INFO:tasks.rados.rados.0.smithi022.stdout:2107: read oid 50 snap -1 2024-09-21T09:38:27.082 INFO:tasks.rados.rados.0.smithi022.stdout:2107: expect (ObjNum 632 snap 189 seq_num 632) 2024-09-21T09:38:27.082 INFO:tasks.rados.rados.0.smithi022.stdout:2108: read oid 26 snap -1 2024-09-21T09:38:27.082 INFO:tasks.rados.rados.0.smithi022.stdout:2108: expect (ObjNum 642 snap 197 seq_num 642) 2024-09-21T09:38:27.082 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 3 current snap is 208 2024-09-21T09:38:27.082 INFO:tasks.rados.rados.0.smithi022.stdout:2106: finishing write tid 1 to smithi02284960-34 2024-09-21T09:38:27.082 INFO:tasks.rados.rados.0.smithi022.stdout:2106: finishing write tid 2 to smithi02284960-34 2024-09-21T09:38:27.082 INFO:tasks.rados.rados.0.smithi022.stdout:2109: seq_num 675 ranges {0=507904,507904=704512,1212416=499712} 2024-09-21T09:38:27.085 INFO:tasks.rados.rados.0.smithi022.stdout:2109: writing smithi02284960-3 from 0 to 507904 tid 1 2024-09-21T09:38:27.089 INFO:tasks.rados.rados.0.smithi022.stdout:2109: writing smithi02284960-3 from 507904 to 1212416 tid 2 2024-09-21T09:38:27.092 INFO:tasks.rados.rados.0.smithi022.stdout:2109: writing smithi02284960-3 from 1212416 to 1712128 tid 3 2024-09-21T09:38:27.092 INFO:tasks.rados.rados.0.smithi022.stdout:2110: snap_remove snap 207 2024-09-21T09:38:27.305 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:38:26 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:38:26.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi022\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi022\", job=\"node\", machine=\"x86_64\", nodename=\"smithi022\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:38:27.307 INFO:tasks.rados.rados.0.smithi022.stdout:2110: done (4 left) 2024-09-21T09:38:27.307 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 4 current snap is 208 2024-09-21T09:38:27.307 INFO:tasks.rados.rados.0.smithi022.stdout:2111: seq_num 676 ranges {9920512=524288,10444800=589824,11034624=450560,11485184=163840} 2024-09-21T09:38:27.364 INFO:tasks.rados.rados.0.smithi022.stdout:2111: writing smithi02284960-4 from 9920512 to 10444800 tid 1 2024-09-21T09:38:27.367 INFO:tasks.rados.rados.0.smithi022.stdout:2111: writing smithi02284960-4 from 10444800 to 11034624 tid 2 2024-09-21T09:38:27.371 INFO:tasks.rados.rados.0.smithi022.stdout:2111: writing smithi02284960-4 from 11034624 to 11485184 tid 3 2024-09-21T09:38:27.373 INFO:tasks.rados.rados.0.smithi022.stdout:2111: writing smithi02284960-4 from 11485184 to 11649024 tid 4 2024-09-21T09:38:27.373 INFO:tasks.rados.rados.0.smithi022.stdout:2112: delete oid 38 current snap is 208 2024-09-21T09:38:27.373 INFO:tasks.rados.rados.0.smithi022.stdout:2106: finishing write tid 3 to smithi02284960-34 2024-09-21T09:38:27.373 INFO:tasks.rados.rados.0.smithi022.stdout:2106: finishing write tid 4 to smithi02284960-34 2024-09-21T09:38:27.373 INFO:tasks.rados.rados.0.smithi022.stdout:2106: finishing write tid 5 to smithi02284960-34 2024-09-21T09:38:27.373 INFO:tasks.rados.rados.0.smithi022.stdout:2107: expect (ObjNum 632 snap 189 seq_num 632) 2024-09-21T09:38:27.401 INFO:tasks.rados.rados.0.smithi022.stdout:2106: finishing write tid 6 to smithi02284960-34 2024-09-21T09:38:27.401 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 34 v 1067 (ObjNum 674 snap 208 seq_num 674) dirty exists 2024-09-21T09:38:27.401 INFO:tasks.rados.rados.0.smithi022.stdout:2106: left oid 34 (ObjNum 674 snap 208 seq_num 674) 2024-09-21T09:38:27.401 INFO:tasks.rados.rados.0.smithi022.stdout:2109: finishing write tid 1 to smithi02284960-3 2024-09-21T09:38:27.401 INFO:tasks.rados.rados.0.smithi022.stdout:2109: finishing write tid 2 to smithi02284960-3 2024-09-21T09:38:27.402 INFO:tasks.rados.rados.0.smithi022.stdout:2106: done (5 left) 2024-09-21T09:38:27.402 INFO:tasks.rados.rados.0.smithi022.stdout:2107: done (4 left) 2024-09-21T09:38:27.402 INFO:tasks.rados.rados.0.smithi022.stdout:2112: done (3 left) 2024-09-21T09:38:27.402 INFO:tasks.rados.rados.0.smithi022.stdout:2113: setattr oid 35 current snap is 208 2024-09-21T09:38:27.402 INFO:tasks.rados.rados.0.smithi022.stdout:2109: finishing write tid 3 to smithi02284960-3 2024-09-21T09:38:27.402 INFO:tasks.rados.rados.0.smithi022.stdout:2109: finishing write tid 4 to smithi02284960-3 2024-09-21T09:38:27.402 INFO:tasks.rados.rados.0.smithi022.stdout:2109: finishing write tid 5 to smithi02284960-3 2024-09-21T09:38:27.402 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 3 v 604 (ObjNum 675 snap 208 seq_num 675) dirty exists 2024-09-21T09:38:27.402 INFO:tasks.rados.rados.0.smithi022.stdout:2109: left oid 3 (ObjNum 675 snap 208 seq_num 675) 2024-09-21T09:38:27.402 INFO:tasks.rados.rados.0.smithi022.stdout:2108: expect (ObjNum 642 snap 197 seq_num 642) 2024-09-21T09:38:27.547 INFO:tasks.rados.rados.0.smithi022.stdout:2111: finishing write tid 1 to smithi02284960-4 2024-09-21T09:38:27.547 INFO:tasks.rados.rados.0.smithi022.stdout:2111: finishing write tid 2 to smithi02284960-4 2024-09-21T09:38:27.547 INFO:tasks.rados.rados.0.smithi022.stdout:2111: finishing write tid 3 to smithi02284960-4 2024-09-21T09:38:27.547 INFO:tasks.rados.rados.0.smithi022.stdout:2111: finishing write tid 4 to smithi02284960-4 2024-09-21T09:38:27.547 INFO:tasks.rados.rados.0.smithi022.stdout:2111: finishing write tid 5 to smithi02284960-4 2024-09-21T09:38:27.547 INFO:tasks.rados.rados.0.smithi022.stdout:2111: finishing write tid 6 to smithi02284960-4 2024-09-21T09:38:27.547 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 4 v 759 (ObjNum 676 snap 208 seq_num 676) dirty exists 2024-09-21T09:38:27.547 INFO:tasks.rados.rados.0.smithi022.stdout:2111: left oid 4 (ObjNum 676 snap 208 seq_num 676) 2024-09-21T09:38:27.547 INFO:tasks.rados.rados.0.smithi022.stdout:2108: done (3 left) 2024-09-21T09:38:27.547 INFO:tasks.rados.rados.0.smithi022.stdout:2109: done (2 left) 2024-09-21T09:38:27.547 INFO:tasks.rados.rados.0.smithi022.stdout:2111: done (1 left) 2024-09-21T09:38:27.547 INFO:tasks.rados.rados.0.smithi022.stdout:2114: rmattr oid 7 current snap is 208 2024-09-21T09:38:27.547 INFO:tasks.rados.rados.0.smithi022.stdout:2114: done (1 left) 2024-09-21T09:38:27.548 INFO:tasks.rados.rados.0.smithi022.stdout:2115: delete oid 49 current snap is 208 2024-09-21T09:38:27.549 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 35 v 566 (ObjNum 609 snap 0 seq_num 3570614048) dirty exists 2024-09-21T09:38:27.550 INFO:tasks.rados.rados.0.smithi022.stdout:2113: done (1 left) 2024-09-21T09:38:27.550 INFO:tasks.rados.rados.0.smithi022.stdout:2115: done (0 left) 2024-09-21T09:38:27.550 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 49 current snap is 208 2024-09-21T09:38:27.550 INFO:tasks.rados.rados.0.smithi022.stdout:2116: seq_num 679 ranges {0=581632,581632=688128,1269760=516096} 2024-09-21T09:38:27.553 INFO:tasks.rados.rados.0.smithi022.stdout:2116: writing smithi02284960-49 from 0 to 581632 tid 1 2024-09-21T09:38:27.557 INFO:tasks.rados.rados.0.smithi022.stdout:2116: writing smithi02284960-49 from 581632 to 1269760 tid 2 2024-09-21T09:38:27.557 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:27 smithi022 ceph-mon[98615]: pgmap v330: 121 pgs: 121 active+clean; 192 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 9.5 MiB/s rd, 1.8 MiB/s wr, 14 op/s; 8.3 MiB/s, 2 objects/s recovering 2024-09-21T09:38:27.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:27 smithi022 ceph-mon[96736]: pgmap v330: 121 pgs: 121 active+clean; 192 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 9.5 MiB/s rd, 1.8 MiB/s wr, 14 op/s; 8.3 MiB/s, 2 objects/s recovering 2024-09-21T09:38:27.561 INFO:tasks.rados.rados.0.smithi022.stdout:2116: writing smithi02284960-49 from 1269760 to 1785856 tid 3 2024-09-21T09:38:27.561 INFO:tasks.rados.rados.0.smithi022.stdout:2117: read oid 38 snap -1 2024-09-21T09:38:27.561 INFO:tasks.rados.rados.0.smithi022.stdout:2117: expect deleted 2024-09-21T09:38:27.561 INFO:tasks.rados.rados.0.smithi022.stdout:2118: delete oid 33 current snap is 208 2024-09-21T09:38:27.561 INFO:tasks.rados.rados.0.smithi022.stdout:2116: finishing write tid 1 to smithi02284960-49 2024-09-21T09:38:27.561 INFO:tasks.rados.rados.0.smithi022.stdout:2116: finishing write tid 2 to smithi02284960-49 2024-09-21T09:38:27.562 INFO:tasks.rados.rados.0.smithi022.stdout:2117: done (2 left) 2024-09-21T09:38:27.563 INFO:tasks.rados.rados.0.smithi022.stdout:2118: done (1 left) 2024-09-21T09:38:27.563 INFO:tasks.rados.rados.0.smithi022.stdout:2119: snap_create 2024-09-21T09:38:27.564 INFO:tasks.rados.rados.0.smithi022.stdout:2116: finishing write tid 3 to smithi02284960-49 2024-09-21T09:38:27.564 INFO:tasks.rados.rados.0.smithi022.stdout:2116: finishing write tid 4 to smithi02284960-49 2024-09-21T09:38:27.568 INFO:tasks.rados.rados.0.smithi022.stdout:2116: finishing write tid 5 to smithi02284960-49 2024-09-21T09:38:27.568 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 49 v 1098 (ObjNum 679 snap 208 seq_num 679) dirty exists 2024-09-21T09:38:27.568 INFO:tasks.rados.rados.0.smithi022.stdout:2116: left oid 49 (ObjNum 679 snap 208 seq_num 679) 2024-09-21T09:38:27.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:27 smithi064 ceph-mon[86709]: pgmap v330: 121 pgs: 121 active+clean; 192 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 9.5 MiB/s rd, 1.8 MiB/s wr, 14 op/s; 8.3 MiB/s, 2 objects/s recovering 2024-09-21T09:38:28.315 INFO:tasks.rados.rados.0.smithi022.stdout:2116: done (1 left) 2024-09-21T09:38:28.315 INFO:tasks.rados.rados.0.smithi022.stdout:2119: done (0 left) 2024-09-21T09:38:28.315 INFO:tasks.rados.rados.0.smithi022.stdout:2120: snap_create 2024-09-21T09:38:28.557 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:28 smithi022 ceph-mon[98615]: osdmap e485: 8 total, 8 up, 8 in 2024-09-21T09:38:28.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:28 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:38:28.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:28 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:38:28.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:28 smithi022 ceph-mon[96736]: osdmap e485: 8 total, 8 up, 8 in 2024-09-21T09:38:28.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:28 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:38:28.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:28 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:38:28.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:28 smithi064 ceph-mon[86709]: osdmap e485: 8 total, 8 up, 8 in 2024-09-21T09:38:28.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:28 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:38:28.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:28 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:38:28.992 INFO:tasks.rados.rados.0.smithi022.stdout:2120: done (0 left) 2024-09-21T09:38:28.992 INFO:tasks.rados.rados.0.smithi022.stdout:2121: read oid 37 snap 200 2024-09-21T09:38:28.993 INFO:tasks.rados.rados.0.smithi022.stdout:2121: expect (ObjNum -725235088 snap 22094 seq_num 3569993328) 2024-09-21T09:38:28.993 INFO:tasks.rados.rados.0.smithi022.stdout:2122: snap_create 2024-09-21T09:38:28.996 INFO:tasks.rados.rados.0.smithi022.stdout:2121: expect (ObjNum -725235088 snap 22094 seq_num 3569993328) 2024-09-21T09:38:29.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:29 smithi064 ceph-mon[86709]: osdmap e486: 8 total, 8 up, 8 in 2024-09-21T09:38:29.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:29 smithi064 ceph-mon[86709]: pgmap v333: 121 pgs: 121 active+clean; 196 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 19 MiB/s rd, 2.7 MiB/s wr, 29 op/s 2024-09-21T09:38:29.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:29 smithi064 ceph-mon[86709]: osdmap e487: 8 total, 8 up, 8 in 2024-09-21T09:38:29.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:29 smithi022 ceph-mon[98615]: osdmap e486: 8 total, 8 up, 8 in 2024-09-21T09:38:29.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:29 smithi022 ceph-mon[98615]: pgmap v333: 121 pgs: 121 active+clean; 196 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 19 MiB/s rd, 2.7 MiB/s wr, 29 op/s 2024-09-21T09:38:29.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:29 smithi022 ceph-mon[98615]: osdmap e487: 8 total, 8 up, 8 in 2024-09-21T09:38:29.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:29 smithi022 ceph-mon[96736]: osdmap e486: 8 total, 8 up, 8 in 2024-09-21T09:38:29.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:29 smithi022 ceph-mon[96736]: pgmap v333: 121 pgs: 121 active+clean; 196 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 19 MiB/s rd, 2.7 MiB/s wr, 29 op/s 2024-09-21T09:38:29.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:29 smithi022 ceph-mon[96736]: osdmap e487: 8 total, 8 up, 8 in 2024-09-21T09:38:29.994 INFO:tasks.rados.rados.0.smithi022.stdout:2121: done (1 left) 2024-09-21T09:38:29.994 INFO:tasks.rados.rados.0.smithi022.stdout:2122: done (0 left) 2024-09-21T09:38:29.994 INFO:tasks.rados.rados.0.smithi022.stdout:2123: snap_remove snap 206 2024-09-21T09:38:30.691 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:38:30 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:38:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:38:31.000 INFO:tasks.rados.rados.0.smithi022.stdout:2123: done (0 left) 2024-09-21T09:38:31.000 INFO:tasks.rados.rados.0.smithi022.stdout:2124: copy_from oid 10 from oid 48 current snap is 211 2024-09-21T09:38:31.000 INFO:tasks.rados.rados.0.smithi022.stdout:2125: snap_create 2024-09-21T09:38:31.045 INFO:tasks.rados.rados.0.smithi022.stdout:2124: finishing copy_from racing read to smithi02284960-10 2024-09-21T09:38:31.048 INFO:tasks.rados.rados.0.smithi022.stdout:2124: finishing copy_from to smithi02284960-10 2024-09-21T09:38:31.048 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 10 v 1247 (ObjNum 623 snap 186 seq_num 623) dirty exists 2024-09-21T09:38:31.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:30 smithi022 ceph-mon[98615]: osdmap e488: 8 total, 8 up, 8 in 2024-09-21T09:38:31.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:30 smithi022 ceph-mon[96736]: osdmap e488: 8 total, 8 up, 8 in 2024-09-21T09:38:31.308 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 21 09:38:31 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-2[113367]: 2024-09-21T09:38:31.014+0000 7f02e436d640 -1 snap_mapper.get_next_objects_to_trim::New Clone-Objects were added to Snap 127 after trimming was started 2024-09-21T09:38:31.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:30 smithi064 ceph-mon[86709]: osdmap e488: 8 total, 8 up, 8 in 2024-09-21T09:38:32.001 INFO:tasks.rados.rados.0.smithi022.stdout:2124: done (1 left) 2024-09-21T09:38:32.001 INFO:tasks.rados.rados.0.smithi022.stdout:2125: done (0 left) 2024-09-21T09:38:32.002 INFO:tasks.rados.rados.0.smithi022.stdout:2126: read oid 11 snap -1 2024-09-21T09:38:32.002 INFO:tasks.rados.rados.0.smithi022.stdout:2126: expect (ObjNum 569 snap 170 seq_num 569) 2024-09-21T09:38:32.002 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 42 current snap is 212 2024-09-21T09:38:32.002 INFO:tasks.rados.rados.0.smithi022.stdout:2127: seq_num 681 ranges {1146880=614400,1761280=458752,2220032=638976,2859008=106496} 2024-09-21T09:38:32.010 INFO:tasks.rados.rados.0.smithi022.stdout:2127: writing smithi02284960-42 from 1146880 to 1761280 tid 1 2024-09-21T09:38:32.013 INFO:tasks.rados.rados.0.smithi022.stdout:2127: writing smithi02284960-42 from 1761280 to 2220032 tid 2 2024-09-21T09:38:32.017 INFO:tasks.rados.rados.0.smithi022.stdout:2127: writing smithi02284960-42 from 2220032 to 2859008 tid 3 2024-09-21T09:38:32.018 INFO:tasks.rados.rados.0.smithi022.stdout:2127: writing smithi02284960-42 from 2859008 to 2965504 tid 4 2024-09-21T09:38:32.019 INFO:tasks.rados.rados.0.smithi022.stdout:2128: copy_from oid 1 from oid 8 current snap is 212 2024-09-21T09:38:32.019 INFO:tasks.rados.rados.0.smithi022.stdout:2127: finishing write tid 1 to smithi02284960-42 2024-09-21T09:38:32.019 INFO:tasks.rados.rados.0.smithi022.stdout:2129: snap_create 2024-09-21T09:38:32.019 INFO:tasks.rados.rados.0.smithi022.stdout:2127: finishing write tid 2 to smithi02284960-42 2024-09-21T09:38:32.022 INFO:tasks.rados.rados.0.smithi022.stdout:2127: finishing write tid 3 to smithi02284960-42 2024-09-21T09:38:32.022 INFO:tasks.rados.rados.0.smithi022.stdout:2127: finishing write tid 4 to smithi02284960-42 2024-09-21T09:38:32.022 INFO:tasks.rados.rados.0.smithi022.stdout:2127: finishing write tid 5 to smithi02284960-42 2024-09-21T09:38:32.025 INFO:tasks.rados.rados.0.smithi022.stdout:2127: finishing write tid 6 to smithi02284960-42 2024-09-21T09:38:32.025 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 42 v 1263 (ObjNum 681 snap 212 seq_num 681) dirty exists 2024-09-21T09:38:32.025 INFO:tasks.rados.rados.0.smithi022.stdout:2127: left oid 42 (ObjNum 681 snap 212 seq_num 681) 2024-09-21T09:38:32.036 INFO:tasks.rados.rados.0.smithi022.stdout:2126: expect (ObjNum 569 snap 170 seq_num 569) 2024-09-21T09:38:32.307 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:32 smithi022 ceph-mon[98615]: pgmap v336: 121 pgs: 121 active+clean; 198 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 21 MiB/s rd, 2.4 MiB/s wr, 23 op/s 2024-09-21T09:38:32.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:32 smithi022 ceph-mon[98615]: osdmap e489: 8 total, 8 up, 8 in 2024-09-21T09:38:32.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:32 smithi022 ceph-mon[96736]: pgmap v336: 121 pgs: 121 active+clean; 198 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 21 MiB/s rd, 2.4 MiB/s wr, 23 op/s 2024-09-21T09:38:32.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:32 smithi022 ceph-mon[96736]: osdmap e489: 8 total, 8 up, 8 in 2024-09-21T09:38:32.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:32 smithi064 ceph-mon[86709]: pgmap v336: 121 pgs: 121 active+clean; 198 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 21 MiB/s rd, 2.4 MiB/s wr, 23 op/s 2024-09-21T09:38:32.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:32 smithi064 ceph-mon[86709]: osdmap e489: 8 total, 8 up, 8 in 2024-09-21T09:38:32.518 INFO:tasks.rados.rados.0.smithi022.stdout:2128: finishing copy_from racing read to smithi02284960-1 2024-09-21T09:38:32.518 INFO:tasks.rados.rados.0.smithi022.stdout:2128: finishing copy_from to smithi02284960-1 2024-09-21T09:38:32.518 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 1 v 745 (ObjNum 670 snap 208 seq_num 670) dirty exists 2024-09-21T09:38:33.013 INFO:tasks.rados.rados.0.smithi022.stdout:2126: done (3 left) 2024-09-21T09:38:33.013 INFO:tasks.rados.rados.0.smithi022.stdout:2127: done (2 left) 2024-09-21T09:38:33.013 INFO:tasks.rados.rados.0.smithi022.stdout:2128: done (1 left) 2024-09-21T09:38:33.013 INFO:tasks.rados.rados.0.smithi022.stdout:2129: done (0 left) 2024-09-21T09:38:33.013 INFO:tasks.rados.rados.0.smithi022.stdout:2130: delete oid 42 current snap is 213 2024-09-21T09:38:33.015 INFO:tasks.rados.rados.0.smithi022.stdout:2130: done (0 left) 2024-09-21T09:38:33.016 INFO:tasks.rados.rados.0.smithi022.stdout:2131: delete oid 48 current snap is 213 2024-09-21T09:38:33.022 INFO:tasks.rados.rados.0.smithi022.stdout:2131: done (0 left) 2024-09-21T09:38:33.022 INFO:tasks.rados.rados.0.smithi022.stdout:2132: rollback oid 47 current snap is 213 2024-09-21T09:38:33.022 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 47 to 201 2024-09-21T09:38:33.022 INFO:tasks.rados.rados.0.smithi022.stdout:2133: read oid 15 snap -1 2024-09-21T09:38:33.022 INFO:tasks.rados.rados.0.smithi022.stdout:2133: expect (ObjNum 642 snap 197 seq_num 642) 2024-09-21T09:38:33.023 INFO:tasks.rados.rados.0.smithi022.stdout:2134: setattr oid 4 current snap is 213 2024-09-21T09:38:33.024 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 29 current snap is 213 2024-09-21T09:38:33.024 INFO:tasks.rados.rados.0.smithi022.stdout:2135: seq_num 684 ranges {3735552=663552,4399104=507904,4907008=409600,5316608=286720} 2024-09-21T09:38:33.048 INFO:tasks.rados.rados.0.smithi022.stdout:2135: writing smithi02284960-29 from 3735552 to 4399104 tid 1 2024-09-21T09:38:33.051 INFO:tasks.rados.rados.0.smithi022.stdout:2135: writing smithi02284960-29 from 4399104 to 4907008 tid 2 2024-09-21T09:38:33.054 INFO:tasks.rados.rados.0.smithi022.stdout:2135: writing smithi02284960-29 from 4907008 to 5316608 tid 3 2024-09-21T09:38:33.056 INFO:tasks.rados.rados.0.smithi022.stdout:2135: writing smithi02284960-29 from 5316608 to 5603328 tid 4 2024-09-21T09:38:33.056 INFO:tasks.rados.rados.0.smithi022.stdout:2136: snap_remove snap 210 2024-09-21T09:38:33.307 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:33 smithi022 ceph-mon[98615]: osdmap e490: 8 total, 8 up, 8 in 2024-09-21T09:38:33.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:33 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:38:33.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:33 smithi022 ceph-mon[96736]: osdmap e490: 8 total, 8 up, 8 in 2024-09-21T09:38:33.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:33 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:38:33.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:33 smithi064 ceph-mon[86709]: osdmap e490: 8 total, 8 up, 8 in 2024-09-21T09:38:33.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:33 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:38:33.995 INFO:tasks.rados.rados.0.smithi022.stdout:2136: done (4 left) 2024-09-21T09:38:33.995 INFO:tasks.rados.rados.0.smithi022.stdout:2137: snap_create 2024-09-21T09:38:33.995 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 4 v 765 (ObjNum 676 snap 208 seq_num 676) dirty exists 2024-09-21T09:38:33.995 INFO:tasks.rados.rados.0.smithi022.stdout:2132: finishing rollback tid 0 to smithi02284960-47 2024-09-21T09:38:33.995 INFO:tasks.rados.rados.0.smithi022.stdout:2132: finishing rollback tid 1 to smithi02284960-47 2024-09-21T09:38:33.995 INFO:tasks.rados.rados.0.smithi022.stdout:2132: finishing rollback tid 2 to smithi02284960-47 2024-09-21T09:38:33.995 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 47 v 1117 (ObjNum 604 snap 183 seq_num 604) dirty exists 2024-09-21T09:38:33.995 INFO:tasks.rados.rados.0.smithi022.stdout:2133: expect (ObjNum 642 snap 197 seq_num 642) 2024-09-21T09:38:34.144 INFO:tasks.rados.rados.0.smithi022.stdout:2135: finishing write tid 1 to smithi02284960-29 2024-09-21T09:38:34.144 INFO:tasks.rados.rados.0.smithi022.stdout:2135: finishing write tid 2 to smithi02284960-29 2024-09-21T09:38:34.145 INFO:tasks.rados.rados.0.smithi022.stdout:2135: finishing write tid 3 to smithi02284960-29 2024-09-21T09:38:34.145 INFO:tasks.rados.rados.0.smithi022.stdout:2135: finishing write tid 4 to smithi02284960-29 2024-09-21T09:38:34.145 INFO:tasks.rados.rados.0.smithi022.stdout:2135: finishing write tid 5 to smithi02284960-29 2024-09-21T09:38:34.145 INFO:tasks.rados.rados.0.smithi022.stdout:2135: finishing write tid 6 to smithi02284960-29 2024-09-21T09:38:34.145 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 29 v 1055 (ObjNum 684 snap 213 seq_num 684) dirty exists 2024-09-21T09:38:34.145 INFO:tasks.rados.rados.0.smithi022.stdout:2135: left oid 29 (ObjNum 684 snap 213 seq_num 684) 2024-09-21T09:38:34.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:34 smithi022 ceph-mon[98615]: pgmap v339: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 199 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 1.2 MiB/s wr, 7 op/s 2024-09-21T09:38:34.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:34 smithi022 ceph-mon[98615]: osdmap e491: 8 total, 8 up, 8 in 2024-09-21T09:38:34.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:34 smithi022 ceph-mon[98615]: osdmap e492: 8 total, 8 up, 8 in 2024-09-21T09:38:34.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:34 smithi022 ceph-mon[96736]: pgmap v339: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 199 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 1.2 MiB/s wr, 7 op/s 2024-09-21T09:38:34.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:34 smithi022 ceph-mon[96736]: osdmap e491: 8 total, 8 up, 8 in 2024-09-21T09:38:34.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:34 smithi022 ceph-mon[96736]: osdmap e492: 8 total, 8 up, 8 in 2024-09-21T09:38:34.343 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:34 smithi064 ceph-mon[86709]: pgmap v339: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 199 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 1.2 MiB/s wr, 7 op/s 2024-09-21T09:38:34.343 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:34 smithi064 ceph-mon[86709]: osdmap e491: 8 total, 8 up, 8 in 2024-09-21T09:38:34.343 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:34 smithi064 ceph-mon[86709]: osdmap e492: 8 total, 8 up, 8 in 2024-09-21T09:38:34.996 INFO:tasks.rados.rados.0.smithi022.stdout:2132: done (4 left) 2024-09-21T09:38:34.996 INFO:tasks.rados.rados.0.smithi022.stdout:2133: done (3 left) 2024-09-21T09:38:34.996 INFO:tasks.rados.rados.0.smithi022.stdout:2134: done (2 left) 2024-09-21T09:38:34.996 INFO:tasks.rados.rados.0.smithi022.stdout:2135: done (1 left) 2024-09-21T09:38:34.996 INFO:tasks.rados.rados.0.smithi022.stdout:2137: done (0 left) 2024-09-21T09:38:34.996 INFO:tasks.rados.rados.0.smithi022.stdout:2138: delete oid 38 current snap is 214 2024-09-21T09:38:35.000 INFO:tasks.rados.rados.0.smithi022.stdout:2138: done (0 left) 2024-09-21T09:38:35.000 INFO:tasks.rados.rados.0.smithi022.stdout:2139: snap_remove snap 212 2024-09-21T09:38:35.999 INFO:tasks.rados.rados.0.smithi022.stdout:2139: done (0 left) 2024-09-21T09:38:35.999 INFO:tasks.rados.rados.0.smithi022.stdout:2140: rmattr oid 12 current snap is 214 2024-09-21T09:38:35.999 INFO:tasks.rados.rados.0.smithi022.stdout:2141: snap_remove snap 209 2024-09-21T09:38:36.000 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:38:35 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:38:35] "GET /metrics HTTP/1.1" 200 36313 "" "Prometheus/2.51.0" 2024-09-21T09:38:36.000 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:35 smithi022 ceph-mon[96736]: pgmap v342: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 211 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 2.2 MiB/s rd, 3.4 MiB/s wr, 7 op/s 2024-09-21T09:38:36.000 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:35 smithi022 ceph-mon[96736]: osdmap e493: 8 total, 8 up, 8 in 2024-09-21T09:38:36.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:35 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16} : dispatch 2024-09-21T09:38:36.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:35 smithi022 ceph-mon[98615]: pgmap v342: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 211 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 2.2 MiB/s rd, 3.4 MiB/s wr, 7 op/s 2024-09-21T09:38:36.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:35 smithi022 ceph-mon[98615]: osdmap e493: 8 total, 8 up, 8 in 2024-09-21T09:38:36.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:35 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16} : dispatch 2024-09-21T09:38:36.317 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:35 smithi064 ceph-mon[86709]: pgmap v342: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 211 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 2.2 MiB/s rd, 3.4 MiB/s wr, 7 op/s 2024-09-21T09:38:36.318 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:35 smithi064 ceph-mon[86709]: osdmap e493: 8 total, 8 up, 8 in 2024-09-21T09:38:36.318 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:35 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16} : dispatch 2024-09-21T09:38:37.055 INFO:tasks.rados.rados.0.smithi022.stdout:2141: done (1 left) 2024-09-21T09:38:37.055 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 45 current snap is 214 2024-09-21T09:38:37.055 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 12 v 1312 (ObjNum 633 snap 189 seq_num 633) dirty exists 2024-09-21T09:38:37.055 INFO:tasks.rados.rados.0.smithi022.stdout:2142: seq_num 686 ranges {3538944=696320,4235264=729088,4964352=425984,5390336=49152} 2024-09-21T09:38:37.078 INFO:tasks.rados.rados.0.smithi022.stdout:2142: writing smithi02284960-45 from 3538944 to 4235264 tid 1 2024-09-21T09:38:37.083 INFO:tasks.rados.rados.0.smithi022.stdout:2142: writing smithi02284960-45 from 4235264 to 4964352 tid 2 2024-09-21T09:38:37.086 INFO:tasks.rados.rados.0.smithi022.stdout:2142: writing smithi02284960-45 from 4964352 to 5390336 tid 3 2024-09-21T09:38:37.087 INFO:tasks.rados.rados.0.smithi022.stdout:2142: writing smithi02284960-45 from 5390336 to 5439488 tid 4 2024-09-21T09:38:37.087 INFO:tasks.rados.rados.0.smithi022.stdout:2140: done (1 left) 2024-09-21T09:38:37.087 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 37 current snap is 214 2024-09-21T09:38:37.087 INFO:tasks.rados.rados.0.smithi022.stdout:2143: seq_num 687 ranges {1662976=712704,2375680=614400,2990080=589824} 2024-09-21T09:38:37.100 INFO:tasks.rados.rados.0.smithi022.stdout:2143: writing smithi02284960-37 from 1662976 to 2375680 tid 1 2024-09-21T09:38:37.104 INFO:tasks.rados.rados.0.smithi022.stdout:2143: writing smithi02284960-37 from 2375680 to 2990080 tid 2 2024-09-21T09:38:37.110 INFO:tasks.rados.rados.0.smithi022.stdout:2143: writing smithi02284960-37 from 2990080 to 3579904 tid 3 2024-09-21T09:38:37.110 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 1 current snap is 214 2024-09-21T09:38:37.110 INFO:tasks.rados.rados.0.smithi022.stdout:2142: finishing write tid 1 to smithi02284960-45 2024-09-21T09:38:37.110 INFO:tasks.rados.rados.0.smithi022.stdout:2144: seq_num 688 ranges {1630208=729088,2359296=499712,2859008=704512} 2024-09-21T09:38:37.120 INFO:tasks.rados.rados.0.smithi022.stdout:2144: writing smithi02284960-1 from 1630208 to 2359296 tid 1 2024-09-21T09:38:37.124 INFO:tasks.rados.rados.0.smithi022.stdout:2144: writing smithi02284960-1 from 2359296 to 2859008 tid 2 2024-09-21T09:38:37.130 INFO:tasks.rados.rados.0.smithi022.stdout:2144: writing smithi02284960-1 from 2859008 to 3563520 tid 3 2024-09-21T09:38:37.130 INFO:tasks.rados.rados.0.smithi022.stdout:2145: rollback oid 33 current snap is 214 2024-09-21T09:38:37.130 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 33 to 208 2024-09-21T09:38:37.130 INFO:tasks.rados.rados.0.smithi022.stdout:2142: finishing write tid 2 to smithi02284960-45 2024-09-21T09:38:37.130 INFO:tasks.rados.rados.0.smithi022.stdout:2142: finishing write tid 3 to smithi02284960-45 2024-09-21T09:38:37.130 INFO:tasks.rados.rados.0.smithi022.stdout:2142: finishing write tid 4 to smithi02284960-45 2024-09-21T09:38:37.130 INFO:tasks.rados.rados.0.smithi022.stdout:2142: finishing write tid 5 to smithi02284960-45 2024-09-21T09:38:37.130 INFO:tasks.rados.rados.0.smithi022.stdout:2142: finishing write tid 6 to smithi02284960-45 2024-09-21T09:38:37.130 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 45 v 1113 (ObjNum 686 snap 214 seq_num 686) dirty exists 2024-09-21T09:38:37.130 INFO:tasks.rados.rados.0.smithi022.stdout:2142: left oid 45 (ObjNum 686 snap 214 seq_num 686) 2024-09-21T09:38:37.130 INFO:tasks.rados.rados.0.smithi022.stdout:2143: finishing write tid 1 to smithi02284960-37 2024-09-21T09:38:37.130 INFO:tasks.rados.rados.0.smithi022.stdout:2142: done (3 left) 2024-09-21T09:38:37.131 INFO:tasks.rados.rados.0.smithi022.stdout:2146: rollback oid 6 current snap is 214 2024-09-21T09:38:37.131 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 6 to 201 2024-09-21T09:38:37.131 INFO:tasks.rados.rados.0.smithi022.stdout:2143: finishing write tid 2 to smithi02284960-37 2024-09-21T09:38:37.131 INFO:tasks.rados.rados.0.smithi022.stdout:2143: finishing write tid 3 to smithi02284960-37 2024-09-21T09:38:37.131 INFO:tasks.rados.rados.0.smithi022.stdout:2143: finishing write tid 4 to smithi02284960-37 2024-09-21T09:38:37.131 INFO:tasks.rados.rados.0.smithi022.stdout:2143: finishing write tid 5 to smithi02284960-37 2024-09-21T09:38:37.131 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 37 v 1319 (ObjNum 687 snap 214 seq_num 687) dirty exists 2024-09-21T09:38:37.131 INFO:tasks.rados.rados.0.smithi022.stdout:2143: left oid 37 (ObjNum 687 snap 214 seq_num 687) 2024-09-21T09:38:37.131 INFO:tasks.rados.rados.0.smithi022.stdout:2144: finishing write tid 1 to smithi02284960-1 2024-09-21T09:38:37.131 INFO:tasks.rados.rados.0.smithi022.stdout:2143: done (3 left) 2024-09-21T09:38:37.131 INFO:tasks.rados.rados.0.smithi022.stdout:2147: copy_from oid 8 from oid 35 current snap is 214 2024-09-21T09:38:37.131 INFO:tasks.rados.rados.0.smithi022.stdout:2144: finishing write tid 2 to smithi02284960-1 2024-09-21T09:38:37.131 INFO:tasks.rados.rados.0.smithi022.stdout:2148: read oid 47 snap -1 2024-09-21T09:38:37.131 INFO:tasks.rados.rados.0.smithi022.stdout:2148: expect (ObjNum 604 snap 183 seq_num 604) 2024-09-21T09:38:37.132 INFO:tasks.rados.rados.0.smithi022.stdout:2149: rollback oid 15 current snap is 214 2024-09-21T09:38:37.132 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 15 to 202 2024-09-21T09:38:37.132 INFO:tasks.rados.rados.0.smithi022.stdout:2150: copy_from oid 49 from oid 4 current snap is 214 2024-09-21T09:38:37.132 INFO:tasks.rados.rados.0.smithi022.stdout:2151: copy_from oid 50 from oid 22 current snap is 214 2024-09-21T09:38:37.132 INFO:tasks.rados.rados.0.smithi022.stdout:2152: read oid 39 snap 202 2024-09-21T09:38:37.132 INFO:tasks.rados.rados.0.smithi022.stdout:2152: expect (ObjNum 538 snap 162 seq_num 538) 2024-09-21T09:38:37.132 INFO:tasks.rados.rados.0.smithi022.stdout:2153: read oid 3 snap -1 2024-09-21T09:38:37.132 INFO:tasks.rados.rados.0.smithi022.stdout:2153: expect (ObjNum 675 snap 208 seq_num 675) 2024-09-21T09:38:37.132 INFO:tasks.rados.rados.0.smithi022.stdout:2154: copy_from oid 21 from oid 19 current snap is 214 2024-09-21T09:38:37.132 INFO:tasks.rados.rados.0.smithi022.stdout:2155: copy_from oid 2 from oid 31 current snap is 214 2024-09-21T09:38:37.132 INFO:tasks.rados.rados.0.smithi022.stdout:2156: snap_remove snap 200 2024-09-21T09:38:37.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:37 smithi022 ceph-mon[98615]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-21T09:38:37.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:37 smithi022 ceph-mon[98615]: Upgrade: osd.5 is safe to restart 2024-09-21T09:38:37.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:37 smithi022 ceph-mon[98615]: osdmap e494: 8 total, 8 up, 8 in 2024-09-21T09:38:37.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:37 smithi022 ceph-mon[96736]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-21T09:38:37.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:37 smithi022 ceph-mon[96736]: Upgrade: osd.5 is safe to restart 2024-09-21T09:38:37.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:37 smithi022 ceph-mon[96736]: osdmap e494: 8 total, 8 up, 8 in 2024-09-21T09:38:37.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:37 smithi064 ceph-mon[86709]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-21T09:38:37.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:37 smithi064 ceph-mon[86709]: Upgrade: osd.5 is safe to restart 2024-09-21T09:38:37.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:37 smithi064 ceph-mon[86709]: osdmap e494: 8 total, 8 up, 8 in 2024-09-21T09:38:38.020 INFO:tasks.rados.rados.0.smithi022.stdout:2145: finishing rollback tid 1 to smithi02284960-33 2024-09-21T09:38:38.020 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 33 v 1266 (ObjNum 33 snap 0 seq_num 3562300792) dirty dne 2024-09-21T09:38:38.020 INFO:tasks.rados.rados.0.smithi022.stdout:2147: finishing copy_from racing read to smithi02284960-8 2024-09-21T09:38:38.020 INFO:tasks.rados.rados.0.smithi022.stdout:2146: finishing rollback tid 1 to smithi02284960-6 2024-09-21T09:38:38.020 INFO:tasks.rados.rados.0.smithi022.stdout:2149: finishing rollback tid 0 to smithi02284960-15 2024-09-21T09:38:38.020 INFO:tasks.rados.rados.0.smithi022.stdout:2149: finishing rollback tid 1 to smithi02284960-15 2024-09-21T09:38:38.020 INFO:tasks.rados.rados.0.smithi022.stdout:2144: finishing write tid 3 to smithi02284960-1 2024-09-21T09:38:38.020 INFO:tasks.rados.rados.0.smithi022.stdout:2144: finishing write tid 4 to smithi02284960-1 2024-09-21T09:38:38.020 INFO:tasks.rados.rados.0.smithi022.stdout:2149: finishing rollback tid 2 to smithi02284960-15 2024-09-21T09:38:38.020 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 15 v 1323 (ObjNum 642 snap 197 seq_num 642) dirty exists 2024-09-21T09:38:38.020 INFO:tasks.rados.rados.0.smithi022.stdout:2146: finishing rollback tid 2 to smithi02284960-6 2024-09-21T09:38:38.020 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 6 v 1283 (ObjNum 423 snap 132 seq_num 423) dirty exists 2024-09-21T09:38:38.021 INFO:tasks.rados.rados.0.smithi022.stdout:2151: finishing copy_from racing read to smithi02284960-50 2024-09-21T09:38:38.021 INFO:tasks.rados.rados.0.smithi022.stdout:2147: finishing copy_from to smithi02284960-8 2024-09-21T09:38:38.021 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 8 v 566 (ObjNum 544235886 snap 171325489 seq_num 304) dirty exists 2024-09-21T09:38:38.021 INFO:tasks.rados.rados.0.smithi022.stdout:2151: finishing copy_from to smithi02284960-50 2024-09-21T09:38:38.021 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 50 v 1255 (ObjNum 619 snap 186 seq_num 619) dirty exists 2024-09-21T09:38:38.021 INFO:tasks.rados.rados.0.smithi022.stdout:2152: expect (ObjNum 538 snap 162 seq_num 538) 2024-09-21T09:38:38.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:38 smithi064 ceph-mon[86709]: pgmap v345: 121 pgs: 121 active+clean; 213 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 8.7 MiB/s rd, 3.3 MiB/s wr, 13 op/s 2024-09-21T09:38:38.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:38 smithi064 ceph-mon[86709]: osdmap e495: 8 total, 8 up, 8 in 2024-09-21T09:38:38.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:38 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:38:38.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:38 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-21T09:38:38.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:38 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:38:38.214 INFO:tasks.rados.rados.0.smithi022.stdout:2153: expect (ObjNum 675 snap 208 seq_num 675) 2024-09-21T09:38:38.260 INFO:tasks.rados.rados.0.smithi022.stdout:2154: finishing copy_from to smithi02284960-21 2024-09-21T09:38:38.260 INFO:tasks.rados.rados.0.smithi022.stdout:2154: got expected ENOENT (src dne) 2024-09-21T09:38:38.260 INFO:tasks.rados.rados.0.smithi022.stdout:2154: finishing copy_from racing read to smithi02284960-21 2024-09-21T09:38:38.260 INFO:tasks.rados.rados.0.smithi022.stdout:2155: finishing copy_from racing read to smithi02284960-2 2024-09-21T09:38:38.260 INFO:tasks.rados.rados.0.smithi022.stdout:2155: finishing copy_from to smithi02284960-2 2024-09-21T09:38:38.260 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 2 v 1254 (ObjNum 590 snap 176 seq_num 590) dirty exists 2024-09-21T09:38:38.260 INFO:tasks.rados.rados.0.smithi022.stdout:2145: done (12 left) 2024-09-21T09:38:38.260 INFO:tasks.rados.rados.0.smithi022.stdout:2146: done (11 left) 2024-09-21T09:38:38.260 INFO:tasks.rados.rados.0.smithi022.stdout:2147: done (10 left) 2024-09-21T09:38:38.260 INFO:tasks.rados.rados.0.smithi022.stdout:2149: done (9 left) 2024-09-21T09:38:38.261 INFO:tasks.rados.rados.0.smithi022.stdout:2151: done (8 left) 2024-09-21T09:38:38.261 INFO:tasks.rados.rados.0.smithi022.stdout:2152: done (7 left) 2024-09-21T09:38:38.261 INFO:tasks.rados.rados.0.smithi022.stdout:2153: done (6 left) 2024-09-21T09:38:38.261 INFO:tasks.rados.rados.0.smithi022.stdout:2154: done (5 left) 2024-09-21T09:38:38.261 INFO:tasks.rados.rados.0.smithi022.stdout:2155: done (4 left) 2024-09-21T09:38:38.261 INFO:tasks.rados.rados.0.smithi022.stdout:2156: done (3 left) 2024-09-21T09:38:38.261 INFO:tasks.rados.rados.0.smithi022.stdout:2157: delete oid 45 current snap is 214 2024-09-21T09:38:38.261 INFO:tasks.rados.rados.0.smithi022.stdout:2148: expect (ObjNum 604 snap 183 seq_num 604) 2024-09-21T09:38:38.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:38 smithi022 ceph-mon[98615]: pgmap v345: 121 pgs: 121 active+clean; 213 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 8.7 MiB/s rd, 3.3 MiB/s wr, 13 op/s 2024-09-21T09:38:38.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:38 smithi022 ceph-mon[98615]: osdmap e495: 8 total, 8 up, 8 in 2024-09-21T09:38:38.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:38 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:38:38.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:38 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-21T09:38:38.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:38 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:38:38.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:38 smithi022 ceph-mon[96736]: pgmap v345: 121 pgs: 121 active+clean; 213 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 8.7 MiB/s rd, 3.3 MiB/s wr, 13 op/s 2024-09-21T09:38:38.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:38 smithi022 ceph-mon[96736]: osdmap e495: 8 total, 8 up, 8 in 2024-09-21T09:38:38.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:38 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:38:38.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:38 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-21T09:38:38.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:38 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:38:38.565 INFO:tasks.rados.rados.0.smithi022.stdout:2144: finishing write tid 5 to smithi02284960-1 2024-09-21T09:38:38.565 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 1 v 1132 (ObjNum 688 snap 214 seq_num 688) dirty exists 2024-09-21T09:38:38.565 INFO:tasks.rados.rados.0.smithi022.stdout:2144: left oid 1 (ObjNum 688 snap 214 seq_num 688) 2024-09-21T09:38:38.565 INFO:tasks.rados.rados.0.smithi022.stdout:2150: finishing copy_from racing read to smithi02284960-49 2024-09-21T09:38:38.565 INFO:tasks.rados.rados.0.smithi022.stdout:2144: done (3 left) 2024-09-21T09:38:38.566 INFO:tasks.rados.rados.0.smithi022.stdout:2148: done (2 left) 2024-09-21T09:38:38.566 INFO:tasks.rados.rados.0.smithi022.stdout:2157: done (1 left) 2024-09-21T09:38:38.566 INFO:tasks.rados.rados.0.smithi022.stdout:2158: snap_create 2024-09-21T09:38:38.566 INFO:tasks.rados.rados.0.smithi022.stdout:2150: finishing copy_from to smithi02284960-49 2024-09-21T09:38:38.566 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 49 v 765 (ObjNum 676 snap 208 seq_num 676) dirty exists 2024-09-21T09:38:38.995 INFO:tasks.rados.rados.0.smithi022.stdout:2150: done (1 left) 2024-09-21T09:38:38.995 INFO:tasks.rados.rados.0.smithi022.stdout:2158: done (0 left) 2024-09-21T09:38:38.995 INFO:tasks.rados.rados.0.smithi022.stdout:2159: delete oid 34 current snap is 215 2024-09-21T09:38:39.000 INFO:tasks.rados.rados.0.smithi022.stdout:2159: done (0 left) 2024-09-21T09:38:39.000 INFO:tasks.rados.rados.0.smithi022.stdout:2160: snap_remove snap 204 2024-09-21T09:38:39.282 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:39 smithi064 ceph-mon[86709]: Upgrade: Updating osd.5 2024-09-21T09:38:39.282 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:39 smithi064 ceph-mon[86709]: Deploying daemon osd.5 on smithi064 2024-09-21T09:38:39.282 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:39 smithi064 ceph-mon[86709]: osdmap e496: 8 total, 8 up, 8 in 2024-09-21T09:38:39.282 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:39 smithi064 ceph-mon[86709]: osdmap e497: 8 total, 8 up, 8 in 2024-09-21T09:38:39.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:39 smithi022 ceph-mon[98615]: Upgrade: Updating osd.5 2024-09-21T09:38:39.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:39 smithi022 ceph-mon[98615]: Deploying daemon osd.5 on smithi064 2024-09-21T09:38:39.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:39 smithi022 ceph-mon[98615]: osdmap e496: 8 total, 8 up, 8 in 2024-09-21T09:38:39.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:39 smithi022 ceph-mon[98615]: osdmap e497: 8 total, 8 up, 8 in 2024-09-21T09:38:39.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:39 smithi022 ceph-mon[96736]: Upgrade: Updating osd.5 2024-09-21T09:38:39.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:39 smithi022 ceph-mon[96736]: Deploying daemon osd.5 on smithi064 2024-09-21T09:38:39.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:39 smithi022 ceph-mon[96736]: osdmap e496: 8 total, 8 up, 8 in 2024-09-21T09:38:39.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:39 smithi022 ceph-mon[96736]: osdmap e497: 8 total, 8 up, 8 in 2024-09-21T09:38:39.997 INFO:tasks.rados.rados.0.smithi022.stdout:2160: done (0 left) 2024-09-21T09:38:39.997 INFO:tasks.rados.rados.0.smithi022.stdout:2161: copy_from oid 3 from oid 28 current snap is 215 2024-09-21T09:38:39.997 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 45 current snap is 215 2024-09-21T09:38:39.997 INFO:tasks.rados.rados.0.smithi022.stdout:2162: seq_num 691 ranges {0=778240,778240=548864,1327104=614400,1941504=40960} 2024-09-21T09:38:40.002 INFO:tasks.rados.rados.0.smithi022.stdout:2162: writing smithi02284960-45 from 0 to 778240 tid 1 2024-09-21T09:38:40.006 INFO:tasks.rados.rados.0.smithi022.stdout:2162: writing smithi02284960-45 from 778240 to 1327104 tid 2 2024-09-21T09:38:40.012 INFO:tasks.rados.rados.0.smithi022.stdout:2162: writing smithi02284960-45 from 1327104 to 1941504 tid 3 2024-09-21T09:38:40.012 INFO:tasks.rados.rados.0.smithi022.stdout:2162: writing smithi02284960-45 from 1941504 to 1982464 tid 4 2024-09-21T09:38:40.012 INFO:tasks.rados.rados.0.smithi022.stdout:2163: read oid 14 snap -1 2024-09-21T09:38:40.012 INFO:tasks.rados.rados.0.smithi022.stdout:2163: expect (ObjNum 580 snap 172 seq_num 580) 2024-09-21T09:38:40.012 INFO:tasks.rados.rados.0.smithi022.stdout:2164: read oid 19 snap -1 2024-09-21T09:38:40.012 INFO:tasks.rados.rados.0.smithi022.stdout:2164: expect deleted 2024-09-21T09:38:40.012 INFO:tasks.rados.rados.0.smithi022.stdout:2165: snap_remove snap 205 2024-09-21T09:38:40.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:40 smithi022 ceph-mon[98615]: pgmap v348: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 227 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 7.6 MiB/s rd, 4.3 MiB/s wr, 14 op/s 2024-09-21T09:38:40.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:40 smithi022 ceph-mon[98615]: osdmap e498: 8 total, 8 up, 8 in 2024-09-21T09:38:40.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:40 smithi022 ceph-mon[96736]: pgmap v348: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 227 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 7.6 MiB/s rd, 4.3 MiB/s wr, 14 op/s 2024-09-21T09:38:40.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:40 smithi022 ceph-mon[96736]: osdmap e498: 8 total, 8 up, 8 in 2024-09-21T09:38:40.384 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:40 smithi064 ceph-mon[86709]: pgmap v348: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 227 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 7.6 MiB/s rd, 4.3 MiB/s wr, 14 op/s 2024-09-21T09:38:40.384 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:40 smithi064 ceph-mon[86709]: osdmap e498: 8 total, 8 up, 8 in 2024-09-21T09:38:40.384 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:38:40 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:38:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:38:40.634 INFO:journalctl@ceph.osd.5.smithi064.stdout:Sep 21 09:38:40 smithi064 systemd[1]: Stopping Ceph osd.5 for d65e801e-77fa-11ef-baf6-efdc52797490... 2024-09-21T09:38:40.941 INFO:journalctl@ceph.osd.5.smithi064.stdout:Sep 21 09:38:40 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-5[42862]: 2024-09-21T09:38:40.632+0000 7f1b60d9f640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T09:38:40.942 INFO:journalctl@ceph.osd.5.smithi064.stdout:Sep 21 09:38:40 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-5[42862]: 2024-09-21T09:38:40.632+0000 7f1b60d9f640 -1 osd.5 498 *** Got signal Terminated *** 2024-09-21T09:38:40.942 INFO:journalctl@ceph.osd.5.smithi064.stdout:Sep 21 09:38:40 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-5[42862]: 2024-09-21T09:38:40.632+0000 7f1b60d9f640 -1 osd.5 498 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T09:38:40.999 INFO:tasks.rados.rados.0.smithi022.stdout:2161: finishing copy_from racing read to smithi02284960-3 2024-09-21T09:38:40.999 INFO:tasks.rados.rados.0.smithi022.stdout:2163: expect (ObjNum 580 snap 172 seq_num 580) 2024-09-21T09:38:41.008 INFO:tasks.rados.rados.0.smithi022.stdout:2161: finishing copy_from to smithi02284960-3 2024-09-21T09:38:41.008 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 3 v 561 (ObjNum 392 snap 122 seq_num 392) dirty exists 2024-09-21T09:38:41.008 INFO:tasks.rados.rados.0.smithi022.stdout:2162: finishing write tid 1 to smithi02284960-45 2024-09-21T09:38:41.008 INFO:tasks.rados.rados.0.smithi022.stdout:2162: finishing write tid 2 to smithi02284960-45 2024-09-21T09:38:41.008 INFO:tasks.rados.rados.0.smithi022.stdout:2162: finishing write tid 3 to smithi02284960-45 2024-09-21T09:38:41.008 INFO:tasks.rados.rados.0.smithi022.stdout:2162: finishing write tid 4 to smithi02284960-45 2024-09-21T09:38:41.008 INFO:tasks.rados.rados.0.smithi022.stdout:2162: finishing write tid 5 to smithi02284960-45 2024-09-21T09:38:41.008 INFO:tasks.rados.rados.0.smithi022.stdout:2162: finishing write tid 6 to smithi02284960-45 2024-09-21T09:38:41.008 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 45 v 1261 (ObjNum 691 snap 215 seq_num 691) dirty exists 2024-09-21T09:38:41.009 INFO:tasks.rados.rados.0.smithi022.stdout:2162: left oid 45 (ObjNum 691 snap 215 seq_num 691) 2024-09-21T09:38:41.009 INFO:tasks.rados.rados.0.smithi022.stdout:2161: done (4 left) 2024-09-21T09:38:41.009 INFO:tasks.rados.rados.0.smithi022.stdout:2162: done (3 left) 2024-09-21T09:38:41.009 INFO:tasks.rados.rados.0.smithi022.stdout:2163: done (2 left) 2024-09-21T09:38:41.009 INFO:tasks.rados.rados.0.smithi022.stdout:2164: done (1 left) 2024-09-21T09:38:41.009 INFO:tasks.rados.rados.0.smithi022.stdout:2165: done (0 left) 2024-09-21T09:38:41.009 INFO:tasks.rados.rados.0.smithi022.stdout:2166: read oid 8 snap -1 2024-09-21T09:38:41.009 INFO:tasks.rados.rados.0.smithi022.stdout:2166: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-21T09:38:41.009 INFO:tasks.rados.rados.0.smithi022.stdout:2167: copy_from oid 27 from oid 37 current snap is 215 2024-09-21T09:38:41.009 INFO:tasks.rados.rados.0.smithi022.stdout:2168: delete oid 15 current snap is 215 2024-09-21T09:38:41.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:41 smithi022 ceph-mon[98615]: osd.5 marked itself down and dead 2024-09-21T09:38:41.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:41 smithi022 ceph-mon[98615]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T09:38:41.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:41 smithi022 ceph-mon[98615]: osdmap e499: 8 total, 7 up, 8 in 2024-09-21T09:38:41.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:41 smithi022 ceph-mon[96736]: osd.5 marked itself down and dead 2024-09-21T09:38:41.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:41 smithi022 ceph-mon[96736]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T09:38:41.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:41 smithi022 ceph-mon[96736]: osdmap e499: 8 total, 7 up, 8 in 2024-09-21T09:38:41.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:41 smithi064 ceph-mon[86709]: osd.5 marked itself down and dead 2024-09-21T09:38:41.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:41 smithi064 ceph-mon[86709]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T09:38:41.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:41 smithi064 ceph-mon[86709]: osdmap e499: 8 total, 7 up, 8 in 2024-09-21T09:38:41.850 INFO:journalctl@ceph.osd.5.smithi064.stdout:Sep 21 09:38:41 smithi064 podman[96294]: 2024-09-21 09:38:41.578584077 +0000 UTC m=+1.045838433 container died 1900c4af5d2856b1841d667842b4149c2b82a3ef1f49144601922ba7e45b381a (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-5, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=quincy-fbd1a6a, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731) 2024-09-21T09:38:42.018 INFO:tasks.rados.rados.0.smithi022.stdout:2166: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-21T09:38:42.021 INFO:tasks.rados.rados.0.smithi022.stdout:2166: done (2 left) 2024-09-21T09:38:42.021 INFO:tasks.rados.rados.0.smithi022.stdout:2168: done (1 left) 2024-09-21T09:38:42.021 INFO:tasks.rados.rados.0.smithi022.stdout:2169: rollback oid 6 current snap is 215 2024-09-21T09:38:42.021 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 6 to 211 2024-09-21T09:38:42.021 INFO:tasks.rados.rados.0.smithi022.stdout:2170: copy_from oid 30 from oid 16 current snap is 215 2024-09-21T09:38:42.021 INFO:tasks.rados.rados.0.smithi022.stdout:2171: delete oid 41 current snap is 215 2024-09-21T09:38:42.024 INFO:tasks.rados.rados.0.smithi022.stdout:2171: done (3 left) 2024-09-21T09:38:42.024 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 7 current snap is 215 2024-09-21T09:38:42.024 INFO:tasks.rados.rados.0.smithi022.stdout:2172: seq_num 694 ranges {0=425984,425984=655360,1081344=786432,1867776=163840} 2024-09-21T09:38:42.027 INFO:tasks.rados.rados.0.smithi022.stdout:2172: writing smithi02284960-7 from 0 to 425984 tid 1 2024-09-21T09:38:42.031 INFO:tasks.rados.rados.0.smithi022.stdout:2172: writing smithi02284960-7 from 425984 to 1081344 tid 2 2024-09-21T09:38:42.036 INFO:tasks.rados.rados.0.smithi022.stdout:2172: writing smithi02284960-7 from 1081344 to 1867776 tid 3 2024-09-21T09:38:42.039 INFO:tasks.rados.rados.0.smithi022.stdout:2172: writing smithi02284960-7 from 1867776 to 2031616 tid 4 2024-09-21T09:38:42.039 INFO:tasks.rados.rados.0.smithi022.stdout:2173: copy_from oid 1 from oid 24 current snap is 215 2024-09-21T09:38:42.039 INFO:tasks.rados.rados.0.smithi022.stdout:2169: finishing rollback tid 0 to smithi02284960-6 2024-09-21T09:38:42.039 INFO:tasks.rados.rados.0.smithi022.stdout:2169: finishing rollback tid 1 to smithi02284960-6 2024-09-21T09:38:42.039 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 6 v 1308 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-09-21T09:38:42.039 INFO:tasks.rados.rados.0.smithi022.stdout:2167: finishing copy_from racing read to smithi02284960-27 2024-09-21T09:38:42.039 INFO:tasks.rados.rados.0.smithi022.stdout:2172: finishing write tid 1 to smithi02284960-7 2024-09-21T09:38:42.039 INFO:tasks.rados.rados.0.smithi022.stdout:2167: finishing copy_from to smithi02284960-27 2024-09-21T09:38:42.039 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 27 v 1319 (ObjNum 687 snap 214 seq_num 687) dirty exists 2024-09-21T09:38:42.039 INFO:tasks.rados.rados.0.smithi022.stdout:2167: done (4 left) 2024-09-21T09:38:42.039 INFO:tasks.rados.rados.0.smithi022.stdout:2169: done (3 left) 2024-09-21T09:38:42.039 INFO:tasks.rados.rados.0.smithi022.stdout:2174: read oid 27 snap -1 2024-09-21T09:38:42.040 INFO:tasks.rados.rados.0.smithi022.stdout:2174: expect (ObjNum 687 snap 214 seq_num 687) 2024-09-21T09:38:42.040 INFO:tasks.rados.rados.0.smithi022.stdout:2175: copy_from oid 20 from oid 36 current snap is 215 2024-09-21T09:38:42.040 INFO:tasks.rados.rados.0.smithi022.stdout:2176: copy_from oid 9 from oid 47 current snap is 215 2024-09-21T09:38:42.040 INFO:tasks.rados.rados.0.smithi022.stdout:2177: delete oid 43 current snap is 215 2024-09-21T09:38:42.041 INFO:tasks.rados.rados.0.smithi022.stdout:2172: finishing write tid 2 to smithi02284960-7 2024-09-21T09:38:42.041 INFO:tasks.rados.rados.0.smithi022.stdout:2177: done (6 left) 2024-09-21T09:38:42.041 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 5 current snap is 215 2024-09-21T09:38:42.041 INFO:tasks.rados.rados.0.smithi022.stdout:2173: finishing copy_from to smithi02284960-1 2024-09-21T09:38:42.041 INFO:tasks.rados.rados.0.smithi022.stdout:2173: got expected ENOENT (src dne) 2024-09-21T09:38:42.041 INFO:tasks.rados.rados.0.smithi022.stdout:2178: seq_num 696 ranges {9723904=458752,10182656=417792,10600448=753664,11354112=434176} 2024-09-21T09:38:42.097 INFO:tasks.rados.rados.0.smithi022.stdout:2178: writing smithi02284960-5 from 9723904 to 10182656 tid 1 2024-09-21T09:38:42.100 INFO:tasks.rados.rados.0.smithi022.stdout:2178: writing smithi02284960-5 from 10182656 to 10600448 tid 2 2024-09-21T09:38:42.104 INFO:tasks.rados.rados.0.smithi022.stdout:2178: writing smithi02284960-5 from 10600448 to 11354112 tid 3 2024-09-21T09:38:42.107 INFO:tasks.rados.rados.0.smithi022.stdout:2178: writing smithi02284960-5 from 11354112 to 11788288 tid 4 2024-09-21T09:38:42.107 INFO:tasks.rados.rados.0.smithi022.stdout:2179: rollback oid 14 current snap is 215 2024-09-21T09:38:42.107 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 14 to 214 2024-09-21T09:38:42.107 INFO:tasks.rados.rados.0.smithi022.stdout:2173: finishing copy_from racing read to smithi02284960-1 2024-09-21T09:38:42.108 INFO:tasks.rados.rados.0.smithi022.stdout:2172: finishing write tid 3 to smithi02284960-7 2024-09-21T09:38:42.108 INFO:tasks.rados.rados.0.smithi022.stdout:2172: finishing write tid 4 to smithi02284960-7 2024-09-21T09:38:42.108 INFO:tasks.rados.rados.0.smithi022.stdout:2172: finishing write tid 5 to smithi02284960-7 2024-09-21T09:38:42.108 INFO:tasks.rados.rados.0.smithi022.stdout:2172: finishing write tid 6 to smithi02284960-7 2024-09-21T09:38:42.108 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 7 v 1313 (ObjNum 694 snap 215 seq_num 694) dirty exists 2024-09-21T09:38:42.108 INFO:tasks.rados.rados.0.smithi022.stdout:2172: left oid 7 (ObjNum 694 snap 215 seq_num 694) 2024-09-21T09:38:42.108 INFO:tasks.rados.rados.0.smithi022.stdout:2174: expect (ObjNum 687 snap 214 seq_num 687) 2024-09-21T09:38:42.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:42 smithi064 ceph-mon[86709]: pgmap v351: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 231 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 4.3 MiB/s rd, 4.4 MiB/s wr, 10 op/s 2024-09-21T09:38:42.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:42 smithi064 ceph-mon[86709]: osdmap e500: 8 total, 7 up, 8 in 2024-09-21T09:38:42.192 INFO:journalctl@ceph.osd.5.smithi064.stdout:Sep 21 09:38:42 smithi064 podman[96294]: 2024-09-21 09:38:42.030900733 +0000 UTC m=+1.498155088 container cleanup 1900c4af5d2856b1841d667842b4149c2b82a3ef1f49144601922ba7e45b381a (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-5, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , RELEASE=quincy-fbd1a6a, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T09:38:42.192 INFO:journalctl@ceph.osd.5.smithi064.stdout:Sep 21 09:38:42 smithi064 bash[96294]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-5 2024-09-21T09:38:42.232 INFO:tasks.rados.rados.0.smithi022.stdout:2176: finishing copy_from racing read to smithi02284960-9 2024-09-21T09:38:42.232 INFO:tasks.rados.rados.0.smithi022.stdout:2175: finishing copy_from racing read to smithi02284960-20 2024-09-21T09:38:42.232 INFO:tasks.rados.rados.0.smithi022.stdout:2176: finishing copy_from to smithi02284960-9 2024-09-21T09:38:42.232 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 9 v 1117 (ObjNum 604 snap 183 seq_num 604) dirty exists 2024-09-21T09:38:42.232 INFO:tasks.rados.rados.0.smithi022.stdout:2170: finishing copy_from racing read to smithi02284960-30 2024-09-21T09:38:42.232 INFO:tasks.rados.rados.0.smithi022.stdout:2175: finishing copy_from to smithi02284960-20 2024-09-21T09:38:42.232 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 20 v 1108 (ObjNum 580 snap 172 seq_num 580) dirty exists 2024-09-21T09:38:42.232 INFO:tasks.rados.rados.0.smithi022.stdout:2170: finishing copy_from to smithi02284960-30 2024-09-21T09:38:42.232 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 30 v 1052 (ObjNum 569 snap 170 seq_num 569) dirty exists 2024-09-21T09:38:42.232 INFO:tasks.rados.rados.0.smithi022.stdout:2178: finishing write tid 1 to smithi02284960-5 2024-09-21T09:38:42.232 INFO:tasks.rados.rados.0.smithi022.stdout:2178: finishing write tid 2 to smithi02284960-5 2024-09-21T09:38:42.233 INFO:tasks.rados.rados.0.smithi022.stdout:2170: done (7 left) 2024-09-21T09:38:42.233 INFO:tasks.rados.rados.0.smithi022.stdout:2172: done (6 left) 2024-09-21T09:38:42.233 INFO:tasks.rados.rados.0.smithi022.stdout:2173: done (5 left) 2024-09-21T09:38:42.233 INFO:tasks.rados.rados.0.smithi022.stdout:2174: done (4 left) 2024-09-21T09:38:42.233 INFO:tasks.rados.rados.0.smithi022.stdout:2175: done (3 left) 2024-09-21T09:38:42.233 INFO:tasks.rados.rados.0.smithi022.stdout:2176: done (2 left) 2024-09-21T09:38:42.233 INFO:tasks.rados.rados.0.smithi022.stdout:2180: read oid 9 snap -1 2024-09-21T09:38:42.233 INFO:tasks.rados.rados.0.smithi022.stdout:2180: expect (ObjNum 604 snap 183 seq_num 604) 2024-09-21T09:38:42.233 INFO:tasks.rados.rados.0.smithi022.stdout:2181: copy_from oid 23 from oid 44 current snap is 215 2024-09-21T09:38:42.233 INFO:tasks.rados.rados.0.smithi022.stdout:2178: finishing write tid 3 to smithi02284960-5 2024-09-21T09:38:42.233 INFO:tasks.rados.rados.0.smithi022.stdout:2178: finishing write tid 4 to smithi02284960-5 2024-09-21T09:38:42.233 INFO:tasks.rados.rados.0.smithi022.stdout:2179: finishing rollback tid 0 to smithi02284960-14 2024-09-21T09:38:42.233 INFO:tasks.rados.rados.0.smithi022.stdout:2178: finishing write tid 5 to smithi02284960-5 2024-09-21T09:38:42.233 INFO:tasks.rados.rados.0.smithi022.stdout:2179: finishing rollback tid 1 to smithi02284960-14 2024-09-21T09:38:42.234 INFO:tasks.rados.rados.0.smithi022.stdout:2179: finishing rollback tid 2 to smithi02284960-14 2024-09-21T09:38:42.234 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 14 v 1101 (ObjNum 580 snap 172 seq_num 580) dirty exists 2024-09-21T09:38:42.234 INFO:tasks.rados.rados.0.smithi022.stdout:2178: finishing write tid 6 to smithi02284960-5 2024-09-21T09:38:42.234 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 5 v 1122 (ObjNum 696 snap 215 seq_num 696) dirty exists 2024-09-21T09:38:42.234 INFO:tasks.rados.rados.0.smithi022.stdout:2178: left oid 5 (ObjNum 696 snap 215 seq_num 696) 2024-09-21T09:38:42.234 INFO:tasks.rados.rados.0.smithi022.stdout:2178: done (3 left) 2024-09-21T09:38:42.234 INFO:tasks.rados.rados.0.smithi022.stdout:2179: done (2 left) 2024-09-21T09:38:42.234 INFO:tasks.rados.rados.0.smithi022.stdout:2182: snap_remove snap 201 2024-09-21T09:38:42.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:42 smithi022 ceph-mon[98615]: pgmap v351: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 231 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 4.3 MiB/s rd, 4.4 MiB/s wr, 10 op/s 2024-09-21T09:38:42.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:42 smithi022 ceph-mon[98615]: osdmap e500: 8 total, 7 up, 8 in 2024-09-21T09:38:42.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:42 smithi022 ceph-mon[96736]: pgmap v351: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 231 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 4.3 MiB/s rd, 4.4 MiB/s wr, 10 op/s 2024-09-21T09:38:42.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:42 smithi022 ceph-mon[96736]: osdmap e500: 8 total, 7 up, 8 in 2024-09-21T09:38:42.806 INFO:journalctl@ceph.osd.5.smithi064.stdout:Sep 21 09:38:42 smithi064 podman[96314]: 2024-09-21 09:38:42.449782895 +0000 UTC m=+0.866338774 container remove 1900c4af5d2856b1841d667842b4149c2b82a3ef1f49144601922ba7e45b381a (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-5, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, RELEASE=quincy-fbd1a6a, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731) 2024-09-21T09:38:43.035 INFO:tasks.rados.rados.0.smithi022.stdout:2182: done (2 left) 2024-09-21T09:38:43.035 INFO:tasks.rados.rados.0.smithi022.stdout:2183: setattr oid 42 current snap is 215 2024-09-21T09:38:43.035 INFO:tasks.rados.rados.0.smithi022.stdout:2181: finishing copy_from to smithi02284960-23 2024-09-21T09:38:43.035 INFO:tasks.rados.rados.0.smithi022.stdout:2181: got expected ENOENT (src dne) 2024-09-21T09:38:43.035 INFO:tasks.rados.rados.0.smithi022.stdout:2181: finishing copy_from racing read to smithi02284960-23 2024-09-21T09:38:43.035 INFO:tasks.rados.rados.0.smithi022.stdout:2180: expect (ObjNum 604 snap 183 seq_num 604) 2024-09-21T09:38:43.129 INFO:journalctl@ceph.osd.5.smithi064.stdout:Sep 21 09:38:42 smithi064 podman[96410]: 2024-09-21 09:38:42.707830383 +0000 UTC m=+0.016706579 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:38:43.129 INFO:journalctl@ceph.osd.5.smithi064.stdout:Sep 21 09:38:42 smithi064 podman[96410]: 2024-09-21 09:38:42.808700423 +0000 UTC m=+0.117576617 container create dee9a50b92ea72d78b58bb2357ef63113b63d5e275c5608c89164002b450aeb7 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-5-deactivate, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=main-62da54b) 2024-09-21T09:38:43.129 INFO:journalctl@ceph.osd.5.smithi064.stdout:Sep 21 09:38:43 smithi064 podman[96410]: 2024-09-21 09:38:43.019665812 +0000 UTC m=+0.328541999 container init dee9a50b92ea72d78b58bb2357ef63113b63d5e275c5608c89164002b450aeb7 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-5-deactivate, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-62da54b, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731) 2024-09-21T09:38:43.129 INFO:journalctl@ceph.osd.5.smithi064.stdout:Sep 21 09:38:43 smithi064 podman[96410]: 2024-09-21 09:38:43.02473381 +0000 UTC m=+0.333610003 container start dee9a50b92ea72d78b58bb2357ef63113b63d5e275c5608c89164002b450aeb7 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-5-deactivate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-62da54b, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, ceph=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-09-21T09:38:43.129 INFO:journalctl@ceph.osd.5.smithi064.stdout:Sep 21 09:38:43 smithi064 podman[96410]: 2024-09-21 09:38:43.127871256 +0000 UTC m=+0.436747453 container attach dee9a50b92ea72d78b58bb2357ef63113b63d5e275c5608c89164002b450aeb7 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-5-deactivate, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, RELEASE=main-62da54b, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_CLEAN=True, ceph=True) 2024-09-21T09:38:43.347 INFO:tasks.rados.rados.0.smithi022.stdout:2180: done (2 left) 2024-09-21T09:38:43.347 INFO:tasks.rados.rados.0.smithi022.stdout:2181: done (1 left) 2024-09-21T09:38:43.347 INFO:tasks.rados.rados.0.smithi022.stdout:2184: copy_from oid 7 from oid 34 current snap is 215 2024-09-21T09:38:43.347 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 20 current snap is 215 2024-09-21T09:38:43.347 INFO:tasks.rados.rados.0.smithi022.stdout:2185: seq_num 697 ranges {114688=475136,589824=704512,1294336=557056,1851392=344064} 2024-09-21T09:38:43.350 INFO:tasks.rados.rados.0.smithi022.stdout:2185: writing smithi02284960-20 from 114688 to 589824 tid 1 2024-09-21T09:38:43.354 INFO:tasks.rados.rados.0.smithi022.stdout:2185: writing smithi02284960-20 from 589824 to 1294336 tid 2 2024-09-21T09:38:43.357 INFO:tasks.rados.rados.0.smithi022.stdout:2185: writing smithi02284960-20 from 1294336 to 1851392 tid 3 2024-09-21T09:38:43.359 INFO:tasks.rados.rados.0.smithi022.stdout:2185: writing smithi02284960-20 from 1851392 to 2195456 tid 4 2024-09-21T09:38:43.360 INFO:tasks.rados.rados.0.smithi022.stdout:2186: rollback oid 48 current snap is 215 2024-09-21T09:38:43.360 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 48 to 213 2024-09-21T09:38:43.360 INFO:tasks.rados.rados.0.smithi022.stdout:2184: finishing copy_from to smithi02284960-7 2024-09-21T09:38:43.360 INFO:tasks.rados.rados.0.smithi022.stdout:2184: got expected ENOENT (src dne) 2024-09-21T09:38:43.360 INFO:tasks.rados.rados.0.smithi022.stdout:2184: finishing copy_from racing read to smithi02284960-7 2024-09-21T09:38:43.360 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 42 v 1320 (ObjNum 881 snap 0 seq_num 3412041792) dirty exists 2024-09-21T09:38:43.360 INFO:tasks.rados.rados.0.smithi022.stdout:2185: finishing write tid 1 to smithi02284960-20 2024-09-21T09:38:43.360 INFO:tasks.rados.rados.0.smithi022.stdout:2185: finishing write tid 2 to smithi02284960-20 2024-09-21T09:38:43.360 INFO:tasks.rados.rados.0.smithi022.stdout:2183: done (3 left) 2024-09-21T09:38:43.360 INFO:tasks.rados.rados.0.smithi022.stdout:2184: done (2 left) 2024-09-21T09:38:43.360 INFO:tasks.rados.rados.0.smithi022.stdout:2187: delete oid 37 current snap is 215 2024-09-21T09:38:43.362 INFO:tasks.rados.rados.0.smithi022.stdout:2187: done (2 left) 2024-09-21T09:38:43.362 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 9 current snap is 215 2024-09-21T09:38:43.362 INFO:tasks.rados.rados.0.smithi022.stdout:2188: seq_num 699 ranges {4300800=516096,4816896=475136,5292032=573440,5865472=557056} 2024-09-21T09:38:43.388 INFO:tasks.rados.rados.0.smithi022.stdout:2188: writing smithi02284960-9 from 4300800 to 4816896 tid 1 2024-09-21T09:38:43.391 INFO:tasks.rados.rados.0.smithi022.stdout:2188: writing smithi02284960-9 from 4816896 to 5292032 tid 2 2024-09-21T09:38:43.394 INFO:tasks.rados.rados.0.smithi022.stdout:2188: writing smithi02284960-9 from 5292032 to 5865472 tid 3 2024-09-21T09:38:43.398 INFO:tasks.rados.rados.0.smithi022.stdout:2188: writing smithi02284960-9 from 5865472 to 6422528 tid 4 2024-09-21T09:38:43.398 INFO:tasks.rados.rados.0.smithi022.stdout:2189: rollback oid 24 current snap is 215 2024-09-21T09:38:43.398 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 24 to 203 2024-09-21T09:38:43.398 INFO:tasks.rados.rados.0.smithi022.stdout:2185: finishing write tid 3 to smithi02284960-20 2024-09-21T09:38:43.398 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 50 current snap is 215 2024-09-21T09:38:43.398 INFO:tasks.rados.rados.0.smithi022.stdout:2185: finishing write tid 4 to smithi02284960-20 2024-09-21T09:38:43.398 INFO:tasks.rados.rados.0.smithi022.stdout:2190: seq_num 700 ranges {770048=532480,1302528=409600,1712128=737280,2449408=458752} 2024-09-21T09:38:43.404 INFO:tasks.rados.rados.0.smithi022.stdout:2190: writing smithi02284960-50 from 770048 to 1302528 tid 1 2024-09-21T09:38:43.407 INFO:tasks.rados.rados.0.smithi022.stdout:2190: writing smithi02284960-50 from 1302528 to 1712128 tid 2 2024-09-21T09:38:43.411 INFO:tasks.rados.rados.0.smithi022.stdout:2190: writing smithi02284960-50 from 1712128 to 2449408 tid 3 2024-09-21T09:38:43.415 INFO:tasks.rados.rados.0.smithi022.stdout:2190: writing smithi02284960-50 from 2449408 to 2908160 tid 4 2024-09-21T09:38:43.415 INFO:tasks.rados.rados.0.smithi022.stdout:2191: rollback oid 3 current snap is 215 2024-09-21T09:38:43.415 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 3 to 208 2024-09-21T09:38:43.415 INFO:tasks.rados.rados.0.smithi022.stdout:2185: finishing write tid 5 to smithi02284960-20 2024-09-21T09:38:43.415 INFO:tasks.rados.rados.0.smithi022.stdout:2185: finishing write tid 6 to smithi02284960-20 2024-09-21T09:38:43.415 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 20 v 1330 (ObjNum 697 snap 215 seq_num 697) dirty exists 2024-09-21T09:38:43.415 INFO:tasks.rados.rados.0.smithi022.stdout:2185: left oid 20 (ObjNum 697 snap 215 seq_num 697) 2024-09-21T09:38:43.415 INFO:tasks.rados.rados.0.smithi022.stdout:2186: finishing rollback tid 1 to smithi02284960-48 2024-09-21T09:38:43.415 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 48 v 1274 (ObjNum 177 snap 0 seq_num 3412041088) dirty dne 2024-09-21T09:38:43.415 INFO:tasks.rados.rados.0.smithi022.stdout:2188: finishing write tid 1 to smithi02284960-9 2024-09-21T09:38:43.415 INFO:tasks.rados.rados.0.smithi022.stdout:2188: finishing write tid 2 to smithi02284960-9 2024-09-21T09:38:43.416 INFO:tasks.rados.rados.0.smithi022.stdout:2188: finishing write tid 3 to smithi02284960-9 2024-09-21T09:38:43.416 INFO:tasks.rados.rados.0.smithi022.stdout:2185: done (5 left) 2024-09-21T09:38:43.416 INFO:tasks.rados.rados.0.smithi022.stdout:2186: done (4 left) 2024-09-21T09:38:43.416 INFO:tasks.rados.rados.0.smithi022.stdout:2192: snap_create 2024-09-21T09:38:43.416 INFO:tasks.rados.rados.0.smithi022.stdout:2189: finishing rollback tid 1 to smithi02284960-24 2024-09-21T09:38:43.416 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 24 v 1256 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-09-21T09:38:43.416 INFO:tasks.rados.rados.0.smithi022.stdout:2188: finishing write tid 4 to smithi02284960-9 2024-09-21T09:38:43.416 INFO:tasks.rados.rados.0.smithi022.stdout:2188: finishing write tid 5 to smithi02284960-9 2024-09-21T09:38:43.416 INFO:tasks.rados.rados.0.smithi022.stdout:2188: finishing write tid 6 to smithi02284960-9 2024-09-21T09:38:43.416 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 9 v 1127 (ObjNum 699 snap 215 seq_num 699) dirty exists 2024-09-21T09:38:43.416 INFO:tasks.rados.rados.0.smithi022.stdout:2188: left oid 9 (ObjNum 699 snap 215 seq_num 699) 2024-09-21T09:38:43.416 INFO:tasks.rados.rados.0.smithi022.stdout:2190: finishing write tid 1 to smithi02284960-50 2024-09-21T09:38:43.416 INFO:tasks.rados.rados.0.smithi022.stdout:2190: finishing write tid 2 to smithi02284960-50 2024-09-21T09:38:43.416 INFO:tasks.rados.rados.0.smithi022.stdout:2190: finishing write tid 3 to smithi02284960-50 2024-09-21T09:38:43.417 INFO:tasks.rados.rados.0.smithi022.stdout:2190: finishing write tid 4 to smithi02284960-50 2024-09-21T09:38:43.418 INFO:tasks.rados.rados.0.smithi022.stdout:2190: finishing write tid 5 to smithi02284960-50 2024-09-21T09:38:43.418 INFO:tasks.rados.rados.0.smithi022.stdout:2191: finishing rollback tid 0 to smithi02284960-3 2024-09-21T09:38:43.418 INFO:tasks.rados.rados.0.smithi022.stdout:2191: finishing rollback tid 1 to smithi02284960-3 2024-09-21T09:38:43.418 INFO:tasks.rados.rados.0.smithi022.stdout:2190: finishing write tid 6 to smithi02284960-50 2024-09-21T09:38:43.418 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 50 v 1266 (ObjNum 700 snap 215 seq_num 700) dirty exists 2024-09-21T09:38:43.418 INFO:tasks.rados.rados.0.smithi022.stdout:2190: left oid 50 (ObjNum 700 snap 215 seq_num 700) 2024-09-21T09:38:43.418 INFO:tasks.rados.rados.0.smithi022.stdout:2191: finishing rollback tid 2 to smithi02284960-3 2024-09-21T09:38:43.418 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 3 v 607 (ObjNum 675 snap 208 seq_num 675) dirty exists 2024-09-21T09:38:43.441 INFO:journalctl@ceph.osd.5.smithi064.stdout:Sep 21 09:38:43 smithi064 podman[96410]: 2024-09-21 09:38:43.225636706 +0000 UTC m=+0.534512893 container died dee9a50b92ea72d78b58bb2357ef63113b63d5e275c5608c89164002b450aeb7 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-5-deactivate, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=main-62da54b, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True) 2024-09-21T09:38:43.822 INFO:journalctl@ceph.osd.5.smithi064.stdout:Sep 21 09:38:43 smithi064 podman[96453]: 2024-09-21 09:38:43.797963897 +0000 UTC m=+0.566212334 container remove dee9a50b92ea72d78b58bb2357ef63113b63d5e275c5608c89164002b450aeb7 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-5-deactivate, ceph=True, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-62da54b, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS) 2024-09-21T09:38:43.823 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:43 smithi064 ceph-mon[86709]: pgmap v354: 121 pgs: 4 active+undersized, 7 peering, 13 stale+active+clean, 3 active+clean+snaptrim, 2 active+undersized+degraded, 92 active+clean; 233 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 6.4 MiB/s rd, 906 KiB/s wr, 4 op/s; 10/414 objects degraded (2.415%) 2024-09-21T09:38:43.823 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:43 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:38:43.823 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:43 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:38:43.823 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:43 smithi064 ceph-mon[86709]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-21T09:38:43.823 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:43 smithi064 ceph-mon[86709]: Health check failed: Degraded data redundancy: 10/414 objects degraded (2.415%), 2 pgs degraded (PG_DEGRADED) 2024-09-21T09:38:43.823 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:43 smithi064 ceph-mon[86709]: osdmap e501: 8 total, 7 up, 8 in 2024-09-21T09:38:43.996 INFO:tasks.rados.rados.0.smithi022.stdout:2188: done (4 left) 2024-09-21T09:38:43.997 INFO:tasks.rados.rados.0.smithi022.stdout:2189: done (3 left) 2024-09-21T09:38:43.997 INFO:tasks.rados.rados.0.smithi022.stdout:2190: done (2 left) 2024-09-21T09:38:43.997 INFO:tasks.rados.rados.0.smithi022.stdout:2191: done (1 left) 2024-09-21T09:38:43.997 INFO:tasks.rados.rados.0.smithi022.stdout:2192: done (0 left) 2024-09-21T09:38:43.997 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 50 current snap is 216 2024-09-21T09:38:43.997 INFO:tasks.rados.rados.0.smithi022.stdout:2193: seq_num 701 ranges {2908160=548864,3457024=696320,4153344=540672,4694016=368640} 2024-09-21T09:38:44.015 INFO:tasks.rados.rados.0.smithi022.stdout:2193: writing smithi02284960-50 from 2908160 to 3457024 tid 1 2024-09-21T09:38:44.020 INFO:tasks.rados.rados.0.smithi022.stdout:2193: writing smithi02284960-50 from 3457024 to 4153344 tid 2 2024-09-21T09:38:44.023 INFO:tasks.rados.rados.0.smithi022.stdout:2193: writing smithi02284960-50 from 4153344 to 4694016 tid 3 2024-09-21T09:38:44.025 INFO:tasks.rados.rados.0.smithi022.stdout:2193: writing smithi02284960-50 from 4694016 to 5062656 tid 4 2024-09-21T09:38:44.025 INFO:tasks.rados.rados.0.smithi022.stdout:2194: read oid 28 snap 208 2024-09-21T09:38:44.025 INFO:tasks.rados.rados.0.smithi022.stdout:2194: expect (ObjNum 392 snap 122 seq_num 392) 2024-09-21T09:38:44.025 INFO:tasks.rados.rados.0.smithi022.stdout:2195: snap_remove snap 214 2024-09-21T09:38:44.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:43 smithi022 ceph-mon[96736]: pgmap v354: 121 pgs: 4 active+undersized, 7 peering, 13 stale+active+clean, 3 active+clean+snaptrim, 2 active+undersized+degraded, 92 active+clean; 233 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 6.4 MiB/s rd, 906 KiB/s wr, 4 op/s; 10/414 objects degraded (2.415%) 2024-09-21T09:38:44.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:43 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:38:44.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:43 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:38:44.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:43 smithi022 ceph-mon[96736]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-21T09:38:44.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:43 smithi022 ceph-mon[96736]: Health check failed: Degraded data redundancy: 10/414 objects degraded (2.415%), 2 pgs degraded (PG_DEGRADED) 2024-09-21T09:38:44.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:43 smithi022 ceph-mon[96736]: osdmap e501: 8 total, 7 up, 8 in 2024-09-21T09:38:44.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:43 smithi022 ceph-mon[98615]: pgmap v354: 121 pgs: 4 active+undersized, 7 peering, 13 stale+active+clean, 3 active+clean+snaptrim, 2 active+undersized+degraded, 92 active+clean; 233 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 6.4 MiB/s rd, 906 KiB/s wr, 4 op/s; 10/414 objects degraded (2.415%) 2024-09-21T09:38:44.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:43 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:38:44.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:43 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:38:44.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:43 smithi022 ceph-mon[98615]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-21T09:38:44.059 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:43 smithi022 ceph-mon[98615]: Health check failed: Degraded data redundancy: 10/414 objects degraded (2.415%), 2 pgs degraded (PG_DEGRADED) 2024-09-21T09:38:44.059 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:43 smithi022 ceph-mon[98615]: osdmap e501: 8 total, 7 up, 8 in 2024-09-21T09:38:44.191 INFO:journalctl@ceph.osd.5.smithi064.stdout:Sep 21 09:38:43 smithi064 systemd[1]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490@osd.5.service: Deactivated successfully. 2024-09-21T09:38:44.191 INFO:journalctl@ceph.osd.5.smithi064.stdout:Sep 21 09:38:43 smithi064 systemd[1]: Stopped Ceph osd.5 for d65e801e-77fa-11ef-baf6-efdc52797490. 2024-09-21T09:38:44.191 INFO:journalctl@ceph.osd.5.smithi064.stdout:Sep 21 09:38:43 smithi064 systemd[1]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490@osd.5.service: Consumed 17.776s CPU time. 2024-09-21T09:38:44.191 INFO:journalctl@ceph.osd.5.smithi064.stdout:Sep 21 09:38:44 smithi064 systemd[1]: Starting Ceph osd.5 for d65e801e-77fa-11ef-baf6-efdc52797490... 2024-09-21T09:38:44.519 INFO:journalctl@ceph.osd.5.smithi064.stdout:Sep 21 09:38:44 smithi064 podman[96560]: 2024-09-21 09:38:44.162880427 +0000 UTC m=+0.016357323 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:38:44.519 INFO:journalctl@ceph.osd.5.smithi064.stdout:Sep 21 09:38:44 smithi064 podman[96560]: 2024-09-21 09:38:44.291216371 +0000 UTC m=+0.144693290 container create e94f3d1d1c5d664fdb3048136822d8b4c9838bbae0122fe1e76a64f356b581fe (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-5-activate, RELEASE=main-62da54b, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_BRANCH=HEAD) 2024-09-21T09:38:44.519 INFO:journalctl@ceph.osd.5.smithi064.stdout:Sep 21 09:38:44 smithi064 podman[96560]: 2024-09-21 09:38:44.518347697 +0000 UTC m=+0.371824580 container init e94f3d1d1c5d664fdb3048136822d8b4c9838bbae0122fe1e76a64f356b581fe (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-5-activate, RELEASE=main-62da54b, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , ceph=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T09:38:44.941 INFO:journalctl@ceph.osd.5.smithi064.stdout:Sep 21 09:38:44 smithi064 podman[96560]: 2024-09-21 09:38:44.523889499 +0000 UTC m=+0.377366380 container start e94f3d1d1c5d664fdb3048136822d8b4c9838bbae0122fe1e76a64f356b581fe (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-5-activate, ceph=True, GIT_BRANCH=HEAD, RELEASE=main-62da54b, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-09-21T09:38:44.942 INFO:journalctl@ceph.osd.5.smithi064.stdout:Sep 21 09:38:44 smithi064 podman[96560]: 2024-09-21 09:38:44.601487478 +0000 UTC m=+0.454964359 container attach e94f3d1d1c5d664fdb3048136822d8b4c9838bbae0122fe1e76a64f356b581fe (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-5-activate, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=main-62da54b, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , ceph=True) 2024-09-21T09:38:44.942 INFO:journalctl@ceph.osd.5.smithi064.stdout:Sep 21 09:38:44 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-5-activate[96576]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T09:38:44.942 INFO:journalctl@ceph.osd.5.smithi064.stdout:Sep 21 09:38:44 smithi064 bash[96560]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T09:38:44.942 INFO:journalctl@ceph.osd.5.smithi064.stdout:Sep 21 09:38:44 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-5-activate[96576]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T09:38:44.942 INFO:journalctl@ceph.osd.5.smithi064.stdout:Sep 21 09:38:44 smithi064 bash[96560]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T09:38:45.004 INFO:tasks.rados.rados.0.smithi022.stdout:2195: done (2 left) 2024-09-21T09:38:45.004 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 38 current snap is 216 2024-09-21T09:38:45.005 INFO:tasks.rados.rados.0.smithi022.stdout:2196: seq_num 702 ranges {0=565248,565248=581632,1146880=745472,1892352=278528} 2024-09-21T09:38:45.008 INFO:tasks.rados.rados.0.smithi022.stdout:2196: writing smithi02284960-38 from 0 to 565248 tid 1 2024-09-21T09:38:45.011 INFO:tasks.rados.rados.0.smithi022.stdout:2196: writing smithi02284960-38 from 565248 to 1146880 tid 2 2024-09-21T09:38:45.016 INFO:tasks.rados.rados.0.smithi022.stdout:2196: writing smithi02284960-38 from 1146880 to 1892352 tid 3 2024-09-21T09:38:45.018 INFO:tasks.rados.rados.0.smithi022.stdout:2196: writing smithi02284960-38 from 1892352 to 2170880 tid 4 2024-09-21T09:38:45.018 INFO:tasks.rados.rados.0.smithi022.stdout:2197: snap_remove snap 215 2024-09-21T09:38:45.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:44 smithi022 ceph-mon[96736]: osdmap e502: 8 total, 7 up, 8 in 2024-09-21T09:38:45.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:44 smithi022 ceph-mon[98615]: osdmap e502: 8 total, 7 up, 8 in 2024-09-21T09:38:45.421 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:44 smithi064 ceph-mon[86709]: osdmap e502: 8 total, 7 up, 8 in 2024-09-21T09:38:45.421 INFO:journalctl@ceph.osd.5.smithi064.stdout:Sep 21 09:38:45 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-5-activate[96576]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-21T09:38:45.421 INFO:journalctl@ceph.osd.5.smithi064.stdout:Sep 21 09:38:45 smithi064 bash[96560]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-21T09:38:45.421 INFO:journalctl@ceph.osd.5.smithi064.stdout:Sep 21 09:38:45 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-5-activate[96576]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-21T09:38:45.421 INFO:journalctl@ceph.osd.5.smithi064.stdout:Sep 21 09:38:45 smithi064 bash[96560]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-21T09:38:45.692 INFO:journalctl@ceph.osd.5.smithi064.stdout:Sep 21 09:38:45 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-5-activate[96576]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-21T09:38:45.692 INFO:journalctl@ceph.osd.5.smithi064.stdout:Sep 21 09:38:45 smithi064 bash[96560]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-21T09:38:45.692 INFO:journalctl@ceph.osd.5.smithi064.stdout:Sep 21 09:38:45 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-5-activate[96576]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-21T09:38:45.692 INFO:journalctl@ceph.osd.5.smithi064.stdout:Sep 21 09:38:45 smithi064 bash[96560]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-21T09:38:45.692 INFO:journalctl@ceph.osd.5.smithi064.stdout:Sep 21 09:38:45 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-5-activate[96576]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-09-21T09:38:45.692 INFO:journalctl@ceph.osd.5.smithi064.stdout:Sep 21 09:38:45 smithi064 bash[96560]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-09-21T09:38:45.692 INFO:journalctl@ceph.osd.5.smithi064.stdout:Sep 21 09:38:45 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-5-activate[96576]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-21T09:38:45.692 INFO:journalctl@ceph.osd.5.smithi064.stdout:Sep 21 09:38:45 smithi064 bash[96560]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-21T09:38:45.692 INFO:journalctl@ceph.osd.5.smithi064.stdout:Sep 21 09:38:45 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-5-activate[96576]: --> ceph-volume raw activate successful for osd ID: 5 2024-09-21T09:38:45.692 INFO:journalctl@ceph.osd.5.smithi064.stdout:Sep 21 09:38:45 smithi064 bash[96560]: --> ceph-volume raw activate successful for osd ID: 5 2024-09-21T09:38:45.692 INFO:journalctl@ceph.osd.5.smithi064.stdout:Sep 21 09:38:45 smithi064 podman[96560]: 2024-09-21 09:38:45.449518549 +0000 UTC m=+1.302995432 container died e94f3d1d1c5d664fdb3048136822d8b4c9838bbae0122fe1e76a64f356b581fe (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-5-activate, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True, RELEASE=main-62da54b, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.vendor=CentOS) 2024-09-21T09:38:45.999 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:38:45 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:38:45] "GET /metrics HTTP/1.1" 200 36296 "" "Prometheus/2.51.0" 2024-09-21T09:38:46.007 INFO:tasks.rados.rados.0.smithi022.stdout:2197: done (3 left) 2024-09-21T09:38:46.007 INFO:tasks.rados.rados.0.smithi022.stdout:2198: copy_from oid 49 from oid 25 current snap is 216 2024-09-21T09:38:46.007 INFO:tasks.rados.rados.0.smithi022.stdout:2193: finishing write tid 1 to smithi02284960-50 2024-09-21T09:38:46.007 INFO:tasks.rados.rados.0.smithi022.stdout:2193: finishing write tid 2 to smithi02284960-50 2024-09-21T09:38:46.007 INFO:tasks.rados.rados.0.smithi022.stdout:2193: finishing write tid 3 to smithi02284960-50 2024-09-21T09:38:46.007 INFO:tasks.rados.rados.0.smithi022.stdout:2193: finishing write tid 4 to smithi02284960-50 2024-09-21T09:38:46.007 INFO:tasks.rados.rados.0.smithi022.stdout:2193: finishing write tid 5 to smithi02284960-50 2024-09-21T09:38:46.007 INFO:tasks.rados.rados.0.smithi022.stdout:2194: expect (ObjNum 392 snap 122 seq_num 392) 2024-09-21T09:38:46.055 INFO:tasks.rados.rados.0.smithi022.stdout:2193: finishing write tid 6 to smithi02284960-50 2024-09-21T09:38:46.055 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 50 v 1271 (ObjNum 701 snap 216 seq_num 701) dirty exists 2024-09-21T09:38:46.055 INFO:tasks.rados.rados.0.smithi022.stdout:2193: left oid 50 (ObjNum 701 snap 216 seq_num 701) 2024-09-21T09:38:46.055 INFO:tasks.rados.rados.0.smithi022.stdout:2193: done (3 left) 2024-09-21T09:38:46.055 INFO:tasks.rados.rados.0.smithi022.stdout:2194: done (2 left) 2024-09-21T09:38:46.055 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 12 current snap is 216 2024-09-21T09:38:46.055 INFO:tasks.rados.rados.0.smithi022.stdout:2196: finishing write tid 1 to smithi02284960-38 2024-09-21T09:38:46.055 INFO:tasks.rados.rados.0.smithi022.stdout:2199: seq_num 703 ranges {5120000=581632,5701632=458752,6160384=557056,6717440=417792,7135232=172032} 2024-09-21T09:38:46.085 INFO:tasks.rados.rados.0.smithi022.stdout:2199: writing smithi02284960-12 from 5120000 to 5701632 tid 1 2024-09-21T09:38:46.088 INFO:tasks.rados.rados.0.smithi022.stdout:2199: writing smithi02284960-12 from 5701632 to 6160384 tid 2 2024-09-21T09:38:46.091 INFO:tasks.rados.rados.0.smithi022.stdout:2199: writing smithi02284960-12 from 6160384 to 6717440 tid 3 2024-09-21T09:38:46.093 INFO:tasks.rados.rados.0.smithi022.stdout:2199: writing smithi02284960-12 from 6717440 to 7135232 tid 4 2024-09-21T09:38:46.095 INFO:tasks.rados.rados.0.smithi022.stdout:2199: writing smithi02284960-12 from 7135232 to 7307264 tid 5 2024-09-21T09:38:46.095 INFO:tasks.rados.rados.0.smithi022.stdout:2200: rollback oid 19 current snap is 216 2024-09-21T09:38:46.095 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 19 to 211 2024-09-21T09:38:46.095 INFO:tasks.rados.rados.0.smithi022.stdout:2196: finishing write tid 2 to smithi02284960-38 2024-09-21T09:38:46.096 INFO:tasks.rados.rados.0.smithi022.stdout:2196: finishing write tid 3 to smithi02284960-38 2024-09-21T09:38:46.096 INFO:tasks.rados.rados.0.smithi022.stdout:2196: finishing write tid 4 to smithi02284960-38 2024-09-21T09:38:46.096 INFO:tasks.rados.rados.0.smithi022.stdout:2196: finishing write tid 5 to smithi02284960-38 2024-09-21T09:38:46.096 INFO:tasks.rados.rados.0.smithi022.stdout:2201: copy_from oid 40 from oid 5 current snap is 216 2024-09-21T09:38:46.096 INFO:tasks.rados.rados.0.smithi022.stdout:2196: finishing write tid 6 to smithi02284960-38 2024-09-21T09:38:46.096 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 38 v 1060 (ObjNum 702 snap 216 seq_num 702) dirty exists 2024-09-21T09:38:46.096 INFO:tasks.rados.rados.0.smithi022.stdout:2196: left oid 38 (ObjNum 702 snap 216 seq_num 702) 2024-09-21T09:38:46.096 INFO:tasks.rados.rados.0.smithi022.stdout:2198: finishing copy_from to smithi02284960-49 2024-09-21T09:38:46.096 INFO:tasks.rados.rados.0.smithi022.stdout:2198: got expected ENOENT (src dne) 2024-09-21T09:38:46.096 INFO:tasks.rados.rados.0.smithi022.stdout:2198: finishing copy_from racing read to smithi02284960-49 2024-09-21T09:38:46.096 INFO:tasks.rados.rados.0.smithi022.stdout:2199: finishing write tid 1 to smithi02284960-12 2024-09-21T09:38:46.096 INFO:tasks.rados.rados.0.smithi022.stdout:2199: finishing write tid 2 to smithi02284960-12 2024-09-21T09:38:46.096 INFO:tasks.rados.rados.0.smithi022.stdout:2199: finishing write tid 3 to smithi02284960-12 2024-09-21T09:38:46.096 INFO:tasks.rados.rados.0.smithi022.stdout:2196: done (4 left) 2024-09-21T09:38:46.096 INFO:tasks.rados.rados.0.smithi022.stdout:2198: done (3 left) 2024-09-21T09:38:46.096 INFO:tasks.rados.rados.0.smithi022.stdout:2202: snap_create 2024-09-21T09:38:46.100 INFO:tasks.rados.rados.0.smithi022.stdout:2199: finishing write tid 4 to smithi02284960-12 2024-09-21T09:38:46.100 INFO:tasks.rados.rados.0.smithi022.stdout:2199: finishing write tid 5 to smithi02284960-12 2024-09-21T09:38:46.100 INFO:tasks.rados.rados.0.smithi022.stdout:2199: finishing write tid 6 to smithi02284960-12 2024-09-21T09:38:46.102 INFO:tasks.rados.rados.0.smithi022.stdout:2200: finishing rollback tid 1 to smithi02284960-19 2024-09-21T09:38:46.102 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 19 v 0 (ObjNum 49 snap 0 seq_num 1953066355) dirty dne 2024-09-21T09:38:46.103 INFO:tasks.rados.rados.0.smithi022.stdout:2199: finishing write tid 7 to smithi02284960-12 2024-09-21T09:38:46.103 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 12 v 1346 (ObjNum 703 snap 216 seq_num 703) dirty exists 2024-09-21T09:38:46.103 INFO:tasks.rados.rados.0.smithi022.stdout:2199: left oid 12 (ObjNum 703 snap 216 seq_num 703) 2024-09-21T09:38:46.142 INFO:tasks.rados.rados.0.smithi022.stdout:2201: finishing copy_from racing read to smithi02284960-40 2024-09-21T09:38:46.144 INFO:tasks.rados.rados.0.smithi022.stdout:2201: finishing copy_from to smithi02284960-40 2024-09-21T09:38:46.144 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 40 v 1122 (ObjNum 696 snap 215 seq_num 696) dirty exists 2024-09-21T09:38:46.147 INFO:teuthology.orchestra.run.smithi022.stdout:true 2024-09-21T09:38:46.274 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:45 smithi064 ceph-mon[86709]: pgmap v357: 121 pgs: 27 active+undersized, 7 peering, 1 stale+active+clean, 1 active+clean+snaptrim, 8 active+undersized+degraded, 77 active+clean; 236 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 5.8 MiB/s rd, 1.7 MiB/s wr, 11 op/s; 44/423 objects degraded (10.402%) 2024-09-21T09:38:46.274 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:45 smithi064 ceph-mon[86709]: osdmap e503: 8 total, 7 up, 8 in 2024-09-21T09:38:46.274 INFO:journalctl@ceph.osd.5.smithi064.stdout:Sep 21 09:38:45 smithi064 podman[96720]: 2024-09-21 09:38:45.996870504 +0000 UTC m=+0.541400734 container remove e94f3d1d1c5d664fdb3048136822d8b4c9838bbae0122fe1e76a64f356b581fe (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-5-activate, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-62da54b, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=) 2024-09-21T09:38:46.274 INFO:journalctl@ceph.osd.5.smithi064.stdout:Sep 21 09:38:46 smithi064 podman[96784]: 2024-09-21 09:38:46.119738728 +0000 UTC m=+0.018471276 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:38:46.307 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:45 smithi022 ceph-mon[98615]: pgmap v357: 121 pgs: 27 active+undersized, 7 peering, 1 stale+active+clean, 1 active+clean+snaptrim, 8 active+undersized+degraded, 77 active+clean; 236 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 5.8 MiB/s rd, 1.7 MiB/s wr, 11 op/s; 44/423 objects degraded (10.402%) 2024-09-21T09:38:46.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:45 smithi022 ceph-mon[98615]: osdmap e503: 8 total, 7 up, 8 in 2024-09-21T09:38:46.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:45 smithi022 ceph-mon[96736]: pgmap v357: 121 pgs: 27 active+undersized, 7 peering, 1 stale+active+clean, 1 active+clean+snaptrim, 8 active+undersized+degraded, 77 active+clean; 236 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 5.8 MiB/s rd, 1.7 MiB/s wr, 11 op/s; 44/423 objects degraded (10.402%) 2024-09-21T09:38:46.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:45 smithi022 ceph-mon[96736]: osdmap e503: 8 total, 7 up, 8 in 2024-09-21T09:38:46.576 INFO:journalctl@ceph.osd.5.smithi064.stdout:Sep 21 09:38:46 smithi064 podman[96784]: 2024-09-21 09:38:46.272849417 +0000 UTC m=+0.171581999 container create b3225b090dba337265d1ede30df4d2a244593ae6b599825f9b9418db3ffa823e (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, ceph=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-62da54b, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731) 2024-09-21T09:38:46.576 INFO:journalctl@ceph.osd.5.smithi064.stdout:Sep 21 09:38:46 smithi064 podman[96784]: 2024-09-21 09:38:46.491540313 +0000 UTC m=+0.390272852 container init b3225b090dba337265d1ede30df4d2a244593ae6b599825f9b9418db3ffa823e (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-5, org.label-schema.build-date=20240731, RELEASE=main-62da54b, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD) 2024-09-21T09:38:46.576 INFO:journalctl@ceph.osd.5.smithi064.stdout:Sep 21 09:38:46 smithi064 podman[96784]: 2024-09-21 09:38:46.4962418 +0000 UTC m=+0.394974340 container start b3225b090dba337265d1ede30df4d2a244593ae6b599825f9b9418db3ffa823e (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-5, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.37.2, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, RELEASE=main-62da54b) 2024-09-21T09:38:46.576 INFO:journalctl@ceph.osd.5.smithi064.stdout:Sep 21 09:38:46 smithi064 ceph-osd[96822]: -- 172.21.15.64:0/1517340356 <== mon.1 v2:172.21.15.64:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55fb4aa49a40 con 0x55fb4b824000 2024-09-21T09:38:46.732 INFO:teuthology.orchestra.run.smithi022.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T09:38:46.732 INFO:teuthology.orchestra.run.smithi022.stdout:alertmanager.a smithi022 *:9093,9094 running (5m) 64s ago 10m 22.0M - 0.27.0 11f11916f8cd caced80ba544 2024-09-21T09:38:46.732 INFO:teuthology.orchestra.run.smithi022.stdout:grafana.a smithi064 *:3000 running (5m) 30s ago 9m 83.5M - 9.4.7 954c08fa6188 527bb657d70d 2024-09-21T09:38:46.732 INFO:teuthology.orchestra.run.smithi022.stdout:mds.a smithi022 running (11m) 64s ago 11m 18.1M - 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 34e4221f3a27 2024-09-21T09:38:46.733 INFO:teuthology.orchestra.run.smithi022.stdout:mds.b smithi064 running (11m) 30s ago 11m 22.5M - 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 0f846b3ec694 2024-09-21T09:38:46.733 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.x smithi064 *:8443,9283,8765 running (4m) 30s ago 14m 455M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 61abc61fea8d 2024-09-21T09:38:46.733 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.y smithi022 *:8443,9283,8765 running (4m) 64s ago 16m 517M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 6c5bee78f09b 2024-09-21T09:38:46.733 INFO:teuthology.orchestra.run.smithi022.stdout:mon.a smithi022 running (4m) 64s ago 16m 58.2M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 0513395c784d 2024-09-21T09:38:46.733 INFO:teuthology.orchestra.run.smithi022.stdout:mon.b smithi064 running (3m) 30s ago 15m 48.0M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 9dab26b6a9e7 2024-09-21T09:38:46.733 INFO:teuthology.orchestra.run.smithi022.stdout:mon.c smithi022 running (3m) 64s ago 15m 49.3M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 c74b58b67bd6 2024-09-21T09:38:46.733 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.a smithi022 *:9100 running (5m) 64s ago 9m 11.1M - 1.7.0 72c9c2088986 593c6158f524 2024-09-21T09:38:46.733 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.b smithi064 *:9100 running (5m) 30s ago 9m 11.1M - 1.7.0 72c9c2088986 a533420b0139 2024-09-21T09:38:46.733 INFO:teuthology.orchestra.run.smithi022.stdout:osd.0 smithi022 running (2m) 64s ago 14m 141M 2481M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 40ce23a2b569 2024-09-21T09:38:46.733 INFO:teuthology.orchestra.run.smithi022.stdout:osd.1 smithi022 running (2m) 64s ago 14m 115M 2481M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 7cd0c18d224c 2024-09-21T09:38:46.733 INFO:teuthology.orchestra.run.smithi022.stdout:osd.2 smithi022 running (102s) 64s ago 13m 108M 2481M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 b8c944306698 2024-09-21T09:38:46.733 INFO:teuthology.orchestra.run.smithi022.stdout:osd.3 smithi022 running (68s) 64s ago 13m 12.2M 2481M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 c90b27f61bb6 2024-09-21T09:38:46.733 INFO:teuthology.orchestra.run.smithi022.stdout:osd.4 smithi064 running (35s) 30s ago 12m 14.6M 2480M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 936099dbca14 2024-09-21T09:38:46.734 INFO:teuthology.orchestra.run.smithi022.stdout:osd.5 smithi064 starting - - - 2480M 2024-09-21T09:38:46.734 INFO:teuthology.orchestra.run.smithi022.stdout:osd.6 smithi064 running (12m) 30s ago 12m 168M 2480M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 3cc4ec816e5b 2024-09-21T09:38:46.734 INFO:teuthology.orchestra.run.smithi022.stdout:osd.7 smithi064 running (11m) 30s ago 11m 197M 2480M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 18ef0be94104 2024-09-21T09:38:46.734 INFO:teuthology.orchestra.run.smithi022.stdout:prometheus.a smithi064 *:9095 running (4m) 30s ago 10m 52.4M - 2.51.0 1d3b7f56885b 80629ffa2179 2024-09-21T09:38:46.941 INFO:journalctl@ceph.osd.5.smithi064.stdout:Sep 21 09:38:46 smithi064 bash[96784]: b3225b090dba337265d1ede30df4d2a244593ae6b599825f9b9418db3ffa823e 2024-09-21T09:38:46.941 INFO:journalctl@ceph.osd.5.smithi064.stdout:Sep 21 09:38:46 smithi064 systemd[1]: Started Ceph osd.5 for d65e801e-77fa-11ef-baf6-efdc52797490. 2024-09-21T09:38:47.009 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:47 smithi022 ceph-mon[96736]: osdmap e504: 8 total, 7 up, 8 in 2024-09-21T09:38:47.012 INFO:tasks.rados.rados.0.smithi022.stdout:2199: done (3 left) 2024-09-21T09:38:47.012 INFO:tasks.rados.rados.0.smithi022.stdout:2200: done (2 left) 2024-09-21T09:38:47.013 INFO:tasks.rados.rados.0.smithi022.stdout:2201: done (1 left) 2024-09-21T09:38:47.013 INFO:tasks.rados.rados.0.smithi022.stdout:2202: done (0 left) 2024-09-21T09:38:47.013 INFO:tasks.rados.rados.0.smithi022.stdout:2203: copy_from oid 35 from oid 2 current snap is 217 2024-09-21T09:38:47.013 INFO:tasks.rados.rados.0.smithi022.stdout:2204: copy_from oid 24 from oid 12 current snap is 217 2024-09-21T09:38:47.013 INFO:tasks.rados.rados.0.smithi022.stdout:2205: copy_from oid 44 from oid 36 current snap is 217 2024-09-21T09:38:47.013 INFO:tasks.rados.rados.0.smithi022.stdout:2206: snap_create 2024-09-21T09:38:47.019 INFO:tasks.rados.rados.0.smithi022.stdout:2203: finishing copy_from racing read to smithi02284960-35 2024-09-21T09:38:47.019 INFO:tasks.rados.rados.0.smithi022.stdout:2205: finishing copy_from racing read to smithi02284960-44 2024-09-21T09:38:47.019 INFO:tasks.rados.rados.0.smithi022.stdout:2203: finishing copy_from to smithi02284960-35 2024-09-21T09:38:47.020 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 35 v 1254 (ObjNum 590 snap 176 seq_num 590) dirty exists 2024-09-21T09:38:47.021 INFO:tasks.rados.rados.0.smithi022.stdout:2205: finishing copy_from to smithi02284960-44 2024-09-21T09:38:47.021 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 44 v 1108 (ObjNum 580 snap 172 seq_num 580) dirty exists 2024-09-21T09:38:47.038 INFO:tasks.rados.rados.0.smithi022.stdout:2204: finishing copy_from racing read to smithi02284960-24 2024-09-21T09:38:47.041 INFO:tasks.rados.rados.0.smithi022.stdout:2204: finishing copy_from to smithi02284960-24 2024-09-21T09:38:47.041 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 24 v 1346 (ObjNum 703 snap 216 seq_num 703) dirty exists 2024-09-21T09:38:47.098 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-09-21T09:38:47.098 INFO:teuthology.orchestra.run.smithi022.stdout: "mon": { 2024-09-21T09:38:47.098 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 3 2024-09-21T09:38:47.098 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-21T09:38:47.098 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr": { 2024-09-21T09:38:47.098 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 2 2024-09-21T09:38:47.099 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-21T09:38:47.099 INFO:teuthology.orchestra.run.smithi022.stdout: "osd": { 2024-09-21T09:38:47.099 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 2, 2024-09-21T09:38:47.099 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 5 2024-09-21T09:38:47.099 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-21T09:38:47.099 INFO:teuthology.orchestra.run.smithi022.stdout: "mds": { 2024-09-21T09:38:47.099 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 2 2024-09-21T09:38:47.099 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-21T09:38:47.099 INFO:teuthology.orchestra.run.smithi022.stdout: "overall": { 2024-09-21T09:38:47.099 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 4, 2024-09-21T09:38:47.099 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 10 2024-09-21T09:38:47.099 INFO:teuthology.orchestra.run.smithi022.stdout: } 2024-09-21T09:38:47.099 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-09-21T09:38:47.261 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:47 smithi064 ceph-mon[86709]: osdmap e504: 8 total, 7 up, 8 in 2024-09-21T09:38:47.262 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:47 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:38:47.262 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:47 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:38:47.262 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:47 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:38:47.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:47 smithi022 ceph-mon[98615]: osdmap e504: 8 total, 7 up, 8 in 2024-09-21T09:38:47.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:47 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:38:47.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:47 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:38:47.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:47 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:38:47.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:47 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:38:47.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:47 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:38:47.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:47 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:38:47.396 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-09-21T09:38:47.396 INFO:teuthology.orchestra.run.smithi022.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47", 2024-09-21T09:38:47.396 INFO:teuthology.orchestra.run.smithi022.stdout: "in_progress": true, 2024-09-21T09:38:47.396 INFO:teuthology.orchestra.run.smithi022.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T09:38:47.396 INFO:teuthology.orchestra.run.smithi022.stdout: "services_complete": [ 2024-09-21T09:38:47.396 INFO:teuthology.orchestra.run.smithi022.stdout: "mon", 2024-09-21T09:38:47.396 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr" 2024-09-21T09:38:47.396 INFO:teuthology.orchestra.run.smithi022.stdout: ], 2024-09-21T09:38:47.396 INFO:teuthology.orchestra.run.smithi022.stdout: "progress": "10/20 daemons upgraded", 2024-09-21T09:38:47.396 INFO:teuthology.orchestra.run.smithi022.stdout: "message": "Currently upgrading osd daemons", 2024-09-21T09:38:47.396 INFO:teuthology.orchestra.run.smithi022.stdout: "is_paused": false 2024-09-21T09:38:47.396 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-09-21T09:38:48.020 INFO:tasks.rados.rados.0.smithi022.stdout:2203: done (3 left) 2024-09-21T09:38:48.020 INFO:tasks.rados.rados.0.smithi022.stdout:2204: done (2 left) 2024-09-21T09:38:48.020 INFO:tasks.rados.rados.0.smithi022.stdout:2205: done (1 left) 2024-09-21T09:38:48.020 INFO:tasks.rados.rados.0.smithi022.stdout:2206: done (0 left) 2024-09-21T09:38:48.020 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 45 current snap is 218 2024-09-21T09:38:48.020 INFO:tasks.rados.rados.0.smithi022.stdout:2207: seq_num 704 ranges {1982464=598016,2580480=745472,3325952=712704,4038656=147456} 2024-09-21T09:38:48.034 INFO:tasks.rados.rados.0.smithi022.stdout:2207: writing smithi02284960-45 from 1982464 to 2580480 tid 1 2024-09-21T09:38:48.038 INFO:tasks.rados.rados.0.smithi022.stdout:2207: writing smithi02284960-45 from 2580480 to 3325952 tid 2 2024-09-21T09:38:48.042 INFO:tasks.rados.rados.0.smithi022.stdout:2207: writing smithi02284960-45 from 3325952 to 4038656 tid 3 2024-09-21T09:38:48.043 INFO:tasks.rados.rados.0.smithi022.stdout:2207: writing smithi02284960-45 from 4038656 to 4186112 tid 4 2024-09-21T09:38:48.043 INFO:tasks.rados.rados.0.smithi022.stdout:2208: rollback oid 15 current snap is 218 2024-09-21T09:38:48.044 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 15 to 208 2024-09-21T09:38:48.044 INFO:tasks.rados.rados.0.smithi022.stdout:2207: finishing write tid 1 to smithi02284960-45 2024-09-21T09:38:48.044 INFO:tasks.rados.rados.0.smithi022.stdout:2207: finishing write tid 2 to smithi02284960-45 2024-09-21T09:38:48.044 INFO:tasks.rados.rados.0.smithi022.stdout:2209: snap_remove snap 213 2024-09-21T09:38:48.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:48 smithi022 ceph-mon[98615]: from='client.34386 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:48.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:48 smithi022 ceph-mon[98615]: pgmap v360: 121 pgs: 36 active+undersized, 13 active+undersized+degraded, 72 active+clean; 257 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 5.7 MiB/s rd, 6.5 MiB/s wr, 20 op/s; 75/432 objects degraded (17.361%) 2024-09-21T09:38:48.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:48 smithi022 ceph-mon[98615]: from='client.44318 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:48.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:48 smithi022 ceph-mon[98615]: from='client.44327 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:48.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:48 smithi022 ceph-mon[98615]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-21T09:38:48.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:48 smithi022 ceph-mon[98615]: osdmap e505: 8 total, 7 up, 8 in 2024-09-21T09:38:48.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:48 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/4281652948' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:38:48.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:48 smithi022 ceph-mon[96736]: from='client.34386 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:48.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:48 smithi022 ceph-mon[96736]: pgmap v360: 121 pgs: 36 active+undersized, 13 active+undersized+degraded, 72 active+clean; 257 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 5.7 MiB/s rd, 6.5 MiB/s wr, 20 op/s; 75/432 objects degraded (17.361%) 2024-09-21T09:38:48.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:48 smithi022 ceph-mon[96736]: from='client.44318 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:48.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:48 smithi022 ceph-mon[96736]: from='client.44327 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:48.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:48 smithi022 ceph-mon[96736]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-21T09:38:48.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:48 smithi022 ceph-mon[96736]: osdmap e505: 8 total, 7 up, 8 in 2024-09-21T09:38:48.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:48 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/4281652948' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:38:48.373 INFO:journalctl@ceph.osd.5.smithi064.stdout:Sep 21 09:38:48 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-5[96798]: 2024-09-21T09:38:48.140+0000 7fe95adcf640 -1 Falling back to public interface 2024-09-21T09:38:48.374 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:48 smithi064 ceph-mon[86709]: from='client.34386 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:48.374 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:48 smithi064 ceph-mon[86709]: pgmap v360: 121 pgs: 36 active+undersized, 13 active+undersized+degraded, 72 active+clean; 257 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 5.7 MiB/s rd, 6.5 MiB/s wr, 20 op/s; 75/432 objects degraded (17.361%) 2024-09-21T09:38:48.374 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:48 smithi064 ceph-mon[86709]: from='client.44318 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:48.374 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:48 smithi064 ceph-mon[86709]: from='client.44327 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:48.374 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:48 smithi064 ceph-mon[86709]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-21T09:38:48.374 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:48 smithi064 ceph-mon[86709]: osdmap e505: 8 total, 7 up, 8 in 2024-09-21T09:38:48.374 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:48 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/4281652948' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:38:48.999 INFO:tasks.rados.rados.0.smithi022.stdout:2209: done (2 left) 2024-09-21T09:38:48.999 INFO:tasks.rados.rados.0.smithi022.stdout:2210: read oid 8 snap 194 2024-09-21T09:38:48.999 INFO:tasks.rados.rados.0.smithi022.stdout:2210: expect deleted 2024-09-21T09:38:48.999 INFO:tasks.rados.rados.0.smithi022.stdout:2211: rollback oid 47 current snap is 218 2024-09-21T09:38:48.999 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 47 to 217 2024-09-21T09:38:49.000 INFO:tasks.rados.rados.0.smithi022.stdout:2207: finishing write tid 3 to smithi02284960-45 2024-09-21T09:38:49.000 INFO:tasks.rados.rados.0.smithi022.stdout:2207: finishing write tid 4 to smithi02284960-45 2024-09-21T09:38:49.000 INFO:tasks.rados.rados.0.smithi022.stdout:2207: finishing write tid 5 to smithi02284960-45 2024-09-21T09:38:49.000 INFO:tasks.rados.rados.0.smithi022.stdout:2207: finishing write tid 6 to smithi02284960-45 2024-09-21T09:38:49.000 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 45 v 1276 (ObjNum 704 snap 218 seq_num 704) dirty exists 2024-09-21T09:38:49.000 INFO:tasks.rados.rados.0.smithi022.stdout:2207: left oid 45 (ObjNum 704 snap 218 seq_num 704) 2024-09-21T09:38:49.000 INFO:tasks.rados.rados.0.smithi022.stdout:2208: finishing rollback tid 1 to smithi02284960-15 2024-09-21T09:38:49.000 INFO:tasks.rados.rados.0.smithi022.stdout:2208: finishing rollback tid 2 to smithi02284960-15 2024-09-21T09:38:49.000 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 15 v 1350 (ObjNum 642 snap 197 seq_num 642) dirty exists 2024-09-21T09:38:49.000 INFO:tasks.rados.rados.0.smithi022.stdout:2207: done (3 left) 2024-09-21T09:38:49.000 INFO:tasks.rados.rados.0.smithi022.stdout:2208: done (2 left) 2024-09-21T09:38:49.000 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 49 current snap is 218 2024-09-21T09:38:49.000 INFO:tasks.rados.rados.0.smithi022.stdout:2212: seq_num 705 ranges {11649024=614400,12263424=630784,12894208=524288,13418496=450560} 2024-09-21T09:38:49.067 INFO:tasks.rados.rados.0.smithi022.stdout:2212: writing smithi02284960-49 from 11649024 to 12263424 tid 1 2024-09-21T09:38:49.071 INFO:tasks.rados.rados.0.smithi022.stdout:2212: writing smithi02284960-49 from 12263424 to 12894208 tid 2 2024-09-21T09:38:49.074 INFO:tasks.rados.rados.0.smithi022.stdout:2212: writing smithi02284960-49 from 12894208 to 13418496 tid 3 2024-09-21T09:38:49.077 INFO:tasks.rados.rados.0.smithi022.stdout:2212: writing smithi02284960-49 from 13418496 to 13869056 tid 4 2024-09-21T09:38:49.077 INFO:tasks.rados.rados.0.smithi022.stdout:2213: copy_from oid 32 from oid 11 current snap is 218 2024-09-21T09:38:49.077 INFO:tasks.rados.rados.0.smithi022.stdout:2211: finishing rollback tid 0 to smithi02284960-47 2024-09-21T09:38:49.077 INFO:tasks.rados.rados.0.smithi022.stdout:2211: finishing rollback tid 1 to smithi02284960-47 2024-09-21T09:38:49.077 INFO:tasks.rados.rados.0.smithi022.stdout:2211: finishing rollback tid 2 to smithi02284960-47 2024-09-21T09:38:49.077 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 47 v 1145 (ObjNum 604 snap 183 seq_num 604) dirty exists 2024-09-21T09:38:49.077 INFO:tasks.rados.rados.0.smithi022.stdout:2212: finishing write tid 1 to smithi02284960-49 2024-09-21T09:38:49.077 INFO:tasks.rados.rados.0.smithi022.stdout:2212: finishing write tid 2 to smithi02284960-49 2024-09-21T09:38:49.077 INFO:tasks.rados.rados.0.smithi022.stdout:2212: finishing write tid 3 to smithi02284960-49 2024-09-21T09:38:49.077 INFO:tasks.rados.rados.0.smithi022.stdout:2210: done (3 left) 2024-09-21T09:38:49.078 INFO:tasks.rados.rados.0.smithi022.stdout:2211: done (2 left) 2024-09-21T09:38:49.078 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 1 current snap is 218 2024-09-21T09:38:49.078 INFO:tasks.rados.rados.0.smithi022.stdout:2214: seq_num 706 ranges {3563520=630784,4194304=516096,4710400=729088,5439488=360448} 2024-09-21T09:38:49.100 INFO:tasks.rados.rados.0.smithi022.stdout:2214: writing smithi02284960-1 from 3563520 to 4194304 tid 1 2024-09-21T09:38:49.103 INFO:tasks.rados.rados.0.smithi022.stdout:2214: writing smithi02284960-1 from 4194304 to 4710400 tid 2 2024-09-21T09:38:49.108 INFO:tasks.rados.rados.0.smithi022.stdout:2214: writing smithi02284960-1 from 4710400 to 5439488 tid 3 2024-09-21T09:38:49.111 INFO:tasks.rados.rados.0.smithi022.stdout:2214: writing smithi02284960-1 from 5439488 to 5799936 tid 4 2024-09-21T09:38:49.111 INFO:tasks.rados.rados.0.smithi022.stdout:2215: snap_remove snap 211 2024-09-21T09:38:49.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:49 smithi022 ceph-mon[98615]: from='client.34413 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:49.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:49 smithi022 ceph-mon[98615]: osdmap e506: 8 total, 7 up, 8 in 2024-09-21T09:38:49.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:49 smithi022 ceph-mon[98615]: Health check update: Degraded data redundancy: 75/432 objects degraded (17.361%), 13 pgs degraded (PG_DEGRADED) 2024-09-21T09:38:49.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:49 smithi022 ceph-mon[98615]: osdmap e507: 8 total, 7 up, 8 in 2024-09-21T09:38:49.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:49 smithi022 ceph-mon[96736]: from='client.34413 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:49.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:49 smithi022 ceph-mon[96736]: osdmap e506: 8 total, 7 up, 8 in 2024-09-21T09:38:49.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:49 smithi022 ceph-mon[96736]: Health check update: Degraded data redundancy: 75/432 objects degraded (17.361%), 13 pgs degraded (PG_DEGRADED) 2024-09-21T09:38:49.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:49 smithi022 ceph-mon[96736]: osdmap e507: 8 total, 7 up, 8 in 2024-09-21T09:38:49.353 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:49 smithi064 ceph-mon[86709]: from='client.34413 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:49.353 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:49 smithi064 ceph-mon[86709]: osdmap e506: 8 total, 7 up, 8 in 2024-09-21T09:38:49.353 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:49 smithi064 ceph-mon[86709]: Health check update: Degraded data redundancy: 75/432 objects degraded (17.361%), 13 pgs degraded (PG_DEGRADED) 2024-09-21T09:38:49.353 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:49 smithi064 ceph-mon[86709]: osdmap e507: 8 total, 7 up, 8 in 2024-09-21T09:38:50.025 INFO:tasks.rados.rados.0.smithi022.stdout:2215: done (3 left) 2024-09-21T09:38:50.026 INFO:tasks.rados.rados.0.smithi022.stdout:2216: snap_remove snap 203 2024-09-21T09:38:50.148 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:50 smithi064 ceph-mon[86709]: pgmap v363: 121 pgs: 36 active+undersized, 13 active+undersized+degraded, 72 active+clean; 287 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 4.4 MiB/s rd, 12 MiB/s wr, 20 op/s; 76/438 objects degraded (17.352%) 2024-09-21T09:38:50.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:50 smithi022 ceph-mon[98615]: pgmap v363: 121 pgs: 36 active+undersized, 13 active+undersized+degraded, 72 active+clean; 287 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 4.4 MiB/s rd, 12 MiB/s wr, 20 op/s; 76/438 objects degraded (17.352%) 2024-09-21T09:38:50.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:50 smithi022 ceph-mon[96736]: pgmap v363: 121 pgs: 36 active+undersized, 13 active+undersized+degraded, 72 active+clean; 287 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 4.4 MiB/s rd, 12 MiB/s wr, 20 op/s; 76/438 objects degraded (17.352%) 2024-09-21T09:38:50.441 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:38:50 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:38:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:38:51.037 INFO:tasks.rados.rados.0.smithi022.stdout:2216: done (3 left) 2024-09-21T09:38:51.037 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 40 current snap is 218 2024-09-21T09:38:51.037 INFO:tasks.rados.rados.0.smithi022.stdout:2212: finishing write tid 4 to smithi02284960-49 2024-09-21T09:38:51.037 INFO:tasks.rados.rados.0.smithi022.stdout:2217: seq_num 707 ranges {11788288=647168,12435456=802816,13238272=532480,13770752=270336} 2024-09-21T09:38:51.105 INFO:tasks.rados.rados.0.smithi022.stdout:2217: writing smithi02284960-40 from 11788288 to 12435456 tid 1 2024-09-21T09:38:51.109 INFO:tasks.rados.rados.0.smithi022.stdout:2217: writing smithi02284960-40 from 12435456 to 13238272 tid 2 2024-09-21T09:38:51.112 INFO:tasks.rados.rados.0.smithi022.stdout:2217: writing smithi02284960-40 from 13238272 to 13770752 tid 3 2024-09-21T09:38:51.114 INFO:tasks.rados.rados.0.smithi022.stdout:2217: writing smithi02284960-40 from 13770752 to 14041088 tid 4 2024-09-21T09:38:51.115 INFO:tasks.rados.rados.0.smithi022.stdout:2218: setattr oid 39 current snap is 218 2024-09-21T09:38:51.115 INFO:tasks.rados.rados.0.smithi022.stdout:2212: finishing write tid 5 to smithi02284960-49 2024-09-21T09:38:51.115 INFO:tasks.rados.rados.0.smithi022.stdout:2212: finishing write tid 6 to smithi02284960-49 2024-09-21T09:38:51.115 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 49 v 1106 (ObjNum 705 snap 218 seq_num 705) dirty exists 2024-09-21T09:38:51.115 INFO:tasks.rados.rados.0.smithi022.stdout:2212: left oid 49 (ObjNum 705 snap 218 seq_num 705) 2024-09-21T09:38:51.115 INFO:tasks.rados.rados.0.smithi022.stdout:2214: finishing write tid 1 to smithi02284960-1 2024-09-21T09:38:51.115 INFO:tasks.rados.rados.0.smithi022.stdout:2214: finishing write tid 2 to smithi02284960-1 2024-09-21T09:38:51.115 INFO:tasks.rados.rados.0.smithi022.stdout:2214: finishing write tid 3 to smithi02284960-1 2024-09-21T09:38:51.115 INFO:tasks.rados.rados.0.smithi022.stdout:2214: finishing write tid 4 to smithi02284960-1 2024-09-21T09:38:51.115 INFO:tasks.rados.rados.0.smithi022.stdout:2214: finishing write tid 5 to smithi02284960-1 2024-09-21T09:38:51.115 INFO:tasks.rados.rados.0.smithi022.stdout:2213: finishing copy_from racing read to smithi02284960-32 2024-09-21T09:38:51.115 INFO:tasks.rados.rados.0.smithi022.stdout:2214: finishing write tid 6 to smithi02284960-1 2024-09-21T09:38:51.115 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 1 v 1151 (ObjNum 706 snap 218 seq_num 706) dirty exists 2024-09-21T09:38:51.115 INFO:tasks.rados.rados.0.smithi022.stdout:2214: left oid 1 (ObjNum 706 snap 218 seq_num 706) 2024-09-21T09:38:51.115 INFO:tasks.rados.rados.0.smithi022.stdout:2213: finishing copy_from to smithi02284960-32 2024-09-21T09:38:51.115 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 32 v 1044 (ObjNum 569 snap 170 seq_num 569) dirty exists 2024-09-21T09:38:51.116 INFO:tasks.rados.rados.0.smithi022.stdout:2217: finishing write tid 1 to smithi02284960-40 2024-09-21T09:38:51.116 INFO:tasks.rados.rados.0.smithi022.stdout:2217: finishing write tid 2 to smithi02284960-40 2024-09-21T09:38:51.116 INFO:tasks.rados.rados.0.smithi022.stdout:2212: done (4 left) 2024-09-21T09:38:51.117 INFO:tasks.rados.rados.0.smithi022.stdout:2213: done (3 left) 2024-09-21T09:38:51.117 INFO:tasks.rados.rados.0.smithi022.stdout:2214: done (2 left) 2024-09-21T09:38:51.117 INFO:tasks.rados.rados.0.smithi022.stdout:2219: rollback oid 11 current snap is 218 2024-09-21T09:38:51.117 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 11 to 217 2024-09-21T09:38:51.117 INFO:tasks.rados.rados.0.smithi022.stdout:2220: read oid 47 snap -1 2024-09-21T09:38:51.117 INFO:tasks.rados.rados.0.smithi022.stdout:2220: expect (ObjNum 604 snap 183 seq_num 604) 2024-09-21T09:38:51.117 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 16 current snap is 218 2024-09-21T09:38:51.117 INFO:tasks.rados.rados.0.smithi022.stdout:2221: seq_num 708 ranges {8929280=663552,9592832=737280,10330112=696320,11026432=172032} 2024-09-21T09:38:51.169 INFO:tasks.rados.rados.0.smithi022.stdout:2221: writing smithi02284960-16 from 8929280 to 9592832 tid 1 2024-09-21T09:38:51.174 INFO:tasks.rados.rados.0.smithi022.stdout:2221: writing smithi02284960-16 from 9592832 to 10330112 tid 2 2024-09-21T09:38:51.178 INFO:tasks.rados.rados.0.smithi022.stdout:2221: writing smithi02284960-16 from 10330112 to 11026432 tid 3 2024-09-21T09:38:51.179 INFO:tasks.rados.rados.0.smithi022.stdout:2221: writing smithi02284960-16 from 11026432 to 11198464 tid 4 2024-09-21T09:38:51.179 INFO:tasks.rados.rados.0.smithi022.stdout:2222: read oid 36 snap -1 2024-09-21T09:38:51.179 INFO:tasks.rados.rados.0.smithi022.stdout:2222: expect (ObjNum 580 snap 172 seq_num 580) 2024-09-21T09:38:51.179 INFO:tasks.rados.rados.0.smithi022.stdout:2223: delete oid 26 current snap is 218 2024-09-21T09:38:51.180 INFO:tasks.rados.rados.0.smithi022.stdout:2217: finishing write tid 3 to smithi02284960-40 2024-09-21T09:38:51.180 INFO:tasks.rados.rados.0.smithi022.stdout:2217: finishing write tid 4 to smithi02284960-40 2024-09-21T09:38:51.180 INFO:tasks.rados.rados.0.smithi022.stdout:2217: finishing write tid 5 to smithi02284960-40 2024-09-21T09:38:51.180 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 39 v 1057 (ObjNum 65 snap 0 seq_num 3288444928) dirty exists 2024-09-21T09:38:51.180 INFO:tasks.rados.rados.0.smithi022.stdout:2217: finishing write tid 6 to smithi02284960-40 2024-09-21T09:38:51.180 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 40 v 1253 (ObjNum 707 snap 218 seq_num 707) dirty exists 2024-09-21T09:38:51.180 INFO:tasks.rados.rados.0.smithi022.stdout:2217: left oid 40 (ObjNum 707 snap 218 seq_num 707) 2024-09-21T09:38:51.180 INFO:tasks.rados.rados.0.smithi022.stdout:2219: finishing rollback tid 0 to smithi02284960-11 2024-09-21T09:38:51.180 INFO:tasks.rados.rados.0.smithi022.stdout:2219: finishing rollback tid 1 to smithi02284960-11 2024-09-21T09:38:51.180 INFO:tasks.rados.rados.0.smithi022.stdout:2219: finishing rollback tid 2 to smithi02284960-11 2024-09-21T09:38:51.180 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 11 v 1155 (ObjNum 569 snap 170 seq_num 569) dirty exists 2024-09-21T09:38:51.180 INFO:tasks.rados.rados.0.smithi022.stdout:2220: expect (ObjNum 604 snap 183 seq_num 604) 2024-09-21T09:38:51.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:51 smithi022 ceph-mon[98615]: osdmap e508: 8 total, 7 up, 8 in 2024-09-21T09:38:51.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:51 smithi022 ceph-mon[96736]: osdmap e508: 8 total, 7 up, 8 in 2024-09-21T09:38:51.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:51 smithi064 ceph-mon[86709]: osdmap e508: 8 total, 7 up, 8 in 2024-09-21T09:38:51.485 INFO:tasks.rados.rados.0.smithi022.stdout:2221: finishing write tid 1 to smithi02284960-16 2024-09-21T09:38:51.485 INFO:tasks.rados.rados.0.smithi022.stdout:2221: finishing write tid 2 to smithi02284960-16 2024-09-21T09:38:51.485 INFO:tasks.rados.rados.0.smithi022.stdout:2222: expect (ObjNum 580 snap 172 seq_num 580) 2024-09-21T09:38:51.490 INFO:tasks.rados.rados.0.smithi022.stdout:2221: finishing write tid 3 to smithi02284960-16 2024-09-21T09:38:51.490 INFO:tasks.rados.rados.0.smithi022.stdout:2221: finishing write tid 4 to smithi02284960-16 2024-09-21T09:38:51.490 INFO:tasks.rados.rados.0.smithi022.stdout:2221: finishing write tid 5 to smithi02284960-16 2024-09-21T09:38:51.490 INFO:tasks.rados.rados.0.smithi022.stdout:2221: finishing write tid 6 to smithi02284960-16 2024-09-21T09:38:51.491 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 16 v 1057 (ObjNum 708 snap 218 seq_num 708) dirty exists 2024-09-21T09:38:51.491 INFO:tasks.rados.rados.0.smithi022.stdout:2221: left oid 16 (ObjNum 708 snap 218 seq_num 708) 2024-09-21T09:38:51.491 INFO:tasks.rados.rados.0.smithi022.stdout:2217: done (6 left) 2024-09-21T09:38:51.491 INFO:tasks.rados.rados.0.smithi022.stdout:2218: done (5 left) 2024-09-21T09:38:51.491 INFO:tasks.rados.rados.0.smithi022.stdout:2219: done (4 left) 2024-09-21T09:38:51.491 INFO:tasks.rados.rados.0.smithi022.stdout:2220: done (3 left) 2024-09-21T09:38:51.491 INFO:tasks.rados.rados.0.smithi022.stdout:2221: done (2 left) 2024-09-21T09:38:51.491 INFO:tasks.rados.rados.0.smithi022.stdout:2222: done (1 left) 2024-09-21T09:38:51.491 INFO:tasks.rados.rados.0.smithi022.stdout:2223: done (0 left) 2024-09-21T09:38:51.491 INFO:tasks.rados.rados.0.smithi022.stdout:2224: read oid 44 snap -1 2024-09-21T09:38:51.491 INFO:tasks.rados.rados.0.smithi022.stdout:2224: expect (ObjNum 580 snap 172 seq_num 580) 2024-09-21T09:38:51.491 INFO:tasks.rados.rados.0.smithi022.stdout:2225: snap_remove snap 217 2024-09-21T09:38:52.214 INFO:tasks.rados.rados.0.smithi022.stdout:2225: done (1 left) 2024-09-21T09:38:52.214 INFO:tasks.rados.rados.0.smithi022.stdout:2226: snap_remove snap 194 2024-09-21T09:38:52.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:52 smithi022 ceph-mon[96736]: pgmap v366: 121 pgs: 36 active+undersized, 13 active+undersized+degraded, 72 active+clean; 295 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1.3 MiB/s rd, 9.4 MiB/s wr, 12 op/s; 79/447 objects degraded (17.673%) 2024-09-21T09:38:52.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:52 smithi022 ceph-mon[96736]: osdmap e509: 8 total, 7 up, 8 in 2024-09-21T09:38:52.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:52 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:38:52.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:52 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:38:52.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:52 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:38:52.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:52 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:38:52.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:52 smithi022 ceph-mon[98615]: pgmap v366: 121 pgs: 36 active+undersized, 13 active+undersized+degraded, 72 active+clean; 295 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1.3 MiB/s rd, 9.4 MiB/s wr, 12 op/s; 79/447 objects degraded (17.673%) 2024-09-21T09:38:52.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:52 smithi022 ceph-mon[98615]: osdmap e509: 8 total, 7 up, 8 in 2024-09-21T09:38:52.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:52 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:38:52.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:52 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:38:52.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:52 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:38:52.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:52 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:38:52.356 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:52 smithi064 ceph-mon[86709]: pgmap v366: 121 pgs: 36 active+undersized, 13 active+undersized+degraded, 72 active+clean; 295 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1.3 MiB/s rd, 9.4 MiB/s wr, 12 op/s; 79/447 objects degraded (17.673%) 2024-09-21T09:38:52.356 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:52 smithi064 ceph-mon[86709]: osdmap e509: 8 total, 7 up, 8 in 2024-09-21T09:38:52.356 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:52 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:38:52.356 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:52 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:38:52.356 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:52 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:38:52.356 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:52 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:38:52.357 INFO:journalctl@ceph.osd.5.smithi064.stdout:Sep 21 09:38:52 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-5[96798]: 2024-09-21T09:38:52.259+0000 7fe95adcf640 -1 osd.5 498 log_to_monitors true 2024-09-21T09:38:53.222 INFO:tasks.rados.rados.0.smithi022.stdout:2224: expect (ObjNum 580 snap 172 seq_num 580) 2024-09-21T09:38:53.226 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:53 smithi064 ceph-mon[86709]: osdmap e510: 8 total, 7 up, 8 in 2024-09-21T09:38:53.227 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:53 smithi064 ceph-mon[86709]: from='osd.5 [v2:172.21.15.64:6808/2760953518,v1:172.21.15.64:6809/2760953518]' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-09-21T09:38:53.227 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:53 smithi064 ceph-mon[86709]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-09-21T09:38:53.227 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:53 smithi064 ceph-mon[86709]: pgmap v369: 121 pgs: 36 active+undersized, 13 active+undersized+degraded, 72 active+clean; 297 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 3.2 MiB/s wr, 4 op/s; 80/450 objects degraded (17.778%) 2024-09-21T09:38:53.229 INFO:tasks.rados.rados.0.smithi022.stdout:2224: done (1 left) 2024-09-21T09:38:53.229 INFO:tasks.rados.rados.0.smithi022.stdout:2226: done (0 left) 2024-09-21T09:38:53.229 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 16 current snap is 218 2024-09-21T09:38:53.230 INFO:tasks.rados.rados.0.smithi022.stdout:2227: seq_num 710 ranges {11198464=696320,11894784=499712,12394496=712704,13107200=393216} 2024-09-21T09:38:53.294 INFO:tasks.rados.rados.0.smithi022.stdout:2227: writing smithi02284960-16 from 11198464 to 11894784 tid 1 2024-09-21T09:38:53.297 INFO:tasks.rados.rados.0.smithi022.stdout:2227: writing smithi02284960-16 from 11894784 to 12394496 tid 2 2024-09-21T09:38:53.303 INFO:tasks.rados.rados.0.smithi022.stdout:2227: writing smithi02284960-16 from 12394496 to 13107200 tid 3 2024-09-21T09:38:53.306 INFO:tasks.rados.rados.0.smithi022.stdout:2227: writing smithi02284960-16 from 13107200 to 13500416 tid 4 2024-09-21T09:38:53.306 INFO:tasks.rados.rados.0.smithi022.stdout:2228: copy_from oid 20 from oid 29 current snap is 218 2024-09-21T09:38:53.306 INFO:tasks.rados.rados.0.smithi022.stdout:2229: rmattr oid 5 current snap is 218 2024-09-21T09:38:53.306 INFO:tasks.rados.rados.0.smithi022.stdout:2230: snap_create 2024-09-21T09:38:53.306 INFO:tasks.rados.rados.0.smithi022.stdout:2227: finishing write tid 1 to smithi02284960-16 2024-09-21T09:38:53.306 INFO:tasks.rados.rados.0.smithi022.stdout:2227: finishing write tid 2 to smithi02284960-16 2024-09-21T09:38:53.310 INFO:tasks.rados.rados.0.smithi022.stdout:2227: finishing write tid 3 to smithi02284960-16 2024-09-21T09:38:53.310 INFO:tasks.rados.rados.0.smithi022.stdout:2227: finishing write tid 4 to smithi02284960-16 2024-09-21T09:38:53.311 INFO:tasks.rados.rados.0.smithi022.stdout:2227: finishing write tid 5 to smithi02284960-16 2024-09-21T09:38:53.311 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 5 v 1128 (ObjNum 696 snap 215 seq_num 696) dirty exists 2024-09-21T09:38:53.313 INFO:tasks.rados.rados.0.smithi022.stdout:2227: finishing write tid 6 to smithi02284960-16 2024-09-21T09:38:53.313 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 16 v 1062 (ObjNum 710 snap 218 seq_num 710) dirty exists 2024-09-21T09:38:53.313 INFO:tasks.rados.rados.0.smithi022.stdout:2227: left oid 16 (ObjNum 710 snap 218 seq_num 710) 2024-09-21T09:38:53.325 INFO:tasks.rados.rados.0.smithi022.stdout:2228: finishing copy_from racing read to smithi02284960-20 2024-09-21T09:38:53.329 INFO:tasks.rados.rados.0.smithi022.stdout:2228: finishing copy_from to smithi02284960-20 2024-09-21T09:38:53.329 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 20 v 1055 (ObjNum 684 snap 213 seq_num 684) dirty exists 2024-09-21T09:38:53.535 INFO:journalctl@ceph.osd.5.smithi064.stdout:Sep 21 09:38:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-5[96798]: 2024-09-21T09:38:53.255+0000 7fe952b79640 -1 osd.5 498 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T09:38:53.557 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:53 smithi022 ceph-mon[98615]: osdmap e510: 8 total, 7 up, 8 in 2024-09-21T09:38:53.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:53 smithi022 ceph-mon[98615]: from='osd.5 [v2:172.21.15.64:6808/2760953518,v1:172.21.15.64:6809/2760953518]' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-09-21T09:38:53.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:53 smithi022 ceph-mon[98615]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-09-21T09:38:53.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:53 smithi022 ceph-mon[98615]: pgmap v369: 121 pgs: 36 active+undersized, 13 active+undersized+degraded, 72 active+clean; 297 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 3.2 MiB/s wr, 4 op/s; 80/450 objects degraded (17.778%) 2024-09-21T09:38:53.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:53 smithi022 ceph-mon[96736]: osdmap e510: 8 total, 7 up, 8 in 2024-09-21T09:38:53.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:53 smithi022 ceph-mon[96736]: from='osd.5 [v2:172.21.15.64:6808/2760953518,v1:172.21.15.64:6809/2760953518]' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-09-21T09:38:53.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:53 smithi022 ceph-mon[96736]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-09-21T09:38:53.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:53 smithi022 ceph-mon[96736]: pgmap v369: 121 pgs: 36 active+undersized, 13 active+undersized+degraded, 72 active+clean; 297 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 3.2 MiB/s wr, 4 op/s; 80/450 objects degraded (17.778%) 2024-09-21T09:38:54.222 INFO:tasks.rados.rados.0.smithi022.stdout:2227: done (3 left) 2024-09-21T09:38:54.222 INFO:tasks.rados.rados.0.smithi022.stdout:2228: done (2 left) 2024-09-21T09:38:54.222 INFO:tasks.rados.rados.0.smithi022.stdout:2229: done (1 left) 2024-09-21T09:38:54.222 INFO:tasks.rados.rados.0.smithi022.stdout:2230: done (0 left) 2024-09-21T09:38:54.222 INFO:tasks.rados.rados.0.smithi022.stdout:2231: delete oid 9 current snap is 219 2024-09-21T09:38:54.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:54 smithi022 ceph-mon[98615]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-21T09:38:54.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:54 smithi022 ceph-mon[98615]: osdmap e511: 8 total, 7 up, 8 in 2024-09-21T09:38:54.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:54 smithi022 ceph-mon[98615]: from='osd.5 [v2:172.21.15.64:6808/2760953518,v1:172.21.15.64:6809/2760953518]' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi064", "root=default"]} : dispatch 2024-09-21T09:38:54.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:54 smithi022 ceph-mon[98615]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi064", "root=default"]} : dispatch 2024-09-21T09:38:54.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:54 smithi022 ceph-mon[98615]: Health check update: Degraded data redundancy: 80/450 objects degraded (17.778%), 13 pgs degraded (PG_DEGRADED) 2024-09-21T09:38:54.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:54 smithi022 ceph-mon[96736]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-21T09:38:54.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:54 smithi022 ceph-mon[96736]: osdmap e511: 8 total, 7 up, 8 in 2024-09-21T09:38:54.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:54 smithi022 ceph-mon[96736]: from='osd.5 [v2:172.21.15.64:6808/2760953518,v1:172.21.15.64:6809/2760953518]' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi064", "root=default"]} : dispatch 2024-09-21T09:38:54.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:54 smithi022 ceph-mon[96736]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi064", "root=default"]} : dispatch 2024-09-21T09:38:54.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:54 smithi022 ceph-mon[96736]: Health check update: Degraded data redundancy: 80/450 objects degraded (17.778%), 13 pgs degraded (PG_DEGRADED) 2024-09-21T09:38:54.624 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:54 smithi064 ceph-mon[86709]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-21T09:38:54.624 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:54 smithi064 ceph-mon[86709]: osdmap e511: 8 total, 7 up, 8 in 2024-09-21T09:38:54.624 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:54 smithi064 ceph-mon[86709]: from='osd.5 [v2:172.21.15.64:6808/2760953518,v1:172.21.15.64:6809/2760953518]' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi064", "root=default"]} : dispatch 2024-09-21T09:38:54.624 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:54 smithi064 ceph-mon[86709]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi064", "root=default"]} : dispatch 2024-09-21T09:38:54.624 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:54 smithi064 ceph-mon[86709]: Health check update: Degraded data redundancy: 80/450 objects degraded (17.778%), 13 pgs degraded (PG_DEGRADED) 2024-09-21T09:38:55.278 INFO:tasks.rados.rados.0.smithi022.stdout:2231: done (0 left) 2024-09-21T09:38:55.278 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 35 current snap is 219 2024-09-21T09:38:55.278 INFO:tasks.rados.rados.0.smithi022.stdout:2232: seq_num 712 ranges {286720=729088,1015808=671744,1687552=671744,2359296=262144} 2024-09-21T09:38:55.284 INFO:tasks.rados.rados.0.smithi022.stdout:2232: writing smithi02284960-35 from 286720 to 1015808 tid 1 2024-09-21T09:38:55.288 INFO:tasks.rados.rados.0.smithi022.stdout:2232: writing smithi02284960-35 from 1015808 to 1687552 tid 2 2024-09-21T09:38:55.292 INFO:tasks.rados.rados.0.smithi022.stdout:2232: writing smithi02284960-35 from 1687552 to 2359296 tid 3 2024-09-21T09:38:55.294 INFO:tasks.rados.rados.0.smithi022.stdout:2232: writing smithi02284960-35 from 2359296 to 2621440 tid 4 2024-09-21T09:38:55.295 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 6 current snap is 219 2024-09-21T09:38:55.295 INFO:tasks.rados.rados.0.smithi022.stdout:2232: finishing write tid 1 to smithi02284960-35 2024-09-21T09:38:55.295 INFO:tasks.rados.rados.0.smithi022.stdout:2232: finishing write tid 2 to smithi02284960-35 2024-09-21T09:38:55.295 INFO:tasks.rados.rados.0.smithi022.stdout:2233: seq_num 713 ranges {0=745472,745472=557056,1302528=483328,1785856=565248} 2024-09-21T09:38:55.298 INFO:tasks.rados.rados.0.smithi022.stdout:2233: writing smithi02284960-6 from 0 to 745472 tid 1 2024-09-21T09:38:55.302 INFO:tasks.rados.rados.0.smithi022.stdout:2233: writing smithi02284960-6 from 745472 to 1302528 tid 2 2024-09-21T09:38:55.305 INFO:tasks.rados.rados.0.smithi022.stdout:2233: writing smithi02284960-6 from 1302528 to 1785856 tid 3 2024-09-21T09:38:55.309 INFO:tasks.rados.rados.0.smithi022.stdout:2233: writing smithi02284960-6 from 1785856 to 2351104 tid 4 2024-09-21T09:38:55.309 INFO:tasks.rados.rados.0.smithi022.stdout:2234: delete oid 37 current snap is 219 2024-09-21T09:38:55.310 INFO:tasks.rados.rados.0.smithi022.stdout:2232: finishing write tid 3 to smithi02284960-35 2024-09-21T09:38:55.310 INFO:tasks.rados.rados.0.smithi022.stdout:2232: finishing write tid 4 to smithi02284960-35 2024-09-21T09:38:55.310 INFO:tasks.rados.rados.0.smithi022.stdout:2232: finishing write tid 5 to smithi02284960-35 2024-09-21T09:38:55.310 INFO:tasks.rados.rados.0.smithi022.stdout:2232: finishing write tid 6 to smithi02284960-35 2024-09-21T09:38:55.310 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 35 v 1259 (ObjNum 712 snap 219 seq_num 712) dirty exists 2024-09-21T09:38:55.310 INFO:tasks.rados.rados.0.smithi022.stdout:2232: left oid 35 (ObjNum 712 snap 219 seq_num 712) 2024-09-21T09:38:55.310 INFO:tasks.rados.rados.0.smithi022.stdout:2233: finishing write tid 1 to smithi02284960-6 2024-09-21T09:38:55.310 INFO:tasks.rados.rados.0.smithi022.stdout:2233: finishing write tid 2 to smithi02284960-6 2024-09-21T09:38:55.310 INFO:tasks.rados.rados.0.smithi022.stdout:2233: finishing write tid 3 to smithi02284960-6 2024-09-21T09:38:55.312 INFO:tasks.rados.rados.0.smithi022.stdout:2233: finishing write tid 4 to smithi02284960-6 2024-09-21T09:38:55.312 INFO:tasks.rados.rados.0.smithi022.stdout:2233: finishing write tid 5 to smithi02284960-6 2024-09-21T09:38:55.313 INFO:tasks.rados.rados.0.smithi022.stdout:2232: done (2 left) 2024-09-21T09:38:55.313 INFO:tasks.rados.rados.0.smithi022.stdout:2234: done (1 left) 2024-09-21T09:38:55.313 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 40 current snap is 219 2024-09-21T09:38:55.313 INFO:tasks.rados.rados.0.smithi022.stdout:2235: seq_num 715 ranges {14041088=778240,14819328=770048,15589376=499712,16089088=335872} 2024-09-21T09:38:55.395 INFO:tasks.rados.rados.0.smithi022.stdout:2235: writing smithi02284960-40 from 14041088 to 14819328 tid 1 2024-09-21T09:38:55.399 INFO:tasks.rados.rados.0.smithi022.stdout:2235: writing smithi02284960-40 from 14819328 to 15589376 tid 2 2024-09-21T09:38:55.402 INFO:tasks.rados.rados.0.smithi022.stdout:2235: writing smithi02284960-40 from 15589376 to 16089088 tid 3 2024-09-21T09:38:55.404 INFO:tasks.rados.rados.0.smithi022.stdout:2235: writing smithi02284960-40 from 16089088 to 16424960 tid 4 2024-09-21T09:38:55.404 INFO:tasks.rados.rados.0.smithi022.stdout:2236: copy_from oid 31 from oid 21 current snap is 219 2024-09-21T09:38:55.405 INFO:tasks.rados.rados.0.smithi022.stdout:2233: finishing write tid 6 to smithi02284960-6 2024-09-21T09:38:55.405 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 6 v 1376 (ObjNum 713 snap 219 seq_num 713) dirty exists 2024-09-21T09:38:55.405 INFO:tasks.rados.rados.0.smithi022.stdout:2233: left oid 6 (ObjNum 713 snap 219 seq_num 713) 2024-09-21T09:38:55.405 INFO:tasks.rados.rados.0.smithi022.stdout:2233: done (2 left) 2024-09-21T09:38:55.405 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 10 current snap is 219 2024-09-21T09:38:55.405 INFO:tasks.rados.rados.0.smithi022.stdout:2237: seq_num 716 ranges {8650752=794624,9445376=655360,10100736=704512,10805248=245760} 2024-09-21T09:38:55.456 INFO:tasks.rados.rados.0.smithi022.stdout:2237: writing smithi02284960-10 from 8650752 to 9445376 tid 1 2024-09-21T09:38:55.459 INFO:tasks.rados.rados.0.smithi022.stdout:2237: writing smithi02284960-10 from 9445376 to 10100736 tid 2 2024-09-21T09:38:55.463 INFO:tasks.rados.rados.0.smithi022.stdout:2237: writing smithi02284960-10 from 10100736 to 10805248 tid 3 2024-09-21T09:38:55.465 INFO:tasks.rados.rados.0.smithi022.stdout:2237: writing smithi02284960-10 from 10805248 to 11051008 tid 4 2024-09-21T09:38:55.465 INFO:tasks.rados.rados.0.smithi022.stdout:2238: copy_from oid 19 from oid 47 current snap is 219 2024-09-21T09:38:55.465 INFO:tasks.rados.rados.0.smithi022.stdout:2236: finishing copy_from to smithi02284960-31 2024-09-21T09:38:55.465 INFO:tasks.rados.rados.0.smithi022.stdout:2236: got expected ENOENT (src dne) 2024-09-21T09:38:55.465 INFO:tasks.rados.rados.0.smithi022.stdout:2239: delete oid 13 current snap is 219 2024-09-21T09:38:55.466 INFO:tasks.rados.rados.0.smithi022.stdout:2236: finishing copy_from racing read to smithi02284960-31 2024-09-21T09:38:55.466 INFO:tasks.rados.rados.0.smithi022.stdout:2235: finishing write tid 1 to smithi02284960-40 2024-09-21T09:38:55.466 INFO:tasks.rados.rados.0.smithi022.stdout:2235: finishing write tid 2 to smithi02284960-40 2024-09-21T09:38:55.466 INFO:tasks.rados.rados.0.smithi022.stdout:2235: finishing write tid 3 to smithi02284960-40 2024-09-21T09:38:55.466 INFO:tasks.rados.rados.0.smithi022.stdout:2235: finishing write tid 4 to smithi02284960-40 2024-09-21T09:38:55.466 INFO:tasks.rados.rados.0.smithi022.stdout:2235: finishing write tid 5 to smithi02284960-40 2024-09-21T09:38:55.468 INFO:tasks.rados.rados.0.smithi022.stdout:2235: finishing write tid 6 to smithi02284960-40 2024-09-21T09:38:55.468 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 40 v 1258 (ObjNum 715 snap 219 seq_num 715) dirty exists 2024-09-21T09:38:55.468 INFO:tasks.rados.rados.0.smithi022.stdout:2235: left oid 40 (ObjNum 715 snap 219 seq_num 715) 2024-09-21T09:38:55.470 INFO:tasks.rados.rados.0.smithi022.stdout:2235: done (4 left) 2024-09-21T09:38:55.470 INFO:tasks.rados.rados.0.smithi022.stdout:2236: done (3 left) 2024-09-21T09:38:55.470 INFO:tasks.rados.rados.0.smithi022.stdout:2239: done (2 left) 2024-09-21T09:38:55.470 INFO:tasks.rados.rados.0.smithi022.stdout:2240: snap_remove snap 216 2024-09-21T09:38:55.557 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:38:55 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: 2024-09-21T09:38:55.100+0000 7feec1b50640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (18 PGs are or would become offline) 2024-09-21T09:38:55.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:55 smithi022 ceph-mon[98615]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T09:38:55.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:55 smithi022 ceph-mon[98615]: osd.5 [v2:172.21.15.64:6808/2760953518,v1:172.21.15.64:6809/2760953518] boot 2024-09-21T09:38:55.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:55 smithi022 ceph-mon[98615]: osdmap e512: 8 total, 8 up, 8 in 2024-09-21T09:38:55.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:55 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-09-21T09:38:55.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:55 smithi022 ceph-mon[98615]: pgmap v372: 121 pgs: 36 active+undersized, 13 active+undersized+degraded, 72 active+clean; 301 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 99 KiB/s rd, 1.6 MiB/s wr, 5 op/s; 81/459 objects degraded (17.647%) 2024-09-21T09:38:55.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:55 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:38:55.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:55 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:38:55.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:55 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:38:55.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:55 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:38:55.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:55 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:38:55.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:55 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:38:55.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:55 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:38:55.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:55 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:38:55.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:55 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:38:55.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:55 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16} : dispatch 2024-09-21T09:38:55.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:55 smithi022 ceph-mon[98615]: osdmap e513: 8 total, 8 up, 8 in 2024-09-21T09:38:55.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:55 smithi022 ceph-mon[96736]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T09:38:55.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:55 smithi022 ceph-mon[96736]: osd.5 [v2:172.21.15.64:6808/2760953518,v1:172.21.15.64:6809/2760953518] boot 2024-09-21T09:38:55.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:55 smithi022 ceph-mon[96736]: osdmap e512: 8 total, 8 up, 8 in 2024-09-21T09:38:55.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:55 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-09-21T09:38:55.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:55 smithi022 ceph-mon[96736]: pgmap v372: 121 pgs: 36 active+undersized, 13 active+undersized+degraded, 72 active+clean; 301 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 99 KiB/s rd, 1.6 MiB/s wr, 5 op/s; 81/459 objects degraded (17.647%) 2024-09-21T09:38:55.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:55 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:38:55.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:55 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:38:55.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:55 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:38:55.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:55 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:38:55.560 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:55 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:38:55.560 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:55 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:38:55.560 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:55 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:38:55.560 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:55 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:38:55.560 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:55 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:38:55.560 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:55 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16} : dispatch 2024-09-21T09:38:55.560 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:55 smithi022 ceph-mon[96736]: osdmap e513: 8 total, 8 up, 8 in 2024-09-21T09:38:55.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:55 smithi064 ceph-mon[86709]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T09:38:55.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:55 smithi064 ceph-mon[86709]: osd.5 [v2:172.21.15.64:6808/2760953518,v1:172.21.15.64:6809/2760953518] boot 2024-09-21T09:38:55.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:55 smithi064 ceph-mon[86709]: osdmap e512: 8 total, 8 up, 8 in 2024-09-21T09:38:55.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:55 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-09-21T09:38:55.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:55 smithi064 ceph-mon[86709]: pgmap v372: 121 pgs: 36 active+undersized, 13 active+undersized+degraded, 72 active+clean; 301 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 99 KiB/s rd, 1.6 MiB/s wr, 5 op/s; 81/459 objects degraded (17.647%) 2024-09-21T09:38:55.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:55 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:38:55.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:55 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:38:55.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:55 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:38:55.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:55 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:38:55.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:55 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:38:55.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:55 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:38:55.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:55 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:38:55.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:55 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:38:55.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:55 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:38:55.693 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:55 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16} : dispatch 2024-09-21T09:38:55.693 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:55 smithi064 ceph-mon[86709]: osdmap e513: 8 total, 8 up, 8 in 2024-09-21T09:38:56.058 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:38:55 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:38:55] "GET /metrics HTTP/1.1" 200 36296 "" "Prometheus/2.51.0" 2024-09-21T09:38:56.241 INFO:tasks.rados.rados.0.smithi022.stdout:2240: done (2 left) 2024-09-21T09:38:56.241 INFO:tasks.rados.rados.0.smithi022.stdout:2241: snap_remove snap 218 2024-09-21T09:38:56.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:56 smithi022 ceph-mon[98615]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-21T09:38:56.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:56 smithi022 ceph-mon[98615]: Upgrade: unsafe to stop osd(s) at this time (18 PGs are or would become offline) 2024-09-21T09:38:56.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:56 smithi022 ceph-mon[96736]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-21T09:38:56.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:56 smithi022 ceph-mon[96736]: Upgrade: unsafe to stop osd(s) at this time (18 PGs are or would become offline) 2024-09-21T09:38:56.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:56 smithi064 ceph-mon[86709]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-21T09:38:56.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:56 smithi064 ceph-mon[86709]: Upgrade: unsafe to stop osd(s) at this time (18 PGs are or would become offline) 2024-09-21T09:38:57.250 INFO:tasks.rados.rados.0.smithi022.stdout:2241: done (2 left) 2024-09-21T09:38:57.250 INFO:tasks.rados.rados.0.smithi022.stdout:2242: copy_from oid 30 from oid 31 current snap is 219 2024-09-21T09:38:57.250 INFO:tasks.rados.rados.0.smithi022.stdout:2237: finishing write tid 1 to smithi02284960-10 2024-09-21T09:38:57.250 INFO:tasks.rados.rados.0.smithi022.stdout:2237: finishing write tid 2 to smithi02284960-10 2024-09-21T09:38:57.250 INFO:tasks.rados.rados.0.smithi022.stdout:2237: finishing write tid 3 to smithi02284960-10 2024-09-21T09:38:57.250 INFO:tasks.rados.rados.0.smithi022.stdout:2237: finishing write tid 4 to smithi02284960-10 2024-09-21T09:38:57.250 INFO:tasks.rados.rados.0.smithi022.stdout:2237: finishing write tid 5 to smithi02284960-10 2024-09-21T09:38:57.250 INFO:tasks.rados.rados.0.smithi022.stdout:2237: finishing write tid 6 to smithi02284960-10 2024-09-21T09:38:57.250 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 10 v 1263 (ObjNum 716 snap 219 seq_num 716) dirty exists 2024-09-21T09:38:57.251 INFO:tasks.rados.rados.0.smithi022.stdout:2237: left oid 10 (ObjNum 716 snap 219 seq_num 716) 2024-09-21T09:38:57.251 INFO:tasks.rados.rados.0.smithi022.stdout:2238: finishing copy_from racing read to smithi02284960-19 2024-09-21T09:38:57.251 INFO:tasks.rados.rados.0.smithi022.stdout:2238: finishing copy_from to smithi02284960-19 2024-09-21T09:38:57.251 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 19 v 1145 (ObjNum 604 snap 183 seq_num 604) dirty exists 2024-09-21T09:38:57.251 INFO:tasks.rados.rados.0.smithi022.stdout:2237: done (2 left) 2024-09-21T09:38:57.251 INFO:tasks.rados.rados.0.smithi022.stdout:2238: done (1 left) 2024-09-21T09:38:57.251 INFO:tasks.rados.rados.0.smithi022.stdout:2243: read oid 21 snap -1 2024-09-21T09:38:57.251 INFO:tasks.rados.rados.0.smithi022.stdout:2243: expect deleted 2024-09-21T09:38:57.251 INFO:tasks.rados.rados.0.smithi022.stdout:2244: setattr oid 28 current snap is 219 2024-09-21T09:38:57.253 INFO:tasks.rados.rados.0.smithi022.stdout:2243: done (2 left) 2024-09-21T09:38:57.253 INFO:tasks.rados.rados.0.smithi022.stdout:2245: snap_remove snap 202 2024-09-21T09:38:57.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:57 smithi022 ceph-mon[98615]: osdmap e514: 8 total, 8 up, 8 in 2024-09-21T09:38:57.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:57 smithi022 ceph-mon[98615]: pgmap v375: 121 pgs: 6 peering, 29 active+undersized, 9 active+undersized+degraded, 77 active+clean; 308 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3.3 MiB/s rd, 3.5 MiB/s wr, 19 op/s; 53/459 objects degraded (11.547%) 2024-09-21T09:38:57.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:57 smithi022 ceph-mon[96736]: osdmap e514: 8 total, 8 up, 8 in 2024-09-21T09:38:57.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:57 smithi022 ceph-mon[96736]: pgmap v375: 121 pgs: 6 peering, 29 active+undersized, 9 active+undersized+degraded, 77 active+clean; 308 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3.3 MiB/s rd, 3.5 MiB/s wr, 19 op/s; 53/459 objects degraded (11.547%) 2024-09-21T09:38:57.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:57 smithi064 ceph-mon[86709]: osdmap e514: 8 total, 8 up, 8 in 2024-09-21T09:38:57.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:57 smithi064 ceph-mon[86709]: pgmap v375: 121 pgs: 6 peering, 29 active+undersized, 9 active+undersized+degraded, 77 active+clean; 308 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3.3 MiB/s rd, 3.5 MiB/s wr, 19 op/s; 53/459 objects degraded (11.547%) 2024-09-21T09:38:58.254 INFO:tasks.rados.rados.0.smithi022.stdout:2245: done (2 left) 2024-09-21T09:38:58.254 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 28 v 1260 (ObjNum 392 snap 122 seq_num 392) dirty exists 2024-09-21T09:38:58.254 INFO:tasks.rados.rados.0.smithi022.stdout:2242: finishing copy_from racing read to smithi02284960-30 2024-09-21T09:38:58.254 INFO:tasks.rados.rados.0.smithi022.stdout:2242: finishing copy_from to smithi02284960-30 2024-09-21T09:38:58.254 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 30 v 1254 (ObjNum 590 snap 176 seq_num 590) dirty exists 2024-09-21T09:38:58.254 INFO:tasks.rados.rados.0.smithi022.stdout:2246: read oid 14 snap 208 2024-09-21T09:38:58.254 INFO:tasks.rados.rados.0.smithi022.stdout:2246: expect (ObjNum 580 snap 172 seq_num 580) 2024-09-21T09:38:58.254 INFO:tasks.rados.rados.0.smithi022.stdout:2242: done (2 left) 2024-09-21T09:38:58.254 INFO:tasks.rados.rados.0.smithi022.stdout:2244: done (1 left) 2024-09-21T09:38:58.254 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 45 current snap is 219 2024-09-21T09:38:58.254 INFO:tasks.rados.rados.0.smithi022.stdout:2247: seq_num 718 ranges {4186112=32768} 2024-09-21T09:38:58.278 INFO:tasks.rados.rados.0.smithi022.stdout:2247: writing smithi02284960-45 from 4186112 to 4218880 tid 1 2024-09-21T09:38:58.278 INFO:tasks.rados.rados.0.smithi022.stdout:2248: delete oid 24 current snap is 219 2024-09-21T09:38:58.278 INFO:tasks.rados.rados.0.smithi022.stdout:2246: expect (ObjNum 580 snap 172 seq_num 580) 2024-09-21T09:38:58.285 INFO:tasks.rados.rados.0.smithi022.stdout:2247: finishing write tid 1 to smithi02284960-45 2024-09-21T09:38:58.285 INFO:tasks.rados.rados.0.smithi022.stdout:2247: finishing write tid 2 to smithi02284960-45 2024-09-21T09:38:58.285 INFO:tasks.rados.rados.0.smithi022.stdout:2247: finishing write tid 3 to smithi02284960-45 2024-09-21T09:38:58.285 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 45 v 1278 (ObjNum 718 snap 219 seq_num 718) dirty exists 2024-09-21T09:38:58.285 INFO:tasks.rados.rados.0.smithi022.stdout:2247: left oid 45 (ObjNum 718 snap 219 seq_num 718) 2024-09-21T09:38:58.285 INFO:tasks.rados.rados.0.smithi022.stdout:2246: done (2 left) 2024-09-21T09:38:58.286 INFO:tasks.rados.rados.0.smithi022.stdout:2247: done (1 left) 2024-09-21T09:38:58.286 INFO:tasks.rados.rados.0.smithi022.stdout:2248: done (0 left) 2024-09-21T09:38:58.286 INFO:tasks.rados.rados.0.smithi022.stdout:2249: rmattr oid 41 current snap is 219 2024-09-21T09:38:58.286 INFO:tasks.rados.rados.0.smithi022.stdout:2249: done (0 left) 2024-09-21T09:38:58.286 INFO:tasks.rados.rados.0.smithi022.stdout:2250: snap_create 2024-09-21T09:38:58.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:58 smithi022 ceph-mon[98615]: osdmap e515: 8 total, 8 up, 8 in 2024-09-21T09:38:58.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:58 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:38:58.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:58 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:38:58.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:58 smithi022 ceph-mon[96736]: osdmap e515: 8 total, 8 up, 8 in 2024-09-21T09:38:58.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:58 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:38:58.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:58 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:38:58.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:58 smithi064 ceph-mon[86709]: osdmap e515: 8 total, 8 up, 8 in 2024-09-21T09:38:58.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:58 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:38:58.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:58 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:38:59.259 INFO:tasks.rados.rados.0.smithi022.stdout:2250: done (0 left) 2024-09-21T09:38:59.259 INFO:tasks.rados.rados.0.smithi022.stdout:2251: rollback oid 36 current snap is 220 2024-09-21T09:38:59.259 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 36 to 208 2024-09-21T09:38:59.259 INFO:tasks.rados.rados.0.smithi022.stdout:2252: delete oid 18 current snap is 220 2024-09-21T09:38:59.267 INFO:tasks.rados.rados.0.smithi022.stdout:2251: finishing rollback tid 0 to smithi02284960-36 2024-09-21T09:38:59.267 INFO:tasks.rados.rados.0.smithi022.stdout:2251: finishing rollback tid 1 to smithi02284960-36 2024-09-21T09:38:59.268 INFO:tasks.rados.rados.0.smithi022.stdout:2252: done (1 left) 2024-09-21T09:38:59.268 INFO:tasks.rados.rados.0.smithi022.stdout:2253: read oid 49 snap -1 2024-09-21T09:38:59.268 INFO:tasks.rados.rados.0.smithi022.stdout:2253: expect (ObjNum 705 snap 218 seq_num 705) 2024-09-21T09:38:59.268 INFO:tasks.rados.rados.0.smithi022.stdout:2254: delete oid 2 current snap is 220 2024-09-21T09:38:59.269 INFO:tasks.rados.rados.0.smithi022.stdout:2251: finishing rollback tid 2 to smithi02284960-36 2024-09-21T09:38:59.269 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 36 v 1281 (ObjNum 580 snap 172 seq_num 580) dirty exists 2024-09-21T09:38:59.274 INFO:tasks.rados.rados.0.smithi022.stdout:2251: done (2 left) 2024-09-21T09:38:59.274 INFO:tasks.rados.rados.0.smithi022.stdout:2254: done (1 left) 2024-09-21T09:38:59.274 INFO:tasks.rados.rados.0.smithi022.stdout:2255: rollback oid 12 current snap is 220 2024-09-21T09:38:59.274 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 12 to 208 2024-09-21T09:38:59.275 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 23 current snap is 220 2024-09-21T09:38:59.275 INFO:tasks.rados.rados.0.smithi022.stdout:2256: seq_num 722 ranges {1294336=98304} 2024-09-21T09:38:59.282 INFO:tasks.rados.rados.0.smithi022.stdout:2256: writing smithi02284960-23 from 1294336 to 1392640 tid 1 2024-09-21T09:38:59.282 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 27 current snap is 220 2024-09-21T09:38:59.282 INFO:tasks.rados.rados.0.smithi022.stdout:2257: seq_num 723 ranges {3579904=114688} 2024-09-21T09:38:59.303 INFO:tasks.rados.rados.0.smithi022.stdout:2257: writing smithi02284960-27 from 3579904 to 3694592 tid 1 2024-09-21T09:38:59.303 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 14 current snap is 220 2024-09-21T09:38:59.303 INFO:tasks.rados.rados.0.smithi022.stdout:2258: seq_num 724 ranges {114688=131072} 2024-09-21T09:38:59.304 INFO:tasks.rados.rados.0.smithi022.stdout:2258: writing smithi02284960-14 from 114688 to 245760 tid 1 2024-09-21T09:38:59.304 INFO:tasks.rados.rados.0.smithi022.stdout:2259: read oid 41 snap -1 2024-09-21T09:38:59.305 INFO:tasks.rados.rados.0.smithi022.stdout:2259: expect deleted 2024-09-21T09:38:59.305 INFO:tasks.rados.rados.0.smithi022.stdout:2260: snap_create 2024-09-21T09:38:59.305 INFO:tasks.rados.rados.0.smithi022.stdout:2255: finishing rollback tid 0 to smithi02284960-12 2024-09-21T09:38:59.305 INFO:tasks.rados.rados.0.smithi022.stdout:2255: finishing rollback tid 1 to smithi02284960-12 2024-09-21T09:38:59.305 INFO:tasks.rados.rados.0.smithi022.stdout:2255: finishing rollback tid 2 to smithi02284960-12 2024-09-21T09:38:59.305 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 12 v 1381 (ObjNum 633 snap 189 seq_num 633) dirty exists 2024-09-21T09:38:59.305 INFO:tasks.rados.rados.0.smithi022.stdout:2256: finishing write tid 1 to smithi02284960-23 2024-09-21T09:38:59.305 INFO:tasks.rados.rados.0.smithi022.stdout:2256: finishing write tid 2 to smithi02284960-23 2024-09-21T09:38:59.305 INFO:tasks.rados.rados.0.smithi022.stdout:2256: finishing write tid 3 to smithi02284960-23 2024-09-21T09:38:59.305 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 23 v 1062 (ObjNum 722 snap 220 seq_num 722) dirty exists 2024-09-21T09:38:59.305 INFO:tasks.rados.rados.0.smithi022.stdout:2256: left oid 23 (ObjNum 722 snap 220 seq_num 722) 2024-09-21T09:38:59.307 INFO:tasks.rados.rados.0.smithi022.stdout:2257: finishing write tid 1 to smithi02284960-27 2024-09-21T09:38:59.307 INFO:tasks.rados.rados.0.smithi022.stdout:2257: finishing write tid 2 to smithi02284960-27 2024-09-21T09:38:59.308 INFO:tasks.rados.rados.0.smithi022.stdout:2257: finishing write tid 3 to smithi02284960-27 2024-09-21T09:38:59.308 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 27 v 1324 (ObjNum 723 snap 220 seq_num 723) dirty exists 2024-09-21T09:38:59.309 INFO:tasks.rados.rados.0.smithi022.stdout:2257: left oid 27 (ObjNum 723 snap 220 seq_num 723) 2024-09-21T09:38:59.338 INFO:tasks.rados.rados.0.smithi022.stdout:2253: expect (ObjNum 705 snap 218 seq_num 705) 2024-09-21T09:38:59.557 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:59 smithi022 ceph-mon[98615]: osdmap e516: 8 total, 8 up, 8 in 2024-09-21T09:38:59.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:59 smithi022 ceph-mon[98615]: pgmap v378: 121 pgs: 6 peering, 115 active+clean; 285 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3.2 MiB/s rd, 5.1 MiB/s wr, 26 op/s; 20 MiB/s, 3 objects/s recovering 2024-09-21T09:38:59.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:38:59 smithi022 ceph-mon[98615]: Health check update: Degraded data redundancy: 53/459 objects degraded (11.547%), 9 pgs degraded (PG_DEGRADED) 2024-09-21T09:38:59.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:59 smithi022 ceph-mon[96736]: osdmap e516: 8 total, 8 up, 8 in 2024-09-21T09:38:59.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:59 smithi022 ceph-mon[96736]: pgmap v378: 121 pgs: 6 peering, 115 active+clean; 285 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3.2 MiB/s rd, 5.1 MiB/s wr, 26 op/s; 20 MiB/s, 3 objects/s recovering 2024-09-21T09:38:59.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:38:59 smithi022 ceph-mon[96736]: Health check update: Degraded data redundancy: 53/459 objects degraded (11.547%), 9 pgs degraded (PG_DEGRADED) 2024-09-21T09:38:59.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:59 smithi064 ceph-mon[86709]: osdmap e516: 8 total, 8 up, 8 in 2024-09-21T09:38:59.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:59 smithi064 ceph-mon[86709]: pgmap v378: 121 pgs: 6 peering, 115 active+clean; 285 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3.2 MiB/s rd, 5.1 MiB/s wr, 26 op/s; 20 MiB/s, 3 objects/s recovering 2024-09-21T09:38:59.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:38:59 smithi064 ceph-mon[86709]: Health check update: Degraded data redundancy: 53/459 objects degraded (11.547%), 9 pgs degraded (PG_DEGRADED) 2024-09-21T09:39:00.464 INFO:tasks.rados.rados.0.smithi022.stdout:2258: finishing write tid 1 to smithi02284960-14 2024-09-21T09:39:00.465 INFO:tasks.rados.rados.0.smithi022.stdout:2258: finishing write tid 2 to smithi02284960-14 2024-09-21T09:39:00.465 INFO:tasks.rados.rados.0.smithi022.stdout:2258: finishing write tid 3 to smithi02284960-14 2024-09-21T09:39:00.465 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 14 v 1108 (ObjNum 724 snap 220 seq_num 724) dirty exists 2024-09-21T09:39:00.465 INFO:tasks.rados.rados.0.smithi022.stdout:2258: left oid 14 (ObjNum 724 snap 220 seq_num 724) 2024-09-21T09:39:00.465 INFO:tasks.rados.rados.0.smithi022.stdout:2253: done (6 left) 2024-09-21T09:39:00.466 INFO:tasks.rados.rados.0.smithi022.stdout:2255: done (5 left) 2024-09-21T09:39:00.466 INFO:tasks.rados.rados.0.smithi022.stdout:2256: done (4 left) 2024-09-21T09:39:00.466 INFO:tasks.rados.rados.0.smithi022.stdout:2257: done (3 left) 2024-09-21T09:39:00.466 INFO:tasks.rados.rados.0.smithi022.stdout:2258: done (2 left) 2024-09-21T09:39:00.466 INFO:tasks.rados.rados.0.smithi022.stdout:2259: done (1 left) 2024-09-21T09:39:00.466 INFO:tasks.rados.rados.0.smithi022.stdout:2260: done (0 left) 2024-09-21T09:39:00.466 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 39 current snap is 221 2024-09-21T09:39:00.466 INFO:tasks.rados.rados.0.smithi022.stdout:2261: seq_num 725 ranges {0=147456} 2024-09-21T09:39:00.466 INFO:tasks.rados.rados.0.smithi022.stdout:2261: writing smithi02284960-39 from 0 to 147456 tid 1 2024-09-21T09:39:00.466 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 38 current snap is 221 2024-09-21T09:39:00.466 INFO:tasks.rados.rados.0.smithi022.stdout:2262: seq_num 726 ranges {2170880=163840} 2024-09-21T09:39:00.479 INFO:tasks.rados.rados.0.smithi022.stdout:2262: writing smithi02284960-38 from 2170880 to 2334720 tid 1 2024-09-21T09:39:00.479 INFO:tasks.rados.rados.0.smithi022.stdout:2263: read oid 35 snap -1 2024-09-21T09:39:00.479 INFO:tasks.rados.rados.0.smithi022.stdout:2263: expect (ObjNum 712 snap 219 seq_num 712) 2024-09-21T09:39:00.479 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 44 current snap is 221 2024-09-21T09:39:00.479 INFO:tasks.rados.rados.0.smithi022.stdout:2261: finishing write tid 1 to smithi02284960-39 2024-09-21T09:39:00.479 INFO:tasks.rados.rados.0.smithi022.stdout:2261: finishing write tid 2 to smithi02284960-39 2024-09-21T09:39:00.479 INFO:tasks.rados.rados.0.smithi022.stdout:2261: finishing write tid 3 to smithi02284960-39 2024-09-21T09:39:00.479 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 39 v 1256 (ObjNum 725 snap 221 seq_num 725) dirty exists 2024-09-21T09:39:00.480 INFO:tasks.rados.rados.0.smithi022.stdout:2261: left oid 39 (ObjNum 725 snap 221 seq_num 725) 2024-09-21T09:39:00.480 INFO:tasks.rados.rados.0.smithi022.stdout:2264: seq_num 727 ranges {114688=180224} 2024-09-21T09:39:00.481 INFO:tasks.rados.rados.0.smithi022.stdout:2264: writing smithi02284960-44 from 114688 to 294912 tid 1 2024-09-21T09:39:00.481 INFO:tasks.rados.rados.0.smithi022.stdout:2261: done (3 left) 2024-09-21T09:39:00.481 INFO:tasks.rados.rados.0.smithi022.stdout:2265: rollback oid 21 current snap is 221 2024-09-21T09:39:00.481 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 21 to 208 2024-09-21T09:39:00.481 INFO:tasks.rados.rados.0.smithi022.stdout:2266: delete oid 16 current snap is 221 2024-09-21T09:39:00.486 INFO:tasks.rados.rados.0.smithi022.stdout:2262: finishing write tid 1 to smithi02284960-38 2024-09-21T09:39:00.487 INFO:tasks.rados.rados.0.smithi022.stdout:2262: finishing write tid 2 to smithi02284960-38 2024-09-21T09:39:00.487 INFO:tasks.rados.rados.0.smithi022.stdout:2265: finishing rollback tid 1 to smithi02284960-21 2024-09-21T09:39:00.487 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 21 v 0 (ObjNum 161 snap 0 seq_num 0) dirty dne 2024-09-21T09:39:00.488 INFO:tasks.rados.rados.0.smithi022.stdout:2265: done (4 left) 2024-09-21T09:39:00.488 INFO:tasks.rados.rados.0.smithi022.stdout:2266: done (3 left) 2024-09-21T09:39:00.488 INFO:tasks.rados.rados.0.smithi022.stdout:2267: setattr oid 16 current snap is 221 2024-09-21T09:39:00.489 INFO:tasks.rados.rados.0.smithi022.stdout:2262: finishing write tid 3 to smithi02284960-38 2024-09-21T09:39:00.489 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 38 v 1064 (ObjNum 726 snap 221 seq_num 726) dirty exists 2024-09-21T09:39:00.489 INFO:tasks.rados.rados.0.smithi022.stdout:2262: left oid 38 (ObjNum 726 snap 221 seq_num 726) 2024-09-21T09:39:00.490 INFO:tasks.rados.rados.0.smithi022.stdout:2263: expect (ObjNum 712 snap 219 seq_num 712) 2024-09-21T09:39:00.557 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:00 smithi022 ceph-mon[98615]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 53/459 objects degraded (11.547%), 9 pgs degraded) 2024-09-21T09:39:00.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:00 smithi022 ceph-mon[98615]: Cluster is now healthy 2024-09-21T09:39:00.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:00 smithi022 ceph-mon[98615]: osdmap e517: 8 total, 8 up, 8 in 2024-09-21T09:39:00.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:00 smithi022 ceph-mon[96736]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 53/459 objects degraded (11.547%), 9 pgs degraded) 2024-09-21T09:39:00.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:00 smithi022 ceph-mon[96736]: Cluster is now healthy 2024-09-21T09:39:00.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:00 smithi022 ceph-mon[96736]: osdmap e517: 8 total, 8 up, 8 in 2024-09-21T09:39:00.566 INFO:tasks.rados.rados.0.smithi022.stdout:2264: finishing write tid 1 to smithi02284960-44 2024-09-21T09:39:00.566 INFO:tasks.rados.rados.0.smithi022.stdout:2264: finishing write tid 2 to smithi02284960-44 2024-09-21T09:39:00.566 INFO:tasks.rados.rados.0.smithi022.stdout:2264: finishing write tid 3 to smithi02284960-44 2024-09-21T09:39:00.566 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 44 v 1182 (ObjNum 727 snap 221 seq_num 727) dirty exists 2024-09-21T09:39:00.566 INFO:tasks.rados.rados.0.smithi022.stdout:2264: left oid 44 (ObjNum 727 snap 221 seq_num 727) 2024-09-21T09:39:00.566 INFO:tasks.rados.rados.0.smithi022.stdout:2262: done (3 left) 2024-09-21T09:39:00.566 INFO:tasks.rados.rados.0.smithi022.stdout:2263: done (2 left) 2024-09-21T09:39:00.566 INFO:tasks.rados.rados.0.smithi022.stdout:2264: done (1 left) 2024-09-21T09:39:00.566 INFO:tasks.rados.rados.0.smithi022.stdout:2268: rollback oid 3 current snap is 221 2024-09-21T09:39:00.567 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 3 to 219 2024-09-21T09:39:00.567 INFO:tasks.rados.rados.0.smithi022.stdout:2269: rollback oid 21 current snap is 221 2024-09-21T09:39:00.567 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 21 to 220 2024-09-21T09:39:00.567 INFO:tasks.rados.rados.0.smithi022.stdout:2270: read oid 43 snap 220 2024-09-21T09:39:00.567 INFO:tasks.rados.rados.0.smithi022.stdout:2270: expect deleted 2024-09-21T09:39:00.567 INFO:tasks.rados.rados.0.smithi022.stdout:2271: delete oid 8 current snap is 221 2024-09-21T09:39:00.568 INFO:tasks.rados.rados.0.smithi022.stdout:2270: done (4 left) 2024-09-21T09:39:00.568 INFO:tasks.rados.rados.0.smithi022.stdout:2271: done (3 left) 2024-09-21T09:39:00.568 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 11 current snap is 221 2024-09-21T09:39:00.568 INFO:tasks.rados.rados.0.smithi022.stdout:2272: seq_num 730 ranges {8929280=229376} 2024-09-21T09:39:00.618 INFO:tasks.rados.rados.0.smithi022.stdout:2272: writing smithi02284960-11 from 8929280 to 9158656 tid 1 2024-09-21T09:39:00.618 INFO:tasks.rados.rados.0.smithi022.stdout:2273: delete oid 48 current snap is 221 2024-09-21T09:39:00.619 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 16 v 1065 (ObjNum 161 snap 0 seq_num 0) dirty exists 2024-09-21T09:39:00.619 INFO:tasks.rados.rados.0.smithi022.stdout:2268: finishing rollback tid 0 to smithi02284960-3 2024-09-21T09:39:00.619 INFO:tasks.rados.rados.0.smithi022.stdout:2268: finishing rollback tid 1 to smithi02284960-3 2024-09-21T09:39:00.619 INFO:tasks.rados.rados.0.smithi022.stdout:2268: finishing rollback tid 2 to smithi02284960-3 2024-09-21T09:39:00.619 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 3 v 610 (ObjNum 675 snap 208 seq_num 675) dirty exists 2024-09-21T09:39:00.619 INFO:tasks.rados.rados.0.smithi022.stdout:2269: finishing rollback tid 1 to smithi02284960-21 2024-09-21T09:39:00.619 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 21 v 0 (ObjNum 161 snap 0 seq_num 0) dirty dne 2024-09-21T09:39:00.620 INFO:tasks.rados.rados.0.smithi022.stdout:2267: done (4 left) 2024-09-21T09:39:00.620 INFO:tasks.rados.rados.0.smithi022.stdout:2268: done (3 left) 2024-09-21T09:39:00.620 INFO:tasks.rados.rados.0.smithi022.stdout:2269: done (2 left) 2024-09-21T09:39:00.620 INFO:tasks.rados.rados.0.smithi022.stdout:2273: done (1 left) 2024-09-21T09:39:00.621 INFO:tasks.rados.rados.0.smithi022.stdout:2274: snap_remove snap 220 2024-09-21T09:39:00.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:00 smithi064 ceph-mon[86709]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 53/459 objects degraded (11.547%), 9 pgs degraded) 2024-09-21T09:39:00.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:00 smithi064 ceph-mon[86709]: Cluster is now healthy 2024-09-21T09:39:00.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:00 smithi064 ceph-mon[86709]: osdmap e517: 8 total, 8 up, 8 in 2024-09-21T09:39:00.692 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:39:00 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:39:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:39:01.270 INFO:tasks.rados.rados.0.smithi022.stdout:2274: done (1 left) 2024-09-21T09:39:01.270 INFO:tasks.rados.rados.0.smithi022.stdout:2275: snap_remove snap 219 2024-09-21T09:39:01.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:01 smithi022 ceph-mon[98615]: osdmap e518: 8 total, 8 up, 8 in 2024-09-21T09:39:01.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:01 smithi022 ceph-mon[98615]: pgmap v381: 121 pgs: 121 active+clean; 271 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1023 B/s rd, 3.3 MiB/s wr, 13 op/s; 38 MiB/s, 6 objects/s recovering 2024-09-21T09:39:01.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:01 smithi022 ceph-mon[96736]: osdmap e518: 8 total, 8 up, 8 in 2024-09-21T09:39:01.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:01 smithi022 ceph-mon[96736]: pgmap v381: 121 pgs: 121 active+clean; 271 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1023 B/s rd, 3.3 MiB/s wr, 13 op/s; 38 MiB/s, 6 objects/s recovering 2024-09-21T09:39:01.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:01 smithi064 ceph-mon[86709]: osdmap e518: 8 total, 8 up, 8 in 2024-09-21T09:39:01.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:01 smithi064 ceph-mon[86709]: pgmap v381: 121 pgs: 121 active+clean; 271 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1023 B/s rd, 3.3 MiB/s wr, 13 op/s; 38 MiB/s, 6 objects/s recovering 2024-09-21T09:39:02.272 INFO:tasks.rados.rados.0.smithi022.stdout:2275: done (1 left) 2024-09-21T09:39:02.272 INFO:tasks.rados.rados.0.smithi022.stdout:2276: rollback oid 39 current snap is 221 2024-09-21T09:39:02.272 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 39 to 208 2024-09-21T09:39:02.272 INFO:tasks.rados.rados.0.smithi022.stdout:2272: finishing write tid 1 to smithi02284960-11 2024-09-21T09:39:02.272 INFO:tasks.rados.rados.0.smithi022.stdout:2272: finishing write tid 2 to smithi02284960-11 2024-09-21T09:39:02.272 INFO:tasks.rados.rados.0.smithi022.stdout:2272: finishing write tid 3 to smithi02284960-11 2024-09-21T09:39:02.272 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 11 v 1185 (ObjNum 730 snap 221 seq_num 730) dirty exists 2024-09-21T09:39:02.272 INFO:tasks.rados.rados.0.smithi022.stdout:2272: left oid 11 (ObjNum 730 snap 221 seq_num 730) 2024-09-21T09:39:02.272 INFO:tasks.rados.rados.0.smithi022.stdout:2272: done (1 left) 2024-09-21T09:39:02.272 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 11 current snap is 221 2024-09-21T09:39:02.272 INFO:tasks.rados.rados.0.smithi022.stdout:2277: seq_num 732 ranges {9158656=262144} 2024-09-21T09:39:02.324 INFO:tasks.rados.rados.0.smithi022.stdout:2277: writing smithi02284960-11 from 9158656 to 9420800 tid 1 2024-09-21T09:39:02.324 INFO:tasks.rados.rados.0.smithi022.stdout:2278: read oid 45 snap -1 2024-09-21T09:39:02.324 INFO:tasks.rados.rados.0.smithi022.stdout:2278: expect (ObjNum 718 snap 219 seq_num 718) 2024-09-21T09:39:02.324 INFO:tasks.rados.rados.0.smithi022.stdout:2279: snap_create 2024-09-21T09:39:02.324 INFO:tasks.rados.rados.0.smithi022.stdout:2276: finishing rollback tid 0 to smithi02284960-39 2024-09-21T09:39:02.324 INFO:tasks.rados.rados.0.smithi022.stdout:2276: finishing rollback tid 1 to smithi02284960-39 2024-09-21T09:39:02.324 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 39 v 1258 (ObjNum 161 snap 0 seq_num 1) dirty dne 2024-09-21T09:39:02.327 INFO:tasks.rados.rados.0.smithi022.stdout:2277: finishing write tid 1 to smithi02284960-11 2024-09-21T09:39:02.327 INFO:tasks.rados.rados.0.smithi022.stdout:2277: finishing write tid 2 to smithi02284960-11 2024-09-21T09:39:02.329 INFO:tasks.rados.rados.0.smithi022.stdout:2277: finishing write tid 3 to smithi02284960-11 2024-09-21T09:39:02.329 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 11 v 1195 (ObjNum 732 snap 221 seq_num 732) dirty exists 2024-09-21T09:39:02.329 INFO:tasks.rados.rados.0.smithi022.stdout:2277: left oid 11 (ObjNum 732 snap 221 seq_num 732) 2024-09-21T09:39:02.337 INFO:tasks.rados.rados.0.smithi022.stdout:2278: expect (ObjNum 718 snap 219 seq_num 718) 2024-09-21T09:39:02.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:02 smithi022 ceph-mon[98615]: osdmap e519: 8 total, 8 up, 8 in 2024-09-21T09:39:02.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:02 smithi022 ceph-mon[96736]: osdmap e519: 8 total, 8 up, 8 in 2024-09-21T09:39:02.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:02 smithi064 ceph-mon[86709]: osdmap e519: 8 total, 8 up, 8 in 2024-09-21T09:39:03.285 INFO:tasks.rados.rados.0.smithi022.stdout:2276: done (3 left) 2024-09-21T09:39:03.285 INFO:tasks.rados.rados.0.smithi022.stdout:2277: done (2 left) 2024-09-21T09:39:03.285 INFO:tasks.rados.rados.0.smithi022.stdout:2278: done (1 left) 2024-09-21T09:39:03.285 INFO:tasks.rados.rados.0.smithi022.stdout:2279: done (0 left) 2024-09-21T09:39:03.285 INFO:tasks.rados.rados.0.smithi022.stdout:2280: delete oid 6 current snap is 222 2024-09-21T09:39:03.290 INFO:tasks.rados.rados.0.smithi022.stdout:2280: done (0 left) 2024-09-21T09:39:03.290 INFO:tasks.rados.rados.0.smithi022.stdout:2281: read oid 47 snap -1 2024-09-21T09:39:03.290 INFO:tasks.rados.rados.0.smithi022.stdout:2281: expect (ObjNum 604 snap 183 seq_num 604) 2024-09-21T09:39:03.290 INFO:tasks.rados.rados.0.smithi022.stdout:2282: read oid 10 snap 208 2024-09-21T09:39:03.291 INFO:tasks.rados.rados.0.smithi022.stdout:2282: expect (ObjNum 664 snap 206 seq_num 664) 2024-09-21T09:39:03.291 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 35 current snap is 222 2024-09-21T09:39:03.291 INFO:tasks.rados.rados.0.smithi022.stdout:2283: seq_num 734 ranges {2621440=294912} 2024-09-21T09:39:03.307 INFO:tasks.rados.rados.0.smithi022.stdout:2283: writing smithi02284960-35 from 2621440 to 2916352 tid 1 2024-09-21T09:39:03.307 INFO:tasks.rados.rados.0.smithi022.stdout:2284: snap_remove snap 221 2024-09-21T09:39:03.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:03 smithi022 ceph-mon[98615]: osdmap e520: 8 total, 8 up, 8 in 2024-09-21T09:39:03.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:03 smithi022 ceph-mon[98615]: pgmap v384: 121 pgs: 121 active+clean; 250 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 712 KiB/s wr, 6 op/s; 18 MiB/s, 2 objects/s recovering 2024-09-21T09:39:03.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:03 smithi022 ceph-mon[96736]: osdmap e520: 8 total, 8 up, 8 in 2024-09-21T09:39:03.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:03 smithi022 ceph-mon[96736]: pgmap v384: 121 pgs: 121 active+clean; 250 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 712 KiB/s wr, 6 op/s; 18 MiB/s, 2 objects/s recovering 2024-09-21T09:39:03.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:03 smithi064 ceph-mon[86709]: osdmap e520: 8 total, 8 up, 8 in 2024-09-21T09:39:03.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:03 smithi064 ceph-mon[86709]: pgmap v384: 121 pgs: 121 active+clean; 250 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 712 KiB/s wr, 6 op/s; 18 MiB/s, 2 objects/s recovering 2024-09-21T09:39:04.002 INFO:tasks.rados.rados.0.smithi022.stdout:2284: done (3 left) 2024-09-21T09:39:04.002 INFO:tasks.rados.rados.0.smithi022.stdout:2285: rollback oid 32 current snap is 222 2024-09-21T09:39:04.002 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 32 to 208 2024-09-21T09:39:04.002 INFO:tasks.rados.rados.0.smithi022.stdout:2281: expect (ObjNum 604 snap 183 seq_num 604) 2024-09-21T09:39:04.311 INFO:tasks.rados.rados.0.smithi022.stdout:2283: finishing write tid 1 to smithi02284960-35 2024-09-21T09:39:04.311 INFO:tasks.rados.rados.0.smithi022.stdout:2283: finishing write tid 2 to smithi02284960-35 2024-09-21T09:39:04.311 INFO:tasks.rados.rados.0.smithi022.stdout:2283: finishing write tid 3 to smithi02284960-35 2024-09-21T09:39:04.311 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 35 v 1262 (ObjNum 734 snap 222 seq_num 734) dirty exists 2024-09-21T09:39:04.311 INFO:tasks.rados.rados.0.smithi022.stdout:2283: left oid 35 (ObjNum 734 snap 222 seq_num 734) 2024-09-21T09:39:04.311 INFO:tasks.rados.rados.0.smithi022.stdout:2282: expect (ObjNum 664 snap 206 seq_num 664) 2024-09-21T09:39:04.557 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:04 smithi022 ceph-mon[98615]: osdmap e521: 8 total, 8 up, 8 in 2024-09-21T09:39:04.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:04 smithi022 ceph-mon[98615]: osdmap e522: 8 total, 8 up, 8 in 2024-09-21T09:39:04.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:04 smithi022 ceph-mon[96736]: osdmap e521: 8 total, 8 up, 8 in 2024-09-21T09:39:04.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:04 smithi022 ceph-mon[96736]: osdmap e522: 8 total, 8 up, 8 in 2024-09-21T09:39:04.593 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:04 smithi064 ceph-mon[86709]: osdmap e521: 8 total, 8 up, 8 in 2024-09-21T09:39:04.593 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:04 smithi064 ceph-mon[86709]: osdmap e522: 8 total, 8 up, 8 in 2024-09-21T09:39:04.695 INFO:tasks.rados.rados.0.smithi022.stdout:2285: finishing rollback tid 0 to smithi02284960-32 2024-09-21T09:39:04.695 INFO:tasks.rados.rados.0.smithi022.stdout:2285: finishing rollback tid 1 to smithi02284960-32 2024-09-21T09:39:04.695 INFO:tasks.rados.rados.0.smithi022.stdout:2285: finishing rollback tid 2 to smithi02284960-32 2024-09-21T09:39:04.695 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 32 v 1396 (ObjNum 648 snap 199 seq_num 648) dirty exists 2024-09-21T09:39:04.695 INFO:tasks.rados.rados.0.smithi022.stdout:2281: done (3 left) 2024-09-21T09:39:04.695 INFO:tasks.rados.rados.0.smithi022.stdout:2282: done (2 left) 2024-09-21T09:39:04.695 INFO:tasks.rados.rados.0.smithi022.stdout:2283: done (1 left) 2024-09-21T09:39:04.695 INFO:tasks.rados.rados.0.smithi022.stdout:2285: done (0 left) 2024-09-21T09:39:04.695 INFO:tasks.rados.rados.0.smithi022.stdout:2286: read oid 41 snap -1 2024-09-21T09:39:04.695 INFO:tasks.rados.rados.0.smithi022.stdout:2286: expect deleted 2024-09-21T09:39:04.695 INFO:tasks.rados.rados.0.smithi022.stdout:2287: snap_create 2024-09-21T09:39:05.291 INFO:tasks.rados.rados.0.smithi022.stdout:2286: done (1 left) 2024-09-21T09:39:05.291 INFO:tasks.rados.rados.0.smithi022.stdout:2287: done (0 left) 2024-09-21T09:39:05.291 INFO:tasks.rados.rados.0.smithi022.stdout:2288: copy_from oid 21 from oid 50 current snap is 223 2024-09-21T09:39:05.291 INFO:tasks.rados.rados.0.smithi022.stdout:2289: read oid 7 snap -1 2024-09-21T09:39:05.291 INFO:tasks.rados.rados.0.smithi022.stdout:2289: expect (ObjNum 694 snap 215 seq_num 694) 2024-09-21T09:39:05.292 INFO:tasks.rados.rados.0.smithi022.stdout:2290: read oid 37 snap -1 2024-09-21T09:39:05.292 INFO:tasks.rados.rados.0.smithi022.stdout:2290: expect deleted 2024-09-21T09:39:05.292 INFO:tasks.rados.rados.0.smithi022.stdout:2291: snap_create 2024-09-21T09:39:05.301 INFO:tasks.rados.rados.0.smithi022.stdout:2289: expect (ObjNum 694 snap 215 seq_num 694) 2024-09-21T09:39:05.355 INFO:tasks.rados.rados.0.smithi022.stdout:2288: finishing copy_from racing read to smithi02284960-21 2024-09-21T09:39:05.355 INFO:tasks.rados.rados.0.smithi022.stdout:2288: finishing copy_from to smithi02284960-21 2024-09-21T09:39:05.355 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 21 v 1271 (ObjNum 701 snap 216 seq_num 701) dirty exists 2024-09-21T09:39:05.557 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:05 smithi022 ceph-mon[98615]: pgmap v387: 121 pgs: 121 active+clean; 233 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 20 MiB/s rd, 238 KiB/s wr, 11 op/s 2024-09-21T09:39:05.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:05 smithi022 ceph-mon[96736]: pgmap v387: 121 pgs: 121 active+clean; 233 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 20 MiB/s rd, 238 KiB/s wr, 11 op/s 2024-09-21T09:39:05.612 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:05 smithi064 ceph-mon[86709]: pgmap v387: 121 pgs: 121 active+clean; 233 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 20 MiB/s rd, 238 KiB/s wr, 11 op/s 2024-09-21T09:39:06.058 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:39:05 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:39:05] "GET /metrics HTTP/1.1" 200 36143 "" "Prometheus/2.51.0" 2024-09-21T09:39:06.293 INFO:tasks.rados.rados.0.smithi022.stdout:2288: done (3 left) 2024-09-21T09:39:06.293 INFO:tasks.rados.rados.0.smithi022.stdout:2289: done (2 left) 2024-09-21T09:39:06.293 INFO:tasks.rados.rados.0.smithi022.stdout:2290: done (1 left) 2024-09-21T09:39:06.293 INFO:tasks.rados.rados.0.smithi022.stdout:2291: done (0 left) 2024-09-21T09:39:06.293 INFO:tasks.rados.rados.0.smithi022.stdout:2292: rollback oid 37 current snap is 224 2024-09-21T09:39:06.293 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 37 to 208 2024-09-21T09:39:06.293 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 48 current snap is 224 2024-09-21T09:39:06.293 INFO:tasks.rados.rados.0.smithi022.stdout:2293: seq_num 735 ranges {0=311296} 2024-09-21T09:39:06.294 INFO:tasks.rados.rados.0.smithi022.stdout:2293: writing smithi02284960-48 from 0 to 311296 tid 1 2024-09-21T09:39:06.294 INFO:tasks.rados.rados.0.smithi022.stdout:2294: snap_create 2024-09-21T09:39:06.300 INFO:tasks.rados.rados.0.smithi022.stdout:2293: finishing write tid 1 to smithi02284960-48 2024-09-21T09:39:06.300 INFO:tasks.rados.rados.0.smithi022.stdout:2293: finishing write tid 2 to smithi02284960-48 2024-09-21T09:39:06.300 INFO:tasks.rados.rados.0.smithi022.stdout:2292: finishing rollback tid 1 to smithi02284960-37 2024-09-21T09:39:06.300 INFO:tasks.rados.rados.0.smithi022.stdout:2292: finishing rollback tid 2 to smithi02284960-37 2024-09-21T09:39:06.300 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 37 v 1385 (ObjNum 672 snap 208 seq_num 672) dirty exists 2024-09-21T09:39:06.301 INFO:tasks.rados.rados.0.smithi022.stdout:2293: finishing write tid 3 to smithi02284960-48 2024-09-21T09:39:06.302 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 48 v 1400 (ObjNum 735 snap 224 seq_num 735) dirty exists 2024-09-21T09:39:06.302 INFO:tasks.rados.rados.0.smithi022.stdout:2293: left oid 48 (ObjNum 735 snap 224 seq_num 735) 2024-09-21T09:39:06.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:06 smithi022 ceph-mon[98615]: osdmap e523: 8 total, 8 up, 8 in 2024-09-21T09:39:06.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:06 smithi022 ceph-mon[96736]: osdmap e523: 8 total, 8 up, 8 in 2024-09-21T09:39:06.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:06 smithi064 ceph-mon[86709]: osdmap e523: 8 total, 8 up, 8 in 2024-09-21T09:39:07.302 INFO:tasks.rados.rados.0.smithi022.stdout:2292: done (2 left) 2024-09-21T09:39:07.302 INFO:tasks.rados.rados.0.smithi022.stdout:2293: done (1 left) 2024-09-21T09:39:07.302 INFO:tasks.rados.rados.0.smithi022.stdout:2294: done (0 left) 2024-09-21T09:39:07.302 INFO:tasks.rados.rados.0.smithi022.stdout:2295: read oid 21 snap -1 2024-09-21T09:39:07.302 INFO:tasks.rados.rados.0.smithi022.stdout:2295: expect (ObjNum 701 snap 216 seq_num 701) 2024-09-21T09:39:07.302 INFO:tasks.rados.rados.0.smithi022.stdout:2296: snap_remove snap 208 2024-09-21T09:39:07.557 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:07 smithi022 ceph-mon[98615]: osdmap e524: 8 total, 8 up, 8 in 2024-09-21T09:39:07.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:07 smithi022 ceph-mon[98615]: pgmap v390: 121 pgs: 121 active+clean; 223 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 264 KiB/s wr, 18 op/s 2024-09-21T09:39:07.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:07 smithi022 ceph-mon[96736]: osdmap e524: 8 total, 8 up, 8 in 2024-09-21T09:39:07.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:07 smithi022 ceph-mon[96736]: pgmap v390: 121 pgs: 121 active+clean; 223 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 264 KiB/s wr, 18 op/s 2024-09-21T09:39:07.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:07 smithi064 ceph-mon[86709]: osdmap e524: 8 total, 8 up, 8 in 2024-09-21T09:39:07.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:07 smithi064 ceph-mon[86709]: pgmap v390: 121 pgs: 121 active+clean; 223 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 264 KiB/s wr, 18 op/s 2024-09-21T09:39:08.309 INFO:tasks.rados.rados.0.smithi022.stdout:2296: done (1 left) 2024-09-21T09:39:08.309 INFO:tasks.rados.rados.0.smithi022.stdout:2297: copy_from oid 1 from oid 37 current snap is 225 2024-09-21T09:39:08.309 INFO:tasks.rados.rados.0.smithi022.stdout:2298: snap_create 2024-09-21T09:39:08.309 INFO:tasks.rados.rados.0.smithi022.stdout:2295: expect (ObjNum 701 snap 216 seq_num 701) 2024-09-21T09:39:08.513 INFO:tasks.rados.rados.0.smithi022.stdout:2297: finishing copy_from racing read to smithi02284960-1 2024-09-21T09:39:08.513 INFO:tasks.rados.rados.0.smithi022.stdout:2297: finishing copy_from to smithi02284960-1 2024-09-21T09:39:08.513 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 1 v 1385 (ObjNum 672 snap 208 seq_num 672) dirty exists 2024-09-21T09:39:08.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:08 smithi064 ceph-mon[86709]: osdmap e525: 8 total, 8 up, 8 in 2024-09-21T09:39:08.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:08 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:39:08.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:08 smithi022 ceph-mon[98615]: osdmap e525: 8 total, 8 up, 8 in 2024-09-21T09:39:08.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:08 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:39:08.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:08 smithi022 ceph-mon[96736]: osdmap e525: 8 total, 8 up, 8 in 2024-09-21T09:39:08.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:08 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:39:09.003 INFO:tasks.rados.rados.0.smithi022.stdout:2295: done (2 left) 2024-09-21T09:39:09.003 INFO:tasks.rados.rados.0.smithi022.stdout:2297: done (1 left) 2024-09-21T09:39:09.003 INFO:tasks.rados.rados.0.smithi022.stdout:2298: done (0 left) 2024-09-21T09:39:09.003 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 21 current snap is 226 2024-09-21T09:39:09.003 INFO:tasks.rados.rados.0.smithi022.stdout:2299: seq_num 736 ranges {5062656=335872} 2024-09-21T09:39:09.032 INFO:tasks.rados.rados.0.smithi022.stdout:2299: writing smithi02284960-21 from 5062656 to 5398528 tid 1 2024-09-21T09:39:09.032 INFO:tasks.rados.rados.0.smithi022.stdout:2300: snap_remove snap 224 2024-09-21T09:39:09.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:09 smithi064 ceph-mon[86709]: osdmap e526: 8 total, 8 up, 8 in 2024-09-21T09:39:09.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:09 smithi064 ceph-mon[86709]: pgmap v393: 121 pgs: 121 active+clean; 210 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 6.7 MiB/s rd, 1.4 MiB/s wr, 12 op/s 2024-09-21T09:39:09.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:09 smithi064 ceph-mon[86709]: osdmap e527: 8 total, 8 up, 8 in 2024-09-21T09:39:09.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:09 smithi022 ceph-mon[98615]: osdmap e526: 8 total, 8 up, 8 in 2024-09-21T09:39:09.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:09 smithi022 ceph-mon[98615]: pgmap v393: 121 pgs: 121 active+clean; 210 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 6.7 MiB/s rd, 1.4 MiB/s wr, 12 op/s 2024-09-21T09:39:09.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:09 smithi022 ceph-mon[98615]: osdmap e527: 8 total, 8 up, 8 in 2024-09-21T09:39:09.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:09 smithi022 ceph-mon[96736]: osdmap e526: 8 total, 8 up, 8 in 2024-09-21T09:39:09.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:09 smithi022 ceph-mon[96736]: pgmap v393: 121 pgs: 121 active+clean; 210 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 6.7 MiB/s rd, 1.4 MiB/s wr, 12 op/s 2024-09-21T09:39:09.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:09 smithi022 ceph-mon[96736]: osdmap e527: 8 total, 8 up, 8 in 2024-09-21T09:39:10.011 INFO:tasks.rados.rados.0.smithi022.stdout:2300: done (1 left) 2024-09-21T09:39:10.011 INFO:tasks.rados.rados.0.smithi022.stdout:2301: read oid 46 snap -1 2024-09-21T09:39:10.011 INFO:tasks.rados.rados.0.smithi022.stdout:2301: expect (ObjNum 609 snap 185 seq_num 609) 2024-09-21T09:39:10.011 INFO:tasks.rados.rados.0.smithi022.stdout:2302: snap_create 2024-09-21T09:39:10.011 INFO:tasks.rados.rados.0.smithi022.stdout:2299: finishing write tid 1 to smithi02284960-21 2024-09-21T09:39:10.011 INFO:tasks.rados.rados.0.smithi022.stdout:2299: finishing write tid 2 to smithi02284960-21 2024-09-21T09:39:10.011 INFO:tasks.rados.rados.0.smithi022.stdout:2299: finishing write tid 3 to smithi02284960-21 2024-09-21T09:39:10.012 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 21 v 1273 (ObjNum 736 snap 226 seq_num 736) dirty exists 2024-09-21T09:39:10.012 INFO:tasks.rados.rados.0.smithi022.stdout:2299: left oid 21 (ObjNum 736 snap 226 seq_num 736) 2024-09-21T09:39:10.017 INFO:tasks.rados.rados.0.smithi022.stdout:2301: expect (ObjNum 609 snap 185 seq_num 609) 2024-09-21T09:39:10.482 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:39:10 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:39:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:39:11.015 INFO:tasks.rados.rados.0.smithi022.stdout:2299: done (2 left) 2024-09-21T09:39:11.015 INFO:tasks.rados.rados.0.smithi022.stdout:2301: done (1 left) 2024-09-21T09:39:11.015 INFO:tasks.rados.rados.0.smithi022.stdout:2302: done (0 left) 2024-09-21T09:39:11.015 INFO:tasks.rados.rados.0.smithi022.stdout:2303: rollback oid 23 current snap is 227 2024-09-21T09:39:11.015 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 23 to 222 2024-09-21T09:39:11.016 INFO:tasks.rados.rados.0.smithi022.stdout:2304: copy_from oid 7 from oid 25 current snap is 227 2024-09-21T09:39:11.016 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 8 current snap is 227 2024-09-21T09:39:11.016 INFO:tasks.rados.rados.0.smithi022.stdout:2305: seq_num 737 ranges {0=352256} 2024-09-21T09:39:11.017 INFO:tasks.rados.rados.0.smithi022.stdout:2305: writing smithi02284960-8 from 0 to 352256 tid 1 2024-09-21T09:39:11.017 INFO:tasks.rados.rados.0.smithi022.stdout:2306: snap_remove snap 223 2024-09-21T09:39:11.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:11 smithi064 ceph-mon[86709]: osdmap e528: 8 total, 8 up, 8 in 2024-09-21T09:39:11.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:11 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16} : dispatch 2024-09-21T09:39:11.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:11 smithi022 ceph-mon[98615]: osdmap e528: 8 total, 8 up, 8 in 2024-09-21T09:39:11.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:11 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16} : dispatch 2024-09-21T09:39:11.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:11 smithi022 ceph-mon[96736]: osdmap e528: 8 total, 8 up, 8 in 2024-09-21T09:39:11.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:11 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16} : dispatch 2024-09-21T09:39:12.022 INFO:tasks.rados.rados.0.smithi022.stdout:2306: done (3 left) 2024-09-21T09:39:12.022 INFO:tasks.rados.rados.0.smithi022.stdout:2307: read oid 31 snap -1 2024-09-21T09:39:12.022 INFO:tasks.rados.rados.0.smithi022.stdout:2307: expect (ObjNum 590 snap 176 seq_num 590) 2024-09-21T09:39:12.022 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 10 current snap is 227 2024-09-21T09:39:12.022 INFO:tasks.rados.rados.0.smithi022.stdout:2304: finishing copy_from to smithi02284960-7 2024-09-21T09:39:12.022 INFO:tasks.rados.rados.0.smithi022.stdout:2304: got expected ENOENT (src dne) 2024-09-21T09:39:12.023 INFO:tasks.rados.rados.0.smithi022.stdout:2304: finishing copy_from racing read to smithi02284960-7 2024-09-21T09:39:12.023 INFO:tasks.rados.rados.0.smithi022.stdout:2305: finishing write tid 1 to smithi02284960-8 2024-09-21T09:39:12.023 INFO:tasks.rados.rados.0.smithi022.stdout:2305: finishing write tid 2 to smithi02284960-8 2024-09-21T09:39:12.023 INFO:tasks.rados.rados.0.smithi022.stdout:2308: seq_num 738 ranges {11051008=368640} 2024-09-21T09:39:12.084 INFO:tasks.rados.rados.0.smithi022.stdout:2308: writing smithi02284960-10 from 11051008 to 11419648 tid 1 2024-09-21T09:39:12.084 INFO:tasks.rados.rados.0.smithi022.stdout:2304: done (4 left) 2024-09-21T09:39:12.084 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 46 current snap is 227 2024-09-21T09:39:12.084 INFO:tasks.rados.rados.0.smithi022.stdout:2303: finishing rollback tid 0 to smithi02284960-23 2024-09-21T09:39:12.084 INFO:tasks.rados.rados.0.smithi022.stdout:2303: finishing rollback tid 1 to smithi02284960-23 2024-09-21T09:39:12.084 INFO:tasks.rados.rados.0.smithi022.stdout:2303: finishing rollback tid 2 to smithi02284960-23 2024-09-21T09:39:12.085 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 23 v 1067 (ObjNum 722 snap 220 seq_num 722) dirty exists 2024-09-21T09:39:12.085 INFO:tasks.rados.rados.0.smithi022.stdout:2305: finishing write tid 3 to smithi02284960-8 2024-09-21T09:39:12.085 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 8 v 1132 (ObjNum 737 snap 227 seq_num 737) dirty exists 2024-09-21T09:39:12.085 INFO:tasks.rados.rados.0.smithi022.stdout:2305: left oid 8 (ObjNum 737 snap 227 seq_num 737) 2024-09-21T09:39:12.085 INFO:tasks.rados.rados.0.smithi022.stdout:2309: seq_num 739 ranges {606208=385024} 2024-09-21T09:39:12.089 INFO:tasks.rados.rados.0.smithi022.stdout:2309: writing smithi02284960-46 from 606208 to 991232 tid 1 2024-09-21T09:39:12.089 INFO:tasks.rados.rados.0.smithi022.stdout:2303: done (4 left) 2024-09-21T09:39:12.090 INFO:tasks.rados.rados.0.smithi022.stdout:2305: done (3 left) 2024-09-21T09:39:12.090 INFO:tasks.rados.rados.0.smithi022.stdout:2310: read oid 22 snap -1 2024-09-21T09:39:12.090 INFO:tasks.rados.rados.0.smithi022.stdout:2310: expect (ObjNum 619 snap 186 seq_num 619) 2024-09-21T09:39:12.090 INFO:tasks.rados.rados.0.smithi022.stdout:2311: setattr oid 2 current snap is 227 2024-09-21T09:39:12.090 INFO:tasks.rados.rados.0.smithi022.stdout:2307: expect (ObjNum 590 snap 176 seq_num 590) 2024-09-21T09:39:12.100 INFO:tasks.rados.rados.0.smithi022.stdout:2308: finishing write tid 1 to smithi02284960-10 2024-09-21T09:39:12.100 INFO:tasks.rados.rados.0.smithi022.stdout:2308: finishing write tid 2 to smithi02284960-10 2024-09-21T09:39:12.100 INFO:tasks.rados.rados.0.smithi022.stdout:2308: finishing write tid 3 to smithi02284960-10 2024-09-21T09:39:12.100 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 10 v 1265 (ObjNum 738 snap 227 seq_num 738) dirty exists 2024-09-21T09:39:12.100 INFO:tasks.rados.rados.0.smithi022.stdout:2308: left oid 10 (ObjNum 738 snap 227 seq_num 738) 2024-09-21T09:39:12.101 INFO:tasks.rados.rados.0.smithi022.stdout:2309: finishing write tid 1 to smithi02284960-46 2024-09-21T09:39:12.101 INFO:tasks.rados.rados.0.smithi022.stdout:2309: finishing write tid 2 to smithi02284960-46 2024-09-21T09:39:12.101 INFO:tasks.rados.rados.0.smithi022.stdout:2310: expect (ObjNum 619 snap 186 seq_num 619) 2024-09-21T09:39:12.121 INFO:tasks.rados.rados.0.smithi022.stdout:2309: finishing write tid 3 to smithi02284960-46 2024-09-21T09:39:12.122 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 46 v 1387 (ObjNum 739 snap 227 seq_num 739) dirty exists 2024-09-21T09:39:12.122 INFO:tasks.rados.rados.0.smithi022.stdout:2309: left oid 46 (ObjNum 739 snap 227 seq_num 739) 2024-09-21T09:39:12.122 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 2 v 1274 (ObjNum 65 snap 0 seq_num 1365405519) dirty exists 2024-09-21T09:39:12.122 INFO:tasks.rados.rados.0.smithi022.stdout:2307: done (4 left) 2024-09-21T09:39:12.122 INFO:tasks.rados.rados.0.smithi022.stdout:2308: done (3 left) 2024-09-21T09:39:12.122 INFO:tasks.rados.rados.0.smithi022.stdout:2309: done (2 left) 2024-09-21T09:39:12.122 INFO:tasks.rados.rados.0.smithi022.stdout:2310: done (1 left) 2024-09-21T09:39:12.122 INFO:tasks.rados.rados.0.smithi022.stdout:2311: done (0 left) 2024-09-21T09:39:12.122 INFO:tasks.rados.rados.0.smithi022.stdout:2312: snap_remove snap 222 2024-09-21T09:39:12.267 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:12 smithi064 ceph-mon[86709]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-21T09:39:12.267 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:12 smithi064 ceph-mon[86709]: Upgrade: osd.6 is safe to restart 2024-09-21T09:39:12.268 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:12 smithi064 ceph-mon[86709]: pgmap v396: 121 pgs: 121 active+clean; 166 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 5.5 MiB/s rd, 1.7 MiB/s wr, 5 op/s 2024-09-21T09:39:12.268 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:12 smithi064 ceph-mon[86709]: osdmap e529: 8 total, 8 up, 8 in 2024-09-21T09:39:12.268 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:12 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:39:12.268 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:12 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-21T09:39:12.268 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:12 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:39:12.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:12 smithi022 ceph-mon[98615]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-21T09:39:12.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:12 smithi022 ceph-mon[98615]: Upgrade: osd.6 is safe to restart 2024-09-21T09:39:12.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:12 smithi022 ceph-mon[98615]: pgmap v396: 121 pgs: 121 active+clean; 166 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 5.5 MiB/s rd, 1.7 MiB/s wr, 5 op/s 2024-09-21T09:39:12.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:12 smithi022 ceph-mon[98615]: osdmap e529: 8 total, 8 up, 8 in 2024-09-21T09:39:12.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:12 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:39:12.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:12 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-21T09:39:12.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:12 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:39:12.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:12 smithi022 ceph-mon[96736]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-21T09:39:12.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:12 smithi022 ceph-mon[96736]: Upgrade: osd.6 is safe to restart 2024-09-21T09:39:12.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:12 smithi022 ceph-mon[96736]: pgmap v396: 121 pgs: 121 active+clean; 166 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 5.5 MiB/s rd, 1.7 MiB/s wr, 5 op/s 2024-09-21T09:39:12.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:12 smithi022 ceph-mon[96736]: osdmap e529: 8 total, 8 up, 8 in 2024-09-21T09:39:12.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:12 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:39:12.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:12 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-21T09:39:12.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:12 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:39:13.027 INFO:tasks.rados.rados.0.smithi022.stdout:2312: done (0 left) 2024-09-21T09:39:13.027 INFO:tasks.rados.rados.0.smithi022.stdout:2313: delete oid 47 current snap is 227 2024-09-21T09:39:13.035 INFO:tasks.rados.rados.0.smithi022.stdout:2313: done (0 left) 2024-09-21T09:39:13.036 INFO:tasks.rados.rados.0.smithi022.stdout:2314: delete oid 13 current snap is 227 2024-09-21T09:39:13.037 INFO:tasks.rados.rados.0.smithi022.stdout:2314: done (0 left) 2024-09-21T09:39:13.037 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 39 current snap is 227 2024-09-21T09:39:13.038 INFO:tasks.rados.rados.0.smithi022.stdout:2315: seq_num 742 ranges {0=434176} 2024-09-21T09:39:13.040 INFO:tasks.rados.rados.0.smithi022.stdout:2315: writing smithi02284960-39 from 0 to 434176 tid 1 2024-09-21T09:39:13.040 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 41 current snap is 227 2024-09-21T09:39:13.040 INFO:tasks.rados.rados.0.smithi022.stdout:2316: seq_num 743 ranges {0=450560} 2024-09-21T09:39:13.043 INFO:tasks.rados.rados.0.smithi022.stdout:2316: writing smithi02284960-41 from 0 to 450560 tid 1 2024-09-21T09:39:13.043 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 20 current snap is 227 2024-09-21T09:39:13.043 INFO:tasks.rados.rados.0.smithi022.stdout:2317: seq_num 744 ranges {5603328=466944} 2024-09-21T09:39:13.077 INFO:tasks.rados.rados.0.smithi022.stdout:2317: writing smithi02284960-20 from 5603328 to 6070272 tid 1 2024-09-21T09:39:13.077 INFO:tasks.rados.rados.0.smithi022.stdout:2318: setattr oid 19 current snap is 227 2024-09-21T09:39:13.077 INFO:tasks.rados.rados.0.smithi022.stdout:2315: finishing write tid 1 to smithi02284960-39 2024-09-21T09:39:13.077 INFO:tasks.rados.rados.0.smithi022.stdout:2315: finishing write tid 2 to smithi02284960-39 2024-09-21T09:39:13.077 INFO:tasks.rados.rados.0.smithi022.stdout:2315: finishing write tid 3 to smithi02284960-39 2024-09-21T09:39:13.077 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 39 v 1260 (ObjNum 742 snap 227 seq_num 742) dirty exists 2024-09-21T09:39:13.077 INFO:tasks.rados.rados.0.smithi022.stdout:2315: left oid 39 (ObjNum 742 snap 227 seq_num 742) 2024-09-21T09:39:13.077 INFO:tasks.rados.rados.0.smithi022.stdout:2316: finishing write tid 1 to smithi02284960-41 2024-09-21T09:39:13.077 INFO:tasks.rados.rados.0.smithi022.stdout:2316: finishing write tid 2 to smithi02284960-41 2024-09-21T09:39:13.078 INFO:tasks.rados.rados.0.smithi022.stdout:2316: finishing write tid 3 to smithi02284960-41 2024-09-21T09:39:13.078 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 41 v 1267 (ObjNum 743 snap 227 seq_num 743) dirty exists 2024-09-21T09:39:13.078 INFO:tasks.rados.rados.0.smithi022.stdout:2316: left oid 41 (ObjNum 743 snap 227 seq_num 743) 2024-09-21T09:39:13.078 INFO:tasks.rados.rados.0.smithi022.stdout:2315: done (3 left) 2024-09-21T09:39:13.078 INFO:tasks.rados.rados.0.smithi022.stdout:2316: done (2 left) 2024-09-21T09:39:13.078 INFO:tasks.rados.rados.0.smithi022.stdout:2319: delete oid 10 current snap is 227 2024-09-21T09:39:13.080 INFO:tasks.rados.rados.0.smithi022.stdout:2319: done (2 left) 2024-09-21T09:39:13.080 INFO:tasks.rados.rados.0.smithi022.stdout:2320: read oid 3 snap -1 2024-09-21T09:39:13.081 INFO:tasks.rados.rados.0.smithi022.stdout:2320: expect (ObjNum 675 snap 208 seq_num 675) 2024-09-21T09:39:13.081 INFO:tasks.rados.rados.0.smithi022.stdout:2321: read oid 28 snap 226 2024-09-21T09:39:13.081 INFO:tasks.rados.rados.0.smithi022.stdout:2321: expect (ObjNum 392 snap 122 seq_num 392) 2024-09-21T09:39:13.081 INFO:tasks.rados.rados.0.smithi022.stdout:2322: delete oid 49 current snap is 227 2024-09-21T09:39:13.081 INFO:tasks.rados.rados.0.smithi022.stdout:2317: finishing write tid 1 to smithi02284960-20 2024-09-21T09:39:13.082 INFO:tasks.rados.rados.0.smithi022.stdout:2317: finishing write tid 2 to smithi02284960-20 2024-09-21T09:39:13.082 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 19 v 1393 (ObjNum 604 snap 183 seq_num 604) dirty exists 2024-09-21T09:39:13.084 INFO:tasks.rados.rados.0.smithi022.stdout:2317: finishing write tid 3 to smithi02284960-20 2024-09-21T09:39:13.084 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 20 v 1409 (ObjNum 744 snap 227 seq_num 744) dirty exists 2024-09-21T09:39:13.084 INFO:tasks.rados.rados.0.smithi022.stdout:2317: left oid 20 (ObjNum 744 snap 227 seq_num 744) 2024-09-21T09:39:13.085 INFO:tasks.rados.rados.0.smithi022.stdout:2317: done (4 left) 2024-09-21T09:39:13.085 INFO:tasks.rados.rados.0.smithi022.stdout:2318: done (3 left) 2024-09-21T09:39:13.085 INFO:tasks.rados.rados.0.smithi022.stdout:2322: done (2 left) 2024-09-21T09:39:13.085 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 4 current snap is 227 2024-09-21T09:39:13.085 INFO:tasks.rados.rados.0.smithi022.stdout:2323: seq_num 747 ranges {11649024=516096} 2024-09-21T09:39:13.152 INFO:tasks.rados.rados.0.smithi022.stdout:2323: writing smithi02284960-4 from 11649024 to 12165120 tid 1 2024-09-21T09:39:13.152 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 27 current snap is 227 2024-09-21T09:39:13.152 INFO:tasks.rados.rados.0.smithi022.stdout:2324: seq_num 748 ranges {3694592=532480} 2024-09-21T09:39:13.175 INFO:tasks.rados.rados.0.smithi022.stdout:2324: writing smithi02284960-27 from 3694592 to 4227072 tid 1 2024-09-21T09:39:13.175 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 11 current snap is 227 2024-09-21T09:39:13.175 INFO:tasks.rados.rados.0.smithi022.stdout:2325: seq_num 749 ranges {9420800=548864} 2024-09-21T09:39:13.229 INFO:tasks.rados.rados.0.smithi022.stdout:2325: writing smithi02284960-11 from 9420800 to 9969664 tid 1 2024-09-21T09:39:13.229 INFO:tasks.rados.rados.0.smithi022.stdout:2326: read oid 37 snap -1 2024-09-21T09:39:13.229 INFO:tasks.rados.rados.0.smithi022.stdout:2326: expect (ObjNum 672 snap 208 seq_num 672) 2024-09-21T09:39:13.229 INFO:tasks.rados.rados.0.smithi022.stdout:2327: rollback oid 31 current snap is 227 2024-09-21T09:39:13.229 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 31 to 226 2024-09-21T09:39:13.230 INFO:tasks.rados.rados.0.smithi022.stdout:2321: expect (ObjNum 392 snap 122 seq_num 392) 2024-09-21T09:39:13.280 INFO:tasks.rados.rados.0.smithi022.stdout:2320: expect (ObjNum 675 snap 208 seq_num 675) 2024-09-21T09:39:13.289 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:13 smithi064 ceph-mon[86709]: Upgrade: Updating osd.6 2024-09-21T09:39:13.289 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:13 smithi064 ceph-mon[86709]: Deploying daemon osd.6 on smithi064 2024-09-21T09:39:13.289 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:13 smithi064 ceph-mon[86709]: osdmap e530: 8 total, 8 up, 8 in 2024-09-21T09:39:13.289 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:13 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:39:13.289 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:13 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:39:13.307 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:13 smithi022 ceph-mon[98615]: Upgrade: Updating osd.6 2024-09-21T09:39:13.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:13 smithi022 ceph-mon[98615]: Deploying daemon osd.6 on smithi064 2024-09-21T09:39:13.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:13 smithi022 ceph-mon[98615]: osdmap e530: 8 total, 8 up, 8 in 2024-09-21T09:39:13.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:13 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:39:13.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:13 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:39:13.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:13 smithi022 ceph-mon[96736]: Upgrade: Updating osd.6 2024-09-21T09:39:13.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:13 smithi022 ceph-mon[96736]: Deploying daemon osd.6 on smithi064 2024-09-21T09:39:13.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:13 smithi022 ceph-mon[96736]: osdmap e530: 8 total, 8 up, 8 in 2024-09-21T09:39:13.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:13 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:39:13.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:13 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:39:13.325 INFO:tasks.rados.rados.0.smithi022.stdout:2323: finishing write tid 1 to smithi02284960-4 2024-09-21T09:39:13.326 INFO:tasks.rados.rados.0.smithi022.stdout:2323: finishing write tid 2 to smithi02284960-4 2024-09-21T09:39:13.326 INFO:tasks.rados.rados.0.smithi022.stdout:2323: finishing write tid 3 to smithi02284960-4 2024-09-21T09:39:13.326 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 4 v 1134 (ObjNum 747 snap 227 seq_num 747) dirty exists 2024-09-21T09:39:13.326 INFO:tasks.rados.rados.0.smithi022.stdout:2323: left oid 4 (ObjNum 747 snap 227 seq_num 747) 2024-09-21T09:39:13.326 INFO:tasks.rados.rados.0.smithi022.stdout:2324: finishing write tid 1 to smithi02284960-27 2024-09-21T09:39:13.326 INFO:tasks.rados.rados.0.smithi022.stdout:2324: finishing write tid 2 to smithi02284960-27 2024-09-21T09:39:13.326 INFO:tasks.rados.rados.0.smithi022.stdout:2320: done (6 left) 2024-09-21T09:39:13.326 INFO:tasks.rados.rados.0.smithi022.stdout:2321: done (5 left) 2024-09-21T09:39:13.326 INFO:tasks.rados.rados.0.smithi022.stdout:2323: done (4 left) 2024-09-21T09:39:13.326 INFO:tasks.rados.rados.0.smithi022.stdout:2328: delete oid 35 current snap is 227 2024-09-21T09:39:13.326 INFO:tasks.rados.rados.0.smithi022.stdout:2324: finishing write tid 3 to smithi02284960-27 2024-09-21T09:39:13.326 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 27 v 1326 (ObjNum 748 snap 227 seq_num 748) dirty exists 2024-09-21T09:39:13.326 INFO:tasks.rados.rados.0.smithi022.stdout:2324: left oid 27 (ObjNum 748 snap 227 seq_num 748) 2024-09-21T09:39:13.326 INFO:tasks.rados.rados.0.smithi022.stdout:2325: finishing write tid 1 to smithi02284960-11 2024-09-21T09:39:13.327 INFO:tasks.rados.rados.0.smithi022.stdout:2325: finishing write tid 2 to smithi02284960-11 2024-09-21T09:39:13.327 INFO:tasks.rados.rados.0.smithi022.stdout:2325: finishing write tid 3 to smithi02284960-11 2024-09-21T09:39:13.327 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 11 v 1390 (ObjNum 749 snap 227 seq_num 749) dirty exists 2024-09-21T09:39:13.327 INFO:tasks.rados.rados.0.smithi022.stdout:2325: left oid 11 (ObjNum 749 snap 227 seq_num 749) 2024-09-21T09:39:13.327 INFO:tasks.rados.rados.0.smithi022.stdout:2326: expect (ObjNum 672 snap 208 seq_num 672) 2024-09-21T09:39:13.371 INFO:tasks.rados.rados.0.smithi022.stdout:2327: finishing rollback tid 0 to smithi02284960-31 2024-09-21T09:39:13.371 INFO:tasks.rados.rados.0.smithi022.stdout:2327: finishing rollback tid 1 to smithi02284960-31 2024-09-21T09:39:13.371 INFO:tasks.rados.rados.0.smithi022.stdout:2327: finishing rollback tid 2 to smithi02284960-31 2024-09-21T09:39:13.371 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 31 v 1397 (ObjNum 590 snap 176 seq_num 590) dirty exists 2024-09-21T09:39:13.371 INFO:tasks.rados.rados.0.smithi022.stdout:2324: done (4 left) 2024-09-21T09:39:13.371 INFO:tasks.rados.rados.0.smithi022.stdout:2325: done (3 left) 2024-09-21T09:39:13.371 INFO:tasks.rados.rados.0.smithi022.stdout:2326: done (2 left) 2024-09-21T09:39:13.371 INFO:tasks.rados.rados.0.smithi022.stdout:2327: done (1 left) 2024-09-21T09:39:13.371 INFO:tasks.rados.rados.0.smithi022.stdout:2328: done (0 left) 2024-09-21T09:39:13.371 INFO:tasks.rados.rados.0.smithi022.stdout:2329: snap_create 2024-09-21T09:39:14.004 INFO:tasks.rados.rados.0.smithi022.stdout:2329: done (0 left) 2024-09-21T09:39:14.004 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 31 current snap is 228 2024-09-21T09:39:14.005 INFO:tasks.rados.rados.0.smithi022.stdout:2330: seq_num 751 ranges {286720=581632} 2024-09-21T09:39:14.009 INFO:tasks.rados.rados.0.smithi022.stdout:2330: writing smithi02284960-31 from 286720 to 868352 tid 1 2024-09-21T09:39:14.009 INFO:tasks.rados.rados.0.smithi022.stdout:2331: read oid 19 snap -1 2024-09-21T09:39:14.009 INFO:tasks.rados.rados.0.smithi022.stdout:2331: expect (ObjNum 604 snap 183 seq_num 604) 2024-09-21T09:39:14.009 INFO:tasks.rados.rados.0.smithi022.stdout:2332: snap_create 2024-09-21T09:39:14.017 INFO:tasks.rados.rados.0.smithi022.stdout:2330: finishing write tid 1 to smithi02284960-31 2024-09-21T09:39:14.020 INFO:tasks.rados.rados.0.smithi022.stdout:2330: finishing write tid 2 to smithi02284960-31 2024-09-21T09:39:14.038 INFO:tasks.rados.rados.0.smithi022.stdout:2331: expect (ObjNum 604 snap 183 seq_num 604) 2024-09-21T09:39:14.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:14 smithi022 ceph-mon[98615]: pgmap v399: 121 pgs: 121 active+clean; 161 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.9 MiB/s rd, 482 KiB/s wr, 3 op/s 2024-09-21T09:39:14.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:14 smithi022 ceph-mon[98615]: osdmap e531: 8 total, 8 up, 8 in 2024-09-21T09:39:14.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:14 smithi022 ceph-mon[98615]: osdmap e532: 8 total, 8 up, 8 in 2024-09-21T09:39:14.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:14 smithi022 ceph-mon[96736]: pgmap v399: 121 pgs: 121 active+clean; 161 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.9 MiB/s rd, 482 KiB/s wr, 3 op/s 2024-09-21T09:39:14.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:14 smithi022 ceph-mon[96736]: osdmap e531: 8 total, 8 up, 8 in 2024-09-21T09:39:14.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:14 smithi022 ceph-mon[96736]: osdmap e532: 8 total, 8 up, 8 in 2024-09-21T09:39:14.343 INFO:tasks.rados.rados.0.smithi022.stdout:2330: finishing write tid 3 to smithi02284960-31 2024-09-21T09:39:14.343 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 31 v 1400 (ObjNum 751 snap 228 seq_num 751) dirty exists 2024-09-21T09:39:14.343 INFO:tasks.rados.rados.0.smithi022.stdout:2330: left oid 31 (ObjNum 751 snap 228 seq_num 751) 2024-09-21T09:39:14.345 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:14 smithi064 ceph-mon[86709]: pgmap v399: 121 pgs: 121 active+clean; 161 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.9 MiB/s rd, 482 KiB/s wr, 3 op/s 2024-09-21T09:39:14.345 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:14 smithi064 ceph-mon[86709]: osdmap e531: 8 total, 8 up, 8 in 2024-09-21T09:39:14.345 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:14 smithi064 ceph-mon[86709]: osdmap e532: 8 total, 8 up, 8 in 2024-09-21T09:39:14.960 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:39:14 smithi064 systemd[1]: Stopping Ceph osd.6 for d65e801e-77fa-11ef-baf6-efdc52797490... 2024-09-21T09:39:15.009 INFO:tasks.rados.rados.0.smithi022.stdout:2330: done (2 left) 2024-09-21T09:39:15.009 INFO:tasks.rados.rados.0.smithi022.stdout:2331: done (1 left) 2024-09-21T09:39:15.009 INFO:tasks.rados.rados.0.smithi022.stdout:2332: done (0 left) 2024-09-21T09:39:15.009 INFO:tasks.rados.rados.0.smithi022.stdout:2333: snap_remove snap 227 2024-09-21T09:39:15.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:15 smithi022 ceph-mon[98615]: osd.6 marked itself down and dead 2024-09-21T09:39:15.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:15 smithi022 ceph-mon[98615]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T09:39:15.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:15 smithi022 ceph-mon[98615]: osdmap e533: 8 total, 7 up, 8 in 2024-09-21T09:39:15.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:15 smithi022 ceph-mon[96736]: osd.6 marked itself down and dead 2024-09-21T09:39:15.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:15 smithi022 ceph-mon[96736]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T09:39:15.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:15 smithi022 ceph-mon[96736]: osdmap e533: 8 total, 7 up, 8 in 2024-09-21T09:39:15.357 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:15 smithi064 ceph-mon[86709]: osd.6 marked itself down and dead 2024-09-21T09:39:15.357 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:15 smithi064 ceph-mon[86709]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T09:39:15.358 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:15 smithi064 ceph-mon[86709]: osdmap e533: 8 total, 7 up, 8 in 2024-09-21T09:39:15.358 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:39:14 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-6[49241]: 2024-09-21T09:39:14.957+0000 7fdb006d5640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T09:39:15.358 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:39:14 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-6[49241]: 2024-09-21T09:39:14.957+0000 7fdb006d5640 -1 osd.6 532 *** Got signal Terminated *** 2024-09-21T09:39:15.358 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:39:14 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-6[49241]: 2024-09-21T09:39:14.957+0000 7fdb006d5640 -1 osd.6 532 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T09:39:15.650 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:39:15 smithi064 podman[100824]: 2024-09-21 09:39:15.356455326 +0000 UTC m=+0.506053215 container died 3cc4ec816e5b30253356dfa8b194503ae443e533df2a638e132819c015f2d9ae (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-6, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-fbd1a6a, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True) 2024-09-21T09:39:15.941 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:39:15 smithi064 podman[100824]: 2024-09-21 09:39:15.748056768 +0000 UTC m=+0.897654646 container cleanup 3cc4ec816e5b30253356dfa8b194503ae443e533df2a638e132819c015f2d9ae (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-6, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, RELEASE=quincy-fbd1a6a, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T09:39:15.942 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:39:15 smithi064 bash[100824]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-6 2024-09-21T09:39:16.009 INFO:tasks.rados.rados.0.smithi022.stdout:2333: done (0 left) 2024-09-21T09:39:16.010 INFO:tasks.rados.rados.0.smithi022.stdout:2334: read oid 8 snap -1 2024-09-21T09:39:16.010 INFO:tasks.rados.rados.0.smithi022.stdout:2334: expect (ObjNum 737 snap 227 seq_num 737) 2024-09-21T09:39:16.010 INFO:tasks.rados.rados.0.smithi022.stdout:2335: read oid 35 snap -1 2024-09-21T09:39:16.010 INFO:tasks.rados.rados.0.smithi022.stdout:2335: expect deleted 2024-09-21T09:39:16.010 INFO:tasks.rados.rados.0.smithi022.stdout:2336: read oid 6 snap -1 2024-09-21T09:39:16.010 INFO:tasks.rados.rados.0.smithi022.stdout:2336: expect deleted 2024-09-21T09:39:16.010 INFO:tasks.rados.rados.0.smithi022.stdout:2337: snap_remove snap 225 2024-09-21T09:39:16.028 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:39:15 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:39:15] "GET /metrics HTTP/1.1" 200 36292 "" "Prometheus/2.51.0" 2024-09-21T09:39:16.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:16 smithi022 ceph-mon[96736]: pgmap v402: 121 pgs: 121 active+clean; 153 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 3.2 MiB/s rd, 412 KiB/s wr, 18 op/s 2024-09-21T09:39:16.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:16 smithi022 ceph-mon[96736]: osdmap e534: 8 total, 7 up, 8 in 2024-09-21T09:39:16.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:16 smithi022 ceph-mon[98615]: pgmap v402: 121 pgs: 121 active+clean; 153 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 3.2 MiB/s rd, 412 KiB/s wr, 18 op/s 2024-09-21T09:39:16.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:16 smithi022 ceph-mon[98615]: osdmap e534: 8 total, 7 up, 8 in 2024-09-21T09:39:16.326 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:16 smithi064 ceph-mon[86709]: pgmap v402: 121 pgs: 121 active+clean; 153 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 3.2 MiB/s rd, 412 KiB/s wr, 18 op/s 2024-09-21T09:39:16.326 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:16 smithi064 ceph-mon[86709]: osdmap e534: 8 total, 7 up, 8 in 2024-09-21T09:39:16.326 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:39:15 smithi064 podman[100844]: 2024-09-21 09:39:15.99927871 +0000 UTC m=+0.638320345 container remove 3cc4ec816e5b30253356dfa8b194503ae443e533df2a638e132819c015f2d9ae (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-6, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=quincy-fbd1a6a, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, ceph=True, org.label-schema.vendor=CentOS) 2024-09-21T09:39:16.653 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:39:16 smithi064 podman[100940]: 2024-09-21 09:39:16.32534339 +0000 UTC m=+0.112016107 container create d96272c732435f3c98b71727ac4e26266bf4ba099df61b3c2a277c18f31263a3 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-6-deactivate, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, RELEASE=main-62da54b, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T09:39:16.653 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:39:16 smithi064 podman[100940]: 2024-09-21 09:39:16.229710127 +0000 UTC m=+0.016382852 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:39:16.654 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:39:16 smithi064 podman[100940]: 2024-09-21 09:39:16.561165671 +0000 UTC m=+0.347838387 container init d96272c732435f3c98b71727ac4e26266bf4ba099df61b3c2a277c18f31263a3 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-6-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, RELEASE=main-62da54b, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0) 2024-09-21T09:39:16.654 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:39:16 smithi064 podman[100940]: 2024-09-21 09:39:16.566500077 +0000 UTC m=+0.353172793 container start d96272c732435f3c98b71727ac4e26266bf4ba099df61b3c2a277c18f31263a3 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-6-deactivate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, RELEASE=main-62da54b, GIT_CLEAN=True, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-09-21T09:39:16.654 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:39:16 smithi064 podman[100940]: 2024-09-21 09:39:16.65241786 +0000 UTC m=+0.439090591 container attach d96272c732435f3c98b71727ac4e26266bf4ba099df61b3c2a277c18f31263a3 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-6-deactivate, GIT_CLEAN=True, org.label-schema.build-date=20240731, ceph=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , RELEASE=main-62da54b, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS) 2024-09-21T09:39:16.941 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:39:16 smithi064 podman[100940]: 2024-09-21 09:39:16.762951321 +0000 UTC m=+0.549624036 container died d96272c732435f3c98b71727ac4e26266bf4ba099df61b3c2a277c18f31263a3 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-6-deactivate, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , RELEASE=main-62da54b, ceph=True, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T09:39:17.011 INFO:tasks.rados.rados.0.smithi022.stdout:2334: expect (ObjNum 737 snap 227 seq_num 737) 2024-09-21T09:39:17.023 INFO:tasks.rados.rados.0.smithi022.stdout:2334: done (3 left) 2024-09-21T09:39:17.023 INFO:tasks.rados.rados.0.smithi022.stdout:2335: done (2 left) 2024-09-21T09:39:17.023 INFO:tasks.rados.rados.0.smithi022.stdout:2336: done (1 left) 2024-09-21T09:39:17.023 INFO:tasks.rados.rados.0.smithi022.stdout:2337: done (0 left) 2024-09-21T09:39:17.023 INFO:tasks.rados.rados.0.smithi022.stdout:2338: snap_create 2024-09-21T09:39:17.692 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:39:17 smithi064 podman[100983]: 2024-09-21 09:39:17.38930748 +0000 UTC m=+0.615699198 container remove d96272c732435f3c98b71727ac4e26266bf4ba099df61b3c2a277c18f31263a3 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-6-deactivate, maintainer=Guillaume Abrioux , ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=main-62da54b, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0) 2024-09-21T09:39:17.692 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:39:17 smithi064 systemd[1]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490@osd.6.service: Deactivated successfully. 2024-09-21T09:39:17.692 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:39:17 smithi064 systemd[1]: Stopped Ceph osd.6 for d65e801e-77fa-11ef-baf6-efdc52797490. 2024-09-21T09:39:17.692 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:39:17 smithi064 systemd[1]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490@osd.6.service: Consumed 14.870s CPU time. 2024-09-21T09:39:17.692 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:39:17 smithi064 systemd[1]: Starting Ceph osd.6 for d65e801e-77fa-11ef-baf6-efdc52797490... 2024-09-21T09:39:17.734 INFO:teuthology.orchestra.run.smithi022.stdout:true 2024-09-21T09:39:18.009 INFO:tasks.rados.rados.0.smithi022.stdout:2338: done (0 left) 2024-09-21T09:39:18.009 INFO:tasks.rados.rados.0.smithi022.stdout:2339: snap_remove snap 228 2024-09-21T09:39:18.144 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:18 smithi064 ceph-mon[86709]: pgmap v405: 121 pgs: 2 activating+undersized, 2 peering, 14 stale+active+clean, 1 active+undersized+degraded, 102 active+clean; 150 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 3.2 MiB/s rd, 1.0 MiB/s wr, 23 op/s; 1/231 objects degraded (0.433%) 2024-09-21T09:39:18.144 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:18 smithi064 ceph-mon[86709]: osdmap e535: 8 total, 7 up, 8 in 2024-09-21T09:39:18.144 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:18 smithi064 ceph-mon[86709]: Health check failed: Degraded data redundancy: 1/231 objects degraded (0.433%), 1 pg degraded (PG_DEGRADED) 2024-09-21T09:39:18.144 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:39:17 smithi064 podman[101086]: 2024-09-21 09:39:17.815476393 +0000 UTC m=+0.104148473 container create a860599dd8a9a1e6b1a234584aeb991b408a0f2b5d611f083b771b321952bb95 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-6-activate, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, RELEASE=main-62da54b, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD) 2024-09-21T09:39:18.144 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:39:17 smithi064 podman[101086]: 2024-09-21 09:39:17.727681912 +0000 UTC m=+0.016354002 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:39:18.144 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:39:18 smithi064 podman[101086]: 2024-09-21 09:39:18.035035312 +0000 UTC m=+0.323707401 container init a860599dd8a9a1e6b1a234584aeb991b408a0f2b5d611f083b771b321952bb95 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-6-activate, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-62da54b, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0) 2024-09-21T09:39:18.145 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:39:18 smithi064 podman[101086]: 2024-09-21 09:39:18.041751226 +0000 UTC m=+0.330423324 container start a860599dd8a9a1e6b1a234584aeb991b408a0f2b5d611f083b771b321952bb95 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-6-activate, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, RELEASE=main-62da54b, ceph=True) 2024-09-21T09:39:18.145 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:39:18 smithi064 podman[101086]: 2024-09-21 09:39:18.143131498 +0000 UTC m=+0.431803594 container attach a860599dd8a9a1e6b1a234584aeb991b408a0f2b5d611f083b771b321952bb95 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-6-activate, RELEASE=main-62da54b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, ceph=True, GIT_CLEAN=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2) 2024-09-21T09:39:18.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:18 smithi022 ceph-mon[98615]: pgmap v405: 121 pgs: 2 activating+undersized, 2 peering, 14 stale+active+clean, 1 active+undersized+degraded, 102 active+clean; 150 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 3.2 MiB/s rd, 1.0 MiB/s wr, 23 op/s; 1/231 objects degraded (0.433%) 2024-09-21T09:39:18.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:18 smithi022 ceph-mon[98615]: osdmap e535: 8 total, 7 up, 8 in 2024-09-21T09:39:18.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:18 smithi022 ceph-mon[98615]: Health check failed: Degraded data redundancy: 1/231 objects degraded (0.433%), 1 pg degraded (PG_DEGRADED) 2024-09-21T09:39:18.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:18 smithi022 ceph-mon[96736]: pgmap v405: 121 pgs: 2 activating+undersized, 2 peering, 14 stale+active+clean, 1 active+undersized+degraded, 102 active+clean; 150 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 3.2 MiB/s rd, 1.0 MiB/s wr, 23 op/s; 1/231 objects degraded (0.433%) 2024-09-21T09:39:18.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:18 smithi022 ceph-mon[96736]: osdmap e535: 8 total, 7 up, 8 in 2024-09-21T09:39:18.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:18 smithi022 ceph-mon[96736]: Health check failed: Degraded data redundancy: 1/231 objects degraded (0.433%), 1 pg degraded (PG_DEGRADED) 2024-09-21T09:39:18.322 INFO:teuthology.orchestra.run.smithi022.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T09:39:18.322 INFO:teuthology.orchestra.run.smithi022.stdout:alertmanager.a smithi022 *:9093,9094 running (6m) 95s ago 10m 22.0M - 0.27.0 11f11916f8cd caced80ba544 2024-09-21T09:39:18.322 INFO:teuthology.orchestra.run.smithi022.stdout:grafana.a smithi064 *:3000 running (5m) 27s ago 10m 83.5M - 9.4.7 954c08fa6188 527bb657d70d 2024-09-21T09:39:18.322 INFO:teuthology.orchestra.run.smithi022.stdout:mds.a smithi022 running (11m) 95s ago 11m 18.1M - 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 34e4221f3a27 2024-09-21T09:39:18.323 INFO:teuthology.orchestra.run.smithi022.stdout:mds.b smithi064 running (11m) 27s ago 11m 22.7M - 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 0f846b3ec694 2024-09-21T09:39:18.323 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.x smithi064 *:8443,9283,8765 running (4m) 27s ago 15m 455M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 61abc61fea8d 2024-09-21T09:39:18.323 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.y smithi022 *:8443,9283,8765 running (5m) 95s ago 17m 517M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 6c5bee78f09b 2024-09-21T09:39:18.323 INFO:teuthology.orchestra.run.smithi022.stdout:mon.a smithi022 running (4m) 95s ago 17m 58.2M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 0513395c784d 2024-09-21T09:39:18.323 INFO:teuthology.orchestra.run.smithi022.stdout:mon.b smithi064 running (4m) 27s ago 15m 48.9M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 9dab26b6a9e7 2024-09-21T09:39:18.323 INFO:teuthology.orchestra.run.smithi022.stdout:mon.c smithi022 running (4m) 95s ago 15m 49.3M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 c74b58b67bd6 2024-09-21T09:39:18.323 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.a smithi022 *:9100 running (5m) 95s ago 10m 11.1M - 1.7.0 72c9c2088986 593c6158f524 2024-09-21T09:39:18.323 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.b smithi064 *:9100 running (5m) 27s ago 10m 11.0M - 1.7.0 72c9c2088986 a533420b0139 2024-09-21T09:39:18.323 INFO:teuthology.orchestra.run.smithi022.stdout:osd.0 smithi022 running (3m) 95s ago 15m 141M 2481M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 40ce23a2b569 2024-09-21T09:39:18.323 INFO:teuthology.orchestra.run.smithi022.stdout:osd.1 smithi022 running (2m) 95s ago 14m 115M 2481M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 7cd0c18d224c 2024-09-21T09:39:18.323 INFO:teuthology.orchestra.run.smithi022.stdout:osd.2 smithi022 running (2m) 95s ago 14m 108M 2481M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 b8c944306698 2024-09-21T09:39:18.323 INFO:teuthology.orchestra.run.smithi022.stdout:osd.3 smithi022 running (100s) 95s ago 13m 12.2M 2481M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 c90b27f61bb6 2024-09-21T09:39:18.323 INFO:teuthology.orchestra.run.smithi022.stdout:osd.4 smithi064 running (66s) 27s ago 13m 107M 2480M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 936099dbca14 2024-09-21T09:39:18.323 INFO:teuthology.orchestra.run.smithi022.stdout:osd.5 smithi064 running (32s) 27s ago 13m 12.0M 2480M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 b3225b090dba 2024-09-21T09:39:18.323 INFO:teuthology.orchestra.run.smithi022.stdout:osd.6 smithi064 running (12m) 27s ago 12m 182M 2480M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 3cc4ec816e5b 2024-09-21T09:39:18.323 INFO:teuthology.orchestra.run.smithi022.stdout:osd.7 smithi064 running (12m) 27s ago 12m 207M 2480M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 18ef0be94104 2024-09-21T09:39:18.324 INFO:teuthology.orchestra.run.smithi022.stdout:prometheus.a smithi064 *:9095 running (5m) 27s ago 10m 52.4M - 2.51.0 1d3b7f56885b 80629ffa2179 2024-09-21T09:39:18.441 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:39:18 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-6-activate[101101]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T09:39:18.441 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:39:18 smithi064 bash[101086]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T09:39:18.442 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:39:18 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-6-activate[101101]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T09:39:18.442 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:39:18 smithi064 bash[101086]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T09:39:18.678 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-09-21T09:39:18.678 INFO:teuthology.orchestra.run.smithi022.stdout: "mon": { 2024-09-21T09:39:18.678 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 3 2024-09-21T09:39:18.678 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-21T09:39:18.678 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr": { 2024-09-21T09:39:18.678 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 2 2024-09-21T09:39:18.679 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-21T09:39:18.679 INFO:teuthology.orchestra.run.smithi022.stdout: "osd": { 2024-09-21T09:39:18.679 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 1, 2024-09-21T09:39:18.679 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 6 2024-09-21T09:39:18.679 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-21T09:39:18.679 INFO:teuthology.orchestra.run.smithi022.stdout: "mds": { 2024-09-21T09:39:18.679 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 2 2024-09-21T09:39:18.679 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-21T09:39:18.679 INFO:teuthology.orchestra.run.smithi022.stdout: "overall": { 2024-09-21T09:39:18.679 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 3, 2024-09-21T09:39:18.679 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 11 2024-09-21T09:39:18.679 INFO:teuthology.orchestra.run.smithi022.stdout: } 2024-09-21T09:39:18.679 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-09-21T09:39:18.941 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:39:18 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-6-activate[101101]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-21T09:39:18.941 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:39:18 smithi064 bash[101086]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-21T09:39:18.941 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:39:18 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-6-activate[101101]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-21T09:39:18.942 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:39:18 smithi064 bash[101086]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-21T09:39:18.977 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-09-21T09:39:18.977 INFO:teuthology.orchestra.run.smithi022.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47", 2024-09-21T09:39:18.978 INFO:teuthology.orchestra.run.smithi022.stdout: "in_progress": true, 2024-09-21T09:39:18.978 INFO:teuthology.orchestra.run.smithi022.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T09:39:18.978 INFO:teuthology.orchestra.run.smithi022.stdout: "services_complete": [ 2024-09-21T09:39:18.978 INFO:teuthology.orchestra.run.smithi022.stdout: "mon", 2024-09-21T09:39:18.978 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr" 2024-09-21T09:39:18.978 INFO:teuthology.orchestra.run.smithi022.stdout: ], 2024-09-21T09:39:18.978 INFO:teuthology.orchestra.run.smithi022.stdout: "progress": "11/20 daemons upgraded", 2024-09-21T09:39:18.978 INFO:teuthology.orchestra.run.smithi022.stdout: "message": "Currently upgrading osd daemons", 2024-09-21T09:39:18.978 INFO:teuthology.orchestra.run.smithi022.stdout: "is_paused": false 2024-09-21T09:39:18.978 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-09-21T09:39:19.005 INFO:tasks.rados.rados.0.smithi022.stdout:2339: done (0 left) 2024-09-21T09:39:19.005 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 6 current snap is 230 2024-09-21T09:39:19.005 INFO:tasks.rados.rados.0.smithi022.stdout:2340: seq_num 752 ranges {0=598016} 2024-09-21T09:39:19.009 INFO:tasks.rados.rados.0.smithi022.stdout:2340: writing smithi02284960-6 from 0 to 598016 tid 1 2024-09-21T09:39:19.009 INFO:tasks.rados.rados.0.smithi022.stdout:2341: copy_from oid 20 from oid 45 current snap is 230 2024-09-21T09:39:19.009 INFO:tasks.rados.rados.0.smithi022.stdout:2342: rollback oid 39 current snap is 230 2024-09-21T09:39:19.009 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 39 to 226 2024-09-21T09:39:19.009 INFO:tasks.rados.rados.0.smithi022.stdout:2343: read oid 30 snap 226 2024-09-21T09:39:19.009 INFO:tasks.rados.rados.0.smithi022.stdout:2343: expect (ObjNum 590 snap 176 seq_num 590) 2024-09-21T09:39:19.009 INFO:tasks.rados.rados.0.smithi022.stdout:2344: snap_create 2024-09-21T09:39:19.024 INFO:tasks.rados.rados.0.smithi022.stdout:2342: finishing rollback tid 0 to smithi02284960-39 2024-09-21T09:39:19.025 INFO:tasks.rados.rados.0.smithi022.stdout:2340: finishing write tid 1 to smithi02284960-6 2024-09-21T09:39:19.025 INFO:tasks.rados.rados.0.smithi022.stdout:2343: expect (ObjNum 590 snap 176 seq_num 590) 2024-09-21T09:39:19.036 INFO:tasks.rados.rados.0.smithi022.stdout:2340: finishing write tid 2 to smithi02284960-6 2024-09-21T09:39:19.036 INFO:tasks.rados.rados.0.smithi022.stdout:2342: finishing rollback tid 1 to smithi02284960-39 2024-09-21T09:39:19.036 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 39 v 1262 (ObjNum 161 snap 0 seq_num 1) dirty dne 2024-09-21T09:39:19.036 INFO:tasks.rados.rados.0.smithi022.stdout:2340: finishing write tid 3 to smithi02284960-6 2024-09-21T09:39:19.036 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 6 v 1413 (ObjNum 752 snap 230 seq_num 752) dirty exists 2024-09-21T09:39:19.036 INFO:tasks.rados.rados.0.smithi022.stdout:2340: left oid 6 (ObjNum 752 snap 230 seq_num 752) 2024-09-21T09:39:19.046 INFO:tasks.rados.rados.0.smithi022.stdout:2341: finishing copy_from racing read to smithi02284960-20 2024-09-21T09:39:19.053 INFO:tasks.rados.rados.0.smithi022.stdout:2341: finishing copy_from to smithi02284960-20 2024-09-21T09:39:19.053 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 20 v 1278 (ObjNum 718 snap 219 seq_num 718) dirty exists 2024-09-21T09:39:19.289 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:19 smithi064 ceph-mon[86709]: from='client.34419 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:19.289 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:19 smithi064 ceph-mon[86709]: osdmap e536: 8 total, 7 up, 8 in 2024-09-21T09:39:19.289 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:19 smithi064 ceph-mon[86709]: from='client.44351 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:19.289 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:19 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/3541982761' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:39:19.289 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:19 smithi064 ceph-mon[86709]: osdmap e537: 8 total, 7 up, 8 in 2024-09-21T09:39:19.290 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:39:18 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-6-activate[101101]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-21T09:39:19.290 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:39:18 smithi064 bash[101086]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-21T09:39:19.290 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:39:18 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-6-activate[101101]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-21T09:39:19.290 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:39:18 smithi064 bash[101086]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-21T09:39:19.290 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:39:18 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-6-activate[101101]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-09-21T09:39:19.290 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:39:18 smithi064 bash[101086]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-09-21T09:39:19.290 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:39:18 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-6-activate[101101]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-21T09:39:19.290 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:39:18 smithi064 bash[101086]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-21T09:39:19.290 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:39:18 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-6-activate[101101]: --> ceph-volume raw activate successful for osd ID: 6 2024-09-21T09:39:19.291 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:39:18 smithi064 bash[101086]: --> ceph-volume raw activate successful for osd ID: 6 2024-09-21T09:39:19.291 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:39:18 smithi064 podman[101086]: 2024-09-21 09:39:18.990075437 +0000 UTC m=+1.278747520 container died a860599dd8a9a1e6b1a234584aeb991b408a0f2b5d611f083b771b321952bb95 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-6-activate, ceph=True, GIT_BRANCH=HEAD, RELEASE=main-62da54b, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T09:39:19.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:19 smithi022 ceph-mon[96736]: from='client.34419 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:19.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:19 smithi022 ceph-mon[96736]: osdmap e536: 8 total, 7 up, 8 in 2024-09-21T09:39:19.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:19 smithi022 ceph-mon[96736]: from='client.44351 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:19.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:19 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/3541982761' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:39:19.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:19 smithi022 ceph-mon[96736]: osdmap e537: 8 total, 7 up, 8 in 2024-09-21T09:39:19.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:19 smithi022 ceph-mon[98615]: from='client.34419 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:19.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:19 smithi022 ceph-mon[98615]: osdmap e536: 8 total, 7 up, 8 in 2024-09-21T09:39:19.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:19 smithi022 ceph-mon[98615]: from='client.44351 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:19.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:19 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/3541982761' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:39:19.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:19 smithi022 ceph-mon[98615]: osdmap e537: 8 total, 7 up, 8 in 2024-09-21T09:39:20.019 INFO:tasks.rados.rados.0.smithi022.stdout:2340: done (4 left) 2024-09-21T09:39:20.019 INFO:tasks.rados.rados.0.smithi022.stdout:2341: done (3 left) 2024-09-21T09:39:20.019 INFO:tasks.rados.rados.0.smithi022.stdout:2342: done (2 left) 2024-09-21T09:39:20.019 INFO:tasks.rados.rados.0.smithi022.stdout:2343: done (1 left) 2024-09-21T09:39:20.019 INFO:tasks.rados.rados.0.smithi022.stdout:2344: done (0 left) 2024-09-21T09:39:20.019 INFO:tasks.rados.rados.0.smithi022.stdout:2345: snap_remove snap 226 2024-09-21T09:39:20.023 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:39:19 smithi064 podman[101258]: 2024-09-21 09:39:19.697329424 +0000 UTC m=+0.701204682 container remove a860599dd8a9a1e6b1a234584aeb991b408a0f2b5d611f083b771b321952bb95 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-6-activate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, RELEASE=main-62da54b, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True) 2024-09-21T09:39:20.023 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:39:19 smithi064 podman[101322]: 2024-09-21 09:39:19.816816939 +0000 UTC m=+0.016652878 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:39:20.023 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:39:19 smithi064 podman[101322]: 2024-09-21 09:39:19.923202787 +0000 UTC m=+0.123038716 container create c6a9a09637e5fdf130945b06cb13437f6df561735be9d226c471a470e7009b59 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-6, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, ceph=True, RELEASE=main-62da54b, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True) 2024-09-21T09:39:20.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:20 smithi022 ceph-mon[98615]: from='client.44357 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:20.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:20 smithi022 ceph-mon[98615]: pgmap v408: 121 pgs: 30 active+undersized, 2 activating+undersized, 2 peering, 9 active+undersized+degraded, 78 active+clean; 150 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4.4 MiB/s rd, 802 KiB/s wr, 8 op/s; 21/234 objects degraded (8.974%) 2024-09-21T09:39:20.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:20 smithi022 ceph-mon[98615]: from='client.34443 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:20.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:20 smithi022 ceph-mon[98615]: osdmap e538: 8 total, 7 up, 8 in 2024-09-21T09:39:20.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:20 smithi022 ceph-mon[96736]: from='client.44357 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:20.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:20 smithi022 ceph-mon[96736]: pgmap v408: 121 pgs: 30 active+undersized, 2 activating+undersized, 2 peering, 9 active+undersized+degraded, 78 active+clean; 150 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4.4 MiB/s rd, 802 KiB/s wr, 8 op/s; 21/234 objects degraded (8.974%) 2024-09-21T09:39:20.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:20 smithi022 ceph-mon[96736]: from='client.34443 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:20.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:20 smithi022 ceph-mon[96736]: osdmap e538: 8 total, 7 up, 8 in 2024-09-21T09:39:20.441 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:39:20 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:39:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:39:20.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:20 smithi064 ceph-mon[86709]: from='client.44357 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:20.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:20 smithi064 ceph-mon[86709]: pgmap v408: 121 pgs: 30 active+undersized, 2 activating+undersized, 2 peering, 9 active+undersized+degraded, 78 active+clean; 150 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4.4 MiB/s rd, 802 KiB/s wr, 8 op/s; 21/234 objects degraded (8.974%) 2024-09-21T09:39:20.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:20 smithi064 ceph-mon[86709]: from='client.34443 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:20.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:20 smithi064 ceph-mon[86709]: osdmap e538: 8 total, 7 up, 8 in 2024-09-21T09:39:20.442 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:39:20 smithi064 podman[101322]: 2024-09-21 09:39:20.126184683 +0000 UTC m=+0.326020627 container init c6a9a09637e5fdf130945b06cb13437f6df561735be9d226c471a470e7009b59 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-6, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, RELEASE=main-62da54b, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux ) 2024-09-21T09:39:20.442 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:39:20 smithi064 podman[101322]: 2024-09-21 09:39:20.131447668 +0000 UTC m=+0.331283598 container start c6a9a09637e5fdf130945b06cb13437f6df561735be9d226c471a470e7009b59 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-6, RELEASE=main-62da54b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.build-date=20240731) 2024-09-21T09:39:20.442 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:39:20 smithi064 ceph-osd[101360]: -- 172.21.15.64:0/4097986367 <== mon.1 v2:172.21.15.64:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x56381d025a40 con 0x56381d003c00 2024-09-21T09:39:20.442 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:39:20 smithi064 bash[101322]: c6a9a09637e5fdf130945b06cb13437f6df561735be9d226c471a470e7009b59 2024-09-21T09:39:20.443 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:39:20 smithi064 systemd[1]: Started Ceph osd.6 for d65e801e-77fa-11ef-baf6-efdc52797490. 2024-09-21T09:39:21.021 INFO:tasks.rados.rados.0.smithi022.stdout:2345: done (0 left) 2024-09-21T09:39:21.021 INFO:tasks.rados.rados.0.smithi022.stdout:2346: rollback oid 32 current snap is 231 2024-09-21T09:39:21.021 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 32 to 229 2024-09-21T09:39:21.021 INFO:tasks.rados.rados.0.smithi022.stdout:2347: read oid 17 snap -1 2024-09-21T09:39:21.021 INFO:tasks.rados.rados.0.smithi022.stdout:2347: expect (ObjNum 666 snap 207 seq_num 666) 2024-09-21T09:39:21.022 INFO:tasks.rados.rados.0.smithi022.stdout:2348: read oid 5 snap -1 2024-09-21T09:39:21.022 INFO:tasks.rados.rados.0.smithi022.stdout:2348: expect (ObjNum 696 snap 215 seq_num 696) 2024-09-21T09:39:21.022 INFO:tasks.rados.rados.0.smithi022.stdout:2349: setattr oid 4 current snap is 231 2024-09-21T09:39:21.022 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 28 current snap is 231 2024-09-21T09:39:21.023 INFO:tasks.rados.rados.0.smithi022.stdout:2350: seq_num 753 ranges {1777664=614400} 2024-09-21T09:39:21.038 INFO:tasks.rados.rados.0.smithi022.stdout:2350: writing smithi02284960-28 from 1777664 to 2392064 tid 1 2024-09-21T09:39:21.038 INFO:tasks.rados.rados.0.smithi022.stdout:2351: snap_remove snap 230 2024-09-21T09:39:21.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:21 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:39:21.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:21 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:39:21.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:21 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:39:21.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:21 smithi022 ceph-mon[98615]: pgmap v411: 121 pgs: 32 active+undersized, 2 activating+undersized, 9 active+undersized+degraded, 78 active+clean; 155 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 4.6 MiB/s rd, 1.3 MiB/s wr, 5 op/s; 21/240 objects degraded (8.750%) 2024-09-21T09:39:21.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:21 smithi022 ceph-mon[98615]: osdmap e539: 8 total, 7 up, 8 in 2024-09-21T09:39:21.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:21 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:39:21.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:21 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:39:21.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:21 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:39:21.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:21 smithi022 ceph-mon[96736]: pgmap v411: 121 pgs: 32 active+undersized, 2 activating+undersized, 9 active+undersized+degraded, 78 active+clean; 155 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 4.6 MiB/s rd, 1.3 MiB/s wr, 5 op/s; 21/240 objects degraded (8.750%) 2024-09-21T09:39:21.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:21 smithi022 ceph-mon[96736]: osdmap e539: 8 total, 7 up, 8 in 2024-09-21T09:39:21.691 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:39:21 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-6[101337]: 2024-09-21T09:39:21.274+0000 7f2afb0d5640 -1 Falling back to public interface 2024-09-21T09:39:21.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:21 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:39:21.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:21 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:39:21.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:21 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:39:21.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:21 smithi064 ceph-mon[86709]: pgmap v411: 121 pgs: 32 active+undersized, 2 activating+undersized, 9 active+undersized+degraded, 78 active+clean; 155 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 4.6 MiB/s rd, 1.3 MiB/s wr, 5 op/s; 21/240 objects degraded (8.750%) 2024-09-21T09:39:21.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:21 smithi064 ceph-mon[86709]: osdmap e539: 8 total, 7 up, 8 in 2024-09-21T09:39:22.023 INFO:tasks.rados.rados.0.smithi022.stdout:2351: done (5 left) 2024-09-21T09:39:22.024 INFO:tasks.rados.rados.0.smithi022.stdout:2352: read oid 13 snap -1 2024-09-21T09:39:22.024 INFO:tasks.rados.rados.0.smithi022.stdout:2352: expect deleted 2024-09-21T09:39:22.024 INFO:tasks.rados.rados.0.smithi022.stdout:2353: delete oid 36 current snap is 231 2024-09-21T09:39:22.024 INFO:tasks.rados.rados.0.smithi022.stdout:2346: finishing rollback tid 0 to smithi02284960-32 2024-09-21T09:39:22.024 INFO:tasks.rados.rados.0.smithi022.stdout:2346: finishing rollback tid 1 to smithi02284960-32 2024-09-21T09:39:22.024 INFO:tasks.rados.rados.0.smithi022.stdout:2346: finishing rollback tid 2 to smithi02284960-32 2024-09-21T09:39:22.024 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 32 v 1421 (ObjNum 648 snap 199 seq_num 648) dirty exists 2024-09-21T09:39:22.025 INFO:tasks.rados.rados.0.smithi022.stdout:2350: finishing write tid 1 to smithi02284960-28 2024-09-21T09:39:22.025 INFO:tasks.rados.rados.0.smithi022.stdout:2350: finishing write tid 2 to smithi02284960-28 2024-09-21T09:39:22.025 INFO:tasks.rados.rados.0.smithi022.stdout:2350: finishing write tid 3 to smithi02284960-28 2024-09-21T09:39:22.025 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 28 v 1265 (ObjNum 753 snap 231 seq_num 753) dirty exists 2024-09-21T09:39:22.025 INFO:tasks.rados.rados.0.smithi022.stdout:2350: left oid 28 (ObjNum 753 snap 231 seq_num 753) 2024-09-21T09:39:22.025 INFO:tasks.rados.rados.0.smithi022.stdout:2347: expect (ObjNum 666 snap 207 seq_num 666) 2024-09-21T09:39:22.555 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 4 v 1135 (ObjNum 747 snap 227 seq_num 747) dirty exists 2024-09-21T09:39:22.555 INFO:tasks.rados.rados.0.smithi022.stdout:2348: expect (ObjNum 696 snap 215 seq_num 696) 2024-09-21T09:39:23.307 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:23 smithi022 ceph-mon[98615]: osdmap e540: 8 total, 7 up, 8 in 2024-09-21T09:39:23.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:23 smithi022 ceph-mon[96736]: osdmap e540: 8 total, 7 up, 8 in 2024-09-21T09:39:23.356 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:23 smithi064 ceph-mon[86709]: osdmap e540: 8 total, 7 up, 8 in 2024-09-21T09:39:23.427 INFO:tasks.rados.rados.0.smithi022.stdout:2346: done (6 left) 2024-09-21T09:39:23.427 INFO:tasks.rados.rados.0.smithi022.stdout:2347: done (5 left) 2024-09-21T09:39:23.427 INFO:tasks.rados.rados.0.smithi022.stdout:2348: done (4 left) 2024-09-21T09:39:23.427 INFO:tasks.rados.rados.0.smithi022.stdout:2349: done (3 left) 2024-09-21T09:39:23.427 INFO:tasks.rados.rados.0.smithi022.stdout:2350: done (2 left) 2024-09-21T09:39:23.427 INFO:tasks.rados.rados.0.smithi022.stdout:2352: done (1 left) 2024-09-21T09:39:23.427 INFO:tasks.rados.rados.0.smithi022.stdout:2353: done (0 left) 2024-09-21T09:39:23.428 INFO:tasks.rados.rados.0.smithi022.stdout:2354: read oid 6 snap 229 2024-09-21T09:39:23.428 INFO:tasks.rados.rados.0.smithi022.stdout:2354: expect deleted 2024-09-21T09:39:23.428 INFO:tasks.rados.rados.0.smithi022.stdout:2355: rollback oid 46 current snap is 231 2024-09-21T09:39:23.428 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 46 to 229 2024-09-21T09:39:23.428 INFO:tasks.rados.rados.0.smithi022.stdout:2356: setattr oid 32 current snap is 231 2024-09-21T09:39:23.428 INFO:tasks.rados.rados.0.smithi022.stdout:2354: done (2 left) 2024-09-21T09:39:23.428 INFO:tasks.rados.rados.0.smithi022.stdout:2357: read oid 48 snap -1 2024-09-21T09:39:23.428 INFO:tasks.rados.rados.0.smithi022.stdout:2357: expect (ObjNum 735 snap 224 seq_num 735) 2024-09-21T09:39:23.429 INFO:tasks.rados.rados.0.smithi022.stdout:2358: snap_create 2024-09-21T09:39:23.430 INFO:tasks.rados.rados.0.smithi022.stdout:2355: finishing rollback tid 0 to smithi02284960-46 2024-09-21T09:39:23.430 INFO:tasks.rados.rados.0.smithi022.stdout:2355: finishing rollback tid 1 to smithi02284960-46 2024-09-21T09:39:23.430 INFO:tasks.rados.rados.0.smithi022.stdout:2355: finishing rollback tid 2 to smithi02284960-46 2024-09-21T09:39:23.430 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 46 v 1393 (ObjNum 739 snap 227 seq_num 739) dirty exists 2024-09-21T09:39:23.430 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 32 v 1424 (ObjNum 648 snap 199 seq_num 648) dirty exists 2024-09-21T09:39:23.432 INFO:tasks.rados.rados.0.smithi022.stdout:2357: expect (ObjNum 735 snap 224 seq_num 735) 2024-09-21T09:39:24.007 INFO:tasks.rados.rados.0.smithi022.stdout:2355: done (3 left) 2024-09-21T09:39:24.007 INFO:tasks.rados.rados.0.smithi022.stdout:2356: done (2 left) 2024-09-21T09:39:24.007 INFO:tasks.rados.rados.0.smithi022.stdout:2357: done (1 left) 2024-09-21T09:39:24.007 INFO:tasks.rados.rados.0.smithi022.stdout:2358: done (0 left) 2024-09-21T09:39:24.007 INFO:tasks.rados.rados.0.smithi022.stdout:2359: rollback oid 44 current snap is 232 2024-09-21T09:39:24.007 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 44 to 231 2024-09-21T09:39:24.007 INFO:tasks.rados.rados.0.smithi022.stdout:2360: rollback oid 40 current snap is 232 2024-09-21T09:39:24.007 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 40 to 231 2024-09-21T09:39:24.007 INFO:tasks.rados.rados.0.smithi022.stdout:2361: snap_remove snap 229 2024-09-21T09:39:24.285 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:24 smithi064 ceph-mon[86709]: pgmap v414: 121 pgs: 34 active+undersized, 9 active+undersized+degraded, 78 active+clean; 142 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 480 KiB/s rd, 1.1 MiB/s wr, 3 op/s; 21/237 objects degraded (8.861%) 2024-09-21T09:39:24.286 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:24 smithi064 ceph-mon[86709]: Health check update: Degraded data redundancy: 21/237 objects degraded (8.861%), 9 pgs degraded (PG_DEGRADED) 2024-09-21T09:39:24.286 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:24 smithi064 ceph-mon[86709]: osdmap e541: 8 total, 7 up, 8 in 2024-09-21T09:39:24.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:24 smithi022 ceph-mon[96736]: pgmap v414: 121 pgs: 34 active+undersized, 9 active+undersized+degraded, 78 active+clean; 142 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 480 KiB/s rd, 1.1 MiB/s wr, 3 op/s; 21/237 objects degraded (8.861%) 2024-09-21T09:39:24.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:24 smithi022 ceph-mon[96736]: Health check update: Degraded data redundancy: 21/237 objects degraded (8.861%), 9 pgs degraded (PG_DEGRADED) 2024-09-21T09:39:24.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:24 smithi022 ceph-mon[96736]: osdmap e541: 8 total, 7 up, 8 in 2024-09-21T09:39:24.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:24 smithi022 ceph-mon[98615]: pgmap v414: 121 pgs: 34 active+undersized, 9 active+undersized+degraded, 78 active+clean; 142 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 480 KiB/s rd, 1.1 MiB/s wr, 3 op/s; 21/237 objects degraded (8.861%) 2024-09-21T09:39:24.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:24 smithi022 ceph-mon[98615]: Health check update: Degraded data redundancy: 21/237 objects degraded (8.861%), 9 pgs degraded (PG_DEGRADED) 2024-09-21T09:39:24.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:24 smithi022 ceph-mon[98615]: osdmap e541: 8 total, 7 up, 8 in 2024-09-21T09:39:25.011 INFO:tasks.rados.rados.0.smithi022.stdout:2361: done (2 left) 2024-09-21T09:39:25.011 INFO:tasks.rados.rados.0.smithi022.stdout:2362: rollback oid 20 current snap is 232 2024-09-21T09:39:25.011 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 20 to 231 2024-09-21T09:39:25.011 INFO:tasks.rados.rados.0.smithi022.stdout:2363: delete oid 19 current snap is 232 2024-09-21T09:39:25.011 INFO:tasks.rados.rados.0.smithi022.stdout:2359: finishing rollback tid 0 to smithi02284960-44 2024-09-21T09:39:25.011 INFO:tasks.rados.rados.0.smithi022.stdout:2359: finishing rollback tid 1 to smithi02284960-44 2024-09-21T09:39:25.011 INFO:tasks.rados.rados.0.smithi022.stdout:2360: finishing rollback tid 0 to smithi02284960-40 2024-09-21T09:39:25.011 INFO:tasks.rados.rados.0.smithi022.stdout:2360: finishing rollback tid 1 to smithi02284960-40 2024-09-21T09:39:25.012 INFO:tasks.rados.rados.0.smithi022.stdout:2359: finishing rollback tid 2 to smithi02284960-44 2024-09-21T09:39:25.012 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 44 v 1396 (ObjNum 727 snap 221 seq_num 727) dirty exists 2024-09-21T09:39:25.012 INFO:tasks.rados.rados.0.smithi022.stdout:2360: finishing rollback tid 2 to smithi02284960-40 2024-09-21T09:39:25.012 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 40 v 1271 (ObjNum 715 snap 219 seq_num 715) dirty exists 2024-09-21T09:39:25.018 INFO:tasks.rados.rados.0.smithi022.stdout:2359: done (3 left) 2024-09-21T09:39:25.018 INFO:tasks.rados.rados.0.smithi022.stdout:2360: done (2 left) 2024-09-21T09:39:25.018 INFO:tasks.rados.rados.0.smithi022.stdout:2363: done (1 left) 2024-09-21T09:39:25.018 INFO:tasks.rados.rados.0.smithi022.stdout:2364: copy_from oid 5 from oid 1 current snap is 232 2024-09-21T09:39:25.018 INFO:tasks.rados.rados.0.smithi022.stdout:2365: snap_create 2024-09-21T09:39:25.020 INFO:tasks.rados.rados.0.smithi022.stdout:2362: finishing rollback tid 0 to smithi02284960-20 2024-09-21T09:39:25.020 INFO:tasks.rados.rados.0.smithi022.stdout:2362: finishing rollback tid 1 to smithi02284960-20 2024-09-21T09:39:25.020 INFO:tasks.rados.rados.0.smithi022.stdout:2362: finishing rollback tid 2 to smithi02284960-20 2024-09-21T09:39:25.020 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 20 v 1432 (ObjNum 718 snap 219 seq_num 718) dirty exists 2024-09-21T09:39:25.033 INFO:tasks.rados.rados.0.smithi022.stdout:2364: finishing copy_from racing read to smithi02284960-5 2024-09-21T09:39:25.067 INFO:tasks.rados.rados.0.smithi022.stdout:2364: finishing copy_from to smithi02284960-5 2024-09-21T09:39:25.067 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 5 v 1385 (ObjNum 672 snap 208 seq_num 672) dirty exists 2024-09-21T09:39:25.441 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:39:24 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-6[101337]: 2024-09-21T09:39:24.981+0000 7f2afb0d5640 -1 osd.6 532 log_to_monitors true 2024-09-21T09:39:25.750 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:39:25 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-6[101337]: 2024-09-21T09:39:25.496+0000 7f2aeee77640 -1 osd.6 532 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T09:39:25.750 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:25 smithi064 ceph-mon[86709]: pgmap v416: 121 pgs: 34 active+undersized, 9 active+undersized+degraded, 78 active+clean; 142 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 439 KiB/s rd, 1.2 MiB/s wr, 4 op/s; 22/234 objects degraded (9.402%) 2024-09-21T09:39:25.750 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:25 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:39:25.750 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:25 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:39:25.750 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:25 smithi064 ceph-mon[86709]: from='osd.6 [v2:172.21.15.64:6816/1881689898,v1:172.21.15.64:6817/1881689898]' entity='osd.6' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} : dispatch 2024-09-21T09:39:25.750 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:25 smithi064 ceph-mon[86709]: from='osd.6 ' entity='osd.6' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} : dispatch 2024-09-21T09:39:25.750 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:25 smithi064 ceph-mon[86709]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-21T09:39:25.751 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:25 smithi064 ceph-mon[86709]: osdmap e542: 8 total, 7 up, 8 in 2024-09-21T09:39:25.751 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:25 smithi064 ceph-mon[86709]: from='osd.6 [v2:172.21.15.64:6816/1881689898,v1:172.21.15.64:6817/1881689898]' entity='osd.6' cmd={"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi064", "root=default"]} : dispatch 2024-09-21T09:39:25.751 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:25 smithi064 ceph-mon[86709]: from='osd.6 ' entity='osd.6' cmd={"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi064", "root=default"]} : dispatch 2024-09-21T09:39:25.751 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:25 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:39:25.751 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:25 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:39:25.751 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:25 smithi064 ceph-mon[86709]: from='osd.6 ' entity='osd.6' 2024-09-21T09:39:25.808 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:39:25 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:39:25] "GET /metrics HTTP/1.1" 200 36292 "" "Prometheus/2.51.0" 2024-09-21T09:39:25.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:25 smithi022 ceph-mon[96736]: pgmap v416: 121 pgs: 34 active+undersized, 9 active+undersized+degraded, 78 active+clean; 142 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 439 KiB/s rd, 1.2 MiB/s wr, 4 op/s; 22/234 objects degraded (9.402%) 2024-09-21T09:39:25.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:25 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:39:25.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:25 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:39:25.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:25 smithi022 ceph-mon[96736]: from='osd.6 [v2:172.21.15.64:6816/1881689898,v1:172.21.15.64:6817/1881689898]' entity='osd.6' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} : dispatch 2024-09-21T09:39:25.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:25 smithi022 ceph-mon[96736]: from='osd.6 ' entity='osd.6' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} : dispatch 2024-09-21T09:39:25.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:25 smithi022 ceph-mon[96736]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-21T09:39:25.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:25 smithi022 ceph-mon[96736]: osdmap e542: 8 total, 7 up, 8 in 2024-09-21T09:39:25.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:25 smithi022 ceph-mon[96736]: from='osd.6 [v2:172.21.15.64:6816/1881689898,v1:172.21.15.64:6817/1881689898]' entity='osd.6' cmd={"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi064", "root=default"]} : dispatch 2024-09-21T09:39:25.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:25 smithi022 ceph-mon[96736]: from='osd.6 ' entity='osd.6' cmd={"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi064", "root=default"]} : dispatch 2024-09-21T09:39:25.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:25 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:39:25.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:25 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:39:25.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:25 smithi022 ceph-mon[96736]: from='osd.6 ' entity='osd.6' 2024-09-21T09:39:25.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:25 smithi022 ceph-mon[98615]: pgmap v416: 121 pgs: 34 active+undersized, 9 active+undersized+degraded, 78 active+clean; 142 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 439 KiB/s rd, 1.2 MiB/s wr, 4 op/s; 22/234 objects degraded (9.402%) 2024-09-21T09:39:25.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:25 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:39:25.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:25 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:39:25.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:25 smithi022 ceph-mon[98615]: from='osd.6 [v2:172.21.15.64:6816/1881689898,v1:172.21.15.64:6817/1881689898]' entity='osd.6' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} : dispatch 2024-09-21T09:39:25.810 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:25 smithi022 ceph-mon[98615]: from='osd.6 ' entity='osd.6' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} : dispatch 2024-09-21T09:39:25.810 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:25 smithi022 ceph-mon[98615]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-21T09:39:25.810 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:25 smithi022 ceph-mon[98615]: osdmap e542: 8 total, 7 up, 8 in 2024-09-21T09:39:25.810 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:25 smithi022 ceph-mon[98615]: from='osd.6 [v2:172.21.15.64:6816/1881689898,v1:172.21.15.64:6817/1881689898]' entity='osd.6' cmd={"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi064", "root=default"]} : dispatch 2024-09-21T09:39:25.810 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:25 smithi022 ceph-mon[98615]: from='osd.6 ' entity='osd.6' cmd={"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi064", "root=default"]} : dispatch 2024-09-21T09:39:25.810 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:25 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:39:25.810 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:25 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:39:25.810 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:25 smithi022 ceph-mon[98615]: from='osd.6 ' entity='osd.6' 2024-09-21T09:39:26.014 INFO:tasks.rados.rados.0.smithi022.stdout:2362: done (2 left) 2024-09-21T09:39:26.014 INFO:tasks.rados.rados.0.smithi022.stdout:2364: done (1 left) 2024-09-21T09:39:26.014 INFO:tasks.rados.rados.0.smithi022.stdout:2365: done (0 left) 2024-09-21T09:39:26.014 INFO:tasks.rados.rados.0.smithi022.stdout:2366: read oid 15 snap -1 2024-09-21T09:39:26.014 INFO:tasks.rados.rados.0.smithi022.stdout:2366: expect (ObjNum 642 snap 197 seq_num 642) 2024-09-21T09:39:26.014 INFO:tasks.rados.rados.0.smithi022.stdout:2367: rmattr oid 36 current snap is 233 2024-09-21T09:39:26.014 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 45 current snap is 233 2024-09-21T09:39:26.014 INFO:tasks.rados.rados.0.smithi022.stdout:2368: seq_num 756 ranges {4218880=671744} 2024-09-21T09:39:26.043 INFO:tasks.rados.rados.0.smithi022.stdout:2368: writing smithi02284960-45 from 4218880 to 4890624 tid 1 2024-09-21T09:39:26.043 INFO:tasks.rados.rados.0.smithi022.stdout:2369: delete oid 2 current snap is 233 2024-09-21T09:39:26.043 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 36 v 0 (ObjNum 81 snap 0 seq_num 1700289381) dirty dne 2024-09-21T09:39:26.043 INFO:tasks.rados.rados.0.smithi022.stdout:2366: expect (ObjNum 642 snap 197 seq_num 642) 2024-09-21T09:39:26.191 INFO:tasks.rados.rados.0.smithi022.stdout:2368: finishing write tid 1 to smithi02284960-45 2024-09-21T09:39:26.191 INFO:tasks.rados.rados.0.smithi022.stdout:2368: finishing write tid 2 to smithi02284960-45 2024-09-21T09:39:26.191 INFO:tasks.rados.rados.0.smithi022.stdout:2368: finishing write tid 3 to smithi02284960-45 2024-09-21T09:39:26.191 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 45 v 1284 (ObjNum 756 snap 233 seq_num 756) dirty exists 2024-09-21T09:39:26.191 INFO:tasks.rados.rados.0.smithi022.stdout:2368: left oid 45 (ObjNum 756 snap 233 seq_num 756) 2024-09-21T09:39:26.191 INFO:tasks.rados.rados.0.smithi022.stdout:2366: done (3 left) 2024-09-21T09:39:26.191 INFO:tasks.rados.rados.0.smithi022.stdout:2367: done (2 left) 2024-09-21T09:39:26.191 INFO:tasks.rados.rados.0.smithi022.stdout:2368: done (1 left) 2024-09-21T09:39:26.191 INFO:tasks.rados.rados.0.smithi022.stdout:2369: done (0 left) 2024-09-21T09:39:26.192 INFO:tasks.rados.rados.0.smithi022.stdout:2370: setattr oid 40 current snap is 233 2024-09-21T09:39:26.192 INFO:tasks.rados.rados.0.smithi022.stdout:2371: read oid 50 snap -1 2024-09-21T09:39:26.192 INFO:tasks.rados.rados.0.smithi022.stdout:2371: expect (ObjNum 701 snap 216 seq_num 701) 2024-09-21T09:39:26.192 INFO:tasks.rados.rados.0.smithi022.stdout:2372: copy_from oid 6 from oid 18 current snap is 233 2024-09-21T09:39:26.192 INFO:tasks.rados.rados.0.smithi022.stdout:2373: read oid 16 snap -1 2024-09-21T09:39:26.192 INFO:tasks.rados.rados.0.smithi022.stdout:2373: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-21T09:39:26.193 INFO:tasks.rados.rados.0.smithi022.stdout:2374: rollback oid 20 current snap is 233 2024-09-21T09:39:26.193 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 20 to 232 2024-09-21T09:39:26.193 INFO:tasks.rados.rados.0.smithi022.stdout:2375: snap_create 2024-09-21T09:39:26.193 INFO:tasks.rados.rados.0.smithi022.stdout:2373: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-21T09:39:26.198 INFO:tasks.rados.rados.0.smithi022.stdout:2372: finishing copy_from to smithi02284960-6 2024-09-21T09:39:26.198 INFO:tasks.rados.rados.0.smithi022.stdout:2372: got expected ENOENT (src dne) 2024-09-21T09:39:26.198 INFO:tasks.rados.rados.0.smithi022.stdout:2372: finishing copy_from racing read to smithi02284960-6 2024-09-21T09:39:26.198 INFO:tasks.rados.rados.0.smithi022.stdout:2374: finishing rollback tid 0 to smithi02284960-20 2024-09-21T09:39:26.199 INFO:tasks.rados.rados.0.smithi022.stdout:2374: finishing rollback tid 1 to smithi02284960-20 2024-09-21T09:39:26.199 INFO:tasks.rados.rados.0.smithi022.stdout:2374: finishing rollback tid 2 to smithi02284960-20 2024-09-21T09:39:26.199 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 20 v 1436 (ObjNum 718 snap 219 seq_num 718) dirty exists 2024-09-21T09:39:26.205 INFO:tasks.rados.rados.0.smithi022.stdout:2371: expect (ObjNum 701 snap 216 seq_num 701) 2024-09-21T09:39:26.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:26 smithi064 ceph-mon[86709]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T09:39:26.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:26 smithi064 ceph-mon[86709]: osd.6 [v2:172.21.15.64:6816/1881689898,v1:172.21.15.64:6817/1881689898] boot 2024-09-21T09:39:26.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:26 smithi064 ceph-mon[86709]: osdmap e543: 8 total, 8 up, 8 in 2024-09-21T09:39:26.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:26 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-09-21T09:39:26.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:26 smithi022 ceph-mon[98615]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T09:39:26.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:26 smithi022 ceph-mon[98615]: osd.6 [v2:172.21.15.64:6816/1881689898,v1:172.21.15.64:6817/1881689898] boot 2024-09-21T09:39:26.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:26 smithi022 ceph-mon[98615]: osdmap e543: 8 total, 8 up, 8 in 2024-09-21T09:39:26.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:26 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-09-21T09:39:26.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:26 smithi022 ceph-mon[96736]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T09:39:26.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:26 smithi022 ceph-mon[96736]: osd.6 [v2:172.21.15.64:6816/1881689898,v1:172.21.15.64:6817/1881689898] boot 2024-09-21T09:39:26.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:26 smithi022 ceph-mon[96736]: osdmap e543: 8 total, 8 up, 8 in 2024-09-21T09:39:26.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:26 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-09-21T09:39:27.015 INFO:tasks.rados.rados.0.smithi022.stdout:2371: done (5 left) 2024-09-21T09:39:27.015 INFO:tasks.rados.rados.0.smithi022.stdout:2372: done (4 left) 2024-09-21T09:39:27.015 INFO:tasks.rados.rados.0.smithi022.stdout:2373: done (3 left) 2024-09-21T09:39:27.016 INFO:tasks.rados.rados.0.smithi022.stdout:2374: done (2 left) 2024-09-21T09:39:27.016 INFO:tasks.rados.rados.0.smithi022.stdout:2375: done (1 left) 2024-09-21T09:39:27.016 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 27 current snap is 234 2024-09-21T09:39:27.016 INFO:tasks.rados.rados.0.smithi022.stdout:2376: seq_num 758 ranges {4227072=704512} 2024-09-21T09:39:27.044 INFO:tasks.rados.rados.0.smithi022.stdout:2376: writing smithi02284960-27 from 4227072 to 4931584 tid 1 2024-09-21T09:39:27.044 INFO:tasks.rados.rados.0.smithi022.stdout:2377: delete oid 23 current snap is 234 2024-09-21T09:39:27.046 INFO:tasks.rados.rados.0.smithi022.stdout:2377: done (2 left) 2024-09-21T09:39:27.046 INFO:tasks.rados.rados.0.smithi022.stdout:2378: snap_remove snap 231 2024-09-21T09:39:28.025 INFO:tasks.rados.rados.0.smithi022.stdout:2378: done (2 left) 2024-09-21T09:39:28.025 INFO:tasks.rados.rados.0.smithi022.stdout:2379: read oid 21 snap 232 2024-09-21T09:39:28.025 INFO:tasks.rados.rados.0.smithi022.stdout:2379: expect (ObjNum 736 snap 226 seq_num 736) 2024-09-21T09:39:28.025 INFO:tasks.rados.rados.0.smithi022.stdout:2380: delete oid 9 current snap is 234 2024-09-21T09:39:28.025 INFO:tasks.rados.rados.0.smithi022.stdout:2376: finishing write tid 1 to smithi02284960-27 2024-09-21T09:39:28.025 INFO:tasks.rados.rados.0.smithi022.stdout:2376: finishing write tid 2 to smithi02284960-27 2024-09-21T09:39:28.025 INFO:tasks.rados.rados.0.smithi022.stdout:2376: finishing write tid 3 to smithi02284960-27 2024-09-21T09:39:28.025 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 27 v 1328 (ObjNum 758 snap 234 seq_num 758) dirty exists 2024-09-21T09:39:28.026 INFO:tasks.rados.rados.0.smithi022.stdout:2376: left oid 27 (ObjNum 758 snap 234 seq_num 758) 2024-09-21T09:39:28.026 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 40 v 1272 (ObjNum 715 snap 219 seq_num 715) dirty exists 2024-09-21T09:39:28.031 INFO:tasks.rados.rados.0.smithi022.stdout:2370: done (3 left) 2024-09-21T09:39:28.031 INFO:tasks.rados.rados.0.smithi022.stdout:2376: done (2 left) 2024-09-21T09:39:28.032 INFO:tasks.rados.rados.0.smithi022.stdout:2380: done (1 left) 2024-09-21T09:39:28.032 INFO:tasks.rados.rados.0.smithi022.stdout:2381: read oid 47 snap -1 2024-09-21T09:39:28.032 INFO:tasks.rados.rados.0.smithi022.stdout:2381: expect deleted 2024-09-21T09:39:28.032 INFO:tasks.rados.rados.0.smithi022.stdout:2382: snap_create 2024-09-21T09:39:28.053 INFO:tasks.rados.rados.0.smithi022.stdout:2379: expect (ObjNum 736 snap 226 seq_num 736) 2024-09-21T09:39:28.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:28 smithi064 ceph-mon[86709]: pgmap v419: 121 pgs: 3 peering, 32 active+undersized, 8 active+undersized+degraded, 78 active+clean; 134 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 508 KiB/s wr, 15 op/s; 21/216 objects degraded (9.722%) 2024-09-21T09:39:28.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:28 smithi064 ceph-mon[86709]: osdmap e544: 8 total, 8 up, 8 in 2024-09-21T09:39:28.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:28 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:39:28.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:28 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:39:28.307 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:28 smithi022 ceph-mon[98615]: pgmap v419: 121 pgs: 3 peering, 32 active+undersized, 8 active+undersized+degraded, 78 active+clean; 134 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 508 KiB/s wr, 15 op/s; 21/216 objects degraded (9.722%) 2024-09-21T09:39:28.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:28 smithi022 ceph-mon[98615]: osdmap e544: 8 total, 8 up, 8 in 2024-09-21T09:39:28.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:28 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:39:28.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:28 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:39:28.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:28 smithi022 ceph-mon[96736]: pgmap v419: 121 pgs: 3 peering, 32 active+undersized, 8 active+undersized+degraded, 78 active+clean; 134 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 508 KiB/s wr, 15 op/s; 21/216 objects degraded (9.722%) 2024-09-21T09:39:28.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:28 smithi022 ceph-mon[96736]: osdmap e544: 8 total, 8 up, 8 in 2024-09-21T09:39:28.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:28 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:39:28.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:28 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:39:29.024 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:39:28 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: 2024-09-21T09:39:28.772+0000 7feec1b50640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (5 PGs are or would become offline) 2024-09-21T09:39:29.025 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:29 smithi022 ceph-mon[96736]: osdmap e545: 8 total, 8 up, 8 in 2024-09-21T09:39:29.028 INFO:tasks.rados.rados.0.smithi022.stdout:2379: done (2 left) 2024-09-21T09:39:29.028 INFO:tasks.rados.rados.0.smithi022.stdout:2381: done (1 left) 2024-09-21T09:39:29.029 INFO:tasks.rados.rados.0.smithi022.stdout:2382: done (0 left) 2024-09-21T09:39:29.029 INFO:tasks.rados.rados.0.smithi022.stdout:2383: read oid 46 snap 234 2024-09-21T09:39:29.029 INFO:tasks.rados.rados.0.smithi022.stdout:2383: expect (ObjNum 739 snap 227 seq_num 739) 2024-09-21T09:39:29.029 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 32 current snap is 235 2024-09-21T09:39:29.029 INFO:tasks.rados.rados.0.smithi022.stdout:2384: seq_num 761 ranges {5799936=753664} 2024-09-21T09:39:29.064 INFO:tasks.rados.rados.0.smithi022.stdout:2384: writing smithi02284960-32 from 5799936 to 6553600 tid 1 2024-09-21T09:39:29.064 INFO:tasks.rados.rados.0.smithi022.stdout:2385: read oid 31 snap -1 2024-09-21T09:39:29.065 INFO:tasks.rados.rados.0.smithi022.stdout:2385: expect (ObjNum 751 snap 228 seq_num 751) 2024-09-21T09:39:29.065 INFO:tasks.rados.rados.0.smithi022.stdout:2386: snap_remove snap 232 2024-09-21T09:39:29.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:29 smithi022 ceph-mon[98615]: osdmap e545: 8 total, 8 up, 8 in 2024-09-21T09:39:29.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:29 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:39:29.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:29 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:39:29.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:29 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:39:29.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:29 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:39:29.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:29 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:39:29.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:29 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:39:29.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:29 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:39:29.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:29 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:39:29.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:29 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:39:29.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:29 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16} : dispatch 2024-09-21T09:39:29.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:29 smithi022 ceph-mon[98615]: Health check update: Degraded data redundancy: 21/216 objects degraded (9.722%), 8 pgs degraded (PG_DEGRADED) 2024-09-21T09:39:29.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:29 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:39:29.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:29 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:39:29.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:29 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:39:29.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:29 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:39:29.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:29 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:39:29.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:29 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:39:29.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:29 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:39:29.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:29 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:39:29.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:29 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:39:29.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:29 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16} : dispatch 2024-09-21T09:39:29.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:29 smithi022 ceph-mon[96736]: Health check update: Degraded data redundancy: 21/216 objects degraded (9.722%), 8 pgs degraded (PG_DEGRADED) 2024-09-21T09:39:29.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:29 smithi064 ceph-mon[86709]: osdmap e545: 8 total, 8 up, 8 in 2024-09-21T09:39:29.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:29 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:39:29.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:29 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:39:29.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:29 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:39:29.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:29 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:39:29.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:29 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:39:29.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:29 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:39:29.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:29 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:39:29.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:29 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:39:29.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:29 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:39:29.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:29 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16} : dispatch 2024-09-21T09:39:29.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:29 smithi064 ceph-mon[86709]: Health check update: Degraded data redundancy: 21/216 objects degraded (9.722%), 8 pgs degraded (PG_DEGRADED) 2024-09-21T09:39:30.034 INFO:tasks.rados.rados.0.smithi022.stdout:2386: done (3 left) 2024-09-21T09:39:30.034 INFO:tasks.rados.rados.0.smithi022.stdout:2387: read oid 40 snap -1 2024-09-21T09:39:30.034 INFO:tasks.rados.rados.0.smithi022.stdout:2387: expect (ObjNum 715 snap 219 seq_num 715) 2024-09-21T09:39:30.035 INFO:tasks.rados.rados.0.smithi022.stdout:2388: read oid 23 snap 233 2024-09-21T09:39:30.035 INFO:tasks.rados.rados.0.smithi022.stdout:2388: expect (ObjNum 722 snap 220 seq_num 722) 2024-09-21T09:39:30.035 INFO:tasks.rados.rados.0.smithi022.stdout:2389: snap_create 2024-09-21T09:39:30.035 INFO:tasks.rados.rados.0.smithi022.stdout:2383: expect (ObjNum 739 snap 227 seq_num 739) 2024-09-21T09:39:30.079 INFO:tasks.rados.rados.0.smithi022.stdout:2385: expect (ObjNum 751 snap 228 seq_num 751) 2024-09-21T09:39:30.109 INFO:tasks.rados.rados.0.smithi022.stdout:2384: finishing write tid 1 to smithi02284960-32 2024-09-21T09:39:30.109 INFO:tasks.rados.rados.0.smithi022.stdout:2384: finishing write tid 2 to smithi02284960-32 2024-09-21T09:39:30.109 INFO:tasks.rados.rados.0.smithi022.stdout:2384: finishing write tid 3 to smithi02284960-32 2024-09-21T09:39:30.109 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 32 v 1441 (ObjNum 761 snap 235 seq_num 761) dirty exists 2024-09-21T09:39:30.109 INFO:tasks.rados.rados.0.smithi022.stdout:2384: left oid 32 (ObjNum 761 snap 235 seq_num 761) 2024-09-21T09:39:30.109 INFO:tasks.rados.rados.0.smithi022.stdout:2388: expect (ObjNum 722 snap 220 seq_num 722) 2024-09-21T09:39:30.167 INFO:tasks.rados.rados.0.smithi022.stdout:2387: expect (ObjNum 715 snap 219 seq_num 715) 2024-09-21T09:39:30.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:30 smithi022 ceph-mon[98615]: pgmap v422: 121 pgs: 1 active+clean+snaptrim, 3 peering, 13 active+undersized, 3 active+undersized+degraded, 101 active+clean; 128 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 24 MiB/s rd, 742 KiB/s wr, 21 op/s; 9/207 objects degraded (4.348%); 1018 KiB/s, 0 objects/s recovering 2024-09-21T09:39:30.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:30 smithi022 ceph-mon[98615]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-21T09:39:30.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:30 smithi022 ceph-mon[98615]: Upgrade: unsafe to stop osd(s) at this time (5 PGs are or would become offline) 2024-09-21T09:39:30.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:30 smithi022 ceph-mon[98615]: osdmap e546: 8 total, 8 up, 8 in 2024-09-21T09:39:30.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:30 smithi022 ceph-mon[96736]: pgmap v422: 121 pgs: 1 active+clean+snaptrim, 3 peering, 13 active+undersized, 3 active+undersized+degraded, 101 active+clean; 128 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 24 MiB/s rd, 742 KiB/s wr, 21 op/s; 9/207 objects degraded (4.348%); 1018 KiB/s, 0 objects/s recovering 2024-09-21T09:39:30.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:30 smithi022 ceph-mon[96736]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-21T09:39:30.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:30 smithi022 ceph-mon[96736]: Upgrade: unsafe to stop osd(s) at this time (5 PGs are or would become offline) 2024-09-21T09:39:30.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:30 smithi022 ceph-mon[96736]: osdmap e546: 8 total, 8 up, 8 in 2024-09-21T09:39:30.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:30 smithi064 ceph-mon[86709]: pgmap v422: 121 pgs: 1 active+clean+snaptrim, 3 peering, 13 active+undersized, 3 active+undersized+degraded, 101 active+clean; 128 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 24 MiB/s rd, 742 KiB/s wr, 21 op/s; 9/207 objects degraded (4.348%); 1018 KiB/s, 0 objects/s recovering 2024-09-21T09:39:30.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:30 smithi064 ceph-mon[86709]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-21T09:39:30.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:30 smithi064 ceph-mon[86709]: Upgrade: unsafe to stop osd(s) at this time (5 PGs are or would become offline) 2024-09-21T09:39:30.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:30 smithi064 ceph-mon[86709]: osdmap e546: 8 total, 8 up, 8 in 2024-09-21T09:39:30.442 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:39:30 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-6[101337]: 2024-09-21T09:39:30.085+0000 7f2ad8630640 -1 snap_mapper.get_next_objects_to_trim::New Clone-Objects were added to Snap 141 after trimming was started 2024-09-21T09:39:30.442 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:39:30 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:39:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:39:31.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:31 smithi022 ceph-mon[96736]: osdmap e547: 8 total, 8 up, 8 in 2024-09-21T09:39:31.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:31 smithi022 ceph-mon[98615]: osdmap e547: 8 total, 8 up, 8 in 2024-09-21T09:39:31.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:31 smithi064 ceph-mon[86709]: osdmap e547: 8 total, 8 up, 8 in 2024-09-21T09:39:31.582 INFO:tasks.rados.rados.0.smithi022.stdout:2383: done (5 left) 2024-09-21T09:39:31.582 INFO:tasks.rados.rados.0.smithi022.stdout:2384: done (4 left) 2024-09-21T09:39:31.582 INFO:tasks.rados.rados.0.smithi022.stdout:2385: done (3 left) 2024-09-21T09:39:31.582 INFO:tasks.rados.rados.0.smithi022.stdout:2387: done (2 left) 2024-09-21T09:39:31.583 INFO:tasks.rados.rados.0.smithi022.stdout:2388: done (1 left) 2024-09-21T09:39:31.583 INFO:tasks.rados.rados.0.smithi022.stdout:2389: done (0 left) 2024-09-21T09:39:31.583 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 40 current snap is 236 2024-09-21T09:39:31.583 INFO:tasks.rados.rados.0.smithi022.stdout:2390: seq_num 762 ranges {16424960=770048} 2024-09-21T09:39:31.676 INFO:tasks.rados.rados.0.smithi022.stdout:2390: writing smithi02284960-40 from 16424960 to 17195008 tid 1 2024-09-21T09:39:31.676 INFO:tasks.rados.rados.0.smithi022.stdout:2391: setattr oid 23 current snap is 236 2024-09-21T09:39:31.677 INFO:tasks.rados.rados.0.smithi022.stdout:2392: delete oid 21 current snap is 236 2024-09-21T09:39:31.680 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 23 v 1069 (ObjNum 273 snap 0 seq_num 0) dirty exists 2024-09-21T09:39:31.680 INFO:tasks.rados.rados.0.smithi022.stdout:2391: done (2 left) 2024-09-21T09:39:31.680 INFO:tasks.rados.rados.0.smithi022.stdout:2392: done (1 left) 2024-09-21T09:39:31.680 INFO:tasks.rados.rados.0.smithi022.stdout:2393: read oid 18 snap -1 2024-09-21T09:39:31.680 INFO:tasks.rados.rados.0.smithi022.stdout:2393: expect deleted 2024-09-21T09:39:31.680 INFO:tasks.rados.rados.0.smithi022.stdout:2394: rollback oid 44 current snap is 236 2024-09-21T09:39:31.680 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 44 to 235 2024-09-21T09:39:31.680 INFO:tasks.rados.rados.0.smithi022.stdout:2395: setattr oid 32 current snap is 236 2024-09-21T09:39:31.681 INFO:tasks.rados.rados.0.smithi022.stdout:2393: done (3 left) 2024-09-21T09:39:31.681 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 10 current snap is 236 2024-09-21T09:39:31.681 INFO:tasks.rados.rados.0.smithi022.stdout:2396: seq_num 764 ranges {0=802816} 2024-09-21T09:39:31.687 INFO:tasks.rados.rados.0.smithi022.stdout:2396: writing smithi02284960-10 from 0 to 802816 tid 1 2024-09-21T09:39:31.687 INFO:tasks.rados.rados.0.smithi022.stdout:2397: copy_from oid 47 from oid 11 current snap is 236 2024-09-21T09:39:31.687 INFO:tasks.rados.rados.0.smithi022.stdout:2390: finishing write tid 1 to smithi02284960-40 2024-09-21T09:39:31.687 INFO:tasks.rados.rados.0.smithi022.stdout:2398: snap_remove snap 233 2024-09-21T09:39:32.044 INFO:tasks.rados.rados.0.smithi022.stdout:2398: done (5 left) 2024-09-21T09:39:32.044 INFO:tasks.rados.rados.0.smithi022.stdout:2399: snap_create 2024-09-21T09:39:32.044 INFO:tasks.rados.rados.0.smithi022.stdout:2394: finishing rollback tid 0 to smithi02284960-44 2024-09-21T09:39:32.044 INFO:tasks.rados.rados.0.smithi022.stdout:2394: finishing rollback tid 1 to smithi02284960-44 2024-09-21T09:39:32.044 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 32 v 1447 (ObjNum 761 snap 235 seq_num 761) dirty exists 2024-09-21T09:39:32.045 INFO:tasks.rados.rados.0.smithi022.stdout:2394: finishing rollback tid 2 to smithi02284960-44 2024-09-21T09:39:32.045 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 44 v 1399 (ObjNum 727 snap 221 seq_num 727) dirty exists 2024-09-21T09:39:32.045 INFO:tasks.rados.rados.0.smithi022.stdout:2390: finishing write tid 2 to smithi02284960-40 2024-09-21T09:39:32.045 INFO:tasks.rados.rados.0.smithi022.stdout:2390: finishing write tid 3 to smithi02284960-40 2024-09-21T09:39:32.045 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 40 v 1274 (ObjNum 762 snap 236 seq_num 762) dirty exists 2024-09-21T09:39:32.045 INFO:tasks.rados.rados.0.smithi022.stdout:2390: left oid 40 (ObjNum 762 snap 236 seq_num 762) 2024-09-21T09:39:32.045 INFO:tasks.rados.rados.0.smithi022.stdout:2396: finishing write tid 1 to smithi02284960-10 2024-09-21T09:39:32.045 INFO:tasks.rados.rados.0.smithi022.stdout:2396: finishing write tid 2 to smithi02284960-10 2024-09-21T09:39:32.045 INFO:tasks.rados.rados.0.smithi022.stdout:2396: finishing write tid 3 to smithi02284960-10 2024-09-21T09:39:32.045 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 10 v 1276 (ObjNum 764 snap 236 seq_num 764) dirty exists 2024-09-21T09:39:32.045 INFO:tasks.rados.rados.0.smithi022.stdout:2396: left oid 10 (ObjNum 764 snap 236 seq_num 764) 2024-09-21T09:39:32.045 INFO:tasks.rados.rados.0.smithi022.stdout:2397: finishing copy_from racing read to smithi02284960-47 2024-09-21T09:39:32.045 INFO:tasks.rados.rados.0.smithi022.stdout:2397: finishing copy_from to smithi02284960-47 2024-09-21T09:39:32.045 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 47 v 1390 (ObjNum 749 snap 227 seq_num 749) dirty exists 2024-09-21T09:39:32.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:32 smithi022 ceph-mon[96736]: pgmap v425: 121 pgs: 1 active+clean+snaptrim, 3 peering, 117 active+clean; 107 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 22 MiB/s rd, 520 KiB/s wr, 15 op/s; 8.8 MiB/s, 0 objects/s recovering 2024-09-21T09:39:32.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:32 smithi022 ceph-mon[96736]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 9/207 objects degraded (4.348%), 3 pgs degraded) 2024-09-21T09:39:32.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:32 smithi022 ceph-mon[96736]: Cluster is now healthy 2024-09-21T09:39:32.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:32 smithi022 ceph-mon[96736]: osdmap e548: 8 total, 8 up, 8 in 2024-09-21T09:39:32.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:32 smithi022 ceph-mon[98615]: pgmap v425: 121 pgs: 1 active+clean+snaptrim, 3 peering, 117 active+clean; 107 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 22 MiB/s rd, 520 KiB/s wr, 15 op/s; 8.8 MiB/s, 0 objects/s recovering 2024-09-21T09:39:32.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:32 smithi022 ceph-mon[98615]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 9/207 objects degraded (4.348%), 3 pgs degraded) 2024-09-21T09:39:32.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:32 smithi022 ceph-mon[98615]: Cluster is now healthy 2024-09-21T09:39:32.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:32 smithi022 ceph-mon[98615]: osdmap e548: 8 total, 8 up, 8 in 2024-09-21T09:39:32.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:32 smithi064 ceph-mon[86709]: pgmap v425: 121 pgs: 1 active+clean+snaptrim, 3 peering, 117 active+clean; 107 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 22 MiB/s rd, 520 KiB/s wr, 15 op/s; 8.8 MiB/s, 0 objects/s recovering 2024-09-21T09:39:32.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:32 smithi064 ceph-mon[86709]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 9/207 objects degraded (4.348%), 3 pgs degraded) 2024-09-21T09:39:32.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:32 smithi064 ceph-mon[86709]: Cluster is now healthy 2024-09-21T09:39:32.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:32 smithi064 ceph-mon[86709]: osdmap e548: 8 total, 8 up, 8 in 2024-09-21T09:39:33.051 INFO:tasks.rados.rados.0.smithi022.stdout:2390: done (5 left) 2024-09-21T09:39:33.051 INFO:tasks.rados.rados.0.smithi022.stdout:2394: done (4 left) 2024-09-21T09:39:33.051 INFO:tasks.rados.rados.0.smithi022.stdout:2395: done (3 left) 2024-09-21T09:39:33.051 INFO:tasks.rados.rados.0.smithi022.stdout:2396: done (2 left) 2024-09-21T09:39:33.051 INFO:tasks.rados.rados.0.smithi022.stdout:2397: done (1 left) 2024-09-21T09:39:33.051 INFO:tasks.rados.rados.0.smithi022.stdout:2399: done (0 left) 2024-09-21T09:39:33.051 INFO:tasks.rados.rados.0.smithi022.stdout:2400: rollback oid 29 current snap is 237 2024-09-21T09:39:33.051 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 29 to 236 2024-09-21T09:39:33.051 INFO:tasks.rados.rados.0.smithi022.stdout:2401: snap_remove snap 234 2024-09-21T09:39:33.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:33 smithi022 ceph-mon[98615]: osdmap e549: 8 total, 8 up, 8 in 2024-09-21T09:39:33.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:33 smithi022 ceph-mon[96736]: osdmap e549: 8 total, 8 up, 8 in 2024-09-21T09:39:33.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:33 smithi064 ceph-mon[86709]: osdmap e549: 8 total, 8 up, 8 in 2024-09-21T09:39:34.011 INFO:tasks.rados.rados.0.smithi022.stdout:2400: finishing rollback tid 0 to smithi02284960-29 2024-09-21T09:39:34.011 INFO:tasks.rados.rados.0.smithi022.stdout:2400: finishing rollback tid 1 to smithi02284960-29 2024-09-21T09:39:34.011 INFO:tasks.rados.rados.0.smithi022.stdout:2400: finishing rollback tid 2 to smithi02284960-29 2024-09-21T09:39:34.011 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 29 v 1072 (ObjNum 684 snap 213 seq_num 684) dirty exists 2024-09-21T09:39:34.011 INFO:tasks.rados.rados.0.smithi022.stdout:2400: done (1 left) 2024-09-21T09:39:34.011 INFO:tasks.rados.rados.0.smithi022.stdout:2401: done (0 left) 2024-09-21T09:39:34.011 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 44 current snap is 237 2024-09-21T09:39:34.011 INFO:tasks.rados.rados.0.smithi022.stdout:2402: seq_num 765 ranges {294912=417792,712704=401408} 2024-09-21T09:39:34.014 INFO:tasks.rados.rados.0.smithi022.stdout:2402: writing smithi02284960-44 from 294912 to 712704 tid 1 2024-09-21T09:39:34.017 INFO:tasks.rados.rados.0.smithi022.stdout:2402: writing smithi02284960-44 from 712704 to 1114112 tid 2 2024-09-21T09:39:34.017 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 49 current snap is 237 2024-09-21T09:39:34.017 INFO:tasks.rados.rados.0.smithi022.stdout:2403: seq_num 766 ranges {0=434176,434176=401408} 2024-09-21T09:39:34.020 INFO:tasks.rados.rados.0.smithi022.stdout:2403: writing smithi02284960-49 from 0 to 434176 tid 1 2024-09-21T09:39:34.023 INFO:tasks.rados.rados.0.smithi022.stdout:2403: writing smithi02284960-49 from 434176 to 835584 tid 2 2024-09-21T09:39:34.023 INFO:tasks.rados.rados.0.smithi022.stdout:2404: rmattr oid 12 current snap is 237 2024-09-21T09:39:34.023 INFO:tasks.rados.rados.0.smithi022.stdout:2405: rmattr oid 34 current snap is 237 2024-09-21T09:39:34.023 INFO:tasks.rados.rados.0.smithi022.stdout:2405: done (3 left) 2024-09-21T09:39:34.024 INFO:tasks.rados.rados.0.smithi022.stdout:2406: copy_from oid 32 from oid 16 current snap is 237 2024-09-21T09:39:34.024 INFO:tasks.rados.rados.0.smithi022.stdout:2407: copy_from oid 13 from oid 34 current snap is 237 2024-09-21T09:39:34.024 INFO:tasks.rados.rados.0.smithi022.stdout:2408: snap_remove snap 236 2024-09-21T09:39:34.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:34 smithi022 ceph-mon[98615]: pgmap v428: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 107 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 184 KiB/s wr, 9 op/s; 7.8 MiB/s, 0 objects/s recovering 2024-09-21T09:39:34.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:34 smithi022 ceph-mon[98615]: osdmap e550: 8 total, 8 up, 8 in 2024-09-21T09:39:34.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:34 smithi022 ceph-mon[98615]: osdmap e551: 8 total, 8 up, 8 in 2024-09-21T09:39:34.308 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 21 09:39:34 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-2[113367]: 2024-09-21T09:39:34.030+0000 7f02ddb60640 -1 snap_mapper.get_next_objects_to_trim::New Clone-Objects were added to Snap 143 after trimming was started 2024-09-21T09:39:34.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:34 smithi022 ceph-mon[96736]: pgmap v428: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 107 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 184 KiB/s wr, 9 op/s; 7.8 MiB/s, 0 objects/s recovering 2024-09-21T09:39:34.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:34 smithi022 ceph-mon[96736]: osdmap e550: 8 total, 8 up, 8 in 2024-09-21T09:39:34.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:34 smithi022 ceph-mon[96736]: osdmap e551: 8 total, 8 up, 8 in 2024-09-21T09:39:34.343 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:34 smithi064 ceph-mon[86709]: pgmap v428: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 107 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 184 KiB/s wr, 9 op/s; 7.8 MiB/s, 0 objects/s recovering 2024-09-21T09:39:34.343 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:34 smithi064 ceph-mon[86709]: osdmap e550: 8 total, 8 up, 8 in 2024-09-21T09:39:34.343 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:34 smithi064 ceph-mon[86709]: osdmap e551: 8 total, 8 up, 8 in 2024-09-21T09:39:35.013 INFO:tasks.rados.rados.0.smithi022.stdout:2407: finishing copy_from to smithi02284960-13 2024-09-21T09:39:35.013 INFO:tasks.rados.rados.0.smithi022.stdout:2407: got expected ENOENT (src dne) 2024-09-21T09:39:35.013 INFO:tasks.rados.rados.0.smithi022.stdout:2407: finishing copy_from racing read to smithi02284960-13 2024-09-21T09:39:35.013 INFO:tasks.rados.rados.0.smithi022.stdout:2406: finishing copy_from racing read to smithi02284960-32 2024-09-21T09:39:35.013 INFO:tasks.rados.rados.0.smithi022.stdout:2402: finishing write tid 1 to smithi02284960-44 2024-09-21T09:39:35.013 INFO:tasks.rados.rados.0.smithi022.stdout:2403: finishing write tid 1 to smithi02284960-49 2024-09-21T09:39:35.013 INFO:tasks.rados.rados.0.smithi022.stdout:2402: finishing write tid 2 to smithi02284960-44 2024-09-21T09:39:35.013 INFO:tasks.rados.rados.0.smithi022.stdout:2403: finishing write tid 2 to smithi02284960-49 2024-09-21T09:39:35.013 INFO:tasks.rados.rados.0.smithi022.stdout:2406: finishing copy_from to smithi02284960-32 2024-09-21T09:39:35.013 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 32 v 1065 (ObjNum 129 snap 0 seq_num 3570430136) dirty exists 2024-09-21T09:39:35.013 INFO:tasks.rados.rados.0.smithi022.stdout:2402: finishing write tid 3 to smithi02284960-44 2024-09-21T09:39:35.014 INFO:tasks.rados.rados.0.smithi022.stdout:2403: finishing write tid 3 to smithi02284960-49 2024-09-21T09:39:35.014 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 12 v 1416 (ObjNum 633 snap 189 seq_num 633) dirty exists 2024-09-21T09:39:35.014 INFO:tasks.rados.rados.0.smithi022.stdout:2402: finishing write tid 4 to smithi02284960-44 2024-09-21T09:39:35.014 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 44 v 1402 (ObjNum 765 snap 237 seq_num 765) dirty exists 2024-09-21T09:39:35.014 INFO:tasks.rados.rados.0.smithi022.stdout:2402: left oid 44 (ObjNum 765 snap 237 seq_num 765) 2024-09-21T09:39:35.014 INFO:tasks.rados.rados.0.smithi022.stdout:2403: finishing write tid 4 to smithi02284960-49 2024-09-21T09:39:35.014 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 49 v 1112 (ObjNum 766 snap 237 seq_num 766) dirty exists 2024-09-21T09:39:35.014 INFO:tasks.rados.rados.0.smithi022.stdout:2403: left oid 49 (ObjNum 766 snap 237 seq_num 766) 2024-09-21T09:39:35.014 INFO:tasks.rados.rados.0.smithi022.stdout:2402: done (5 left) 2024-09-21T09:39:35.014 INFO:tasks.rados.rados.0.smithi022.stdout:2403: done (4 left) 2024-09-21T09:39:35.014 INFO:tasks.rados.rados.0.smithi022.stdout:2404: done (3 left) 2024-09-21T09:39:35.014 INFO:tasks.rados.rados.0.smithi022.stdout:2406: done (2 left) 2024-09-21T09:39:35.014 INFO:tasks.rados.rados.0.smithi022.stdout:2407: done (1 left) 2024-09-21T09:39:35.014 INFO:tasks.rados.rados.0.smithi022.stdout:2408: done (0 left) 2024-09-21T09:39:35.015 INFO:tasks.rados.rados.0.smithi022.stdout:2409: rollback oid 17 current snap is 237 2024-09-21T09:39:35.015 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 17 to 235 2024-09-21T09:39:35.015 INFO:tasks.rados.rados.0.smithi022.stdout:2410: rollback oid 9 current snap is 237 2024-09-21T09:39:35.015 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 9 to 235 2024-09-21T09:39:35.015 INFO:tasks.rados.rados.0.smithi022.stdout:2411: read oid 48 snap -1 2024-09-21T09:39:35.015 INFO:tasks.rados.rados.0.smithi022.stdout:2411: expect (ObjNum 735 snap 224 seq_num 735) 2024-09-21T09:39:35.015 INFO:tasks.rados.rados.0.smithi022.stdout:2412: rollback oid 47 current snap is 237 2024-09-21T09:39:35.015 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 47 to 235 2024-09-21T09:39:35.015 INFO:tasks.rados.rados.0.smithi022.stdout:2413: rollback oid 40 current snap is 237 2024-09-21T09:39:35.015 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 40 to 235 2024-09-21T09:39:35.015 INFO:tasks.rados.rados.0.smithi022.stdout:2414: delete oid 24 current snap is 237 2024-09-21T09:39:35.016 INFO:tasks.rados.rados.0.smithi022.stdout:2410: finishing rollback tid 1 to smithi02284960-9 2024-09-21T09:39:35.016 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 9 v 0 (ObjNum 49 snap 0 seq_num 1953066355) dirty dne 2024-09-21T09:39:35.017 INFO:tasks.rados.rados.0.smithi022.stdout:2409: finishing rollback tid 0 to smithi02284960-17 2024-09-21T09:39:35.017 INFO:tasks.rados.rados.0.smithi022.stdout:2409: finishing rollback tid 1 to smithi02284960-17 2024-09-21T09:39:35.018 INFO:tasks.rados.rados.0.smithi022.stdout:2409: finishing rollback tid 2 to smithi02284960-17 2024-09-21T09:39:35.018 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 17 v 1265 (ObjNum 666 snap 207 seq_num 666) dirty exists 2024-09-21T09:39:35.020 INFO:tasks.rados.rados.0.smithi022.stdout:2411: expect (ObjNum 735 snap 224 seq_num 735) 2024-09-21T09:39:35.031 INFO:tasks.rados.rados.0.smithi022.stdout:2412: finishing rollback tid 0 to smithi02284960-47 2024-09-21T09:39:35.032 INFO:tasks.rados.rados.0.smithi022.stdout:2412: finishing rollback tid 1 to smithi02284960-47 2024-09-21T09:39:35.032 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 47 v 1404 (ObjNum 225 snap 0 seq_num 3412041136) dirty dne 2024-09-21T09:39:35.032 INFO:tasks.rados.rados.0.smithi022.stdout:2413: finishing rollback tid 0 to smithi02284960-40 2024-09-21T09:39:35.032 INFO:tasks.rados.rados.0.smithi022.stdout:2413: finishing rollback tid 1 to smithi02284960-40 2024-09-21T09:39:35.032 INFO:tasks.rados.rados.0.smithi022.stdout:2413: finishing rollback tid 2 to smithi02284960-40 2024-09-21T09:39:35.032 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 40 v 1279 (ObjNum 715 snap 219 seq_num 715) dirty exists 2024-09-21T09:39:35.032 INFO:tasks.rados.rados.0.smithi022.stdout:2409: done (5 left) 2024-09-21T09:39:35.032 INFO:tasks.rados.rados.0.smithi022.stdout:2410: done (4 left) 2024-09-21T09:39:35.032 INFO:tasks.rados.rados.0.smithi022.stdout:2411: done (3 left) 2024-09-21T09:39:35.032 INFO:tasks.rados.rados.0.smithi022.stdout:2412: done (2 left) 2024-09-21T09:39:35.032 INFO:tasks.rados.rados.0.smithi022.stdout:2413: done (1 left) 2024-09-21T09:39:35.032 INFO:tasks.rados.rados.0.smithi022.stdout:2414: done (0 left) 2024-09-21T09:39:35.032 INFO:tasks.rados.rados.0.smithi022.stdout:2415: rollback oid 11 current snap is 237 2024-09-21T09:39:35.032 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 11 to 235 2024-09-21T09:39:35.033 INFO:tasks.rados.rados.0.smithi022.stdout:2416: snap_create 2024-09-21T09:39:35.035 INFO:tasks.rados.rados.0.smithi022.stdout:2415: finishing rollback tid 0 to smithi02284960-11 2024-09-21T09:39:35.036 INFO:tasks.rados.rados.0.smithi022.stdout:2415: finishing rollback tid 1 to smithi02284960-11 2024-09-21T09:39:35.037 INFO:tasks.rados.rados.0.smithi022.stdout:2415: finishing rollback tid 2 to smithi02284960-11 2024-09-21T09:39:35.037 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 11 v 1407 (ObjNum 749 snap 227 seq_num 749) dirty exists 2024-09-21T09:39:36.008 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:39:35 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:39:35] "GET /metrics HTTP/1.1" 200 36143 "" "Prometheus/2.51.0" 2024-09-21T09:39:36.016 INFO:tasks.rados.rados.0.smithi022.stdout:2415: done (1 left) 2024-09-21T09:39:36.016 INFO:tasks.rados.rados.0.smithi022.stdout:2416: done (0 left) 2024-09-21T09:39:36.016 INFO:tasks.rados.rados.0.smithi022.stdout:2417: snap_create 2024-09-21T09:39:36.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:36 smithi022 ceph-mon[98615]: pgmap v431: 121 pgs: 121 active+clean; 106 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 1.6 MiB/s rd, 0 B/s wr, 7 op/s 2024-09-21T09:39:36.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:36 smithi022 ceph-mon[98615]: osdmap e552: 8 total, 8 up, 8 in 2024-09-21T09:39:36.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:36 smithi022 ceph-mon[96736]: pgmap v431: 121 pgs: 121 active+clean; 106 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 1.6 MiB/s rd, 0 B/s wr, 7 op/s 2024-09-21T09:39:36.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:36 smithi022 ceph-mon[96736]: osdmap e552: 8 total, 8 up, 8 in 2024-09-21T09:39:36.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:36 smithi064 ceph-mon[86709]: pgmap v431: 121 pgs: 121 active+clean; 106 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 1.6 MiB/s rd, 0 B/s wr, 7 op/s 2024-09-21T09:39:36.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:36 smithi064 ceph-mon[86709]: osdmap e552: 8 total, 8 up, 8 in 2024-09-21T09:39:37.021 INFO:tasks.rados.rados.0.smithi022.stdout:2417: done (0 left) 2024-09-21T09:39:37.022 INFO:tasks.rados.rados.0.smithi022.stdout:2418: snap_create 2024-09-21T09:39:37.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:37 smithi022 ceph-mon[98615]: osdmap e553: 8 total, 8 up, 8 in 2024-09-21T09:39:37.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:37 smithi022 ceph-mon[96736]: osdmap e553: 8 total, 8 up, 8 in 2024-09-21T09:39:37.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:37 smithi064 ceph-mon[86709]: osdmap e553: 8 total, 8 up, 8 in 2024-09-21T09:39:38.023 INFO:tasks.rados.rados.0.smithi022.stdout:2418: done (0 left) 2024-09-21T09:39:38.024 INFO:tasks.rados.rados.0.smithi022.stdout:2419: read oid 21 snap -1 2024-09-21T09:39:38.024 INFO:tasks.rados.rados.0.smithi022.stdout:2419: expect deleted 2024-09-21T09:39:38.024 INFO:tasks.rados.rados.0.smithi022.stdout:2420: delete oid 10 current snap is 240 2024-09-21T09:39:38.033 INFO:tasks.rados.rados.0.smithi022.stdout:2419: done (1 left) 2024-09-21T09:39:38.033 INFO:tasks.rados.rados.0.smithi022.stdout:2420: done (0 left) 2024-09-21T09:39:38.033 INFO:tasks.rados.rados.0.smithi022.stdout:2421: rollback oid 38 current snap is 240 2024-09-21T09:39:38.033 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 38 to 238 2024-09-21T09:39:38.033 INFO:tasks.rados.rados.0.smithi022.stdout:2422: snap_remove snap 237 2024-09-21T09:39:38.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:38 smithi022 ceph-mon[98615]: pgmap v434: 121 pgs: 121 active+clean; 107 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 1.9 MiB/s rd, 2.9 MiB/s wr, 19 op/s 2024-09-21T09:39:38.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:38 smithi022 ceph-mon[98615]: osdmap e554: 8 total, 8 up, 8 in 2024-09-21T09:39:38.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:38 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:39:38.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:38 smithi022 ceph-mon[96736]: pgmap v434: 121 pgs: 121 active+clean; 107 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 1.9 MiB/s rd, 2.9 MiB/s wr, 19 op/s 2024-09-21T09:39:38.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:38 smithi022 ceph-mon[96736]: osdmap e554: 8 total, 8 up, 8 in 2024-09-21T09:39:38.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:38 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:39:38.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:38 smithi064 ceph-mon[86709]: pgmap v434: 121 pgs: 121 active+clean; 107 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 1.9 MiB/s rd, 2.9 MiB/s wr, 19 op/s 2024-09-21T09:39:38.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:38 smithi064 ceph-mon[86709]: osdmap e554: 8 total, 8 up, 8 in 2024-09-21T09:39:38.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:38 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:39:39.011 INFO:tasks.rados.rados.0.smithi022.stdout:2422: done (1 left) 2024-09-21T09:39:39.011 INFO:tasks.rados.rados.0.smithi022.stdout:2423: snap_create 2024-09-21T09:39:39.011 INFO:tasks.rados.rados.0.smithi022.stdout:2421: finishing rollback tid 0 to smithi02284960-38 2024-09-21T09:39:39.011 INFO:tasks.rados.rados.0.smithi022.stdout:2421: finishing rollback tid 1 to smithi02284960-38 2024-09-21T09:39:39.011 INFO:tasks.rados.rados.0.smithi022.stdout:2421: finishing rollback tid 2 to smithi02284960-38 2024-09-21T09:39:39.011 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 38 v 1075 (ObjNum 726 snap 221 seq_num 726) dirty exists 2024-09-21T09:39:39.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:39 smithi022 ceph-mon[98615]: osdmap e555: 8 total, 8 up, 8 in 2024-09-21T09:39:39.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:39 smithi022 ceph-mon[98615]: osdmap e556: 8 total, 8 up, 8 in 2024-09-21T09:39:39.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:39 smithi022 ceph-mon[96736]: osdmap e555: 8 total, 8 up, 8 in 2024-09-21T09:39:39.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:39 smithi022 ceph-mon[96736]: osdmap e556: 8 total, 8 up, 8 in 2024-09-21T09:39:39.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:39 smithi064 ceph-mon[86709]: osdmap e555: 8 total, 8 up, 8 in 2024-09-21T09:39:39.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:39 smithi064 ceph-mon[86709]: osdmap e556: 8 total, 8 up, 8 in 2024-09-21T09:39:40.013 INFO:tasks.rados.rados.0.smithi022.stdout:2421: done (1 left) 2024-09-21T09:39:40.013 INFO:tasks.rados.rados.0.smithi022.stdout:2423: done (0 left) 2024-09-21T09:39:40.013 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 40 current snap is 241 2024-09-21T09:39:40.013 INFO:tasks.rados.rados.0.smithi022.stdout:2424: seq_num 769 ranges {16424960=483328,16908288=401408} 2024-09-21T09:39:40.105 INFO:tasks.rados.rados.0.smithi022.stdout:2424: writing smithi02284960-40 from 16424960 to 16908288 tid 1 2024-09-21T09:39:40.107 INFO:tasks.rados.rados.0.smithi022.stdout:2424: writing smithi02284960-40 from 16908288 to 17309696 tid 2 2024-09-21T09:39:40.107 INFO:tasks.rados.rados.0.smithi022.stdout:2425: setattr oid 18 current snap is 241 2024-09-21T09:39:40.108 INFO:tasks.rados.rados.0.smithi022.stdout:2426: setattr oid 7 current snap is 241 2024-09-21T09:39:40.111 INFO:tasks.rados.rados.0.smithi022.stdout:2427: rollback oid 12 current snap is 241 2024-09-21T09:39:40.111 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 12 to 235 2024-09-21T09:39:40.111 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 4 current snap is 241 2024-09-21T09:39:40.111 INFO:tasks.rados.rados.0.smithi022.stdout:2428: seq_num 770 ranges {12165120=499712,12664832=401408} 2024-09-21T09:39:40.181 INFO:tasks.rados.rados.0.smithi022.stdout:2428: writing smithi02284960-4 from 12165120 to 12664832 tid 1 2024-09-21T09:39:40.184 INFO:tasks.rados.rados.0.smithi022.stdout:2428: writing smithi02284960-4 from 12664832 to 13066240 tid 2 2024-09-21T09:39:40.184 INFO:tasks.rados.rados.0.smithi022.stdout:2429: read oid 32 snap 240 2024-09-21T09:39:40.184 INFO:tasks.rados.rados.0.smithi022.stdout:2429: expect (ObjNum -725390496 snap 22094 seq_num 3570292256) 2024-09-21T09:39:40.184 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 14 current snap is 241 2024-09-21T09:39:40.184 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 18 v 1066 (ObjNum 129 snap 0 seq_num 3570201872) dirty exists 2024-09-21T09:39:40.184 INFO:tasks.rados.rados.0.smithi022.stdout:2430: seq_num 771 ranges {245760=516096,761856=401408} 2024-09-21T09:39:40.188 INFO:tasks.rados.rados.0.smithi022.stdout:2430: writing smithi02284960-14 from 245760 to 761856 tid 1 2024-09-21T09:39:40.191 INFO:tasks.rados.rados.0.smithi022.stdout:2430: writing smithi02284960-14 from 761856 to 1163264 tid 2 2024-09-21T09:39:40.191 INFO:tasks.rados.rados.0.smithi022.stdout:2425: done (6 left) 2024-09-21T09:39:40.191 INFO:tasks.rados.rados.0.smithi022.stdout:2431: rollback oid 27 current snap is 241 2024-09-21T09:39:40.191 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 27 to 239 2024-09-21T09:39:40.191 INFO:tasks.rados.rados.0.smithi022.stdout:2424: finishing write tid 1 to smithi02284960-40 2024-09-21T09:39:40.191 INFO:tasks.rados.rados.0.smithi022.stdout:2424: finishing write tid 2 to smithi02284960-40 2024-09-21T09:39:40.192 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 7 v 1457 (ObjNum 694 snap 215 seq_num 694) dirty exists 2024-09-21T09:39:40.192 INFO:tasks.rados.rados.0.smithi022.stdout:2424: finishing write tid 3 to smithi02284960-40 2024-09-21T09:39:40.192 INFO:tasks.rados.rados.0.smithi022.stdout:2427: finishing rollback tid 0 to smithi02284960-12 2024-09-21T09:39:40.192 INFO:tasks.rados.rados.0.smithi022.stdout:2427: finishing rollback tid 1 to smithi02284960-12 2024-09-21T09:39:40.192 INFO:tasks.rados.rados.0.smithi022.stdout:2424: finishing write tid 4 to smithi02284960-40 2024-09-21T09:39:40.192 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 40 v 1283 (ObjNum 769 snap 241 seq_num 769) dirty exists 2024-09-21T09:39:40.192 INFO:tasks.rados.rados.0.smithi022.stdout:2424: left oid 40 (ObjNum 769 snap 241 seq_num 769) 2024-09-21T09:39:40.192 INFO:tasks.rados.rados.0.smithi022.stdout:2427: finishing rollback tid 2 to smithi02284960-12 2024-09-21T09:39:40.192 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 12 v 1423 (ObjNum 633 snap 189 seq_num 633) dirty exists 2024-09-21T09:39:40.192 INFO:tasks.rados.rados.0.smithi022.stdout:2429: expect (ObjNum -725390496 snap 22094 seq_num 3570292256) 2024-09-21T09:39:40.195 INFO:tasks.rados.rados.0.smithi022.stdout:2428: finishing write tid 1 to smithi02284960-4 2024-09-21T09:39:40.195 INFO:tasks.rados.rados.0.smithi022.stdout:2428: finishing write tid 2 to smithi02284960-4 2024-09-21T09:39:40.195 INFO:tasks.rados.rados.0.smithi022.stdout:2428: finishing write tid 3 to smithi02284960-4 2024-09-21T09:39:40.195 INFO:tasks.rados.rados.0.smithi022.stdout:2428: finishing write tid 4 to smithi02284960-4 2024-09-21T09:39:40.195 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 4 v 1388 (ObjNum 770 snap 241 seq_num 770) dirty exists 2024-09-21T09:39:40.195 INFO:tasks.rados.rados.0.smithi022.stdout:2428: left oid 4 (ObjNum 770 snap 241 seq_num 770) 2024-09-21T09:39:40.195 INFO:tasks.rados.rados.0.smithi022.stdout:2430: finishing write tid 1 to smithi02284960-14 2024-09-21T09:39:40.195 INFO:tasks.rados.rados.0.smithi022.stdout:2430: finishing write tid 2 to smithi02284960-14 2024-09-21T09:39:40.195 INFO:tasks.rados.rados.0.smithi022.stdout:2424: done (6 left) 2024-09-21T09:39:40.195 INFO:tasks.rados.rados.0.smithi022.stdout:2426: done (5 left) 2024-09-21T09:39:40.195 INFO:tasks.rados.rados.0.smithi022.stdout:2427: done (4 left) 2024-09-21T09:39:40.195 INFO:tasks.rados.rados.0.smithi022.stdout:2428: done (3 left) 2024-09-21T09:39:40.195 INFO:tasks.rados.rados.0.smithi022.stdout:2429: done (2 left) 2024-09-21T09:39:40.195 INFO:tasks.rados.rados.0.smithi022.stdout:2432: delete oid 43 current snap is 241 2024-09-21T09:39:40.196 INFO:tasks.rados.rados.0.smithi022.stdout:2430: finishing write tid 3 to smithi02284960-14 2024-09-21T09:39:40.197 INFO:tasks.rados.rados.0.smithi022.stdout:2430: finishing write tid 4 to smithi02284960-14 2024-09-21T09:39:40.197 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 14 v 1115 (ObjNum 771 snap 241 seq_num 771) dirty exists 2024-09-21T09:39:40.197 INFO:tasks.rados.rados.0.smithi022.stdout:2430: left oid 14 (ObjNum 771 snap 241 seq_num 771) 2024-09-21T09:39:40.197 INFO:tasks.rados.rados.0.smithi022.stdout:2431: finishing rollback tid 0 to smithi02284960-27 2024-09-21T09:39:40.197 INFO:tasks.rados.rados.0.smithi022.stdout:2431: finishing rollback tid 1 to smithi02284960-27 2024-09-21T09:39:40.197 INFO:tasks.rados.rados.0.smithi022.stdout:2430: done (2 left) 2024-09-21T09:39:40.197 INFO:tasks.rados.rados.0.smithi022.stdout:2432: done (1 left) 2024-09-21T09:39:40.197 INFO:tasks.rados.rados.0.smithi022.stdout:2433: rollback oid 25 current snap is 241 2024-09-21T09:39:40.198 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 25 to 239 2024-09-21T09:39:40.198 INFO:tasks.rados.rados.0.smithi022.stdout:2434: snap_remove snap 240 2024-09-21T09:39:40.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:40 smithi022 ceph-mon[98615]: pgmap v437: 121 pgs: 121 active+clean; 108 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 229 KiB/s rd, 3.1 MiB/s wr, 15 op/s 2024-09-21T09:39:40.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:40 smithi022 ceph-mon[98615]: osdmap e557: 8 total, 8 up, 8 in 2024-09-21T09:39:40.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:40 smithi022 ceph-mon[96736]: pgmap v437: 121 pgs: 121 active+clean; 108 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 229 KiB/s rd, 3.1 MiB/s wr, 15 op/s 2024-09-21T09:39:40.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:40 smithi022 ceph-mon[96736]: osdmap e557: 8 total, 8 up, 8 in 2024-09-21T09:39:40.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:40 smithi064 ceph-mon[86709]: pgmap v437: 121 pgs: 121 active+clean; 108 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 229 KiB/s rd, 3.1 MiB/s wr, 15 op/s 2024-09-21T09:39:40.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:40 smithi064 ceph-mon[86709]: osdmap e557: 8 total, 8 up, 8 in 2024-09-21T09:39:40.441 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:39:40 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:39:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:39:41.040 INFO:tasks.rados.rados.0.smithi022.stdout:2434: done (2 left) 2024-09-21T09:39:41.040 INFO:tasks.rados.rados.0.smithi022.stdout:2435: snap_create 2024-09-21T09:39:41.040 INFO:tasks.rados.rados.0.smithi022.stdout:2431: finishing rollback tid 2 to smithi02284960-27 2024-09-21T09:39:41.040 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 27 v 1331 (ObjNum 758 snap 234 seq_num 758) dirty exists 2024-09-21T09:39:41.040 INFO:tasks.rados.rados.0.smithi022.stdout:2433: finishing rollback tid 1 to smithi02284960-25 2024-09-21T09:39:41.040 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 25 v 0 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-09-21T09:39:42.044 INFO:tasks.rados.rados.0.smithi022.stdout:2431: done (2 left) 2024-09-21T09:39:42.044 INFO:tasks.rados.rados.0.smithi022.stdout:2433: done (1 left) 2024-09-21T09:39:42.044 INFO:tasks.rados.rados.0.smithi022.stdout:2435: done (0 left) 2024-09-21T09:39:42.044 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 27 current snap is 242 2024-09-21T09:39:42.045 INFO:tasks.rados.rados.0.smithi022.stdout:2436: seq_num 773 ranges {4931584=548864,5480448=401408} 2024-09-21T09:39:42.074 INFO:tasks.rados.rados.0.smithi022.stdout:2436: writing smithi02284960-27 from 4931584 to 5480448 tid 1 2024-09-21T09:39:42.077 INFO:tasks.rados.rados.0.smithi022.stdout:2436: writing smithi02284960-27 from 5480448 to 5881856 tid 2 2024-09-21T09:39:42.077 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 34 current snap is 242 2024-09-21T09:39:42.077 INFO:tasks.rados.rados.0.smithi022.stdout:2437: seq_num 774 ranges {0=565248,565248=401408} 2024-09-21T09:39:42.080 INFO:tasks.rados.rados.0.smithi022.stdout:2437: writing smithi02284960-34 from 0 to 565248 tid 1 2024-09-21T09:39:42.082 INFO:tasks.rados.rados.0.smithi022.stdout:2437: writing smithi02284960-34 from 565248 to 966656 tid 2 2024-09-21T09:39:42.083 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 38 current snap is 242 2024-09-21T09:39:42.083 INFO:tasks.rados.rados.0.smithi022.stdout:2436: finishing write tid 1 to smithi02284960-27 2024-09-21T09:39:42.083 INFO:tasks.rados.rados.0.smithi022.stdout:2436: finishing write tid 2 to smithi02284960-27 2024-09-21T09:39:42.083 INFO:tasks.rados.rados.0.smithi022.stdout:2436: finishing write tid 3 to smithi02284960-27 2024-09-21T09:39:42.083 INFO:tasks.rados.rados.0.smithi022.stdout:2438: seq_num 775 ranges {2334720=589824,2924544=401408} 2024-09-21T09:39:42.098 INFO:tasks.rados.rados.0.smithi022.stdout:2438: writing smithi02284960-38 from 2334720 to 2924544 tid 1 2024-09-21T09:39:42.102 INFO:tasks.rados.rados.0.smithi022.stdout:2438: writing smithi02284960-38 from 2924544 to 3325952 tid 2 2024-09-21T09:39:42.102 INFO:tasks.rados.rados.0.smithi022.stdout:2439: rollback oid 13 current snap is 242 2024-09-21T09:39:42.102 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 13 to 235 2024-09-21T09:39:42.102 INFO:tasks.rados.rados.0.smithi022.stdout:2436: finishing write tid 4 to smithi02284960-27 2024-09-21T09:39:42.102 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 27 v 1334 (ObjNum 773 snap 242 seq_num 773) dirty exists 2024-09-21T09:39:42.102 INFO:tasks.rados.rados.0.smithi022.stdout:2436: left oid 27 (ObjNum 773 snap 242 seq_num 773) 2024-09-21T09:39:42.102 INFO:tasks.rados.rados.0.smithi022.stdout:2437: finishing write tid 1 to smithi02284960-34 2024-09-21T09:39:42.102 INFO:tasks.rados.rados.0.smithi022.stdout:2437: finishing write tid 2 to smithi02284960-34 2024-09-21T09:39:42.102 INFO:tasks.rados.rados.0.smithi022.stdout:2437: finishing write tid 3 to smithi02284960-34 2024-09-21T09:39:42.103 INFO:tasks.rados.rados.0.smithi022.stdout:2437: finishing write tid 4 to smithi02284960-34 2024-09-21T09:39:42.103 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 34 v 1279 (ObjNum 774 snap 242 seq_num 774) dirty exists 2024-09-21T09:39:42.103 INFO:tasks.rados.rados.0.smithi022.stdout:2437: left oid 34 (ObjNum 774 snap 242 seq_num 774) 2024-09-21T09:39:42.103 INFO:tasks.rados.rados.0.smithi022.stdout:2436: done (3 left) 2024-09-21T09:39:42.103 INFO:tasks.rados.rados.0.smithi022.stdout:2437: done (2 left) 2024-09-21T09:39:42.103 INFO:tasks.rados.rados.0.smithi022.stdout:2440: rollback oid 15 current snap is 242 2024-09-21T09:39:42.103 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 15 to 239 2024-09-21T09:39:42.103 INFO:tasks.rados.rados.0.smithi022.stdout:2441: read oid 28 snap -1 2024-09-21T09:39:42.103 INFO:tasks.rados.rados.0.smithi022.stdout:2441: expect (ObjNum 753 snap 231 seq_num 753) 2024-09-21T09:39:42.103 INFO:tasks.rados.rados.0.smithi022.stdout:2442: delete oid 3 current snap is 242 2024-09-21T09:39:42.104 INFO:tasks.rados.rados.0.smithi022.stdout:2439: finishing rollback tid 1 to smithi02284960-13 2024-09-21T09:39:42.104 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 13 v 0 (ObjNum 33 snap 0 seq_num 2956867829) dirty dne 2024-09-21T09:39:42.105 INFO:tasks.rados.rados.0.smithi022.stdout:2439: done (4 left) 2024-09-21T09:39:42.105 INFO:tasks.rados.rados.0.smithi022.stdout:2442: done (3 left) 2024-09-21T09:39:42.105 INFO:tasks.rados.rados.0.smithi022.stdout:2443: read oid 25 snap -1 2024-09-21T09:39:42.105 INFO:tasks.rados.rados.0.smithi022.stdout:2443: expect deleted 2024-09-21T09:39:42.105 INFO:tasks.rados.rados.0.smithi022.stdout:2444: rmattr oid 36 current snap is 242 2024-09-21T09:39:42.105 INFO:tasks.rados.rados.0.smithi022.stdout:2444: done (4 left) 2024-09-21T09:39:42.105 INFO:tasks.rados.rados.0.smithi022.stdout:2445: read oid 49 snap -1 2024-09-21T09:39:42.105 INFO:tasks.rados.rados.0.smithi022.stdout:2445: expect (ObjNum 766 snap 237 seq_num 766) 2024-09-21T09:39:42.106 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 24 current snap is 242 2024-09-21T09:39:42.106 INFO:tasks.rados.rados.0.smithi022.stdout:2446: seq_num 777 ranges {0=622592,622592=401408} 2024-09-21T09:39:42.109 INFO:tasks.rados.rados.0.smithi022.stdout:2446: writing smithi02284960-24 from 0 to 622592 tid 1 2024-09-21T09:39:42.112 INFO:tasks.rados.rados.0.smithi022.stdout:2446: writing smithi02284960-24 from 622592 to 1024000 tid 2 2024-09-21T09:39:42.112 INFO:tasks.rados.rados.0.smithi022.stdout:2447: copy_from oid 17 from oid 40 current snap is 242 2024-09-21T09:39:42.112 INFO:tasks.rados.rados.0.smithi022.stdout:2438: finishing write tid 1 to smithi02284960-38 2024-09-21T09:39:42.112 INFO:tasks.rados.rados.0.smithi022.stdout:2438: finishing write tid 2 to smithi02284960-38 2024-09-21T09:39:42.112 INFO:tasks.rados.rados.0.smithi022.stdout:2443: done (6 left) 2024-09-21T09:39:42.112 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 20 current snap is 242 2024-09-21T09:39:42.112 INFO:tasks.rados.rados.0.smithi022.stdout:2438: finishing write tid 3 to smithi02284960-38 2024-09-21T09:39:42.112 INFO:tasks.rados.rados.0.smithi022.stdout:2440: finishing rollback tid 0 to smithi02284960-15 2024-09-21T09:39:42.113 INFO:tasks.rados.rados.0.smithi022.stdout:2440: finishing rollback tid 1 to smithi02284960-15 2024-09-21T09:39:42.113 INFO:tasks.rados.rados.0.smithi022.stdout:2440: finishing rollback tid 2 to smithi02284960-15 2024-09-21T09:39:42.113 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 15 v 1429 (ObjNum 642 snap 197 seq_num 642) dirty exists 2024-09-21T09:39:42.113 INFO:tasks.rados.rados.0.smithi022.stdout:2438: finishing write tid 4 to smithi02284960-38 2024-09-21T09:39:42.113 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 38 v 1078 (ObjNum 775 snap 242 seq_num 775) dirty exists 2024-09-21T09:39:42.113 INFO:tasks.rados.rados.0.smithi022.stdout:2438: left oid 38 (ObjNum 775 snap 242 seq_num 775) 2024-09-21T09:39:42.113 INFO:tasks.rados.rados.0.smithi022.stdout:2441: expect (ObjNum 753 snap 231 seq_num 753) 2024-09-21T09:39:42.215 INFO:tasks.rados.rados.0.smithi022.stdout:2445: expect (ObjNum 766 snap 237 seq_num 766) 2024-09-21T09:39:42.237 INFO:tasks.rados.rados.0.smithi022.stdout:2446: finishing write tid 1 to smithi02284960-24 2024-09-21T09:39:42.237 INFO:tasks.rados.rados.0.smithi022.stdout:2446: finishing write tid 2 to smithi02284960-24 2024-09-21T09:39:42.237 INFO:tasks.rados.rados.0.smithi022.stdout:2446: finishing write tid 3 to smithi02284960-24 2024-09-21T09:39:42.237 INFO:tasks.rados.rados.0.smithi022.stdout:2446: finishing write tid 4 to smithi02284960-24 2024-09-21T09:39:42.237 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 24 v 1432 (ObjNum 777 snap 242 seq_num 777) dirty exists 2024-09-21T09:39:42.237 INFO:tasks.rados.rados.0.smithi022.stdout:2446: left oid 24 (ObjNum 777 snap 242 seq_num 777) 2024-09-21T09:39:42.237 INFO:tasks.rados.rados.0.smithi022.stdout:2447: finishing copy_from racing read to smithi02284960-17 2024-09-21T09:39:42.238 INFO:tasks.rados.rados.0.smithi022.stdout:2447: finishing copy_from to smithi02284960-17 2024-09-21T09:39:42.238 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 17 v 1283 (ObjNum 769 snap 241 seq_num 769) dirty exists 2024-09-21T09:39:42.238 INFO:tasks.rados.rados.0.smithi022.stdout:2448: seq_num 778 ranges {4218880=638976,4857856=401408} 2024-09-21T09:39:42.263 INFO:tasks.rados.rados.0.smithi022.stdout:2448: writing smithi02284960-20 from 4218880 to 4857856 tid 1 2024-09-21T09:39:42.266 INFO:tasks.rados.rados.0.smithi022.stdout:2448: writing smithi02284960-20 from 4857856 to 5259264 tid 2 2024-09-21T09:39:42.266 INFO:tasks.rados.rados.0.smithi022.stdout:2438: done (6 left) 2024-09-21T09:39:42.266 INFO:tasks.rados.rados.0.smithi022.stdout:2440: done (5 left) 2024-09-21T09:39:42.266 INFO:tasks.rados.rados.0.smithi022.stdout:2441: done (4 left) 2024-09-21T09:39:42.267 INFO:tasks.rados.rados.0.smithi022.stdout:2445: done (3 left) 2024-09-21T09:39:42.267 INFO:tasks.rados.rados.0.smithi022.stdout:2446: done (2 left) 2024-09-21T09:39:42.267 INFO:tasks.rados.rados.0.smithi022.stdout:2447: done (1 left) 2024-09-21T09:39:42.267 INFO:tasks.rados.rados.0.smithi022.stdout:2449: read oid 25 snap -1 2024-09-21T09:39:42.267 INFO:tasks.rados.rados.0.smithi022.stdout:2449: expect deleted 2024-09-21T09:39:42.267 INFO:tasks.rados.rados.0.smithi022.stdout:2450: read oid 28 snap -1 2024-09-21T09:39:42.267 INFO:tasks.rados.rados.0.smithi022.stdout:2450: expect (ObjNum 753 snap 231 seq_num 753) 2024-09-21T09:39:42.267 INFO:tasks.rados.rados.0.smithi022.stdout:2451: setattr oid 10 current snap is 242 2024-09-21T09:39:42.268 INFO:tasks.rados.rados.0.smithi022.stdout:2449: done (3 left) 2024-09-21T09:39:42.268 INFO:tasks.rados.rados.0.smithi022.stdout:2452: rmattr oid 34 current snap is 242 2024-09-21T09:39:42.268 INFO:tasks.rados.rados.0.smithi022.stdout:2452: done (3 left) 2024-09-21T09:39:42.268 INFO:tasks.rados.rados.0.smithi022.stdout:2453: read oid 45 snap -1 2024-09-21T09:39:42.268 INFO:tasks.rados.rados.0.smithi022.stdout:2453: expect (ObjNum 756 snap 233 seq_num 756) 2024-09-21T09:39:42.268 INFO:tasks.rados.rados.0.smithi022.stdout:2454: read oid 33 snap -1 2024-09-21T09:39:42.269 INFO:tasks.rados.rados.0.smithi022.stdout:2454: expect deleted 2024-09-21T09:39:42.269 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 2 current snap is 242 2024-09-21T09:39:42.269 INFO:tasks.rados.rados.0.smithi022.stdout:2455: seq_num 779 ranges {0=655360,655360=401408} 2024-09-21T09:39:42.273 INFO:tasks.rados.rados.0.smithi022.stdout:2455: writing smithi02284960-2 from 0 to 655360 tid 1 2024-09-21T09:39:42.276 INFO:tasks.rados.rados.0.smithi022.stdout:2455: writing smithi02284960-2 from 655360 to 1056768 tid 2 2024-09-21T09:39:42.276 INFO:tasks.rados.rados.0.smithi022.stdout:2456: snap_create 2024-09-21T09:39:42.276 INFO:tasks.rados.rados.0.smithi022.stdout:2448: finishing write tid 1 to smithi02284960-20 2024-09-21T09:39:42.276 INFO:tasks.rados.rados.0.smithi022.stdout:2448: finishing write tid 2 to smithi02284960-20 2024-09-21T09:39:42.276 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 10 v 1284 (ObjNum 33 snap 0 seq_num 3569784736) dirty exists 2024-09-21T09:39:42.276 INFO:tasks.rados.rados.0.smithi022.stdout:2448: finishing write tid 3 to smithi02284960-20 2024-09-21T09:39:42.276 INFO:tasks.rados.rados.0.smithi022.stdout:2450: expect (ObjNum 753 snap 231 seq_num 753) 2024-09-21T09:39:42.307 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:42 smithi022 ceph-mon[98615]: pgmap v440: 121 pgs: 121 active+clean; 108 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 204 KiB/s wr, 13 op/s 2024-09-21T09:39:42.307 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:42 smithi022 ceph-mon[98615]: osdmap e558: 8 total, 8 up, 8 in 2024-09-21T09:39:42.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:42 smithi022 ceph-mon[96736]: pgmap v440: 121 pgs: 121 active+clean; 108 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 204 KiB/s wr, 13 op/s 2024-09-21T09:39:42.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:42 smithi022 ceph-mon[96736]: osdmap e558: 8 total, 8 up, 8 in 2024-09-21T09:39:42.372 INFO:tasks.rados.rados.0.smithi022.stdout:2448: finishing write tid 4 to smithi02284960-20 2024-09-21T09:39:42.372 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 20 v 1463 (ObjNum 778 snap 242 seq_num 778) dirty exists 2024-09-21T09:39:42.372 INFO:tasks.rados.rados.0.smithi022.stdout:2448: left oid 20 (ObjNum 778 snap 242 seq_num 778) 2024-09-21T09:39:42.372 INFO:tasks.rados.rados.0.smithi022.stdout:2455: finishing write tid 1 to smithi02284960-2 2024-09-21T09:39:42.372 INFO:tasks.rados.rados.0.smithi022.stdout:2455: finishing write tid 2 to smithi02284960-2 2024-09-21T09:39:42.372 INFO:tasks.rados.rados.0.smithi022.stdout:2455: finishing write tid 3 to smithi02284960-2 2024-09-21T09:39:42.372 INFO:tasks.rados.rados.0.smithi022.stdout:2455: finishing write tid 4 to smithi02284960-2 2024-09-21T09:39:42.372 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 2 v 1282 (ObjNum 779 snap 242 seq_num 779) dirty exists 2024-09-21T09:39:42.372 INFO:tasks.rados.rados.0.smithi022.stdout:2455: left oid 2 (ObjNum 779 snap 242 seq_num 779) 2024-09-21T09:39:42.372 INFO:tasks.rados.rados.0.smithi022.stdout:2453: expect (ObjNum 756 snap 233 seq_num 756) 2024-09-21T09:39:42.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:42 smithi064 ceph-mon[86709]: pgmap v440: 121 pgs: 121 active+clean; 108 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 204 KiB/s wr, 13 op/s 2024-09-21T09:39:42.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:42 smithi064 ceph-mon[86709]: osdmap e558: 8 total, 8 up, 8 in 2024-09-21T09:39:43.049 INFO:tasks.rados.rados.0.smithi022.stdout:2448: done (6 left) 2024-09-21T09:39:43.049 INFO:tasks.rados.rados.0.smithi022.stdout:2450: done (5 left) 2024-09-21T09:39:43.049 INFO:tasks.rados.rados.0.smithi022.stdout:2451: done (4 left) 2024-09-21T09:39:43.049 INFO:tasks.rados.rados.0.smithi022.stdout:2453: done (3 left) 2024-09-21T09:39:43.049 INFO:tasks.rados.rados.0.smithi022.stdout:2454: done (2 left) 2024-09-21T09:39:43.049 INFO:tasks.rados.rados.0.smithi022.stdout:2455: done (1 left) 2024-09-21T09:39:43.049 INFO:tasks.rados.rados.0.smithi022.stdout:2456: done (0 left) 2024-09-21T09:39:43.049 INFO:tasks.rados.rados.0.smithi022.stdout:2457: copy_from oid 27 from oid 9 current snap is 243 2024-09-21T09:39:43.050 INFO:tasks.rados.rados.0.smithi022.stdout:2458: read oid 16 snap 241 2024-09-21T09:39:43.050 INFO:tasks.rados.rados.0.smithi022.stdout:2458: expect (ObjNum -724791152 snap 22094 seq_num 3569737600) 2024-09-21T09:39:43.050 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 36 current snap is 243 2024-09-21T09:39:43.050 INFO:tasks.rados.rados.0.smithi022.stdout:2459: seq_num 780 ranges {0=671744,671744=401408} 2024-09-21T09:39:43.052 INFO:tasks.rados.rados.0.smithi022.stdout:2459: writing smithi02284960-36 from 0 to 671744 tid 1 2024-09-21T09:39:43.055 INFO:tasks.rados.rados.0.smithi022.stdout:2459: writing smithi02284960-36 from 671744 to 1073152 tid 2 2024-09-21T09:39:43.055 INFO:tasks.rados.rados.0.smithi022.stdout:2460: snap_create 2024-09-21T09:39:43.055 INFO:tasks.rados.rados.0.smithi022.stdout:2458: expect (ObjNum -724791152 snap 22094 seq_num 3569737600) 2024-09-21T09:39:43.060 INFO:tasks.rados.rados.0.smithi022.stdout:2457: finishing copy_from to smithi02284960-27 2024-09-21T09:39:43.060 INFO:tasks.rados.rados.0.smithi022.stdout:2457: got expected ENOENT (src dne) 2024-09-21T09:39:43.060 INFO:tasks.rados.rados.0.smithi022.stdout:2457: finishing copy_from racing read to smithi02284960-27 2024-09-21T09:39:43.060 INFO:tasks.rados.rados.0.smithi022.stdout:2459: finishing write tid 1 to smithi02284960-36 2024-09-21T09:39:43.060 INFO:tasks.rados.rados.0.smithi022.stdout:2459: finishing write tid 2 to smithi02284960-36 2024-09-21T09:39:43.060 INFO:tasks.rados.rados.0.smithi022.stdout:2459: finishing write tid 3 to smithi02284960-36 2024-09-21T09:39:43.060 INFO:tasks.rados.rados.0.smithi022.stdout:2459: finishing write tid 4 to smithi02284960-36 2024-09-21T09:39:43.061 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 36 v 1287 (ObjNum 780 snap 243 seq_num 780) dirty exists 2024-09-21T09:39:43.061 INFO:tasks.rados.rados.0.smithi022.stdout:2459: left oid 36 (ObjNum 780 snap 243 seq_num 780) 2024-09-21T09:39:43.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:43 smithi022 ceph-mon[98615]: osdmap e559: 8 total, 8 up, 8 in 2024-09-21T09:39:43.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:43 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:39:43.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:43 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:39:43.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:43 smithi022 ceph-mon[96736]: osdmap e559: 8 total, 8 up, 8 in 2024-09-21T09:39:43.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:43 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:39:43.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:43 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:39:43.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:43 smithi064 ceph-mon[86709]: osdmap e559: 8 total, 8 up, 8 in 2024-09-21T09:39:43.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:43 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:39:43.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:43 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:39:44.012 INFO:tasks.rados.rados.0.smithi022.stdout:2457: done (3 left) 2024-09-21T09:39:44.013 INFO:tasks.rados.rados.0.smithi022.stdout:2458: done (2 left) 2024-09-21T09:39:44.013 INFO:tasks.rados.rados.0.smithi022.stdout:2459: done (1 left) 2024-09-21T09:39:44.013 INFO:tasks.rados.rados.0.smithi022.stdout:2460: done (0 left) 2024-09-21T09:39:44.013 INFO:tasks.rados.rados.0.smithi022.stdout:2461: snap_remove snap 243 2024-09-21T09:39:44.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:44 smithi022 ceph-mon[98615]: pgmap v443: 121 pgs: 121 active+clean; 109 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 612 KiB/s rd, 224 KiB/s wr, 12 op/s 2024-09-21T09:39:44.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:44 smithi022 ceph-mon[98615]: osdmap e560: 8 total, 8 up, 8 in 2024-09-21T09:39:44.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:44 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16} : dispatch 2024-09-21T09:39:44.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:44 smithi022 ceph-mon[98615]: osdmap e561: 8 total, 8 up, 8 in 2024-09-21T09:39:44.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:44 smithi022 ceph-mon[96736]: pgmap v443: 121 pgs: 121 active+clean; 109 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 612 KiB/s rd, 224 KiB/s wr, 12 op/s 2024-09-21T09:39:44.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:44 smithi022 ceph-mon[96736]: osdmap e560: 8 total, 8 up, 8 in 2024-09-21T09:39:44.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:44 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16} : dispatch 2024-09-21T09:39:44.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:44 smithi022 ceph-mon[96736]: osdmap e561: 8 total, 8 up, 8 in 2024-09-21T09:39:44.404 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:44 smithi064 ceph-mon[86709]: pgmap v443: 121 pgs: 121 active+clean; 109 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 612 KiB/s rd, 224 KiB/s wr, 12 op/s 2024-09-21T09:39:44.404 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:44 smithi064 ceph-mon[86709]: osdmap e560: 8 total, 8 up, 8 in 2024-09-21T09:39:44.404 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:44 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16} : dispatch 2024-09-21T09:39:44.404 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:44 smithi064 ceph-mon[86709]: osdmap e561: 8 total, 8 up, 8 in 2024-09-21T09:39:45.015 INFO:tasks.rados.rados.0.smithi022.stdout:2461: done (0 left) 2024-09-21T09:39:45.015 INFO:tasks.rados.rados.0.smithi022.stdout:2462: read oid 16 snap -1 2024-09-21T09:39:45.015 INFO:tasks.rados.rados.0.smithi022.stdout:2462: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-21T09:39:45.015 INFO:tasks.rados.rados.0.smithi022.stdout:2463: rollback oid 31 current snap is 244 2024-09-21T09:39:45.015 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 31 to 242 2024-09-21T09:39:45.015 INFO:tasks.rados.rados.0.smithi022.stdout:2464: rollback oid 36 current snap is 244 2024-09-21T09:39:45.015 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 36 to 241 2024-09-21T09:39:45.015 INFO:tasks.rados.rados.0.smithi022.stdout:2465: rollback oid 28 current snap is 244 2024-09-21T09:39:45.015 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 28 to 242 2024-09-21T09:39:45.015 INFO:tasks.rados.rados.0.smithi022.stdout:2466: snap_create 2024-09-21T09:39:45.018 INFO:tasks.rados.rados.0.smithi022.stdout:2462: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-21T09:39:45.024 INFO:tasks.rados.rados.0.smithi022.stdout:2463: finishing rollback tid 0 to smithi02284960-31 2024-09-21T09:39:45.024 INFO:tasks.rados.rados.0.smithi022.stdout:2463: finishing rollback tid 1 to smithi02284960-31 2024-09-21T09:39:45.024 INFO:tasks.rados.rados.0.smithi022.stdout:2463: finishing rollback tid 2 to smithi02284960-31 2024-09-21T09:39:45.024 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 31 v 1436 (ObjNum 751 snap 228 seq_num 751) dirty exists 2024-09-21T09:39:45.024 INFO:tasks.rados.rados.0.smithi022.stdout:2464: finishing rollback tid 0 to smithi02284960-36 2024-09-21T09:39:45.024 INFO:tasks.rados.rados.0.smithi022.stdout:2464: finishing rollback tid 1 to smithi02284960-36 2024-09-21T09:39:45.024 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 36 v 1289 (ObjNum 161 snap 0 seq_num 0) dirty dne 2024-09-21T09:39:45.025 INFO:tasks.rados.rados.0.smithi022.stdout:2465: finishing rollback tid 0 to smithi02284960-28 2024-09-21T09:39:45.025 INFO:tasks.rados.rados.0.smithi022.stdout:2465: finishing rollback tid 1 to smithi02284960-28 2024-09-21T09:39:45.025 INFO:tasks.rados.rados.0.smithi022.stdout:2465: finishing rollback tid 2 to smithi02284960-28 2024-09-21T09:39:45.026 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 28 v 1268 (ObjNum 753 snap 231 seq_num 753) dirty exists 2024-09-21T09:39:45.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:45 smithi022 ceph-mon[98615]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-21T09:39:45.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:45 smithi022 ceph-mon[98615]: Upgrade: osd.7 is safe to restart 2024-09-21T09:39:45.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:45 smithi022 ceph-mon[98615]: osdmap e562: 8 total, 8 up, 8 in 2024-09-21T09:39:45.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:45 smithi022 ceph-mon[96736]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-21T09:39:45.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:45 smithi022 ceph-mon[96736]: Upgrade: osd.7 is safe to restart 2024-09-21T09:39:45.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:45 smithi022 ceph-mon[96736]: osdmap e562: 8 total, 8 up, 8 in 2024-09-21T09:39:45.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:45 smithi064 ceph-mon[86709]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-21T09:39:45.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:45 smithi064 ceph-mon[86709]: Upgrade: osd.7 is safe to restart 2024-09-21T09:39:45.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:45 smithi064 ceph-mon[86709]: osdmap e562: 8 total, 8 up, 8 in 2024-09-21T09:39:46.016 INFO:tasks.rados.rados.0.smithi022.stdout:2462: done (4 left) 2024-09-21T09:39:46.016 INFO:tasks.rados.rados.0.smithi022.stdout:2463: done (3 left) 2024-09-21T09:39:46.016 INFO:tasks.rados.rados.0.smithi022.stdout:2464: done (2 left) 2024-09-21T09:39:46.016 INFO:tasks.rados.rados.0.smithi022.stdout:2465: done (1 left) 2024-09-21T09:39:46.016 INFO:tasks.rados.rados.0.smithi022.stdout:2466: done (0 left) 2024-09-21T09:39:46.016 INFO:tasks.rados.rados.0.smithi022.stdout:2467: rollback oid 20 current snap is 245 2024-09-21T09:39:46.016 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 20 to 244 2024-09-21T09:39:46.016 INFO:tasks.rados.rados.0.smithi022.stdout:2468: copy_from oid 33 from oid 28 current snap is 245 2024-09-21T09:39:46.017 INFO:tasks.rados.rados.0.smithi022.stdout:2469: read oid 50 snap 238 2024-09-21T09:39:46.017 INFO:tasks.rados.rados.0.smithi022.stdout:2469: expect (ObjNum 701 snap 216 seq_num 701) 2024-09-21T09:39:46.017 INFO:tasks.rados.rados.0.smithi022.stdout:2470: delete oid 29 current snap is 245 2024-09-21T09:39:46.025 INFO:tasks.rados.rados.0.smithi022.stdout:2470: done (3 left) 2024-09-21T09:39:46.025 INFO:tasks.rados.rados.0.smithi022.stdout:2471: rmattr oid 15 current snap is 245 2024-09-21T09:39:46.026 INFO:tasks.rados.rados.0.smithi022.stdout:2471: done (3 left) 2024-09-21T09:39:46.026 INFO:tasks.rados.rados.0.smithi022.stdout:2472: delete oid 8 current snap is 245 2024-09-21T09:39:46.026 INFO:tasks.rados.rados.0.smithi022.stdout:2467: finishing rollback tid 0 to smithi02284960-20 2024-09-21T09:39:46.026 INFO:tasks.rados.rados.0.smithi022.stdout:2467: finishing rollback tid 1 to smithi02284960-20 2024-09-21T09:39:46.028 INFO:tasks.rados.rados.0.smithi022.stdout:2467: finishing rollback tid 2 to smithi02284960-20 2024-09-21T09:39:46.028 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 20 v 1467 (ObjNum 778 snap 242 seq_num 778) dirty exists 2024-09-21T09:39:46.029 INFO:tasks.rados.rados.0.smithi022.stdout:2467: done (3 left) 2024-09-21T09:39:46.029 INFO:tasks.rados.rados.0.smithi022.stdout:2472: done (2 left) 2024-09-21T09:39:46.029 INFO:tasks.rados.rados.0.smithi022.stdout:2473: copy_from oid 38 from oid 44 current snap is 245 2024-09-21T09:39:46.029 INFO:tasks.rados.rados.0.smithi022.stdout:2474: read oid 15 snap 241 2024-09-21T09:39:46.030 INFO:tasks.rados.rados.0.smithi022.stdout:2474: expect (ObjNum 642 snap 197 seq_num 642) 2024-09-21T09:39:46.030 INFO:tasks.rados.rados.0.smithi022.stdout:2475: read oid 35 snap -1 2024-09-21T09:39:46.030 INFO:tasks.rados.rados.0.smithi022.stdout:2475: expect deleted 2024-09-21T09:39:46.030 INFO:tasks.rados.rados.0.smithi022.stdout:2476: snap_create 2024-09-21T09:39:46.043 INFO:tasks.rados.rados.0.smithi022.stdout:2473: finishing copy_from racing read to smithi02284960-38 2024-09-21T09:39:46.043 INFO:tasks.rados.rados.0.smithi022.stdout:2468: finishing copy_from racing read to smithi02284960-33 2024-09-21T09:39:46.045 INFO:tasks.rados.rados.0.smithi022.stdout:2468: finishing copy_from to smithi02284960-33 2024-09-21T09:39:46.045 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 33 v 1268 (ObjNum 753 snap 231 seq_num 753) dirty exists 2024-09-21T09:39:46.045 INFO:tasks.rados.rados.0.smithi022.stdout:2469: expect (ObjNum 701 snap 216 seq_num 701) 2024-09-21T09:39:46.052 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:39:45 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:39:45] "GET /metrics HTTP/1.1" 200 36291 "" "Prometheus/2.51.0" 2024-09-21T09:39:46.239 INFO:tasks.rados.rados.0.smithi022.stdout:2474: expect (ObjNum 642 snap 197 seq_num 642) 2024-09-21T09:39:46.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:46 smithi022 ceph-mon[96736]: pgmap v446: 121 pgs: 121 active+clean; 113 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 7.5 MiB/s rd, 1.4 MiB/s wr, 11 op/s 2024-09-21T09:39:46.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:46 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:39:46.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:46 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-21T09:39:46.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:46 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:39:46.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:46 smithi022 ceph-mon[96736]: osdmap e563: 8 total, 8 up, 8 in 2024-09-21T09:39:46.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:46 smithi022 ceph-mon[98615]: pgmap v446: 121 pgs: 121 active+clean; 113 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 7.5 MiB/s rd, 1.4 MiB/s wr, 11 op/s 2024-09-21T09:39:46.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:46 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:39:46.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:46 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-21T09:39:46.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:46 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:39:46.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:46 smithi022 ceph-mon[98615]: osdmap e563: 8 total, 8 up, 8 in 2024-09-21T09:39:46.364 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:46 smithi064 ceph-mon[86709]: pgmap v446: 121 pgs: 121 active+clean; 113 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 7.5 MiB/s rd, 1.4 MiB/s wr, 11 op/s 2024-09-21T09:39:46.364 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:46 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:39:46.364 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:46 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-21T09:39:46.364 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:46 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:39:46.364 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:46 smithi064 ceph-mon[86709]: osdmap e563: 8 total, 8 up, 8 in 2024-09-21T09:39:46.384 INFO:tasks.rados.rados.0.smithi022.stdout:2473: finishing copy_from to smithi02284960-38 2024-09-21T09:39:46.384 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 38 v 1402 (ObjNum 765 snap 237 seq_num 765) dirty exists 2024-09-21T09:39:47.017 INFO:tasks.rados.rados.0.smithi022.stdout:2468: done (5 left) 2024-09-21T09:39:47.018 INFO:tasks.rados.rados.0.smithi022.stdout:2469: done (4 left) 2024-09-21T09:39:47.018 INFO:tasks.rados.rados.0.smithi022.stdout:2473: done (3 left) 2024-09-21T09:39:47.018 INFO:tasks.rados.rados.0.smithi022.stdout:2474: done (2 left) 2024-09-21T09:39:47.018 INFO:tasks.rados.rados.0.smithi022.stdout:2475: done (1 left) 2024-09-21T09:39:47.018 INFO:tasks.rados.rados.0.smithi022.stdout:2476: done (0 left) 2024-09-21T09:39:47.018 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 24 current snap is 246 2024-09-21T09:39:47.018 INFO:tasks.rados.rados.0.smithi022.stdout:2477: seq_num 783 ranges {1024000=720896,1744896=401408} 2024-09-21T09:39:47.027 INFO:tasks.rados.rados.0.smithi022.stdout:2477: writing smithi02284960-24 from 1024000 to 1744896 tid 1 2024-09-21T09:39:47.029 INFO:tasks.rados.rados.0.smithi022.stdout:2477: writing smithi02284960-24 from 1744896 to 2146304 tid 2 2024-09-21T09:39:47.029 INFO:tasks.rados.rados.0.smithi022.stdout:2478: delete oid 9 current snap is 246 2024-09-21T09:39:47.034 INFO:tasks.rados.rados.0.smithi022.stdout:2478: done (1 left) 2024-09-21T09:39:47.034 INFO:tasks.rados.rados.0.smithi022.stdout:2479: read oid 27 snap -1 2024-09-21T09:39:47.034 INFO:tasks.rados.rados.0.smithi022.stdout:2479: expect (ObjNum 773 snap 242 seq_num 773) 2024-09-21T09:39:47.034 INFO:tasks.rados.rados.0.smithi022.stdout:2480: read oid 18 snap -1 2024-09-21T09:39:47.034 INFO:tasks.rados.rados.0.smithi022.stdout:2480: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-21T09:39:47.034 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 7 current snap is 246 2024-09-21T09:39:47.034 INFO:tasks.rados.rados.0.smithi022.stdout:2481: seq_num 785 ranges {2031616=753664,2785280=401408} 2024-09-21T09:39:47.049 INFO:tasks.rados.rados.0.smithi022.stdout:2481: writing smithi02284960-7 from 2031616 to 2785280 tid 1 2024-09-21T09:39:47.051 INFO:tasks.rados.rados.0.smithi022.stdout:2481: writing smithi02284960-7 from 2785280 to 3186688 tid 2 2024-09-21T09:39:47.051 INFO:tasks.rados.rados.0.smithi022.stdout:2482: read oid 15 snap -1 2024-09-21T09:39:47.051 INFO:tasks.rados.rados.0.smithi022.stdout:2482: expect (ObjNum 642 snap 197 seq_num 642) 2024-09-21T09:39:47.051 INFO:tasks.rados.rados.0.smithi022.stdout:2483: delete oid 2 current snap is 246 2024-09-21T09:39:47.052 INFO:tasks.rados.rados.0.smithi022.stdout:2480: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-21T09:39:47.057 INFO:tasks.rados.rados.0.smithi022.stdout:2477: finishing write tid 1 to smithi02284960-24 2024-09-21T09:39:47.067 INFO:tasks.rados.rados.0.smithi022.stdout:2477: finishing write tid 2 to smithi02284960-24 2024-09-21T09:39:47.067 INFO:tasks.rados.rados.0.smithi022.stdout:2477: finishing write tid 3 to smithi02284960-24 2024-09-21T09:39:47.067 INFO:tasks.rados.rados.0.smithi022.stdout:2477: finishing write tid 4 to smithi02284960-24 2024-09-21T09:39:47.067 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 24 v 1440 (ObjNum 783 snap 246 seq_num 783) dirty exists 2024-09-21T09:39:47.067 INFO:tasks.rados.rados.0.smithi022.stdout:2477: left oid 24 (ObjNum 783 snap 246 seq_num 783) 2024-09-21T09:39:47.068 INFO:tasks.rados.rados.0.smithi022.stdout:2477: done (5 left) 2024-09-21T09:39:47.068 INFO:tasks.rados.rados.0.smithi022.stdout:2480: done (4 left) 2024-09-21T09:39:47.068 INFO:tasks.rados.rados.0.smithi022.stdout:2483: done (3 left) 2024-09-21T09:39:47.068 INFO:tasks.rados.rados.0.smithi022.stdout:2484: rmattr oid 8 current snap is 246 2024-09-21T09:39:47.068 INFO:tasks.rados.rados.0.smithi022.stdout:2484: done (3 left) 2024-09-21T09:39:47.068 INFO:tasks.rados.rados.0.smithi022.stdout:2485: snap_remove snap 241 2024-09-21T09:39:47.235 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:47 smithi064 ceph-mon[86709]: Upgrade: Updating osd.7 2024-09-21T09:39:47.236 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:47 smithi064 ceph-mon[86709]: Deploying daemon osd.7 on smithi064 2024-09-21T09:39:47.236 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:47 smithi064 ceph-mon[86709]: osdmap e564: 8 total, 8 up, 8 in 2024-09-21T09:39:47.557 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:47 smithi022 ceph-mon[98615]: Upgrade: Updating osd.7 2024-09-21T09:39:47.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:47 smithi022 ceph-mon[98615]: Deploying daemon osd.7 on smithi064 2024-09-21T09:39:47.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:47 smithi022 ceph-mon[98615]: osdmap e564: 8 total, 8 up, 8 in 2024-09-21T09:39:47.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:47 smithi022 ceph-mon[96736]: Upgrade: Updating osd.7 2024-09-21T09:39:47.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:47 smithi022 ceph-mon[96736]: Deploying daemon osd.7 on smithi064 2024-09-21T09:39:47.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:47 smithi022 ceph-mon[96736]: osdmap e564: 8 total, 8 up, 8 in 2024-09-21T09:39:48.071 INFO:tasks.rados.rados.0.smithi022.stdout:2481: finishing write tid 1 to smithi02284960-7 2024-09-21T09:39:48.071 INFO:tasks.rados.rados.0.smithi022.stdout:2479: expect (ObjNum 773 snap 242 seq_num 773) 2024-09-21T09:39:48.346 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:48 smithi064 ceph-mon[86709]: pgmap v449: 121 pgs: 121 active+clean; 133 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 7.0 MiB/s rd, 6.0 MiB/s wr, 21 op/s 2024-09-21T09:39:48.526 INFO:tasks.rados.rados.0.smithi022.stdout:2481: finishing write tid 2 to smithi02284960-7 2024-09-21T09:39:48.526 INFO:tasks.rados.rados.0.smithi022.stdout:2481: finishing write tid 3 to smithi02284960-7 2024-09-21T09:39:48.526 INFO:tasks.rados.rados.0.smithi022.stdout:2481: finishing write tid 4 to smithi02284960-7 2024-09-21T09:39:48.526 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 7 v 1472 (ObjNum 785 snap 246 seq_num 785) dirty exists 2024-09-21T09:39:48.526 INFO:tasks.rados.rados.0.smithi022.stdout:2481: left oid 7 (ObjNum 785 snap 246 seq_num 785) 2024-09-21T09:39:48.527 INFO:tasks.rados.rados.0.smithi022.stdout:2482: expect (ObjNum 642 snap 197 seq_num 642) 2024-09-21T09:39:48.557 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:48 smithi022 ceph-mon[98615]: pgmap v449: 121 pgs: 121 active+clean; 133 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 7.0 MiB/s rd, 6.0 MiB/s wr, 21 op/s 2024-09-21T09:39:48.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:48 smithi022 ceph-mon[96736]: pgmap v449: 121 pgs: 121 active+clean; 133 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 7.0 MiB/s rd, 6.0 MiB/s wr, 21 op/s 2024-09-21T09:39:48.599 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:39:48 smithi064 systemd[1]: Stopping Ceph osd.7 for d65e801e-77fa-11ef-baf6-efdc52797490... 2024-09-21T09:39:48.671 INFO:tasks.rados.rados.0.smithi022.stdout:2479: done (3 left) 2024-09-21T09:39:48.671 INFO:tasks.rados.rados.0.smithi022.stdout:2481: done (2 left) 2024-09-21T09:39:48.672 INFO:tasks.rados.rados.0.smithi022.stdout:2482: done (1 left) 2024-09-21T09:39:48.672 INFO:tasks.rados.rados.0.smithi022.stdout:2485: done (0 left) 2024-09-21T09:39:48.672 INFO:tasks.rados.rados.0.smithi022.stdout:2486: snap_create 2024-09-21T09:39:48.941 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:39:48 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7[55468]: 2024-09-21T09:39:48.596+0000 7f2d92950640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T09:39:48.941 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:39:48 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7[55468]: 2024-09-21T09:39:48.596+0000 7f2d92950640 -1 osd.7 565 *** Got signal Terminated *** 2024-09-21T09:39:48.941 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:39:48 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7[55468]: 2024-09-21T09:39:48.596+0000 7f2d92950640 -1 osd.7 565 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T09:39:49.015 INFO:tasks.rados.rados.0.smithi022.stdout:2486: done (0 left) 2024-09-21T09:39:49.015 INFO:tasks.rados.rados.0.smithi022.stdout:2487: rollback oid 30 current snap is 247 2024-09-21T09:39:49.015 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 30 to 238 2024-09-21T09:39:49.015 INFO:tasks.rados.rados.0.smithi022.stdout:2488: rollback oid 21 current snap is 247 2024-09-21T09:39:49.016 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 21 to 235 2024-09-21T09:39:49.016 INFO:tasks.rados.rados.0.smithi022.stdout:2489: delete oid 10 current snap is 247 2024-09-21T09:39:49.028 INFO:tasks.rados.rados.0.smithi022.stdout:2488: finishing rollback tid 1 to smithi02284960-21 2024-09-21T09:39:49.028 INFO:tasks.rados.rados.0.smithi022.stdout:2487: finishing rollback tid 0 to smithi02284960-30 2024-09-21T09:39:49.028 INFO:tasks.rados.rados.0.smithi022.stdout:2488: finishing rollback tid 2 to smithi02284960-21 2024-09-21T09:39:49.028 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 21 v 1285 (ObjNum 736 snap 226 seq_num 736) dirty exists 2024-09-21T09:39:49.028 INFO:tasks.rados.rados.0.smithi022.stdout:2487: finishing rollback tid 1 to smithi02284960-30 2024-09-21T09:39:49.028 INFO:tasks.rados.rados.0.smithi022.stdout:2487: finishing rollback tid 2 to smithi02284960-30 2024-09-21T09:39:49.029 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 30 v 1286 (ObjNum 590 snap 176 seq_num 590) dirty exists 2024-09-21T09:39:49.029 INFO:tasks.rados.rados.0.smithi022.stdout:2487: done (2 left) 2024-09-21T09:39:49.029 INFO:tasks.rados.rados.0.smithi022.stdout:2488: done (1 left) 2024-09-21T09:39:49.029 INFO:tasks.rados.rados.0.smithi022.stdout:2489: done (0 left) 2024-09-21T09:39:49.029 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 15 current snap is 247 2024-09-21T09:39:49.029 INFO:tasks.rados.rados.0.smithi022.stdout:2490: seq_num 788 ranges {3121152=802816,3923968=401408} 2024-09-21T09:39:49.051 INFO:tasks.rados.rados.0.smithi022.stdout:2490: writing smithi02284960-15 from 3121152 to 3923968 tid 1 2024-09-21T09:39:49.053 INFO:tasks.rados.rados.0.smithi022.stdout:2490: writing smithi02284960-15 from 3923968 to 4325376 tid 2 2024-09-21T09:39:49.054 INFO:tasks.rados.rados.0.smithi022.stdout:2491: snap_create 2024-09-21T09:39:49.315 INFO:teuthology.orchestra.run.smithi022.stdout:true 2024-09-21T09:39:49.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:49 smithi064 ceph-mon[86709]: osdmap e565: 8 total, 8 up, 8 in 2024-09-21T09:39:49.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:49 smithi064 ceph-mon[86709]: osd.7 marked itself down and dead 2024-09-21T09:39:49.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:49 smithi064 ceph-mon[86709]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T09:39:49.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:49 smithi064 ceph-mon[86709]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-09-21T09:39:49.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:49 smithi064 ceph-mon[86709]: osdmap e566: 8 total, 7 up, 8 in 2024-09-21T09:39:49.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:49 smithi022 ceph-mon[96736]: osdmap e565: 8 total, 8 up, 8 in 2024-09-21T09:39:49.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:49 smithi022 ceph-mon[96736]: osd.7 marked itself down and dead 2024-09-21T09:39:49.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:49 smithi022 ceph-mon[96736]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T09:39:49.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:49 smithi022 ceph-mon[96736]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-09-21T09:39:49.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:49 smithi022 ceph-mon[96736]: osdmap e566: 8 total, 7 up, 8 in 2024-09-21T09:39:49.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:49 smithi022 ceph-mon[98615]: osdmap e565: 8 total, 8 up, 8 in 2024-09-21T09:39:49.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:49 smithi022 ceph-mon[98615]: osd.7 marked itself down and dead 2024-09-21T09:39:49.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:49 smithi022 ceph-mon[98615]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T09:39:49.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:49 smithi022 ceph-mon[98615]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-09-21T09:39:49.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:49 smithi022 ceph-mon[98615]: osdmap e566: 8 total, 7 up, 8 in 2024-09-21T09:39:49.899 INFO:teuthology.orchestra.run.smithi022.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T09:39:49.899 INFO:teuthology.orchestra.run.smithi022.stdout:alertmanager.a smithi022 *:9093,9094 running (6m) 2m ago 11m 22.0M - 0.27.0 11f11916f8cd caced80ba544 2024-09-21T09:39:49.899 INFO:teuthology.orchestra.run.smithi022.stdout:grafana.a smithi064 *:3000 running (6m) 25s ago 10m 83.5M - 9.4.7 954c08fa6188 527bb657d70d 2024-09-21T09:39:49.900 INFO:teuthology.orchestra.run.smithi022.stdout:mds.a smithi022 running (12m) 2m ago 12m 18.1M - 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 34e4221f3a27 2024-09-21T09:39:49.900 INFO:teuthology.orchestra.run.smithi022.stdout:mds.b smithi064 running (12m) 25s ago 12m 22.9M - 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 0f846b3ec694 2024-09-21T09:39:49.900 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.x smithi064 *:8443,9283,8765 running (5m) 25s ago 15m 455M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 61abc61fea8d 2024-09-21T09:39:49.900 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.y smithi022 *:8443,9283,8765 running (5m) 2m ago 17m 517M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 6c5bee78f09b 2024-09-21T09:39:49.900 INFO:teuthology.orchestra.run.smithi022.stdout:mon.a smithi022 running (5m) 2m ago 17m 58.2M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 0513395c784d 2024-09-21T09:39:49.900 INFO:teuthology.orchestra.run.smithi022.stdout:mon.b smithi064 running (4m) 25s ago 16m 51.9M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 9dab26b6a9e7 2024-09-21T09:39:49.900 INFO:teuthology.orchestra.run.smithi022.stdout:mon.c smithi022 running (4m) 2m ago 16m 49.3M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 c74b58b67bd6 2024-09-21T09:39:49.900 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.a smithi022 *:9100 running (6m) 2m ago 10m 11.1M - 1.7.0 72c9c2088986 593c6158f524 2024-09-21T09:39:49.900 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.b smithi064 *:9100 running (6m) 25s ago 10m 10.8M - 1.7.0 72c9c2088986 a533420b0139 2024-09-21T09:39:49.900 INFO:teuthology.orchestra.run.smithi022.stdout:osd.0 smithi022 running (3m) 2m ago 15m 141M 2481M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 40ce23a2b569 2024-09-21T09:39:49.900 INFO:teuthology.orchestra.run.smithi022.stdout:osd.1 smithi022 running (3m) 2m ago 15m 115M 2481M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 7cd0c18d224c 2024-09-21T09:39:49.900 INFO:teuthology.orchestra.run.smithi022.stdout:osd.2 smithi022 running (2m) 2m ago 14m 108M 2481M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 b8c944306698 2024-09-21T09:39:49.900 INFO:teuthology.orchestra.run.smithi022.stdout:osd.3 smithi022 running (2m) 2m ago 14m 12.2M 2481M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 c90b27f61bb6 2024-09-21T09:39:49.900 INFO:teuthology.orchestra.run.smithi022.stdout:osd.4 smithi064 running (98s) 25s ago 14m 116M 2480M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 936099dbca14 2024-09-21T09:39:49.901 INFO:teuthology.orchestra.run.smithi022.stdout:osd.5 smithi064 running (63s) 25s ago 13m 125M 2480M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 b3225b090dba 2024-09-21T09:39:49.901 INFO:teuthology.orchestra.run.smithi022.stdout:osd.6 smithi064 running (30s) 25s ago 13m 14.6M 2480M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 c6a9a09637e5 2024-09-21T09:39:49.901 INFO:teuthology.orchestra.run.smithi022.stdout:osd.7 smithi064 running (12m) 25s ago 12m 223M 2480M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 18ef0be94104 2024-09-21T09:39:49.901 INFO:teuthology.orchestra.run.smithi022.stdout:prometheus.a smithi064 *:9095 running (5m) 25s ago 11m 54.5M - 2.51.0 1d3b7f56885b 80629ffa2179 2024-09-21T09:39:49.901 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:39:49 smithi064 podman[105520]: 2024-09-21 09:39:49.579955087 +0000 UTC m=+1.087042358 container died 18ef0be94104099b6cf6703aee63afe8f883748327b3028412a83e50c697c792 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, RELEASE=quincy-fbd1a6a, ceph=True) 2024-09-21T09:39:50.021 INFO:tasks.rados.rados.0.smithi022.stdout:2491: done (1 left) 2024-09-21T09:39:50.021 INFO:tasks.rados.rados.0.smithi022.stdout:2492: rmattr oid 45 current snap is 248 2024-09-21T09:39:50.021 INFO:tasks.rados.rados.0.smithi022.stdout:2492: done (1 left) 2024-09-21T09:39:50.021 INFO:tasks.rados.rados.0.smithi022.stdout:2493: delete oid 14 current snap is 248 2024-09-21T09:39:50.030 INFO:tasks.rados.rados.0.smithi022.stdout:2493: done (1 left) 2024-09-21T09:39:50.030 INFO:tasks.rados.rados.0.smithi022.stdout:2494: read oid 17 snap -1 2024-09-21T09:39:50.030 INFO:tasks.rados.rados.0.smithi022.stdout:2494: expect (ObjNum 769 snap 241 seq_num 769) 2024-09-21T09:39:50.030 INFO:tasks.rados.rados.0.smithi022.stdout:2495: delete oid 26 current snap is 248 2024-09-21T09:39:50.032 INFO:tasks.rados.rados.0.smithi022.stdout:2495: done (2 left) 2024-09-21T09:39:50.032 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 26 current snap is 248 2024-09-21T09:39:50.033 INFO:tasks.rados.rados.0.smithi022.stdout:2496: seq_num 791 ranges {0=450560,450560=475136,925696=327680} 2024-09-21T09:39:50.035 INFO:tasks.rados.rados.0.smithi022.stdout:2496: writing smithi02284960-26 from 0 to 450560 tid 1 2024-09-21T09:39:50.038 INFO:tasks.rados.rados.0.smithi022.stdout:2496: writing smithi02284960-26 from 450560 to 925696 tid 2 2024-09-21T09:39:50.040 INFO:tasks.rados.rados.0.smithi022.stdout:2496: writing smithi02284960-26 from 925696 to 1253376 tid 3 2024-09-21T09:39:50.041 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 32 current snap is 248 2024-09-21T09:39:50.041 INFO:tasks.rados.rados.0.smithi022.stdout:2490: finishing write tid 1 to smithi02284960-15 2024-09-21T09:39:50.041 INFO:tasks.rados.rados.0.smithi022.stdout:2490: finishing write tid 2 to smithi02284960-15 2024-09-21T09:39:50.041 INFO:tasks.rados.rados.0.smithi022.stdout:2490: finishing write tid 3 to smithi02284960-15 2024-09-21T09:39:50.041 INFO:tasks.rados.rados.0.smithi022.stdout:2490: finishing write tid 4 to smithi02284960-15 2024-09-21T09:39:50.041 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 15 v 1448 (ObjNum 788 snap 247 seq_num 788) dirty exists 2024-09-21T09:39:50.041 INFO:tasks.rados.rados.0.smithi022.stdout:2490: left oid 15 (ObjNum 788 snap 247 seq_num 788) 2024-09-21T09:39:50.041 INFO:tasks.rados.rados.0.smithi022.stdout:2497: seq_num 792 ranges {0=466944,466944=761856,1228800=40960} 2024-09-21T09:39:50.043 INFO:tasks.rados.rados.0.smithi022.stdout:2497: writing smithi02284960-32 from 0 to 466944 tid 1 2024-09-21T09:39:50.048 INFO:tasks.rados.rados.0.smithi022.stdout:2497: writing smithi02284960-32 from 466944 to 1228800 tid 2 2024-09-21T09:39:50.049 INFO:tasks.rados.rados.0.smithi022.stdout:2497: writing smithi02284960-32 from 1228800 to 1269760 tid 3 2024-09-21T09:39:50.049 INFO:tasks.rados.rados.0.smithi022.stdout:2490: done (3 left) 2024-09-21T09:39:50.049 INFO:tasks.rados.rados.0.smithi022.stdout:2498: snap_remove snap 242 2024-09-21T09:39:50.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:50 smithi064 ceph-mon[86709]: pgmap v452: 121 pgs: 121 active+clean; 135 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 5.6 MiB/s wr, 19 op/s 2024-09-21T09:39:50.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:50 smithi064 ceph-mon[86709]: osdmap e567: 8 total, 7 up, 8 in 2024-09-21T09:39:50.292 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-09-21T09:39:50.292 INFO:teuthology.orchestra.run.smithi022.stdout: "mon": { 2024-09-21T09:39:50.292 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 3 2024-09-21T09:39:50.292 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-21T09:39:50.292 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr": { 2024-09-21T09:39:50.292 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 2 2024-09-21T09:39:50.293 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-21T09:39:50.293 INFO:teuthology.orchestra.run.smithi022.stdout: "osd": { 2024-09-21T09:39:50.293 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 7 2024-09-21T09:39:50.293 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-21T09:39:50.293 INFO:teuthology.orchestra.run.smithi022.stdout: "mds": { 2024-09-21T09:39:50.293 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 2 2024-09-21T09:39:50.293 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-21T09:39:50.293 INFO:teuthology.orchestra.run.smithi022.stdout: "overall": { 2024-09-21T09:39:50.293 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 2, 2024-09-21T09:39:50.293 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 12 2024-09-21T09:39:50.293 INFO:teuthology.orchestra.run.smithi022.stdout: } 2024-09-21T09:39:50.293 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-09-21T09:39:50.294 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:50 smithi022 ceph-mon[98615]: pgmap v452: 121 pgs: 121 active+clean; 135 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 5.6 MiB/s wr, 19 op/s 2024-09-21T09:39:50.294 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:50 smithi022 ceph-mon[98615]: osdmap e567: 8 total, 7 up, 8 in 2024-09-21T09:39:50.294 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:50 smithi022 ceph-mon[96736]: pgmap v452: 121 pgs: 121 active+clean; 135 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 5.6 MiB/s wr, 19 op/s 2024-09-21T09:39:50.294 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:50 smithi022 ceph-mon[96736]: osdmap e567: 8 total, 7 up, 8 in 2024-09-21T09:39:50.591 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-09-21T09:39:50.591 INFO:teuthology.orchestra.run.smithi022.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47", 2024-09-21T09:39:50.591 INFO:teuthology.orchestra.run.smithi022.stdout: "in_progress": true, 2024-09-21T09:39:50.591 INFO:teuthology.orchestra.run.smithi022.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T09:39:50.591 INFO:teuthology.orchestra.run.smithi022.stdout: "services_complete": [ 2024-09-21T09:39:50.591 INFO:teuthology.orchestra.run.smithi022.stdout: "mon", 2024-09-21T09:39:50.591 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr" 2024-09-21T09:39:50.591 INFO:teuthology.orchestra.run.smithi022.stdout: ], 2024-09-21T09:39:50.591 INFO:teuthology.orchestra.run.smithi022.stdout: "progress": "12/20 daemons upgraded", 2024-09-21T09:39:50.591 INFO:teuthology.orchestra.run.smithi022.stdout: "message": "Currently upgrading osd daemons", 2024-09-21T09:39:50.591 INFO:teuthology.orchestra.run.smithi022.stdout: "is_paused": false 2024-09-21T09:39:50.591 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-09-21T09:39:50.691 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:39:50 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:39:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:39:50.691 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:39:50 smithi064 podman[105520]: 2024-09-21 09:39:50.191183088 +0000 UTC m=+1.698270357 container cleanup 18ef0be94104099b6cf6703aee63afe8f883748327b3028412a83e50c697c792 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, RELEASE=quincy-fbd1a6a, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T09:39:50.692 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:39:50 smithi064 bash[105520]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7 2024-09-21T09:39:50.692 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:39:50 smithi064 podman[105540]: 2024-09-21 09:39:50.385534987 +0000 UTC m=+0.798388682 container remove 18ef0be94104099b6cf6703aee63afe8f883748327b3028412a83e50c697c792 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=quincy-fbd1a6a, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.license=GPLv2, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD) 2024-09-21T09:39:51.018 INFO:tasks.rados.rados.0.smithi022.stdout:2496: finishing write tid 1 to smithi02284960-26 2024-09-21T09:39:51.018 INFO:tasks.rados.rados.0.smithi022.stdout:2496: finishing write tid 2 to smithi02284960-26 2024-09-21T09:39:51.018 INFO:tasks.rados.rados.0.smithi022.stdout:2496: finishing write tid 3 to smithi02284960-26 2024-09-21T09:39:51.018 INFO:tasks.rados.rados.0.smithi022.stdout:2496: finishing write tid 4 to smithi02284960-26 2024-09-21T09:39:51.019 INFO:tasks.rados.rados.0.smithi022.stdout:2496: finishing write tid 5 to smithi02284960-26 2024-09-21T09:39:51.019 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 26 v 1338 (ObjNum 791 snap 248 seq_num 791) dirty exists 2024-09-21T09:39:51.019 INFO:tasks.rados.rados.0.smithi022.stdout:2496: left oid 26 (ObjNum 791 snap 248 seq_num 791) 2024-09-21T09:39:51.019 INFO:tasks.rados.rados.0.smithi022.stdout:2497: finishing write tid 1 to smithi02284960-32 2024-09-21T09:39:51.019 INFO:tasks.rados.rados.0.smithi022.stdout:2497: finishing write tid 2 to smithi02284960-32 2024-09-21T09:39:51.019 INFO:tasks.rados.rados.0.smithi022.stdout:2497: finishing write tid 3 to smithi02284960-32 2024-09-21T09:39:51.019 INFO:tasks.rados.rados.0.smithi022.stdout:2497: finishing write tid 4 to smithi02284960-32 2024-09-21T09:39:51.019 INFO:tasks.rados.rados.0.smithi022.stdout:2497: finishing write tid 5 to smithi02284960-32 2024-09-21T09:39:51.019 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 32 v 1481 (ObjNum 792 snap 248 seq_num 792) dirty exists 2024-09-21T09:39:51.019 INFO:tasks.rados.rados.0.smithi022.stdout:2497: left oid 32 (ObjNum 792 snap 248 seq_num 792) 2024-09-21T09:39:51.019 INFO:tasks.rados.rados.0.smithi022.stdout:2494: expect (ObjNum 769 snap 241 seq_num 769) 2024-09-21T09:39:51.079 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:39:50 smithi064 podman[105633]: 2024-09-21 09:39:50.811443935 +0000 UTC m=+0.108155290 container create 83e44f38b7e9878e2883fe4c09bca033991ddce5354f11673866e7be792931de (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7-deactivate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, RELEASE=main-62da54b, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=) 2024-09-21T09:39:51.079 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:39:50 smithi064 podman[105633]: 2024-09-21 09:39:50.719395062 +0000 UTC m=+0.016106348 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:39:51.339 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:51 smithi064 ceph-mon[86709]: from='client.44381 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:51.339 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:51 smithi064 ceph-mon[86709]: from='client.44387 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:51.339 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:51 smithi064 ceph-mon[86709]: from='client.34464 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:51.339 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:51 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/532788405' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:39:51.339 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:51 smithi064 ceph-mon[86709]: osdmap e568: 8 total, 7 up, 8 in 2024-09-21T09:39:51.339 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:39:51 smithi064 podman[105633]: 2024-09-21 09:39:51.130766233 +0000 UTC m=+0.427477506 container init 83e44f38b7e9878e2883fe4c09bca033991ddce5354f11673866e7be792931de (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7-deactivate, RELEASE=main-62da54b, GIT_CLEAN=True, org.label-schema.license=GPLv2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-09-21T09:39:51.340 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:39:51 smithi064 podman[105633]: 2024-09-21 09:39:51.135863529 +0000 UTC m=+0.432574812 container start 83e44f38b7e9878e2883fe4c09bca033991ddce5354f11673866e7be792931de (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240731, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-62da54b, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2) 2024-09-21T09:39:51.340 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:39:51 smithi064 podman[105633]: 2024-09-21 09:39:51.222108258 +0000 UTC m=+0.518819546 container attach 83e44f38b7e9878e2883fe4c09bca033991ddce5354f11673866e7be792931de (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7-deactivate, org.label-schema.vendor=CentOS, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, RELEASE=main-62da54b, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux ) 2024-09-21T09:39:51.557 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:51 smithi022 ceph-mon[98615]: from='client.44381 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:51.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:51 smithi022 ceph-mon[98615]: from='client.44387 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:51.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:51 smithi022 ceph-mon[98615]: from='client.34464 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:51.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:51 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/532788405' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:39:51.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:51 smithi022 ceph-mon[98615]: osdmap e568: 8 total, 7 up, 8 in 2024-09-21T09:39:51.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:51 smithi022 ceph-mon[96736]: from='client.44381 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:51.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:51 smithi022 ceph-mon[96736]: from='client.44387 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:51.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:51 smithi022 ceph-mon[96736]: from='client.34464 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:51.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:51 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/532788405' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:39:51.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:51 smithi022 ceph-mon[96736]: osdmap e568: 8 total, 7 up, 8 in 2024-09-21T09:39:51.635 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:39:51 smithi064 podman[105633]: 2024-09-21 09:39:51.337775031 +0000 UTC m=+0.634486311 container died 83e44f38b7e9878e2883fe4c09bca033991ddce5354f11673866e7be792931de (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7-deactivate, org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=main-62da54b, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731) 2024-09-21T09:39:52.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:52 smithi064 ceph-mon[86709]: pgmap v455: 121 pgs: 7 active+undersized, 14 stale+active+clean, 6 active+undersized+degraded, 94 active+clean; 142 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 2.5 MiB/s wr, 15 op/s; 25/297 objects degraded (8.418%) 2024-09-21T09:39:52.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:52 smithi064 ceph-mon[86709]: from='client.34476 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:52.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:52 smithi064 ceph-mon[86709]: Health check failed: Degraded data redundancy: 25/297 objects degraded (8.418%), 6 pgs degraded (PG_DEGRADED) 2024-09-21T09:39:52.192 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:39:51 smithi064 podman[105675]: 2024-09-21 09:39:51.897902936 +0000 UTC m=+0.554063289 container remove 83e44f38b7e9878e2883fe4c09bca033991ddce5354f11673866e7be792931de (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7-deactivate, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, ceph=True, GIT_CLEAN=True, RELEASE=main-62da54b) 2024-09-21T09:39:52.192 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:39:51 smithi064 systemd[1]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490@osd.7.service: Deactivated successfully. 2024-09-21T09:39:52.192 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:39:51 smithi064 systemd[1]: Stopped Ceph osd.7 for d65e801e-77fa-11ef-baf6-efdc52797490. 2024-09-21T09:39:52.192 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:39:51 smithi064 systemd[1]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490@osd.7.service: Consumed 19.192s CPU time. 2024-09-21T09:39:52.192 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:39:52 smithi064 systemd[1]: Starting Ceph osd.7 for d65e801e-77fa-11ef-baf6-efdc52797490... 2024-09-21T09:39:52.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:52 smithi022 ceph-mon[98615]: pgmap v455: 121 pgs: 7 active+undersized, 14 stale+active+clean, 6 active+undersized+degraded, 94 active+clean; 142 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 2.5 MiB/s wr, 15 op/s; 25/297 objects degraded (8.418%) 2024-09-21T09:39:52.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:52 smithi022 ceph-mon[98615]: from='client.34476 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:52.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:52 smithi022 ceph-mon[98615]: Health check failed: Degraded data redundancy: 25/297 objects degraded (8.418%), 6 pgs degraded (PG_DEGRADED) 2024-09-21T09:39:52.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:52 smithi022 ceph-mon[96736]: pgmap v455: 121 pgs: 7 active+undersized, 14 stale+active+clean, 6 active+undersized+degraded, 94 active+clean; 142 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 2.5 MiB/s wr, 15 op/s; 25/297 objects degraded (8.418%) 2024-09-21T09:39:52.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:52 smithi022 ceph-mon[96736]: from='client.34476 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:52.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:52 smithi022 ceph-mon[96736]: Health check failed: Degraded data redundancy: 25/297 objects degraded (8.418%), 6 pgs degraded (PG_DEGRADED) 2024-09-21T09:39:52.617 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:39:52 smithi064 podman[105783]: 2024-09-21 09:39:52.237005448 +0000 UTC m=+0.016694997 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:39:52.617 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:39:52 smithi064 podman[105783]: 2024-09-21 09:39:52.357240073 +0000 UTC m=+0.136929611 container create 511bf8fd081efbe428069cbba0bdb13b65aa57b3b2268e0d1a01a1af1f6e4d9a (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7-activate, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, RELEASE=main-62da54b, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, ceph=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS) 2024-09-21T09:39:52.617 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:39:52 smithi064 podman[105783]: 2024-09-21 09:39:52.615904183 +0000 UTC m=+0.395593720 container init 511bf8fd081efbe428069cbba0bdb13b65aa57b3b2268e0d1a01a1af1f6e4d9a (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7-activate, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, RELEASE=main-62da54b, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, ceph=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0) 2024-09-21T09:39:52.731 INFO:tasks.rados.rados.0.smithi022.stdout:2494: done (3 left) 2024-09-21T09:39:52.731 INFO:tasks.rados.rados.0.smithi022.stdout:2496: done (2 left) 2024-09-21T09:39:52.731 INFO:tasks.rados.rados.0.smithi022.stdout:2497: done (1 left) 2024-09-21T09:39:52.731 INFO:tasks.rados.rados.0.smithi022.stdout:2498: done (0 left) 2024-09-21T09:39:52.731 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 34 current snap is 248 2024-09-21T09:39:52.731 INFO:tasks.rados.rados.0.smithi022.stdout:2499: seq_num 793 ranges {966656=483328,1449984=647168,2097152=155648} 2024-09-21T09:39:52.738 INFO:tasks.rados.rados.0.smithi022.stdout:2499: writing smithi02284960-34 from 966656 to 1449984 tid 1 2024-09-21T09:39:52.742 INFO:tasks.rados.rados.0.smithi022.stdout:2499: writing smithi02284960-34 from 1449984 to 2097152 tid 2 2024-09-21T09:39:52.744 INFO:tasks.rados.rados.0.smithi022.stdout:2499: writing smithi02284960-34 from 2097152 to 2252800 tid 3 2024-09-21T09:39:52.744 INFO:tasks.rados.rados.0.smithi022.stdout:2500: read oid 36 snap 235 2024-09-21T09:39:52.744 INFO:tasks.rados.rados.0.smithi022.stdout:2500: expect deleted 2024-09-21T09:39:52.744 INFO:tasks.rados.rados.0.smithi022.stdout:2501: delete oid 26 current snap is 248 2024-09-21T09:39:52.744 INFO:tasks.rados.rados.0.smithi022.stdout:2499: finishing write tid 1 to smithi02284960-34 2024-09-21T09:39:52.745 INFO:tasks.rados.rados.0.smithi022.stdout:2500: done (2 left) 2024-09-21T09:39:52.745 INFO:tasks.rados.rados.0.smithi022.stdout:2501: done (1 left) 2024-09-21T09:39:52.745 INFO:tasks.rados.rados.0.smithi022.stdout:2502: delete oid 38 current snap is 248 2024-09-21T09:39:52.747 INFO:tasks.rados.rados.0.smithi022.stdout:2499: finishing write tid 2 to smithi02284960-34 2024-09-21T09:39:52.747 INFO:tasks.rados.rados.0.smithi022.stdout:2499: finishing write tid 3 to smithi02284960-34 2024-09-21T09:39:52.747 INFO:tasks.rados.rados.0.smithi022.stdout:2499: finishing write tid 4 to smithi02284960-34 2024-09-21T09:39:52.747 INFO:tasks.rados.rados.0.smithi022.stdout:2502: done (1 left) 2024-09-21T09:39:52.747 INFO:tasks.rados.rados.0.smithi022.stdout:2503: setattr oid 37 current snap is 248 2024-09-21T09:39:52.748 INFO:tasks.rados.rados.0.smithi022.stdout:2499: finishing write tid 5 to smithi02284960-34 2024-09-21T09:39:52.748 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 34 v 1289 (ObjNum 793 snap 248 seq_num 793) dirty exists 2024-09-21T09:39:52.748 INFO:tasks.rados.rados.0.smithi022.stdout:2499: left oid 34 (ObjNum 793 snap 248 seq_num 793) 2024-09-21T09:39:52.750 INFO:tasks.rados.rados.0.smithi022.stdout:2499: done (1 left) 2024-09-21T09:39:52.750 INFO:tasks.rados.rados.0.smithi022.stdout:2504: snap_remove snap 244 2024-09-21T09:39:52.941 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:39:52 smithi064 podman[105783]: 2024-09-21 09:39:52.621505589 +0000 UTC m=+0.401195126 container start 511bf8fd081efbe428069cbba0bdb13b65aa57b3b2268e0d1a01a1af1f6e4d9a (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7-activate, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-62da54b, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=) 2024-09-21T09:39:52.941 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:39:52 smithi064 podman[105783]: 2024-09-21 09:39:52.709152237 +0000 UTC m=+0.488841774 container attach 511bf8fd081efbe428069cbba0bdb13b65aa57b3b2268e0d1a01a1af1f6e4d9a (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7-activate, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240731, RELEASE=main-62da54b, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-09-21T09:39:52.942 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:39:52 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7-activate[105798]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T09:39:52.942 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:39:52 smithi064 bash[105783]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T09:39:52.942 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:39:52 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7-activate[105798]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T09:39:52.942 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:39:52 smithi064 bash[105783]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T09:39:53.086 INFO:tasks.rados.rados.0.smithi022.stdout:2504: done (1 left) 2024-09-21T09:39:53.086 INFO:tasks.rados.rados.0.smithi022.stdout:2505: snap_create 2024-09-21T09:39:53.086 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 37 v 1450 (ObjNum 672 snap 208 seq_num 672) dirty exists 2024-09-21T09:39:53.528 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:39:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7-activate[105798]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-21T09:39:53.528 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:39:53 smithi064 bash[105783]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-21T09:39:53.529 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:39:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7-activate[105798]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-21T09:39:53.529 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:39:53 smithi064 bash[105783]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-21T09:39:53.529 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:39:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7-activate[105798]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-21T09:39:53.848 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:39:53 smithi064 bash[105783]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-21T09:39:53.848 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:39:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7-activate[105798]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-21T09:39:53.848 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:39:53 smithi064 bash[105783]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-21T09:39:53.848 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:39:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7-activate[105798]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-09-21T09:39:53.848 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:39:53 smithi064 bash[105783]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-09-21T09:39:53.848 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:39:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7-activate[105798]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-21T09:39:53.848 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:39:53 smithi064 bash[105783]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-21T09:39:53.849 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:39:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7-activate[105798]: --> ceph-volume raw activate successful for osd ID: 7 2024-09-21T09:39:53.849 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:39:53 smithi064 bash[105783]: --> ceph-volume raw activate successful for osd ID: 7 2024-09-21T09:39:53.849 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:39:53 smithi064 podman[105783]: 2024-09-21 09:39:53.557054434 +0000 UTC m=+1.336743980 container died 511bf8fd081efbe428069cbba0bdb13b65aa57b3b2268e0d1a01a1af1f6e4d9a (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7-activate, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-62da54b, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True) 2024-09-21T09:39:54.017 INFO:tasks.rados.rados.0.smithi022.stdout:2503: done (1 left) 2024-09-21T09:39:54.017 INFO:tasks.rados.rados.0.smithi022.stdout:2505: done (0 left) 2024-09-21T09:39:54.018 INFO:tasks.rados.rados.0.smithi022.stdout:2506: read oid 42 snap -1 2024-09-21T09:39:54.018 INFO:tasks.rados.rados.0.smithi022.stdout:2506: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-21T09:39:54.018 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 20 current snap is 249 2024-09-21T09:39:54.018 INFO:tasks.rados.rados.0.smithi022.stdout:2507: seq_num 796 ranges {5259264=540672,5799936=704512,6504448=98304} 2024-09-21T09:39:54.049 INFO:tasks.rados.rados.0.smithi022.stdout:2507: writing smithi02284960-20 from 5259264 to 5799936 tid 1 2024-09-21T09:39:54.053 INFO:tasks.rados.rados.0.smithi022.stdout:2507: writing smithi02284960-20 from 5799936 to 6504448 tid 2 2024-09-21T09:39:54.054 INFO:tasks.rados.rados.0.smithi022.stdout:2507: writing smithi02284960-20 from 6504448 to 6602752 tid 3 2024-09-21T09:39:54.054 INFO:tasks.rados.rados.0.smithi022.stdout:2508: read oid 32 snap 245 2024-09-21T09:39:54.055 INFO:tasks.rados.rados.0.smithi022.stdout:2508: expect (ObjNum -724231152 snap 22094 seq_num 3569445440) 2024-09-21T09:39:54.055 INFO:tasks.rados.rados.0.smithi022.stdout:2509: delete oid 10 current snap is 249 2024-09-21T09:39:54.055 INFO:tasks.rados.rados.0.smithi022.stdout:2506: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-21T09:39:54.057 INFO:tasks.rados.rados.0.smithi022.stdout:2506: done (3 left) 2024-09-21T09:39:54.057 INFO:tasks.rados.rados.0.smithi022.stdout:2509: done (2 left) 2024-09-21T09:39:54.057 INFO:tasks.rados.rados.0.smithi022.stdout:2510: read oid 28 snap -1 2024-09-21T09:39:54.057 INFO:tasks.rados.rados.0.smithi022.stdout:2510: expect (ObjNum 753 snap 231 seq_num 753) 2024-09-21T09:39:54.057 INFO:tasks.rados.rados.0.smithi022.stdout:2511: snap_create 2024-09-21T09:39:54.057 INFO:tasks.rados.rados.0.smithi022.stdout:2507: finishing write tid 1 to smithi02284960-20 2024-09-21T09:39:54.061 INFO:tasks.rados.rados.0.smithi022.stdout:2507: finishing write tid 2 to smithi02284960-20 2024-09-21T09:39:54.061 INFO:tasks.rados.rados.0.smithi022.stdout:2508: expect (ObjNum -724231152 snap 22094 seq_num 3569445440) 2024-09-21T09:39:54.066 INFO:tasks.rados.rados.0.smithi022.stdout:2507: finishing write tid 3 to smithi02284960-20 2024-09-21T09:39:54.066 INFO:tasks.rados.rados.0.smithi022.stdout:2507: finishing write tid 4 to smithi02284960-20 2024-09-21T09:39:54.066 INFO:tasks.rados.rados.0.smithi022.stdout:2507: finishing write tid 5 to smithi02284960-20 2024-09-21T09:39:54.066 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 20 v 1498 (ObjNum 796 snap 249 seq_num 796) dirty exists 2024-09-21T09:39:54.066 INFO:tasks.rados.rados.0.smithi022.stdout:2507: left oid 20 (ObjNum 796 snap 249 seq_num 796) 2024-09-21T09:39:54.067 INFO:tasks.rados.rados.0.smithi022.stdout:2510: expect (ObjNum 753 snap 231 seq_num 753) 2024-09-21T09:39:54.160 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:54 smithi064 ceph-mon[86709]: pgmap v457: 121 pgs: 1 active+clean+snaptrim, 19 active+undersized, 7 stale+active+clean, 8 active+undersized+degraded, 86 active+clean; 142 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 19 MiB/s rd, 2.1 MiB/s wr, 13 op/s; 27/303 objects degraded (8.911%) 2024-09-21T09:39:54.160 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:54 smithi064 ceph-mon[86709]: osdmap e569: 8 total, 7 up, 8 in 2024-09-21T09:39:54.160 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:54 smithi064 ceph-mon[86709]: osdmap e570: 8 total, 7 up, 8 in 2024-09-21T09:39:54.360 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:54 smithi022 ceph-mon[98615]: pgmap v457: 121 pgs: 1 active+clean+snaptrim, 19 active+undersized, 7 stale+active+clean, 8 active+undersized+degraded, 86 active+clean; 142 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 19 MiB/s rd, 2.1 MiB/s wr, 13 op/s; 27/303 objects degraded (8.911%) 2024-09-21T09:39:54.360 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:54 smithi022 ceph-mon[98615]: osdmap e569: 8 total, 7 up, 8 in 2024-09-21T09:39:54.360 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:54 smithi022 ceph-mon[98615]: osdmap e570: 8 total, 7 up, 8 in 2024-09-21T09:39:54.360 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:54 smithi022 ceph-mon[96736]: pgmap v457: 121 pgs: 1 active+clean+snaptrim, 19 active+undersized, 7 stale+active+clean, 8 active+undersized+degraded, 86 active+clean; 142 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 19 MiB/s rd, 2.1 MiB/s wr, 13 op/s; 27/303 objects degraded (8.911%) 2024-09-21T09:39:54.360 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:54 smithi022 ceph-mon[96736]: osdmap e569: 8 total, 7 up, 8 in 2024-09-21T09:39:54.360 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:54 smithi022 ceph-mon[96736]: osdmap e570: 8 total, 7 up, 8 in 2024-09-21T09:39:54.441 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:39:54 smithi064 podman[105942]: 2024-09-21 09:39:54.159259507 +0000 UTC m=+0.594599407 container remove 511bf8fd081efbe428069cbba0bdb13b65aa57b3b2268e0d1a01a1af1f6e4d9a (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7-activate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, RELEASE=main-62da54b, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=) 2024-09-21T09:39:54.442 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:39:54 smithi064 podman[106007]: 2024-09-21 09:39:54.36010156 +0000 UTC m=+0.104559223 container create 6f82655605e3f8dd46c90697549f33f1b6367e048120204dddeb9a008bd0a65f (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, RELEASE=main-62da54b, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS) 2024-09-21T09:39:54.442 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:39:54 smithi064 podman[106007]: 2024-09-21 09:39:54.272141574 +0000 UTC m=+0.016599246 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:39:54.941 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:39:54 smithi064 podman[106007]: 2024-09-21 09:39:54.570621442 +0000 UTC m=+0.315079108 container init 6f82655605e3f8dd46c90697549f33f1b6367e048120204dddeb9a008bd0a65f (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-62da54b, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_CLEAN=True, ceph=True, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=) 2024-09-21T09:39:54.941 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:39:54 smithi064 podman[106007]: 2024-09-21 09:39:54.575616477 +0000 UTC m=+0.320074149 container start 6f82655605e3f8dd46c90697549f33f1b6367e048120204dddeb9a008bd0a65f (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , ceph=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, RELEASE=main-62da54b) 2024-09-21T09:39:54.942 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:39:54 smithi064 ceph-osd[106045]: -- 172.21.15.64:0/889905583 <== mon.1 v2:172.21.15.64:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x563a32de1a40 con 0x563a33bb4000 2024-09-21T09:39:54.942 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:39:54 smithi064 bash[106007]: 6f82655605e3f8dd46c90697549f33f1b6367e048120204dddeb9a008bd0a65f 2024-09-21T09:39:54.942 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:39:54 smithi064 systemd[1]: Started Ceph osd.7 for d65e801e-77fa-11ef-baf6-efdc52797490. 2024-09-21T09:39:55.017 INFO:tasks.rados.rados.0.smithi022.stdout:2507: done (3 left) 2024-09-21T09:39:55.017 INFO:tasks.rados.rados.0.smithi022.stdout:2508: done (2 left) 2024-09-21T09:39:55.018 INFO:tasks.rados.rados.0.smithi022.stdout:2510: done (1 left) 2024-09-21T09:39:55.018 INFO:tasks.rados.rados.0.smithi022.stdout:2511: done (0 left) 2024-09-21T09:39:55.018 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 33 current snap is 250 2024-09-21T09:39:55.018 INFO:tasks.rados.rados.0.smithi022.stdout:2512: seq_num 798 ranges {2392064=573440,2965504=466944,3432448=335872} 2024-09-21T09:39:55.033 INFO:tasks.rados.rados.0.smithi022.stdout:2512: writing smithi02284960-33 from 2392064 to 2965504 tid 1 2024-09-21T09:39:55.036 INFO:tasks.rados.rados.0.smithi022.stdout:2512: writing smithi02284960-33 from 2965504 to 3432448 tid 2 2024-09-21T09:39:55.039 INFO:tasks.rados.rados.0.smithi022.stdout:2512: writing smithi02284960-33 from 3432448 to 3768320 tid 3 2024-09-21T09:39:55.039 INFO:tasks.rados.rados.0.smithi022.stdout:2513: snap_create 2024-09-21T09:39:55.040 INFO:tasks.rados.rados.0.smithi022.stdout:2512: finishing write tid 1 to smithi02284960-33 2024-09-21T09:39:55.043 INFO:tasks.rados.rados.0.smithi022.stdout:2512: finishing write tid 2 to smithi02284960-33 2024-09-21T09:39:55.043 INFO:tasks.rados.rados.0.smithi022.stdout:2512: finishing write tid 3 to smithi02284960-33 2024-09-21T09:39:55.044 INFO:tasks.rados.rados.0.smithi022.stdout:2512: finishing write tid 4 to smithi02284960-33 2024-09-21T09:39:55.045 INFO:tasks.rados.rados.0.smithi022.stdout:2512: finishing write tid 5 to smithi02284960-33 2024-09-21T09:39:55.045 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 33 v 1503 (ObjNum 798 snap 250 seq_num 798) dirty exists 2024-09-21T09:39:55.045 INFO:tasks.rados.rados.0.smithi022.stdout:2512: left oid 33 (ObjNum 798 snap 250 seq_num 798) 2024-09-21T09:39:56.019 INFO:tasks.rados.rados.0.smithi022.stdout:2512: done (1 left) 2024-09-21T09:39:56.019 INFO:tasks.rados.rados.0.smithi022.stdout:2513: done (0 left) 2024-09-21T09:39:56.019 INFO:tasks.rados.rados.0.smithi022.stdout:2514: delete oid 46 current snap is 251 2024-09-21T09:39:56.023 INFO:tasks.rados.rados.0.smithi022.stdout:2514: done (0 left) 2024-09-21T09:39:56.023 INFO:tasks.rados.rados.0.smithi022.stdout:2515: delete oid 40 current snap is 251 2024-09-21T09:39:56.029 INFO:tasks.rados.rados.0.smithi022.stdout:2515: done (0 left) 2024-09-21T09:39:56.029 INFO:tasks.rados.rados.0.smithi022.stdout:2516: snap_create 2024-09-21T09:39:56.057 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:39:55 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:39:55] "GET /metrics HTTP/1.1" 200 36291 "" "Prometheus/2.51.0" 2024-09-21T09:39:56.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:55 smithi022 ceph-mon[98615]: pgmap v460: 121 pgs: 1 active+clean+snaptrim, 38 active+undersized, 12 active+undersized+degraded, 70 active+clean; 147 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 1.9 MiB/s wr, 7 op/s; 33/306 objects degraded (10.784%) 2024-09-21T09:39:56.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:55 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:39:56.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:55 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:39:56.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:55 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:39:56.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:55 smithi022 ceph-mon[98615]: osdmap e571: 8 total, 7 up, 8 in 2024-09-21T09:39:56.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:55 smithi022 ceph-mon[96736]: pgmap v460: 121 pgs: 1 active+clean+snaptrim, 38 active+undersized, 12 active+undersized+degraded, 70 active+clean; 147 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 1.9 MiB/s wr, 7 op/s; 33/306 objects degraded (10.784%) 2024-09-21T09:39:56.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:55 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:39:56.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:55 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:39:56.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:55 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:39:56.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:55 smithi022 ceph-mon[96736]: osdmap e571: 8 total, 7 up, 8 in 2024-09-21T09:39:56.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:55 smithi064 ceph-mon[86709]: pgmap v460: 121 pgs: 1 active+clean+snaptrim, 38 active+undersized, 12 active+undersized+degraded, 70 active+clean; 147 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 1.9 MiB/s wr, 7 op/s; 33/306 objects degraded (10.784%) 2024-09-21T09:39:56.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:55 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:39:56.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:55 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:39:56.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:55 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:39:56.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:55 smithi064 ceph-mon[86709]: osdmap e571: 8 total, 7 up, 8 in 2024-09-21T09:39:56.569 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:39:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7[106022]: 2024-09-21T09:39:56.214+0000 7f33982a9640 -1 Falling back to public interface 2024-09-21T09:39:57.025 INFO:tasks.rados.rados.0.smithi022.stdout:2516: done (0 left) 2024-09-21T09:39:57.026 INFO:tasks.rados.rados.0.smithi022.stdout:2517: read oid 50 snap -1 2024-09-21T09:39:57.026 INFO:tasks.rados.rados.0.smithi022.stdout:2517: expect (ObjNum 701 snap 216 seq_num 701) 2024-09-21T09:39:57.026 INFO:tasks.rados.rados.0.smithi022.stdout:2518: read oid 18 snap -1 2024-09-21T09:39:57.026 INFO:tasks.rados.rados.0.smithi022.stdout:2518: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-21T09:39:57.026 INFO:tasks.rados.rados.0.smithi022.stdout:2519: setattr oid 7 current snap is 252 2024-09-21T09:39:57.027 INFO:tasks.rados.rados.0.smithi022.stdout:2520: snap_create 2024-09-21T09:39:57.029 INFO:tasks.rados.rados.0.smithi022.stdout:2518: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-21T09:39:57.035 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 7 v 1505 (ObjNum 785 snap 246 seq_num 785) dirty exists 2024-09-21T09:39:57.042 INFO:tasks.rados.rados.0.smithi022.stdout:2517: expect (ObjNum 701 snap 216 seq_num 701) 2024-09-21T09:39:57.312 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:57 smithi022 ceph-mon[98615]: osdmap e572: 8 total, 7 up, 8 in 2024-09-21T09:39:57.327 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:57 smithi022 ceph-mon[96736]: osdmap e572: 8 total, 7 up, 8 in 2024-09-21T09:39:57.327 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:57 smithi064 ceph-mon[86709]: osdmap e572: 8 total, 7 up, 8 in 2024-09-21T09:39:58.032 INFO:tasks.rados.rados.0.smithi022.stdout:2517: done (3 left) 2024-09-21T09:39:58.032 INFO:tasks.rados.rados.0.smithi022.stdout:2518: done (2 left) 2024-09-21T09:39:58.032 INFO:tasks.rados.rados.0.smithi022.stdout:2519: done (1 left) 2024-09-21T09:39:58.032 INFO:tasks.rados.rados.0.smithi022.stdout:2520: done (0 left) 2024-09-21T09:39:58.032 INFO:tasks.rados.rados.0.smithi022.stdout:2521: read oid 21 snap -1 2024-09-21T09:39:58.032 INFO:tasks.rados.rados.0.smithi022.stdout:2521: expect (ObjNum 736 snap 226 seq_num 736) 2024-09-21T09:39:58.032 INFO:tasks.rados.rados.0.smithi022.stdout:2522: read oid 16 snap -1 2024-09-21T09:39:58.032 INFO:tasks.rados.rados.0.smithi022.stdout:2522: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-21T09:39:58.032 INFO:tasks.rados.rados.0.smithi022.stdout:2523: delete oid 22 current snap is 253 2024-09-21T09:39:58.034 INFO:tasks.rados.rados.0.smithi022.stdout:2522: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-21T09:39:58.039 INFO:tasks.rados.rados.0.smithi022.stdout:2522: done (2 left) 2024-09-21T09:39:58.039 INFO:tasks.rados.rados.0.smithi022.stdout:2523: done (1 left) 2024-09-21T09:39:58.039 INFO:tasks.rados.rados.0.smithi022.stdout:2524: snap_create 2024-09-21T09:39:58.059 INFO:tasks.rados.rados.0.smithi022.stdout:2521: expect (ObjNum 736 snap 226 seq_num 736) 2024-09-21T09:39:58.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:58 smithi022 ceph-mon[98615]: pgmap v463: 121 pgs: 38 active+undersized, 12 active+undersized+degraded, 71 active+clean; 149 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 978 KiB/s wr, 8 op/s; 33/309 objects degraded (10.680%) 2024-09-21T09:39:58.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:58 smithi022 ceph-mon[98615]: Health check update: Degraded data redundancy: 33/309 objects degraded (10.680%), 12 pgs degraded (PG_DEGRADED) 2024-09-21T09:39:58.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:58 smithi022 ceph-mon[98615]: osdmap e573: 8 total, 7 up, 8 in 2024-09-21T09:39:58.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:58 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:39:58.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:58 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:39:58.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:58 smithi022 ceph-mon[96736]: pgmap v463: 121 pgs: 38 active+undersized, 12 active+undersized+degraded, 71 active+clean; 149 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 978 KiB/s wr, 8 op/s; 33/309 objects degraded (10.680%) 2024-09-21T09:39:58.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:58 smithi022 ceph-mon[96736]: Health check update: Degraded data redundancy: 33/309 objects degraded (10.680%), 12 pgs degraded (PG_DEGRADED) 2024-09-21T09:39:58.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:58 smithi022 ceph-mon[96736]: osdmap e573: 8 total, 7 up, 8 in 2024-09-21T09:39:58.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:58 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:39:58.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:58 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:39:58.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:58 smithi064 ceph-mon[86709]: pgmap v463: 121 pgs: 38 active+undersized, 12 active+undersized+degraded, 71 active+clean; 149 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 978 KiB/s wr, 8 op/s; 33/309 objects degraded (10.680%) 2024-09-21T09:39:58.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:58 smithi064 ceph-mon[86709]: Health check update: Degraded data redundancy: 33/309 objects degraded (10.680%), 12 pgs degraded (PG_DEGRADED) 2024-09-21T09:39:58.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:58 smithi064 ceph-mon[86709]: osdmap e573: 8 total, 7 up, 8 in 2024-09-21T09:39:58.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:58 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:39:58.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:58 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:39:59.018 INFO:tasks.rados.rados.0.smithi022.stdout:2521: done (1 left) 2024-09-21T09:39:59.018 INFO:tasks.rados.rados.0.smithi022.stdout:2524: done (0 left) 2024-09-21T09:39:59.019 INFO:tasks.rados.rados.0.smithi022.stdout:2525: snap_create 2024-09-21T09:39:59.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:59 smithi022 ceph-mon[98615]: osdmap e574: 8 total, 7 up, 8 in 2024-09-21T09:39:59.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:39:59 smithi022 ceph-mon[98615]: osdmap e575: 8 total, 7 up, 8 in 2024-09-21T09:39:59.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:59 smithi022 ceph-mon[96736]: osdmap e574: 8 total, 7 up, 8 in 2024-09-21T09:39:59.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:39:59 smithi022 ceph-mon[96736]: osdmap e575: 8 total, 7 up, 8 in 2024-09-21T09:39:59.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:59 smithi064 ceph-mon[86709]: osdmap e574: 8 total, 7 up, 8 in 2024-09-21T09:39:59.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:39:59 smithi064 ceph-mon[86709]: osdmap e575: 8 total, 7 up, 8 in 2024-09-21T09:40:00.019 INFO:tasks.rados.rados.0.smithi022.stdout:2525: done (0 left) 2024-09-21T09:40:00.020 INFO:tasks.rados.rados.0.smithi022.stdout:2526: delete oid 49 current snap is 255 2024-09-21T09:40:00.021 INFO:tasks.rados.rados.0.smithi022.stdout:2526: done (0 left) 2024-09-21T09:40:00.022 INFO:tasks.rados.rados.0.smithi022.stdout:2527: delete oid 41 current snap is 255 2024-09-21T09:40:00.027 INFO:tasks.rados.rados.0.smithi022.stdout:2527: done (0 left) 2024-09-21T09:40:00.027 INFO:tasks.rados.rados.0.smithi022.stdout:2528: copy_from oid 45 from oid 18 current snap is 255 2024-09-21T09:40:00.027 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 6 current snap is 255 2024-09-21T09:40:00.027 INFO:tasks.rados.rados.0.smithi022.stdout:2529: seq_num 804 ranges {598016=671744,1269760=622592,1892352=180224} 2024-09-21T09:40:00.034 INFO:tasks.rados.rados.0.smithi022.stdout:2529: writing smithi02284960-6 from 598016 to 1269760 tid 1 2024-09-21T09:40:00.037 INFO:tasks.rados.rados.0.smithi022.stdout:2529: writing smithi02284960-6 from 1269760 to 1892352 tid 2 2024-09-21T09:40:00.039 INFO:tasks.rados.rados.0.smithi022.stdout:2529: writing smithi02284960-6 from 1892352 to 2072576 tid 3 2024-09-21T09:40:00.039 INFO:tasks.rados.rados.0.smithi022.stdout:2530: copy_from oid 41 from oid 11 current snap is 255 2024-09-21T09:40:00.039 INFO:tasks.rados.rados.0.smithi022.stdout:2528: finishing copy_from racing read to smithi02284960-45 2024-09-21T09:40:00.039 INFO:tasks.rados.rados.0.smithi022.stdout:2531: setattr oid 37 current snap is 255 2024-09-21T09:40:00.039 INFO:tasks.rados.rados.0.smithi022.stdout:2528: finishing copy_from to smithi02284960-45 2024-09-21T09:40:00.039 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 45 v 1066 (ObjNum 49 snap 0 seq_num 1953066355) dirty exists 2024-09-21T09:40:00.039 INFO:tasks.rados.rados.0.smithi022.stdout:2529: finishing write tid 1 to smithi02284960-6 2024-09-21T09:40:00.041 INFO:tasks.rados.rados.0.smithi022.stdout:2528: done (3 left) 2024-09-21T09:40:00.041 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 47 current snap is 255 2024-09-21T09:40:00.041 INFO:tasks.rados.rados.0.smithi022.stdout:2532: seq_num 805 ranges {0=688128,688128=507904,1196032=294912} 2024-09-21T09:40:00.045 INFO:tasks.rados.rados.0.smithi022.stdout:2532: writing smithi02284960-47 from 0 to 688128 tid 1 2024-09-21T09:40:00.048 INFO:tasks.rados.rados.0.smithi022.stdout:2532: writing smithi02284960-47 from 688128 to 1196032 tid 2 2024-09-21T09:40:00.051 INFO:tasks.rados.rados.0.smithi022.stdout:2532: writing smithi02284960-47 from 1196032 to 1490944 tid 3 2024-09-21T09:40:00.051 INFO:tasks.rados.rados.0.smithi022.stdout:2533: rollback oid 31 current snap is 255 2024-09-21T09:40:00.051 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 31 to 239 2024-09-21T09:40:00.051 INFO:tasks.rados.rados.0.smithi022.stdout:2529: finishing write tid 2 to smithi02284960-6 2024-09-21T09:40:00.051 INFO:tasks.rados.rados.0.smithi022.stdout:2534: setattr oid 26 current snap is 255 2024-09-21T09:40:00.051 INFO:tasks.rados.rados.0.smithi022.stdout:2529: finishing write tid 3 to smithi02284960-6 2024-09-21T09:40:00.052 INFO:tasks.rados.rados.0.smithi022.stdout:2529: finishing write tid 4 to smithi02284960-6 2024-09-21T09:40:00.052 INFO:tasks.rados.rados.0.smithi022.stdout:2529: finishing write tid 5 to smithi02284960-6 2024-09-21T09:40:00.052 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 6 v 1510 (ObjNum 804 snap 255 seq_num 804) dirty exists 2024-09-21T09:40:00.052 INFO:tasks.rados.rados.0.smithi022.stdout:2529: left oid 6 (ObjNum 804 snap 255 seq_num 804) 2024-09-21T09:40:00.052 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 37 v 1452 (ObjNum 672 snap 208 seq_num 672) dirty exists 2024-09-21T09:40:00.054 INFO:tasks.rados.rados.0.smithi022.stdout:2529: done (5 left) 2024-09-21T09:40:00.054 INFO:tasks.rados.rados.0.smithi022.stdout:2531: done (4 left) 2024-09-21T09:40:00.054 INFO:tasks.rados.rados.0.smithi022.stdout:2535: copy_from oid 15 from oid 49 current snap is 255 2024-09-21T09:40:00.054 INFO:tasks.rados.rados.0.smithi022.stdout:2536: rmattr oid 39 current snap is 255 2024-09-21T09:40:00.054 INFO:tasks.rados.rados.0.smithi022.stdout:2536: done (5 left) 2024-09-21T09:40:00.054 INFO:tasks.rados.rados.0.smithi022.stdout:2537: copy_from oid 50 from oid 9 current snap is 255 2024-09-21T09:40:00.055 INFO:tasks.rados.rados.0.smithi022.stdout:2538: read oid 28 snap -1 2024-09-21T09:40:00.055 INFO:tasks.rados.rados.0.smithi022.stdout:2538: expect (ObjNum 753 snap 231 seq_num 753) 2024-09-21T09:40:00.055 INFO:tasks.rados.rados.0.smithi022.stdout:2539: delete oid 18 current snap is 255 2024-09-21T09:40:00.055 INFO:tasks.rados.rados.0.smithi022.stdout:2537: finishing copy_from to smithi02284960-50 2024-09-21T09:40:00.055 INFO:tasks.rados.rados.0.smithi022.stdout:2537: got expected ENOENT (src dne) 2024-09-21T09:40:00.056 INFO:tasks.rados.rados.0.smithi022.stdout:2533: finishing rollback tid 0 to smithi02284960-31 2024-09-21T09:40:00.056 INFO:tasks.rados.rados.0.smithi022.stdout:2537: finishing copy_from racing read to smithi02284960-50 2024-09-21T09:40:00.056 INFO:tasks.rados.rados.0.smithi022.stdout:2533: finishing rollback tid 1 to smithi02284960-31 2024-09-21T09:40:00.056 INFO:tasks.rados.rados.0.smithi022.stdout:2533: finishing rollback tid 2 to smithi02284960-31 2024-09-21T09:40:00.056 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 31 v 1456 (ObjNum 751 snap 228 seq_num 751) dirty exists 2024-09-21T09:40:00.057 INFO:tasks.rados.rados.0.smithi022.stdout:2532: finishing write tid 1 to smithi02284960-47 2024-09-21T09:40:00.058 INFO:tasks.rados.rados.0.smithi022.stdout:2535: finishing copy_from to smithi02284960-15 2024-09-21T09:40:00.058 INFO:tasks.rados.rados.0.smithi022.stdout:2535: got expected ENOENT (src dne) 2024-09-21T09:40:00.058 INFO:tasks.rados.rados.0.smithi022.stdout:2535: finishing copy_from racing read to smithi02284960-15 2024-09-21T09:40:00.058 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 26 v 1341 (ObjNum 33 snap 0 seq_num 3570162416) dirty exists 2024-09-21T09:40:00.061 INFO:tasks.rados.rados.0.smithi022.stdout:2532: finishing write tid 2 to smithi02284960-47 2024-09-21T09:40:00.061 INFO:tasks.rados.rados.0.smithi022.stdout:2532: finishing write tid 3 to smithi02284960-47 2024-09-21T09:40:00.061 INFO:tasks.rados.rados.0.smithi022.stdout:2532: finishing write tid 4 to smithi02284960-47 2024-09-21T09:40:00.062 INFO:tasks.rados.rados.0.smithi022.stdout:2533: done (7 left) 2024-09-21T09:40:00.062 INFO:tasks.rados.rados.0.smithi022.stdout:2534: done (6 left) 2024-09-21T09:40:00.062 INFO:tasks.rados.rados.0.smithi022.stdout:2535: done (5 left) 2024-09-21T09:40:00.062 INFO:tasks.rados.rados.0.smithi022.stdout:2537: done (4 left) 2024-09-21T09:40:00.062 INFO:tasks.rados.rados.0.smithi022.stdout:2539: done (3 left) 2024-09-21T09:40:00.062 INFO:tasks.rados.rados.0.smithi022.stdout:2540: snap_remove snap 235 2024-09-21T09:40:00.273 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:40:00 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7[106022]: 2024-09-21T09:40:00.014+0000 7f33982a9640 -1 osd.7 565 log_to_monitors true 2024-09-21T09:40:00.274 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:40:00 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:40:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:40:00.274 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:00 smithi064 ceph-mon[86709]: pgmap v466: 121 pgs: 38 active+undersized, 12 active+undersized+degraded, 71 active+clean; 149 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 9.2 MiB/s rd, 664 KiB/s wr, 12 op/s; 33/309 objects degraded (10.680%) 2024-09-21T09:40:00.274 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:00 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:00.274 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:00 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:00.274 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:00 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:00.274 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:00 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:00.274 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:00 smithi064 ceph-mon[86709]: Health detail: HEALTH_WARN 1 osds down; all OSDs are running squid or later but require_osd_release < squid; Degraded data redundancy: 33/309 objects degraded (10.680%), 12 pgs degraded 2024-09-21T09:40:00.274 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:00 smithi064 ceph-mon[86709]: [WRN] OSD_DOWN: 1 osds down 2024-09-21T09:40:00.275 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:00 smithi064 ceph-mon[86709]: osd.7 (root=default,host=smithi064) is down 2024-09-21T09:40:00.275 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:00 smithi064 ceph-mon[86709]: [WRN] OSD_UPGRADE_FINISHED: all OSDs are running squid or later but require_osd_release < squid 2024-09-21T09:40:00.275 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:00 smithi064 ceph-mon[86709]: all OSDs are running squid or later but require_osd_release < squid 2024-09-21T09:40:00.275 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:00 smithi064 ceph-mon[86709]: [WRN] PG_DEGRADED: Degraded data redundancy: 33/309 objects degraded (10.680%), 12 pgs degraded 2024-09-21T09:40:00.275 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:00 smithi064 ceph-mon[86709]: pg 1.0 is active+undersized+degraded, acting [0,6] 2024-09-21T09:40:00.275 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:00 smithi064 ceph-mon[86709]: pg 2.7 is active+undersized+degraded, acting [6,2] 2024-09-21T09:40:00.275 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:00 smithi064 ceph-mon[86709]: pg 2.b is active+undersized+degraded, acting [4,5] 2024-09-21T09:40:00.275 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:00 smithi064 ceph-mon[86709]: pg 2.e is active+undersized+degraded, acting [2,3] 2024-09-21T09:40:00.275 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:00 smithi064 ceph-mon[86709]: pg 2.f is active+undersized+degraded, acting [4,0] 2024-09-21T09:40:00.275 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:00 smithi064 ceph-mon[86709]: pg 2.1c is active+undersized+degraded, acting [5,2] 2024-09-21T09:40:00.275 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:00 smithi064 ceph-mon[86709]: pg 2.1d is active+undersized+degraded, acting [6,0] 2024-09-21T09:40:00.275 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:00 smithi064 ceph-mon[86709]: pg 2.1f is active+undersized+degraded, acting [0,4] 2024-09-21T09:40:00.275 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:00 smithi064 ceph-mon[86709]: pg 4.3 is active+undersized+degraded, acting [0,5] 2024-09-21T09:40:00.275 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:00 smithi064 ceph-mon[86709]: pg 5.4 is active+undersized+degraded, acting [NONE,2,5] 2024-09-21T09:40:00.275 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:00 smithi064 ceph-mon[86709]: pg 5.9 is active+undersized+degraded, acting [NONE,6,1] 2024-09-21T09:40:00.275 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:00 smithi064 ceph-mon[86709]: pg 5.d is active+undersized+degraded, acting [2,NONE,5] 2024-09-21T09:40:00.276 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:00 smithi064 ceph-mon[86709]: osdmap e576: 8 total, 7 up, 8 in 2024-09-21T09:40:00.276 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:00 smithi064 ceph-mon[86709]: from='osd.7 [v2:172.21.15.64:6824/927270672,v1:172.21.15.64:6825/927270672]' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-09-21T09:40:00.276 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:00 smithi064 ceph-mon[86709]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-09-21T09:40:00.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:00 smithi022 ceph-mon[98615]: pgmap v466: 121 pgs: 38 active+undersized, 12 active+undersized+degraded, 71 active+clean; 149 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 9.2 MiB/s rd, 664 KiB/s wr, 12 op/s; 33/309 objects degraded (10.680%) 2024-09-21T09:40:00.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:00 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:00.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:00 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:00.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:00 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:00.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:00 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:00.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:00 smithi022 ceph-mon[98615]: Health detail: HEALTH_WARN 1 osds down; all OSDs are running squid or later but require_osd_release < squid; Degraded data redundancy: 33/309 objects degraded (10.680%), 12 pgs degraded 2024-09-21T09:40:00.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:00 smithi022 ceph-mon[98615]: [WRN] OSD_DOWN: 1 osds down 2024-09-21T09:40:00.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:00 smithi022 ceph-mon[98615]: osd.7 (root=default,host=smithi064) is down 2024-09-21T09:40:00.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:00 smithi022 ceph-mon[98615]: [WRN] OSD_UPGRADE_FINISHED: all OSDs are running squid or later but require_osd_release < squid 2024-09-21T09:40:00.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:00 smithi022 ceph-mon[98615]: all OSDs are running squid or later but require_osd_release < squid 2024-09-21T09:40:00.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:00 smithi022 ceph-mon[98615]: [WRN] PG_DEGRADED: Degraded data redundancy: 33/309 objects degraded (10.680%), 12 pgs degraded 2024-09-21T09:40:00.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:00 smithi022 ceph-mon[98615]: pg 1.0 is active+undersized+degraded, acting [0,6] 2024-09-21T09:40:00.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:00 smithi022 ceph-mon[98615]: pg 2.7 is active+undersized+degraded, acting [6,2] 2024-09-21T09:40:00.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:00 smithi022 ceph-mon[98615]: pg 2.b is active+undersized+degraded, acting [4,5] 2024-09-21T09:40:00.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:00 smithi022 ceph-mon[98615]: pg 2.e is active+undersized+degraded, acting [2,3] 2024-09-21T09:40:00.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:00 smithi022 ceph-mon[98615]: pg 2.f is active+undersized+degraded, acting [4,0] 2024-09-21T09:40:00.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:00 smithi022 ceph-mon[98615]: pg 2.1c is active+undersized+degraded, acting [5,2] 2024-09-21T09:40:00.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:00 smithi022 ceph-mon[98615]: pg 2.1d is active+undersized+degraded, acting [6,0] 2024-09-21T09:40:00.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:00 smithi022 ceph-mon[98615]: pg 2.1f is active+undersized+degraded, acting [0,4] 2024-09-21T09:40:00.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:00 smithi022 ceph-mon[98615]: pg 4.3 is active+undersized+degraded, acting [0,5] 2024-09-21T09:40:00.560 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:00 smithi022 ceph-mon[98615]: pg 5.4 is active+undersized+degraded, acting [NONE,2,5] 2024-09-21T09:40:00.560 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:00 smithi022 ceph-mon[98615]: pg 5.9 is active+undersized+degraded, acting [NONE,6,1] 2024-09-21T09:40:00.560 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:00 smithi022 ceph-mon[98615]: pg 5.d is active+undersized+degraded, acting [2,NONE,5] 2024-09-21T09:40:00.560 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:00 smithi022 ceph-mon[98615]: osdmap e576: 8 total, 7 up, 8 in 2024-09-21T09:40:00.560 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:00 smithi022 ceph-mon[98615]: from='osd.7 [v2:172.21.15.64:6824/927270672,v1:172.21.15.64:6825/927270672]' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-09-21T09:40:00.560 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:00 smithi022 ceph-mon[98615]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-09-21T09:40:00.560 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:00 smithi022 ceph-mon[96736]: pgmap v466: 121 pgs: 38 active+undersized, 12 active+undersized+degraded, 71 active+clean; 149 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 9.2 MiB/s rd, 664 KiB/s wr, 12 op/s; 33/309 objects degraded (10.680%) 2024-09-21T09:40:00.560 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:00 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:00.560 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:00 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:00.560 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:00 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:00.560 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:00 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:00.560 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:00 smithi022 ceph-mon[96736]: Health detail: HEALTH_WARN 1 osds down; all OSDs are running squid or later but require_osd_release < squid; Degraded data redundancy: 33/309 objects degraded (10.680%), 12 pgs degraded 2024-09-21T09:40:00.560 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:00 smithi022 ceph-mon[96736]: [WRN] OSD_DOWN: 1 osds down 2024-09-21T09:40:00.560 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:00 smithi022 ceph-mon[96736]: osd.7 (root=default,host=smithi064) is down 2024-09-21T09:40:00.561 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:00 smithi022 ceph-mon[96736]: [WRN] OSD_UPGRADE_FINISHED: all OSDs are running squid or later but require_osd_release < squid 2024-09-21T09:40:00.561 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:00 smithi022 ceph-mon[96736]: all OSDs are running squid or later but require_osd_release < squid 2024-09-21T09:40:00.561 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:00 smithi022 ceph-mon[96736]: [WRN] PG_DEGRADED: Degraded data redundancy: 33/309 objects degraded (10.680%), 12 pgs degraded 2024-09-21T09:40:00.561 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:00 smithi022 ceph-mon[96736]: pg 1.0 is active+undersized+degraded, acting [0,6] 2024-09-21T09:40:00.561 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:00 smithi022 ceph-mon[96736]: pg 2.7 is active+undersized+degraded, acting [6,2] 2024-09-21T09:40:00.561 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:00 smithi022 ceph-mon[96736]: pg 2.b is active+undersized+degraded, acting [4,5] 2024-09-21T09:40:00.561 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:00 smithi022 ceph-mon[96736]: pg 2.e is active+undersized+degraded, acting [2,3] 2024-09-21T09:40:00.561 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:00 smithi022 ceph-mon[96736]: pg 2.f is active+undersized+degraded, acting [4,0] 2024-09-21T09:40:00.561 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:00 smithi022 ceph-mon[96736]: pg 2.1c is active+undersized+degraded, acting [5,2] 2024-09-21T09:40:00.561 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:00 smithi022 ceph-mon[96736]: pg 2.1d is active+undersized+degraded, acting [6,0] 2024-09-21T09:40:00.561 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:00 smithi022 ceph-mon[96736]: pg 2.1f is active+undersized+degraded, acting [0,4] 2024-09-21T09:40:00.561 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:00 smithi022 ceph-mon[96736]: pg 4.3 is active+undersized+degraded, acting [0,5] 2024-09-21T09:40:00.561 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:00 smithi022 ceph-mon[96736]: pg 5.4 is active+undersized+degraded, acting [NONE,2,5] 2024-09-21T09:40:00.561 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:00 smithi022 ceph-mon[96736]: pg 5.9 is active+undersized+degraded, acting [NONE,6,1] 2024-09-21T09:40:00.561 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:00 smithi022 ceph-mon[96736]: pg 5.d is active+undersized+degraded, acting [2,NONE,5] 2024-09-21T09:40:00.562 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:00 smithi022 ceph-mon[96736]: osdmap e576: 8 total, 7 up, 8 in 2024-09-21T09:40:00.562 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:00 smithi022 ceph-mon[96736]: from='osd.7 [v2:172.21.15.64:6824/927270672,v1:172.21.15.64:6825/927270672]' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-09-21T09:40:00.562 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:00 smithi022 ceph-mon[96736]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-09-21T09:40:01.022 INFO:tasks.rados.rados.0.smithi022.stdout:2540: done (3 left) 2024-09-21T09:40:01.023 INFO:tasks.rados.rados.0.smithi022.stdout:2532: finishing write tid 5 to smithi02284960-47 2024-09-21T09:40:01.024 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 47 v 1412 (ObjNum 805 snap 255 seq_num 805) dirty exists 2024-09-21T09:40:01.024 INFO:tasks.rados.rados.0.smithi022.stdout:2532: left oid 47 (ObjNum 805 snap 255 seq_num 805) 2024-09-21T09:40:01.024 INFO:tasks.rados.rados.0.smithi022.stdout:2538: expect (ObjNum 753 snap 231 seq_num 753) 2024-09-21T09:40:01.123 INFO:tasks.rados.rados.0.smithi022.stdout:2530: finishing copy_from racing read to smithi02284960-41 2024-09-21T09:40:01.123 INFO:tasks.rados.rados.0.smithi022.stdout:2530: finishing copy_from to smithi02284960-41 2024-09-21T09:40:01.123 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 41 v 1407 (ObjNum 749 snap 227 seq_num 749) dirty exists 2024-09-21T09:40:01.123 INFO:tasks.rados.rados.0.smithi022.stdout:2541: read oid 24 snap 245 2024-09-21T09:40:01.123 INFO:tasks.rados.rados.0.smithi022.stdout:2541: expect (ObjNum 777 snap 242 seq_num 777) 2024-09-21T09:40:01.124 INFO:tasks.rados.rados.0.smithi022.stdout:2530: done (3 left) 2024-09-21T09:40:01.124 INFO:tasks.rados.rados.0.smithi022.stdout:2532: done (2 left) 2024-09-21T09:40:01.124 INFO:tasks.rados.rados.0.smithi022.stdout:2538: done (1 left) 2024-09-21T09:40:01.124 INFO:tasks.rados.rados.0.smithi022.stdout:2542: snap_remove snap 250 2024-09-21T09:40:01.942 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:40:01 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7[106022]: 2024-09-21T09:40:01.496+0000 7f338c04b640 -1 osd.7 565 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T09:40:02.106 INFO:tasks.rados.rados.0.smithi022.stdout:2542: done (1 left) 2024-09-21T09:40:02.106 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 48 current snap is 255 2024-09-21T09:40:02.106 INFO:tasks.rados.rados.0.smithi022.stdout:2543: seq_num 807 ranges {311296=720896,1032192=729088,1761280=73728} 2024-09-21T09:40:02.113 INFO:tasks.rados.rados.0.smithi022.stdout:2543: writing smithi02284960-48 from 311296 to 1032192 tid 1 2024-09-21T09:40:02.119 INFO:tasks.rados.rados.0.smithi022.stdout:2543: writing smithi02284960-48 from 1032192 to 1761280 tid 2 2024-09-21T09:40:02.119 INFO:tasks.rados.rados.0.smithi022.stdout:2543: writing smithi02284960-48 from 1761280 to 1835008 tid 3 2024-09-21T09:40:02.121 INFO:tasks.rados.rados.0.smithi022.stdout:2541: expect (ObjNum 777 snap 242 seq_num 777) 2024-09-21T09:40:02.154 INFO:tasks.rados.rados.0.smithi022.stdout:2543: finishing write tid 1 to smithi02284960-48 2024-09-21T09:40:02.154 INFO:tasks.rados.rados.0.smithi022.stdout:2543: finishing write tid 2 to smithi02284960-48 2024-09-21T09:40:02.154 INFO:tasks.rados.rados.0.smithi022.stdout:2543: finishing write tid 3 to smithi02284960-48 2024-09-21T09:40:02.154 INFO:tasks.rados.rados.0.smithi022.stdout:2543: finishing write tid 4 to smithi02284960-48 2024-09-21T09:40:02.154 INFO:tasks.rados.rados.0.smithi022.stdout:2543: finishing write tid 5 to smithi02284960-48 2024-09-21T09:40:02.154 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 48 v 1527 (ObjNum 807 snap 255 seq_num 807) dirty exists 2024-09-21T09:40:02.154 INFO:tasks.rados.rados.0.smithi022.stdout:2543: left oid 48 (ObjNum 807 snap 255 seq_num 807) 2024-09-21T09:40:02.155 INFO:tasks.rados.rados.0.smithi022.stdout:2541: done (1 left) 2024-09-21T09:40:02.155 INFO:tasks.rados.rados.0.smithi022.stdout:2543: done (0 left) 2024-09-21T09:40:02.155 INFO:tasks.rados.rados.0.smithi022.stdout:2544: rollback oid 6 current snap is 255 2024-09-21T09:40:02.155 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 6 to 247 2024-09-21T09:40:02.155 INFO:tasks.rados.rados.0.smithi022.stdout:2545: read oid 37 snap -1 2024-09-21T09:40:02.155 INFO:tasks.rados.rados.0.smithi022.stdout:2545: expect (ObjNum 672 snap 208 seq_num 672) 2024-09-21T09:40:02.155 INFO:tasks.rados.rados.0.smithi022.stdout:2546: delete oid 39 current snap is 255 2024-09-21T09:40:02.156 INFO:tasks.rados.rados.0.smithi022.stdout:2546: done (2 left) 2024-09-21T09:40:02.156 INFO:tasks.rados.rados.0.smithi022.stdout:2547: snap_remove snap 251 2024-09-21T09:40:02.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:02 smithi022 ceph-mon[98615]: pgmap v469: 121 pgs: 38 active+undersized, 12 active+undersized+degraded, 71 active+clean; 161 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 9.2 MiB/s rd, 3.1 MiB/s wr, 24 op/s; 39/342 objects degraded (11.404%) 2024-09-21T09:40:02.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:02 smithi022 ceph-mon[98615]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-21T09:40:02.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:02 smithi022 ceph-mon[98615]: osdmap e577: 8 total, 7 up, 8 in 2024-09-21T09:40:02.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:02 smithi022 ceph-mon[98615]: from='osd.7 [v2:172.21.15.64:6824/927270672,v1:172.21.15.64:6825/927270672]' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi064", "root=default"]} : dispatch 2024-09-21T09:40:02.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:02 smithi022 ceph-mon[98615]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi064", "root=default"]} : dispatch 2024-09-21T09:40:02.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:02 smithi022 ceph-mon[98615]: from='osd.7 ' entity='osd.7' 2024-09-21T09:40:02.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:02 smithi022 ceph-mon[96736]: pgmap v469: 121 pgs: 38 active+undersized, 12 active+undersized+degraded, 71 active+clean; 161 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 9.2 MiB/s rd, 3.1 MiB/s wr, 24 op/s; 39/342 objects degraded (11.404%) 2024-09-21T09:40:02.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:02 smithi022 ceph-mon[96736]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-21T09:40:02.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:02 smithi022 ceph-mon[96736]: osdmap e577: 8 total, 7 up, 8 in 2024-09-21T09:40:02.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:02 smithi022 ceph-mon[96736]: from='osd.7 [v2:172.21.15.64:6824/927270672,v1:172.21.15.64:6825/927270672]' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi064", "root=default"]} : dispatch 2024-09-21T09:40:02.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:02 smithi022 ceph-mon[96736]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi064", "root=default"]} : dispatch 2024-09-21T09:40:02.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:02 smithi022 ceph-mon[96736]: from='osd.7 ' entity='osd.7' 2024-09-21T09:40:02.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:02 smithi064 ceph-mon[86709]: pgmap v469: 121 pgs: 38 active+undersized, 12 active+undersized+degraded, 71 active+clean; 161 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 9.2 MiB/s rd, 3.1 MiB/s wr, 24 op/s; 39/342 objects degraded (11.404%) 2024-09-21T09:40:02.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:02 smithi064 ceph-mon[86709]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-21T09:40:02.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:02 smithi064 ceph-mon[86709]: osdmap e577: 8 total, 7 up, 8 in 2024-09-21T09:40:02.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:02 smithi064 ceph-mon[86709]: from='osd.7 [v2:172.21.15.64:6824/927270672,v1:172.21.15.64:6825/927270672]' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi064", "root=default"]} : dispatch 2024-09-21T09:40:02.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:02 smithi064 ceph-mon[86709]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi064", "root=default"]} : dispatch 2024-09-21T09:40:02.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:02 smithi064 ceph-mon[86709]: from='osd.7 ' entity='osd.7' 2024-09-21T09:40:03.111 INFO:tasks.rados.rados.0.smithi022.stdout:2544: finishing rollback tid 0 to smithi02284960-6 2024-09-21T09:40:03.111 INFO:tasks.rados.rados.0.smithi022.stdout:2544: finishing rollback tid 1 to smithi02284960-6 2024-09-21T09:40:03.111 INFO:tasks.rados.rados.0.smithi022.stdout:2544: finishing rollback tid 2 to smithi02284960-6 2024-09-21T09:40:03.111 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 6 v 1530 (ObjNum 752 snap 230 seq_num 752) dirty exists 2024-09-21T09:40:03.111 INFO:tasks.rados.rados.0.smithi022.stdout:2544: done (2 left) 2024-09-21T09:40:03.111 INFO:tasks.rados.rados.0.smithi022.stdout:2547: done (1 left) 2024-09-21T09:40:03.111 INFO:tasks.rados.rados.0.smithi022.stdout:2548: rollback oid 28 current snap is 255 2024-09-21T09:40:03.111 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 28 to 252 2024-09-21T09:40:03.111 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 15 current snap is 255 2024-09-21T09:40:03.111 INFO:tasks.rados.rados.0.smithi022.stdout:2549: seq_num 809 ranges {4325376=753664,5079040=499712,5578752=303104} 2024-09-21T09:40:03.141 INFO:tasks.rados.rados.0.smithi022.stdout:2549: writing smithi02284960-15 from 4325376 to 5079040 tid 1 2024-09-21T09:40:03.144 INFO:tasks.rados.rados.0.smithi022.stdout:2549: writing smithi02284960-15 from 5079040 to 5578752 tid 2 2024-09-21T09:40:03.146 INFO:tasks.rados.rados.0.smithi022.stdout:2549: writing smithi02284960-15 from 5578752 to 5881856 tid 3 2024-09-21T09:40:03.146 INFO:tasks.rados.rados.0.smithi022.stdout:2548: finishing rollback tid 0 to smithi02284960-28 2024-09-21T09:40:03.146 INFO:tasks.rados.rados.0.smithi022.stdout:2548: finishing rollback tid 1 to smithi02284960-28 2024-09-21T09:40:03.146 INFO:tasks.rados.rados.0.smithi022.stdout:2548: finishing rollback tid 2 to smithi02284960-28 2024-09-21T09:40:03.146 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 28 v 1271 (ObjNum 753 snap 231 seq_num 753) dirty exists 2024-09-21T09:40:03.146 INFO:tasks.rados.rados.0.smithi022.stdout:2548: done (2 left) 2024-09-21T09:40:03.146 INFO:tasks.rados.rados.0.smithi022.stdout:2550: snap_create 2024-09-21T09:40:03.160 INFO:tasks.rados.rados.0.smithi022.stdout:2545: expect (ObjNum 672 snap 208 seq_num 672) 2024-09-21T09:40:03.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:03 smithi064 ceph-mon[86709]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T09:40:03.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:03 smithi064 ceph-mon[86709]: osd.7 [v2:172.21.15.64:6824/927270672,v1:172.21.15.64:6825/927270672] boot 2024-09-21T09:40:03.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:03 smithi064 ceph-mon[86709]: osdmap e578: 8 total, 8 up, 8 in 2024-09-21T09:40:03.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:03 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-09-21T09:40:03.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:03 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:03.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:03 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:03.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:03 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:40:03.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:03 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:40:03.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:03 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:03.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:03 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:40:03.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:03 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:40:03.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:03 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:40:03.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:03 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:40:03.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:03 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:40:03.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:03 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:03.193 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:03 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.0"} : dispatch 2024-09-21T09:40:03.193 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:03 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-09-21T09:40:03.193 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:03 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.1"} : dispatch 2024-09-21T09:40:03.193 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:03 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-09-21T09:40:03.193 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:03 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.2"} : dispatch 2024-09-21T09:40:03.193 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:03 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-09-21T09:40:03.193 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:03 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.3"} : dispatch 2024-09-21T09:40:03.193 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:03 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-09-21T09:40:03.193 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:03 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.4"} : dispatch 2024-09-21T09:40:03.193 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:03 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-09-21T09:40:03.193 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:03 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.5"} : dispatch 2024-09-21T09:40:03.193 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:03 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-09-21T09:40:03.213 INFO:tasks.rados.rados.0.smithi022.stdout:2549: finishing write tid 1 to smithi02284960-15 2024-09-21T09:40:03.213 INFO:tasks.rados.rados.0.smithi022.stdout:2549: finishing write tid 2 to smithi02284960-15 2024-09-21T09:40:03.213 INFO:tasks.rados.rados.0.smithi022.stdout:2549: finishing write tid 3 to smithi02284960-15 2024-09-21T09:40:03.213 INFO:tasks.rados.rados.0.smithi022.stdout:2549: finishing write tid 4 to smithi02284960-15 2024-09-21T09:40:03.213 INFO:tasks.rados.rados.0.smithi022.stdout:2549: finishing write tid 5 to smithi02284960-15 2024-09-21T09:40:03.213 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 15 v 1461 (ObjNum 809 snap 255 seq_num 809) dirty exists 2024-09-21T09:40:03.213 INFO:tasks.rados.rados.0.smithi022.stdout:2549: left oid 15 (ObjNum 809 snap 255 seq_num 809) 2024-09-21T09:40:03.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:03 smithi022 ceph-mon[98615]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T09:40:03.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:03 smithi022 ceph-mon[98615]: osd.7 [v2:172.21.15.64:6824/927270672,v1:172.21.15.64:6825/927270672] boot 2024-09-21T09:40:03.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:03 smithi022 ceph-mon[98615]: osdmap e578: 8 total, 8 up, 8 in 2024-09-21T09:40:03.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:03 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-09-21T09:40:03.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:03 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:03.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:03 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:03.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:03 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:40:03.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:03 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:40:03.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:03 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:03.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:03 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:40:03.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:03 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:40:03.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:03 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:40:03.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:03 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:40:03.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:03 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:40:03.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:03 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:03.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:03 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.0"} : dispatch 2024-09-21T09:40:03.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:03 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-09-21T09:40:03.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:03 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.1"} : dispatch 2024-09-21T09:40:03.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:03 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-09-21T09:40:03.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:03 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.2"} : dispatch 2024-09-21T09:40:03.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:03 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-09-21T09:40:03.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:03 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.3"} : dispatch 2024-09-21T09:40:03.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:03 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-09-21T09:40:03.310 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:03 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.4"} : dispatch 2024-09-21T09:40:03.310 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:03 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-09-21T09:40:03.310 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:03 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.5"} : dispatch 2024-09-21T09:40:03.310 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:03 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-09-21T09:40:03.310 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:03 smithi022 ceph-mon[96736]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T09:40:03.310 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:03 smithi022 ceph-mon[96736]: osd.7 [v2:172.21.15.64:6824/927270672,v1:172.21.15.64:6825/927270672] boot 2024-09-21T09:40:03.310 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:03 smithi022 ceph-mon[96736]: osdmap e578: 8 total, 8 up, 8 in 2024-09-21T09:40:03.310 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:03 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-09-21T09:40:03.310 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:03 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:03.310 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:03 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:03.310 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:03 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:40:03.311 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:03 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:40:03.311 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:03 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:03.311 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:03 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:40:03.311 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:03 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:40:03.311 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:03 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:40:03.311 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:03 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:40:03.311 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:03 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:40:03.311 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:03 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:03.311 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:03 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.0"} : dispatch 2024-09-21T09:40:03.311 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:03 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-09-21T09:40:03.311 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:03 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.1"} : dispatch 2024-09-21T09:40:03.311 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:03 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-09-21T09:40:03.311 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:03 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.2"} : dispatch 2024-09-21T09:40:03.311 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:03 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-09-21T09:40:03.312 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:03 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.3"} : dispatch 2024-09-21T09:40:03.312 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:03 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-09-21T09:40:03.312 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:03 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.4"} : dispatch 2024-09-21T09:40:03.312 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:03 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-09-21T09:40:03.312 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:03 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.5"} : dispatch 2024-09-21T09:40:03.312 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:03 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-09-21T09:40:04.111 INFO:tasks.rados.rados.0.smithi022.stdout:2545: done (2 left) 2024-09-21T09:40:04.111 INFO:tasks.rados.rados.0.smithi022.stdout:2549: done (1 left) 2024-09-21T09:40:04.111 INFO:tasks.rados.rados.0.smithi022.stdout:2550: done (0 left) 2024-09-21T09:40:04.111 INFO:tasks.rados.rados.0.smithi022.stdout:2551: read oid 10 snap -1 2024-09-21T09:40:04.111 INFO:tasks.rados.rados.0.smithi022.stdout:2551: expect deleted 2024-09-21T09:40:04.111 INFO:tasks.rados.rados.0.smithi022.stdout:2552: snap_create 2024-09-21T09:40:04.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:04 smithi022 ceph-mon[96736]: pgmap v472: 121 pgs: 4 peering, 1 active+clean+snaptrim, 35 active+undersized, 11 active+undersized+degraded, 70 active+clean; 161 MiB data, 945 MiB used, 714 GiB / 715 GiB avail; 43 KiB/s rd, 3.1 MiB/s wr, 22 op/s; 38/345 objects degraded (11.014%) 2024-09-21T09:40:04.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:04 smithi022 ceph-mon[96736]: Upgrade: Setting container_image for all osd 2024-09-21T09:40:04.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:04 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.6"} : dispatch 2024-09-21T09:40:04.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:04 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-09-21T09:40:04.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:04 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.7"} : dispatch 2024-09-21T09:40:04.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:04 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-09-21T09:40:04.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:04 smithi022 ceph-mon[96736]: Upgrade: Setting require_osd_release to 19 squid 2024-09-21T09:40:04.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:04 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd require-osd-release", "release": "squid"} : dispatch 2024-09-21T09:40:04.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:04 smithi022 ceph-mon[96736]: Health check update: Degraded data redundancy: 38/345 objects degraded (11.014%), 11 pgs degraded (PG_DEGRADED) 2024-09-21T09:40:04.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:04 smithi022 ceph-mon[96736]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-09-21T09:40:04.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:04 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-09-21T09:40:04.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:04 smithi022 ceph-mon[96736]: osdmap e579: 8 total, 8 up, 8 in 2024-09-21T09:40:04.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:04 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "mds ok-to-stop", "ids": ["a"]} : dispatch 2024-09-21T09:40:04.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:04 smithi022 ceph-mon[98615]: pgmap v472: 121 pgs: 4 peering, 1 active+clean+snaptrim, 35 active+undersized, 11 active+undersized+degraded, 70 active+clean; 161 MiB data, 945 MiB used, 714 GiB / 715 GiB avail; 43 KiB/s rd, 3.1 MiB/s wr, 22 op/s; 38/345 objects degraded (11.014%) 2024-09-21T09:40:04.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:04 smithi022 ceph-mon[98615]: Upgrade: Setting container_image for all osd 2024-09-21T09:40:04.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:04 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.6"} : dispatch 2024-09-21T09:40:04.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:04 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-09-21T09:40:04.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:04 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.7"} : dispatch 2024-09-21T09:40:04.310 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:04 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-09-21T09:40:04.310 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:04 smithi022 ceph-mon[98615]: Upgrade: Setting require_osd_release to 19 squid 2024-09-21T09:40:04.310 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:04 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd require-osd-release", "release": "squid"} : dispatch 2024-09-21T09:40:04.310 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:04 smithi022 ceph-mon[98615]: Health check update: Degraded data redundancy: 38/345 objects degraded (11.014%), 11 pgs degraded (PG_DEGRADED) 2024-09-21T09:40:04.310 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:04 smithi022 ceph-mon[98615]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-09-21T09:40:04.310 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:04 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-09-21T09:40:04.310 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:04 smithi022 ceph-mon[98615]: osdmap e579: 8 total, 8 up, 8 in 2024-09-21T09:40:04.310 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:04 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "mds ok-to-stop", "ids": ["a"]} : dispatch 2024-09-21T09:40:04.377 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:04 smithi064 ceph-mon[86709]: pgmap v472: 121 pgs: 4 peering, 1 active+clean+snaptrim, 35 active+undersized, 11 active+undersized+degraded, 70 active+clean; 161 MiB data, 945 MiB used, 714 GiB / 715 GiB avail; 43 KiB/s rd, 3.1 MiB/s wr, 22 op/s; 38/345 objects degraded (11.014%) 2024-09-21T09:40:04.377 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:04 smithi064 ceph-mon[86709]: Upgrade: Setting container_image for all osd 2024-09-21T09:40:04.377 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:04 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.6"} : dispatch 2024-09-21T09:40:04.377 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:04 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-09-21T09:40:04.377 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:04 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.7"} : dispatch 2024-09-21T09:40:04.377 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:04 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-09-21T09:40:04.377 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:04 smithi064 ceph-mon[86709]: Upgrade: Setting require_osd_release to 19 squid 2024-09-21T09:40:04.377 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:04 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd require-osd-release", "release": "squid"} : dispatch 2024-09-21T09:40:04.377 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:04 smithi064 ceph-mon[86709]: Health check update: Degraded data redundancy: 38/345 objects degraded (11.014%), 11 pgs degraded (PG_DEGRADED) 2024-09-21T09:40:04.377 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:04 smithi064 ceph-mon[86709]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-09-21T09:40:04.377 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:04 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-09-21T09:40:04.378 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:04 smithi064 ceph-mon[86709]: osdmap e579: 8 total, 8 up, 8 in 2024-09-21T09:40:04.378 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:04 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "mds ok-to-stop", "ids": ["a"]} : dispatch 2024-09-21T09:40:05.113 INFO:tasks.rados.rados.0.smithi022.stdout:2551: done (1 left) 2024-09-21T09:40:05.114 INFO:tasks.rados.rados.0.smithi022.stdout:2552: done (0 left) 2024-09-21T09:40:05.114 INFO:tasks.rados.rados.0.smithi022.stdout:2553: read oid 28 snap 246 2024-09-21T09:40:05.114 INFO:tasks.rados.rados.0.smithi022.stdout:2553: expect (ObjNum 753 snap 231 seq_num 753) 2024-09-21T09:40:05.114 INFO:tasks.rados.rados.0.smithi022.stdout:2554: snap_create 2024-09-21T09:40:05.120 INFO:tasks.rados.rados.0.smithi022.stdout:2553: expect (ObjNum 753 snap 231 seq_num 753) 2024-09-21T09:40:05.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:05 smithi022 ceph-mon[98615]: Upgrade: It appears safe to stop mds.a 2024-09-21T09:40:05.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:05 smithi022 ceph-mon[98615]: osdmap e580: 8 total, 8 up, 8 in 2024-09-21T09:40:05.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:05 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:05.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:05 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-21T09:40:05.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:05 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:40:05.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:05 smithi022 ceph-mon[96736]: Upgrade: It appears safe to stop mds.a 2024-09-21T09:40:05.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:05 smithi022 ceph-mon[96736]: osdmap e580: 8 total, 8 up, 8 in 2024-09-21T09:40:05.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:05 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:05.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:05 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-21T09:40:05.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:05 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:40:05.389 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:05 smithi064 ceph-mon[86709]: Upgrade: It appears safe to stop mds.a 2024-09-21T09:40:05.389 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:05 smithi064 ceph-mon[86709]: osdmap e580: 8 total, 8 up, 8 in 2024-09-21T09:40:05.389 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:05 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:05.389 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:05 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-21T09:40:05.389 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:05 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:40:06.024 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:40:05 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:40:05] "GET /metrics HTTP/1.1" 200 36363 "" "Prometheus/2.51.0" 2024-09-21T09:40:06.117 INFO:tasks.rados.rados.0.smithi022.stdout:2553: done (1 left) 2024-09-21T09:40:06.117 INFO:tasks.rados.rados.0.smithi022.stdout:2554: done (0 left) 2024-09-21T09:40:06.117 INFO:tasks.rados.rados.0.smithi022.stdout:2555: rollback oid 50 current snap is 258 2024-09-21T09:40:06.117 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 50 to 239 2024-09-21T09:40:06.117 INFO:tasks.rados.rados.0.smithi022.stdout:2556: read oid 14 snap -1 2024-09-21T09:40:06.117 INFO:tasks.rados.rados.0.smithi022.stdout:2556: expect deleted 2024-09-21T09:40:06.117 INFO:tasks.rados.rados.0.smithi022.stdout:2557: setattr oid 45 current snap is 258 2024-09-21T09:40:06.118 INFO:tasks.rados.rados.0.smithi022.stdout:2556: done (2 left) 2024-09-21T09:40:06.118 INFO:tasks.rados.rados.0.smithi022.stdout:2558: read oid 24 snap 248 2024-09-21T09:40:06.118 INFO:tasks.rados.rados.0.smithi022.stdout:2558: expect (ObjNum 783 snap 246 seq_num 783) 2024-09-21T09:40:06.118 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 4 current snap is 258 2024-09-21T09:40:06.118 INFO:tasks.rados.rados.0.smithi022.stdout:2559: seq_num 810 ranges {13066240=770048,13836288=778240,14614528=24576} 2024-09-21T09:40:06.194 INFO:tasks.rados.rados.0.smithi022.stdout:2559: writing smithi02284960-4 from 13066240 to 13836288 tid 1 2024-09-21T09:40:06.198 INFO:tasks.rados.rados.0.smithi022.stdout:2559: writing smithi02284960-4 from 13836288 to 14614528 tid 2 2024-09-21T09:40:06.198 INFO:tasks.rados.rados.0.smithi022.stdout:2559: writing smithi02284960-4 from 14614528 to 14639104 tid 3 2024-09-21T09:40:06.198 INFO:tasks.rados.rados.0.smithi022.stdout:2560: snap_create 2024-09-21T09:40:06.199 INFO:tasks.rados.rados.0.smithi022.stdout:2555: finishing rollback tid 0 to smithi02284960-50 2024-09-21T09:40:06.199 INFO:tasks.rados.rados.0.smithi022.stdout:2555: finishing rollback tid 1 to smithi02284960-50 2024-09-21T09:40:06.199 INFO:tasks.rados.rados.0.smithi022.stdout:2555: finishing rollback tid 2 to smithi02284960-50 2024-09-21T09:40:06.199 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 50 v 1292 (ObjNum 701 snap 216 seq_num 701) dirty exists 2024-09-21T09:40:06.199 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 45 v 1293 (ObjNum 289 snap 0 seq_num 0) dirty exists 2024-09-21T09:40:06.199 INFO:tasks.rados.rados.0.smithi022.stdout:2558: expect (ObjNum 783 snap 246 seq_num 783) 2024-09-21T09:40:06.274 INFO:tasks.rados.rados.0.smithi022.stdout:2559: finishing write tid 1 to smithi02284960-4 2024-09-21T09:40:06.274 INFO:tasks.rados.rados.0.smithi022.stdout:2559: finishing write tid 2 to smithi02284960-4 2024-09-21T09:40:06.274 INFO:tasks.rados.rados.0.smithi022.stdout:2559: finishing write tid 3 to smithi02284960-4 2024-09-21T09:40:06.274 INFO:tasks.rados.rados.0.smithi022.stdout:2559: finishing write tid 4 to smithi02284960-4 2024-09-21T09:40:06.274 INFO:tasks.rados.rados.0.smithi022.stdout:2559: finishing write tid 5 to smithi02284960-4 2024-09-21T09:40:06.274 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 4 v 1393 (ObjNum 810 snap 258 seq_num 810) dirty exists 2024-09-21T09:40:06.274 INFO:tasks.rados.rados.0.smithi022.stdout:2559: left oid 4 (ObjNum 810 snap 258 seq_num 810) 2024-09-21T09:40:06.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:06 smithi022 ceph-mon[98615]: pgmap v475: 121 pgs: 2 activating, 9 peering, 1 active+clean+snaptrim, 21 active+undersized, 9 active+undersized+degraded, 79 active+clean; 156 MiB data, 971 MiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 0 B/s wr, 2 op/s; 35/342 objects degraded (10.234%) 2024-09-21T09:40:06.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:06 smithi022 ceph-mon[98615]: Upgrade: Updating mds.a 2024-09-21T09:40:06.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:06 smithi022 ceph-mon[98615]: Deploying daemon mds.a on smithi022 2024-09-21T09:40:06.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:06 smithi022 ceph-mon[98615]: osdmap e581: 8 total, 8 up, 8 in 2024-09-21T09:40:06.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:06 smithi022 ceph-mon[96736]: pgmap v475: 121 pgs: 2 activating, 9 peering, 1 active+clean+snaptrim, 21 active+undersized, 9 active+undersized+degraded, 79 active+clean; 156 MiB data, 971 MiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 0 B/s wr, 2 op/s; 35/342 objects degraded (10.234%) 2024-09-21T09:40:06.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:06 smithi022 ceph-mon[96736]: Upgrade: Updating mds.a 2024-09-21T09:40:06.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:06 smithi022 ceph-mon[96736]: Deploying daemon mds.a on smithi022 2024-09-21T09:40:06.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:06 smithi022 ceph-mon[96736]: osdmap e581: 8 total, 8 up, 8 in 2024-09-21T09:40:06.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:06 smithi064 ceph-mon[86709]: pgmap v475: 121 pgs: 2 activating, 9 peering, 1 active+clean+snaptrim, 21 active+undersized, 9 active+undersized+degraded, 79 active+clean; 156 MiB data, 971 MiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 0 B/s wr, 2 op/s; 35/342 objects degraded (10.234%) 2024-09-21T09:40:06.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:06 smithi064 ceph-mon[86709]: Upgrade: Updating mds.a 2024-09-21T09:40:06.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:06 smithi064 ceph-mon[86709]: Deploying daemon mds.a on smithi022 2024-09-21T09:40:06.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:06 smithi064 ceph-mon[86709]: osdmap e581: 8 total, 8 up, 8 in 2024-09-21T09:40:06.800 INFO:journalctl@ceph.mds.a.smithi022.stdout:Sep 21 09:40:06 smithi022 systemd[1]: Stopping Ceph mds.a for d65e801e-77fa-11ef-baf6-efdc52797490... 2024-09-21T09:40:07.058 INFO:journalctl@ceph.mds.a.smithi022.stdout:Sep 21 09:40:06 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mds-a[68145]: 2024-09-21T09:40:06.797+0000 7fb30c51e640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mds -n mds.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T09:40:07.058 INFO:journalctl@ceph.mds.a.smithi022.stdout:Sep 21 09:40:06 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mds-a[68145]: 2024-09-21T09:40:06.797+0000 7fb30c51e640 -1 mds.a *** got signal Terminated *** 2024-09-21T09:40:07.118 INFO:tasks.rados.rados.0.smithi022.stdout:2555: done (4 left) 2024-09-21T09:40:07.118 INFO:tasks.rados.rados.0.smithi022.stdout:2557: done (3 left) 2024-09-21T09:40:07.119 INFO:tasks.rados.rados.0.smithi022.stdout:2558: done (2 left) 2024-09-21T09:40:07.119 INFO:tasks.rados.rados.0.smithi022.stdout:2559: done (1 left) 2024-09-21T09:40:07.119 INFO:tasks.rados.rados.0.smithi022.stdout:2560: done (0 left) 2024-09-21T09:40:07.119 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 9 current snap is 259 2024-09-21T09:40:07.119 INFO:tasks.rados.rados.0.smithi022.stdout:2561: seq_num 811 ranges {0=786432,786432=663552,1449984=139264} 2024-09-21T09:40:07.123 INFO:tasks.rados.rados.0.smithi022.stdout:2561: writing smithi02284960-9 from 0 to 786432 tid 1 2024-09-21T09:40:07.127 INFO:tasks.rados.rados.0.smithi022.stdout:2561: writing smithi02284960-9 from 786432 to 1449984 tid 2 2024-09-21T09:40:07.128 INFO:tasks.rados.rados.0.smithi022.stdout:2561: writing smithi02284960-9 from 1449984 to 1589248 tid 3 2024-09-21T09:40:07.128 INFO:tasks.rados.rados.0.smithi022.stdout:2562: read oid 18 snap -1 2024-09-21T09:40:07.128 INFO:tasks.rados.rados.0.smithi022.stdout:2562: expect deleted 2024-09-21T09:40:07.128 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 21 current snap is 259 2024-09-21T09:40:07.128 INFO:tasks.rados.rados.0.smithi022.stdout:2563: seq_num 812 ranges {5398528=802816,6201344=548864,6750208=253952} 2024-09-21T09:40:07.162 INFO:tasks.rados.rados.0.smithi022.stdout:2563: writing smithi02284960-21 from 5398528 to 6201344 tid 1 2024-09-21T09:40:07.165 INFO:tasks.rados.rados.0.smithi022.stdout:2563: writing smithi02284960-21 from 6201344 to 6750208 tid 2 2024-09-21T09:40:07.167 INFO:tasks.rados.rados.0.smithi022.stdout:2563: writing smithi02284960-21 from 6750208 to 7004160 tid 3 2024-09-21T09:40:07.167 INFO:tasks.rados.rados.0.smithi022.stdout:2564: snap_create 2024-09-21T09:40:07.167 INFO:tasks.rados.rados.0.smithi022.stdout:2561: finishing write tid 1 to smithi02284960-9 2024-09-21T09:40:07.167 INFO:tasks.rados.rados.0.smithi022.stdout:2561: finishing write tid 2 to smithi02284960-9 2024-09-21T09:40:07.167 INFO:tasks.rados.rados.0.smithi022.stdout:2561: finishing write tid 3 to smithi02284960-9 2024-09-21T09:40:07.167 INFO:tasks.rados.rados.0.smithi022.stdout:2561: finishing write tid 4 to smithi02284960-9 2024-09-21T09:40:07.167 INFO:tasks.rados.rados.0.smithi022.stdout:2561: finishing write tid 5 to smithi02284960-9 2024-09-21T09:40:07.167 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 9 v 1397 (ObjNum 811 snap 259 seq_num 811) dirty exists 2024-09-21T09:40:07.167 INFO:tasks.rados.rados.0.smithi022.stdout:2561: left oid 9 (ObjNum 811 snap 259 seq_num 811) 2024-09-21T09:40:07.168 INFO:tasks.rados.rados.0.smithi022.stdout:2563: finishing write tid 1 to smithi02284960-21 2024-09-21T09:40:07.169 INFO:tasks.rados.rados.0.smithi022.stdout:2563: finishing write tid 2 to smithi02284960-21 2024-09-21T09:40:07.170 INFO:tasks.rados.rados.0.smithi022.stdout:2563: finishing write tid 3 to smithi02284960-21 2024-09-21T09:40:07.170 INFO:tasks.rados.rados.0.smithi022.stdout:2563: finishing write tid 4 to smithi02284960-21 2024-09-21T09:40:07.171 INFO:tasks.rados.rados.0.smithi022.stdout:2563: finishing write tid 5 to smithi02284960-21 2024-09-21T09:40:07.171 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 21 v 1293 (ObjNum 812 snap 259 seq_num 812) dirty exists 2024-09-21T09:40:07.171 INFO:tasks.rados.rados.0.smithi022.stdout:2563: left oid 21 (ObjNum 812 snap 259 seq_num 812) 2024-09-21T09:40:07.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:07 smithi064 ceph-mon[86709]: osdmap e582: 8 total, 8 up, 8 in 2024-09-21T09:40:07.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:07 smithi064 ceph-mon[86709]: pgmap v478: 121 pgs: 2 activating, 9 peering, 110 active+clean; 151 MiB data, 985 MiB used, 714 GiB / 715 GiB avail; 4.5 MiB/s rd, 752 KiB/s wr, 6 op/s; 4.2 MiB/s, 3 objects/s recovering 2024-09-21T09:40:07.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:07 smithi022 ceph-mon[96736]: osdmap e582: 8 total, 8 up, 8 in 2024-09-21T09:40:07.573 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:07 smithi022 ceph-mon[96736]: pgmap v478: 121 pgs: 2 activating, 9 peering, 110 active+clean; 151 MiB data, 985 MiB used, 714 GiB / 715 GiB avail; 4.5 MiB/s rd, 752 KiB/s wr, 6 op/s; 4.2 MiB/s, 3 objects/s recovering 2024-09-21T09:40:07.574 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:07 smithi022 ceph-mon[98615]: osdmap e582: 8 total, 8 up, 8 in 2024-09-21T09:40:07.574 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:07 smithi022 ceph-mon[98615]: pgmap v478: 121 pgs: 2 activating, 9 peering, 110 active+clean; 151 MiB data, 985 MiB used, 714 GiB / 715 GiB avail; 4.5 MiB/s rd, 752 KiB/s wr, 6 op/s; 4.2 MiB/s, 3 objects/s recovering 2024-09-21T09:40:08.132 INFO:tasks.rados.rados.0.smithi022.stdout:2561: done (3 left) 2024-09-21T09:40:08.133 INFO:tasks.rados.rados.0.smithi022.stdout:2562: done (2 left) 2024-09-21T09:40:08.133 INFO:tasks.rados.rados.0.smithi022.stdout:2563: done (1 left) 2024-09-21T09:40:08.133 INFO:tasks.rados.rados.0.smithi022.stdout:2564: done (0 left) 2024-09-21T09:40:08.133 INFO:tasks.rados.rados.0.smithi022.stdout:2565: rmattr oid 11 current snap is 260 2024-09-21T09:40:08.133 INFO:tasks.rados.rados.0.smithi022.stdout:2566: read oid 40 snap -1 2024-09-21T09:40:08.133 INFO:tasks.rados.rados.0.smithi022.stdout:2566: expect deleted 2024-09-21T09:40:08.133 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 47 current snap is 260 2024-09-21T09:40:08.133 INFO:tasks.rados.rados.0.smithi022.stdout:2567: seq_num 813 ranges {1490944=417792,1908736=434176,2342912=442368,2785280=327680} 2024-09-21T09:40:08.143 INFO:tasks.rados.rados.0.smithi022.stdout:2567: writing smithi02284960-47 from 1490944 to 1908736 tid 1 2024-09-21T09:40:08.146 INFO:tasks.rados.rados.0.smithi022.stdout:2567: writing smithi02284960-47 from 1908736 to 2342912 tid 2 2024-09-21T09:40:08.149 INFO:tasks.rados.rados.0.smithi022.stdout:2567: writing smithi02284960-47 from 2342912 to 2785280 tid 3 2024-09-21T09:40:08.151 INFO:tasks.rados.rados.0.smithi022.stdout:2567: writing smithi02284960-47 from 2785280 to 3112960 tid 4 2024-09-21T09:40:08.151 INFO:tasks.rados.rados.0.smithi022.stdout:2568: read oid 10 snap -1 2024-09-21T09:40:08.152 INFO:tasks.rados.rados.0.smithi022.stdout:2568: expect deleted 2024-09-21T09:40:08.152 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 25 current snap is 260 2024-09-21T09:40:08.152 INFO:tasks.rados.rados.0.smithi022.stdout:2569: seq_num 814 ranges {0=442368,442368=770048,1212416=434176} 2024-09-21T09:40:08.154 INFO:tasks.rados.rados.0.smithi022.stdout:2569: writing smithi02284960-25 from 0 to 442368 tid 1 2024-09-21T09:40:08.158 INFO:tasks.rados.rados.0.smithi022.stdout:2569: writing smithi02284960-25 from 442368 to 1212416 tid 2 2024-09-21T09:40:08.161 INFO:tasks.rados.rados.0.smithi022.stdout:2569: writing smithi02284960-25 from 1212416 to 1646592 tid 3 2024-09-21T09:40:08.161 INFO:tasks.rados.rados.0.smithi022.stdout:2570: snap_create 2024-09-21T09:40:08.161 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 11 v 1413 (ObjNum 749 snap 227 seq_num 749) dirty exists 2024-09-21T09:40:08.161 INFO:tasks.rados.rados.0.smithi022.stdout:2567: finishing write tid 1 to smithi02284960-47 2024-09-21T09:40:08.161 INFO:tasks.rados.rados.0.smithi022.stdout:2567: finishing write tid 2 to smithi02284960-47 2024-09-21T09:40:08.161 INFO:tasks.rados.rados.0.smithi022.stdout:2567: finishing write tid 3 to smithi02284960-47 2024-09-21T09:40:08.161 INFO:tasks.rados.rados.0.smithi022.stdout:2567: finishing write tid 4 to smithi02284960-47 2024-09-21T09:40:08.161 INFO:tasks.rados.rados.0.smithi022.stdout:2567: finishing write tid 5 to smithi02284960-47 2024-09-21T09:40:08.162 INFO:tasks.rados.rados.0.smithi022.stdout:2567: finishing write tid 6 to smithi02284960-47 2024-09-21T09:40:08.162 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 47 v 1418 (ObjNum 813 snap 260 seq_num 813) dirty exists 2024-09-21T09:40:08.162 INFO:tasks.rados.rados.0.smithi022.stdout:2567: left oid 47 (ObjNum 813 snap 260 seq_num 813) 2024-09-21T09:40:08.162 INFO:tasks.rados.rados.0.smithi022.stdout:2569: finishing write tid 1 to smithi02284960-25 2024-09-21T09:40:08.163 INFO:tasks.rados.rados.0.smithi022.stdout:2569: finishing write tid 2 to smithi02284960-25 2024-09-21T09:40:08.164 INFO:tasks.rados.rados.0.smithi022.stdout:2569: finishing write tid 3 to smithi02284960-25 2024-09-21T09:40:08.164 INFO:tasks.rados.rados.0.smithi022.stdout:2569: finishing write tid 4 to smithi02284960-25 2024-09-21T09:40:08.165 INFO:tasks.rados.rados.0.smithi022.stdout:2569: finishing write tid 5 to smithi02284960-25 2024-09-21T09:40:08.166 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 25 v 1411 (ObjNum 814 snap 260 seq_num 814) dirty exists 2024-09-21T09:40:08.166 INFO:tasks.rados.rados.0.smithi022.stdout:2569: left oid 25 (ObjNum 814 snap 260 seq_num 814) 2024-09-21T09:40:08.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:08 smithi064 ceph-mon[86709]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 35/342 objects degraded (10.234%), 9 pgs degraded) 2024-09-21T09:40:08.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:08 smithi064 ceph-mon[86709]: Cluster is now healthy 2024-09-21T09:40:08.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:08 smithi064 ceph-mon[86709]: osdmap e583: 8 total, 8 up, 8 in 2024-09-21T09:40:08.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:08 smithi064 ceph-mon[86709]: fsmap cephfs:1 {0=b=up:active} 2024-09-21T09:40:08.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:08 smithi022 ceph-mon[96736]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 35/342 objects degraded (10.234%), 9 pgs degraded) 2024-09-21T09:40:08.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:08 smithi022 ceph-mon[96736]: Cluster is now healthy 2024-09-21T09:40:08.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:08 smithi022 ceph-mon[96736]: osdmap e583: 8 total, 8 up, 8 in 2024-09-21T09:40:08.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:08 smithi022 ceph-mon[96736]: fsmap cephfs:1 {0=b=up:active} 2024-09-21T09:40:08.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:08 smithi022 ceph-mon[98615]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 35/342 objects degraded (10.234%), 9 pgs degraded) 2024-09-21T09:40:08.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:08 smithi022 ceph-mon[98615]: Cluster is now healthy 2024-09-21T09:40:08.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:08 smithi022 ceph-mon[98615]: osdmap e583: 8 total, 8 up, 8 in 2024-09-21T09:40:08.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:08 smithi022 ceph-mon[98615]: fsmap cephfs:1 {0=b=up:active} 2024-09-21T09:40:09.138 INFO:tasks.rados.rados.0.smithi022.stdout:2565: done (5 left) 2024-09-21T09:40:09.138 INFO:tasks.rados.rados.0.smithi022.stdout:2566: done (4 left) 2024-09-21T09:40:09.138 INFO:tasks.rados.rados.0.smithi022.stdout:2567: done (3 left) 2024-09-21T09:40:09.138 INFO:tasks.rados.rados.0.smithi022.stdout:2568: done (2 left) 2024-09-21T09:40:09.138 INFO:tasks.rados.rados.0.smithi022.stdout:2569: done (1 left) 2024-09-21T09:40:09.138 INFO:tasks.rados.rados.0.smithi022.stdout:2570: done (0 left) 2024-09-21T09:40:09.138 INFO:tasks.rados.rados.0.smithi022.stdout:2571: delete oid 37 current snap is 261 2024-09-21T09:40:09.145 INFO:tasks.rados.rados.0.smithi022.stdout:2571: done (0 left) 2024-09-21T09:40:09.145 INFO:tasks.rados.rados.0.smithi022.stdout:2572: delete oid 8 current snap is 261 2024-09-21T09:40:09.147 INFO:tasks.rados.rados.0.smithi022.stdout:2572: done (0 left) 2024-09-21T09:40:09.147 INFO:tasks.rados.rados.0.smithi022.stdout:2573: read oid 49 snap 239 2024-09-21T09:40:09.147 INFO:tasks.rados.rados.0.smithi022.stdout:2573: expect (ObjNum 766 snap 237 seq_num 766) 2024-09-21T09:40:09.147 INFO:tasks.rados.rados.0.smithi022.stdout:2574: read oid 36 snap 255 2024-09-21T09:40:09.147 INFO:tasks.rados.rados.0.smithi022.stdout:2574: expect deleted 2024-09-21T09:40:09.148 INFO:tasks.rados.rados.0.smithi022.stdout:2575: copy_from oid 43 from oid 5 current snap is 261 2024-09-21T09:40:09.148 INFO:tasks.rados.rados.0.smithi022.stdout:2576: snap_remove snap 238 2024-09-21T09:40:09.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:09 smithi064 ceph-mon[86709]: osdmap e584: 8 total, 8 up, 8 in 2024-09-21T09:40:09.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:09 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "a"} : dispatch 2024-09-21T09:40:09.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:09 smithi064 ceph-mon[86709]: pgmap v481: 121 pgs: 121 active+clean; 153 MiB data, 1006 MiB used, 714 GiB / 715 GiB avail; 4.5 MiB/s rd, 1.1 MiB/s wr, 13 op/s; 4.2 MiB/s, 3 objects/s recovering 2024-09-21T09:40:09.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:09 smithi022 ceph-mon[98615]: osdmap e584: 8 total, 8 up, 8 in 2024-09-21T09:40:09.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:09 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "a"} : dispatch 2024-09-21T09:40:09.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:09 smithi022 ceph-mon[98615]: pgmap v481: 121 pgs: 121 active+clean; 153 MiB data, 1006 MiB used, 714 GiB / 715 GiB avail; 4.5 MiB/s rd, 1.1 MiB/s wr, 13 op/s; 4.2 MiB/s, 3 objects/s recovering 2024-09-21T09:40:09.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:09 smithi022 ceph-mon[96736]: osdmap e584: 8 total, 8 up, 8 in 2024-09-21T09:40:09.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:09 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "a"} : dispatch 2024-09-21T09:40:09.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:09 smithi022 ceph-mon[96736]: pgmap v481: 121 pgs: 121 active+clean; 153 MiB data, 1006 MiB used, 714 GiB / 715 GiB avail; 4.5 MiB/s rd, 1.1 MiB/s wr, 13 op/s; 4.2 MiB/s, 3 objects/s recovering 2024-09-21T09:40:10.144 INFO:tasks.rados.rados.0.smithi022.stdout:2573: expect (ObjNum 766 snap 237 seq_num 766) 2024-09-21T09:40:10.170 INFO:tasks.rados.rados.0.smithi022.stdout:2575: finishing copy_from racing read to smithi02284960-43 2024-09-21T09:40:10.170 INFO:tasks.rados.rados.0.smithi022.stdout:2575: finishing copy_from to smithi02284960-43 2024-09-21T09:40:10.170 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 43 v 1385 (ObjNum 672 snap 208 seq_num 672) dirty exists 2024-09-21T09:40:10.170 INFO:tasks.rados.rados.0.smithi022.stdout:2573: done (3 left) 2024-09-21T09:40:10.170 INFO:tasks.rados.rados.0.smithi022.stdout:2574: done (2 left) 2024-09-21T09:40:10.170 INFO:tasks.rados.rados.0.smithi022.stdout:2575: done (1 left) 2024-09-21T09:40:10.170 INFO:tasks.rados.rados.0.smithi022.stdout:2576: done (0 left) 2024-09-21T09:40:10.170 INFO:tasks.rados.rados.0.smithi022.stdout:2577: copy_from oid 21 from oid 25 current snap is 261 2024-09-21T09:40:10.171 INFO:tasks.rados.rados.0.smithi022.stdout:2578: read oid 38 snap -1 2024-09-21T09:40:10.171 INFO:tasks.rados.rados.0.smithi022.stdout:2578: expect deleted 2024-09-21T09:40:10.171 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 4 current snap is 261 2024-09-21T09:40:10.171 INFO:tasks.rados.rados.0.smithi022.stdout:2579: seq_num 817 ranges {14639104=491520,15130624=417792,15548416=425984,15974400=360448} 2024-09-21T09:40:10.252 INFO:tasks.rados.rados.0.smithi022.stdout:2579: writing smithi02284960-4 from 14639104 to 15130624 tid 1 2024-09-21T09:40:10.255 INFO:tasks.rados.rados.0.smithi022.stdout:2579: writing smithi02284960-4 from 15130624 to 15548416 tid 2 2024-09-21T09:40:10.258 INFO:tasks.rados.rados.0.smithi022.stdout:2579: writing smithi02284960-4 from 15548416 to 15974400 tid 3 2024-09-21T09:40:10.261 INFO:tasks.rados.rados.0.smithi022.stdout:2579: writing smithi02284960-4 from 15974400 to 16334848 tid 4 2024-09-21T09:40:10.261 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 1 current snap is 261 2024-09-21T09:40:10.261 INFO:tasks.rados.rados.0.smithi022.stdout:2580: seq_num 818 ranges {1662976=507904,2170880=704512,2875392=499712} 2024-09-21T09:40:10.273 INFO:tasks.rados.rados.0.smithi022.stdout:2580: writing smithi02284960-1 from 1662976 to 2170880 tid 1 2024-09-21T09:40:10.277 INFO:tasks.rados.rados.0.smithi022.stdout:2580: writing smithi02284960-1 from 2170880 to 2875392 tid 2 2024-09-21T09:40:10.281 INFO:tasks.rados.rados.0.smithi022.stdout:2580: writing smithi02284960-1 from 2875392 to 3375104 tid 3 2024-09-21T09:40:10.281 INFO:tasks.rados.rados.0.smithi022.stdout:2581: delete oid 44 current snap is 261 2024-09-21T09:40:10.281 INFO:tasks.rados.rados.0.smithi022.stdout:2577: finishing copy_from racing read to smithi02284960-21 2024-09-21T09:40:10.281 INFO:tasks.rados.rados.0.smithi022.stdout:2577: finishing copy_from to smithi02284960-21 2024-09-21T09:40:10.281 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 21 v 1411 (ObjNum 814 snap 260 seq_num 814) dirty exists 2024-09-21T09:40:10.281 INFO:tasks.rados.rados.0.smithi022.stdout:2579: finishing write tid 1 to smithi02284960-4 2024-09-21T09:40:10.281 INFO:tasks.rados.rados.0.smithi022.stdout:2579: finishing write tid 2 to smithi02284960-4 2024-09-21T09:40:10.281 INFO:tasks.rados.rados.0.smithi022.stdout:2579: finishing write tid 3 to smithi02284960-4 2024-09-21T09:40:10.281 INFO:tasks.rados.rados.0.smithi022.stdout:2579: finishing write tid 4 to smithi02284960-4 2024-09-21T09:40:10.281 INFO:tasks.rados.rados.0.smithi022.stdout:2579: finishing write tid 5 to smithi02284960-4 2024-09-21T09:40:10.281 INFO:tasks.rados.rados.0.smithi022.stdout:2579: finishing write tid 6 to smithi02284960-4 2024-09-21T09:40:10.281 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 4 v 1402 (ObjNum 817 snap 261 seq_num 817) dirty exists 2024-09-21T09:40:10.282 INFO:tasks.rados.rados.0.smithi022.stdout:2579: left oid 4 (ObjNum 817 snap 261 seq_num 817) 2024-09-21T09:40:10.282 INFO:tasks.rados.rados.0.smithi022.stdout:2580: finishing write tid 1 to smithi02284960-1 2024-09-21T09:40:10.282 INFO:tasks.rados.rados.0.smithi022.stdout:2580: finishing write tid 2 to smithi02284960-1 2024-09-21T09:40:10.283 INFO:tasks.rados.rados.0.smithi022.stdout:2580: finishing write tid 3 to smithi02284960-1 2024-09-21T09:40:10.284 INFO:tasks.rados.rados.0.smithi022.stdout:2580: finishing write tid 4 to smithi02284960-1 2024-09-21T09:40:10.284 INFO:tasks.rados.rados.0.smithi022.stdout:2577: done (4 left) 2024-09-21T09:40:10.284 INFO:tasks.rados.rados.0.smithi022.stdout:2578: done (3 left) 2024-09-21T09:40:10.284 INFO:tasks.rados.rados.0.smithi022.stdout:2579: done (2 left) 2024-09-21T09:40:10.284 INFO:tasks.rados.rados.0.smithi022.stdout:2581: done (1 left) 2024-09-21T09:40:10.284 INFO:tasks.rados.rados.0.smithi022.stdout:2582: setattr oid 40 current snap is 261 2024-09-21T09:40:10.285 INFO:tasks.rados.rados.0.smithi022.stdout:2580: finishing write tid 5 to smithi02284960-1 2024-09-21T09:40:10.285 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 1 v 1422 (ObjNum 818 snap 261 seq_num 818) dirty exists 2024-09-21T09:40:10.285 INFO:tasks.rados.rados.0.smithi022.stdout:2580: left oid 1 (ObjNum 818 snap 261 seq_num 818) 2024-09-21T09:40:10.286 INFO:tasks.rados.rados.0.smithi022.stdout:2580: done (1 left) 2024-09-21T09:40:10.286 INFO:tasks.rados.rados.0.smithi022.stdout:2583: read oid 1 snap -1 2024-09-21T09:40:10.286 INFO:tasks.rados.rados.0.smithi022.stdout:2583: expect (ObjNum 818 snap 261 seq_num 818) 2024-09-21T09:40:10.286 INFO:tasks.rados.rados.0.smithi022.stdout:2584: rollback oid 42 current snap is 261 2024-09-21T09:40:10.286 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 42 to 253 2024-09-21T09:40:10.286 INFO:tasks.rados.rados.0.smithi022.stdout:2585: snap_create 2024-09-21T09:40:10.288 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 40 v 1412 (ObjNum 177 snap 0 seq_num 0) dirty exists 2024-09-21T09:40:10.289 INFO:tasks.rados.rados.0.smithi022.stdout:2584: finishing rollback tid 0 to smithi02284960-42 2024-09-21T09:40:10.289 INFO:tasks.rados.rados.0.smithi022.stdout:2584: finishing rollback tid 1 to smithi02284960-42 2024-09-21T09:40:10.290 INFO:tasks.rados.rados.0.smithi022.stdout:2584: finishing rollback tid 2 to smithi02284960-42 2024-09-21T09:40:10.290 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 42 v 1344 (ObjNum 1265 snap 0 seq_num 3570304656) dirty exists 2024-09-21T09:40:10.298 INFO:tasks.rados.rados.0.smithi022.stdout:2583: expect (ObjNum 818 snap 261 seq_num 818) 2024-09-21T09:40:10.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:10 smithi064 ceph-mon[86709]: osdmap e585: 8 total, 8 up, 8 in 2024-09-21T09:40:10.442 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:40:10 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:40:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:40:10.557 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:10 smithi022 ceph-mon[98615]: osdmap e585: 8 total, 8 up, 8 in 2024-09-21T09:40:10.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:10 smithi022 ceph-mon[96736]: osdmap e585: 8 total, 8 up, 8 in 2024-09-21T09:40:11.123 INFO:journalctl@ceph.mds.a.smithi022.stdout:Sep 21 09:40:10 smithi022 podman[122841]: 2024-09-21 09:40:10.810043806 +0000 UTC m=+4.119603369 container died 34e4221f3a2777f03221291ba32091db4dc7dbfda30062481512245696e66950 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mds-a, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, ceph=True, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=quincy-fbd1a6a, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T09:40:11.148 INFO:tasks.rados.rados.0.smithi022.stdout:2582: done (3 left) 2024-09-21T09:40:11.148 INFO:tasks.rados.rados.0.smithi022.stdout:2583: done (2 left) 2024-09-21T09:40:11.149 INFO:tasks.rados.rados.0.smithi022.stdout:2584: done (1 left) 2024-09-21T09:40:11.149 INFO:tasks.rados.rados.0.smithi022.stdout:2585: done (0 left) 2024-09-21T09:40:11.149 INFO:tasks.rados.rados.0.smithi022.stdout:2586: copy_from oid 16 from oid 7 current snap is 262 2024-09-21T09:40:11.149 INFO:tasks.rados.rados.0.smithi022.stdout:2587: read oid 14 snap -1 2024-09-21T09:40:11.149 INFO:tasks.rados.rados.0.smithi022.stdout:2587: expect deleted 2024-09-21T09:40:11.149 INFO:tasks.rados.rados.0.smithi022.stdout:2588: delete oid 40 current snap is 262 2024-09-21T09:40:11.154 INFO:tasks.rados.rados.0.smithi022.stdout:2587: done (2 left) 2024-09-21T09:40:11.154 INFO:tasks.rados.rados.0.smithi022.stdout:2588: done (1 left) 2024-09-21T09:40:11.154 INFO:tasks.rados.rados.0.smithi022.stdout:2589: snap_remove snap 253 2024-09-21T09:40:11.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:11 smithi064 ceph-mon[86709]: osdmap e586: 8 total, 8 up, 8 in 2024-09-21T09:40:11.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:11 smithi064 ceph-mon[86709]: pgmap v484: 121 pgs: 121 active+clean; 160 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 2.3 MiB/s wr, 17 op/s 2024-09-21T09:40:11.515 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:11 smithi022 ceph-mon[98615]: osdmap e586: 8 total, 8 up, 8 in 2024-09-21T09:40:11.515 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:11 smithi022 ceph-mon[98615]: pgmap v484: 121 pgs: 121 active+clean; 160 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 2.3 MiB/s wr, 17 op/s 2024-09-21T09:40:11.516 INFO:journalctl@ceph.mds.a.smithi022.stdout:Sep 21 09:40:11 smithi022 podman[122841]: 2024-09-21 09:40:11.279664164 +0000 UTC m=+4.589223707 container cleanup 34e4221f3a2777f03221291ba32091db4dc7dbfda30062481512245696e66950 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mds-a, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=quincy-fbd1a6a, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_BRANCH=HEAD) 2024-09-21T09:40:11.516 INFO:journalctl@ceph.mds.a.smithi022.stdout:Sep 21 09:40:11 smithi022 bash[122841]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mds-a 2024-09-21T09:40:11.516 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:11 smithi022 ceph-mon[96736]: osdmap e586: 8 total, 8 up, 8 in 2024-09-21T09:40:11.516 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:11 smithi022 ceph-mon[96736]: pgmap v484: 121 pgs: 121 active+clean; 160 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 2.3 MiB/s wr, 17 op/s 2024-09-21T09:40:11.808 INFO:journalctl@ceph.mds.a.smithi022.stdout:Sep 21 09:40:11 smithi022 podman[122858]: 2024-09-21 09:40:11.514510392 +0000 UTC m=+0.699374581 container remove 34e4221f3a2777f03221291ba32091db4dc7dbfda30062481512245696e66950 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mds-a, RELEASE=quincy-fbd1a6a, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2) 2024-09-21T09:40:11.808 INFO:journalctl@ceph.mds.a.smithi022.stdout:Sep 21 09:40:11 smithi022 systemd[1]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490@mds.a.service: Deactivated successfully. 2024-09-21T09:40:11.808 INFO:journalctl@ceph.mds.a.smithi022.stdout:Sep 21 09:40:11 smithi022 systemd[1]: Stopped Ceph mds.a for d65e801e-77fa-11ef-baf6-efdc52797490. 2024-09-21T09:40:11.808 INFO:journalctl@ceph.mds.a.smithi022.stdout:Sep 21 09:40:11 smithi022 systemd[1]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490@mds.a.service: Consumed 1.202s CPU time. 2024-09-21T09:40:12.149 INFO:tasks.rados.rados.0.smithi022.stdout:2589: done (1 left) 2024-09-21T09:40:12.149 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 6 current snap is 262 2024-09-21T09:40:12.149 INFO:tasks.rados.rados.0.smithi022.stdout:2586: finishing copy_from racing read to smithi02284960-16 2024-09-21T09:40:12.149 INFO:tasks.rados.rados.0.smithi022.stdout:2590: seq_num 821 ranges {598016=557056,1155072=761856,1916928=442368} 2024-09-21T09:40:12.153 INFO:journalctl@ceph.mds.a.smithi022.stdout:Sep 21 09:40:11 smithi022 systemd[1]: Starting Ceph mds.a for d65e801e-77fa-11ef-baf6-efdc52797490... 2024-09-21T09:40:12.154 INFO:journalctl@ceph.mds.a.smithi022.stdout:Sep 21 09:40:12 smithi022 podman[122993]: 2024-09-21 09:40:12.057649918 +0000 UTC m=+0.110707720 container create 88fab662525b5563af835556fb30bfb0e941628136c8e9b276067425e5f7466f (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mds-a, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, ceph=True, GIT_CLEAN=True, RELEASE=main-62da54b, CEPH_POINT_RELEASE=) 2024-09-21T09:40:12.154 INFO:journalctl@ceph.mds.a.smithi022.stdout:Sep 21 09:40:12 smithi022 podman[122993]: 2024-09-21 09:40:11.963501511 +0000 UTC m=+0.016559323 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:40:12.156 INFO:tasks.rados.rados.0.smithi022.stdout:2590: writing smithi02284960-6 from 598016 to 1155072 tid 1 2024-09-21T09:40:12.161 INFO:tasks.rados.rados.0.smithi022.stdout:2590: writing smithi02284960-6 from 1155072 to 1916928 tid 2 2024-09-21T09:40:12.165 INFO:tasks.rados.rados.0.smithi022.stdout:2590: writing smithi02284960-6 from 1916928 to 2359296 tid 3 2024-09-21T09:40:12.165 INFO:tasks.rados.rados.0.smithi022.stdout:2591: snap_remove snap 255 2024-09-21T09:40:12.430 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:12 smithi022 ceph-mon[98615]: osdmap e587: 8 total, 8 up, 8 in 2024-09-21T09:40:12.430 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:12 smithi022 ceph-mon[96736]: osdmap e587: 8 total, 8 up, 8 in 2024-09-21T09:40:12.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:12 smithi064 ceph-mon[86709]: osdmap e587: 8 total, 8 up, 8 in 2024-09-21T09:40:12.808 INFO:journalctl@ceph.mds.a.smithi022.stdout:Sep 21 09:40:12 smithi022 podman[122993]: 2024-09-21 09:40:12.429206866 +0000 UTC m=+0.482264677 container init 88fab662525b5563af835556fb30bfb0e941628136c8e9b276067425e5f7466f (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mds-a, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-62da54b, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True) 2024-09-21T09:40:12.808 INFO:journalctl@ceph.mds.a.smithi022.stdout:Sep 21 09:40:12 smithi022 podman[122993]: 2024-09-21 09:40:12.434621367 +0000 UTC m=+0.487679169 container start 88fab662525b5563af835556fb30bfb0e941628136c8e9b276067425e5f7466f (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mds-a, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-62da54b, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS) 2024-09-21T09:40:12.808 INFO:journalctl@ceph.mds.a.smithi022.stdout:Sep 21 09:40:12 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mds-a[123008]: starting mds.a at 2024-09-21T09:40:12.808 INFO:journalctl@ceph.mds.a.smithi022.stdout:Sep 21 09:40:12 smithi022 bash[122993]: 88fab662525b5563af835556fb30bfb0e941628136c8e9b276067425e5f7466f 2024-09-21T09:40:12.808 INFO:journalctl@ceph.mds.a.smithi022.stdout:Sep 21 09:40:12 smithi022 systemd[1]: Started Ceph mds.a for d65e801e-77fa-11ef-baf6-efdc52797490. 2024-09-21T09:40:13.169 INFO:tasks.rados.rados.0.smithi022.stdout:2591: done (2 left) 2024-09-21T09:40:13.169 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 26 current snap is 262 2024-09-21T09:40:13.170 INFO:tasks.rados.rados.0.smithi022.stdout:2586: finishing copy_from to smithi02284960-16 2024-09-21T09:40:13.170 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 16 v 1505 (ObjNum 785 snap 246 seq_num 785) dirty exists 2024-09-21T09:40:13.170 INFO:tasks.rados.rados.0.smithi022.stdout:2592: seq_num 822 ranges {0=573440,573440=696320,1269760=507904} 2024-09-21T09:40:13.173 INFO:tasks.rados.rados.0.smithi022.stdout:2592: writing smithi02284960-26 from 0 to 573440 tid 1 2024-09-21T09:40:13.177 INFO:tasks.rados.rados.0.smithi022.stdout:2592: writing smithi02284960-26 from 573440 to 1269760 tid 2 2024-09-21T09:40:13.181 INFO:tasks.rados.rados.0.smithi022.stdout:2592: writing smithi02284960-26 from 1269760 to 1777664 tid 3 2024-09-21T09:40:13.181 INFO:tasks.rados.rados.0.smithi022.stdout:2586: done (2 left) 2024-09-21T09:40:13.181 INFO:tasks.rados.rados.0.smithi022.stdout:2593: copy_from oid 1 from oid 7 current snap is 262 2024-09-21T09:40:13.182 INFO:tasks.rados.rados.0.smithi022.stdout:2590: finishing write tid 1 to smithi02284960-6 2024-09-21T09:40:13.182 INFO:tasks.rados.rados.0.smithi022.stdout:2590: finishing write tid 2 to smithi02284960-6 2024-09-21T09:40:13.182 INFO:tasks.rados.rados.0.smithi022.stdout:2590: finishing write tid 3 to smithi02284960-6 2024-09-21T09:40:13.182 INFO:tasks.rados.rados.0.smithi022.stdout:2590: finishing write tid 4 to smithi02284960-6 2024-09-21T09:40:13.182 INFO:tasks.rados.rados.0.smithi022.stdout:2590: finishing write tid 5 to smithi02284960-6 2024-09-21T09:40:13.182 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 6 v 1555 (ObjNum 821 snap 262 seq_num 821) dirty exists 2024-09-21T09:40:13.182 INFO:tasks.rados.rados.0.smithi022.stdout:2590: left oid 6 (ObjNum 821 snap 262 seq_num 821) 2024-09-21T09:40:13.182 INFO:tasks.rados.rados.0.smithi022.stdout:2590: done (2 left) 2024-09-21T09:40:13.182 INFO:tasks.rados.rados.0.smithi022.stdout:2594: rollback oid 47 current snap is 262 2024-09-21T09:40:13.182 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 47 to 259 2024-09-21T09:40:13.182 INFO:tasks.rados.rados.0.smithi022.stdout:2595: delete oid 23 current snap is 262 2024-09-21T09:40:13.185 INFO:tasks.rados.rados.0.smithi022.stdout:2595: done (3 left) 2024-09-21T09:40:13.185 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 36 current snap is 262 2024-09-21T09:40:13.185 INFO:tasks.rados.rados.0.smithi022.stdout:2596: seq_num 824 ranges {0=606208,606208=458752,1064960=573440,1638400=172032} 2024-09-21T09:40:13.189 INFO:tasks.rados.rados.0.smithi022.stdout:2596: writing smithi02284960-36 from 0 to 606208 tid 1 2024-09-21T09:40:13.192 INFO:tasks.rados.rados.0.smithi022.stdout:2596: writing smithi02284960-36 from 606208 to 1064960 tid 2 2024-09-21T09:40:13.196 INFO:tasks.rados.rados.0.smithi022.stdout:2596: writing smithi02284960-36 from 1064960 to 1638400 tid 3 2024-09-21T09:40:13.198 INFO:tasks.rados.rados.0.smithi022.stdout:2596: writing smithi02284960-36 from 1638400 to 1810432 tid 4 2024-09-21T09:40:13.199 INFO:tasks.rados.rados.0.smithi022.stdout:2597: copy_from oid 18 from oid 41 current snap is 262 2024-09-21T09:40:13.199 INFO:tasks.rados.rados.0.smithi022.stdout:2592: finishing write tid 1 to smithi02284960-26 2024-09-21T09:40:13.199 INFO:tasks.rados.rados.0.smithi022.stdout:2592: finishing write tid 2 to smithi02284960-26 2024-09-21T09:40:13.199 INFO:tasks.rados.rados.0.smithi022.stdout:2592: finishing write tid 3 to smithi02284960-26 2024-09-21T09:40:13.199 INFO:tasks.rados.rados.0.smithi022.stdout:2592: finishing write tid 4 to smithi02284960-26 2024-09-21T09:40:13.199 INFO:tasks.rados.rados.0.smithi022.stdout:2594: finishing rollback tid 0 to smithi02284960-47 2024-09-21T09:40:13.199 INFO:tasks.rados.rados.0.smithi022.stdout:2594: finishing rollback tid 1 to smithi02284960-47 2024-09-21T09:40:13.199 INFO:tasks.rados.rados.0.smithi022.stdout:2594: finishing rollback tid 2 to smithi02284960-47 2024-09-21T09:40:13.199 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 47 v 1426 (ObjNum 805 snap 255 seq_num 805) dirty exists 2024-09-21T09:40:13.199 INFO:tasks.rados.rados.0.smithi022.stdout:2592: finishing write tid 5 to smithi02284960-26 2024-09-21T09:40:13.199 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 26 v 1348 (ObjNum 822 snap 262 seq_num 822) dirty exists 2024-09-21T09:40:13.199 INFO:tasks.rados.rados.0.smithi022.stdout:2592: left oid 26 (ObjNum 822 snap 262 seq_num 822) 2024-09-21T09:40:13.199 INFO:tasks.rados.rados.0.smithi022.stdout:2596: finishing write tid 1 to smithi02284960-36 2024-09-21T09:40:13.200 INFO:tasks.rados.rados.0.smithi022.stdout:2596: finishing write tid 2 to smithi02284960-36 2024-09-21T09:40:13.200 INFO:tasks.rados.rados.0.smithi022.stdout:2592: done (4 left) 2024-09-21T09:40:13.200 INFO:tasks.rados.rados.0.smithi022.stdout:2594: done (3 left) 2024-09-21T09:40:13.200 INFO:tasks.rados.rados.0.smithi022.stdout:2598: snap_remove snap 260 2024-09-21T09:40:13.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:13 smithi064 ceph-mon[86709]: osdmap e588: 8 total, 8 up, 8 in 2024-09-21T09:40:13.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:13 smithi064 ceph-mon[86709]: pgmap v487: 121 pgs: 121 active+clean; 160 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 613 KiB/s rd, 1.9 MiB/s wr, 11 op/s 2024-09-21T09:40:13.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:13 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:13.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:13 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:13.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:13 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:40:13.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:13 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:13.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:13 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:40:13.442 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:13 smithi022 ceph-mon[98615]: osdmap e588: 8 total, 8 up, 8 in 2024-09-21T09:40:13.442 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:13 smithi022 ceph-mon[98615]: pgmap v487: 121 pgs: 121 active+clean; 160 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 613 KiB/s rd, 1.9 MiB/s wr, 11 op/s 2024-09-21T09:40:13.442 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:13 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:13.443 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:13 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:13.443 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:13 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:40:13.443 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:13 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:13.443 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:13 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:40:13.443 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:13 smithi022 ceph-mon[96736]: osdmap e588: 8 total, 8 up, 8 in 2024-09-21T09:40:13.443 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:13 smithi022 ceph-mon[96736]: pgmap v487: 121 pgs: 121 active+clean; 160 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 613 KiB/s rd, 1.9 MiB/s wr, 11 op/s 2024-09-21T09:40:13.443 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:13 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:13.443 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:13 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:13.443 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:13 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:40:13.443 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:13 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:13.443 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:13 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:40:14.023 INFO:tasks.rados.rados.0.smithi022.stdout:2598: done (3 left) 2024-09-21T09:40:14.023 INFO:tasks.rados.rados.0.smithi022.stdout:2599: read oid 37 snap -1 2024-09-21T09:40:14.023 INFO:tasks.rados.rados.0.smithi022.stdout:2599: expect deleted 2024-09-21T09:40:14.023 INFO:tasks.rados.rados.0.smithi022.stdout:2600: snap_remove snap 261 2024-09-21T09:40:14.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:14 smithi064 ceph-mon[86709]: mds.? [v2:172.21.15.22:6834/4150258157,v1:172.21.15.22:6835/4150258157] up:boot 2024-09-21T09:40:14.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:14 smithi064 ceph-mon[86709]: fsmap cephfs:1 {0=b=up:active} 1 up:standby 2024-09-21T09:40:14.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:14 smithi064 ceph-mon[86709]: osdmap e589: 8 total, 8 up, 8 in 2024-09-21T09:40:14.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:14 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "a"} : dispatch 2024-09-21T09:40:14.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:14 smithi064 ceph-mon[86709]: osdmap e590: 8 total, 8 up, 8 in 2024-09-21T09:40:14.464 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:14 smithi022 ceph-mon[98615]: mds.? [v2:172.21.15.22:6834/4150258157,v1:172.21.15.22:6835/4150258157] up:boot 2024-09-21T09:40:14.464 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:14 smithi022 ceph-mon[98615]: fsmap cephfs:1 {0=b=up:active} 1 up:standby 2024-09-21T09:40:14.464 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:14 smithi022 ceph-mon[98615]: osdmap e589: 8 total, 8 up, 8 in 2024-09-21T09:40:14.464 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:14 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "a"} : dispatch 2024-09-21T09:40:14.464 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:14 smithi022 ceph-mon[98615]: osdmap e590: 8 total, 8 up, 8 in 2024-09-21T09:40:14.464 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:14 smithi022 ceph-mon[96736]: mds.? [v2:172.21.15.22:6834/4150258157,v1:172.21.15.22:6835/4150258157] up:boot 2024-09-21T09:40:14.464 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:14 smithi022 ceph-mon[96736]: fsmap cephfs:1 {0=b=up:active} 1 up:standby 2024-09-21T09:40:14.464 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:14 smithi022 ceph-mon[96736]: osdmap e589: 8 total, 8 up, 8 in 2024-09-21T09:40:14.464 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:14 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "a"} : dispatch 2024-09-21T09:40:14.464 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:14 smithi022 ceph-mon[96736]: osdmap e590: 8 total, 8 up, 8 in 2024-09-21T09:40:15.024 INFO:tasks.rados.rados.0.smithi022.stdout:2600: done (4 left) 2024-09-21T09:40:15.024 INFO:tasks.rados.rados.0.smithi022.stdout:2601: read oid 25 snap -1 2024-09-21T09:40:15.025 INFO:tasks.rados.rados.0.smithi022.stdout:2601: expect (ObjNum 814 snap 260 seq_num 814) 2024-09-21T09:40:15.025 INFO:tasks.rados.rados.0.smithi022.stdout:2602: rollback oid 50 current snap is 262 2024-09-21T09:40:15.025 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 50 to 248 2024-09-21T09:40:15.025 INFO:tasks.rados.rados.0.smithi022.stdout:2603: copy_from oid 35 from oid 9 current snap is 262 2024-09-21T09:40:15.025 INFO:tasks.rados.rados.0.smithi022.stdout:2604: snap_remove snap 247 2024-09-21T09:40:16.025 INFO:tasks.rados.rados.0.smithi022.stdout:2604: done (7 left) 2024-09-21T09:40:16.025 INFO:tasks.rados.rados.0.smithi022.stdout:2605: read oid 21 snap -1 2024-09-21T09:40:16.026 INFO:tasks.rados.rados.0.smithi022.stdout:2605: expect (ObjNum 814 snap 260 seq_num 814) 2024-09-21T09:40:16.026 INFO:tasks.rados.rados.0.smithi022.stdout:2606: snap_create 2024-09-21T09:40:16.026 INFO:tasks.rados.rados.0.smithi022.stdout:2593: finishing copy_from racing read to smithi02284960-1 2024-09-21T09:40:16.026 INFO:tasks.rados.rados.0.smithi022.stdout:2596: finishing write tid 3 to smithi02284960-36 2024-09-21T09:40:16.026 INFO:tasks.rados.rados.0.smithi022.stdout:2596: finishing write tid 4 to smithi02284960-36 2024-09-21T09:40:16.026 INFO:tasks.rados.rados.0.smithi022.stdout:2596: finishing write tid 5 to smithi02284960-36 2024-09-21T09:40:16.026 INFO:tasks.rados.rados.0.smithi022.stdout:2596: finishing write tid 6 to smithi02284960-36 2024-09-21T09:40:16.026 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 36 v 1298 (ObjNum 824 snap 262 seq_num 824) dirty exists 2024-09-21T09:40:16.026 INFO:tasks.rados.rados.0.smithi022.stdout:2596: left oid 36 (ObjNum 824 snap 262 seq_num 824) 2024-09-21T09:40:16.026 INFO:tasks.rados.rados.0.smithi022.stdout:2593: finishing copy_from to smithi02284960-1 2024-09-21T09:40:16.026 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 1 v 1505 (ObjNum 785 snap 246 seq_num 785) dirty exists 2024-09-21T09:40:16.026 INFO:tasks.rados.rados.0.smithi022.stdout:2597: finishing copy_from racing read to smithi02284960-18 2024-09-21T09:40:16.026 INFO:tasks.rados.rados.0.smithi022.stdout:2597: finishing copy_from to smithi02284960-18 2024-09-21T09:40:16.026 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 18 v 1407 (ObjNum 749 snap 227 seq_num 749) dirty exists 2024-09-21T09:40:16.027 INFO:tasks.rados.rados.0.smithi022.stdout:2602: finishing rollback tid 0 to smithi02284960-50 2024-09-21T09:40:16.027 INFO:tasks.rados.rados.0.smithi022.stdout:2602: finishing rollback tid 1 to smithi02284960-50 2024-09-21T09:40:16.027 INFO:tasks.rados.rados.0.smithi022.stdout:2602: finishing rollback tid 2 to smithi02284960-50 2024-09-21T09:40:16.027 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 50 v 1301 (ObjNum 701 snap 216 seq_num 701) dirty exists 2024-09-21T09:40:16.027 INFO:tasks.rados.rados.0.smithi022.stdout:2601: expect (ObjNum 814 snap 260 seq_num 814) 2024-09-21T09:40:16.036 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:40:15 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:40:15] "GET /metrics HTTP/1.1" 200 36120 "" "Prometheus/2.51.0" 2024-09-21T09:40:16.082 INFO:tasks.rados.rados.0.smithi022.stdout:2603: finishing copy_from racing read to smithi02284960-35 2024-09-21T09:40:16.082 INFO:tasks.rados.rados.0.smithi022.stdout:2603: finishing copy_from to smithi02284960-35 2024-09-21T09:40:16.083 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 35 v 1397 (ObjNum 811 snap 259 seq_num 811) dirty exists 2024-09-21T09:40:16.083 INFO:tasks.rados.rados.0.smithi022.stdout:2605: expect (ObjNum 814 snap 260 seq_num 814) 2024-09-21T09:40:16.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:16 smithi022 ceph-mon[98615]: pgmap v490: 121 pgs: 121 active+clean; 162 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 612 KiB/s rd, 402 KiB/s wr, 2 op/s 2024-09-21T09:40:16.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:16 smithi022 ceph-mon[98615]: osdmap e591: 8 total, 8 up, 8 in 2024-09-21T09:40:16.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:16 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:16.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:16 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:16.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:16 smithi022 ceph-mon[96736]: pgmap v490: 121 pgs: 121 active+clean; 162 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 612 KiB/s rd, 402 KiB/s wr, 2 op/s 2024-09-21T09:40:16.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:16 smithi022 ceph-mon[96736]: osdmap e591: 8 total, 8 up, 8 in 2024-09-21T09:40:16.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:16 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:16.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:16 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:16.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:16 smithi064 ceph-mon[86709]: pgmap v490: 121 pgs: 121 active+clean; 162 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 612 KiB/s rd, 402 KiB/s wr, 2 op/s 2024-09-21T09:40:16.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:16 smithi064 ceph-mon[86709]: osdmap e591: 8 total, 8 up, 8 in 2024-09-21T09:40:16.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:16 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:16.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:16 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:17.026 INFO:tasks.rados.rados.0.smithi022.stdout:2593: done (8 left) 2024-09-21T09:40:17.026 INFO:tasks.rados.rados.0.smithi022.stdout:2596: done (7 left) 2024-09-21T09:40:17.026 INFO:tasks.rados.rados.0.smithi022.stdout:2597: done (6 left) 2024-09-21T09:40:17.026 INFO:tasks.rados.rados.0.smithi022.stdout:2599: done (5 left) 2024-09-21T09:40:17.026 INFO:tasks.rados.rados.0.smithi022.stdout:2601: done (4 left) 2024-09-21T09:40:17.026 INFO:tasks.rados.rados.0.smithi022.stdout:2602: done (3 left) 2024-09-21T09:40:17.027 INFO:tasks.rados.rados.0.smithi022.stdout:2603: done (2 left) 2024-09-21T09:40:17.027 INFO:tasks.rados.rados.0.smithi022.stdout:2605: done (1 left) 2024-09-21T09:40:17.027 INFO:tasks.rados.rados.0.smithi022.stdout:2606: done (0 left) 2024-09-21T09:40:17.027 INFO:tasks.rados.rados.0.smithi022.stdout:2607: copy_from oid 43 from oid 4 current snap is 263 2024-09-21T09:40:17.027 INFO:tasks.rados.rados.0.smithi022.stdout:2608: setattr oid 12 current snap is 263 2024-09-21T09:40:17.027 INFO:tasks.rados.rados.0.smithi022.stdout:2609: read oid 2 snap -1 2024-09-21T09:40:17.028 INFO:tasks.rados.rados.0.smithi022.stdout:2609: expect deleted 2024-09-21T09:40:17.028 INFO:tasks.rados.rados.0.smithi022.stdout:2610: setattr oid 29 current snap is 263 2024-09-21T09:40:17.029 INFO:tasks.rados.rados.0.smithi022.stdout:2611: read oid 33 snap -1 2024-09-21T09:40:17.029 INFO:tasks.rados.rados.0.smithi022.stdout:2611: expect (ObjNum 798 snap 250 seq_num 798) 2024-09-21T09:40:17.029 INFO:tasks.rados.rados.0.smithi022.stdout:2612: snap_remove snap 248 2024-09-21T09:40:17.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:17 smithi022 ceph-mon[98615]: osdmap e592: 8 total, 8 up, 8 in 2024-09-21T09:40:17.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:17 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:17.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:17 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:17.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:17 smithi022 ceph-mon[98615]: osdmap e593: 8 total, 8 up, 8 in 2024-09-21T09:40:17.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:17 smithi022 ceph-mon[96736]: osdmap e592: 8 total, 8 up, 8 in 2024-09-21T09:40:17.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:17 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:17.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:17 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:17.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:17 smithi022 ceph-mon[96736]: osdmap e593: 8 total, 8 up, 8 in 2024-09-21T09:40:17.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:17 smithi064 ceph-mon[86709]: osdmap e592: 8 total, 8 up, 8 in 2024-09-21T09:40:17.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:17 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:17.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:17 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:17.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:17 smithi064 ceph-mon[86709]: osdmap e593: 8 total, 8 up, 8 in 2024-09-21T09:40:18.029 INFO:tasks.rados.rados.0.smithi022.stdout:2612: done (5 left) 2024-09-21T09:40:18.029 INFO:tasks.rados.rados.0.smithi022.stdout:2613: read oid 19 snap -1 2024-09-21T09:40:18.029 INFO:tasks.rados.rados.0.smithi022.stdout:2613: expect deleted 2024-09-21T09:40:18.029 INFO:tasks.rados.rados.0.smithi022.stdout:2614: read oid 46 snap -1 2024-09-21T09:40:18.029 INFO:tasks.rados.rados.0.smithi022.stdout:2614: expect deleted 2024-09-21T09:40:18.029 INFO:tasks.rados.rados.0.smithi022.stdout:2615: snap_create 2024-09-21T09:40:18.029 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 29 v 1405 (ObjNum 49 snap 0 seq_num 3570269824) dirty exists 2024-09-21T09:40:18.029 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 12 v 1505 (ObjNum 633 snap 189 seq_num 633) dirty exists 2024-09-21T09:40:18.029 INFO:tasks.rados.rados.0.smithi022.stdout:2611: expect (ObjNum 798 snap 250 seq_num 798) 2024-09-21T09:40:18.213 INFO:tasks.rados.rados.0.smithi022.stdout:2607: finishing copy_from racing read to smithi02284960-43 2024-09-21T09:40:18.213 INFO:tasks.rados.rados.0.smithi022.stdout:2607: finishing copy_from to smithi02284960-43 2024-09-21T09:40:18.213 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 43 v 1402 (ObjNum 817 snap 261 seq_num 817) dirty exists 2024-09-21T09:40:18.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:18 smithi022 ceph-mon[98615]: pgmap v493: 121 pgs: 121 active+clean; 181 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 3.6 MiB/s rd, 5.9 MiB/s wr, 15 op/s 2024-09-21T09:40:18.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:18 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:18.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:18 smithi022 ceph-mon[98615]: osdmap e594: 8 total, 8 up, 8 in 2024-09-21T09:40:18.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:18 smithi022 ceph-mon[96736]: pgmap v493: 121 pgs: 121 active+clean; 181 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 3.6 MiB/s rd, 5.9 MiB/s wr, 15 op/s 2024-09-21T09:40:18.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:18 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:18.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:18 smithi022 ceph-mon[96736]: osdmap e594: 8 total, 8 up, 8 in 2024-09-21T09:40:18.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:18 smithi064 ceph-mon[86709]: pgmap v493: 121 pgs: 121 active+clean; 181 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 3.6 MiB/s rd, 5.9 MiB/s wr, 15 op/s 2024-09-21T09:40:18.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:18 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:18.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:18 smithi064 ceph-mon[86709]: osdmap e594: 8 total, 8 up, 8 in 2024-09-21T09:40:19.025 INFO:tasks.rados.rados.0.smithi022.stdout:2607: done (7 left) 2024-09-21T09:40:19.025 INFO:tasks.rados.rados.0.smithi022.stdout:2608: done (6 left) 2024-09-21T09:40:19.026 INFO:tasks.rados.rados.0.smithi022.stdout:2609: done (5 left) 2024-09-21T09:40:19.026 INFO:tasks.rados.rados.0.smithi022.stdout:2610: done (4 left) 2024-09-21T09:40:19.026 INFO:tasks.rados.rados.0.smithi022.stdout:2611: done (3 left) 2024-09-21T09:40:19.026 INFO:tasks.rados.rados.0.smithi022.stdout:2613: done (2 left) 2024-09-21T09:40:19.026 INFO:tasks.rados.rados.0.smithi022.stdout:2614: done (1 left) 2024-09-21T09:40:19.026 INFO:tasks.rados.rados.0.smithi022.stdout:2615: done (0 left) 2024-09-21T09:40:19.026 INFO:tasks.rados.rados.0.smithi022.stdout:2616: copy_from oid 44 from oid 8 current snap is 264 2024-09-21T09:40:19.026 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 35 current snap is 264 2024-09-21T09:40:19.026 INFO:tasks.rados.rados.0.smithi022.stdout:2617: seq_num 825 ranges {1589248=622592,2211840=745472,2957312=458752} 2024-09-21T09:40:19.037 INFO:tasks.rados.rados.0.smithi022.stdout:2617: writing smithi02284960-35 from 1589248 to 2211840 tid 1 2024-09-21T09:40:19.042 INFO:tasks.rados.rados.0.smithi022.stdout:2617: writing smithi02284960-35 from 2211840 to 2957312 tid 2 2024-09-21T09:40:19.046 INFO:tasks.rados.rados.0.smithi022.stdout:2617: writing smithi02284960-35 from 2957312 to 3416064 tid 3 2024-09-21T09:40:19.046 INFO:tasks.rados.rados.0.smithi022.stdout:2618: read oid 11 snap -1 2024-09-21T09:40:19.046 INFO:tasks.rados.rados.0.smithi022.stdout:2618: expect (ObjNum 749 snap 227 seq_num 749) 2024-09-21T09:40:19.046 INFO:tasks.rados.rados.0.smithi022.stdout:2619: copy_from oid 18 from oid 45 current snap is 264 2024-09-21T09:40:19.046 INFO:tasks.rados.rados.0.smithi022.stdout:2616: finishing copy_from to smithi02284960-44 2024-09-21T09:40:19.046 INFO:tasks.rados.rados.0.smithi022.stdout:2616: got expected ENOENT (src dne) 2024-09-21T09:40:19.046 INFO:tasks.rados.rados.0.smithi022.stdout:2616: finishing copy_from racing read to smithi02284960-44 2024-09-21T09:40:19.046 INFO:tasks.rados.rados.0.smithi022.stdout:2617: finishing write tid 1 to smithi02284960-35 2024-09-21T09:40:19.047 INFO:tasks.rados.rados.0.smithi022.stdout:2616: done (3 left) 2024-09-21T09:40:19.047 INFO:tasks.rados.rados.0.smithi022.stdout:2620: rollback oid 19 current snap is 264 2024-09-21T09:40:19.047 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 19 to 254 2024-09-21T09:40:19.047 INFO:tasks.rados.rados.0.smithi022.stdout:2621: read oid 28 snap -1 2024-09-21T09:40:19.047 INFO:tasks.rados.rados.0.smithi022.stdout:2621: expect (ObjNum 753 snap 231 seq_num 753) 2024-09-21T09:40:19.047 INFO:tasks.rados.rados.0.smithi022.stdout:2622: delete oid 31 current snap is 264 2024-09-21T09:40:19.047 INFO:tasks.rados.rados.0.smithi022.stdout:2617: finishing write tid 2 to smithi02284960-35 2024-09-21T09:40:19.049 INFO:tasks.rados.rados.0.smithi022.stdout:2619: finishing copy_from racing read to smithi02284960-18 2024-09-21T09:40:19.051 INFO:tasks.rados.rados.0.smithi022.stdout:2617: finishing write tid 3 to smithi02284960-35 2024-09-21T09:40:19.051 INFO:tasks.rados.rados.0.smithi022.stdout:2617: finishing write tid 4 to smithi02284960-35 2024-09-21T09:40:19.053 INFO:tasks.rados.rados.0.smithi022.stdout:2619: finishing copy_from to smithi02284960-18 2024-09-21T09:40:19.053 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 18 v 1293 (ObjNum 33 snap 0 seq_num 3569436464) dirty exists 2024-09-21T09:40:19.053 INFO:tasks.rados.rados.0.smithi022.stdout:2620: finishing rollback tid 1 to smithi02284960-19 2024-09-21T09:40:19.053 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 19 v 0 (ObjNum 945 snap 0 seq_num 3570147760) dirty dne 2024-09-21T09:40:19.054 INFO:tasks.rados.rados.0.smithi022.stdout:2619: done (5 left) 2024-09-21T09:40:19.054 INFO:tasks.rados.rados.0.smithi022.stdout:2620: done (4 left) 2024-09-21T09:40:19.054 INFO:tasks.rados.rados.0.smithi022.stdout:2622: done (3 left) 2024-09-21T09:40:19.054 INFO:tasks.rados.rados.0.smithi022.stdout:2623: rollback oid 13 current snap is 264 2024-09-21T09:40:19.054 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 13 to 258 2024-09-21T09:40:19.054 INFO:tasks.rados.rados.0.smithi022.stdout:2624: snap_remove snap 239 2024-09-21T09:40:20.047 INFO:tasks.rados.rados.0.smithi022.stdout:2624: done (4 left) 2024-09-21T09:40:20.047 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 29 current snap is 264 2024-09-21T09:40:20.047 INFO:tasks.rados.rados.0.smithi022.stdout:2625: seq_num 827 ranges {0=655360,655360=516096,1171456=688128} 2024-09-21T09:40:20.052 INFO:tasks.rados.rados.0.smithi022.stdout:2625: writing smithi02284960-29 from 0 to 655360 tid 1 2024-09-21T09:40:20.056 INFO:tasks.rados.rados.0.smithi022.stdout:2625: writing smithi02284960-29 from 655360 to 1171456 tid 2 2024-09-21T09:40:20.061 INFO:tasks.rados.rados.0.smithi022.stdout:2625: writing smithi02284960-29 from 1171456 to 1859584 tid 3 2024-09-21T09:40:20.061 INFO:tasks.rados.rados.0.smithi022.stdout:2626: snap_create 2024-09-21T09:40:20.061 INFO:tasks.rados.rados.0.smithi022.stdout:2623: finishing rollback tid 1 to smithi02284960-13 2024-09-21T09:40:20.061 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 13 v 0 (ObjNum 33 snap 0 seq_num 2955068410) dirty dne 2024-09-21T09:40:20.062 INFO:tasks.rados.rados.0.smithi022.stdout:2617: finishing write tid 5 to smithi02284960-35 2024-09-21T09:40:20.062 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 35 v 1401 (ObjNum 825 snap 264 seq_num 825) dirty exists 2024-09-21T09:40:20.062 INFO:tasks.rados.rados.0.smithi022.stdout:2617: left oid 35 (ObjNum 825 snap 264 seq_num 825) 2024-09-21T09:40:20.062 INFO:tasks.rados.rados.0.smithi022.stdout:2621: expect (ObjNum 753 snap 231 seq_num 753) 2024-09-21T09:40:20.156 INFO:tasks.rados.rados.0.smithi022.stdout:2618: expect (ObjNum 749 snap 227 seq_num 749) 2024-09-21T09:40:20.164 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:20 smithi064 ceph-mon[86709]: pgmap v496: 121 pgs: 121 active+clean; 184 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 4.8 MiB/s rd, 6.4 MiB/s wr, 23 op/s 2024-09-21T09:40:20.164 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:20 smithi064 ceph-mon[86709]: osdmap e595: 8 total, 8 up, 8 in 2024-09-21T09:40:20.164 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:20 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:20.164 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:20 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:20.164 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:20 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:40:20.164 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:20 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:40:20.164 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:20 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:20.164 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:20 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:40:20.164 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:20 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:40:20.164 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:20 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:40:20.164 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:20 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:40:20.164 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:20 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:40:20.165 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:20 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "mds ok-to-stop", "ids": ["b"]} : dispatch 2024-09-21T09:40:20.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:20 smithi022 ceph-mon[98615]: pgmap v496: 121 pgs: 121 active+clean; 184 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 4.8 MiB/s rd, 6.4 MiB/s wr, 23 op/s 2024-09-21T09:40:20.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:20 smithi022 ceph-mon[98615]: osdmap e595: 8 total, 8 up, 8 in 2024-09-21T09:40:20.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:20 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:20.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:20 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:20.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:20 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:40:20.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:20 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:40:20.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:20 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:20.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:20 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:40:20.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:20 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:40:20.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:20 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:40:20.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:20 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:40:20.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:20 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:40:20.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:20 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "mds ok-to-stop", "ids": ["b"]} : dispatch 2024-09-21T09:40:20.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:20 smithi022 ceph-mon[96736]: pgmap v496: 121 pgs: 121 active+clean; 184 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 4.8 MiB/s rd, 6.4 MiB/s wr, 23 op/s 2024-09-21T09:40:20.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:20 smithi022 ceph-mon[96736]: osdmap e595: 8 total, 8 up, 8 in 2024-09-21T09:40:20.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:20 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:20.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:20 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:20.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:20 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:40:20.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:20 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:40:20.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:20 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:20.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:20 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:40:20.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:20 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:40:20.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:20 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:40:20.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:20 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:40:20.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:20 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:40:20.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:20 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "mds ok-to-stop", "ids": ["b"]} : dispatch 2024-09-21T09:40:20.441 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:40:20 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:40:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:40:20.903 INFO:teuthology.orchestra.run.smithi022.stdout:true 2024-09-21T09:40:21.115 INFO:tasks.rados.rados.0.smithi022.stdout:2625: finishing write tid 1 to smithi02284960-29 2024-09-21T09:40:21.115 INFO:tasks.rados.rados.0.smithi022.stdout:2625: finishing write tid 2 to smithi02284960-29 2024-09-21T09:40:21.115 INFO:tasks.rados.rados.0.smithi022.stdout:2625: finishing write tid 3 to smithi02284960-29 2024-09-21T09:40:21.115 INFO:tasks.rados.rados.0.smithi022.stdout:2625: finishing write tid 4 to smithi02284960-29 2024-09-21T09:40:21.115 INFO:tasks.rados.rados.0.smithi022.stdout:2625: finishing write tid 5 to smithi02284960-29 2024-09-21T09:40:21.115 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 29 v 1409 (ObjNum 827 snap 264 seq_num 827) dirty exists 2024-09-21T09:40:21.116 INFO:tasks.rados.rados.0.smithi022.stdout:2625: left oid 29 (ObjNum 827 snap 264 seq_num 827) 2024-09-21T09:40:21.116 INFO:tasks.rados.rados.0.smithi022.stdout:2617: done (5 left) 2024-09-21T09:40:21.116 INFO:tasks.rados.rados.0.smithi022.stdout:2618: done (4 left) 2024-09-21T09:40:21.116 INFO:tasks.rados.rados.0.smithi022.stdout:2621: done (3 left) 2024-09-21T09:40:21.116 INFO:tasks.rados.rados.0.smithi022.stdout:2623: done (2 left) 2024-09-21T09:40:21.116 INFO:tasks.rados.rados.0.smithi022.stdout:2625: done (1 left) 2024-09-21T09:40:21.116 INFO:tasks.rados.rados.0.smithi022.stdout:2626: done (0 left) 2024-09-21T09:40:21.116 INFO:tasks.rados.rados.0.smithi022.stdout:2627: snap_create 2024-09-21T09:40:21.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:21 smithi022 ceph-mon[98615]: Detected new or changed devices on smithi022 2024-09-21T09:40:21.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:21 smithi022 ceph-mon[98615]: Upgrade: It appears safe to stop mds.b 2024-09-21T09:40:21.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:21 smithi022 ceph-mon[98615]: osdmap e596: 8 total, 8 up, 8 in 2024-09-21T09:40:21.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:21 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:21.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:21 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-21T09:40:21.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:21 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:40:21.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:21 smithi022 ceph-mon[96736]: Detected new or changed devices on smithi022 2024-09-21T09:40:21.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:21 smithi022 ceph-mon[96736]: Upgrade: It appears safe to stop mds.b 2024-09-21T09:40:21.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:21 smithi022 ceph-mon[96736]: osdmap e596: 8 total, 8 up, 8 in 2024-09-21T09:40:21.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:21 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:21.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:21 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-21T09:40:21.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:21 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:40:21.369 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:21 smithi064 ceph-mon[86709]: Detected new or changed devices on smithi022 2024-09-21T09:40:21.369 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:21 smithi064 ceph-mon[86709]: Upgrade: It appears safe to stop mds.b 2024-09-21T09:40:21.369 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:21 smithi064 ceph-mon[86709]: osdmap e596: 8 total, 8 up, 8 in 2024-09-21T09:40:21.369 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:21 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:21.369 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:21 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-21T09:40:21.369 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:21 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:40:21.502 INFO:teuthology.orchestra.run.smithi022.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T09:40:21.502 INFO:teuthology.orchestra.run.smithi022.stdout:alertmanager.a smithi022 *:9093,9094 running (7m) 5s ago 11m 22.0M - 0.27.0 11f11916f8cd caced80ba544 2024-09-21T09:40:21.503 INFO:teuthology.orchestra.run.smithi022.stdout:grafana.a smithi064 *:3000 running (6m) 22s ago 11m 83.5M - 9.4.7 954c08fa6188 527bb657d70d 2024-09-21T09:40:21.503 INFO:teuthology.orchestra.run.smithi022.stdout:mds.a smithi022 running (9s) 5s ago 13m 11.0M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 88fab662525b 2024-09-21T09:40:21.503 INFO:teuthology.orchestra.run.smithi022.stdout:mds.b smithi064 running (13m) 22s ago 12m 23.2M - 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 0f846b3ec694 2024-09-21T09:40:21.503 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.x smithi064 *:8443,9283,8765 running (6m) 22s ago 16m 456M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 61abc61fea8d 2024-09-21T09:40:21.503 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.y smithi022 *:8443,9283,8765 running (6m) 5s ago 18m 521M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 6c5bee78f09b 2024-09-21T09:40:21.503 INFO:teuthology.orchestra.run.smithi022.stdout:mon.a smithi022 running (5m) 5s ago 18m 69.2M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 0513395c784d 2024-09-21T09:40:21.503 INFO:teuthology.orchestra.run.smithi022.stdout:mon.b smithi064 running (5m) 22s ago 16m 54.7M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 9dab26b6a9e7 2024-09-21T09:40:21.503 INFO:teuthology.orchestra.run.smithi022.stdout:mon.c smithi022 running (5m) 5s ago 16m 55.9M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 c74b58b67bd6 2024-09-21T09:40:21.503 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.a smithi022 *:9100 running (6m) 5s ago 11m 11.5M - 1.7.0 72c9c2088986 593c6158f524 2024-09-21T09:40:21.503 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.b smithi064 *:9100 running (6m) 22s ago 11m 10.6M - 1.7.0 72c9c2088986 a533420b0139 2024-09-21T09:40:21.503 INFO:teuthology.orchestra.run.smithi022.stdout:osd.0 smithi022 running (4m) 5s ago 16m 213M 2481M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 40ce23a2b569 2024-09-21T09:40:21.503 INFO:teuthology.orchestra.run.smithi022.stdout:osd.1 smithi022 running (3m) 5s ago 15m 200M 2481M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 7cd0c18d224c 2024-09-21T09:40:21.503 INFO:teuthology.orchestra.run.smithi022.stdout:osd.2 smithi022 running (3m) 5s ago 15m 208M 2481M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 b8c944306698 2024-09-21T09:40:21.504 INFO:teuthology.orchestra.run.smithi022.stdout:osd.3 smithi022 running (2m) 5s ago 14m 104M 2481M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 c90b27f61bb6 2024-09-21T09:40:21.504 INFO:teuthology.orchestra.run.smithi022.stdout:osd.4 smithi064 running (2m) 22s ago 14m 125M 2480M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 936099dbca14 2024-09-21T09:40:21.504 INFO:teuthology.orchestra.run.smithi022.stdout:osd.5 smithi064 running (95s) 22s ago 14m 137M 2480M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 b3225b090dba 2024-09-21T09:40:21.504 INFO:teuthology.orchestra.run.smithi022.stdout:osd.6 smithi064 running (61s) 22s ago 13m 87.7M 2480M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 c6a9a09637e5 2024-09-21T09:40:21.504 INFO:teuthology.orchestra.run.smithi022.stdout:osd.7 smithi064 running (27s) 22s ago 13m 14.4M 2480M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 6f82655605e3 2024-09-21T09:40:21.504 INFO:teuthology.orchestra.run.smithi022.stdout:prometheus.a smithi064 *:9095 running (6m) 22s ago 12m 56.6M - 2.51.0 1d3b7f56885b 80629ffa2179 2024-09-21T09:40:21.857 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-09-21T09:40:21.857 INFO:teuthology.orchestra.run.smithi022.stdout: "mon": { 2024-09-21T09:40:21.857 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 3 2024-09-21T09:40:21.857 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-21T09:40:21.857 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr": { 2024-09-21T09:40:21.857 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 2 2024-09-21T09:40:21.857 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-21T09:40:21.857 INFO:teuthology.orchestra.run.smithi022.stdout: "osd": { 2024-09-21T09:40:21.858 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 8 2024-09-21T09:40:21.858 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-21T09:40:21.858 INFO:teuthology.orchestra.run.smithi022.stdout: "mds": { 2024-09-21T09:40:21.858 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 1, 2024-09-21T09:40:21.858 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 1 2024-09-21T09:40:21.858 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-21T09:40:21.858 INFO:teuthology.orchestra.run.smithi022.stdout: "overall": { 2024-09-21T09:40:21.858 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 1, 2024-09-21T09:40:21.858 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 14 2024-09-21T09:40:21.858 INFO:teuthology.orchestra.run.smithi022.stdout: } 2024-09-21T09:40:21.858 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-09-21T09:40:22.057 INFO:tasks.rados.rados.0.smithi022.stdout:2627: done (0 left) 2024-09-21T09:40:22.057 INFO:tasks.rados.rados.0.smithi022.stdout:2628: rollback oid 45 current snap is 266 2024-09-21T09:40:22.057 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 45 to 246 2024-09-21T09:40:22.057 INFO:tasks.rados.rados.0.smithi022.stdout:2629: read oid 14 snap -1 2024-09-21T09:40:22.057 INFO:tasks.rados.rados.0.smithi022.stdout:2629: expect deleted 2024-09-21T09:40:22.057 INFO:tasks.rados.rados.0.smithi022.stdout:2630: read oid 3 snap -1 2024-09-21T09:40:22.057 INFO:tasks.rados.rados.0.smithi022.stdout:2630: expect deleted 2024-09-21T09:40:22.057 INFO:tasks.rados.rados.0.smithi022.stdout:2631: delete oid 25 current snap is 266 2024-09-21T09:40:22.061 INFO:tasks.rados.rados.0.smithi022.stdout:2628: finishing rollback tid 0 to smithi02284960-45 2024-09-21T09:40:22.062 INFO:tasks.rados.rados.0.smithi022.stdout:2628: finishing rollback tid 1 to smithi02284960-45 2024-09-21T09:40:22.062 INFO:tasks.rados.rados.0.smithi022.stdout:2629: done (3 left) 2024-09-21T09:40:22.062 INFO:tasks.rados.rados.0.smithi022.stdout:2630: done (2 left) 2024-09-21T09:40:22.062 INFO:tasks.rados.rados.0.smithi022.stdout:2631: done (1 left) 2024-09-21T09:40:22.062 INFO:tasks.rados.rados.0.smithi022.stdout:2632: delete oid 32 current snap is 266 2024-09-21T09:40:22.062 INFO:tasks.rados.rados.0.smithi022.stdout:2628: finishing rollback tid 2 to smithi02284960-45 2024-09-21T09:40:22.062 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 45 v 1304 (ObjNum 756 snap 233 seq_num 756) dirty exists 2024-09-21T09:40:22.065 INFO:tasks.rados.rados.0.smithi022.stdout:2628: done (1 left) 2024-09-21T09:40:22.065 INFO:tasks.rados.rados.0.smithi022.stdout:2632: done (0 left) 2024-09-21T09:40:22.065 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 21 current snap is 266 2024-09-21T09:40:22.065 INFO:tasks.rados.rados.0.smithi022.stdout:2633: seq_num 830 ranges {1646592=704512,2351104=614400,2965504=573440,3538944=16384} 2024-09-21T09:40:22.078 INFO:tasks.rados.rados.0.smithi022.stdout:2633: writing smithi02284960-21 from 1646592 to 2351104 tid 1 2024-09-21T09:40:22.081 INFO:tasks.rados.rados.0.smithi022.stdout:2633: writing smithi02284960-21 from 2351104 to 2965504 tid 2 2024-09-21T09:40:22.085 INFO:tasks.rados.rados.0.smithi022.stdout:2633: writing smithi02284960-21 from 2965504 to 3538944 tid 3 2024-09-21T09:40:22.085 INFO:tasks.rados.rados.0.smithi022.stdout:2633: writing smithi02284960-21 from 3538944 to 3555328 tid 4 2024-09-21T09:40:22.085 INFO:tasks.rados.rados.0.smithi022.stdout:2634: rollback oid 1 current snap is 266 2024-09-21T09:40:22.085 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 1 to 257 2024-09-21T09:40:22.085 INFO:tasks.rados.rados.0.smithi022.stdout:2633: finishing write tid 1 to smithi02284960-21 2024-09-21T09:40:22.085 INFO:tasks.rados.rados.0.smithi022.stdout:2633: finishing write tid 2 to smithi02284960-21 2024-09-21T09:40:22.085 INFO:tasks.rados.rados.0.smithi022.stdout:2635: snap_create 2024-09-21T09:40:22.088 INFO:tasks.rados.rados.0.smithi022.stdout:2633: finishing write tid 3 to smithi02284960-21 2024-09-21T09:40:22.088 INFO:tasks.rados.rados.0.smithi022.stdout:2633: finishing write tid 4 to smithi02284960-21 2024-09-21T09:40:22.090 INFO:tasks.rados.rados.0.smithi022.stdout:2633: finishing write tid 5 to smithi02284960-21 2024-09-21T09:40:22.090 INFO:tasks.rados.rados.0.smithi022.stdout:2634: finishing rollback tid 0 to smithi02284960-1 2024-09-21T09:40:22.090 INFO:tasks.rados.rados.0.smithi022.stdout:2634: finishing rollback tid 1 to smithi02284960-1 2024-09-21T09:40:22.090 INFO:tasks.rados.rados.0.smithi022.stdout:2633: finishing write tid 6 to smithi02284960-21 2024-09-21T09:40:22.090 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 21 v 1416 (ObjNum 830 snap 266 seq_num 830) dirty exists 2024-09-21T09:40:22.090 INFO:tasks.rados.rados.0.smithi022.stdout:2633: left oid 21 (ObjNum 830 snap 266 seq_num 830) 2024-09-21T09:40:22.091 INFO:tasks.rados.rados.0.smithi022.stdout:2634: finishing rollback tid 2 to smithi02284960-1 2024-09-21T09:40:22.091 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 1 v 1508 (ObjNum 672 snap 208 seq_num 672) dirty exists 2024-09-21T09:40:22.169 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-09-21T09:40:22.169 INFO:teuthology.orchestra.run.smithi022.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47", 2024-09-21T09:40:22.169 INFO:teuthology.orchestra.run.smithi022.stdout: "in_progress": true, 2024-09-21T09:40:22.169 INFO:teuthology.orchestra.run.smithi022.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T09:40:22.169 INFO:teuthology.orchestra.run.smithi022.stdout: "services_complete": [ 2024-09-21T09:40:22.169 INFO:teuthology.orchestra.run.smithi022.stdout: "mon", 2024-09-21T09:40:22.169 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr", 2024-09-21T09:40:22.169 INFO:teuthology.orchestra.run.smithi022.stdout: "osd" 2024-09-21T09:40:22.169 INFO:teuthology.orchestra.run.smithi022.stdout: ], 2024-09-21T09:40:22.169 INFO:teuthology.orchestra.run.smithi022.stdout: "progress": "14/20 daemons upgraded", 2024-09-21T09:40:22.169 INFO:teuthology.orchestra.run.smithi022.stdout: "message": "Currently upgrading mds daemons", 2024-09-21T09:40:22.170 INFO:teuthology.orchestra.run.smithi022.stdout: "is_paused": false 2024-09-21T09:40:22.170 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-09-21T09:40:22.170 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:22 smithi022 ceph-mon[96736]: pgmap v499: 121 pgs: 121 active+clean; 199 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 4.7 MiB/s wr, 11 op/s 2024-09-21T09:40:22.170 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:22 smithi022 ceph-mon[96736]: from='client.34491 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:40:22.170 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:22 smithi022 ceph-mon[96736]: Upgrade: Updating mds.b 2024-09-21T09:40:22.170 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:22 smithi022 ceph-mon[96736]: Deploying daemon mds.b on smithi064 2024-09-21T09:40:22.170 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:22 smithi022 ceph-mon[96736]: osdmap e597: 8 total, 8 up, 8 in 2024-09-21T09:40:22.170 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:22 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/1391618876' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:40:22.171 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:22 smithi022 ceph-mon[98615]: pgmap v499: 121 pgs: 121 active+clean; 199 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 4.7 MiB/s wr, 11 op/s 2024-09-21T09:40:22.171 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:22 smithi022 ceph-mon[98615]: from='client.34491 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:40:22.171 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:22 smithi022 ceph-mon[98615]: Upgrade: Updating mds.b 2024-09-21T09:40:22.171 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:22 smithi022 ceph-mon[98615]: Deploying daemon mds.b on smithi064 2024-09-21T09:40:22.171 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:22 smithi022 ceph-mon[98615]: osdmap e597: 8 total, 8 up, 8 in 2024-09-21T09:40:22.171 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:22 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/1391618876' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:40:22.425 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:22 smithi064 ceph-mon[86709]: pgmap v499: 121 pgs: 121 active+clean; 199 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 4.7 MiB/s wr, 11 op/s 2024-09-21T09:40:22.425 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:22 smithi064 ceph-mon[86709]: from='client.34491 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:40:22.425 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:22 smithi064 ceph-mon[86709]: Upgrade: Updating mds.b 2024-09-21T09:40:22.425 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:22 smithi064 ceph-mon[86709]: Deploying daemon mds.b on smithi064 2024-09-21T09:40:22.425 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:22 smithi064 ceph-mon[86709]: osdmap e597: 8 total, 8 up, 8 in 2024-09-21T09:40:22.425 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:22 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/1391618876' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:40:22.990 INFO:journalctl@ceph.mds.b.smithi064.stdout:Sep 21 09:40:22 smithi064 systemd[1]: Stopping Ceph mds.b for d65e801e-77fa-11ef-baf6-efdc52797490... 2024-09-21T09:40:22.997 INFO:tasks.rados.rados.0.smithi022.stdout:2633: done (2 left) 2024-09-21T09:40:22.997 INFO:tasks.rados.rados.0.smithi022.stdout:2634: done (1 left) 2024-09-21T09:40:22.997 INFO:tasks.rados.rados.0.smithi022.stdout:2635: done (0 left) 2024-09-21T09:40:22.997 INFO:tasks.rados.rados.0.smithi022.stdout:2636: snap_create 2024-09-21T09:40:23.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:23 smithi022 ceph-mon[98615]: from='client.44426 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:40:23.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:23 smithi022 ceph-mon[98615]: from='client.44432 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:40:23.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:23 smithi022 ceph-mon[98615]: osdmap e598: 8 total, 8 up, 8 in 2024-09-21T09:40:23.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:23 smithi022 ceph-mon[98615]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-21T09:40:23.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:23 smithi022 ceph-mon[98615]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-21T09:40:23.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:23 smithi022 ceph-mon[98615]: osdmap e599: 8 total, 8 up, 8 in 2024-09-21T09:40:23.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:23 smithi022 ceph-mon[98615]: Standby daemon mds.a assigned to filesystem cephfs as rank 0 2024-09-21T09:40:23.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:23 smithi022 ceph-mon[98615]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-21T09:40:23.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:23 smithi022 ceph-mon[98615]: fsmap cephfs:0/1 1 up:standby, 1 failed 2024-09-21T09:40:23.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:23 smithi022 ceph-mon[98615]: fsmap cephfs:1/1 {0=a=up:replay} 2024-09-21T09:40:23.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:22 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mon-a[96691]: 2024-09-21T09:40:22.988+0000 7fe95466e640 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-21T09:40:23.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:23 smithi022 ceph-mon[96736]: from='client.44426 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:40:23.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:23 smithi022 ceph-mon[96736]: from='client.44432 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:40:23.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:23 smithi022 ceph-mon[96736]: osdmap e598: 8 total, 8 up, 8 in 2024-09-21T09:40:23.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:23 smithi022 ceph-mon[96736]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-21T09:40:23.310 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:23 smithi022 ceph-mon[96736]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-21T09:40:23.310 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:23 smithi022 ceph-mon[96736]: osdmap e599: 8 total, 8 up, 8 in 2024-09-21T09:40:23.310 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:23 smithi022 ceph-mon[96736]: Standby daemon mds.a assigned to filesystem cephfs as rank 0 2024-09-21T09:40:23.310 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:23 smithi022 ceph-mon[96736]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-21T09:40:23.310 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:23 smithi022 ceph-mon[96736]: fsmap cephfs:0/1 1 up:standby, 1 failed 2024-09-21T09:40:23.310 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:23 smithi022 ceph-mon[96736]: fsmap cephfs:1/1 {0=a=up:replay} 2024-09-21T09:40:23.441 INFO:journalctl@ceph.mds.b.smithi064.stdout:Sep 21 09:40:22 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mds-b[59631]: 2024-09-21T09:40:22.988+0000 7f76e8f39640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mds -n mds.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T09:40:23.442 INFO:journalctl@ceph.mds.b.smithi064.stdout:Sep 21 09:40:22 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mds-b[59631]: 2024-09-21T09:40:22.988+0000 7f76e8f39640 -1 mds.b *** got signal Terminated *** 2024-09-21T09:40:23.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:23 smithi064 ceph-mon[86709]: from='client.44426 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:40:23.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:23 smithi064 ceph-mon[86709]: from='client.44432 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:40:23.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:23 smithi064 ceph-mon[86709]: osdmap e598: 8 total, 8 up, 8 in 2024-09-21T09:40:23.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:23 smithi064 ceph-mon[86709]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-21T09:40:23.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:23 smithi064 ceph-mon[86709]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-21T09:40:23.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:23 smithi064 ceph-mon[86709]: osdmap e599: 8 total, 8 up, 8 in 2024-09-21T09:40:23.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:23 smithi064 ceph-mon[86709]: Standby daemon mds.a assigned to filesystem cephfs as rank 0 2024-09-21T09:40:23.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:23 smithi064 ceph-mon[86709]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-21T09:40:23.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:23 smithi064 ceph-mon[86709]: fsmap cephfs:0/1 1 up:standby, 1 failed 2024-09-21T09:40:23.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:23 smithi064 ceph-mon[86709]: fsmap cephfs:1/1 {0=a=up:replay} 2024-09-21T09:40:24.008 INFO:tasks.rados.rados.0.smithi022.stdout:2636: done (0 left) 2024-09-21T09:40:24.008 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 36 current snap is 268 2024-09-21T09:40:24.008 INFO:tasks.rados.rados.0.smithi022.stdout:2637: seq_num 831 ranges {1810432=720896,2531328=499712,3031040=704512} 2024-09-21T09:40:24.021 INFO:tasks.rados.rados.0.smithi022.stdout:2637: writing smithi02284960-36 from 1810432 to 2531328 tid 1 2024-09-21T09:40:24.025 INFO:tasks.rados.rados.0.smithi022.stdout:2637: writing smithi02284960-36 from 2531328 to 3031040 tid 2 2024-09-21T09:40:24.030 INFO:tasks.rados.rados.0.smithi022.stdout:2637: writing smithi02284960-36 from 3031040 to 3735552 tid 3 2024-09-21T09:40:24.030 INFO:tasks.rados.rados.0.smithi022.stdout:2638: snap_create 2024-09-21T09:40:24.031 INFO:tasks.rados.rados.0.smithi022.stdout:2637: finishing write tid 1 to smithi02284960-36 2024-09-21T09:40:24.031 INFO:tasks.rados.rados.0.smithi022.stdout:2637: finishing write tid 2 to smithi02284960-36 2024-09-21T09:40:24.033 INFO:tasks.rados.rados.0.smithi022.stdout:2637: finishing write tid 3 to smithi02284960-36 2024-09-21T09:40:24.033 INFO:tasks.rados.rados.0.smithi022.stdout:2637: finishing write tid 4 to smithi02284960-36 2024-09-21T09:40:24.034 INFO:tasks.rados.rados.0.smithi022.stdout:2637: finishing write tid 5 to smithi02284960-36 2024-09-21T09:40:24.034 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 36 v 1308 (ObjNum 831 snap 268 seq_num 831) dirty exists 2024-09-21T09:40:24.034 INFO:tasks.rados.rados.0.smithi022.stdout:2637: left oid 36 (ObjNum 831 snap 268 seq_num 831) 2024-09-21T09:40:24.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:24 smithi064 ceph-mon[86709]: from='client.34515 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:40:24.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:24 smithi064 ceph-mon[86709]: pgmap v502: 121 pgs: 121 active+clean; 191 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 9.9 MiB/s rd, 3.9 MiB/s wr, 8 op/s 2024-09-21T09:40:24.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:24 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "b"} : dispatch 2024-09-21T09:40:24.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:24 smithi064 ceph-mon[86709]: mds.? [v2:172.21.15.22:6834/4150258157,v1:172.21.15.22:6835/4150258157] up:reconnect 2024-09-21T09:40:24.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:24 smithi064 ceph-mon[86709]: fsmap cephfs:1/1 {0=a=up:reconnect} 2024-09-21T09:40:24.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:24 smithi064 ceph-mon[86709]: osdmap e600: 8 total, 8 up, 8 in 2024-09-21T09:40:24.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:24 smithi064 ceph-mon[86709]: osdmap e601: 8 total, 8 up, 8 in 2024-09-21T09:40:24.498 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:24 smithi022 ceph-mon[98615]: from='client.34515 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:40:24.498 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:24 smithi022 ceph-mon[98615]: pgmap v502: 121 pgs: 121 active+clean; 191 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 9.9 MiB/s rd, 3.9 MiB/s wr, 8 op/s 2024-09-21T09:40:24.499 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:24 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "b"} : dispatch 2024-09-21T09:40:24.499 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:24 smithi022 ceph-mon[98615]: mds.? [v2:172.21.15.22:6834/4150258157,v1:172.21.15.22:6835/4150258157] up:reconnect 2024-09-21T09:40:24.499 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:24 smithi022 ceph-mon[98615]: fsmap cephfs:1/1 {0=a=up:reconnect} 2024-09-21T09:40:24.499 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:24 smithi022 ceph-mon[98615]: osdmap e600: 8 total, 8 up, 8 in 2024-09-21T09:40:24.499 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:24 smithi022 ceph-mon[98615]: osdmap e601: 8 total, 8 up, 8 in 2024-09-21T09:40:24.499 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:24 smithi022 ceph-mon[96736]: from='client.34515 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:40:24.499 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:24 smithi022 ceph-mon[96736]: pgmap v502: 121 pgs: 121 active+clean; 191 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 9.9 MiB/s rd, 3.9 MiB/s wr, 8 op/s 2024-09-21T09:40:24.499 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:24 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "b"} : dispatch 2024-09-21T09:40:24.499 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:24 smithi022 ceph-mon[96736]: mds.? [v2:172.21.15.22:6834/4150258157,v1:172.21.15.22:6835/4150258157] up:reconnect 2024-09-21T09:40:24.499 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:24 smithi022 ceph-mon[96736]: fsmap cephfs:1/1 {0=a=up:reconnect} 2024-09-21T09:40:24.500 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:24 smithi022 ceph-mon[96736]: osdmap e600: 8 total, 8 up, 8 in 2024-09-21T09:40:24.500 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:24 smithi022 ceph-mon[96736]: osdmap e601: 8 total, 8 up, 8 in 2024-09-21T09:40:25.027 INFO:tasks.rados.rados.0.smithi022.stdout:2637: done (1 left) 2024-09-21T09:40:25.027 INFO:tasks.rados.rados.0.smithi022.stdout:2638: done (0 left) 2024-09-21T09:40:25.027 INFO:tasks.rados.rados.0.smithi022.stdout:2639: read oid 5 snap 257 2024-09-21T09:40:25.027 INFO:tasks.rados.rados.0.smithi022.stdout:2639: expect (ObjNum 672 snap 208 seq_num 672) 2024-09-21T09:40:25.027 INFO:tasks.rados.rados.0.smithi022.stdout:2640: rollback oid 9 current snap is 269 2024-09-21T09:40:25.027 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 9 to 246 2024-09-21T09:40:25.028 INFO:tasks.rados.rados.0.smithi022.stdout:2641: read oid 24 snap -1 2024-09-21T09:40:25.028 INFO:tasks.rados.rados.0.smithi022.stdout:2641: expect (ObjNum 783 snap 246 seq_num 783) 2024-09-21T09:40:25.028 INFO:tasks.rados.rados.0.smithi022.stdout:2642: delete oid 26 current snap is 269 2024-09-21T09:40:25.035 INFO:tasks.rados.rados.0.smithi022.stdout:2640: finishing rollback tid 0 to smithi02284960-9 2024-09-21T09:40:25.035 INFO:tasks.rados.rados.0.smithi022.stdout:2640: finishing rollback tid 1 to smithi02284960-9 2024-09-21T09:40:25.035 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 9 v 1404 (ObjNum 33 snap 0 seq_num 3569625360) dirty dne 2024-09-21T09:40:25.037 INFO:tasks.rados.rados.0.smithi022.stdout:2639: expect (ObjNum 672 snap 208 seq_num 672) 2024-09-21T09:40:25.085 INFO:tasks.rados.rados.0.smithi022.stdout:2641: expect (ObjNum 783 snap 246 seq_num 783) 2024-09-21T09:40:25.158 INFO:tasks.rados.rados.0.smithi022.stdout:2639: done (3 left) 2024-09-21T09:40:25.158 INFO:tasks.rados.rados.0.smithi022.stdout:2640: done (2 left) 2024-09-21T09:40:25.159 INFO:tasks.rados.rados.0.smithi022.stdout:2641: done (1 left) 2024-09-21T09:40:25.159 INFO:tasks.rados.rados.0.smithi022.stdout:2642: done (0 left) 2024-09-21T09:40:25.159 INFO:tasks.rados.rados.0.smithi022.stdout:2643: delete oid 40 current snap is 269 2024-09-21T09:40:25.161 INFO:tasks.rados.rados.0.smithi022.stdout:2643: done (0 left) 2024-09-21T09:40:25.161 INFO:tasks.rados.rados.0.smithi022.stdout:2644: snap_remove snap 258 2024-09-21T09:40:25.808 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:40:25 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:40:25] "GET /metrics HTTP/1.1" 200 36120 "" "Prometheus/2.51.0" 2024-09-21T09:40:26.067 INFO:tasks.rados.rados.0.smithi022.stdout:2644: done (0 left) 2024-09-21T09:40:26.067 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 5 current snap is 269 2024-09-21T09:40:26.067 INFO:tasks.rados.rados.0.smithi022.stdout:2645: seq_num 834 ranges {1662976=778240,2441216=557056,2998272=548864,3547136=98304} 2024-09-21T09:40:26.080 INFO:tasks.rados.rados.0.smithi022.stdout:2645: writing smithi02284960-5 from 1662976 to 2441216 tid 1 2024-09-21T09:40:26.084 INFO:tasks.rados.rados.0.smithi022.stdout:2645: writing smithi02284960-5 from 2441216 to 2998272 tid 2 2024-09-21T09:40:26.088 INFO:tasks.rados.rados.0.smithi022.stdout:2645: writing smithi02284960-5 from 2998272 to 3547136 tid 3 2024-09-21T09:40:26.089 INFO:tasks.rados.rados.0.smithi022.stdout:2645: writing smithi02284960-5 from 3547136 to 3645440 tid 4 2024-09-21T09:40:26.089 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 24 current snap is 269 2024-09-21T09:40:26.089 INFO:tasks.rados.rados.0.smithi022.stdout:2646: seq_num 835 ranges {2146304=794624,2940928=442368,3383296=712704,4096000=49152} 2024-09-21T09:40:26.106 INFO:tasks.rados.rados.0.smithi022.stdout:2646: writing smithi02284960-24 from 2146304 to 2940928 tid 1 2024-09-21T09:40:26.108 INFO:tasks.rados.rados.0.smithi022.stdout:2646: writing smithi02284960-24 from 2940928 to 3383296 tid 2 2024-09-21T09:40:26.112 INFO:tasks.rados.rados.0.smithi022.stdout:2646: writing smithi02284960-24 from 3383296 to 4096000 tid 3 2024-09-21T09:40:26.113 INFO:tasks.rados.rados.0.smithi022.stdout:2646: writing smithi02284960-24 from 4096000 to 4145152 tid 4 2024-09-21T09:40:26.113 INFO:tasks.rados.rados.0.smithi022.stdout:2647: rollback oid 13 current snap is 269 2024-09-21T09:40:26.113 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 13 to 254 2024-09-21T09:40:26.113 INFO:tasks.rados.rados.0.smithi022.stdout:2645: finishing write tid 1 to smithi02284960-5 2024-09-21T09:40:26.113 INFO:tasks.rados.rados.0.smithi022.stdout:2645: finishing write tid 2 to smithi02284960-5 2024-09-21T09:40:26.113 INFO:tasks.rados.rados.0.smithi022.stdout:2645: finishing write tid 3 to smithi02284960-5 2024-09-21T09:40:26.113 INFO:tasks.rados.rados.0.smithi022.stdout:2645: finishing write tid 4 to smithi02284960-5 2024-09-21T09:40:26.113 INFO:tasks.rados.rados.0.smithi022.stdout:2645: finishing write tid 5 to smithi02284960-5 2024-09-21T09:40:26.114 INFO:tasks.rados.rados.0.smithi022.stdout:2645: finishing write tid 6 to smithi02284960-5 2024-09-21T09:40:26.114 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 5 v 1409 (ObjNum 834 snap 269 seq_num 834) dirty exists 2024-09-21T09:40:26.114 INFO:tasks.rados.rados.0.smithi022.stdout:2645: left oid 5 (ObjNum 834 snap 269 seq_num 834) 2024-09-21T09:40:26.114 INFO:tasks.rados.rados.0.smithi022.stdout:2646: finishing write tid 1 to smithi02284960-24 2024-09-21T09:40:26.114 INFO:tasks.rados.rados.0.smithi022.stdout:2646: finishing write tid 2 to smithi02284960-24 2024-09-21T09:40:26.114 INFO:tasks.rados.rados.0.smithi022.stdout:2645: done (2 left) 2024-09-21T09:40:26.114 INFO:tasks.rados.rados.0.smithi022.stdout:2648: delete oid 45 current snap is 269 2024-09-21T09:40:26.115 INFO:tasks.rados.rados.0.smithi022.stdout:2648: done (2 left) 2024-09-21T09:40:26.115 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 9 current snap is 269 2024-09-21T09:40:26.115 INFO:tasks.rados.rados.0.smithi022.stdout:2649: seq_num 837 ranges {0=425984,425984=655360,1081344=729088,1810432=221184} 2024-09-21T09:40:26.117 INFO:tasks.rados.rados.0.smithi022.stdout:2649: writing smithi02284960-9 from 0 to 425984 tid 1 2024-09-21T09:40:26.121 INFO:tasks.rados.rados.0.smithi022.stdout:2649: writing smithi02284960-9 from 425984 to 1081344 tid 2 2024-09-21T09:40:26.125 INFO:tasks.rados.rados.0.smithi022.stdout:2649: writing smithi02284960-9 from 1081344 to 1810432 tid 3 2024-09-21T09:40:26.128 INFO:tasks.rados.rados.0.smithi022.stdout:2649: writing smithi02284960-9 from 1810432 to 2031616 tid 4 2024-09-21T09:40:26.128 INFO:tasks.rados.rados.0.smithi022.stdout:2650: delete oid 44 current snap is 269 2024-09-21T09:40:26.128 INFO:tasks.rados.rados.0.smithi022.stdout:2647: finishing rollback tid 1 to smithi02284960-13 2024-09-21T09:40:26.128 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 13 v 0 (ObjNum 161 snap 0 seq_num 1) dirty dne 2024-09-21T09:40:26.128 INFO:tasks.rados.rados.0.smithi022.stdout:2646: finishing write tid 3 to smithi02284960-24 2024-09-21T09:40:26.128 INFO:tasks.rados.rados.0.smithi022.stdout:2646: finishing write tid 4 to smithi02284960-24 2024-09-21T09:40:26.128 INFO:tasks.rados.rados.0.smithi022.stdout:2646: finishing write tid 5 to smithi02284960-24 2024-09-21T09:40:26.128 INFO:tasks.rados.rados.0.smithi022.stdout:2646: finishing write tid 6 to smithi02284960-24 2024-09-21T09:40:26.128 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 24 v 1536 (ObjNum 835 snap 269 seq_num 835) dirty exists 2024-09-21T09:40:26.128 INFO:tasks.rados.rados.0.smithi022.stdout:2646: left oid 24 (ObjNum 835 snap 269 seq_num 835) 2024-09-21T09:40:26.128 INFO:tasks.rados.rados.0.smithi022.stdout:2649: finishing write tid 1 to smithi02284960-9 2024-09-21T09:40:26.128 INFO:tasks.rados.rados.0.smithi022.stdout:2649: finishing write tid 2 to smithi02284960-9 2024-09-21T09:40:26.129 INFO:tasks.rados.rados.0.smithi022.stdout:2646: done (3 left) 2024-09-21T09:40:26.129 INFO:tasks.rados.rados.0.smithi022.stdout:2647: done (2 left) 2024-09-21T09:40:26.129 INFO:tasks.rados.rados.0.smithi022.stdout:2650: done (1 left) 2024-09-21T09:40:26.129 INFO:tasks.rados.rados.0.smithi022.stdout:2651: snap_remove snap 263 2024-09-21T09:40:26.307 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:26 smithi022 ceph-mon[96736]: pgmap v506: 121 pgs: 121 active+clean; 192 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 1.6 MiB/s wr, 13 op/s 2024-09-21T09:40:26.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:26 smithi022 ceph-mon[96736]: mds.? [v2:172.21.15.22:6834/4150258157,v1:172.21.15.22:6835/4150258157] up:rejoin 2024-09-21T09:40:26.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:26 smithi022 ceph-mon[96736]: fsmap cephfs:1/1 {0=a=up:rejoin} 2024-09-21T09:40:26.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:26 smithi022 ceph-mon[96736]: daemon mds.a is now active in filesystem cephfs as rank 0 2024-09-21T09:40:26.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:26 smithi022 ceph-mon[96736]: osdmap e602: 8 total, 8 up, 8 in 2024-09-21T09:40:26.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:26 smithi022 ceph-mon[98615]: pgmap v506: 121 pgs: 121 active+clean; 192 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 1.6 MiB/s wr, 13 op/s 2024-09-21T09:40:26.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:26 smithi022 ceph-mon[98615]: mds.? [v2:172.21.15.22:6834/4150258157,v1:172.21.15.22:6835/4150258157] up:rejoin 2024-09-21T09:40:26.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:26 smithi022 ceph-mon[98615]: fsmap cephfs:1/1 {0=a=up:rejoin} 2024-09-21T09:40:26.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:26 smithi022 ceph-mon[98615]: daemon mds.a is now active in filesystem cephfs as rank 0 2024-09-21T09:40:26.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:26 smithi022 ceph-mon[98615]: osdmap e602: 8 total, 8 up, 8 in 2024-09-21T09:40:26.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:26 smithi064 ceph-mon[86709]: pgmap v506: 121 pgs: 121 active+clean; 192 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 1.6 MiB/s wr, 13 op/s 2024-09-21T09:40:26.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:26 smithi064 ceph-mon[86709]: mds.? [v2:172.21.15.22:6834/4150258157,v1:172.21.15.22:6835/4150258157] up:rejoin 2024-09-21T09:40:26.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:26 smithi064 ceph-mon[86709]: fsmap cephfs:1/1 {0=a=up:rejoin} 2024-09-21T09:40:26.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:26 smithi064 ceph-mon[86709]: daemon mds.a is now active in filesystem cephfs as rank 0 2024-09-21T09:40:26.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:26 smithi064 ceph-mon[86709]: osdmap e602: 8 total, 8 up, 8 in 2024-09-21T09:40:27.075 INFO:tasks.rados.rados.0.smithi022.stdout:2651: done (1 left) 2024-09-21T09:40:27.075 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 38 current snap is 269 2024-09-21T09:40:27.075 INFO:tasks.rados.rados.0.smithi022.stdout:2652: seq_num 839 ranges {0=458752,458752=425984,884736=688128,1572864=491520} 2024-09-21T09:40:27.078 INFO:tasks.rados.rados.0.smithi022.stdout:2652: writing smithi02284960-38 from 0 to 458752 tid 1 2024-09-21T09:40:27.081 INFO:tasks.rados.rados.0.smithi022.stdout:2652: writing smithi02284960-38 from 458752 to 884736 tid 2 2024-09-21T09:40:27.086 INFO:tasks.rados.rados.0.smithi022.stdout:2652: writing smithi02284960-38 from 884736 to 1572864 tid 3 2024-09-21T09:40:27.090 INFO:tasks.rados.rados.0.smithi022.stdout:2652: writing smithi02284960-38 from 1572864 to 2064384 tid 4 2024-09-21T09:40:27.090 INFO:tasks.rados.rados.0.smithi022.stdout:2653: snap_remove snap 259 2024-09-21T09:40:27.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:27 smithi022 ceph-mon[98615]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-21T09:40:27.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:27 smithi022 ceph-mon[98615]: Cluster is now healthy 2024-09-21T09:40:27.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:27 smithi022 ceph-mon[98615]: mds.? [v2:172.21.15.22:6834/4150258157,v1:172.21.15.22:6835/4150258157] up:active 2024-09-21T09:40:27.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:27 smithi022 ceph-mon[98615]: fsmap cephfs:1 {0=a=up:active} 2024-09-21T09:40:27.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:27 smithi022 ceph-mon[98615]: osdmap e603: 8 total, 8 up, 8 in 2024-09-21T09:40:27.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:27 smithi022 ceph-mon[96736]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-21T09:40:27.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:27 smithi022 ceph-mon[96736]: Cluster is now healthy 2024-09-21T09:40:27.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:27 smithi022 ceph-mon[96736]: mds.? [v2:172.21.15.22:6834/4150258157,v1:172.21.15.22:6835/4150258157] up:active 2024-09-21T09:40:27.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:27 smithi022 ceph-mon[96736]: fsmap cephfs:1 {0=a=up:active} 2024-09-21T09:40:27.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:27 smithi022 ceph-mon[96736]: osdmap e603: 8 total, 8 up, 8 in 2024-09-21T09:40:27.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:27 smithi064 ceph-mon[86709]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-21T09:40:27.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:27 smithi064 ceph-mon[86709]: Cluster is now healthy 2024-09-21T09:40:27.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:27 smithi064 ceph-mon[86709]: mds.? [v2:172.21.15.22:6834/4150258157,v1:172.21.15.22:6835/4150258157] up:active 2024-09-21T09:40:27.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:27 smithi064 ceph-mon[86709]: fsmap cephfs:1 {0=a=up:active} 2024-09-21T09:40:27.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:27 smithi064 ceph-mon[86709]: osdmap e603: 8 total, 8 up, 8 in 2024-09-21T09:40:28.081 INFO:tasks.rados.rados.0.smithi022.stdout:2653: done (2 left) 2024-09-21T09:40:28.081 INFO:tasks.rados.rados.0.smithi022.stdout:2654: copy_from oid 29 from oid 2 current snap is 269 2024-09-21T09:40:28.081 INFO:tasks.rados.rados.0.smithi022.stdout:2649: finishing write tid 3 to smithi02284960-9 2024-09-21T09:40:28.081 INFO:tasks.rados.rados.0.smithi022.stdout:2649: finishing write tid 4 to smithi02284960-9 2024-09-21T09:40:28.081 INFO:tasks.rados.rados.0.smithi022.stdout:2649: finishing write tid 5 to smithi02284960-9 2024-09-21T09:40:28.081 INFO:tasks.rados.rados.0.smithi022.stdout:2649: finishing write tid 6 to smithi02284960-9 2024-09-21T09:40:28.081 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 9 v 1414 (ObjNum 837 snap 269 seq_num 837) dirty exists 2024-09-21T09:40:28.081 INFO:tasks.rados.rados.0.smithi022.stdout:2649: left oid 9 (ObjNum 837 snap 269 seq_num 837) 2024-09-21T09:40:28.082 INFO:tasks.rados.rados.0.smithi022.stdout:2652: finishing write tid 1 to smithi02284960-38 2024-09-21T09:40:28.082 INFO:tasks.rados.rados.0.smithi022.stdout:2652: finishing write tid 2 to smithi02284960-38 2024-09-21T09:40:28.082 INFO:tasks.rados.rados.0.smithi022.stdout:2652: finishing write tid 3 to smithi02284960-38 2024-09-21T09:40:28.082 INFO:tasks.rados.rados.0.smithi022.stdout:2652: finishing write tid 4 to smithi02284960-38 2024-09-21T09:40:28.082 INFO:tasks.rados.rados.0.smithi022.stdout:2652: finishing write tid 5 to smithi02284960-38 2024-09-21T09:40:28.082 INFO:tasks.rados.rados.0.smithi022.stdout:2652: finishing write tid 6 to smithi02284960-38 2024-09-21T09:40:28.082 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 38 v 1414 (ObjNum 839 snap 269 seq_num 839) dirty exists 2024-09-21T09:40:28.082 INFO:tasks.rados.rados.0.smithi022.stdout:2652: left oid 38 (ObjNum 839 snap 269 seq_num 839) 2024-09-21T09:40:28.082 INFO:tasks.rados.rados.0.smithi022.stdout:2649: done (2 left) 2024-09-21T09:40:28.082 INFO:tasks.rados.rados.0.smithi022.stdout:2652: done (1 left) 2024-09-21T09:40:28.082 INFO:tasks.rados.rados.0.smithi022.stdout:2655: read oid 42 snap 265 2024-09-21T09:40:28.082 INFO:tasks.rados.rados.0.smithi022.stdout:2655: expect (ObjNum -724420080 snap 22094 seq_num 3569522432) 2024-09-21T09:40:28.083 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 37 current snap is 269 2024-09-21T09:40:28.083 INFO:tasks.rados.rados.0.smithi022.stdout:2656: seq_num 840 ranges {0=475136,475136=712704,1187840=499712,1687552=393216} 2024-09-21T09:40:28.084 INFO:tasks.rados.rados.0.smithi022.stdout:2656: writing smithi02284960-37 from 0 to 475136 tid 1 2024-09-21T09:40:28.089 INFO:tasks.rados.rados.0.smithi022.stdout:2656: writing smithi02284960-37 from 475136 to 1187840 tid 2 2024-09-21T09:40:28.092 INFO:tasks.rados.rados.0.smithi022.stdout:2656: writing smithi02284960-37 from 1187840 to 1687552 tid 3 2024-09-21T09:40:28.095 INFO:tasks.rados.rados.0.smithi022.stdout:2656: writing smithi02284960-37 from 1687552 to 2080768 tid 4 2024-09-21T09:40:28.095 INFO:tasks.rados.rados.0.smithi022.stdout:2657: snap_remove snap 268 2024-09-21T09:40:28.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:28 smithi064 ceph-mon[86709]: pgmap v509: 121 pgs: 121 active+clean; 195 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 5.2 MiB/s rd, 2.1 MiB/s wr, 22 op/s 2024-09-21T09:40:28.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:28 smithi064 ceph-mon[86709]: osdmap e604: 8 total, 8 up, 8 in 2024-09-21T09:40:28.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:28 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:40:28.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:28 smithi022 ceph-mon[98615]: pgmap v509: 121 pgs: 121 active+clean; 195 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 5.2 MiB/s rd, 2.1 MiB/s wr, 22 op/s 2024-09-21T09:40:28.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:28 smithi022 ceph-mon[98615]: osdmap e604: 8 total, 8 up, 8 in 2024-09-21T09:40:28.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:28 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:40:28.558 INFO:journalctl@ceph.mds.a.smithi022.stdout:Sep 21 09:40:28 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mds-a[123008]: 2024-09-21T09:40:28.170+0000 7ff8ba1d6640 -1 mds.pinger is_rank_lagging: rank=0 was never sent ping request. 2024-09-21T09:40:28.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:28 smithi022 ceph-mon[96736]: pgmap v509: 121 pgs: 121 active+clean; 195 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 5.2 MiB/s rd, 2.1 MiB/s wr, 22 op/s 2024-09-21T09:40:28.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:28 smithi022 ceph-mon[96736]: osdmap e604: 8 total, 8 up, 8 in 2024-09-21T09:40:28.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:28 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:40:28.941 INFO:journalctl@ceph.mds.b.smithi064.stdout:Sep 21 09:40:28 smithi064 podman[109991]: 2024-09-21 09:40:28.682691829 +0000 UTC m=+5.815451843 container died 0f846b3ec6947704b96b602066283327e11f659fdf60a4ce5c5e0f40a4c41781 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mds-b, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=quincy-fbd1a6a, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, ceph=True) 2024-09-21T09:40:29.029 INFO:tasks.rados.rados.0.smithi022.stdout:2657: done (3 left) 2024-09-21T09:40:29.029 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 28 current snap is 269 2024-09-21T09:40:29.029 INFO:tasks.rados.rados.0.smithi022.stdout:2658: seq_num 841 ranges {2392064=491520,2883584=598016,3481600=704512,4186112=303104} 2024-09-21T09:40:29.045 INFO:tasks.rados.rados.0.smithi022.stdout:2658: writing smithi02284960-28 from 2392064 to 2883584 tid 1 2024-09-21T09:40:29.049 INFO:tasks.rados.rados.0.smithi022.stdout:2658: writing smithi02284960-28 from 2883584 to 3481600 tid 2 2024-09-21T09:40:29.053 INFO:tasks.rados.rados.0.smithi022.stdout:2658: writing smithi02284960-28 from 3481600 to 4186112 tid 3 2024-09-21T09:40:29.055 INFO:tasks.rados.rados.0.smithi022.stdout:2658: writing smithi02284960-28 from 4186112 to 4489216 tid 4 2024-09-21T09:40:29.055 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 17 current snap is 269 2024-09-21T09:40:29.055 INFO:tasks.rados.rados.0.smithi022.stdout:2659: seq_num 842 ranges {17309696=507904,17817600=475136,18292736=516096,18808832=499712,19308544=114688} 2024-09-21T09:40:29.153 INFO:tasks.rados.rados.0.smithi022.stdout:2659: writing smithi02284960-17 from 17309696 to 17817600 tid 1 2024-09-21T09:40:29.156 INFO:tasks.rados.rados.0.smithi022.stdout:2659: writing smithi02284960-17 from 17817600 to 18292736 tid 2 2024-09-21T09:40:29.159 INFO:tasks.rados.rados.0.smithi022.stdout:2659: writing smithi02284960-17 from 18292736 to 18808832 tid 3 2024-09-21T09:40:29.162 INFO:tasks.rados.rados.0.smithi022.stdout:2659: writing smithi02284960-17 from 18808832 to 19308544 tid 4 2024-09-21T09:40:29.163 INFO:tasks.rados.rados.0.smithi022.stdout:2659: writing smithi02284960-17 from 19308544 to 19423232 tid 5 2024-09-21T09:40:29.163 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 4 current snap is 269 2024-09-21T09:40:29.163 INFO:tasks.rados.rados.0.smithi022.stdout:2655: expect (ObjNum -724420080 snap 22094 seq_num 3569522432) 2024-09-21T09:40:29.165 INFO:tasks.rados.rados.0.smithi022.stdout:2654: finishing copy_from to smithi02284960-29 2024-09-21T09:40:29.165 INFO:tasks.rados.rados.0.smithi022.stdout:2654: got expected ENOENT (src dne) 2024-09-21T09:40:29.166 INFO:tasks.rados.rados.0.smithi022.stdout:2654: finishing copy_from racing read to smithi02284960-29 2024-09-21T09:40:29.166 INFO:tasks.rados.rados.0.smithi022.stdout:2656: finishing write tid 1 to smithi02284960-37 2024-09-21T09:40:29.166 INFO:tasks.rados.rados.0.smithi022.stdout:2656: finishing write tid 2 to smithi02284960-37 2024-09-21T09:40:29.166 INFO:tasks.rados.rados.0.smithi022.stdout:2656: finishing write tid 3 to smithi02284960-37 2024-09-21T09:40:29.166 INFO:tasks.rados.rados.0.smithi022.stdout:2656: finishing write tid 4 to smithi02284960-37 2024-09-21T09:40:29.166 INFO:tasks.rados.rados.0.smithi022.stdout:2656: finishing write tid 5 to smithi02284960-37 2024-09-21T09:40:29.166 INFO:tasks.rados.rados.0.smithi022.stdout:2656: finishing write tid 6 to smithi02284960-37 2024-09-21T09:40:29.166 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 37 v 1553 (ObjNum 840 snap 269 seq_num 840) dirty exists 2024-09-21T09:40:29.166 INFO:tasks.rados.rados.0.smithi022.stdout:2656: left oid 37 (ObjNum 840 snap 269 seq_num 840) 2024-09-21T09:40:29.166 INFO:tasks.rados.rados.0.smithi022.stdout:2660: seq_num 843 ranges {16334848=524288,16859136=761856,17620992=671744,18292736=172032} 2024-09-21T09:40:29.257 INFO:tasks.rados.rados.0.smithi022.stdout:2660: writing smithi02284960-4 from 16334848 to 16859136 tid 1 2024-09-21T09:40:29.262 INFO:tasks.rados.rados.0.smithi022.stdout:2660: writing smithi02284960-4 from 16859136 to 17620992 tid 2 2024-09-21T09:40:29.265 INFO:tasks.rados.rados.0.smithi022.stdout:2660: writing smithi02284960-4 from 17620992 to 18292736 tid 3 2024-09-21T09:40:29.267 INFO:tasks.rados.rados.0.smithi022.stdout:2660: writing smithi02284960-4 from 18292736 to 18464768 tid 4 2024-09-21T09:40:29.267 INFO:tasks.rados.rados.0.smithi022.stdout:2654: done (5 left) 2024-09-21T09:40:29.267 INFO:tasks.rados.rados.0.smithi022.stdout:2655: done (4 left) 2024-09-21T09:40:29.267 INFO:tasks.rados.rados.0.smithi022.stdout:2656: done (3 left) 2024-09-21T09:40:29.268 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 48 current snap is 269 2024-09-21T09:40:29.268 INFO:tasks.rados.rados.0.smithi022.stdout:2658: finishing write tid 1 to smithi02284960-28 2024-09-21T09:40:29.268 INFO:tasks.rados.rados.0.smithi022.stdout:2658: finishing write tid 2 to smithi02284960-28 2024-09-21T09:40:29.268 INFO:tasks.rados.rados.0.smithi022.stdout:2658: finishing write tid 3 to smithi02284960-28 2024-09-21T09:40:29.268 INFO:tasks.rados.rados.0.smithi022.stdout:2658: finishing write tid 4 to smithi02284960-28 2024-09-21T09:40:29.268 INFO:tasks.rados.rados.0.smithi022.stdout:2658: finishing write tid 5 to smithi02284960-28 2024-09-21T09:40:29.268 INFO:tasks.rados.rados.0.smithi022.stdout:2658: finishing write tid 6 to smithi02284960-28 2024-09-21T09:40:29.268 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 28 v 1406 (ObjNum 841 snap 269 seq_num 841) dirty exists 2024-09-21T09:40:29.268 INFO:tasks.rados.rados.0.smithi022.stdout:2658: left oid 28 (ObjNum 841 snap 269 seq_num 841) 2024-09-21T09:40:29.268 INFO:tasks.rados.rados.0.smithi022.stdout:2659: finishing write tid 1 to smithi02284960-17 2024-09-21T09:40:29.268 INFO:tasks.rados.rados.0.smithi022.stdout:2659: finishing write tid 2 to smithi02284960-17 2024-09-21T09:40:29.268 INFO:tasks.rados.rados.0.smithi022.stdout:2659: finishing write tid 3 to smithi02284960-17 2024-09-21T09:40:29.268 INFO:tasks.rados.rados.0.smithi022.stdout:2661: seq_num 844 ranges {1835008=540672,2375680=696320,3072000=475136,3547136=434176} 2024-09-21T09:40:29.279 INFO:tasks.rados.rados.0.smithi022.stdout:2661: writing smithi02284960-48 from 1835008 to 2375680 tid 1 2024-09-21T09:40:29.284 INFO:tasks.rados.rados.0.smithi022.stdout:2661: writing smithi02284960-48 from 2375680 to 3072000 tid 2 2024-09-21T09:40:29.286 INFO:tasks.rados.rados.0.smithi022.stdout:2661: writing smithi02284960-48 from 3072000 to 3547136 tid 3 2024-09-21T09:40:29.290 INFO:tasks.rados.rados.0.smithi022.stdout:2661: writing smithi02284960-48 from 3547136 to 3981312 tid 4 2024-09-21T09:40:29.290 INFO:tasks.rados.rados.0.smithi022.stdout:2658: done (3 left) 2024-09-21T09:40:29.290 INFO:tasks.rados.rados.0.smithi022.stdout:2662: rollback oid 1 current snap is 269 2024-09-21T09:40:29.290 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 1 to 266 2024-09-21T09:40:29.290 INFO:tasks.rados.rados.0.smithi022.stdout:2659: finishing write tid 4 to smithi02284960-17 2024-09-21T09:40:29.290 INFO:tasks.rados.rados.0.smithi022.stdout:2659: finishing write tid 5 to smithi02284960-17 2024-09-21T09:40:29.290 INFO:tasks.rados.rados.0.smithi022.stdout:2659: finishing write tid 6 to smithi02284960-17 2024-09-21T09:40:29.291 INFO:tasks.rados.rados.0.smithi022.stdout:2659: finishing write tid 7 to smithi02284960-17 2024-09-21T09:40:29.291 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 17 v 1292 (ObjNum 842 snap 269 seq_num 842) dirty exists 2024-09-21T09:40:29.291 INFO:tasks.rados.rados.0.smithi022.stdout:2659: left oid 17 (ObjNum 842 snap 269 seq_num 842) 2024-09-21T09:40:29.291 INFO:tasks.rados.rados.0.smithi022.stdout:2660: finishing write tid 1 to smithi02284960-4 2024-09-21T09:40:29.291 INFO:tasks.rados.rados.0.smithi022.stdout:2660: finishing write tid 2 to smithi02284960-4 2024-09-21T09:40:29.291 INFO:tasks.rados.rados.0.smithi022.stdout:2659: done (3 left) 2024-09-21T09:40:29.291 INFO:tasks.rados.rados.0.smithi022.stdout:2663: snap_create 2024-09-21T09:40:29.291 INFO:tasks.rados.rados.0.smithi022.stdout:2660: finishing write tid 3 to smithi02284960-4 2024-09-21T09:40:29.291 INFO:tasks.rados.rados.0.smithi022.stdout:2660: finishing write tid 4 to smithi02284960-4 2024-09-21T09:40:29.291 INFO:tasks.rados.rados.0.smithi022.stdout:2660: finishing write tid 5 to smithi02284960-4 2024-09-21T09:40:29.291 INFO:tasks.rados.rados.0.smithi022.stdout:2660: finishing write tid 6 to smithi02284960-4 2024-09-21T09:40:29.291 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 4 v 1419 (ObjNum 843 snap 269 seq_num 843) dirty exists 2024-09-21T09:40:29.291 INFO:tasks.rados.rados.0.smithi022.stdout:2660: left oid 4 (ObjNum 843 snap 269 seq_num 843) 2024-09-21T09:40:29.291 INFO:tasks.rados.rados.0.smithi022.stdout:2661: finishing write tid 1 to smithi02284960-48 2024-09-21T09:40:29.292 INFO:tasks.rados.rados.0.smithi022.stdout:2661: finishing write tid 2 to smithi02284960-48 2024-09-21T09:40:29.292 INFO:tasks.rados.rados.0.smithi022.stdout:2661: finishing write tid 3 to smithi02284960-48 2024-09-21T09:40:29.292 INFO:tasks.rados.rados.0.smithi022.stdout:2661: finishing write tid 4 to smithi02284960-48 2024-09-21T09:40:29.292 INFO:tasks.rados.rados.0.smithi022.stdout:2661: finishing write tid 5 to smithi02284960-48 2024-09-21T09:40:29.294 INFO:tasks.rados.rados.0.smithi022.stdout:2661: finishing write tid 6 to smithi02284960-48 2024-09-21T09:40:29.294 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 48 v 1611 (ObjNum 844 snap 269 seq_num 844) dirty exists 2024-09-21T09:40:29.294 INFO:tasks.rados.rados.0.smithi022.stdout:2661: left oid 48 (ObjNum 844 snap 269 seq_num 844) 2024-09-21T09:40:29.294 INFO:tasks.rados.rados.0.smithi022.stdout:2662: finishing rollback tid 0 to smithi02284960-1 2024-09-21T09:40:29.294 INFO:tasks.rados.rados.0.smithi022.stdout:2662: finishing rollback tid 1 to smithi02284960-1 2024-09-21T09:40:29.294 INFO:tasks.rados.rados.0.smithi022.stdout:2662: finishing rollback tid 2 to smithi02284960-1 2024-09-21T09:40:29.294 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 1 v 1511 (ObjNum 672 snap 208 seq_num 672) dirty exists 2024-09-21T09:40:29.355 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:29 smithi064 ceph-mon[86709]: osdmap e605: 8 total, 8 up, 8 in 2024-09-21T09:40:29.355 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:29 smithi064 ceph-mon[86709]: osdmap e606: 8 total, 8 up, 8 in 2024-09-21T09:40:29.355 INFO:journalctl@ceph.mds.b.smithi064.stdout:Sep 21 09:40:29 smithi064 podman[109991]: 2024-09-21 09:40:29.093823644 +0000 UTC m=+6.226583650 container cleanup 0f846b3ec6947704b96b602066283327e11f659fdf60a4ce5c5e0f40a4c41781 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mds-b, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, RELEASE=quincy-fbd1a6a, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.37.2) 2024-09-21T09:40:29.355 INFO:journalctl@ceph.mds.b.smithi064.stdout:Sep 21 09:40:29 smithi064 bash[109991]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mds-b 2024-09-21T09:40:29.557 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:29 smithi022 ceph-mon[98615]: osdmap e605: 8 total, 8 up, 8 in 2024-09-21T09:40:29.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:29 smithi022 ceph-mon[98615]: osdmap e606: 8 total, 8 up, 8 in 2024-09-21T09:40:29.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:29 smithi022 ceph-mon[96736]: osdmap e605: 8 total, 8 up, 8 in 2024-09-21T09:40:29.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:29 smithi022 ceph-mon[96736]: osdmap e606: 8 total, 8 up, 8 in 2024-09-21T09:40:29.691 INFO:journalctl@ceph.mds.b.smithi064.stdout:Sep 21 09:40:29 smithi064 podman[110010]: 2024-09-21 09:40:29.353727272 +0000 UTC m=+0.669152141 container remove 0f846b3ec6947704b96b602066283327e11f659fdf60a4ce5c5e0f40a4c41781 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mds-b, GIT_CLEAN=True, RELEASE=quincy-fbd1a6a, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=) 2024-09-21T09:40:29.691 INFO:journalctl@ceph.mds.b.smithi064.stdout:Sep 21 09:40:29 smithi064 systemd[1]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490@mds.b.service: Deactivated successfully. 2024-09-21T09:40:29.691 INFO:journalctl@ceph.mds.b.smithi064.stdout:Sep 21 09:40:29 smithi064 systemd[1]: Stopped Ceph mds.b for d65e801e-77fa-11ef-baf6-efdc52797490. 2024-09-21T09:40:29.691 INFO:journalctl@ceph.mds.b.smithi064.stdout:Sep 21 09:40:29 smithi064 systemd[1]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490@mds.b.service: Consumed 1.744s CPU time. 2024-09-21T09:40:30.075 INFO:journalctl@ceph.mds.b.smithi064.stdout:Sep 21 09:40:29 smithi064 systemd[1]: Starting Ceph mds.b for d65e801e-77fa-11ef-baf6-efdc52797490... 2024-09-21T09:40:30.075 INFO:journalctl@ceph.mds.b.smithi064.stdout:Sep 21 09:40:29 smithi064 podman[110145]: 2024-09-21 09:40:29.847415456 +0000 UTC m=+0.016297754 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:40:30.075 INFO:journalctl@ceph.mds.b.smithi064.stdout:Sep 21 09:40:29 smithi064 podman[110145]: 2024-09-21 09:40:29.954913079 +0000 UTC m=+0.123795359 container create e520bfb5f2948d1e31fc65348bbb963b39eb4ccf21df32b6be89c0ed902e3fd7 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mds-b, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, RELEASE=main-62da54b) 2024-09-21T09:40:30.089 INFO:tasks.rados.rados.0.smithi022.stdout:2660: done (3 left) 2024-09-21T09:40:30.089 INFO:tasks.rados.rados.0.smithi022.stdout:2661: done (2 left) 2024-09-21T09:40:30.089 INFO:tasks.rados.rados.0.smithi022.stdout:2662: done (1 left) 2024-09-21T09:40:30.089 INFO:tasks.rados.rados.0.smithi022.stdout:2663: done (0 left) 2024-09-21T09:40:30.089 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 10 current snap is 270 2024-09-21T09:40:30.089 INFO:tasks.rados.rados.0.smithi022.stdout:2664: seq_num 845 ranges {0=557056,557056=581632,1138688=688128,1826816=335872} 2024-09-21T09:40:30.092 INFO:tasks.rados.rados.0.smithi022.stdout:2664: writing smithi02284960-10 from 0 to 557056 tid 1 2024-09-21T09:40:30.095 INFO:tasks.rados.rados.0.smithi022.stdout:2664: writing smithi02284960-10 from 557056 to 1138688 tid 2 2024-09-21T09:40:30.099 INFO:tasks.rados.rados.0.smithi022.stdout:2664: writing smithi02284960-10 from 1138688 to 1826816 tid 3 2024-09-21T09:40:30.101 INFO:tasks.rados.rados.0.smithi022.stdout:2664: writing smithi02284960-10 from 1826816 to 2162688 tid 4 2024-09-21T09:40:30.101 INFO:tasks.rados.rados.0.smithi022.stdout:2665: read oid 21 snap -1 2024-09-21T09:40:30.102 INFO:tasks.rados.rados.0.smithi022.stdout:2665: expect (ObjNum 830 snap 266 seq_num 830) 2024-09-21T09:40:30.102 INFO:tasks.rados.rados.0.smithi022.stdout:2666: snap_remove snap 269 2024-09-21T09:40:30.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:30 smithi064 ceph-mon[86709]: pgmap v512: 121 pgs: 121 active+clean; 198 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 2.7 MiB/s rd, 1.4 MiB/s wr, 18 op/s 2024-09-21T09:40:30.441 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:40:30 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:40:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:40:30.442 INFO:journalctl@ceph.mds.b.smithi064.stdout:Sep 21 09:40:30 smithi064 podman[110145]: 2024-09-21 09:40:30.190995015 +0000 UTC m=+0.359877305 container init e520bfb5f2948d1e31fc65348bbb963b39eb4ccf21df32b6be89c0ed902e3fd7 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mds-b, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, RELEASE=main-62da54b, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, io.buildah.version=1.37.2) 2024-09-21T09:40:30.442 INFO:journalctl@ceph.mds.b.smithi064.stdout:Sep 21 09:40:30 smithi064 podman[110145]: 2024-09-21 09:40:30.196088938 +0000 UTC m=+0.364971219 container start e520bfb5f2948d1e31fc65348bbb963b39eb4ccf21df32b6be89c0ed902e3fd7 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mds-b, RELEASE=main-62da54b, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux ) 2024-09-21T09:40:30.442 INFO:journalctl@ceph.mds.b.smithi064.stdout:Sep 21 09:40:30 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mds-b[110160]: starting mds.b at 2024-09-21T09:40:30.442 INFO:journalctl@ceph.mds.b.smithi064.stdout:Sep 21 09:40:30 smithi064 bash[110145]: e520bfb5f2948d1e31fc65348bbb963b39eb4ccf21df32b6be89c0ed902e3fd7 2024-09-21T09:40:30.442 INFO:journalctl@ceph.mds.b.smithi064.stdout:Sep 21 09:40:30 smithi064 systemd[1]: Started Ceph mds.b for d65e801e-77fa-11ef-baf6-efdc52797490. 2024-09-21T09:40:30.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:30 smithi022 ceph-mon[98615]: pgmap v512: 121 pgs: 121 active+clean; 198 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 2.7 MiB/s rd, 1.4 MiB/s wr, 18 op/s 2024-09-21T09:40:30.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:30 smithi022 ceph-mon[96736]: pgmap v512: 121 pgs: 121 active+clean; 198 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 2.7 MiB/s rd, 1.4 MiB/s wr, 18 op/s 2024-09-21T09:40:31.096 INFO:tasks.rados.rados.0.smithi022.stdout:2666: done (2 left) 2024-09-21T09:40:31.096 INFO:tasks.rados.rados.0.smithi022.stdout:2667: read oid 23 snap -1 2024-09-21T09:40:31.096 INFO:tasks.rados.rados.0.smithi022.stdout:2667: expect deleted 2024-09-21T09:40:31.096 INFO:tasks.rados.rados.0.smithi022.stdout:2668: snap_create 2024-09-21T09:40:31.096 INFO:tasks.rados.rados.0.smithi022.stdout:2664: finishing write tid 1 to smithi02284960-10 2024-09-21T09:40:31.097 INFO:tasks.rados.rados.0.smithi022.stdout:2664: finishing write tid 2 to smithi02284960-10 2024-09-21T09:40:31.097 INFO:tasks.rados.rados.0.smithi022.stdout:2664: finishing write tid 3 to smithi02284960-10 2024-09-21T09:40:31.097 INFO:tasks.rados.rados.0.smithi022.stdout:2664: finishing write tid 4 to smithi02284960-10 2024-09-21T09:40:31.097 INFO:tasks.rados.rados.0.smithi022.stdout:2664: finishing write tid 5 to smithi02284960-10 2024-09-21T09:40:31.097 INFO:tasks.rados.rados.0.smithi022.stdout:2664: finishing write tid 6 to smithi02284960-10 2024-09-21T09:40:31.097 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 10 v 1419 (ObjNum 845 snap 270 seq_num 845) dirty exists 2024-09-21T09:40:31.097 INFO:tasks.rados.rados.0.smithi022.stdout:2664: left oid 10 (ObjNum 845 snap 270 seq_num 845) 2024-09-21T09:40:31.097 INFO:tasks.rados.rados.0.smithi022.stdout:2665: expect (ObjNum 830 snap 266 seq_num 830) 2024-09-21T09:40:31.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:31 smithi064 ceph-mon[86709]: osdmap e607: 8 total, 8 up, 8 in 2024-09-21T09:40:31.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:31 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:31.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:31 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:31.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:31 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:40:31.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:31 smithi022 ceph-mon[98615]: osdmap e607: 8 total, 8 up, 8 in 2024-09-21T09:40:31.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:31 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:31.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:31 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:31.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:31 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:40:31.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:31 smithi022 ceph-mon[96736]: osdmap e607: 8 total, 8 up, 8 in 2024-09-21T09:40:31.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:31 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:31.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:31 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:31.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:31 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:40:32.097 INFO:tasks.rados.rados.0.smithi022.stdout:2664: done (3 left) 2024-09-21T09:40:32.097 INFO:tasks.rados.rados.0.smithi022.stdout:2665: done (2 left) 2024-09-21T09:40:32.097 INFO:tasks.rados.rados.0.smithi022.stdout:2667: done (1 left) 2024-09-21T09:40:32.097 INFO:tasks.rados.rados.0.smithi022.stdout:2668: done (0 left) 2024-09-21T09:40:32.097 INFO:tasks.rados.rados.0.smithi022.stdout:2669: rollback oid 30 current snap is 271 2024-09-21T09:40:32.097 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 30 to 265 2024-09-21T09:40:32.097 INFO:tasks.rados.rados.0.smithi022.stdout:2670: delete oid 21 current snap is 271 2024-09-21T09:40:32.102 INFO:tasks.rados.rados.0.smithi022.stdout:2669: finishing rollback tid 0 to smithi02284960-30 2024-09-21T09:40:32.102 INFO:tasks.rados.rados.0.smithi022.stdout:2670: done (1 left) 2024-09-21T09:40:32.102 INFO:tasks.rados.rados.0.smithi022.stdout:2671: snap_create 2024-09-21T09:40:32.103 INFO:tasks.rados.rados.0.smithi022.stdout:2669: finishing rollback tid 1 to smithi02284960-30 2024-09-21T09:40:32.103 INFO:tasks.rados.rados.0.smithi022.stdout:2669: finishing rollback tid 2 to smithi02284960-30 2024-09-21T09:40:32.103 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 30 v 1295 (ObjNum 590 snap 176 seq_num 590) dirty exists 2024-09-21T09:40:32.266 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:32 smithi064 ceph-mon[86709]: pgmap v515: 121 pgs: 121 active+clean; 208 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 3.4 MiB/s wr, 23 op/s 2024-09-21T09:40:32.266 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:32 smithi064 ceph-mon[86709]: mds.? [v2:172.21.15.64:6832/1932960939,v1:172.21.15.64:6833/1932960939] up:boot 2024-09-21T09:40:32.266 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:32 smithi064 ceph-mon[86709]: fsmap cephfs:1 {0=a=up:active} 1 up:standby 2024-09-21T09:40:32.266 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:32 smithi064 ceph-mon[86709]: osdmap e608: 8 total, 8 up, 8 in 2024-09-21T09:40:32.266 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:32 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "b"} : dispatch 2024-09-21T09:40:32.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:32 smithi022 ceph-mon[98615]: pgmap v515: 121 pgs: 121 active+clean; 208 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 3.4 MiB/s wr, 23 op/s 2024-09-21T09:40:32.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:32 smithi022 ceph-mon[98615]: mds.? [v2:172.21.15.64:6832/1932960939,v1:172.21.15.64:6833/1932960939] up:boot 2024-09-21T09:40:32.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:32 smithi022 ceph-mon[98615]: fsmap cephfs:1 {0=a=up:active} 1 up:standby 2024-09-21T09:40:32.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:32 smithi022 ceph-mon[98615]: osdmap e608: 8 total, 8 up, 8 in 2024-09-21T09:40:32.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:32 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "b"} : dispatch 2024-09-21T09:40:32.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:32 smithi022 ceph-mon[96736]: pgmap v515: 121 pgs: 121 active+clean; 208 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 3.4 MiB/s wr, 23 op/s 2024-09-21T09:40:32.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:32 smithi022 ceph-mon[96736]: mds.? [v2:172.21.15.64:6832/1932960939,v1:172.21.15.64:6833/1932960939] up:boot 2024-09-21T09:40:32.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:32 smithi022 ceph-mon[96736]: fsmap cephfs:1 {0=a=up:active} 1 up:standby 2024-09-21T09:40:32.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:32 smithi022 ceph-mon[96736]: osdmap e608: 8 total, 8 up, 8 in 2024-09-21T09:40:32.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:32 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "b"} : dispatch 2024-09-21T09:40:33.102 INFO:tasks.rados.rados.0.smithi022.stdout:2669: done (1 left) 2024-09-21T09:40:33.102 INFO:tasks.rados.rados.0.smithi022.stdout:2671: done (0 left) 2024-09-21T09:40:33.102 INFO:tasks.rados.rados.0.smithi022.stdout:2672: snap_create 2024-09-21T09:40:33.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:33 smithi064 ceph-mon[86709]: osdmap e609: 8 total, 8 up, 8 in 2024-09-21T09:40:33.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:33 smithi064 ceph-mon[86709]: pgmap v518: 121 pgs: 121 active+clean; 212 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 24 KiB/s rd, 3.5 MiB/s wr, 22 op/s 2024-09-21T09:40:33.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:33 smithi022 ceph-mon[98615]: osdmap e609: 8 total, 8 up, 8 in 2024-09-21T09:40:33.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:33 smithi022 ceph-mon[98615]: pgmap v518: 121 pgs: 121 active+clean; 212 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 24 KiB/s rd, 3.5 MiB/s wr, 22 op/s 2024-09-21T09:40:33.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:33 smithi022 ceph-mon[96736]: osdmap e609: 8 total, 8 up, 8 in 2024-09-21T09:40:33.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:33 smithi022 ceph-mon[96736]: pgmap v518: 121 pgs: 121 active+clean; 212 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 24 KiB/s rd, 3.5 MiB/s wr, 22 op/s 2024-09-21T09:40:34.031 INFO:tasks.rados.rados.0.smithi022.stdout:2672: done (0 left) 2024-09-21T09:40:34.031 INFO:tasks.rados.rados.0.smithi022.stdout:2673: delete oid 36 current snap is 273 2024-09-21T09:40:34.035 INFO:tasks.rados.rados.0.smithi022.stdout:2673: done (0 left) 2024-09-21T09:40:34.035 INFO:tasks.rados.rados.0.smithi022.stdout:2674: delete oid 1 current snap is 273 2024-09-21T09:40:34.043 INFO:tasks.rados.rados.0.smithi022.stdout:2674: done (0 left) 2024-09-21T09:40:34.043 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 21 current snap is 273 2024-09-21T09:40:34.043 INFO:tasks.rados.rados.0.smithi022.stdout:2675: seq_num 849 ranges {0=622592,622592=516096,1138688=663552,1802240=425984} 2024-09-21T09:40:34.046 INFO:tasks.rados.rados.0.smithi022.stdout:2675: writing smithi02284960-21 from 0 to 622592 tid 1 2024-09-21T09:40:34.049 INFO:tasks.rados.rados.0.smithi022.stdout:2675: writing smithi02284960-21 from 622592 to 1138688 tid 2 2024-09-21T09:40:34.053 INFO:tasks.rados.rados.0.smithi022.stdout:2675: writing smithi02284960-21 from 1138688 to 1802240 tid 3 2024-09-21T09:40:34.056 INFO:tasks.rados.rados.0.smithi022.stdout:2675: writing smithi02284960-21 from 1802240 to 2228224 tid 4 2024-09-21T09:40:34.057 INFO:tasks.rados.rados.0.smithi022.stdout:2676: copy_from oid 19 from oid 49 current snap is 273 2024-09-21T09:40:34.057 INFO:tasks.rados.rados.0.smithi022.stdout:2677: read oid 9 snap -1 2024-09-21T09:40:34.057 INFO:tasks.rados.rados.0.smithi022.stdout:2677: expect (ObjNum 837 snap 269 seq_num 837) 2024-09-21T09:40:34.057 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 5 current snap is 273 2024-09-21T09:40:34.057 INFO:tasks.rados.rados.0.smithi022.stdout:2678: seq_num 850 ranges {3645440=638976,4284416=802816,5087232=475136,5562368=327680} 2024-09-21T09:40:34.080 INFO:tasks.rados.rados.0.smithi022.stdout:2678: writing smithi02284960-5 from 3645440 to 4284416 tid 1 2024-09-21T09:40:34.085 INFO:tasks.rados.rados.0.smithi022.stdout:2678: writing smithi02284960-5 from 4284416 to 5087232 tid 2 2024-09-21T09:40:34.088 INFO:tasks.rados.rados.0.smithi022.stdout:2678: writing smithi02284960-5 from 5087232 to 5562368 tid 3 2024-09-21T09:40:34.091 INFO:tasks.rados.rados.0.smithi022.stdout:2678: writing smithi02284960-5 from 5562368 to 5890048 tid 4 2024-09-21T09:40:34.091 INFO:tasks.rados.rados.0.smithi022.stdout:2679: read oid 46 snap -1 2024-09-21T09:40:34.091 INFO:tasks.rados.rados.0.smithi022.stdout:2679: expect deleted 2024-09-21T09:40:34.091 INFO:tasks.rados.rados.0.smithi022.stdout:2680: delete oid 16 current snap is 273 2024-09-21T09:40:34.091 INFO:tasks.rados.rados.0.smithi022.stdout:2675: finishing write tid 1 to smithi02284960-21 2024-09-21T09:40:34.092 INFO:tasks.rados.rados.0.smithi022.stdout:2675: finishing write tid 2 to smithi02284960-21 2024-09-21T09:40:34.092 INFO:tasks.rados.rados.0.smithi022.stdout:2676: finishing copy_from to smithi02284960-19 2024-09-21T09:40:34.092 INFO:tasks.rados.rados.0.smithi022.stdout:2676: got expected ENOENT (src dne) 2024-09-21T09:40:34.092 INFO:tasks.rados.rados.0.smithi022.stdout:2676: finishing copy_from racing read to smithi02284960-19 2024-09-21T09:40:34.092 INFO:tasks.rados.rados.0.smithi022.stdout:2675: finishing write tid 3 to smithi02284960-21 2024-09-21T09:40:34.092 INFO:tasks.rados.rados.0.smithi022.stdout:2675: finishing write tid 4 to smithi02284960-21 2024-09-21T09:40:34.092 INFO:tasks.rados.rados.0.smithi022.stdout:2675: finishing write tid 5 to smithi02284960-21 2024-09-21T09:40:34.092 INFO:tasks.rados.rados.0.smithi022.stdout:2675: finishing write tid 6 to smithi02284960-21 2024-09-21T09:40:34.092 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 21 v 1422 (ObjNum 849 snap 273 seq_num 849) dirty exists 2024-09-21T09:40:34.092 INFO:tasks.rados.rados.0.smithi022.stdout:2675: left oid 21 (ObjNum 849 snap 273 seq_num 849) 2024-09-21T09:40:34.092 INFO:tasks.rados.rados.0.smithi022.stdout:2677: expect (ObjNum 837 snap 269 seq_num 837) 2024-09-21T09:40:34.147 INFO:tasks.rados.rados.0.smithi022.stdout:2678: finishing write tid 1 to smithi02284960-5 2024-09-21T09:40:34.147 INFO:tasks.rados.rados.0.smithi022.stdout:2678: finishing write tid 2 to smithi02284960-5 2024-09-21T09:40:34.147 INFO:tasks.rados.rados.0.smithi022.stdout:2678: finishing write tid 3 to smithi02284960-5 2024-09-21T09:40:34.147 INFO:tasks.rados.rados.0.smithi022.stdout:2675: done (5 left) 2024-09-21T09:40:34.147 INFO:tasks.rados.rados.0.smithi022.stdout:2676: done (4 left) 2024-09-21T09:40:34.148 INFO:tasks.rados.rados.0.smithi022.stdout:2677: done (3 left) 2024-09-21T09:40:34.148 INFO:tasks.rados.rados.0.smithi022.stdout:2680: done (2 left) 2024-09-21T09:40:34.148 INFO:tasks.rados.rados.0.smithi022.stdout:2681: setattr oid 27 current snap is 273 2024-09-21T09:40:34.148 INFO:tasks.rados.rados.0.smithi022.stdout:2678: finishing write tid 4 to smithi02284960-5 2024-09-21T09:40:34.148 INFO:tasks.rados.rados.0.smithi022.stdout:2678: finishing write tid 5 to smithi02284960-5 2024-09-21T09:40:34.148 INFO:tasks.rados.rados.0.smithi022.stdout:2678: finishing write tid 6 to smithi02284960-5 2024-09-21T09:40:34.148 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 5 v 1424 (ObjNum 850 snap 273 seq_num 850) dirty exists 2024-09-21T09:40:34.148 INFO:tasks.rados.rados.0.smithi022.stdout:2678: left oid 5 (ObjNum 850 snap 273 seq_num 850) 2024-09-21T09:40:34.149 INFO:tasks.rados.rados.0.smithi022.stdout:2678: done (2 left) 2024-09-21T09:40:34.149 INFO:tasks.rados.rados.0.smithi022.stdout:2679: done (1 left) 2024-09-21T09:40:34.149 INFO:tasks.rados.rados.0.smithi022.stdout:2682: read oid 14 snap -1 2024-09-21T09:40:34.149 INFO:tasks.rados.rados.0.smithi022.stdout:2682: expect deleted 2024-09-21T09:40:34.149 INFO:tasks.rados.rados.0.smithi022.stdout:2683: snap_create 2024-09-21T09:40:34.153 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 27 v 1350 (ObjNum 773 snap 242 seq_num 773) dirty exists 2024-09-21T09:40:34.381 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:34 smithi064 ceph-mon[86709]: osdmap e610: 8 total, 8 up, 8 in 2024-09-21T09:40:34.381 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:34 smithi064 ceph-mon[86709]: osdmap e611: 8 total, 8 up, 8 in 2024-09-21T09:40:34.557 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:34 smithi022 ceph-mon[96736]: osdmap e610: 8 total, 8 up, 8 in 2024-09-21T09:40:34.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:34 smithi022 ceph-mon[96736]: osdmap e611: 8 total, 8 up, 8 in 2024-09-21T09:40:34.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:34 smithi022 ceph-mon[98615]: osdmap e610: 8 total, 8 up, 8 in 2024-09-21T09:40:34.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:34 smithi022 ceph-mon[98615]: osdmap e611: 8 total, 8 up, 8 in 2024-09-21T09:40:35.103 INFO:tasks.rados.rados.0.smithi022.stdout:2681: done (2 left) 2024-09-21T09:40:35.103 INFO:tasks.rados.rados.0.smithi022.stdout:2682: done (1 left) 2024-09-21T09:40:35.103 INFO:tasks.rados.rados.0.smithi022.stdout:2683: done (0 left) 2024-09-21T09:40:35.104 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 23 current snap is 274 2024-09-21T09:40:35.104 INFO:tasks.rados.rados.0.smithi022.stdout:2684: seq_num 852 ranges {0=671744,671744=622592,1294336=442368,1736704=540672} 2024-09-21T09:40:35.106 INFO:tasks.rados.rados.0.smithi022.stdout:2684: writing smithi02284960-23 from 0 to 671744 tid 1 2024-09-21T09:40:35.110 INFO:tasks.rados.rados.0.smithi022.stdout:2684: writing smithi02284960-23 from 671744 to 1294336 tid 2 2024-09-21T09:40:35.113 INFO:tasks.rados.rados.0.smithi022.stdout:2684: writing smithi02284960-23 from 1294336 to 1736704 tid 3 2024-09-21T09:40:35.116 INFO:tasks.rados.rados.0.smithi022.stdout:2684: writing smithi02284960-23 from 1736704 to 2277376 tid 4 2024-09-21T09:40:35.117 INFO:tasks.rados.rados.0.smithi022.stdout:2685: snap_create 2024-09-21T09:40:35.117 INFO:tasks.rados.rados.0.smithi022.stdout:2684: finishing write tid 1 to smithi02284960-23 2024-09-21T09:40:35.117 INFO:tasks.rados.rados.0.smithi022.stdout:2684: finishing write tid 2 to smithi02284960-23 2024-09-21T09:40:35.117 INFO:tasks.rados.rados.0.smithi022.stdout:2684: finishing write tid 3 to smithi02284960-23 2024-09-21T09:40:35.119 INFO:tasks.rados.rados.0.smithi022.stdout:2684: finishing write tid 4 to smithi02284960-23 2024-09-21T09:40:35.119 INFO:tasks.rados.rados.0.smithi022.stdout:2684: finishing write tid 5 to smithi02284960-23 2024-09-21T09:40:35.120 INFO:tasks.rados.rados.0.smithi022.stdout:2684: finishing write tid 6 to smithi02284960-23 2024-09-21T09:40:35.120 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 23 v 1419 (ObjNum 852 snap 274 seq_num 852) dirty exists 2024-09-21T09:40:35.120 INFO:tasks.rados.rados.0.smithi022.stdout:2684: left oid 23 (ObjNum 852 snap 274 seq_num 852) 2024-09-21T09:40:35.519 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:35 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:35.519 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:35 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:35.519 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:35 smithi064 ceph-mon[86709]: pgmap v521: 121 pgs: 121 active+clean; 214 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 2.6 MiB/s rd, 1.5 MiB/s wr, 7 op/s 2024-09-21T09:40:35.519 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:35 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:35.519 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:35 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:35.519 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:35 smithi064 ceph-mon[86709]: osdmap e612: 8 total, 8 up, 8 in 2024-09-21T09:40:35.557 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:35 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:35.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:35 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:35.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:35 smithi022 ceph-mon[98615]: pgmap v521: 121 pgs: 121 active+clean; 214 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 2.6 MiB/s rd, 1.5 MiB/s wr, 7 op/s 2024-09-21T09:40:35.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:35 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:35.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:35 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:35.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:35 smithi022 ceph-mon[98615]: osdmap e612: 8 total, 8 up, 8 in 2024-09-21T09:40:35.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:35 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:35.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:35 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:35.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:35 smithi022 ceph-mon[96736]: pgmap v521: 121 pgs: 121 active+clean; 214 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 2.6 MiB/s rd, 1.5 MiB/s wr, 7 op/s 2024-09-21T09:40:35.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:35 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:35.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:35 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:35.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:35 smithi022 ceph-mon[96736]: osdmap e612: 8 total, 8 up, 8 in 2024-09-21T09:40:36.058 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:40:35 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:40:35] "GET /metrics HTTP/1.1" 200 36124 "" "Prometheus/2.51.0" 2024-09-21T09:40:36.105 INFO:tasks.rados.rados.0.smithi022.stdout:2684: done (1 left) 2024-09-21T09:40:36.105 INFO:tasks.rados.rados.0.smithi022.stdout:2685: done (0 left) 2024-09-21T09:40:36.106 INFO:tasks.rados.rados.0.smithi022.stdout:2686: delete oid 5 current snap is 275 2024-09-21T09:40:36.109 INFO:tasks.rados.rados.0.smithi022.stdout:2686: done (0 left) 2024-09-21T09:40:36.109 INFO:tasks.rados.rados.0.smithi022.stdout:2687: delete oid 6 current snap is 275 2024-09-21T09:40:36.112 INFO:tasks.rados.rados.0.smithi022.stdout:2687: done (0 left) 2024-09-21T09:40:36.112 INFO:tasks.rados.rados.0.smithi022.stdout:2688: read oid 20 snap 254 2024-09-21T09:40:36.112 INFO:tasks.rados.rados.0.smithi022.stdout:2688: expect (ObjNum 796 snap 249 seq_num 796) 2024-09-21T09:40:36.112 INFO:tasks.rados.rados.0.smithi022.stdout:2689: read oid 16 snap -1 2024-09-21T09:40:36.112 INFO:tasks.rados.rados.0.smithi022.stdout:2689: expect deleted 2024-09-21T09:40:36.112 INFO:tasks.rados.rados.0.smithi022.stdout:2690: snap_remove snap 266 2024-09-21T09:40:37.112 INFO:tasks.rados.rados.0.smithi022.stdout:2690: done (2 left) 2024-09-21T09:40:37.112 INFO:tasks.rados.rados.0.smithi022.stdout:2691: copy_from oid 34 from oid 21 current snap is 275 2024-09-21T09:40:37.112 INFO:tasks.rados.rados.0.smithi022.stdout:2688: expect (ObjNum 796 snap 249 seq_num 796) 2024-09-21T09:40:37.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:37 smithi064 ceph-mon[86709]: osdmap e613: 8 total, 8 up, 8 in 2024-09-21T09:40:37.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:37 smithi064 ceph-mon[86709]: pgmap v524: 121 pgs: 121 active+clean; 216 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 4.0 MiB/s rd, 1.0 MiB/s wr, 11 op/s 2024-09-21T09:40:37.546 INFO:tasks.rados.rados.0.smithi022.stdout:2691: finishing copy_from racing read to smithi02284960-34 2024-09-21T09:40:37.546 INFO:tasks.rados.rados.0.smithi022.stdout:2691: finishing copy_from to smithi02284960-34 2024-09-21T09:40:37.546 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 34 v 1422 (ObjNum 849 snap 273 seq_num 849) dirty exists 2024-09-21T09:40:37.546 INFO:tasks.rados.rados.0.smithi022.stdout:2688: done (2 left) 2024-09-21T09:40:37.546 INFO:tasks.rados.rados.0.smithi022.stdout:2689: done (1 left) 2024-09-21T09:40:37.546 INFO:tasks.rados.rados.0.smithi022.stdout:2691: done (0 left) 2024-09-21T09:40:37.546 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 8 current snap is 275 2024-09-21T09:40:37.546 INFO:tasks.rados.rados.0.smithi022.stdout:2692: seq_num 855 ranges {0=729088,729088=671744,1400832=663552,2064384=270336} 2024-09-21T09:40:37.549 INFO:tasks.rados.rados.0.smithi022.stdout:2692: writing smithi02284960-8 from 0 to 729088 tid 1 2024-09-21T09:40:37.553 INFO:tasks.rados.rados.0.smithi022.stdout:2692: writing smithi02284960-8 from 729088 to 1400832 tid 2 2024-09-21T09:40:37.557 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:37 smithi022 ceph-mon[96736]: osdmap e613: 8 total, 8 up, 8 in 2024-09-21T09:40:37.557 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:37 smithi022 ceph-mon[96736]: pgmap v524: 121 pgs: 121 active+clean; 216 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 4.0 MiB/s rd, 1.0 MiB/s wr, 11 op/s 2024-09-21T09:40:37.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:37 smithi022 ceph-mon[98615]: osdmap e613: 8 total, 8 up, 8 in 2024-09-21T09:40:37.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:37 smithi022 ceph-mon[98615]: pgmap v524: 121 pgs: 121 active+clean; 216 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 4.0 MiB/s rd, 1.0 MiB/s wr, 11 op/s 2024-09-21T09:40:37.558 INFO:tasks.rados.rados.0.smithi022.stdout:2692: writing smithi02284960-8 from 1400832 to 2064384 tid 3 2024-09-21T09:40:37.559 INFO:tasks.rados.rados.0.smithi022.stdout:2692: writing smithi02284960-8 from 2064384 to 2334720 tid 4 2024-09-21T09:40:37.559 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 30 current snap is 275 2024-09-21T09:40:37.559 INFO:tasks.rados.rados.0.smithi022.stdout:2692: finishing write tid 1 to smithi02284960-8 2024-09-21T09:40:37.559 INFO:tasks.rados.rados.0.smithi022.stdout:2693: seq_num 856 ranges {286720=745472,1032192=557056,1589248=417792,2007040=434176,2441216=196608} 2024-09-21T09:40:37.565 INFO:tasks.rados.rados.0.smithi022.stdout:2693: writing smithi02284960-30 from 286720 to 1032192 tid 1 2024-09-21T09:40:37.568 INFO:tasks.rados.rados.0.smithi022.stdout:2693: writing smithi02284960-30 from 1032192 to 1589248 tid 2 2024-09-21T09:40:37.571 INFO:tasks.rados.rados.0.smithi022.stdout:2693: writing smithi02284960-30 from 1589248 to 2007040 tid 3 2024-09-21T09:40:37.574 INFO:tasks.rados.rados.0.smithi022.stdout:2693: writing smithi02284960-30 from 2007040 to 2441216 tid 4 2024-09-21T09:40:37.575 INFO:tasks.rados.rados.0.smithi022.stdout:2693: writing smithi02284960-30 from 2441216 to 2637824 tid 5 2024-09-21T09:40:37.576 INFO:tasks.rados.rados.0.smithi022.stdout:2692: finishing write tid 2 to smithi02284960-8 2024-09-21T09:40:37.576 INFO:tasks.rados.rados.0.smithi022.stdout:2694: read oid 44 snap -1 2024-09-21T09:40:37.576 INFO:tasks.rados.rados.0.smithi022.stdout:2694: expect deleted 2024-09-21T09:40:37.576 INFO:tasks.rados.rados.0.smithi022.stdout:2695: snap_remove snap 262 2024-09-21T09:40:38.117 INFO:tasks.rados.rados.0.smithi022.stdout:2695: done (3 left) 2024-09-21T09:40:38.117 INFO:tasks.rados.rados.0.smithi022.stdout:2696: read oid 39 snap 273 2024-09-21T09:40:38.117 INFO:tasks.rados.rados.0.smithi022.stdout:2696: expect deleted 2024-09-21T09:40:38.117 INFO:tasks.rados.rados.0.smithi022.stdout:2697: snap_create 2024-09-21T09:40:38.117 INFO:tasks.rados.rados.0.smithi022.stdout:2692: finishing write tid 3 to smithi02284960-8 2024-09-21T09:40:38.117 INFO:tasks.rados.rados.0.smithi022.stdout:2692: finishing write tid 4 to smithi02284960-8 2024-09-21T09:40:38.117 INFO:tasks.rados.rados.0.smithi022.stdout:2692: finishing write tid 5 to smithi02284960-8 2024-09-21T09:40:38.117 INFO:tasks.rados.rados.0.smithi022.stdout:2692: finishing write tid 6 to smithi02284960-8 2024-09-21T09:40:38.117 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 8 v 1430 (ObjNum 855 snap 275 seq_num 855) dirty exists 2024-09-21T09:40:38.118 INFO:tasks.rados.rados.0.smithi022.stdout:2692: left oid 8 (ObjNum 855 snap 275 seq_num 855) 2024-09-21T09:40:38.118 INFO:tasks.rados.rados.0.smithi022.stdout:2693: finishing write tid 1 to smithi02284960-30 2024-09-21T09:40:38.118 INFO:tasks.rados.rados.0.smithi022.stdout:2693: finishing write tid 2 to smithi02284960-30 2024-09-21T09:40:38.118 INFO:tasks.rados.rados.0.smithi022.stdout:2693: finishing write tid 3 to smithi02284960-30 2024-09-21T09:40:38.118 INFO:tasks.rados.rados.0.smithi022.stdout:2693: finishing write tid 4 to smithi02284960-30 2024-09-21T09:40:38.118 INFO:tasks.rados.rados.0.smithi022.stdout:2693: finishing write tid 5 to smithi02284960-30 2024-09-21T09:40:38.118 INFO:tasks.rados.rados.0.smithi022.stdout:2693: finishing write tid 6 to smithi02284960-30 2024-09-21T09:40:38.118 INFO:tasks.rados.rados.0.smithi022.stdout:2693: finishing write tid 7 to smithi02284960-30 2024-09-21T09:40:38.118 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 30 v 1301 (ObjNum 856 snap 275 seq_num 856) dirty exists 2024-09-21T09:40:38.118 INFO:tasks.rados.rados.0.smithi022.stdout:2693: left oid 30 (ObjNum 856 snap 275 seq_num 856) 2024-09-21T09:40:38.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:38 smithi064 ceph-mon[86709]: osdmap e614: 8 total, 8 up, 8 in 2024-09-21T09:40:38.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:38 smithi022 ceph-mon[98615]: osdmap e614: 8 total, 8 up, 8 in 2024-09-21T09:40:38.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:38 smithi022 ceph-mon[96736]: osdmap e614: 8 total, 8 up, 8 in 2024-09-21T09:40:39.034 INFO:tasks.rados.rados.0.smithi022.stdout:2692: done (4 left) 2024-09-21T09:40:39.034 INFO:tasks.rados.rados.0.smithi022.stdout:2693: done (3 left) 2024-09-21T09:40:39.034 INFO:tasks.rados.rados.0.smithi022.stdout:2694: done (2 left) 2024-09-21T09:40:39.034 INFO:tasks.rados.rados.0.smithi022.stdout:2696: done (1 left) 2024-09-21T09:40:39.034 INFO:tasks.rados.rados.0.smithi022.stdout:2697: done (0 left) 2024-09-21T09:40:39.034 INFO:tasks.rados.rados.0.smithi022.stdout:2698: setattr oid 44 current snap is 276 2024-09-21T09:40:39.035 INFO:tasks.rados.rados.0.smithi022.stdout:2699: rollback oid 17 current snap is 276 2024-09-21T09:40:39.036 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 17 to 275 2024-09-21T09:40:39.036 INFO:tasks.rados.rados.0.smithi022.stdout:2700: snap_remove snap 272 2024-09-21T09:40:39.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:39 smithi064 ceph-mon[86709]: osdmap e615: 8 total, 8 up, 8 in 2024-09-21T09:40:39.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:39 smithi064 ceph-mon[86709]: Detected new or changed devices on smithi064 2024-09-21T09:40:39.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:39 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:39.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:39 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:39.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:39 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:40:39.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:39 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:40:39.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:39 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:39.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:39 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:40:39.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:39 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:40:39.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:39 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:40:39.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:39 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:40:39.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:39 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:40:39.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:39 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:40:39.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:39 smithi064 ceph-mon[86709]: Upgrade: Setting container_image for all mds 2024-09-21T09:40:39.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:39 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:39.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:39 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds.a"} : dispatch 2024-09-21T09:40:39.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:39 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.a"}]': finished 2024-09-21T09:40:39.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:39 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds.b"} : dispatch 2024-09-21T09:40:39.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:39 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.b"}]': finished 2024-09-21T09:40:39.444 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:39 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:40:39.444 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:39 smithi064 ceph-mon[86709]: Upgrade: Setting container_image for all rgw 2024-09-21T09:40:39.444 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:39 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:39.444 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:39 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:40:39.444 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:39 smithi064 ceph-mon[86709]: Upgrade: Setting container_image for all rbd-mirror 2024-09-21T09:40:39.444 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:39 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:39.444 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:39 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:40:39.444 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:39 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:40:39.444 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:39 smithi064 ceph-mon[86709]: Upgrade: Setting container_image for all ceph-exporter 2024-09-21T09:40:39.444 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:39 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:39.444 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:39 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:40:39.444 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:39 smithi064 ceph-mon[86709]: Upgrade: Setting container_image for all iscsi 2024-09-21T09:40:39.444 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:39 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:39.444 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:39 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:40:39.445 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:39 smithi064 ceph-mon[86709]: pgmap v527: 121 pgs: 121 active+clean; 222 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 1.5 MiB/s rd, 2.1 MiB/s wr, 12 op/s 2024-09-21T09:40:39.445 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:39 smithi064 ceph-mon[86709]: Upgrade: Setting container_image for all nfs 2024-09-21T09:40:39.445 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:39 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:39.445 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:39 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:40:39.445 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:39 smithi064 ceph-mon[86709]: Upgrade: Setting container_image for all nvmeof 2024-09-21T09:40:39.445 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:39 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:39.445 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:39 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:40:39.445 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:39 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:40:39.445 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:39 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:40:39.445 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:39 smithi064 ceph-mon[86709]: osdmap e616: 8 total, 8 up, 8 in 2024-09-21T09:40:39.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:39 smithi022 ceph-mon[96736]: osdmap e615: 8 total, 8 up, 8 in 2024-09-21T09:40:39.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:39 smithi022 ceph-mon[96736]: Detected new or changed devices on smithi064 2024-09-21T09:40:39.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:39 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:39.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:39 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:39.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:39 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:40:39.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:39 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:40:39.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:39 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:39.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:39 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:40:39.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:39 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:40:39.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:39 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:40:39.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:39 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:40:39.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:39 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:40:39.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:39 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:40:39.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:39 smithi022 ceph-mon[96736]: Upgrade: Setting container_image for all mds 2024-09-21T09:40:39.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:39 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:39.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:39 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds.a"} : dispatch 2024-09-21T09:40:39.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:39 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.a"}]': finished 2024-09-21T09:40:39.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:39 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds.b"} : dispatch 2024-09-21T09:40:39.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:39 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.b"}]': finished 2024-09-21T09:40:39.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:39 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:40:39.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:39 smithi022 ceph-mon[96736]: Upgrade: Setting container_image for all rgw 2024-09-21T09:40:39.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:39 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:39.560 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:39 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:40:39.560 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:39 smithi022 ceph-mon[96736]: Upgrade: Setting container_image for all rbd-mirror 2024-09-21T09:40:39.560 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:39 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:39.560 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:39 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:40:39.560 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:39 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:40:39.560 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:39 smithi022 ceph-mon[96736]: Upgrade: Setting container_image for all ceph-exporter 2024-09-21T09:40:39.560 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:39 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:39.560 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:39 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:40:39.560 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:39 smithi022 ceph-mon[96736]: Upgrade: Setting container_image for all iscsi 2024-09-21T09:40:39.560 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:39 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:39.560 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:39 smithi022 ceph-mon[98615]: osdmap e615: 8 total, 8 up, 8 in 2024-09-21T09:40:39.561 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:39 smithi022 ceph-mon[98615]: Detected new or changed devices on smithi064 2024-09-21T09:40:39.561 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:39 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:39.561 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:39 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:39.561 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:39 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:40:39.561 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:39 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:40:39.561 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:39 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:39.561 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:39 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:40:39.561 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:39 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:40:39.561 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:39 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:40:39.561 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:39 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:40:39.561 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:39 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:40:39.561 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:39 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:40:39.561 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:39 smithi022 ceph-mon[98615]: Upgrade: Setting container_image for all mds 2024-09-21T09:40:39.561 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:39 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:39.562 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:39 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds.a"} : dispatch 2024-09-21T09:40:39.562 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:39 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.a"}]': finished 2024-09-21T09:40:39.562 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:39 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds.b"} : dispatch 2024-09-21T09:40:39.562 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:39 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.b"}]': finished 2024-09-21T09:40:39.562 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:39 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:40:39.562 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:39 smithi022 ceph-mon[98615]: Upgrade: Setting container_image for all rgw 2024-09-21T09:40:39.562 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:39 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:39.562 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:39 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:40:39.562 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:39 smithi022 ceph-mon[98615]: Upgrade: Setting container_image for all rbd-mirror 2024-09-21T09:40:39.562 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:39 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:39.562 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:39 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:40:39.562 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:39 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:40:39.562 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:39 smithi022 ceph-mon[98615]: Upgrade: Setting container_image for all ceph-exporter 2024-09-21T09:40:39.562 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:39 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:39.563 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:39 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:40:39.563 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:39 smithi022 ceph-mon[98615]: Upgrade: Setting container_image for all iscsi 2024-09-21T09:40:39.563 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:39 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:39.563 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:39 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:40:39.563 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:39 smithi022 ceph-mon[96736]: pgmap v527: 121 pgs: 121 active+clean; 222 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 1.5 MiB/s rd, 2.1 MiB/s wr, 12 op/s 2024-09-21T09:40:39.563 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:39 smithi022 ceph-mon[96736]: Upgrade: Setting container_image for all nfs 2024-09-21T09:40:39.563 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:39 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:39.563 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:39 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:40:39.563 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:39 smithi022 ceph-mon[96736]: Upgrade: Setting container_image for all nvmeof 2024-09-21T09:40:39.563 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:39 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:39.563 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:39 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:40:39.564 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:39 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:40:39.564 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:39 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:40:39.564 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:39 smithi022 ceph-mon[96736]: osdmap e616: 8 total, 8 up, 8 in 2024-09-21T09:40:39.564 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:39 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:40:39.564 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:39 smithi022 ceph-mon[98615]: pgmap v527: 121 pgs: 121 active+clean; 222 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 1.5 MiB/s rd, 2.1 MiB/s wr, 12 op/s 2024-09-21T09:40:39.564 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:39 smithi022 ceph-mon[98615]: Upgrade: Setting container_image for all nfs 2024-09-21T09:40:39.564 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:39 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:39.564 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:39 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:40:39.564 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:39 smithi022 ceph-mon[98615]: Upgrade: Setting container_image for all nvmeof 2024-09-21T09:40:39.564 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:39 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:39.564 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:39 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:40:39.564 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:39 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:40:39.564 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:39 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:40:39.565 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:39 smithi022 ceph-mon[98615]: osdmap e616: 8 total, 8 up, 8 in 2024-09-21T09:40:40.036 INFO:tasks.rados.rados.0.smithi022.stdout:2700: done (2 left) 2024-09-21T09:40:40.036 INFO:tasks.rados.rados.0.smithi022.stdout:2701: delete oid 48 current snap is 276 2024-09-21T09:40:40.036 INFO:tasks.rados.rados.0.smithi022.stdout:2699: finishing rollback tid 0 to smithi02284960-17 2024-09-21T09:40:40.036 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 44 v 1513 (ObjNum 289 snap 0 seq_num 2954912973) dirty exists 2024-09-21T09:40:40.036 INFO:tasks.rados.rados.0.smithi022.stdout:2699: finishing rollback tid 1 to smithi02284960-17 2024-09-21T09:40:40.036 INFO:tasks.rados.rados.0.smithi022.stdout:2699: finishing rollback tid 2 to smithi02284960-17 2024-09-21T09:40:40.036 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 17 v 1304 (ObjNum 842 snap 269 seq_num 842) dirty exists 2024-09-21T09:40:40.045 INFO:tasks.rados.rados.0.smithi022.stdout:2698: done (2 left) 2024-09-21T09:40:40.045 INFO:tasks.rados.rados.0.smithi022.stdout:2699: done (1 left) 2024-09-21T09:40:40.045 INFO:tasks.rados.rados.0.smithi022.stdout:2701: done (0 left) 2024-09-21T09:40:40.045 INFO:tasks.rados.rados.0.smithi022.stdout:2702: setattr oid 14 current snap is 276 2024-09-21T09:40:40.046 INFO:tasks.rados.rados.0.smithi022.stdout:2703: snap_remove snap 254 2024-09-21T09:40:40.691 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:40:40 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:40:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:40:41.044 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 14 v 1118 (ObjNum 177 snap 0 seq_num 1) dirty exists 2024-09-21T09:40:41.045 INFO:tasks.rados.rados.0.smithi022.stdout:2702: done (1 left) 2024-09-21T09:40:41.045 INFO:tasks.rados.rados.0.smithi022.stdout:2703: done (0 left) 2024-09-21T09:40:41.045 INFO:tasks.rados.rados.0.smithi022.stdout:2704: rollback oid 38 current snap is 276 2024-09-21T09:40:41.045 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 38 to 273 2024-09-21T09:40:41.045 INFO:tasks.rados.rados.0.smithi022.stdout:2705: snap_remove snap 249 2024-09-21T09:40:41.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:41 smithi022 ceph-mon[98615]: Upgrade: Updating grafana.a 2024-09-21T09:40:41.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:41 smithi022 ceph-mon[98615]: Deploying daemon grafana.a on smithi064 2024-09-21T09:40:41.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:41 smithi022 ceph-mon[98615]: osdmap e617: 8 total, 8 up, 8 in 2024-09-21T09:40:41.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:41 smithi022 ceph-mon[96736]: Upgrade: Updating grafana.a 2024-09-21T09:40:41.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:41 smithi022 ceph-mon[96736]: Deploying daemon grafana.a on smithi064 2024-09-21T09:40:41.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:41 smithi022 ceph-mon[96736]: osdmap e617: 8 total, 8 up, 8 in 2024-09-21T09:40:41.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:41 smithi064 ceph-mon[86709]: Upgrade: Updating grafana.a 2024-09-21T09:40:41.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:41 smithi064 ceph-mon[86709]: Deploying daemon grafana.a on smithi064 2024-09-21T09:40:41.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:41 smithi064 ceph-mon[86709]: osdmap e617: 8 total, 8 up, 8 in 2024-09-21T09:40:42.043 INFO:tasks.rados.rados.0.smithi022.stdout:2704: finishing rollback tid 0 to smithi02284960-38 2024-09-21T09:40:42.043 INFO:tasks.rados.rados.0.smithi022.stdout:2704: finishing rollback tid 1 to smithi02284960-38 2024-09-21T09:40:42.043 INFO:tasks.rados.rados.0.smithi022.stdout:2704: finishing rollback tid 2 to smithi02284960-38 2024-09-21T09:40:42.043 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 38 v 1422 (ObjNum 839 snap 269 seq_num 839) dirty exists 2024-09-21T09:40:42.043 INFO:tasks.rados.rados.0.smithi022.stdout:2704: done (1 left) 2024-09-21T09:40:42.043 INFO:tasks.rados.rados.0.smithi022.stdout:2705: done (0 left) 2024-09-21T09:40:42.043 INFO:tasks.rados.rados.0.smithi022.stdout:2706: copy_from oid 42 from oid 7 current snap is 276 2024-09-21T09:40:42.043 INFO:tasks.rados.rados.0.smithi022.stdout:2707: read oid 14 snap -1 2024-09-21T09:40:42.043 INFO:tasks.rados.rados.0.smithi022.stdout:2707: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-21T09:40:42.044 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 41 current snap is 276 2024-09-21T09:40:42.044 INFO:tasks.rados.rados.0.smithi022.stdout:2708: seq_num 858 ranges {9969664=778240,10747904=729088,11476992=434176,11911168=442368} 2024-09-21T09:40:42.104 INFO:tasks.rados.rados.0.smithi022.stdout:2708: writing smithi02284960-41 from 9969664 to 10747904 tid 1 2024-09-21T09:40:42.108 INFO:tasks.rados.rados.0.smithi022.stdout:2708: writing smithi02284960-41 from 10747904 to 11476992 tid 2 2024-09-21T09:40:42.110 INFO:tasks.rados.rados.0.smithi022.stdout:2708: writing smithi02284960-41 from 11476992 to 11911168 tid 3 2024-09-21T09:40:42.113 INFO:tasks.rados.rados.0.smithi022.stdout:2708: writing smithi02284960-41 from 11911168 to 12353536 tid 4 2024-09-21T09:40:42.113 INFO:tasks.rados.rados.0.smithi022.stdout:2709: snap_create 2024-09-21T09:40:42.113 INFO:tasks.rados.rados.0.smithi022.stdout:2707: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-21T09:40:42.115 INFO:tasks.rados.rados.0.smithi022.stdout:2706: finishing copy_from racing read to smithi02284960-42 2024-09-21T09:40:42.115 INFO:tasks.rados.rados.0.smithi022.stdout:2706: finishing copy_from to smithi02284960-42 2024-09-21T09:40:42.115 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 42 v 1505 (ObjNum 785 snap 246 seq_num 785) dirty exists 2024-09-21T09:40:42.115 INFO:tasks.rados.rados.0.smithi022.stdout:2708: finishing write tid 1 to smithi02284960-41 2024-09-21T09:40:42.115 INFO:tasks.rados.rados.0.smithi022.stdout:2708: finishing write tid 2 to smithi02284960-41 2024-09-21T09:40:42.116 INFO:tasks.rados.rados.0.smithi022.stdout:2708: finishing write tid 3 to smithi02284960-41 2024-09-21T09:40:42.116 INFO:tasks.rados.rados.0.smithi022.stdout:2708: finishing write tid 4 to smithi02284960-41 2024-09-21T09:40:42.117 INFO:tasks.rados.rados.0.smithi022.stdout:2708: finishing write tid 5 to smithi02284960-41 2024-09-21T09:40:42.120 INFO:tasks.rados.rados.0.smithi022.stdout:2708: finishing write tid 6 to smithi02284960-41 2024-09-21T09:40:42.120 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 41 v 1424 (ObjNum 858 snap 276 seq_num 858) dirty exists 2024-09-21T09:40:42.120 INFO:tasks.rados.rados.0.smithi022.stdout:2708: left oid 41 (ObjNum 858 snap 276 seq_num 858) 2024-09-21T09:40:42.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:42 smithi022 ceph-mon[98615]: pgmap v530: 121 pgs: 121 active+clean; 213 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 4.7 MiB/s rd, 2.2 MiB/s wr, 12 op/s 2024-09-21T09:40:42.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:42 smithi022 ceph-mon[98615]: osdmap e618: 8 total, 8 up, 8 in 2024-09-21T09:40:42.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:42 smithi022 ceph-mon[96736]: pgmap v530: 121 pgs: 121 active+clean; 213 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 4.7 MiB/s rd, 2.2 MiB/s wr, 12 op/s 2024-09-21T09:40:42.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:42 smithi022 ceph-mon[96736]: osdmap e618: 8 total, 8 up, 8 in 2024-09-21T09:40:42.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:42 smithi064 ceph-mon[86709]: pgmap v530: 121 pgs: 121 active+clean; 213 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 4.7 MiB/s rd, 2.2 MiB/s wr, 12 op/s 2024-09-21T09:40:42.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:42 smithi064 ceph-mon[86709]: osdmap e618: 8 total, 8 up, 8 in 2024-09-21T09:40:43.056 INFO:tasks.rados.rados.0.smithi022.stdout:2706: done (3 left) 2024-09-21T09:40:43.057 INFO:tasks.rados.rados.0.smithi022.stdout:2707: done (2 left) 2024-09-21T09:40:43.057 INFO:tasks.rados.rados.0.smithi022.stdout:2708: done (1 left) 2024-09-21T09:40:43.057 INFO:tasks.rados.rados.0.smithi022.stdout:2709: done (0 left) 2024-09-21T09:40:43.057 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 48 current snap is 277 2024-09-21T09:40:43.057 INFO:tasks.rados.rados.0.smithi022.stdout:2710: seq_num 859 ranges {0=794624,794624=614400,1409024=638976,2048000=352256} 2024-09-21T09:40:43.060 INFO:tasks.rados.rados.0.smithi022.stdout:2710: writing smithi02284960-48 from 0 to 794624 tid 1 2024-09-21T09:40:43.064 INFO:tasks.rados.rados.0.smithi022.stdout:2710: writing smithi02284960-48 from 794624 to 1409024 tid 2 2024-09-21T09:40:43.068 INFO:tasks.rados.rados.0.smithi022.stdout:2710: writing smithi02284960-48 from 1409024 to 2048000 tid 3 2024-09-21T09:40:43.071 INFO:tasks.rados.rados.0.smithi022.stdout:2710: writing smithi02284960-48 from 2048000 to 2400256 tid 4 2024-09-21T09:40:43.071 INFO:tasks.rados.rados.0.smithi022.stdout:2711: delete oid 9 current snap is 277 2024-09-21T09:40:43.071 INFO:tasks.rados.rados.0.smithi022.stdout:2710: finishing write tid 1 to smithi02284960-48 2024-09-21T09:40:43.071 INFO:tasks.rados.rados.0.smithi022.stdout:2710: finishing write tid 2 to smithi02284960-48 2024-09-21T09:40:43.074 INFO:tasks.rados.rados.0.smithi022.stdout:2711: done (1 left) 2024-09-21T09:40:43.074 INFO:tasks.rados.rados.0.smithi022.stdout:2712: read oid 25 snap -1 2024-09-21T09:40:43.074 INFO:tasks.rados.rados.0.smithi022.stdout:2712: expect deleted 2024-09-21T09:40:43.074 INFO:tasks.rados.rados.0.smithi022.stdout:2713: read oid 18 snap -1 2024-09-21T09:40:43.074 INFO:tasks.rados.rados.0.smithi022.stdout:2713: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-21T09:40:43.074 INFO:tasks.rados.rados.0.smithi022.stdout:2714: setattr oid 20 current snap is 277 2024-09-21T09:40:43.075 INFO:tasks.rados.rados.0.smithi022.stdout:2713: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-21T09:40:43.080 INFO:tasks.rados.rados.0.smithi022.stdout:2710: finishing write tid 3 to smithi02284960-48 2024-09-21T09:40:43.080 INFO:tasks.rados.rados.0.smithi022.stdout:2710: finishing write tid 4 to smithi02284960-48 2024-09-21T09:40:43.080 INFO:tasks.rados.rados.0.smithi022.stdout:2710: finishing write tid 5 to smithi02284960-48 2024-09-21T09:40:43.080 INFO:tasks.rados.rados.0.smithi022.stdout:2710: finishing write tid 6 to smithi02284960-48 2024-09-21T09:40:43.080 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 48 v 1648 (ObjNum 859 snap 277 seq_num 859) dirty exists 2024-09-21T09:40:43.080 INFO:tasks.rados.rados.0.smithi022.stdout:2710: left oid 48 (ObjNum 859 snap 277 seq_num 859) 2024-09-21T09:40:43.081 INFO:tasks.rados.rados.0.smithi022.stdout:2710: done (3 left) 2024-09-21T09:40:43.081 INFO:tasks.rados.rados.0.smithi022.stdout:2712: done (2 left) 2024-09-21T09:40:43.081 INFO:tasks.rados.rados.0.smithi022.stdout:2713: done (1 left) 2024-09-21T09:40:43.081 INFO:tasks.rados.rados.0.smithi022.stdout:2715: delete oid 14 current snap is 277 2024-09-21T09:40:43.082 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 20 v 1650 (ObjNum 796 snap 249 seq_num 796) dirty exists 2024-09-21T09:40:43.084 INFO:tasks.rados.rados.0.smithi022.stdout:2714: done (1 left) 2024-09-21T09:40:43.084 INFO:tasks.rados.rados.0.smithi022.stdout:2715: done (0 left) 2024-09-21T09:40:43.084 INFO:tasks.rados.rados.0.smithi022.stdout:2716: delete oid 26 current snap is 277 2024-09-21T09:40:43.086 INFO:tasks.rados.rados.0.smithi022.stdout:2716: done (0 left) 2024-09-21T09:40:43.086 INFO:tasks.rados.rados.0.smithi022.stdout:2717: read oid 28 snap -1 2024-09-21T09:40:43.086 INFO:tasks.rados.rados.0.smithi022.stdout:2717: expect (ObjNum 841 snap 269 seq_num 841) 2024-09-21T09:40:43.086 INFO:tasks.rados.rados.0.smithi022.stdout:2718: copy_from oid 14 from oid 9 current snap is 277 2024-09-21T09:40:43.087 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 41 current snap is 277 2024-09-21T09:40:43.087 INFO:tasks.rados.rados.0.smithi022.stdout:2719: seq_num 863 ranges {12353536=57344} 2024-09-21T09:40:43.155 INFO:tasks.rados.rados.0.smithi022.stdout:2719: writing smithi02284960-41 from 12353536 to 12410880 tid 1 2024-09-21T09:40:43.155 INFO:tasks.rados.rados.0.smithi022.stdout:2720: delete oid 38 current snap is 277 2024-09-21T09:40:43.155 INFO:tasks.rados.rados.0.smithi022.stdout:2718: finishing copy_from to smithi02284960-14 2024-09-21T09:40:43.155 INFO:tasks.rados.rados.0.smithi022.stdout:2718: got expected ENOENT (src dne) 2024-09-21T09:40:43.155 INFO:tasks.rados.rados.0.smithi022.stdout:2718: finishing copy_from racing read to smithi02284960-14 2024-09-21T09:40:43.155 INFO:tasks.rados.rados.0.smithi022.stdout:2717: expect (ObjNum 841 snap 269 seq_num 841) 2024-09-21T09:40:43.307 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:43 smithi022 ceph-mon[98615]: osdmap e619: 8 total, 8 up, 8 in 2024-09-21T09:40:43.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:43 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:40:43.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:43 smithi022 ceph-mon[96736]: osdmap e619: 8 total, 8 up, 8 in 2024-09-21T09:40:43.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:43 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:40:43.345 INFO:tasks.rados.rados.0.smithi022.stdout:2719: finishing write tid 1 to smithi02284960-41 2024-09-21T09:40:43.345 INFO:tasks.rados.rados.0.smithi022.stdout:2719: finishing write tid 2 to smithi02284960-41 2024-09-21T09:40:43.345 INFO:tasks.rados.rados.0.smithi022.stdout:2719: finishing write tid 3 to smithi02284960-41 2024-09-21T09:40:43.345 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 41 v 1426 (ObjNum 863 snap 277 seq_num 863) dirty exists 2024-09-21T09:40:43.345 INFO:tasks.rados.rados.0.smithi022.stdout:2719: left oid 41 (ObjNum 863 snap 277 seq_num 863) 2024-09-21T09:40:43.345 INFO:tasks.rados.rados.0.smithi022.stdout:2717: done (3 left) 2024-09-21T09:40:43.345 INFO:tasks.rados.rados.0.smithi022.stdout:2718: done (2 left) 2024-09-21T09:40:43.346 INFO:tasks.rados.rados.0.smithi022.stdout:2719: done (1 left) 2024-09-21T09:40:43.346 INFO:tasks.rados.rados.0.smithi022.stdout:2720: done (0 left) 2024-09-21T09:40:43.346 INFO:tasks.rados.rados.0.smithi022.stdout:2721: rollback oid 2 current snap is 277 2024-09-21T09:40:43.346 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 2 to 276 2024-09-21T09:40:43.346 INFO:tasks.rados.rados.0.smithi022.stdout:2722: rollback oid 12 current snap is 277 2024-09-21T09:40:43.346 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 12 to 252 2024-09-21T09:40:43.346 INFO:tasks.rados.rados.0.smithi022.stdout:2723: rollback oid 50 current snap is 277 2024-09-21T09:40:43.346 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 50 to 252 2024-09-21T09:40:43.346 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 46 current snap is 277 2024-09-21T09:40:43.346 INFO:tasks.rados.rados.0.smithi022.stdout:2724: seq_num 865 ranges {0=90112} 2024-09-21T09:40:43.346 INFO:tasks.rados.rados.0.smithi022.stdout:2724: writing smithi02284960-46 from 0 to 90112 tid 1 2024-09-21T09:40:43.346 INFO:tasks.rados.rados.0.smithi022.stdout:2725: snap_create 2024-09-21T09:40:43.347 INFO:tasks.rados.rados.0.smithi022.stdout:2721: finishing rollback tid 1 to smithi02284960-2 2024-09-21T09:40:43.347 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 2 v 1283 (ObjNum 49 snap 0 seq_num 1953066355) dirty dne 2024-09-21T09:40:43.349 INFO:tasks.rados.rados.0.smithi022.stdout:2724: finishing write tid 1 to smithi02284960-46 2024-09-21T09:40:43.349 INFO:tasks.rados.rados.0.smithi022.stdout:2723: finishing rollback tid 0 to smithi02284960-50 2024-09-21T09:40:43.349 INFO:tasks.rados.rados.0.smithi022.stdout:2724: finishing write tid 2 to smithi02284960-46 2024-09-21T09:40:43.349 INFO:tasks.rados.rados.0.smithi022.stdout:2723: finishing rollback tid 1 to smithi02284960-50 2024-09-21T09:40:43.349 INFO:tasks.rados.rados.0.smithi022.stdout:2723: finishing rollback tid 2 to smithi02284960-50 2024-09-21T09:40:43.349 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 50 v 1313 (ObjNum 701 snap 216 seq_num 701) dirty exists 2024-09-21T09:40:43.350 INFO:tasks.rados.rados.0.smithi022.stdout:2722: finishing rollback tid 0 to smithi02284960-12 2024-09-21T09:40:43.350 INFO:tasks.rados.rados.0.smithi022.stdout:2724: finishing write tid 3 to smithi02284960-46 2024-09-21T09:40:43.350 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 46 v 1515 (ObjNum 865 snap 277 seq_num 865) dirty exists 2024-09-21T09:40:43.350 INFO:tasks.rados.rados.0.smithi022.stdout:2724: left oid 46 (ObjNum 865 snap 277 seq_num 865) 2024-09-21T09:40:43.350 INFO:tasks.rados.rados.0.smithi022.stdout:2722: finishing rollback tid 1 to smithi02284960-12 2024-09-21T09:40:43.350 INFO:tasks.rados.rados.0.smithi022.stdout:2722: finishing rollback tid 2 to smithi02284960-12 2024-09-21T09:40:43.350 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 12 v 1587 (ObjNum 633 snap 189 seq_num 633) dirty exists 2024-09-21T09:40:43.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:43 smithi064 ceph-mon[86709]: osdmap e619: 8 total, 8 up, 8 in 2024-09-21T09:40:43.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:43 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:40:44.034 INFO:tasks.rados.rados.0.smithi022.stdout:2721: done (4 left) 2024-09-21T09:40:44.035 INFO:tasks.rados.rados.0.smithi022.stdout:2722: done (3 left) 2024-09-21T09:40:44.035 INFO:tasks.rados.rados.0.smithi022.stdout:2723: done (2 left) 2024-09-21T09:40:44.035 INFO:tasks.rados.rados.0.smithi022.stdout:2724: done (1 left) 2024-09-21T09:40:44.035 INFO:tasks.rados.rados.0.smithi022.stdout:2725: done (0 left) 2024-09-21T09:40:44.035 INFO:tasks.rados.rados.0.smithi022.stdout:2726: read oid 19 snap -1 2024-09-21T09:40:44.035 INFO:tasks.rados.rados.0.smithi022.stdout:2726: expect deleted 2024-09-21T09:40:44.035 INFO:tasks.rados.rados.0.smithi022.stdout:2727: snap_remove snap 271 2024-09-21T09:40:44.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:44 smithi022 ceph-mon[96736]: pgmap v533: 121 pgs: 121 active+clean; 216 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 4.7 MiB/s rd, 1.1 MiB/s wr, 16 op/s 2024-09-21T09:40:44.310 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:44 smithi022 ceph-mon[96736]: osdmap e620: 8 total, 8 up, 8 in 2024-09-21T09:40:44.310 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:44 smithi022 ceph-mon[96736]: osdmap e621: 8 total, 8 up, 8 in 2024-09-21T09:40:44.310 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:44 smithi022 ceph-mon[98615]: pgmap v533: 121 pgs: 121 active+clean; 216 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 4.7 MiB/s rd, 1.1 MiB/s wr, 16 op/s 2024-09-21T09:40:44.310 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:44 smithi022 ceph-mon[98615]: osdmap e620: 8 total, 8 up, 8 in 2024-09-21T09:40:44.310 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:44 smithi022 ceph-mon[98615]: osdmap e621: 8 total, 8 up, 8 in 2024-09-21T09:40:44.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:44 smithi064 ceph-mon[86709]: pgmap v533: 121 pgs: 121 active+clean; 216 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 4.7 MiB/s rd, 1.1 MiB/s wr, 16 op/s 2024-09-21T09:40:44.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:44 smithi064 ceph-mon[86709]: osdmap e620: 8 total, 8 up, 8 in 2024-09-21T09:40:44.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:44 smithi064 ceph-mon[86709]: osdmap e621: 8 total, 8 up, 8 in 2024-09-21T09:40:45.037 INFO:tasks.rados.rados.0.smithi022.stdout:2727: done (1 left) 2024-09-21T09:40:45.037 INFO:tasks.rados.rados.0.smithi022.stdout:2728: snap_create 2024-09-21T09:40:46.038 INFO:tasks.rados.rados.0.smithi022.stdout:2726: done (1 left) 2024-09-21T09:40:46.038 INFO:tasks.rados.rados.0.smithi022.stdout:2728: done (0 left) 2024-09-21T09:40:46.038 INFO:tasks.rados.rados.0.smithi022.stdout:2729: copy_from oid 50 from oid 43 current snap is 279 2024-09-21T09:40:46.038 INFO:tasks.rados.rados.0.smithi022.stdout:2730: snap_create 2024-09-21T09:40:46.039 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:40:45 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:40:45] "GET /metrics HTTP/1.1" 200 36357 "" "Prometheus/2.51.0" 2024-09-21T09:40:46.120 INFO:tasks.rados.rados.0.smithi022.stdout:2729: finishing copy_from racing read to smithi02284960-50 2024-09-21T09:40:46.124 INFO:tasks.rados.rados.0.smithi022.stdout:2729: finishing copy_from to smithi02284960-50 2024-09-21T09:40:46.124 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 50 v 1402 (ObjNum 817 snap 261 seq_num 817) dirty exists 2024-09-21T09:40:46.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:46 smithi022 ceph-mon[96736]: pgmap v536: 121 pgs: 121 active+clean; 216 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 574 KiB/s wr, 8 op/s 2024-09-21T09:40:46.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:46 smithi022 ceph-mon[96736]: osdmap e622: 8 total, 8 up, 8 in 2024-09-21T09:40:46.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:46 smithi022 ceph-mon[98615]: pgmap v536: 121 pgs: 121 active+clean; 216 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 574 KiB/s wr, 8 op/s 2024-09-21T09:40:46.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:46 smithi022 ceph-mon[98615]: osdmap e622: 8 total, 8 up, 8 in 2024-09-21T09:40:46.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:46 smithi064 ceph-mon[86709]: pgmap v536: 121 pgs: 121 active+clean; 216 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 574 KiB/s wr, 8 op/s 2024-09-21T09:40:46.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:46 smithi064 ceph-mon[86709]: osdmap e622: 8 total, 8 up, 8 in 2024-09-21T09:40:47.050 INFO:tasks.rados.rados.0.smithi022.stdout:2729: done (1 left) 2024-09-21T09:40:47.051 INFO:tasks.rados.rados.0.smithi022.stdout:2730: done (0 left) 2024-09-21T09:40:47.051 INFO:tasks.rados.rados.0.smithi022.stdout:2731: rollback oid 44 current snap is 280 2024-09-21T09:40:47.051 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 44 to 267 2024-09-21T09:40:47.051 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 42 current snap is 280 2024-09-21T09:40:47.051 INFO:tasks.rados.rados.0.smithi022.stdout:2732: seq_num 866 ranges {3186688=106496} 2024-09-21T09:40:47.068 INFO:tasks.rados.rados.0.smithi022.stdout:2732: writing smithi02284960-42 from 3186688 to 3293184 tid 1 2024-09-21T09:40:47.069 INFO:tasks.rados.rados.0.smithi022.stdout:2733: rollback oid 5 current snap is 280 2024-09-21T09:40:47.069 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 5 to 252 2024-09-21T09:40:47.069 INFO:tasks.rados.rados.0.smithi022.stdout:2731: finishing rollback tid 0 to smithi02284960-44 2024-09-21T09:40:47.069 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 50 current snap is 280 2024-09-21T09:40:47.069 INFO:tasks.rados.rados.0.smithi022.stdout:2734: seq_num 867 ranges {16334848=122880} 2024-09-21T09:40:47.159 INFO:tasks.rados.rados.0.smithi022.stdout:2734: writing smithi02284960-50 from 16334848 to 16457728 tid 1 2024-09-21T09:40:47.159 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 32 current snap is 280 2024-09-21T09:40:47.159 INFO:tasks.rados.rados.0.smithi022.stdout:2735: seq_num 868 ranges {0=139264} 2024-09-21T09:40:47.160 INFO:tasks.rados.rados.0.smithi022.stdout:2735: writing smithi02284960-32 from 0 to 139264 tid 1 2024-09-21T09:40:47.160 INFO:tasks.rados.rados.0.smithi022.stdout:2736: read oid 41 snap -1 2024-09-21T09:40:47.160 INFO:tasks.rados.rados.0.smithi022.stdout:2736: expect (ObjNum 863 snap 277 seq_num 863) 2024-09-21T09:40:47.160 INFO:tasks.rados.rados.0.smithi022.stdout:2737: delete oid 30 current snap is 280 2024-09-21T09:40:47.160 INFO:tasks.rados.rados.0.smithi022.stdout:2731: finishing rollback tid 1 to smithi02284960-44 2024-09-21T09:40:47.160 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 44 v 1517 (ObjNum 161 snap 0 seq_num 0) dirty dne 2024-09-21T09:40:47.160 INFO:tasks.rados.rados.0.smithi022.stdout:2733: finishing rollback tid 1 to smithi02284960-5 2024-09-21T09:40:47.160 INFO:tasks.rados.rados.0.smithi022.stdout:2733: finishing rollback tid 2 to smithi02284960-5 2024-09-21T09:40:47.160 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 5 v 1433 (ObjNum 672 snap 208 seq_num 672) dirty exists 2024-09-21T09:40:47.160 INFO:tasks.rados.rados.0.smithi022.stdout:2732: finishing write tid 1 to smithi02284960-42 2024-09-21T09:40:47.160 INFO:tasks.rados.rados.0.smithi022.stdout:2732: finishing write tid 2 to smithi02284960-42 2024-09-21T09:40:47.160 INFO:tasks.rados.rados.0.smithi022.stdout:2732: finishing write tid 3 to smithi02284960-42 2024-09-21T09:40:47.161 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 42 v 1507 (ObjNum 866 snap 280 seq_num 866) dirty exists 2024-09-21T09:40:47.161 INFO:tasks.rados.rados.0.smithi022.stdout:2732: left oid 42 (ObjNum 866 snap 280 seq_num 866) 2024-09-21T09:40:47.165 INFO:tasks.rados.rados.0.smithi022.stdout:2734: finishing write tid 1 to smithi02284960-50 2024-09-21T09:40:47.165 INFO:tasks.rados.rados.0.smithi022.stdout:2735: finishing write tid 1 to smithi02284960-32 2024-09-21T09:40:47.165 INFO:tasks.rados.rados.0.smithi022.stdout:2734: finishing write tid 2 to smithi02284960-50 2024-09-21T09:40:47.165 INFO:tasks.rados.rados.0.smithi022.stdout:2735: finishing write tid 2 to smithi02284960-32 2024-09-21T09:40:47.165 INFO:tasks.rados.rados.0.smithi022.stdout:2731: done (6 left) 2024-09-21T09:40:47.165 INFO:tasks.rados.rados.0.smithi022.stdout:2732: done (5 left) 2024-09-21T09:40:47.166 INFO:tasks.rados.rados.0.smithi022.stdout:2733: done (4 left) 2024-09-21T09:40:47.166 INFO:tasks.rados.rados.0.smithi022.stdout:2737: done (3 left) 2024-09-21T09:40:47.166 INFO:tasks.rados.rados.0.smithi022.stdout:2738: delete oid 18 current snap is 280 2024-09-21T09:40:47.167 INFO:tasks.rados.rados.0.smithi022.stdout:2735: finishing write tid 3 to smithi02284960-32 2024-09-21T09:40:47.167 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 32 v 1658 (ObjNum 868 snap 280 seq_num 868) dirty exists 2024-09-21T09:40:47.167 INFO:tasks.rados.rados.0.smithi022.stdout:2735: left oid 32 (ObjNum 868 snap 280 seq_num 868) 2024-09-21T09:40:47.167 INFO:tasks.rados.rados.0.smithi022.stdout:2734: finishing write tid 3 to smithi02284960-50 2024-09-21T09:40:47.168 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 50 v 1404 (ObjNum 867 snap 280 seq_num 867) dirty exists 2024-09-21T09:40:47.168 INFO:tasks.rados.rados.0.smithi022.stdout:2734: left oid 50 (ObjNum 867 snap 280 seq_num 867) 2024-09-21T09:40:47.169 INFO:tasks.rados.rados.0.smithi022.stdout:2734: done (3 left) 2024-09-21T09:40:47.170 INFO:tasks.rados.rados.0.smithi022.stdout:2735: done (2 left) 2024-09-21T09:40:47.170 INFO:tasks.rados.rados.0.smithi022.stdout:2738: done (1 left) 2024-09-21T09:40:47.170 INFO:tasks.rados.rados.0.smithi022.stdout:2739: rollback oid 38 current snap is 280 2024-09-21T09:40:47.170 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 38 to 276 2024-09-21T09:40:47.170 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 47 current snap is 280 2024-09-21T09:40:47.170 INFO:tasks.rados.rados.0.smithi022.stdout:2740: seq_num 871 ranges {1490944=188416} 2024-09-21T09:40:47.179 INFO:tasks.rados.rados.0.smithi022.stdout:2740: writing smithi02284960-47 from 1490944 to 1679360 tid 1 2024-09-21T09:40:47.180 INFO:tasks.rados.rados.0.smithi022.stdout:2741: read oid 7 snap -1 2024-09-21T09:40:47.180 INFO:tasks.rados.rados.0.smithi022.stdout:2741: expect (ObjNum 785 snap 246 seq_num 785) 2024-09-21T09:40:47.180 INFO:tasks.rados.rados.0.smithi022.stdout:2742: rmattr oid 1 current snap is 280 2024-09-21T09:40:47.180 INFO:tasks.rados.rados.0.smithi022.stdout:2742: done (4 left) 2024-09-21T09:40:47.180 INFO:tasks.rados.rados.0.smithi022.stdout:2743: setattr oid 3 current snap is 280 2024-09-21T09:40:47.180 INFO:tasks.rados.rados.0.smithi022.stdout:2739: finishing rollback tid 1 to smithi02284960-38 2024-09-21T09:40:47.180 INFO:tasks.rados.rados.0.smithi022.stdout:2739: finishing rollback tid 2 to smithi02284960-38 2024-09-21T09:40:47.180 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 38 v 1425 (ObjNum 839 snap 269 seq_num 839) dirty exists 2024-09-21T09:40:47.183 INFO:tasks.rados.rados.0.smithi022.stdout:2739: done (4 left) 2024-09-21T09:40:47.183 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 19 current snap is 280 2024-09-21T09:40:47.183 INFO:tasks.rados.rados.0.smithi022.stdout:2744: seq_num 872 ranges {0=212992} 2024-09-21T09:40:47.185 INFO:tasks.rados.rados.0.smithi022.stdout:2744: writing smithi02284960-19 from 0 to 212992 tid 1 2024-09-21T09:40:47.185 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 33 current snap is 280 2024-09-21T09:40:47.185 INFO:tasks.rados.rados.0.smithi022.stdout:2745: seq_num 873 ranges {3768320=229376} 2024-09-21T09:40:47.207 INFO:tasks.rados.rados.0.smithi022.stdout:2745: writing smithi02284960-33 from 3768320 to 3997696 tid 1 2024-09-21T09:40:47.207 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 10 current snap is 280 2024-09-21T09:40:47.207 INFO:tasks.rados.rados.0.smithi022.stdout:2746: seq_num 874 ranges {2162688=245760} 2024-09-21T09:40:47.220 INFO:tasks.rados.rados.0.smithi022.stdout:2746: writing smithi02284960-10 from 2162688 to 2408448 tid 1 2024-09-21T09:40:47.221 INFO:tasks.rados.rados.0.smithi022.stdout:2747: rollback oid 36 current snap is 280 2024-09-21T09:40:47.221 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 36 to 276 2024-09-21T09:40:47.221 INFO:tasks.rados.rados.0.smithi022.stdout:2740: finishing write tid 1 to smithi02284960-47 2024-09-21T09:40:47.221 INFO:tasks.rados.rados.0.smithi022.stdout:2740: finishing write tid 2 to smithi02284960-47 2024-09-21T09:40:47.221 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 3 v 612 (ObjNum 593 snap 0 seq_num 3571194888) dirty exists 2024-09-21T09:40:47.221 INFO:tasks.rados.rados.0.smithi022.stdout:2741: expect (ObjNum 785 snap 246 seq_num 785) 2024-09-21T09:40:47.307 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:47 smithi022 ceph-mon[96736]: osdmap e623: 8 total, 8 up, 8 in 2024-09-21T09:40:47.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:47 smithi022 ceph-mon[98615]: osdmap e623: 8 total, 8 up, 8 in 2024-09-21T09:40:47.343 INFO:tasks.rados.rados.0.smithi022.stdout:2740: finishing write tid 3 to smithi02284960-47 2024-09-21T09:40:47.343 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 47 v 1519 (ObjNum 871 snap 280 seq_num 871) dirty exists 2024-09-21T09:40:47.343 INFO:tasks.rados.rados.0.smithi022.stdout:2740: left oid 47 (ObjNum 871 snap 280 seq_num 871) 2024-09-21T09:40:47.343 INFO:tasks.rados.rados.0.smithi022.stdout:2744: finishing write tid 1 to smithi02284960-19 2024-09-21T09:40:47.343 INFO:tasks.rados.rados.0.smithi022.stdout:2740: done (7 left) 2024-09-21T09:40:47.344 INFO:tasks.rados.rados.0.smithi022.stdout:2741: done (6 left) 2024-09-21T09:40:47.344 INFO:tasks.rados.rados.0.smithi022.stdout:2743: done (5 left) 2024-09-21T09:40:47.344 INFO:tasks.rados.rados.0.smithi022.stdout:2748: delete oid 29 current snap is 280 2024-09-21T09:40:47.344 INFO:tasks.rados.rados.0.smithi022.stdout:2744: finishing write tid 2 to smithi02284960-19 2024-09-21T09:40:47.344 INFO:tasks.rados.rados.0.smithi022.stdout:2744: finishing write tid 3 to smithi02284960-19 2024-09-21T09:40:47.344 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 19 v 1591 (ObjNum 872 snap 280 seq_num 872) dirty exists 2024-09-21T09:40:47.344 INFO:tasks.rados.rados.0.smithi022.stdout:2744: left oid 19 (ObjNum 872 snap 280 seq_num 872) 2024-09-21T09:40:47.344 INFO:tasks.rados.rados.0.smithi022.stdout:2745: finishing write tid 1 to smithi02284960-33 2024-09-21T09:40:47.344 INFO:tasks.rados.rados.0.smithi022.stdout:2745: finishing write tid 2 to smithi02284960-33 2024-09-21T09:40:47.344 INFO:tasks.rados.rados.0.smithi022.stdout:2745: finishing write tid 3 to smithi02284960-33 2024-09-21T09:40:47.344 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 33 v 1661 (ObjNum 873 snap 280 seq_num 873) dirty exists 2024-09-21T09:40:47.344 INFO:tasks.rados.rados.0.smithi022.stdout:2745: left oid 33 (ObjNum 873 snap 280 seq_num 873) 2024-09-21T09:40:47.344 INFO:tasks.rados.rados.0.smithi022.stdout:2736: expect (ObjNum 863 snap 277 seq_num 863) 2024-09-21T09:40:47.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:47 smithi064 ceph-mon[86709]: osdmap e623: 8 total, 8 up, 8 in 2024-09-21T09:40:48.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:48 smithi064 ceph-mon[86709]: pgmap v539: 121 pgs: 121 active+clean; 223 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 29 KiB/s rd, 1.9 MiB/s wr, 12 op/s 2024-09-21T09:40:48.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:48 smithi064 ceph-mon[86709]: osdmap e624: 8 total, 8 up, 8 in 2024-09-21T09:40:48.307 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:48 smithi022 ceph-mon[96736]: pgmap v539: 121 pgs: 121 active+clean; 223 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 29 KiB/s rd, 1.9 MiB/s wr, 12 op/s 2024-09-21T09:40:48.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:48 smithi022 ceph-mon[96736]: osdmap e624: 8 total, 8 up, 8 in 2024-09-21T09:40:48.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:48 smithi022 ceph-mon[98615]: pgmap v539: 121 pgs: 121 active+clean; 223 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 29 KiB/s rd, 1.9 MiB/s wr, 12 op/s 2024-09-21T09:40:48.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:48 smithi022 ceph-mon[98615]: osdmap e624: 8 total, 8 up, 8 in 2024-09-21T09:40:48.727 INFO:tasks.rados.rados.0.smithi022.stdout:2747: finishing rollback tid 1 to smithi02284960-36 2024-09-21T09:40:48.727 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 36 v 1310 (ObjNum 65 snap 0 seq_num 3570790704) dirty dne 2024-09-21T09:40:48.727 INFO:tasks.rados.rados.0.smithi022.stdout:2746: finishing write tid 1 to smithi02284960-10 2024-09-21T09:40:48.727 INFO:tasks.rados.rados.0.smithi022.stdout:2746: finishing write tid 2 to smithi02284960-10 2024-09-21T09:40:48.727 INFO:tasks.rados.rados.0.smithi022.stdout:2746: finishing write tid 3 to smithi02284960-10 2024-09-21T09:40:48.727 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 10 v 1428 (ObjNum 874 snap 280 seq_num 874) dirty exists 2024-09-21T09:40:48.727 INFO:tasks.rados.rados.0.smithi022.stdout:2746: left oid 10 (ObjNum 874 snap 280 seq_num 874) 2024-09-21T09:40:48.727 INFO:tasks.rados.rados.0.smithi022.stdout:2736: done (5 left) 2024-09-21T09:40:48.727 INFO:tasks.rados.rados.0.smithi022.stdout:2744: done (4 left) 2024-09-21T09:40:48.727 INFO:tasks.rados.rados.0.smithi022.stdout:2745: done (3 left) 2024-09-21T09:40:48.728 INFO:tasks.rados.rados.0.smithi022.stdout:2746: done (2 left) 2024-09-21T09:40:48.728 INFO:tasks.rados.rados.0.smithi022.stdout:2747: done (1 left) 2024-09-21T09:40:48.728 INFO:tasks.rados.rados.0.smithi022.stdout:2748: done (0 left) 2024-09-21T09:40:48.728 INFO:tasks.rados.rados.0.smithi022.stdout:2749: snap_create 2024-09-21T09:40:49.035 INFO:tasks.rados.rados.0.smithi022.stdout:2749: done (0 left) 2024-09-21T09:40:49.035 INFO:tasks.rados.rados.0.smithi022.stdout:2750: copy_from oid 8 from oid 3 current snap is 281 2024-09-21T09:40:49.036 INFO:tasks.rados.rados.0.smithi022.stdout:2751: setattr oid 13 current snap is 281 2024-09-21T09:40:49.036 INFO:tasks.rados.rados.0.smithi022.stdout:2752: delete oid 23 current snap is 281 2024-09-21T09:40:49.040 INFO:tasks.rados.rados.0.smithi022.stdout:2752: done (2 left) 2024-09-21T09:40:49.040 INFO:tasks.rados.rados.0.smithi022.stdout:2753: copy_from oid 2 from oid 24 current snap is 281 2024-09-21T09:40:49.041 INFO:tasks.rados.rados.0.smithi022.stdout:2754: snap_create 2024-09-21T09:40:49.046 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 13 v 1508 (ObjNum 33 snap 0 seq_num 3570880080) dirty exists 2024-09-21T09:40:49.046 INFO:tasks.rados.rados.0.smithi022.stdout:2750: finishing copy_from racing read to smithi02284960-8 2024-09-21T09:40:49.048 INFO:tasks.rados.rados.0.smithi022.stdout:2750: finishing copy_from to smithi02284960-8 2024-09-21T09:40:49.048 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 8 v 612 (ObjNum 161 snap 0 seq_num 0) dirty exists 2024-09-21T09:40:49.066 INFO:tasks.rados.rados.0.smithi022.stdout:2753: finishing copy_from racing read to smithi02284960-2 2024-09-21T09:40:49.068 INFO:tasks.rados.rados.0.smithi022.stdout:2753: finishing copy_from to smithi02284960-2 2024-09-21T09:40:49.068 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 2 v 1536 (ObjNum 835 snap 269 seq_num 835) dirty exists 2024-09-21T09:40:50.041 INFO:tasks.rados.rados.0.smithi022.stdout:2750: done (3 left) 2024-09-21T09:40:50.041 INFO:tasks.rados.rados.0.smithi022.stdout:2751: done (2 left) 2024-09-21T09:40:50.041 INFO:tasks.rados.rados.0.smithi022.stdout:2753: done (1 left) 2024-09-21T09:40:50.041 INFO:tasks.rados.rados.0.smithi022.stdout:2754: done (0 left) 2024-09-21T09:40:50.041 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 12 current snap is 282 2024-09-21T09:40:50.041 INFO:tasks.rados.rados.0.smithi022.stdout:2755: seq_num 877 ranges {5120000=294912} 2024-09-21T09:40:50.071 INFO:tasks.rados.rados.0.smithi022.stdout:2755: writing smithi02284960-12 from 5120000 to 5414912 tid 1 2024-09-21T09:40:50.071 INFO:tasks.rados.rados.0.smithi022.stdout:2756: read oid 33 snap -1 2024-09-21T09:40:50.071 INFO:tasks.rados.rados.0.smithi022.stdout:2756: expect (ObjNum 873 snap 280 seq_num 873) 2024-09-21T09:40:50.071 INFO:tasks.rados.rados.0.smithi022.stdout:2757: read oid 50 snap 245 2024-09-21T09:40:50.071 INFO:tasks.rados.rados.0.smithi022.stdout:2757: expect (ObjNum 701 snap 216 seq_num 701) 2024-09-21T09:40:50.071 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 45 current snap is 282 2024-09-21T09:40:50.071 INFO:tasks.rados.rados.0.smithi022.stdout:2758: seq_num 878 ranges {0=311296} 2024-09-21T09:40:50.073 INFO:tasks.rados.rados.0.smithi022.stdout:2758: writing smithi02284960-45 from 0 to 311296 tid 1 2024-09-21T09:40:50.073 INFO:tasks.rados.rados.0.smithi022.stdout:2759: rollback oid 36 current snap is 282 2024-09-21T09:40:50.073 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 36 to 265 2024-09-21T09:40:50.073 INFO:tasks.rados.rados.0.smithi022.stdout:2760: snap_remove snap 246 2024-09-21T09:40:50.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:50 smithi064 ceph-mon[86709]: pgmap v541: 121 pgs: 121 active+clean; 241 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 3.0 MiB/s rd, 5.3 MiB/s wr, 19 op/s 2024-09-21T09:40:50.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:50 smithi064 ceph-mon[86709]: osdmap e625: 8 total, 8 up, 8 in 2024-09-21T09:40:50.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:50 smithi022 ceph-mon[98615]: pgmap v541: 121 pgs: 121 active+clean; 241 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 3.0 MiB/s rd, 5.3 MiB/s wr, 19 op/s 2024-09-21T09:40:50.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:50 smithi022 ceph-mon[98615]: osdmap e625: 8 total, 8 up, 8 in 2024-09-21T09:40:50.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:50 smithi022 ceph-mon[96736]: pgmap v541: 121 pgs: 121 active+clean; 241 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 3.0 MiB/s rd, 5.3 MiB/s wr, 19 op/s 2024-09-21T09:40:50.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:50 smithi022 ceph-mon[96736]: osdmap e625: 8 total, 8 up, 8 in 2024-09-21T09:40:50.521 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:40:50 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:40:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:40:50.522 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:50 smithi064 systemd[1]: Stopping Ceph grafana.a for d65e801e-77fa-11ef-baf6-efdc52797490... 2024-09-21T09:40:50.809 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:50 smithi064 podman[114294]: 2024-09-21 09:40:50.519939832 +0000 UTC m=+0.153141363 container died 527bb657d70d6d420e657297c4d3f77a9c60385cf52e4028a1c7dc70b92aecf7 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a, io.openshift.expose-services=, io.k8s.display-name=Red Hat Universal Base Image 8, name=ubi8, vcs-type=git, io.openshift.tags=base rhel8, maintainer=Paul Cuzner , release=239.1651231664, com.redhat.component=ubi8-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vendor=Red Hat, Inc., com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.buildah.version=1.32.0, architecture=x86_64, build-date=2022-04-29T11:48:11.055897, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Grafana Container configured for Ceph mgr/dashboard integration, description=Ceph Grafana Container, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, version=8.5, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com) 2024-09-21T09:40:51.048 INFO:tasks.rados.rados.0.smithi022.stdout:2760: done (5 left) 2024-09-21T09:40:51.048 INFO:tasks.rados.rados.0.smithi022.stdout:2761: delete oid 18 current snap is 282 2024-09-21T09:40:51.048 INFO:tasks.rados.rados.0.smithi022.stdout:2755: finishing write tid 1 to smithi02284960-12 2024-09-21T09:40:51.048 INFO:tasks.rados.rados.0.smithi022.stdout:2755: finishing write tid 2 to smithi02284960-12 2024-09-21T09:40:51.048 INFO:tasks.rados.rados.0.smithi022.stdout:2755: finishing write tid 3 to smithi02284960-12 2024-09-21T09:40:51.048 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 12 v 1594 (ObjNum 877 snap 282 seq_num 877) dirty exists 2024-09-21T09:40:51.049 INFO:tasks.rados.rados.0.smithi022.stdout:2755: left oid 12 (ObjNum 877 snap 282 seq_num 877) 2024-09-21T09:40:51.049 INFO:tasks.rados.rados.0.smithi022.stdout:2756: expect (ObjNum 873 snap 280 seq_num 873) 2024-09-21T09:40:51.176 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:51 smithi064 ceph-mon[86709]: osdmap e626: 8 total, 8 up, 8 in 2024-09-21T09:40:51.176 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:50 smithi064 podman[114294]: 2024-09-21 09:40:50.906807705 +0000 UTC m=+0.540009234 container cleanup 527bb657d70d6d420e657297c4d3f77a9c60385cf52e4028a1c7dc70b92aecf7 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a, distribution-scope=public, maintainer=Paul Cuzner , com.redhat.component=ubi8-container, summary=Grafana Container configured for Ceph mgr/dashboard integration, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.tags=base rhel8, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, name=ubi8, version=8.5, description=Ceph Grafana Container, vendor=Red Hat, Inc., build-date=2022-04-29T11:48:11.055897, architecture=x86_64, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.buildah.version=1.32.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, release=239.1651231664) 2024-09-21T09:40:51.176 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:50 smithi064 bash[114294]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a 2024-09-21T09:40:51.296 INFO:tasks.rados.rados.0.smithi022.stdout:2757: expect (ObjNum 701 snap 216 seq_num 701) 2024-09-21T09:40:51.307 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:51 smithi022 ceph-mon[98615]: osdmap e626: 8 total, 8 up, 8 in 2024-09-21T09:40:51.307 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:51 smithi022 ceph-mon[96736]: osdmap e626: 8 total, 8 up, 8 in 2024-09-21T09:40:51.441 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:51 smithi064 podman[114311]: 2024-09-21 09:40:51.174858697 +0000 UTC m=+0.653826616 container remove 527bb657d70d6d420e657297c4d3f77a9c60385cf52e4028a1c7dc70b92aecf7 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a, io.buildah.version=1.32.0, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, version=8.5, io.k8s.display-name=Red Hat Universal Base Image 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=ubi8, vendor=Red Hat, Inc., distribution-scope=public, maintainer=Paul Cuzner , release=239.1651231664, io.openshift.tags=base rhel8, com.redhat.component=ubi8-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, architecture=x86_64, build-date=2022-04-29T11:48:11.055897, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-type=git, description=Ceph Grafana Container, io.openshift.expose-services=) 2024-09-21T09:40:51.441 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:51 smithi064 systemd[1]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490@grafana.a.service: Main process exited, code=exited, status=143/n/a 2024-09-21T09:40:51.441 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:51 smithi064 systemd[1]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490@grafana.a.service: Failed with result 'exit-code'. 2024-09-21T09:40:51.442 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:51 smithi064 systemd[1]: Stopped Ceph grafana.a for d65e801e-77fa-11ef-baf6-efdc52797490. 2024-09-21T09:40:51.442 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:51 smithi064 systemd[1]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490@grafana.a.service: Consumed 7.323s CPU time. 2024-09-21T09:40:51.490 INFO:tasks.rados.rados.0.smithi022.stdout:2758: finishing write tid 1 to smithi02284960-45 2024-09-21T09:40:51.490 INFO:tasks.rados.rados.0.smithi022.stdout:2758: finishing write tid 2 to smithi02284960-45 2024-09-21T09:40:51.490 INFO:tasks.rados.rados.0.smithi022.stdout:2759: finishing rollback tid 1 to smithi02284960-36 2024-09-21T09:40:51.490 INFO:tasks.rados.rados.0.smithi022.stdout:2758: finishing write tid 3 to smithi02284960-45 2024-09-21T09:40:51.490 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 45 v 1406 (ObjNum 878 snap 282 seq_num 878) dirty exists 2024-09-21T09:40:51.490 INFO:tasks.rados.rados.0.smithi022.stdout:2758: left oid 45 (ObjNum 878 snap 282 seq_num 878) 2024-09-21T09:40:51.490 INFO:tasks.rados.rados.0.smithi022.stdout:2759: finishing rollback tid 2 to smithi02284960-36 2024-09-21T09:40:51.490 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 36 v 1408 (ObjNum 824 snap 262 seq_num 824) dirty exists 2024-09-21T09:40:51.490 INFO:tasks.rados.rados.0.smithi022.stdout:2755: done (5 left) 2024-09-21T09:40:51.490 INFO:tasks.rados.rados.0.smithi022.stdout:2756: done (4 left) 2024-09-21T09:40:51.490 INFO:tasks.rados.rados.0.smithi022.stdout:2757: done (3 left) 2024-09-21T09:40:51.490 INFO:tasks.rados.rados.0.smithi022.stdout:2758: done (2 left) 2024-09-21T09:40:51.490 INFO:tasks.rados.rados.0.smithi022.stdout:2759: done (1 left) 2024-09-21T09:40:51.490 INFO:tasks.rados.rados.0.smithi022.stdout:2761: done (0 left) 2024-09-21T09:40:51.491 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 10 current snap is 282 2024-09-21T09:40:51.491 INFO:tasks.rados.rados.0.smithi022.stdout:2762: seq_num 880 ranges {2408448=344064} 2024-09-21T09:40:51.505 INFO:tasks.rados.rados.0.smithi022.stdout:2762: writing smithi02284960-10 from 2408448 to 2752512 tid 1 2024-09-21T09:40:51.505 INFO:tasks.rados.rados.0.smithi022.stdout:2763: read oid 6 snap -1 2024-09-21T09:40:51.505 INFO:tasks.rados.rados.0.smithi022.stdout:2763: expect deleted 2024-09-21T09:40:51.505 INFO:tasks.rados.rados.0.smithi022.stdout:2764: rollback oid 37 current snap is 282 2024-09-21T09:40:51.505 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 37 to 265 2024-09-21T09:40:51.505 INFO:tasks.rados.rados.0.smithi022.stdout:2765: read oid 17 snap -1 2024-09-21T09:40:51.505 INFO:tasks.rados.rados.0.smithi022.stdout:2765: expect (ObjNum 842 snap 269 seq_num 842) 2024-09-21T09:40:51.505 INFO:tasks.rados.rados.0.smithi022.stdout:2766: rollback oid 30 current snap is 282 2024-09-21T09:40:51.505 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 30 to 270 2024-09-21T09:40:51.505 INFO:tasks.rados.rados.0.smithi022.stdout:2767: delete oid 3 current snap is 282 2024-09-21T09:40:51.512 INFO:tasks.rados.rados.0.smithi022.stdout:2764: finishing rollback tid 0 to smithi02284960-37 2024-09-21T09:40:51.512 INFO:tasks.rados.rados.0.smithi022.stdout:2764: finishing rollback tid 1 to smithi02284960-37 2024-09-21T09:40:51.513 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 37 v 1607 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-09-21T09:40:51.513 INFO:tasks.rados.rados.0.smithi022.stdout:2763: done (5 left) 2024-09-21T09:40:51.513 INFO:tasks.rados.rados.0.smithi022.stdout:2764: done (4 left) 2024-09-21T09:40:51.513 INFO:tasks.rados.rados.0.smithi022.stdout:2767: done (3 left) 2024-09-21T09:40:51.513 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 23 current snap is 282 2024-09-21T09:40:51.513 INFO:tasks.rados.rados.0.smithi022.stdout:2768: seq_num 882 ranges {0=376832} 2024-09-21T09:40:51.515 INFO:tasks.rados.rados.0.smithi022.stdout:2768: writing smithi02284960-23 from 0 to 376832 tid 1 2024-09-21T09:40:51.515 INFO:tasks.rados.rados.0.smithi022.stdout:2769: delete oid 19 current snap is 282 2024-09-21T09:40:51.515 INFO:tasks.rados.rados.0.smithi022.stdout:2762: finishing write tid 1 to smithi02284960-10 2024-09-21T09:40:51.516 INFO:tasks.rados.rados.0.smithi022.stdout:2762: finishing write tid 2 to smithi02284960-10 2024-09-21T09:40:51.516 INFO:tasks.rados.rados.0.smithi022.stdout:2762: finishing write tid 3 to smithi02284960-10 2024-09-21T09:40:51.516 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 10 v 1430 (ObjNum 880 snap 282 seq_num 880) dirty exists 2024-09-21T09:40:51.516 INFO:tasks.rados.rados.0.smithi022.stdout:2762: left oid 10 (ObjNum 880 snap 282 seq_num 880) 2024-09-21T09:40:51.519 INFO:tasks.rados.rados.0.smithi022.stdout:2762: done (4 left) 2024-09-21T09:40:51.519 INFO:tasks.rados.rados.0.smithi022.stdout:2769: done (3 left) 2024-09-21T09:40:51.519 INFO:tasks.rados.rados.0.smithi022.stdout:2770: read oid 46 snap -1 2024-09-21T09:40:51.519 INFO:tasks.rados.rados.0.smithi022.stdout:2770: expect (ObjNum 865 snap 277 seq_num 865) 2024-09-21T09:40:51.519 INFO:tasks.rados.rados.0.smithi022.stdout:2771: copy_from oid 9 from oid 20 current snap is 282 2024-09-21T09:40:51.519 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 42 current snap is 282 2024-09-21T09:40:51.520 INFO:tasks.rados.rados.0.smithi022.stdout:2772: seq_num 884 ranges {3293184=409600} 2024-09-21T09:40:51.541 INFO:tasks.rados.rados.0.smithi022.stdout:2772: writing smithi02284960-42 from 3293184 to 3702784 tid 1 2024-09-21T09:40:51.541 INFO:tasks.rados.rados.0.smithi022.stdout:2768: finishing write tid 1 to smithi02284960-23 2024-09-21T09:40:51.542 INFO:tasks.rados.rados.0.smithi022.stdout:2773: delete oid 25 current snap is 282 2024-09-21T09:40:51.542 INFO:tasks.rados.rados.0.smithi022.stdout:2768: finishing write tid 2 to smithi02284960-23 2024-09-21T09:40:51.542 INFO:tasks.rados.rados.0.smithi022.stdout:2768: finishing write tid 3 to smithi02284960-23 2024-09-21T09:40:51.542 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 23 v 1429 (ObjNum 882 snap 282 seq_num 882) dirty exists 2024-09-21T09:40:51.542 INFO:tasks.rados.rados.0.smithi022.stdout:2768: left oid 23 (ObjNum 882 snap 282 seq_num 882) 2024-09-21T09:40:51.542 INFO:tasks.rados.rados.0.smithi022.stdout:2770: expect (ObjNum 865 snap 277 seq_num 865) 2024-09-21T09:40:51.546 INFO:tasks.rados.rados.0.smithi022.stdout:2768: done (6 left) 2024-09-21T09:40:51.546 INFO:tasks.rados.rados.0.smithi022.stdout:2770: done (5 left) 2024-09-21T09:40:51.546 INFO:tasks.rados.rados.0.smithi022.stdout:2773: done (4 left) 2024-09-21T09:40:51.546 INFO:tasks.rados.rados.0.smithi022.stdout:2774: setattr oid 5 current snap is 282 2024-09-21T09:40:51.546 INFO:tasks.rados.rados.0.smithi022.stdout:2771: finishing copy_from racing read to smithi02284960-9 2024-09-21T09:40:51.547 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 44 current snap is 282 2024-09-21T09:40:51.547 INFO:tasks.rados.rados.0.smithi022.stdout:2775: seq_num 886 ranges {0=442368} 2024-09-21T09:40:51.551 INFO:tasks.rados.rados.0.smithi022.stdout:2775: writing smithi02284960-44 from 0 to 442368 tid 1 2024-09-21T09:40:51.551 INFO:tasks.rados.rados.0.smithi022.stdout:2776: setattr oid 16 current snap is 282 2024-09-21T09:40:51.551 INFO:tasks.rados.rados.0.smithi022.stdout:2771: finishing copy_from to smithi02284960-9 2024-09-21T09:40:51.551 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 9 v 1650 (ObjNum 796 snap 249 seq_num 796) dirty exists 2024-09-21T09:40:51.552 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 5 v 1651 (ObjNum 672 snap 208 seq_num 672) dirty exists 2024-09-21T09:40:51.553 INFO:tasks.rados.rados.0.smithi022.stdout:2771: done (6 left) 2024-09-21T09:40:51.553 INFO:tasks.rados.rados.0.smithi022.stdout:2774: done (5 left) 2024-09-21T09:40:51.553 INFO:tasks.rados.rados.0.smithi022.stdout:2777: rollback oid 3 current snap is 282 2024-09-21T09:40:51.553 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 3 to 264 2024-09-21T09:40:51.553 INFO:tasks.rados.rados.0.smithi022.stdout:2778: read oid 24 snap -1 2024-09-21T09:40:51.553 INFO:tasks.rados.rados.0.smithi022.stdout:2778: expect (ObjNum 835 snap 269 seq_num 835) 2024-09-21T09:40:51.553 INFO:tasks.rados.rados.0.smithi022.stdout:2779: read oid 5 snap -1 2024-09-21T09:40:51.553 INFO:tasks.rados.rados.0.smithi022.stdout:2779: expect (ObjNum 672 snap 208 seq_num 672) 2024-09-21T09:40:51.553 INFO:tasks.rados.rados.0.smithi022.stdout:2780: snap_create 2024-09-21T09:40:51.556 INFO:tasks.rados.rados.0.smithi022.stdout:2777: finishing rollback tid 1 to smithi02284960-3 2024-09-21T09:40:51.556 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 3 v 613 (ObjNum 593 snap 0 seq_num 3570792600) dirty dne 2024-09-21T09:40:51.556 INFO:tasks.rados.rados.0.smithi022.stdout:2775: finishing write tid 1 to smithi02284960-44 2024-09-21T09:40:51.556 INFO:tasks.rados.rados.0.smithi022.stdout:2772: finishing write tid 1 to smithi02284960-42 2024-09-21T09:40:51.556 INFO:tasks.rados.rados.0.smithi022.stdout:2775: finishing write tid 2 to smithi02284960-44 2024-09-21T09:40:51.556 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 16 v 1508 (ObjNum 161 snap 0 seq_num 0) dirty exists 2024-09-21T09:40:51.556 INFO:tasks.rados.rados.0.smithi022.stdout:2772: finishing write tid 2 to smithi02284960-42 2024-09-21T09:40:51.558 INFO:tasks.rados.rados.0.smithi022.stdout:2772: finishing write tid 3 to smithi02284960-42 2024-09-21T09:40:51.558 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 42 v 1510 (ObjNum 884 snap 282 seq_num 884) dirty exists 2024-09-21T09:40:51.558 INFO:tasks.rados.rados.0.smithi022.stdout:2772: left oid 42 (ObjNum 884 snap 282 seq_num 884) 2024-09-21T09:40:51.559 INFO:tasks.rados.rados.0.smithi022.stdout:2775: finishing write tid 3 to smithi02284960-44 2024-09-21T09:40:51.559 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 44 v 1521 (ObjNum 886 snap 282 seq_num 886) dirty exists 2024-09-21T09:40:51.559 INFO:tasks.rados.rados.0.smithi022.stdout:2775: left oid 44 (ObjNum 886 snap 282 seq_num 886) 2024-09-21T09:40:51.563 INFO:tasks.rados.rados.0.smithi022.stdout:2779: expect (ObjNum 672 snap 208 seq_num 672) 2024-09-21T09:40:51.614 INFO:tasks.rados.rados.0.smithi022.stdout:2766: finishing rollback tid 1 to smithi02284960-30 2024-09-21T09:40:51.614 INFO:tasks.rados.rados.0.smithi022.stdout:2766: finishing rollback tid 2 to smithi02284960-30 2024-09-21T09:40:51.614 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 30 v 1307 (ObjNum 590 snap 176 seq_num 590) dirty exists 2024-09-21T09:40:51.615 INFO:tasks.rados.rados.0.smithi022.stdout:2778: expect (ObjNum 835 snap 269 seq_num 835) 2024-09-21T09:40:51.774 INFO:tasks.rados.rados.0.smithi022.stdout:2765: expect (ObjNum 842 snap 269 seq_num 842) 2024-09-21T09:40:51.941 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:51 smithi064 systemd[1]: Starting Ceph grafana.a for d65e801e-77fa-11ef-baf6-efdc52797490... 2024-09-21T09:40:51.941 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:51 smithi064 podman[114442]: 2024-09-21 09:40:51.749744569 +0000 UTC m=+0.017450761 image pull 5aad1d7cf004c80a4acb170e1c80fb85781b022f00f08c6bcd22608882b4c748 quay.io/ceph/grafana:10.4.8 2024-09-21T09:40:52.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:52 smithi022 ceph-mon[98615]: pgmap v544: 121 pgs: 121 active+clean; 243 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 16 MiB/s rd, 4.5 MiB/s wr, 16 op/s 2024-09-21T09:40:52.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:52 smithi022 ceph-mon[98615]: osdmap e627: 8 total, 8 up, 8 in 2024-09-21T09:40:52.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:52 smithi022 ceph-mon[96736]: pgmap v544: 121 pgs: 121 active+clean; 243 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 16 MiB/s rd, 4.5 MiB/s wr, 16 op/s 2024-09-21T09:40:52.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:52 smithi022 ceph-mon[96736]: osdmap e627: 8 total, 8 up, 8 in 2024-09-21T09:40:52.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:52 smithi064 ceph-mon[86709]: pgmap v544: 121 pgs: 121 active+clean; 243 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 16 MiB/s rd, 4.5 MiB/s wr, 16 op/s 2024-09-21T09:40:52.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:52 smithi064 ceph-mon[86709]: osdmap e627: 8 total, 8 up, 8 in 2024-09-21T09:40:52.484 INFO:teuthology.orchestra.run.smithi022.stdout:true 2024-09-21T09:40:53.074 INFO:teuthology.orchestra.run.smithi022.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T09:40:53.075 INFO:teuthology.orchestra.run.smithi022.stdout:alertmanager.a smithi022 *:9093,9094 running (7m) 37s ago 12m 22.0M - 0.27.0 11f11916f8cd caced80ba544 2024-09-21T09:40:53.075 INFO:teuthology.orchestra.run.smithi022.stdout:grafana.a smithi064 *:3000 running (7m) 18s ago 11m 83.5M - 9.4.7 954c08fa6188 527bb657d70d 2024-09-21T09:40:53.075 INFO:teuthology.orchestra.run.smithi022.stdout:mds.a smithi022 running (41s) 37s ago 13m 11.0M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 88fab662525b 2024-09-21T09:40:53.075 INFO:teuthology.orchestra.run.smithi022.stdout:mds.b smithi064 running (23s) 18s ago 13m 11.0M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 e520bfb5f294 2024-09-21T09:40:53.075 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.x smithi064 *:8443,9283,8765 running (6m) 18s ago 17m 456M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 61abc61fea8d 2024-09-21T09:40:53.075 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.y smithi022 *:8443,9283,8765 running (7m) 37s ago 18m 521M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 6c5bee78f09b 2024-09-21T09:40:53.075 INFO:teuthology.orchestra.run.smithi022.stdout:mon.a smithi022 running (6m) 37s ago 18m 69.2M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 0513395c784d 2024-09-21T09:40:53.075 INFO:teuthology.orchestra.run.smithi022.stdout:mon.b smithi064 running (5m) 18s ago 17m 55.9M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 9dab26b6a9e7 2024-09-21T09:40:53.075 INFO:teuthology.orchestra.run.smithi022.stdout:mon.c smithi022 running (5m) 37s ago 17m 55.9M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 c74b58b67bd6 2024-09-21T09:40:53.075 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.a smithi022 *:9100 running (7m) 37s ago 11m 11.5M - 1.7.0 72c9c2088986 593c6158f524 2024-09-21T09:40:53.075 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.b smithi064 *:9100 running (7m) 18s ago 11m 10.5M - 1.7.0 72c9c2088986 a533420b0139 2024-09-21T09:40:53.075 INFO:teuthology.orchestra.run.smithi022.stdout:osd.0 smithi022 running (4m) 37s ago 16m 213M 2481M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 40ce23a2b569 2024-09-21T09:40:53.075 INFO:teuthology.orchestra.run.smithi022.stdout:osd.1 smithi022 running (4m) 37s ago 16m 200M 2481M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 7cd0c18d224c 2024-09-21T09:40:53.076 INFO:teuthology.orchestra.run.smithi022.stdout:osd.2 smithi022 running (3m) 37s ago 15m 208M 2481M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 b8c944306698 2024-09-21T09:40:53.076 INFO:teuthology.orchestra.run.smithi022.stdout:osd.3 smithi022 running (3m) 37s ago 15m 104M 2481M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 c90b27f61bb6 2024-09-21T09:40:53.076 INFO:teuthology.orchestra.run.smithi022.stdout:osd.4 smithi064 running (2m) 18s ago 15m 137M 2480M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 936099dbca14 2024-09-21T09:40:53.076 INFO:teuthology.orchestra.run.smithi022.stdout:osd.5 smithi064 running (2m) 18s ago 14m 176M 2480M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 b3225b090dba 2024-09-21T09:40:53.076 INFO:teuthology.orchestra.run.smithi022.stdout:osd.6 smithi064 running (93s) 18s ago 14m 111M 2480M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 c6a9a09637e5 2024-09-21T09:40:53.076 INFO:teuthology.orchestra.run.smithi022.stdout:osd.7 smithi064 running (58s) 18s ago 13m 101M 2480M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 6f82655605e3 2024-09-21T09:40:53.076 INFO:teuthology.orchestra.run.smithi022.stdout:prometheus.a smithi064 *:9095 running (6m) 18s ago 12m 56.6M - 2.51.0 1d3b7f56885b 80629ffa2179 2024-09-21T09:40:53.307 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:53 smithi022 ceph-mon[98615]: osdmap e628: 8 total, 8 up, 8 in 2024-09-21T09:40:53.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:53 smithi022 ceph-mon[96736]: osdmap e628: 8 total, 8 up, 8 in 2024-09-21T09:40:53.377 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-mon[86709]: osdmap e628: 8 total, 8 up, 8 in 2024-09-21T09:40:53.377 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 podman[114442]: 2024-09-21 09:40:53.051224875 +0000 UTC m=+1.318931063 container create 0f60e823c29b66897b00eb3b410e0d32a44b30f76265da993fa3a4de35425a5e (image=quay.io/ceph/grafana:10.4.8, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a, maintainer=Grafana Labs ) 2024-09-21T09:40:53.433 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-09-21T09:40:53.433 INFO:teuthology.orchestra.run.smithi022.stdout: "mon": { 2024-09-21T09:40:53.433 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 3 2024-09-21T09:40:53.433 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-21T09:40:53.433 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr": { 2024-09-21T09:40:53.433 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 2 2024-09-21T09:40:53.433 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-21T09:40:53.433 INFO:teuthology.orchestra.run.smithi022.stdout: "osd": { 2024-09-21T09:40:53.434 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 8 2024-09-21T09:40:53.434 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-21T09:40:53.434 INFO:teuthology.orchestra.run.smithi022.stdout: "mds": { 2024-09-21T09:40:53.434 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 2 2024-09-21T09:40:53.434 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-21T09:40:53.434 INFO:teuthology.orchestra.run.smithi022.stdout: "overall": { 2024-09-21T09:40:53.434 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 15 2024-09-21T09:40:53.434 INFO:teuthology.orchestra.run.smithi022.stdout: } 2024-09-21T09:40:53.434 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-09-21T09:40:53.733 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-09-21T09:40:53.733 INFO:teuthology.orchestra.run.smithi022.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47", 2024-09-21T09:40:53.733 INFO:teuthology.orchestra.run.smithi022.stdout: "in_progress": true, 2024-09-21T09:40:53.733 INFO:teuthology.orchestra.run.smithi022.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T09:40:53.733 INFO:teuthology.orchestra.run.smithi022.stdout: "services_complete": [ 2024-09-21T09:40:53.734 INFO:teuthology.orchestra.run.smithi022.stdout: "mon", 2024-09-21T09:40:53.734 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr", 2024-09-21T09:40:53.734 INFO:teuthology.orchestra.run.smithi022.stdout: "osd", 2024-09-21T09:40:53.734 INFO:teuthology.orchestra.run.smithi022.stdout: "mds" 2024-09-21T09:40:53.734 INFO:teuthology.orchestra.run.smithi022.stdout: ], 2024-09-21T09:40:53.734 INFO:teuthology.orchestra.run.smithi022.stdout: "progress": "15/20 daemons upgraded", 2024-09-21T09:40:53.734 INFO:teuthology.orchestra.run.smithi022.stdout: "message": "Currently upgrading grafana daemons", 2024-09-21T09:40:53.734 INFO:teuthology.orchestra.run.smithi022.stdout: "is_paused": false 2024-09-21T09:40:53.734 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-09-21T09:40:53.828 INFO:tasks.rados.rados.0.smithi022.stdout:2765: done (8 left) 2024-09-21T09:40:53.828 INFO:tasks.rados.rados.0.smithi022.stdout:2766: done (7 left) 2024-09-21T09:40:53.828 INFO:tasks.rados.rados.0.smithi022.stdout:2772: done (6 left) 2024-09-21T09:40:53.828 INFO:tasks.rados.rados.0.smithi022.stdout:2775: done (5 left) 2024-09-21T09:40:53.828 INFO:tasks.rados.rados.0.smithi022.stdout:2776: done (4 left) 2024-09-21T09:40:53.828 INFO:tasks.rados.rados.0.smithi022.stdout:2777: done (3 left) 2024-09-21T09:40:53.828 INFO:tasks.rados.rados.0.smithi022.stdout:2778: done (2 left) 2024-09-21T09:40:53.828 INFO:tasks.rados.rados.0.smithi022.stdout:2779: done (1 left) 2024-09-21T09:40:53.828 INFO:tasks.rados.rados.0.smithi022.stdout:2780: done (0 left) 2024-09-21T09:40:53.829 INFO:tasks.rados.rados.0.smithi022.stdout:2781: rmattr oid 49 current snap is 283 2024-09-21T09:40:53.829 INFO:tasks.rados.rados.0.smithi022.stdout:2781: done (0 left) 2024-09-21T09:40:53.829 INFO:tasks.rados.rados.0.smithi022.stdout:2782: snap_remove snap 278 2024-09-21T09:40:53.832 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 podman[114442]: 2024-09-21 09:40:53.57914048 +0000 UTC m=+1.846846671 container init 0f60e823c29b66897b00eb3b410e0d32a44b30f76265da993fa3a4de35425a5e (image=quay.io/ceph/grafana:10.4.8, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a, maintainer=Grafana Labs ) 2024-09-21T09:40:53.832 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 podman[114442]: 2024-09-21 09:40:53.585279889 +0000 UTC m=+1.852986084 container start 0f60e823c29b66897b00eb3b410e0d32a44b30f76265da993fa3a4de35425a5e (image=quay.io/ceph/grafana:10.4.8, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a, maintainer=Grafana Labs ) 2024-09-21T09:40:53.832 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 bash[114442]: 0f60e823c29b66897b00eb3b410e0d32a44b30f76265da993fa3a4de35425a5e 2024-09-21T09:40:53.832 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 systemd[1]: Started Ceph grafana.a for d65e801e-77fa-11ef-baf6-efdc52797490. 2024-09-21T09:40:53.832 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=settings t=2024-09-21T09:40:53.714848099Z level=info msg="Starting Grafana" version=10.4.8 commit=8c384b71bde142a25ddc01ed5a183ca572b2b7f7 branch=v10.4.x compiled=2024-09-21T09:40:53Z 2024-09-21T09:40:53.832 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=settings t=2024-09-21T09:40:53.715073551Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2024-09-21T09:40:53.832 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=settings t=2024-09-21T09:40:53.715082755Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2024-09-21T09:40:53.832 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=settings t=2024-09-21T09:40:53.715086316Z level=info msg="Config overridden from command line" arg="default.paths.data=/var/lib/grafana" 2024-09-21T09:40:53.832 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=settings t=2024-09-21T09:40:53.715089544Z level=info msg="Config overridden from command line" arg="default.paths.logs=/var/log/grafana" 2024-09-21T09:40:53.833 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=settings t=2024-09-21T09:40:53.715092806Z level=info msg="Config overridden from command line" arg="default.paths.plugins=/var/lib/grafana/plugins" 2024-09-21T09:40:53.833 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=settings t=2024-09-21T09:40:53.715095713Z level=info msg="Config overridden from command line" arg="default.paths.provisioning=/etc/grafana/provisioning" 2024-09-21T09:40:53.833 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=settings t=2024-09-21T09:40:53.715098614Z level=info msg="Config overridden from command line" arg="default.log.mode=console" 2024-09-21T09:40:53.833 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=settings t=2024-09-21T09:40:53.715102882Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2024-09-21T09:40:53.833 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=settings t=2024-09-21T09:40:53.715106053Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2024-09-21T09:40:53.833 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=settings t=2024-09-21T09:40:53.715108745Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-09-21T09:40:53.833 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=settings t=2024-09-21T09:40:53.715111424Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-09-21T09:40:53.833 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=settings t=2024-09-21T09:40:53.715114502Z level=info msg=Target target=[all] 2024-09-21T09:40:53.833 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=settings t=2024-09-21T09:40:53.71512151Z level=info msg="Path Home" path=/usr/share/grafana 2024-09-21T09:40:53.834 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=settings t=2024-09-21T09:40:53.715124481Z level=info msg="Path Data" path=/var/lib/grafana 2024-09-21T09:40:53.834 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=settings t=2024-09-21T09:40:53.715127173Z level=info msg="Path Logs" path=/var/log/grafana 2024-09-21T09:40:53.834 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=settings t=2024-09-21T09:40:53.7151299Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2024-09-21T09:40:53.834 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=settings t=2024-09-21T09:40:53.715132736Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2024-09-21T09:40:53.834 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=settings t=2024-09-21T09:40:53.715135549Z level=info msg="App mode production" 2024-09-21T09:40:53.834 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=sqlstore t=2024-09-21T09:40:53.715380165Z level=info msg="Connecting to DB" dbtype=sqlite3 2024-09-21T09:40:53.834 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=sqlstore t=2024-09-21T09:40:53.715396035Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-09-21T09:40:53.834 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.715850936Z level=info msg="Starting DB migrations" 2024-09-21T09:40:53.835 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.731648796Z level=info msg="Executing migration" id="Add uid column to user" 2024-09-21T09:40:53.835 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.73773798Z level=info msg="Migration successfully executed" id="Add uid column to user" duration=6.085608ms 2024-09-21T09:40:53.835 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.738252967Z level=info msg="Executing migration" id="Update uid column values for users" 2024-09-21T09:40:53.835 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.738406766Z level=info msg="Migration successfully executed" id="Update uid column values for users" duration=152.97µs 2024-09-21T09:40:53.835 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.738834707Z level=info msg="Executing migration" id="Add unique index user_uid" 2024-09-21T09:40:53.835 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.739686295Z level=info msg="Migration successfully executed" id="Add unique index user_uid" duration=850.485µs 2024-09-21T09:40:53.835 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.741760966Z level=info msg="Executing migration" id="update login field with orgid to allow for multiple service accounts with same name across orgs" 2024-09-21T09:40:53.835 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.742112793Z level=info msg="Migration successfully executed" id="update login field with orgid to allow for multiple service accounts with same name across orgs" duration=353.299µs 2024-09-21T09:40:53.835 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.742956917Z level=info msg="Executing migration" id="Add playlist column created_at" 2024-09-21T09:40:53.835 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.748455685Z level=info msg="Migration successfully executed" id="Add playlist column created_at" duration=5.496648ms 2024-09-21T09:40:53.835 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.748834293Z level=info msg="Executing migration" id="Add playlist column updated_at" 2024-09-21T09:40:53.835 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.754898564Z level=info msg="Migration successfully executed" id="Add playlist column updated_at" duration=6.064058ms 2024-09-21T09:40:53.835 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.755356544Z level=info msg="Executing migration" id="Add column uid in team" 2024-09-21T09:40:53.836 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.760845863Z level=info msg="Migration successfully executed" id="Add column uid in team" duration=5.488466ms 2024-09-21T09:40:53.836 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.761207743Z level=info msg="Executing migration" id="Update uid column values in team" 2024-09-21T09:40:53.836 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.761335108Z level=info msg="Migration successfully executed" id="Update uid column values in team" duration=127.574µs 2024-09-21T09:40:53.836 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.761782008Z level=info msg="Executing migration" id="Add unique index team_org_id_uid" 2024-09-21T09:40:53.836 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.762643652Z level=info msg="Migration successfully executed" id="Add unique index team_org_id_uid" duration=860.668µs 2024-09-21T09:40:53.836 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.76315028Z level=info msg="Executing migration" id="add index user_auth_token.revoked_at" 2024-09-21T09:40:53.836 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.763988602Z level=info msg="Migration successfully executed" id="add index user_auth_token.revoked_at" duration=838.317µs 2024-09-21T09:40:53.836 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.764404723Z level=info msg="Executing migration" id="add result_fingerprint column to alert_instance" 2024-09-21T09:40:53.836 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.77107963Z level=info msg="Migration successfully executed" id="add result_fingerprint column to alert_instance" duration=6.674447ms 2024-09-21T09:40:53.836 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.771492187Z level=info msg="Executing migration" id="add last_applied column to alert_configuration_history" 2024-09-21T09:40:53.836 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.776849515Z level=info msg="Migration successfully executed" id="add last_applied column to alert_configuration_history" duration=5.354367ms 2024-09-21T09:40:53.836 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.777243473Z level=info msg="Executing migration" id="alter library_element model to mediumtext" 2024-09-21T09:40:53.836 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.777285472Z level=info msg="Migration successfully executed" id="alter library_element model to mediumtext" duration=41.893µs 2024-09-21T09:40:53.836 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.77770795Z level=info msg="Executing migration" id="add library_element folder uid" 2024-09-21T09:40:53.836 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.783005968Z level=info msg="Migration successfully executed" id="add library_element folder uid" duration=5.297559ms 2024-09-21T09:40:53.837 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.783374488Z level=info msg="Executing migration" id="populate library_element folder_uid" 2024-09-21T09:40:53.837 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.783660725Z level=info msg="Migration successfully executed" id="populate library_element folder_uid" duration=286.273µs 2024-09-21T09:40:53.837 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.786138741Z level=info msg="Executing migration" id="add index library_element org_id-folder_uid-name-kind" 2024-09-21T09:40:53.837 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.78702186Z level=info msg="Migration successfully executed" id="add index library_element org_id-folder_uid-name-kind" duration=882.758µs 2024-09-21T09:40:53.837 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.787434043Z level=info msg="Executing migration" id="permission kind migration" 2024-09-21T09:40:53.837 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.79294518Z level=info msg="Migration successfully executed" id="permission kind migration" duration=5.510079ms 2024-09-21T09:40:53.837 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.793306566Z level=info msg="Executing migration" id="permission attribute migration" 2024-09-21T09:40:53.837 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.798805523Z level=info msg="Migration successfully executed" id="permission attribute migration" duration=5.49855ms 2024-09-21T09:40:53.837 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.799186125Z level=info msg="Executing migration" id="permission identifier migration" 2024-09-21T09:40:53.837 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.804486994Z level=info msg="Migration successfully executed" id="permission identifier migration" duration=5.298822ms 2024-09-21T09:40:53.837 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.804874405Z level=info msg="Executing migration" id="add permission identifier index" 2024-09-21T09:40:53.837 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.805644074Z level=info msg="Migration successfully executed" id="add permission identifier index" duration=770.395µs 2024-09-21T09:40:53.837 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.806001991Z level=info msg="Executing migration" id="add permission action scope role_id index" 2024-09-21T09:40:53.837 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.806790437Z level=info msg="Migration successfully executed" id="add permission action scope role_id index" duration=788.089µs 2024-09-21T09:40:53.837 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.807155962Z level=info msg="Executing migration" id="remove permission role_id action scope index" 2024-09-21T09:40:53.837 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.807887966Z level=info msg="Migration successfully executed" id="remove permission role_id action scope index" duration=731.864µs 2024-09-21T09:40:53.838 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.809990409Z level=info msg="Executing migration" id="rbac disabled migrator" 2024-09-21T09:40:53.838 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.810009557Z level=info msg="Migration successfully executed" id="rbac disabled migrator" duration=20.263µs 2024-09-21T09:40:53.838 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.810433618Z level=info msg="Executing migration" id="drop index IDX_correlation_uid - v1" 2024-09-21T09:40:53.838 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.811149629Z level=info msg="Migration successfully executed" id="drop index IDX_correlation_uid - v1" duration=715.436µs 2024-09-21T09:40:53.838 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.811492315Z level=info msg="Executing migration" id="drop index IDX_correlation_source_uid - v1" 2024-09-21T09:40:53.838 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.812208828Z level=info msg="Migration successfully executed" id="drop index IDX_correlation_source_uid - v1" duration=716.419µs 2024-09-21T09:40:53.838 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.812546598Z level=info msg="Executing migration" id="Rename table correlation to correlation_tmp_qwerty - v1" 2024-09-21T09:40:53.838 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.830954313Z level=info msg="Migration successfully executed" id="Rename table correlation to correlation_tmp_qwerty - v1" duration=18.404936ms 2024-09-21T09:40:53.838 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.83157465Z level=info msg="Executing migration" id="create correlation v2" 2024-09-21T09:40:53.838 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.832740278Z level=info msg="Migration successfully executed" id="create correlation v2" duration=1.166293ms 2024-09-21T09:40:54.039 INFO:tasks.rados.rados.0.smithi022.stdout:2782: done (0 left) 2024-09-21T09:40:54.039 INFO:tasks.rados.rados.0.smithi022.stdout:2783: rollback oid 11 current snap is 283 2024-09-21T09:40:54.039 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 11 to 280 2024-09-21T09:40:54.039 INFO:tasks.rados.rados.0.smithi022.stdout:2784: delete oid 36 current snap is 283 2024-09-21T09:40:54.042 INFO:tasks.rados.rados.0.smithi022.stdout:2784: done (1 left) 2024-09-21T09:40:54.042 INFO:tasks.rados.rados.0.smithi022.stdout:2785: copy_from oid 1 from oid 9 current snap is 283 2024-09-21T09:40:54.042 INFO:tasks.rados.rados.0.smithi022.stdout:2786: snap_remove snap 273 2024-09-21T09:40:54.097 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:54 smithi064 ceph-mon[86709]: pgmap v547: 121 pgs: 121 active+clean; 244 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 346 KiB/s wr, 15 op/s 2024-09-21T09:40:54.097 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:54 smithi064 ceph-mon[86709]: from='client.44453 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:40:54.097 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:54 smithi064 ceph-mon[86709]: from='client.34530 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:40:54.097 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:54 smithi064 ceph-mon[86709]: from='client.34536 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:40:54.097 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:54 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/2000171309' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:40:54.097 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:54 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:54.097 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:54 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:54.098 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:54 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:40:54.098 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:54 smithi064 ceph-mon[86709]: osdmap e629: 8 total, 8 up, 8 in 2024-09-21T09:40:54.098 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.833220406Z level=info msg="Executing migration" id="create index IDX_correlation_uid - v2" 2024-09-21T09:40:54.098 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.834310533Z level=info msg="Migration successfully executed" id="create index IDX_correlation_uid - v2" duration=1.089796ms 2024-09-21T09:40:54.099 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.835993052Z level=info msg="Executing migration" id="create index IDX_correlation_source_uid - v2" 2024-09-21T09:40:54.099 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.837155951Z level=info msg="Migration successfully executed" id="create index IDX_correlation_source_uid - v2" duration=1.161991ms 2024-09-21T09:40:54.099 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.837661841Z level=info msg="Executing migration" id="create index IDX_correlation_org_id - v2" 2024-09-21T09:40:54.099 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.83870006Z level=info msg="Migration successfully executed" id="create index IDX_correlation_org_id - v2" duration=1.038403ms 2024-09-21T09:40:54.099 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.840410918Z level=info msg="Executing migration" id="copy correlation v1 to v2" 2024-09-21T09:40:54.099 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.840594367Z level=info msg="Migration successfully executed" id="copy correlation v1 to v2" duration=183.36µs 2024-09-21T09:40:54.099 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.843302016Z level=info msg="Executing migration" id="drop correlation_tmp_qwerty" 2024-09-21T09:40:54.099 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.843948443Z level=info msg="Migration successfully executed" id="drop correlation_tmp_qwerty" duration=646.241µs 2024-09-21T09:40:54.099 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.844306681Z level=info msg="Executing migration" id="add provisioning column" 2024-09-21T09:40:54.099 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.850009033Z level=info msg="Migration successfully executed" id="add provisioning column" duration=5.702018ms 2024-09-21T09:40:54.099 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.85039977Z level=info msg="Executing migration" id="backfill empty share column fields with default of public" 2024-09-21T09:40:54.099 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.850546158Z level=info msg="Migration successfully executed" id="backfill empty share column fields with default of public" duration=145.487µs 2024-09-21T09:40:54.099 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.851263377Z level=info msg="Executing migration" id="migrate contents column to mediumblob for MySQL" 2024-09-21T09:40:54.100 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.851303035Z level=info msg="Migration successfully executed" id="migrate contents column to mediumblob for MySQL" duration=39.57µs 2024-09-21T09:40:54.100 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.85212624Z level=info msg="Executing migration" id="add origin column to seed_assignment" 2024-09-21T09:40:54.100 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.857721389Z level=info msg="Migration successfully executed" id="add origin column to seed_assignment" duration=5.594208ms 2024-09-21T09:40:54.100 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.858087187Z level=info msg="Executing migration" id="add origin to plugin seed_assignment" 2024-09-21T09:40:54.100 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.858289198Z level=info msg="Migration successfully executed" id="add origin to plugin seed_assignment" duration=199.67µs 2024-09-21T09:40:54.100 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.858681374Z level=info msg="Executing migration" id="prevent seeding OnCall access" 2024-09-21T09:40:54.100 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.858836269Z level=info msg="Migration successfully executed" id="prevent seeding OnCall access" duration=154.957µs 2024-09-21T09:40:54.100 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.859628473Z level=info msg="Executing migration" id="managed folder permissions library panel actions migration" 2024-09-21T09:40:54.100 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.860564825Z level=info msg="Migration successfully executed" id="managed folder permissions library panel actions migration" duration=935.685µs 2024-09-21T09:40:54.100 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.860930985Z level=info msg="Executing migration" id="Remove unique index for folder.title and folder.parent_uid" 2024-09-21T09:40:54.100 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.861646416Z level=info msg="Migration successfully executed" id="Remove unique index for folder.title and folder.parent_uid" duration=714.851µs 2024-09-21T09:40:54.100 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.862004219Z level=info msg="Executing migration" id="Add unique index for title, parent_uid, and org_id" 2024-09-21T09:40:54.100 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.862746119Z level=info msg="Migration successfully executed" id="Add unique index for title, parent_uid, and org_id" duration=740.699µs 2024-09-21T09:40:54.101 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.863094173Z level=info msg="Executing migration" id="Sync dashboard and folder table" 2024-09-21T09:40:54.101 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.863499754Z level=info msg="Migration successfully executed" id="Sync dashboard and folder table" duration=405.355µs 2024-09-21T09:40:54.101 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.863952422Z level=info msg="Executing migration" id="Remove ghost folders from the folder table" 2024-09-21T09:40:54.101 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.864148758Z level=info msg="Migration successfully executed" id="Remove ghost folders from the folder table" duration=196.247µs 2024-09-21T09:40:54.101 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.864829101Z level=info msg="Executing migration" id="Remove unique index UQE_folder_uid_org_id" 2024-09-21T09:40:54.101 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.865530423Z level=info msg="Migration successfully executed" id="Remove unique index UQE_folder_uid_org_id" duration=699.728µs 2024-09-21T09:40:54.101 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.865914799Z level=info msg="Executing migration" id="Add unique index UQE_folder_org_id_uid" 2024-09-21T09:40:54.101 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.866667225Z level=info msg="Migration successfully executed" id="Add unique index UQE_folder_org_id_uid" duration=752.11µs 2024-09-21T09:40:54.101 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.867018047Z level=info msg="Executing migration" id="Remove unique index UQE_folder_title_parent_uid_org_id" 2024-09-21T09:40:54.101 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.867718504Z level=info msg="Migration successfully executed" id="Remove unique index UQE_folder_title_parent_uid_org_id" duration=700.216µs 2024-09-21T09:40:54.101 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.868069884Z level=info msg="Executing migration" id="Add unique index UQE_folder_org_id_parent_uid_title" 2024-09-21T09:40:54.101 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.868825746Z level=info msg="Migration successfully executed" id="Add unique index UQE_folder_org_id_parent_uid_title" duration=755.452µs 2024-09-21T09:40:54.101 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.869172923Z level=info msg="Executing migration" id="Remove index IDX_folder_parent_uid_org_id" 2024-09-21T09:40:54.101 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.8698918Z level=info msg="Migration successfully executed" id="Remove index IDX_folder_parent_uid_org_id" duration=718.754µs 2024-09-21T09:40:54.102 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.87024914Z level=info msg="Executing migration" id="create anon_device table" 2024-09-21T09:40:54.102 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.870856087Z level=info msg="Migration successfully executed" id="create anon_device table" duration=604.716µs 2024-09-21T09:40:54.102 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.871210737Z level=info msg="Executing migration" id="add unique index anon_device.device_id" 2024-09-21T09:40:54.102 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.871924593Z level=info msg="Migration successfully executed" id="add unique index anon_device.device_id" duration=713.517µs 2024-09-21T09:40:54.102 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.872274221Z level=info msg="Executing migration" id="add index anon_device.updated_at" 2024-09-21T09:40:54.102 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.872993067Z level=info msg="Migration successfully executed" id="add index anon_device.updated_at" duration=718.552µs 2024-09-21T09:40:54.102 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.873345895Z level=info msg="Executing migration" id="create signing_key table" 2024-09-21T09:40:54.102 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.874025066Z level=info msg="Migration successfully executed" id="create signing_key table" duration=678.903µs 2024-09-21T09:40:54.102 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.876264514Z level=info msg="Executing migration" id="add unique index signing_key.key_id" 2024-09-21T09:40:54.102 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.876961093Z level=info msg="Migration successfully executed" id="add unique index signing_key.key_id" duration=696.253µs 2024-09-21T09:40:54.102 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.877316962Z level=info msg="Executing migration" id="set legacy alert migration status in kvstore" 2024-09-21T09:40:54.102 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.877979112Z level=info msg="Migration successfully executed" id="set legacy alert migration status in kvstore" duration=662.12µs 2024-09-21T09:40:54.102 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.880128228Z level=info msg="Executing migration" id="migrate record of created folders during legacy migration to kvstore" 2024-09-21T09:40:54.102 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.880594321Z level=info msg="Migration successfully executed" id="migrate record of created folders during legacy migration to kvstore" duration=466.613µs 2024-09-21T09:40:54.103 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.882921369Z level=info msg="Executing migration" id="Add folder_uid for dashboard" 2024-09-21T09:40:54.103 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.888860185Z level=info msg="Migration successfully executed" id="Add folder_uid for dashboard" duration=5.938042ms 2024-09-21T09:40:54.103 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.889241466Z level=info msg="Executing migration" id="Populate dashboard folder_uid column" 2024-09-21T09:40:54.103 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.891138795Z level=info msg="Migration successfully executed" id="Populate dashboard folder_uid column" duration=1.897933ms 2024-09-21T09:40:54.103 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.891703646Z level=info msg="Executing migration" id="Add unique index for dashboard_org_id_folder_uid_title" 2024-09-21T09:40:54.103 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.891715424Z level=info msg="Migration successfully executed" id="Add unique index for dashboard_org_id_folder_uid_title" duration=12.508µs 2024-09-21T09:40:54.103 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.89440346Z level=info msg="Executing migration" id="Delete unique index for dashboard_org_id_folder_id_title" 2024-09-21T09:40:54.103 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.895147562Z level=info msg="Migration successfully executed" id="Delete unique index for dashboard_org_id_folder_id_title" duration=744.059µs 2024-09-21T09:40:54.103 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.895484289Z level=info msg="Executing migration" id="Delete unique index for dashboard_org_id_folder_uid_title" 2024-09-21T09:40:54.103 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.895494748Z level=info msg="Migration successfully executed" id="Delete unique index for dashboard_org_id_folder_uid_title" duration=9.952µs 2024-09-21T09:40:54.103 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.89836945Z level=info msg="Executing migration" id="Add unique index for dashboard_org_id_folder_uid_title_is_folder" 2024-09-21T09:40:54.104 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.899223634Z level=info msg="Migration successfully executed" id="Add unique index for dashboard_org_id_folder_uid_title_is_folder" duration=854.809µs 2024-09-21T09:40:54.104 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.899597483Z level=info msg="Executing migration" id="Restore index for dashboard_org_id_folder_id_title" 2024-09-21T09:40:54.104 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.900422599Z level=info msg="Migration successfully executed" id="Restore index for dashboard_org_id_folder_id_title" duration=824.704µs 2024-09-21T09:40:54.105 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.902491713Z level=info msg="Executing migration" id="create sso_setting table" 2024-09-21T09:40:54.105 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.903321708Z level=info msg="Migration successfully executed" id="create sso_setting table" duration=827.433µs 2024-09-21T09:40:54.105 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.90527146Z level=info msg="Executing migration" id="copy kvstore migration status to each org" 2024-09-21T09:40:54.105 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.905937515Z level=info msg="Migration successfully executed" id="copy kvstore migration status to each org" duration=665.912µs 2024-09-21T09:40:54.105 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.908073315Z level=info msg="Executing migration" id="add back entry for orgid=0 migrated status" 2024-09-21T09:40:54.105 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.908468448Z level=info msg="Migration successfully executed" id="add back entry for orgid=0 migrated status" duration=394.909µs 2024-09-21T09:40:54.105 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.908818884Z level=info msg="Executing migration" id="alter kv_store.value to longtext" 2024-09-21T09:40:54.105 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.908858101Z level=info msg="Migration successfully executed" id="alter kv_store.value to longtext" duration=39.597µs 2024-09-21T09:40:54.105 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.911715658Z level=info msg="Executing migration" id="add notification_settings column to alert_rule table" 2024-09-21T09:40:54.105 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.917275824Z level=info msg="Migration successfully executed" id="add notification_settings column to alert_rule table" duration=5.558475ms 2024-09-21T09:40:54.105 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.917638497Z level=info msg="Executing migration" id="add notification_settings column to alert_rule_version table" 2024-09-21T09:40:54.105 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.923594597Z level=info msg="Migration successfully executed" id="add notification_settings column to alert_rule_version table" duration=5.955222ms 2024-09-21T09:40:54.105 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.923983636Z level=info msg="Executing migration" id="removing scope from alert.instances:read action migration" 2024-09-21T09:40:54.106 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.924216301Z level=info msg="Migration successfully executed" id="removing scope from alert.instances:read action migration" duration=232.57µs 2024-09-21T09:40:54.106 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=migrator t=2024-09-21T09:40:53.924596946Z level=info msg="migrations completed" performed=69 skipped=482 duration=193.024841ms 2024-09-21T09:40:54.106 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=sqlstore t=2024-09-21T09:40:53.925266245Z level=info msg="Created default organization" 2024-09-21T09:40:54.106 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=secrets t=2024-09-21T09:40:53.927298274Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2024-09-21T09:40:54.106 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=plugin.store t=2024-09-21T09:40:53.94182898Z level=info msg="Loading plugins..." 2024-09-21T09:40:54.106 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=local.finder t=2024-09-21T09:40:53.992879192Z level=warn msg="Skipping finding plugins as directory does not exist" path=/usr/share/grafana/plugins-bundled 2024-09-21T09:40:54.106 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=plugin.store t=2024-09-21T09:40:53.9929447Z level=info msg="Plugins loaded" count=55 duration=51.115735ms 2024-09-21T09:40:54.106 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=query_data t=2024-09-21T09:40:53.997188389Z level=info msg="Query Service initialization" 2024-09-21T09:40:54.106 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:54 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=live.push_http t=2024-09-21T09:40:53.999985608Z level=info msg="Live Push Gateway initialization" 2024-09-21T09:40:54.106 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:54 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=ngalert.migration t=2024-09-21T09:40:54.001104005Z level=info msg=Starting 2024-09-21T09:40:54.106 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:54 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=ngalert.state.manager t=2024-09-21T09:40:54.008000877Z level=info msg="Running in alternative execution of Error/NoData mode" 2024-09-21T09:40:54.106 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:54 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=infra.usagestats.collector t=2024-09-21T09:40:54.010271223Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2024-09-21T09:40:54.106 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:54 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=provisioning.datasources t=2024-09-21T09:40:54.013792107Z level=info msg="deleted datasource based on configuration" name=Dashboard1 2024-09-21T09:40:54.106 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:54 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=provisioning.datasources t=2024-09-21T09:40:54.01418643Z level=info msg="inserting datasource from configuration" name=Dashboard1 uid=P43CA22E17D0F9596 2024-09-21T09:40:54.107 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:54 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=provisioning.alerting t=2024-09-21T09:40:54.051674402Z level=info msg="starting to provision alerting" 2024-09-21T09:40:54.107 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:54 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=provisioning.alerting t=2024-09-21T09:40:54.051954732Z level=info msg="finished to provision alerting" 2024-09-21T09:40:54.107 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:54 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=grafanaStorageLogger t=2024-09-21T09:40:54.053065425Z level=info msg="Storage starting" 2024-09-21T09:40:54.107 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:54 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=ngalert.state.manager t=2024-09-21T09:40:54.053501837Z level=info msg="Warming state cache for startup" 2024-09-21T09:40:54.107 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:54 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=ngalert.state.manager t=2024-09-21T09:40:54.054606403Z level=info msg="State cache has been initialized" states=0 duration=1.102539ms 2024-09-21T09:40:54.107 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:54 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=ngalert.scheduler t=2024-09-21T09:40:54.054794234Z level=info msg="Starting scheduler" tickInterval=10s maxAttempts=1 2024-09-21T09:40:54.107 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:54 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=http.server t=2024-09-21T09:40:54.054962807Z level=info msg="HTTP Server TLS settings" MinTLSVersion=TLS1.2 configuredciphers=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_256_CBC_SHA 2024-09-21T09:40:54.107 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:54 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=ngalert.multiorg.alertmanager t=2024-09-21T09:40:54.05532504Z level=info msg="Starting MultiOrg Alertmanager" 2024-09-21T09:40:54.107 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:54 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=http.server t=2024-09-21T09:40:54.055591469Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2024-09-21T09:40:54.107 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:54 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=ticker t=2024-09-21T09:40:54.054975216Z level=info msg=starting first_tick=2024-09-21T09:41:00Z 2024-09-21T09:40:54.107 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:54 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=provisioning.dashboard t=2024-09-21T09:40:54.096348293Z level=info msg="starting to provision dashboards" 2024-09-21T09:40:54.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:54 smithi022 ceph-mon[96736]: pgmap v547: 121 pgs: 121 active+clean; 244 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 346 KiB/s wr, 15 op/s 2024-09-21T09:40:54.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:54 smithi022 ceph-mon[96736]: from='client.44453 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:40:54.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:54 smithi022 ceph-mon[96736]: from='client.34530 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:40:54.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:54 smithi022 ceph-mon[96736]: from='client.34536 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:40:54.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:54 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/2000171309' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:40:54.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:54 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:54.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:54 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:54.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:54 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:40:54.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:54 smithi022 ceph-mon[96736]: osdmap e629: 8 total, 8 up, 8 in 2024-09-21T09:40:54.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:54 smithi022 ceph-mon[98615]: pgmap v547: 121 pgs: 121 active+clean; 244 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 346 KiB/s wr, 15 op/s 2024-09-21T09:40:54.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:54 smithi022 ceph-mon[98615]: from='client.44453 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:40:54.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:54 smithi022 ceph-mon[98615]: from='client.34530 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:40:54.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:54 smithi022 ceph-mon[98615]: from='client.34536 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:40:54.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:54 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/2000171309' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:40:54.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:54 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:54.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:54 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:54.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:54 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:40:54.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:54 smithi022 ceph-mon[98615]: osdmap e629: 8 total, 8 up, 8 in 2024-09-21T09:40:54.417 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:54 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=grafana.update.checker t=2024-09-21T09:40:54.135368108Z level=info msg="Update check succeeded" duration=79.228783ms 2024-09-21T09:40:54.417 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:54 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=plugins.update.checker t=2024-09-21T09:40:54.14238023Z level=info msg="Update check succeeded" duration=86.632921ms 2024-09-21T09:40:54.417 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:54 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=provisioning.dashboard t=2024-09-21T09:40:54.274877029Z level=info msg="finished to provision dashboards" 2024-09-21T09:40:54.691 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:54 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=grafana-apiserver t=2024-09-21T09:40:54.659221812Z level=info msg="Adding GroupVersion playlist.grafana.app v0alpha1 to ResourceManager" 2024-09-21T09:40:54.691 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:40:54 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=grafana-apiserver t=2024-09-21T09:40:54.660487859Z level=info msg="Adding GroupVersion featuretoggle.grafana.app v0alpha1 to ResourceManager" 2024-09-21T09:40:55.042 INFO:tasks.rados.rados.0.smithi022.stdout:2786: done (2 left) 2024-09-21T09:40:55.042 INFO:tasks.rados.rados.0.smithi022.stdout:2787: setattr oid 38 current snap is 283 2024-09-21T09:40:55.042 INFO:tasks.rados.rados.0.smithi022.stdout:2783: finishing rollback tid 0 to smithi02284960-11 2024-09-21T09:40:55.042 INFO:tasks.rados.rados.0.smithi022.stdout:2783: finishing rollback tid 1 to smithi02284960-11 2024-09-21T09:40:55.042 INFO:tasks.rados.rados.0.smithi022.stdout:2783: finishing rollback tid 2 to smithi02284960-11 2024-09-21T09:40:55.042 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 11 v 1524 (ObjNum 749 snap 227 seq_num 749) dirty exists 2024-09-21T09:40:55.042 INFO:tasks.rados.rados.0.smithi022.stdout:2785: finishing copy_from racing read to smithi02284960-1 2024-09-21T09:40:55.042 INFO:tasks.rados.rados.0.smithi022.stdout:2785: finishing copy_from to smithi02284960-1 2024-09-21T09:40:55.042 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 1 v 1650 (ObjNum 796 snap 249 seq_num 796) dirty exists 2024-09-21T09:40:55.043 INFO:tasks.rados.rados.0.smithi022.stdout:2783: done (2 left) 2024-09-21T09:40:55.043 INFO:tasks.rados.rados.0.smithi022.stdout:2785: done (1 left) 2024-09-21T09:40:55.043 INFO:tasks.rados.rados.0.smithi022.stdout:2788: snap_create 2024-09-21T09:40:55.050 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 38 v 1430 (ObjNum 839 snap 269 seq_num 839) dirty exists 2024-09-21T09:40:55.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:55 smithi064 ceph-mon[86709]: from='client.34548 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:40:55.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:55 smithi064 ceph-mon[86709]: osdmap e630: 8 total, 8 up, 8 in 2024-09-21T09:40:55.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:55 smithi022 ceph-mon[98615]: from='client.34548 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:40:55.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:55 smithi022 ceph-mon[98615]: osdmap e630: 8 total, 8 up, 8 in 2024-09-21T09:40:55.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:55 smithi022 ceph-mon[96736]: from='client.34548 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:40:55.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:55 smithi022 ceph-mon[96736]: osdmap e630: 8 total, 8 up, 8 in 2024-09-21T09:40:56.043 INFO:tasks.rados.rados.0.smithi022.stdout:2787: done (1 left) 2024-09-21T09:40:56.043 INFO:tasks.rados.rados.0.smithi022.stdout:2788: done (0 left) 2024-09-21T09:40:56.043 INFO:tasks.rados.rados.0.smithi022.stdout:2789: snap_remove snap 276 2024-09-21T09:40:56.057 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:40:55 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:40:55] "GET /metrics HTTP/1.1" 200 36357 "" "Prometheus/2.51.0" 2024-09-21T09:40:56.357 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:56 smithi064 ceph-mon[86709]: pgmap v549: 121 pgs: 121 active+clean; 250 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 16 MiB/s rd, 1.4 MiB/s wr, 18 op/s 2024-09-21T09:40:56.358 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:56 smithi064 ceph-mon[86709]: osdmap e631: 8 total, 8 up, 8 in 2024-09-21T09:40:56.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:56 smithi022 ceph-mon[96736]: pgmap v549: 121 pgs: 121 active+clean; 250 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 16 MiB/s rd, 1.4 MiB/s wr, 18 op/s 2024-09-21T09:40:56.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:56 smithi022 ceph-mon[96736]: osdmap e631: 8 total, 8 up, 8 in 2024-09-21T09:40:56.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:56 smithi022 ceph-mon[98615]: pgmap v549: 121 pgs: 121 active+clean; 250 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 16 MiB/s rd, 1.4 MiB/s wr, 18 op/s 2024-09-21T09:40:56.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:56 smithi022 ceph-mon[98615]: osdmap e631: 8 total, 8 up, 8 in 2024-09-21T09:40:57.047 INFO:tasks.rados.rados.0.smithi022.stdout:2789: done (0 left) 2024-09-21T09:40:57.047 INFO:tasks.rados.rados.0.smithi022.stdout:2790: delete oid 29 current snap is 284 2024-09-21T09:40:57.054 INFO:tasks.rados.rados.0.smithi022.stdout:2790: done (0 left) 2024-09-21T09:40:57.055 INFO:tasks.rados.rados.0.smithi022.stdout:2791: copy_from oid 14 from oid 22 current snap is 284 2024-09-21T09:40:57.055 INFO:tasks.rados.rados.0.smithi022.stdout:2792: delete oid 40 current snap is 284 2024-09-21T09:40:57.059 INFO:tasks.rados.rados.0.smithi022.stdout:2791: finishing copy_from to smithi02284960-14 2024-09-21T09:40:57.059 INFO:tasks.rados.rados.0.smithi022.stdout:2791: got expected ENOENT (src dne) 2024-09-21T09:40:57.059 INFO:tasks.rados.rados.0.smithi022.stdout:2791: finishing copy_from racing read to smithi02284960-14 2024-09-21T09:40:57.059 INFO:tasks.rados.rados.0.smithi022.stdout:2791: done (1 left) 2024-09-21T09:40:57.059 INFO:tasks.rados.rados.0.smithi022.stdout:2792: done (0 left) 2024-09-21T09:40:57.059 INFO:tasks.rados.rados.0.smithi022.stdout:2793: delete oid 19 current snap is 284 2024-09-21T09:40:57.066 INFO:tasks.rados.rados.0.smithi022.stdout:2793: done (0 left) 2024-09-21T09:40:57.066 INFO:tasks.rados.rados.0.smithi022.stdout:2794: rmattr oid 35 current snap is 284 2024-09-21T09:40:57.066 INFO:tasks.rados.rados.0.smithi022.stdout:2794: done (0 left) 2024-09-21T09:40:57.066 INFO:tasks.rados.rados.0.smithi022.stdout:2795: read oid 38 snap 264 2024-09-21T09:40:57.066 INFO:tasks.rados.rados.0.smithi022.stdout:2795: expect deleted 2024-09-21T09:40:57.066 INFO:tasks.rados.rados.0.smithi022.stdout:2796: copy_from oid 36 from oid 40 current snap is 284 2024-09-21T09:40:57.066 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 4 current snap is 284 2024-09-21T09:40:57.066 INFO:tasks.rados.rados.0.smithi022.stdout:2797: seq_num 891 ranges {18464768=532480} 2024-09-21T09:40:57.169 INFO:tasks.rados.rados.0.smithi022.stdout:2797: writing smithi02284960-4 from 18464768 to 18997248 tid 1 2024-09-21T09:40:57.170 INFO:tasks.rados.rados.0.smithi022.stdout:2798: read oid 32 snap -1 2024-09-21T09:40:57.170 INFO:tasks.rados.rados.0.smithi022.stdout:2798: expect (ObjNum 868 snap 280 seq_num 868) 2024-09-21T09:40:57.170 INFO:tasks.rados.rados.0.smithi022.stdout:2799: snap_remove snap 280 2024-09-21T09:40:58.066 INFO:tasks.rados.rados.0.smithi022.stdout:2799: done (4 left) 2024-09-21T09:40:58.066 INFO:tasks.rados.rados.0.smithi022.stdout:2800: copy_from oid 41 from oid 9 current snap is 284 2024-09-21T09:40:58.066 INFO:tasks.rados.rados.0.smithi022.stdout:2796: finishing copy_from to smithi02284960-36 2024-09-21T09:40:58.067 INFO:tasks.rados.rados.0.smithi022.stdout:2796: got expected ENOENT (src dne) 2024-09-21T09:40:58.067 INFO:tasks.rados.rados.0.smithi022.stdout:2796: finishing copy_from racing read to smithi02284960-36 2024-09-21T09:40:58.067 INFO:tasks.rados.rados.0.smithi022.stdout:2798: expect (ObjNum 868 snap 280 seq_num 868) 2024-09-21T09:40:58.070 INFO:tasks.rados.rados.0.smithi022.stdout:2797: finishing write tid 1 to smithi02284960-4 2024-09-21T09:40:58.071 INFO:tasks.rados.rados.0.smithi022.stdout:2797: finishing write tid 2 to smithi02284960-4 2024-09-21T09:40:58.071 INFO:tasks.rados.rados.0.smithi022.stdout:2797: finishing write tid 3 to smithi02284960-4 2024-09-21T09:40:58.071 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 4 v 1653 (ObjNum 891 snap 284 seq_num 891) dirty exists 2024-09-21T09:40:58.071 INFO:tasks.rados.rados.0.smithi022.stdout:2797: left oid 4 (ObjNum 891 snap 284 seq_num 891) 2024-09-21T09:40:58.071 INFO:tasks.rados.rados.0.smithi022.stdout:2795: done (4 left) 2024-09-21T09:40:58.071 INFO:tasks.rados.rados.0.smithi022.stdout:2796: done (3 left) 2024-09-21T09:40:58.071 INFO:tasks.rados.rados.0.smithi022.stdout:2797: done (2 left) 2024-09-21T09:40:58.071 INFO:tasks.rados.rados.0.smithi022.stdout:2798: done (1 left) 2024-09-21T09:40:58.071 INFO:tasks.rados.rados.0.smithi022.stdout:2801: read oid 39 snap 245 2024-09-21T09:40:58.071 INFO:tasks.rados.rados.0.smithi022.stdout:2801: expect deleted 2024-09-21T09:40:58.071 INFO:tasks.rados.rados.0.smithi022.stdout:2802: delete oid 24 current snap is 284 2024-09-21T09:40:58.081 INFO:tasks.rados.rados.0.smithi022.stdout:2801: done (2 left) 2024-09-21T09:40:58.082 INFO:tasks.rados.rados.0.smithi022.stdout:2802: done (1 left) 2024-09-21T09:40:58.082 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 47 current snap is 284 2024-09-21T09:40:58.082 INFO:tasks.rados.rados.0.smithi022.stdout:2803: seq_num 893 ranges {1679360=565248} 2024-09-21T09:40:58.094 INFO:tasks.rados.rados.0.smithi022.stdout:2803: writing smithi02284960-47 from 1679360 to 2244608 tid 1 2024-09-21T09:40:58.094 INFO:tasks.rados.rados.0.smithi022.stdout:2804: delete oid 26 current snap is 284 2024-09-21T09:40:58.096 INFO:tasks.rados.rados.0.smithi022.stdout:2804: done (2 left) 2024-09-21T09:40:58.097 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 28 current snap is 284 2024-09-21T09:40:58.097 INFO:tasks.rados.rados.0.smithi022.stdout:2805: seq_num 895 ranges {4489216=598016} 2024-09-21T09:40:58.125 INFO:tasks.rados.rados.0.smithi022.stdout:2805: writing smithi02284960-28 from 4489216 to 5087232 tid 1 2024-09-21T09:40:58.125 INFO:tasks.rados.rados.0.smithi022.stdout:2806: delete oid 25 current snap is 284 2024-09-21T09:40:58.125 INFO:tasks.rados.rados.0.smithi022.stdout:2803: finishing write tid 1 to smithi02284960-47 2024-09-21T09:40:58.125 INFO:tasks.rados.rados.0.smithi022.stdout:2803: finishing write tid 2 to smithi02284960-47 2024-09-21T09:40:58.125 INFO:tasks.rados.rados.0.smithi022.stdout:2803: finishing write tid 3 to smithi02284960-47 2024-09-21T09:40:58.125 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 47 v 1652 (ObjNum 893 snap 284 seq_num 893) dirty exists 2024-09-21T09:40:58.125 INFO:tasks.rados.rados.0.smithi022.stdout:2803: left oid 47 (ObjNum 893 snap 284 seq_num 893) 2024-09-21T09:40:58.125 INFO:tasks.rados.rados.0.smithi022.stdout:2800: finishing copy_from racing read to smithi02284960-41 2024-09-21T09:40:58.125 INFO:tasks.rados.rados.0.smithi022.stdout:2800: finishing copy_from to smithi02284960-41 2024-09-21T09:40:58.126 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 41 v 1650 (ObjNum 796 snap 249 seq_num 796) dirty exists 2024-09-21T09:40:58.127 INFO:tasks.rados.rados.0.smithi022.stdout:2800: done (3 left) 2024-09-21T09:40:58.127 INFO:tasks.rados.rados.0.smithi022.stdout:2803: done (2 left) 2024-09-21T09:40:58.127 INFO:tasks.rados.rados.0.smithi022.stdout:2806: done (1 left) 2024-09-21T09:40:58.127 INFO:tasks.rados.rados.0.smithi022.stdout:2807: snap_remove snap 274 2024-09-21T09:40:58.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:58 smithi022 ceph-mon[98615]: pgmap v552: 121 pgs: 121 active+clean; 264 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 20 MiB/s rd, 4.2 MiB/s wr, 21 op/s 2024-09-21T09:40:58.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:58 smithi022 ceph-mon[98615]: osdmap e632: 8 total, 8 up, 8 in 2024-09-21T09:40:58.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:58 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:58.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:58 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:58.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:58 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:40:58.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:58 smithi022 ceph-mon[96736]: pgmap v552: 121 pgs: 121 active+clean; 264 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 20 MiB/s rd, 4.2 MiB/s wr, 21 op/s 2024-09-21T09:40:58.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:58 smithi022 ceph-mon[96736]: osdmap e632: 8 total, 8 up, 8 in 2024-09-21T09:40:58.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:58 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:58.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:58 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:58.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:58 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:40:58.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:58 smithi064 ceph-mon[86709]: pgmap v552: 121 pgs: 121 active+clean; 264 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 20 MiB/s rd, 4.2 MiB/s wr, 21 op/s 2024-09-21T09:40:58.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:58 smithi064 ceph-mon[86709]: osdmap e632: 8 total, 8 up, 8 in 2024-09-21T09:40:58.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:58 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:58.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:58 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:58.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:58 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:40:59.040 INFO:tasks.rados.rados.0.smithi022.stdout:2805: finishing write tid 1 to smithi02284960-28 2024-09-21T09:40:59.040 INFO:tasks.rados.rados.0.smithi022.stdout:2807: done (1 left) 2024-09-21T09:40:59.040 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 8 current snap is 284 2024-09-21T09:40:59.040 INFO:tasks.rados.rados.0.smithi022.stdout:2805: finishing write tid 2 to smithi02284960-28 2024-09-21T09:40:59.040 INFO:tasks.rados.rados.0.smithi022.stdout:2805: finishing write tid 3 to smithi02284960-28 2024-09-21T09:40:59.040 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 28 v 1408 (ObjNum 895 snap 284 seq_num 895) dirty exists 2024-09-21T09:40:59.041 INFO:tasks.rados.rados.0.smithi022.stdout:2805: left oid 28 (ObjNum 895 snap 284 seq_num 895) 2024-09-21T09:40:59.041 INFO:tasks.rados.rados.0.smithi022.stdout:2808: seq_num 897 ranges {0=630784} 2024-09-21T09:40:59.044 INFO:tasks.rados.rados.0.smithi022.stdout:2808: writing smithi02284960-8 from 0 to 630784 tid 1 2024-09-21T09:40:59.045 INFO:tasks.rados.rados.0.smithi022.stdout:2805: done (1 left) 2024-09-21T09:40:59.045 INFO:tasks.rados.rados.0.smithi022.stdout:2809: read oid 4 snap -1 2024-09-21T09:40:59.045 INFO:tasks.rados.rados.0.smithi022.stdout:2809: expect (ObjNum 891 snap 284 seq_num 891) 2024-09-21T09:40:59.046 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 22 current snap is 284 2024-09-21T09:40:59.046 INFO:tasks.rados.rados.0.smithi022.stdout:2810: seq_num 898 ranges {0=647168} 2024-09-21T09:40:59.050 INFO:tasks.rados.rados.0.smithi022.stdout:2810: writing smithi02284960-22 from 0 to 647168 tid 1 2024-09-21T09:40:59.051 INFO:tasks.rados.rados.0.smithi022.stdout:2811: snap_remove snap 256 2024-09-21T09:40:59.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:59 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:59.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:59 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:59.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:59 smithi022 ceph-mon[96736]: osdmap e633: 8 total, 8 up, 8 in 2024-09-21T09:40:59.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:40:59 smithi022 ceph-mon[96736]: osdmap e634: 8 total, 8 up, 8 in 2024-09-21T09:40:59.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:59 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:59.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:59 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:59.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:59 smithi022 ceph-mon[98615]: osdmap e633: 8 total, 8 up, 8 in 2024-09-21T09:40:59.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:40:59 smithi022 ceph-mon[98615]: osdmap e634: 8 total, 8 up, 8 in 2024-09-21T09:40:59.427 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:59 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:59.427 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:59 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:40:59.427 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:59 smithi064 ceph-mon[86709]: osdmap e633: 8 total, 8 up, 8 in 2024-09-21T09:40:59.427 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:40:59 smithi064 ceph-mon[86709]: osdmap e634: 8 total, 8 up, 8 in 2024-09-21T09:41:00.054 INFO:tasks.rados.rados.0.smithi022.stdout:2811: done (3 left) 2024-09-21T09:41:00.054 INFO:tasks.rados.rados.0.smithi022.stdout:2812: snap_remove snap 245 2024-09-21T09:41:00.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:00 smithi064 ceph-mon[86709]: pgmap v555: 121 pgs: 121 active+clean; 264 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 18 MiB/s rd, 3.4 MiB/s wr, 23 op/s 2024-09-21T09:41:00.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:00 smithi064 ceph-mon[86709]: osdmap e635: 8 total, 8 up, 8 in 2024-09-21T09:41:00.442 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:41:00 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:41:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:41:00.557 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:00 smithi022 ceph-mon[96736]: pgmap v555: 121 pgs: 121 active+clean; 264 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 18 MiB/s rd, 3.4 MiB/s wr, 23 op/s 2024-09-21T09:41:00.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:00 smithi022 ceph-mon[96736]: osdmap e635: 8 total, 8 up, 8 in 2024-09-21T09:41:00.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:00 smithi022 ceph-mon[98615]: pgmap v555: 121 pgs: 121 active+clean; 264 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 18 MiB/s rd, 3.4 MiB/s wr, 23 op/s 2024-09-21T09:41:00.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:00 smithi022 ceph-mon[98615]: osdmap e635: 8 total, 8 up, 8 in 2024-09-21T09:41:01.056 INFO:tasks.rados.rados.0.smithi022.stdout:2812: done (3 left) 2024-09-21T09:41:01.056 INFO:tasks.rados.rados.0.smithi022.stdout:2813: delete oid 7 current snap is 284 2024-09-21T09:41:01.056 INFO:tasks.rados.rados.0.smithi022.stdout:2810: finishing write tid 1 to smithi02284960-22 2024-09-21T09:41:01.056 INFO:tasks.rados.rados.0.smithi022.stdout:2810: finishing write tid 2 to smithi02284960-22 2024-09-21T09:41:01.056 INFO:tasks.rados.rados.0.smithi022.stdout:2810: finishing write tid 3 to smithi02284960-22 2024-09-21T09:41:01.056 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 22 v 1512 (ObjNum 898 snap 284 seq_num 898) dirty exists 2024-09-21T09:41:01.056 INFO:tasks.rados.rados.0.smithi022.stdout:2810: left oid 22 (ObjNum 898 snap 284 seq_num 898) 2024-09-21T09:41:01.059 INFO:tasks.rados.rados.0.smithi022.stdout:2808: finishing write tid 1 to smithi02284960-8 2024-09-21T09:41:01.059 INFO:tasks.rados.rados.0.smithi022.stdout:2808: finishing write tid 2 to smithi02284960-8 2024-09-21T09:41:01.061 INFO:tasks.rados.rados.0.smithi022.stdout:2809: expect (ObjNum 891 snap 284 seq_num 891) 2024-09-21T09:41:02.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[98615]: pgmap v558: 121 pgs: 121 active+clean; 270 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 2.0 MiB/s wr, 9 op/s 2024-09-21T09:41:02.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[98615]: osdmap e636: 8 total, 8 up, 8 in 2024-09-21T09:41:02.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:41:02.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:41:02.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:41:02.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:41:02.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:41:02.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "dashboard get-grafana-api-url"} : dispatch 2024-09-21T09:41:02.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:41:02.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:41:02.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:41:02.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:41:02.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:41:02.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:41:02.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:41:02.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:41:02.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:41:02.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:41:02.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:41:02.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:41:02.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:41:02.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:41:02.310 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:41:02.310 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:41:02.310 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:41:02.310 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:41:02.310 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:41:02.310 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:41:02.310 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-21T09:41:02.310 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[96736]: pgmap v558: 121 pgs: 121 active+clean; 270 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 2.0 MiB/s wr, 9 op/s 2024-09-21T09:41:02.310 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[96736]: osdmap e636: 8 total, 8 up, 8 in 2024-09-21T09:41:02.310 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:41:02.310 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:41:02.311 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:41:02.311 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:41:02.311 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:41:02.311 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "dashboard get-grafana-api-url"} : dispatch 2024-09-21T09:41:02.311 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:41:02.311 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:41:02.311 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:41:02.311 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:41:02.311 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:41:02.311 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:41:02.311 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:41:02.311 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:41:02.311 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:41:02.311 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:41:02.311 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:41:02.312 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:41:02.312 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:41:02.312 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:41:02.312 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:41:02.312 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:41:02.312 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:41:02.312 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:41:02.312 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:41:02.312 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:41:02.312 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-21T09:41:02.312 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-21T09:41:02.313 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:41:02.313 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-21T09:41:02.313 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-21T09:41:02.313 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-21T09:41:02.313 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-21T09:41:02.313 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-09-21T09:41:02.313 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-21T09:41:02.313 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-21T09:41:02.313 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-21T09:41:02.313 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-09-21T09:41:02.313 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-21T09:41:02.313 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-21T09:41:02.313 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:41:02.313 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-21T09:41:02.314 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-21T09:41:02.314 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-21T09:41:02.314 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-09-21T09:41:02.314 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-21T09:41:02.314 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-21T09:41:02.314 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-21T09:41:02.314 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-21T09:41:02.314 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-21T09:41:02.314 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:41:02.314 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-21T09:41:02.314 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-21T09:41:02.314 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-21T09:41:02.315 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-21T09:41:02.315 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-09-21T09:41:02.315 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-21T09:41:02.315 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-21T09:41:02.315 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-21T09:41:02.315 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-09-21T09:41:02.315 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-21T09:41:02.315 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-21T09:41:02.315 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:41:02.315 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-21T09:41:02.315 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-21T09:41:02.315 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-21T09:41:02.315 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-09-21T09:41:02.315 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-21T09:41:02.316 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-21T09:41:02.316 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-21T09:41:02.316 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-21T09:41:02.316 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:41:02.316 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:41:02.316 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:41:02.316 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:41:02.316 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:41:02.316 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:41:02.316 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-21T09:41:02.316 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-21T09:41:02.316 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:41:02.317 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:41:02.317 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:41:02.317 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:41:02.317 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:41:02.317 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:41:02.317 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:41:02.317 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:41:02.317 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:41:02.317 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:41:02.317 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:41:02.317 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:41:02.317 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:41:02.317 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:41:02.317 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-21T09:41:02.318 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-21T09:41:02.318 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:41:02.318 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:41:02.318 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:41:02.318 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:41:02.318 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:41:02.318 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:41:02.318 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:41:02.318 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:02 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:41:02.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:02 smithi064 ceph-mon[86709]: pgmap v558: 121 pgs: 121 active+clean; 270 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 2.0 MiB/s wr, 9 op/s 2024-09-21T09:41:02.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:02 smithi064 ceph-mon[86709]: osdmap e636: 8 total, 8 up, 8 in 2024-09-21T09:41:02.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:02 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:41:02.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:02 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:41:02.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:02 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:41:02.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:02 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:41:02.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:02 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:41:02.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:02 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "dashboard get-grafana-api-url"} : dispatch 2024-09-21T09:41:02.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:02 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:41:02.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:02 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:41:02.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:02 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:41:02.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:02 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:41:02.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:02 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:41:02.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:02 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:41:02.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:02 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:41:02.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:02 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:41:02.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:02 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:41:02.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:02 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:41:02.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:02 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:41:02.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:02 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:41:02.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:02 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:41:02.443 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:02 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:41:02.444 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:02 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:41:02.444 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:02 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:41:02.444 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:02 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:41:02.444 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:02 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:41:02.444 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:02 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:41:02.444 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:02 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:41:02.444 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:02 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-21T09:41:02.444 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:02 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-21T09:41:02.444 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:02 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:41:02.444 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:02 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-21T09:41:02.444 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:02 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-21T09:41:02.444 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:02 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-21T09:41:02.444 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:02 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-21T09:41:02.445 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:02 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-09-21T09:41:02.445 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:02 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-21T09:41:02.445 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:02 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-21T09:41:02.445 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:02 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-21T09:41:02.445 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:02 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-09-21T09:41:02.445 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:02 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-21T09:41:02.445 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:02 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-21T09:41:02.445 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:02 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:41:02.445 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:02 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-21T09:41:02.445 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:02 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-21T09:41:02.445 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:02 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-21T09:41:02.445 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:02 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-09-21T09:41:02.445 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:02 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-21T09:41:02.446 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:02 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-21T09:41:02.446 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:02 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-21T09:41:02.446 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:02 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-21T09:41:02.446 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:02 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:41:02.446 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:02 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:41:02.446 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:02 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:41:02.446 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:02 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:41:02.446 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:02 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:41:02.446 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:02 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:41:02.446 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:02 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-21T09:41:02.446 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:02 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-21T09:41:02.446 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:02 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:41:02.447 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:02 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:41:02.447 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:02 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:41:02.447 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:02 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:41:02.447 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:02 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:41:02.447 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:02 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:41:02.447 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:02 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:41:02.447 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:02 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:41:03.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:03 smithi022 ceph-mon[96736]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T09:41:03.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:03 smithi022 ceph-mon[96736]: Upgrade: Finalizing container_image settings 2024-09-21T09:41:03.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:03 smithi022 ceph-mon[96736]: Upgrade: Complete! 2024-09-21T09:41:03.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:03 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:41:03.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:03 smithi022 ceph-mon[98615]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T09:41:03.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:03 smithi022 ceph-mon[98615]: Upgrade: Finalizing container_image settings 2024-09-21T09:41:03.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:03 smithi022 ceph-mon[98615]: Upgrade: Complete! 2024-09-21T09:41:03.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:03 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:41:03.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:03 smithi064 ceph-mon[86709]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T09:41:03.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:03 smithi064 ceph-mon[86709]: Upgrade: Finalizing container_image settings 2024-09-21T09:41:03.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:03 smithi064 ceph-mon[86709]: Upgrade: Complete! 2024-09-21T09:41:03.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:03 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:41:03.564 INFO:tasks.rados.rados.0.smithi022.stdout:2808: finishing write tid 3 to smithi02284960-8 2024-09-21T09:41:03.565 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 8 v 1655 (ObjNum 897 snap 284 seq_num 897) dirty exists 2024-09-21T09:41:03.565 INFO:tasks.rados.rados.0.smithi022.stdout:2808: left oid 8 (ObjNum 897 snap 284 seq_num 897) 2024-09-21T09:41:03.565 INFO:tasks.rados.rados.0.smithi022.stdout:2808: done (3 left) 2024-09-21T09:41:03.565 INFO:tasks.rados.rados.0.smithi022.stdout:2809: done (2 left) 2024-09-21T09:41:03.565 INFO:tasks.rados.rados.0.smithi022.stdout:2810: done (1 left) 2024-09-21T09:41:03.565 INFO:tasks.rados.rados.0.smithi022.stdout:2813: done (0 left) 2024-09-21T09:41:03.565 INFO:tasks.rados.rados.0.smithi022.stdout:2814: delete oid 19 current snap is 284 2024-09-21T09:41:03.567 INFO:tasks.rados.rados.0.smithi022.stdout:2814: done (0 left) 2024-09-21T09:41:03.567 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 18 current snap is 284 2024-09-21T09:41:03.567 INFO:tasks.rados.rados.0.smithi022.stdout:2815: seq_num 901 ranges {0=696320} 2024-09-21T09:41:03.571 INFO:tasks.rados.rados.0.smithi022.stdout:2815: writing smithi02284960-18 from 0 to 696320 tid 1 2024-09-21T09:41:03.571 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 14 current snap is 284 2024-09-21T09:41:03.571 INFO:tasks.rados.rados.0.smithi022.stdout:2816: seq_num 902 ranges {0=712704} 2024-09-21T09:41:03.575 INFO:tasks.rados.rados.0.smithi022.stdout:2816: writing smithi02284960-14 from 0 to 712704 tid 1 2024-09-21T09:41:03.575 INFO:tasks.rados.rados.0.smithi022.stdout:2817: delete oid 35 current snap is 284 2024-09-21T09:41:03.575 INFO:tasks.rados.rados.0.smithi022.stdout:2815: finishing write tid 1 to smithi02284960-18 2024-09-21T09:41:03.575 INFO:tasks.rados.rados.0.smithi022.stdout:2815: finishing write tid 2 to smithi02284960-18 2024-09-21T09:41:03.576 INFO:tasks.rados.rados.0.smithi022.stdout:2815: finishing write tid 3 to smithi02284960-18 2024-09-21T09:41:03.576 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 18 v 1510 (ObjNum 901 snap 284 seq_num 901) dirty exists 2024-09-21T09:41:03.576 INFO:tasks.rados.rados.0.smithi022.stdout:2815: left oid 18 (ObjNum 901 snap 284 seq_num 901) 2024-09-21T09:41:03.578 INFO:tasks.rados.rados.0.smithi022.stdout:2815: done (2 left) 2024-09-21T09:41:03.578 INFO:tasks.rados.rados.0.smithi022.stdout:2817: done (1 left) 2024-09-21T09:41:03.578 INFO:tasks.rados.rados.0.smithi022.stdout:2818: setattr oid 35 current snap is 284 2024-09-21T09:41:03.579 INFO:tasks.rados.rados.0.smithi022.stdout:2819: copy_from oid 26 from oid 1 current snap is 284 2024-09-21T09:41:03.579 INFO:tasks.rados.rados.0.smithi022.stdout:2820: copy_from oid 17 from oid 29 current snap is 284 2024-09-21T09:41:03.579 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 27 current snap is 284 2024-09-21T09:41:03.580 INFO:tasks.rados.rados.0.smithi022.stdout:2821: seq_num 904 ranges {5881856=745472} 2024-09-21T09:41:03.616 INFO:tasks.rados.rados.0.smithi022.stdout:2821: writing smithi02284960-27 from 5881856 to 6627328 tid 1 2024-09-21T09:41:03.616 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 2 current snap is 284 2024-09-21T09:41:03.616 INFO:tasks.rados.rados.0.smithi022.stdout:2822: seq_num 905 ranges {4145152=761856} 2024-09-21T09:41:03.644 INFO:tasks.rados.rados.0.smithi022.stdout:2822: writing smithi02284960-2 from 4145152 to 4907008 tid 1 2024-09-21T09:41:03.644 INFO:tasks.rados.rados.0.smithi022.stdout:2823: read oid 9 snap -1 2024-09-21T09:41:03.644 INFO:tasks.rados.rados.0.smithi022.stdout:2823: expect (ObjNum 796 snap 249 seq_num 796) 2024-09-21T09:41:03.644 INFO:tasks.rados.rados.0.smithi022.stdout:2824: rollback oid 45 current snap is 284 2024-09-21T09:41:03.644 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 45 to 275 2024-09-21T09:41:03.644 INFO:tasks.rados.rados.0.smithi022.stdout:2816: finishing write tid 1 to smithi02284960-14 2024-09-21T09:41:03.644 INFO:tasks.rados.rados.0.smithi022.stdout:2825: read oid 46 snap -1 2024-09-21T09:41:03.644 INFO:tasks.rados.rados.0.smithi022.stdout:2825: expect (ObjNum 865 snap 277 seq_num 865) 2024-09-21T09:41:03.644 INFO:tasks.rados.rados.0.smithi022.stdout:2826: read oid 32 snap -1 2024-09-21T09:41:03.644 INFO:tasks.rados.rados.0.smithi022.stdout:2826: expect (ObjNum 868 snap 280 seq_num 868) 2024-09-21T09:41:03.644 INFO:tasks.rados.rados.0.smithi022.stdout:2827: read oid 24 snap -1 2024-09-21T09:41:03.644 INFO:tasks.rados.rados.0.smithi022.stdout:2827: expect deleted 2024-09-21T09:41:03.645 INFO:tasks.rados.rados.0.smithi022.stdout:2828: delete oid 6 current snap is 284 2024-09-21T09:41:03.645 INFO:tasks.rados.rados.0.smithi022.stdout:2816: finishing write tid 2 to smithi02284960-14 2024-09-21T09:41:03.645 INFO:tasks.rados.rados.0.smithi022.stdout:2820: finishing copy_from to smithi02284960-17 2024-09-21T09:41:03.645 INFO:tasks.rados.rados.0.smithi022.stdout:2820: got expected ENOENT (src dne) 2024-09-21T09:41:03.645 INFO:tasks.rados.rados.0.smithi022.stdout:2820: finishing copy_from racing read to smithi02284960-17 2024-09-21T09:41:03.645 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 35 v 1410 (ObjNum 289 snap 0 seq_num 0) dirty exists 2024-09-21T09:41:03.645 INFO:tasks.rados.rados.0.smithi022.stdout:2816: finishing write tid 3 to smithi02284960-14 2024-09-21T09:41:03.645 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 14 v 1121 (ObjNum 902 snap 284 seq_num 902) dirty exists 2024-09-21T09:41:03.645 INFO:tasks.rados.rados.0.smithi022.stdout:2816: left oid 14 (ObjNum 902 snap 284 seq_num 902) 2024-09-21T09:41:03.645 INFO:tasks.rados.rados.0.smithi022.stdout:2819: finishing copy_from racing read to smithi02284960-26 2024-09-21T09:41:03.645 INFO:tasks.rados.rados.0.smithi022.stdout:2819: finishing copy_from to smithi02284960-26 2024-09-21T09:41:03.645 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 26 v 1650 (ObjNum 796 snap 249 seq_num 796) dirty exists 2024-09-21T09:41:03.645 INFO:tasks.rados.rados.0.smithi022.stdout:2821: finishing write tid 1 to smithi02284960-27 2024-09-21T09:41:03.645 INFO:tasks.rados.rados.0.smithi022.stdout:2821: finishing write tid 2 to smithi02284960-27 2024-09-21T09:41:03.646 INFO:tasks.rados.rados.0.smithi022.stdout:2821: finishing write tid 3 to smithi02284960-27 2024-09-21T09:41:03.646 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 27 v 1652 (ObjNum 904 snap 284 seq_num 904) dirty exists 2024-09-21T09:41:03.646 INFO:tasks.rados.rados.0.smithi022.stdout:2821: left oid 27 (ObjNum 904 snap 284 seq_num 904) 2024-09-21T09:41:03.646 INFO:tasks.rados.rados.0.smithi022.stdout:2825: expect (ObjNum 865 snap 277 seq_num 865) 2024-09-21T09:41:03.649 INFO:tasks.rados.rados.0.smithi022.stdout:2826: expect (ObjNum 868 snap 280 seq_num 868) 2024-09-21T09:41:03.654 INFO:tasks.rados.rados.0.smithi022.stdout:2824: finishing rollback tid 0 to smithi02284960-45 2024-09-21T09:41:03.654 INFO:tasks.rados.rados.0.smithi022.stdout:2824: finishing rollback tid 1 to smithi02284960-45 2024-09-21T09:41:03.654 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 45 v 1411 (ObjNum 4113 snap 0 seq_num 1196846152) dirty dne 2024-09-21T09:41:03.654 INFO:tasks.rados.rados.0.smithi022.stdout:2822: finishing write tid 1 to smithi02284960-2 2024-09-21T09:41:03.654 INFO:tasks.rados.rados.0.smithi022.stdout:2822: finishing write tid 2 to smithi02284960-2 2024-09-21T09:41:03.654 INFO:tasks.rados.rados.0.smithi022.stdout:2816: done (11 left) 2024-09-21T09:41:03.654 INFO:tasks.rados.rados.0.smithi022.stdout:2818: done (10 left) 2024-09-21T09:41:03.654 INFO:tasks.rados.rados.0.smithi022.stdout:2819: done (9 left) 2024-09-21T09:41:03.655 INFO:tasks.rados.rados.0.smithi022.stdout:2820: done (8 left) 2024-09-21T09:41:03.655 INFO:tasks.rados.rados.0.smithi022.stdout:2821: done (7 left) 2024-09-21T09:41:03.655 INFO:tasks.rados.rados.0.smithi022.stdout:2824: done (6 left) 2024-09-21T09:41:03.655 INFO:tasks.rados.rados.0.smithi022.stdout:2825: done (5 left) 2024-09-21T09:41:03.655 INFO:tasks.rados.rados.0.smithi022.stdout:2826: done (4 left) 2024-09-21T09:41:03.655 INFO:tasks.rados.rados.0.smithi022.stdout:2827: done (3 left) 2024-09-21T09:41:03.655 INFO:tasks.rados.rados.0.smithi022.stdout:2828: done (2 left) 2024-09-21T09:41:03.655 INFO:tasks.rados.rados.0.smithi022.stdout:2829: copy_from oid 30 from oid 40 current snap is 284 2024-09-21T09:41:03.655 INFO:tasks.rados.rados.0.smithi022.stdout:2822: finishing write tid 3 to smithi02284960-2 2024-09-21T09:41:03.655 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 2 v 1538 (ObjNum 905 snap 284 seq_num 905) dirty exists 2024-09-21T09:41:03.655 INFO:tasks.rados.rados.0.smithi022.stdout:2822: left oid 2 (ObjNum 905 snap 284 seq_num 905) 2024-09-21T09:41:03.655 INFO:tasks.rados.rados.0.smithi022.stdout:2822: done (2 left) 2024-09-21T09:41:03.655 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 13 current snap is 284 2024-09-21T09:41:03.655 INFO:tasks.rados.rados.0.smithi022.stdout:2830: seq_num 907 ranges {0=794624} 2024-09-21T09:41:03.658 INFO:tasks.rados.rados.0.smithi022.stdout:2830: writing smithi02284960-13 from 0 to 794624 tid 1 2024-09-21T09:41:03.658 INFO:tasks.rados.rados.0.smithi022.stdout:2831: delete oid 26 current snap is 284 2024-09-21T09:41:03.658 INFO:tasks.rados.rados.0.smithi022.stdout:2829: finishing copy_from to smithi02284960-30 2024-09-21T09:41:03.658 INFO:tasks.rados.rados.0.smithi022.stdout:2829: got expected ENOENT (src dne) 2024-09-21T09:41:03.658 INFO:tasks.rados.rados.0.smithi022.stdout:2829: finishing copy_from racing read to smithi02284960-30 2024-09-21T09:41:03.664 INFO:tasks.rados.rados.0.smithi022.stdout:2830: finishing write tid 1 to smithi02284960-13 2024-09-21T09:41:03.664 INFO:tasks.rados.rados.0.smithi022.stdout:2830: finishing write tid 2 to smithi02284960-13 2024-09-21T09:41:03.664 INFO:tasks.rados.rados.0.smithi022.stdout:2823: expect (ObjNum 796 snap 249 seq_num 796) 2024-09-21T09:41:04.095 INFO:tasks.rados.rados.0.smithi022.stdout:2830: finishing write tid 3 to smithi02284960-13 2024-09-21T09:41:04.095 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 13 v 1654 (ObjNum 907 snap 284 seq_num 907) dirty exists 2024-09-21T09:41:04.095 INFO:tasks.rados.rados.0.smithi022.stdout:2830: left oid 13 (ObjNum 907 snap 284 seq_num 907) 2024-09-21T09:41:04.095 INFO:tasks.rados.rados.0.smithi022.stdout:2823: done (3 left) 2024-09-21T09:41:04.095 INFO:tasks.rados.rados.0.smithi022.stdout:2829: done (2 left) 2024-09-21T09:41:04.095 INFO:tasks.rados.rados.0.smithi022.stdout:2830: done (1 left) 2024-09-21T09:41:04.095 INFO:tasks.rados.rados.0.smithi022.stdout:2831: done (0 left) 2024-09-21T09:41:04.095 INFO:tasks.rados.rados.0.smithi022.stdout:2832: read oid 33 snap 267 2024-09-21T09:41:04.095 INFO:tasks.rados.rados.0.smithi022.stdout:2832: expect (ObjNum 798 snap 250 seq_num 798) 2024-09-21T09:41:04.095 INFO:tasks.rados.rados.0.smithi022.stdout:2833: delete oid 18 current snap is 284 2024-09-21T09:41:04.098 INFO:tasks.rados.rados.0.smithi022.stdout:2833: done (1 left) 2024-09-21T09:41:04.099 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 3 current snap is 284 2024-09-21T09:41:04.099 INFO:tasks.rados.rados.0.smithi022.stdout:2834: seq_num 910 ranges {0=442368,442368=401408} 2024-09-21T09:41:04.101 INFO:tasks.rados.rados.0.smithi022.stdout:2834: writing smithi02284960-3 from 0 to 442368 tid 1 2024-09-21T09:41:04.104 INFO:tasks.rados.rados.0.smithi022.stdout:2834: writing smithi02284960-3 from 442368 to 843776 tid 2 2024-09-21T09:41:04.104 INFO:tasks.rados.rados.0.smithi022.stdout:2835: copy_from oid 35 from oid 12 current snap is 284 2024-09-21T09:41:04.104 INFO:tasks.rados.rados.0.smithi022.stdout:2836: read oid 15 snap 279 2024-09-21T09:41:04.104 INFO:tasks.rados.rados.0.smithi022.stdout:2836: expect (ObjNum 809 snap 255 seq_num 809) 2024-09-21T09:41:04.105 INFO:tasks.rados.rados.0.smithi022.stdout:2837: rmattr oid 4 current snap is 284 2024-09-21T09:41:04.105 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 39 current snap is 284 2024-09-21T09:41:04.105 INFO:tasks.rados.rados.0.smithi022.stdout:2838: seq_num 911 ranges {0=466944,466944=401408} 2024-09-21T09:41:04.108 INFO:tasks.rados.rados.0.smithi022.stdout:2838: writing smithi02284960-39 from 0 to 466944 tid 1 2024-09-21T09:41:04.111 INFO:tasks.rados.rados.0.smithi022.stdout:2838: writing smithi02284960-39 from 466944 to 868352 tid 2 2024-09-21T09:41:04.111 INFO:tasks.rados.rados.0.smithi022.stdout:2839: copy_from oid 31 from oid 42 current snap is 284 2024-09-21T09:41:04.111 INFO:tasks.rados.rados.0.smithi022.stdout:2832: expect (ObjNum 798 snap 250 seq_num 798) 2024-09-21T09:41:04.283 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 4 v 1656 (ObjNum 891 snap 284 seq_num 891) dirty exists 2024-09-21T09:41:04.283 INFO:tasks.rados.rados.0.smithi022.stdout:2834: finishing write tid 1 to smithi02284960-3 2024-09-21T09:41:04.283 INFO:tasks.rados.rados.0.smithi022.stdout:2834: finishing write tid 2 to smithi02284960-3 2024-09-21T09:41:04.283 INFO:tasks.rados.rados.0.smithi022.stdout:2834: finishing write tid 3 to smithi02284960-3 2024-09-21T09:41:04.283 INFO:tasks.rados.rados.0.smithi022.stdout:2838: finishing write tid 1 to smithi02284960-39 2024-09-21T09:41:04.283 INFO:tasks.rados.rados.0.smithi022.stdout:2834: finishing write tid 4 to smithi02284960-3 2024-09-21T09:41:04.284 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 3 v 616 (ObjNum 910 snap 284 seq_num 910) dirty exists 2024-09-21T09:41:04.284 INFO:tasks.rados.rados.0.smithi022.stdout:2834: left oid 3 (ObjNum 910 snap 284 seq_num 910) 2024-09-21T09:41:04.284 INFO:tasks.rados.rados.0.smithi022.stdout:2838: finishing write tid 2 to smithi02284960-39 2024-09-21T09:41:04.284 INFO:tasks.rados.rados.0.smithi022.stdout:2832: done (6 left) 2024-09-21T09:41:04.284 INFO:tasks.rados.rados.0.smithi022.stdout:2834: done (5 left) 2024-09-21T09:41:04.284 INFO:tasks.rados.rados.0.smithi022.stdout:2837: done (4 left) 2024-09-21T09:41:04.284 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 26 current snap is 284 2024-09-21T09:41:04.284 INFO:tasks.rados.rados.0.smithi022.stdout:2838: finishing write tid 3 to smithi02284960-39 2024-09-21T09:41:04.284 INFO:tasks.rados.rados.0.smithi022.stdout:2840: seq_num 912 ranges {0=483328,483328=401408} 2024-09-21T09:41:04.285 INFO:tasks.rados.rados.0.smithi022.stdout:2840: writing smithi02284960-26 from 0 to 483328 tid 1 2024-09-21T09:41:04.288 INFO:tasks.rados.rados.0.smithi022.stdout:2840: writing smithi02284960-26 from 483328 to 884736 tid 2 2024-09-21T09:41:04.288 INFO:tasks.rados.rados.0.smithi022.stdout:2841: snap_remove snap 267 2024-09-21T09:41:04.342 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:04 smithi064 ceph-mon[86709]: pgmap v560: 121 pgs: 121 active+clean; 270 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 2.0 MiB/s wr, 10 op/s 2024-09-21T09:41:04.343 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:04 smithi064 ceph-mon[86709]: osdmap e637: 8 total, 8 up, 8 in 2024-09-21T09:41:04.557 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:04 smithi022 ceph-mon[98615]: pgmap v560: 121 pgs: 121 active+clean; 270 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 2.0 MiB/s wr, 10 op/s 2024-09-21T09:41:04.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:04 smithi022 ceph-mon[98615]: osdmap e637: 8 total, 8 up, 8 in 2024-09-21T09:41:04.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:04 smithi022 ceph-mon[96736]: pgmap v560: 121 pgs: 121 active+clean; 270 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 2.0 MiB/s wr, 10 op/s 2024-09-21T09:41:04.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:04 smithi022 ceph-mon[96736]: osdmap e637: 8 total, 8 up, 8 in 2024-09-21T09:41:05.068 INFO:tasks.rados.rados.0.smithi022.stdout:2841: done (5 left) 2024-09-21T09:41:05.068 INFO:tasks.rados.rados.0.smithi022.stdout:2842: delete oid 27 current snap is 284 2024-09-21T09:41:05.069 INFO:tasks.rados.rados.0.smithi022.stdout:2838: finishing write tid 4 to smithi02284960-39 2024-09-21T09:41:05.069 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 39 v 1310 (ObjNum 911 snap 284 seq_num 911) dirty exists 2024-09-21T09:41:05.069 INFO:tasks.rados.rados.0.smithi022.stdout:2838: left oid 39 (ObjNum 911 snap 284 seq_num 911) 2024-09-21T09:41:05.069 INFO:tasks.rados.rados.0.smithi022.stdout:2836: expect (ObjNum 809 snap 255 seq_num 809) 2024-09-21T09:41:05.420 INFO:tasks.rados.rados.0.smithi022.stdout:2839: finishing copy_from racing read to smithi02284960-31 2024-09-21T09:41:05.420 INFO:tasks.rados.rados.0.smithi022.stdout:2839: finishing copy_from to smithi02284960-31 2024-09-21T09:41:05.420 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 31 v 1510 (ObjNum 884 snap 282 seq_num 884) dirty exists 2024-09-21T09:41:05.420 INFO:tasks.rados.rados.0.smithi022.stdout:2835: finishing copy_from racing read to smithi02284960-35 2024-09-21T09:41:05.421 INFO:tasks.rados.rados.0.smithi022.stdout:2835: finishing copy_from to smithi02284960-35 2024-09-21T09:41:05.421 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 35 v 1594 (ObjNum 877 snap 282 seq_num 877) dirty exists 2024-09-21T09:41:05.421 INFO:tasks.rados.rados.0.smithi022.stdout:2840: finishing write tid 1 to smithi02284960-26 2024-09-21T09:41:05.421 INFO:tasks.rados.rados.0.smithi022.stdout:2835: done (5 left) 2024-09-21T09:41:05.421 INFO:tasks.rados.rados.0.smithi022.stdout:2836: done (4 left) 2024-09-21T09:41:05.421 INFO:tasks.rados.rados.0.smithi022.stdout:2838: done (3 left) 2024-09-21T09:41:05.421 INFO:tasks.rados.rados.0.smithi022.stdout:2839: done (2 left) 2024-09-21T09:41:05.421 INFO:tasks.rados.rados.0.smithi022.stdout:2842: done (1 left) 2024-09-21T09:41:05.421 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 22 current snap is 284 2024-09-21T09:41:05.421 INFO:tasks.rados.rados.0.smithi022.stdout:2840: finishing write tid 2 to smithi02284960-26 2024-09-21T09:41:05.421 INFO:tasks.rados.rados.0.smithi022.stdout:2840: finishing write tid 3 to smithi02284960-26 2024-09-21T09:41:05.421 INFO:tasks.rados.rados.0.smithi022.stdout:2840: finishing write tid 4 to smithi02284960-26 2024-09-21T09:41:05.421 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 26 v 1658 (ObjNum 912 snap 284 seq_num 912) dirty exists 2024-09-21T09:41:05.421 INFO:tasks.rados.rados.0.smithi022.stdout:2840: left oid 26 (ObjNum 912 snap 284 seq_num 912) 2024-09-21T09:41:05.421 INFO:tasks.rados.rados.0.smithi022.stdout:2843: seq_num 914 ranges {647168=516096,1163264=401408} 2024-09-21T09:41:05.426 INFO:tasks.rados.rados.0.smithi022.stdout:2843: writing smithi02284960-22 from 647168 to 1163264 tid 1 2024-09-21T09:41:05.428 INFO:tasks.rados.rados.0.smithi022.stdout:2843: writing smithi02284960-22 from 1163264 to 1564672 tid 2 2024-09-21T09:41:05.428 INFO:tasks.rados.rados.0.smithi022.stdout:2840: done (1 left) 2024-09-21T09:41:05.428 INFO:tasks.rados.rados.0.smithi022.stdout:2844: snap_remove snap 252 2024-09-21T09:41:06.058 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:41:05 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:41:05] "GET /metrics HTTP/1.1" 200 36360 "" "Prometheus/2.51.0" 2024-09-21T09:41:06.069 INFO:tasks.rados.rados.0.smithi022.stdout:2843: finishing write tid 1 to smithi02284960-22 2024-09-21T09:41:06.069 INFO:tasks.rados.rados.0.smithi022.stdout:2843: finishing write tid 2 to smithi02284960-22 2024-09-21T09:41:06.070 INFO:tasks.rados.rados.0.smithi022.stdout:2843: finishing write tid 3 to smithi02284960-22 2024-09-21T09:41:06.070 INFO:tasks.rados.rados.0.smithi022.stdout:2843: finishing write tid 4 to smithi02284960-22 2024-09-21T09:41:06.070 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 22 v 1662 (ObjNum 914 snap 284 seq_num 914) dirty exists 2024-09-21T09:41:06.070 INFO:tasks.rados.rados.0.smithi022.stdout:2843: left oid 22 (ObjNum 914 snap 284 seq_num 914) 2024-09-21T09:41:06.070 INFO:tasks.rados.rados.0.smithi022.stdout:2843: done (1 left) 2024-09-21T09:41:06.070 INFO:tasks.rados.rados.0.smithi022.stdout:2844: done (0 left) 2024-09-21T09:41:06.070 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 26 current snap is 284 2024-09-21T09:41:06.070 INFO:tasks.rados.rados.0.smithi022.stdout:2845: seq_num 915 ranges {884736=532480,1417216=401408} 2024-09-21T09:41:06.078 INFO:tasks.rados.rados.0.smithi022.stdout:2845: writing smithi02284960-26 from 884736 to 1417216 tid 1 2024-09-21T09:41:06.081 INFO:tasks.rados.rados.0.smithi022.stdout:2845: writing smithi02284960-26 from 1417216 to 1818624 tid 2 2024-09-21T09:41:06.081 INFO:tasks.rados.rados.0.smithi022.stdout:2846: rmattr oid 36 current snap is 284 2024-09-21T09:41:06.081 INFO:tasks.rados.rados.0.smithi022.stdout:2846: done (1 left) 2024-09-21T09:41:06.081 INFO:tasks.rados.rados.0.smithi022.stdout:2847: snap_remove snap 277 2024-09-21T09:41:06.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:06 smithi064 ceph-mon[86709]: pgmap v562: 121 pgs: 121 active+clean; 268 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 1.7 MiB/s wr, 5 op/s 2024-09-21T09:41:06.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:06 smithi064 ceph-mon[86709]: osdmap e638: 8 total, 8 up, 8 in 2024-09-21T09:41:06.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:06 smithi022 ceph-mon[98615]: pgmap v562: 121 pgs: 121 active+clean; 268 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 1.7 MiB/s wr, 5 op/s 2024-09-21T09:41:06.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:06 smithi022 ceph-mon[98615]: osdmap e638: 8 total, 8 up, 8 in 2024-09-21T09:41:06.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:06 smithi022 ceph-mon[96736]: pgmap v562: 121 pgs: 121 active+clean; 268 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 1.7 MiB/s wr, 5 op/s 2024-09-21T09:41:06.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:06 smithi022 ceph-mon[96736]: osdmap e638: 8 total, 8 up, 8 in 2024-09-21T09:41:07.082 INFO:tasks.rados.rados.0.smithi022.stdout:2845: finishing write tid 1 to smithi02284960-26 2024-09-21T09:41:07.082 INFO:tasks.rados.rados.0.smithi022.stdout:2845: finishing write tid 2 to smithi02284960-26 2024-09-21T09:41:07.083 INFO:tasks.rados.rados.0.smithi022.stdout:2845: finishing write tid 3 to smithi02284960-26 2024-09-21T09:41:07.083 INFO:tasks.rados.rados.0.smithi022.stdout:2845: finishing write tid 4 to smithi02284960-26 2024-09-21T09:41:07.083 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 26 v 1665 (ObjNum 915 snap 284 seq_num 915) dirty exists 2024-09-21T09:41:07.083 INFO:tasks.rados.rados.0.smithi022.stdout:2845: left oid 26 (ObjNum 915 snap 284 seq_num 915) 2024-09-21T09:41:07.083 INFO:tasks.rados.rados.0.smithi022.stdout:2845: done (1 left) 2024-09-21T09:41:07.083 INFO:tasks.rados.rados.0.smithi022.stdout:2847: done (0 left) 2024-09-21T09:41:07.083 INFO:tasks.rados.rados.0.smithi022.stdout:2848: snap_remove snap 257 2024-09-21T09:41:07.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:07 smithi064 ceph-mon[86709]: osdmap e639: 8 total, 8 up, 8 in 2024-09-21T09:41:07.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:07 smithi022 ceph-mon[98615]: osdmap e639: 8 total, 8 up, 8 in 2024-09-21T09:41:07.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:07 smithi022 ceph-mon[96736]: osdmap e639: 8 total, 8 up, 8 in 2024-09-21T09:41:08.088 INFO:tasks.rados.rados.0.smithi022.stdout:2848: done (0 left) 2024-09-21T09:41:08.088 INFO:tasks.rados.rados.0.smithi022.stdout:2849: rmattr oid 22 current snap is 284 2024-09-21T09:41:08.088 INFO:tasks.rados.rados.0.smithi022.stdout:2849: done (0 left) 2024-09-21T09:41:08.088 INFO:tasks.rados.rados.0.smithi022.stdout:2850: read oid 19 snap -1 2024-09-21T09:41:08.088 INFO:tasks.rados.rados.0.smithi022.stdout:2850: expect deleted 2024-09-21T09:41:08.088 INFO:tasks.rados.rados.0.smithi022.stdout:2851: read oid 50 snap -1 2024-09-21T09:41:08.088 INFO:tasks.rados.rados.0.smithi022.stdout:2851: expect (ObjNum 867 snap 280 seq_num 867) 2024-09-21T09:41:08.088 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 28 current snap is 284 2024-09-21T09:41:08.088 INFO:tasks.rados.rados.0.smithi022.stdout:2852: seq_num 916 ranges {5087232=548864,5636096=401408} 2024-09-21T09:41:08.120 INFO:tasks.rados.rados.0.smithi022.stdout:2852: writing smithi02284960-28 from 5087232 to 5636096 tid 1 2024-09-21T09:41:08.122 INFO:tasks.rados.rados.0.smithi022.stdout:2852: writing smithi02284960-28 from 5636096 to 6037504 tid 2 2024-09-21T09:41:08.123 INFO:tasks.rados.rados.0.smithi022.stdout:2853: read oid 36 snap -1 2024-09-21T09:41:08.123 INFO:tasks.rados.rados.0.smithi022.stdout:2853: expect deleted 2024-09-21T09:41:08.123 INFO:tasks.rados.rados.0.smithi022.stdout:2854: rmattr oid 15 current snap is 284 2024-09-21T09:41:08.123 INFO:tasks.rados.rados.0.smithi022.stdout:2854: done (4 left) 2024-09-21T09:41:08.123 INFO:tasks.rados.rados.0.smithi022.stdout:2855: snap_remove snap 282 2024-09-21T09:41:08.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:08 smithi064 ceph-mon[86709]: pgmap v565: 121 pgs: 121 active+clean; 258 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 8.8 MiB/s rd, 3.1 MiB/s wr, 14 op/s 2024-09-21T09:41:08.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:08 smithi064 ceph-mon[86709]: osdmap e640: 8 total, 8 up, 8 in 2024-09-21T09:41:08.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:08 smithi022 ceph-mon[98615]: pgmap v565: 121 pgs: 121 active+clean; 258 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 8.8 MiB/s rd, 3.1 MiB/s wr, 14 op/s 2024-09-21T09:41:08.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:08 smithi022 ceph-mon[98615]: osdmap e640: 8 total, 8 up, 8 in 2024-09-21T09:41:08.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:08 smithi022 ceph-mon[96736]: pgmap v565: 121 pgs: 121 active+clean; 258 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 8.8 MiB/s rd, 3.1 MiB/s wr, 14 op/s 2024-09-21T09:41:08.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:08 smithi022 ceph-mon[96736]: osdmap e640: 8 total, 8 up, 8 in 2024-09-21T09:41:09.043 INFO:tasks.rados.rados.0.smithi022.stdout:2852: finishing write tid 1 to smithi02284960-28 2024-09-21T09:41:09.043 INFO:tasks.rados.rados.0.smithi022.stdout:2852: finishing write tid 2 to smithi02284960-28 2024-09-21T09:41:09.043 INFO:tasks.rados.rados.0.smithi022.stdout:2852: finishing write tid 3 to smithi02284960-28 2024-09-21T09:41:09.043 INFO:tasks.rados.rados.0.smithi022.stdout:2852: finishing write tid 4 to smithi02284960-28 2024-09-21T09:41:09.043 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 28 v 1597 (ObjNum 916 snap 284 seq_num 916) dirty exists 2024-09-21T09:41:09.043 INFO:tasks.rados.rados.0.smithi022.stdout:2852: left oid 28 (ObjNum 916 snap 284 seq_num 916) 2024-09-21T09:41:09.043 INFO:tasks.rados.rados.0.smithi022.stdout:2851: expect (ObjNum 867 snap 280 seq_num 867) 2024-09-21T09:41:09.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:09 smithi064 ceph-mon[86709]: osdmap e641: 8 total, 8 up, 8 in 2024-09-21T09:41:09.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:09 smithi064 ceph-mon[86709]: osdmap e642: 8 total, 8 up, 8 in 2024-09-21T09:41:09.557 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:09 smithi022 ceph-mon[98615]: osdmap e641: 8 total, 8 up, 8 in 2024-09-21T09:41:09.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:09 smithi022 ceph-mon[98615]: osdmap e642: 8 total, 8 up, 8 in 2024-09-21T09:41:09.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:09 smithi022 ceph-mon[96736]: osdmap e641: 8 total, 8 up, 8 in 2024-09-21T09:41:09.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:09 smithi022 ceph-mon[96736]: osdmap e642: 8 total, 8 up, 8 in 2024-09-21T09:41:10.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:10 smithi064 ceph-mon[86709]: pgmap v568: 121 pgs: 121 active+clean; 250 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 5.7 MiB/s wr, 26 op/s 2024-09-21T09:41:10.441 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:41:10 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:41:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:41:10.557 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:10 smithi022 ceph-mon[98615]: pgmap v568: 121 pgs: 121 active+clean; 250 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 5.7 MiB/s wr, 26 op/s 2024-09-21T09:41:10.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:10 smithi022 ceph-mon[96736]: pgmap v568: 121 pgs: 121 active+clean; 250 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 5.7 MiB/s wr, 26 op/s 2024-09-21T09:41:11.160 INFO:tasks.rados.rados.0.smithi022.stdout:2850: done (4 left) 2024-09-21T09:41:11.160 INFO:tasks.rados.rados.0.smithi022.stdout:2851: done (3 left) 2024-09-21T09:41:11.160 INFO:tasks.rados.rados.0.smithi022.stdout:2852: done (2 left) 2024-09-21T09:41:11.160 INFO:tasks.rados.rados.0.smithi022.stdout:2853: done (1 left) 2024-09-21T09:41:11.160 INFO:tasks.rados.rados.0.smithi022.stdout:2855: done (0 left) 2024-09-21T09:41:11.161 INFO:tasks.rados.rados.0.smithi022.stdout:2856: snap_create 2024-09-21T09:41:12.098 INFO:tasks.rados.rados.0.smithi022.stdout:2856: done (0 left) 2024-09-21T09:41:12.098 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 20 current snap is 285 2024-09-21T09:41:12.098 INFO:tasks.rados.rados.0.smithi022.stdout:2857: seq_num 917 ranges {6602752=565248,7168000=401408} 2024-09-21T09:41:12.137 INFO:tasks.rados.rados.0.smithi022.stdout:2857: writing smithi02284960-20 from 6602752 to 7168000 tid 1 2024-09-21T09:41:12.140 INFO:tasks.rados.rados.0.smithi022.stdout:2857: writing smithi02284960-20 from 7168000 to 7569408 tid 2 2024-09-21T09:41:12.140 INFO:tasks.rados.rados.0.smithi022.stdout:2858: read oid 9 snap 270 2024-09-21T09:41:12.140 INFO:tasks.rados.rados.0.smithi022.stdout:2858: expect (ObjNum 837 snap 269 seq_num 837) 2024-09-21T09:41:12.141 INFO:tasks.rados.rados.0.smithi022.stdout:2859: read oid 30 snap -1 2024-09-21T09:41:12.141 INFO:tasks.rados.rados.0.smithi022.stdout:2859: expect (ObjNum 590 snap 176 seq_num 590) 2024-09-21T09:41:12.141 INFO:tasks.rados.rados.0.smithi022.stdout:2860: snap_create 2024-09-21T09:41:12.144 INFO:tasks.rados.rados.0.smithi022.stdout:2859: expect (ObjNum 590 snap 176 seq_num 590) 2024-09-21T09:41:12.155 INFO:tasks.rados.rados.0.smithi022.stdout:2857: finishing write tid 1 to smithi02284960-20 2024-09-21T09:41:12.155 INFO:tasks.rados.rados.0.smithi022.stdout:2857: finishing write tid 2 to smithi02284960-20 2024-09-21T09:41:12.155 INFO:tasks.rados.rados.0.smithi022.stdout:2857: finishing write tid 3 to smithi02284960-20 2024-09-21T09:41:12.155 INFO:tasks.rados.rados.0.smithi022.stdout:2857: finishing write tid 4 to smithi02284960-20 2024-09-21T09:41:12.155 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 20 v 1764 (ObjNum 917 snap 285 seq_num 917) dirty exists 2024-09-21T09:41:12.155 INFO:tasks.rados.rados.0.smithi022.stdout:2857: left oid 20 (ObjNum 917 snap 285 seq_num 917) 2024-09-21T09:41:12.155 INFO:tasks.rados.rados.0.smithi022.stdout:2858: expect (ObjNum 837 snap 269 seq_num 837) 2024-09-21T09:41:12.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:12 smithi064 ceph-mon[86709]: pgmap v570: 121 pgs: 121 active+clean; 247 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3.9 MiB/s rd, 4.9 MiB/s wr, 14 op/s 2024-09-21T09:41:12.557 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:12 smithi022 ceph-mon[98615]: pgmap v570: 121 pgs: 121 active+clean; 247 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3.9 MiB/s rd, 4.9 MiB/s wr, 14 op/s 2024-09-21T09:41:12.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:12 smithi022 ceph-mon[96736]: pgmap v570: 121 pgs: 121 active+clean; 247 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3.9 MiB/s rd, 4.9 MiB/s wr, 14 op/s 2024-09-21T09:41:13.104 INFO:tasks.rados.rados.0.smithi022.stdout:2857: done (3 left) 2024-09-21T09:41:13.104 INFO:tasks.rados.rados.0.smithi022.stdout:2858: done (2 left) 2024-09-21T09:41:13.105 INFO:tasks.rados.rados.0.smithi022.stdout:2859: done (1 left) 2024-09-21T09:41:13.105 INFO:tasks.rados.rados.0.smithi022.stdout:2860: done (0 left) 2024-09-21T09:41:13.105 INFO:tasks.rados.rados.0.smithi022.stdout:2861: read oid 34 snap -1 2024-09-21T09:41:13.105 INFO:tasks.rados.rados.0.smithi022.stdout:2861: expect (ObjNum 849 snap 273 seq_num 849) 2024-09-21T09:41:13.105 INFO:tasks.rados.rados.0.smithi022.stdout:2862: delete oid 2 current snap is 286 2024-09-21T09:41:13.117 INFO:tasks.rados.rados.0.smithi022.stdout:2862: done (1 left) 2024-09-21T09:41:13.117 INFO:tasks.rados.rados.0.smithi022.stdout:2863: snap_remove snap 265 2024-09-21T09:41:13.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:13 smithi064 ceph-mon[86709]: osdmap e643: 8 total, 8 up, 8 in 2024-09-21T09:41:13.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:13 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:41:13.557 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:13 smithi022 ceph-mon[96736]: osdmap e643: 8 total, 8 up, 8 in 2024-09-21T09:41:13.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:13 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:41:13.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:13 smithi022 ceph-mon[98615]: osdmap e643: 8 total, 8 up, 8 in 2024-09-21T09:41:13.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:13 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:41:14.044 INFO:tasks.rados.rados.0.smithi022.stdout:2863: done (1 left) 2024-09-21T09:41:14.045 INFO:tasks.rados.rados.0.smithi022.stdout:2864: read oid 4 snap -1 2024-09-21T09:41:14.045 INFO:tasks.rados.rados.0.smithi022.stdout:2864: expect (ObjNum 891 snap 284 seq_num 891) 2024-09-21T09:41:14.045 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 36 current snap is 286 2024-09-21T09:41:14.045 INFO:tasks.rados.rados.0.smithi022.stdout:2865: seq_num 919 ranges {0=598016,598016=401408} 2024-09-21T09:41:14.052 INFO:tasks.rados.rados.0.smithi022.stdout:2865: writing smithi02284960-36 from 0 to 598016 tid 1 2024-09-21T09:41:14.052 INFO:tasks.rados.rados.0.smithi022.stdout:2865: writing smithi02284960-36 from 598016 to 999424 tid 2 2024-09-21T09:41:14.052 INFO:tasks.rados.rados.0.smithi022.stdout:2866: read oid 22 snap -1 2024-09-21T09:41:14.052 INFO:tasks.rados.rados.0.smithi022.stdout:2866: expect (ObjNum 914 snap 284 seq_num 914) 2024-09-21T09:41:14.052 INFO:tasks.rados.rados.0.smithi022.stdout:2867: rollback oid 24 current snap is 286 2024-09-21T09:41:14.052 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 24 to 270 2024-09-21T09:41:14.052 INFO:tasks.rados.rados.0.smithi022.stdout:2861: expect (ObjNum 849 snap 273 seq_num 849) 2024-09-21T09:41:14.129 INFO:tasks.rados.rados.0.smithi022.stdout:2867: finishing rollback tid 1 to smithi02284960-24 2024-09-21T09:41:14.131 INFO:tasks.rados.rados.0.smithi022.stdout:2867: finishing rollback tid 2 to smithi02284960-24 2024-09-21T09:41:14.131 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 24 v 1688 (ObjNum 835 snap 269 seq_num 835) dirty exists 2024-09-21T09:41:14.131 INFO:tasks.rados.rados.0.smithi022.stdout:2865: finishing write tid 1 to smithi02284960-36 2024-09-21T09:41:14.131 INFO:tasks.rados.rados.0.smithi022.stdout:2865: finishing write tid 2 to smithi02284960-36 2024-09-21T09:41:14.131 INFO:tasks.rados.rados.0.smithi022.stdout:2865: finishing write tid 3 to smithi02284960-36 2024-09-21T09:41:14.131 INFO:tasks.rados.rados.0.smithi022.stdout:2861: done (4 left) 2024-09-21T09:41:14.131 INFO:tasks.rados.rados.0.smithi022.stdout:2867: done (3 left) 2024-09-21T09:41:14.131 INFO:tasks.rados.rados.0.smithi022.stdout:2868: rmattr oid 5 current snap is 286 2024-09-21T09:41:14.131 INFO:tasks.rados.rados.0.smithi022.stdout:2866: expect (ObjNum 914 snap 284 seq_num 914) 2024-09-21T09:41:14.182 INFO:tasks.rados.rados.0.smithi022.stdout:2865: finishing write tid 4 to smithi02284960-36 2024-09-21T09:41:14.182 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 36 v 1414 (ObjNum 919 snap 286 seq_num 919) dirty exists 2024-09-21T09:41:14.182 INFO:tasks.rados.rados.0.smithi022.stdout:2865: left oid 36 (ObjNum 919 snap 286 seq_num 919) 2024-09-21T09:41:14.182 INFO:tasks.rados.rados.0.smithi022.stdout:2864: expect (ObjNum 891 snap 284 seq_num 891) 2024-09-21T09:41:14.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:14 smithi064 ceph-mon[86709]: pgmap v572: 121 pgs: 121 active+clean; 244 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 574 B/s rd, 1.4 MiB/s wr, 6 op/s 2024-09-21T09:41:14.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:14 smithi064 ceph-mon[86709]: osdmap e644: 8 total, 8 up, 8 in 2024-09-21T09:41:14.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:14 smithi064 ceph-mon[86709]: osdmap e645: 8 total, 8 up, 8 in 2024-09-21T09:41:14.557 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:14 smithi022 ceph-mon[96736]: pgmap v572: 121 pgs: 121 active+clean; 244 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 574 B/s rd, 1.4 MiB/s wr, 6 op/s 2024-09-21T09:41:14.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:14 smithi022 ceph-mon[96736]: osdmap e644: 8 total, 8 up, 8 in 2024-09-21T09:41:14.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:14 smithi022 ceph-mon[96736]: osdmap e645: 8 total, 8 up, 8 in 2024-09-21T09:41:14.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:14 smithi022 ceph-mon[98615]: pgmap v572: 121 pgs: 121 active+clean; 244 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 574 B/s rd, 1.4 MiB/s wr, 6 op/s 2024-09-21T09:41:14.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:14 smithi022 ceph-mon[98615]: osdmap e644: 8 total, 8 up, 8 in 2024-09-21T09:41:14.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:14 smithi022 ceph-mon[98615]: osdmap e645: 8 total, 8 up, 8 in 2024-09-21T09:41:15.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:15 smithi064 ceph-mon[86709]: pgmap v575: 121 pgs: 121 active+clean; 243 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 9.9 MiB/s rd, 172 KiB/s wr, 2 op/s 2024-09-21T09:41:15.557 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:15 smithi022 ceph-mon[98615]: pgmap v575: 121 pgs: 121 active+clean; 243 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 9.9 MiB/s rd, 172 KiB/s wr, 2 op/s 2024-09-21T09:41:15.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:15 smithi022 ceph-mon[96736]: pgmap v575: 121 pgs: 121 active+clean; 243 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 9.9 MiB/s rd, 172 KiB/s wr, 2 op/s 2024-09-21T09:41:16.057 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:41:15 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:41:15] "GET /metrics HTTP/1.1" 200 36353 "" "Prometheus/2.51.0" 2024-09-21T09:41:16.662 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 5 v 1657 (ObjNum 672 snap 208 seq_num 672) dirty exists 2024-09-21T09:41:16.662 INFO:tasks.rados.rados.0.smithi022.stdout:2864: done (3 left) 2024-09-21T09:41:16.662 INFO:tasks.rados.rados.0.smithi022.stdout:2865: done (2 left) 2024-09-21T09:41:16.663 INFO:tasks.rados.rados.0.smithi022.stdout:2866: done (1 left) 2024-09-21T09:41:16.663 INFO:tasks.rados.rados.0.smithi022.stdout:2868: done (0 left) 2024-09-21T09:41:16.663 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 20 current snap is 286 2024-09-21T09:41:16.663 INFO:tasks.rados.rados.0.smithi022.stdout:2869: seq_num 920 ranges {7569408=614400,8183808=401408} 2024-09-21T09:41:16.706 INFO:tasks.rados.rados.0.smithi022.stdout:2869: writing smithi02284960-20 from 7569408 to 8183808 tid 1 2024-09-21T09:41:16.709 INFO:tasks.rados.rados.0.smithi022.stdout:2869: writing smithi02284960-20 from 8183808 to 8585216 tid 2 2024-09-21T09:41:16.709 INFO:tasks.rados.rados.0.smithi022.stdout:2870: copy_from oid 48 from oid 37 current snap is 286 2024-09-21T09:41:16.709 INFO:tasks.rados.rados.0.smithi022.stdout:2871: read oid 39 snap -1 2024-09-21T09:41:16.709 INFO:tasks.rados.rados.0.smithi022.stdout:2871: expect (ObjNum 911 snap 284 seq_num 911) 2024-09-21T09:41:16.709 INFO:tasks.rados.rados.0.smithi022.stdout:2872: setattr oid 38 current snap is 286 2024-09-21T09:41:16.712 INFO:tasks.rados.rados.0.smithi022.stdout:2873: rmattr oid 10 current snap is 286 2024-09-21T09:41:16.712 INFO:tasks.rados.rados.0.smithi022.stdout:2873: done (4 left) 2024-09-21T09:41:16.712 INFO:tasks.rados.rados.0.smithi022.stdout:2874: read oid 23 snap 264 2024-09-21T09:41:16.712 INFO:tasks.rados.rados.0.smithi022.stdout:2874: expect deleted 2024-09-21T09:41:16.713 INFO:tasks.rados.rados.0.smithi022.stdout:2875: snap_create 2024-09-21T09:41:16.714 INFO:tasks.rados.rados.0.smithi022.stdout:2869: finishing write tid 1 to smithi02284960-20 2024-09-21T09:41:16.714 INFO:tasks.rados.rados.0.smithi022.stdout:2871: expect (ObjNum 911 snap 284 seq_num 911) 2024-09-21T09:41:16.738 INFO:tasks.rados.rados.0.smithi022.stdout:2869: finishing write tid 2 to smithi02284960-20 2024-09-21T09:41:16.738 INFO:tasks.rados.rados.0.smithi022.stdout:2869: finishing write tid 3 to smithi02284960-20 2024-09-21T09:41:16.738 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 38 v 1431 (ObjNum 839 snap 269 seq_num 839) dirty exists 2024-09-21T09:41:16.738 INFO:tasks.rados.rados.0.smithi022.stdout:2869: finishing write tid 4 to smithi02284960-20 2024-09-21T09:41:16.738 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 20 v 1780 (ObjNum 920 snap 286 seq_num 920) dirty exists 2024-09-21T09:41:16.738 INFO:tasks.rados.rados.0.smithi022.stdout:2869: left oid 20 (ObjNum 920 snap 286 seq_num 920) 2024-09-21T09:41:16.739 INFO:tasks.rados.rados.0.smithi022.stdout:2870: finishing copy_from to smithi02284960-48 2024-09-21T09:41:16.739 INFO:tasks.rados.rados.0.smithi022.stdout:2870: got expected ENOENT (src dne) 2024-09-21T09:41:16.739 INFO:tasks.rados.rados.0.smithi022.stdout:2870: finishing copy_from racing read to smithi02284960-48 2024-09-21T09:41:17.482 INFO:tasks.rados.rados.0.smithi022.stdout:2869: done (5 left) 2024-09-21T09:41:17.483 INFO:tasks.rados.rados.0.smithi022.stdout:2870: done (4 left) 2024-09-21T09:41:17.483 INFO:tasks.rados.rados.0.smithi022.stdout:2871: done (3 left) 2024-09-21T09:41:17.483 INFO:tasks.rados.rados.0.smithi022.stdout:2872: done (2 left) 2024-09-21T09:41:17.483 INFO:tasks.rados.rados.0.smithi022.stdout:2874: done (1 left) 2024-09-21T09:41:17.483 INFO:tasks.rados.rados.0.smithi022.stdout:2875: done (0 left) 2024-09-21T09:41:17.483 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 15 current snap is 287 2024-09-21T09:41:17.483 INFO:tasks.rados.rados.0.smithi022.stdout:2876: seq_num 921 ranges {5881856=630784,6512640=401408} 2024-09-21T09:41:17.518 INFO:tasks.rados.rados.0.smithi022.stdout:2876: writing smithi02284960-15 from 5881856 to 6512640 tid 1 2024-09-21T09:41:17.520 INFO:tasks.rados.rados.0.smithi022.stdout:2876: writing smithi02284960-15 from 6512640 to 6914048 tid 2 2024-09-21T09:41:17.520 INFO:tasks.rados.rados.0.smithi022.stdout:2877: delete oid 23 current snap is 287 2024-09-21T09:41:17.525 INFO:tasks.rados.rados.0.smithi022.stdout:2877: done (1 left) 2024-09-21T09:41:17.525 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 28 current snap is 287 2024-09-21T09:41:17.525 INFO:tasks.rados.rados.0.smithi022.stdout:2878: seq_num 923 ranges {6037504=663552,6701056=401408} 2024-09-21T09:41:17.561 INFO:tasks.rados.rados.0.smithi022.stdout:2878: writing smithi02284960-28 from 6037504 to 6701056 tid 1 2024-09-21T09:41:17.564 INFO:tasks.rados.rados.0.smithi022.stdout:2878: writing smithi02284960-28 from 6701056 to 7102464 tid 2 2024-09-21T09:41:17.564 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 37 current snap is 287 2024-09-21T09:41:17.564 INFO:tasks.rados.rados.0.smithi022.stdout:2876: finishing write tid 1 to smithi02284960-15 2024-09-21T09:41:17.564 INFO:tasks.rados.rados.0.smithi022.stdout:2879: seq_num 924 ranges {0=679936,679936=401408} 2024-09-21T09:41:17.568 INFO:tasks.rados.rados.0.smithi022.stdout:2879: writing smithi02284960-37 from 0 to 679936 tid 1 2024-09-21T09:41:17.571 INFO:tasks.rados.rados.0.smithi022.stdout:2879: writing smithi02284960-37 from 679936 to 1081344 tid 2 2024-09-21T09:41:17.571 INFO:tasks.rados.rados.0.smithi022.stdout:2880: read oid 42 snap -1 2024-09-21T09:41:17.571 INFO:tasks.rados.rados.0.smithi022.stdout:2880: expect (ObjNum 884 snap 282 seq_num 884) 2024-09-21T09:41:17.571 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 48 current snap is 287 2024-09-21T09:41:17.572 INFO:tasks.rados.rados.0.smithi022.stdout:2876: finishing write tid 2 to smithi02284960-15 2024-09-21T09:41:17.572 INFO:tasks.rados.rados.0.smithi022.stdout:2876: finishing write tid 3 to smithi02284960-15 2024-09-21T09:41:17.572 INFO:tasks.rados.rados.0.smithi022.stdout:2876: finishing write tid 4 to smithi02284960-15 2024-09-21T09:41:17.572 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 15 v 1696 (ObjNum 921 snap 287 seq_num 921) dirty exists 2024-09-21T09:41:17.572 INFO:tasks.rados.rados.0.smithi022.stdout:2876: left oid 15 (ObjNum 921 snap 287 seq_num 921) 2024-09-21T09:41:17.572 INFO:tasks.rados.rados.0.smithi022.stdout:2881: seq_num 925 ranges {2400256=696320,3096576=401408} 2024-09-21T09:41:17.588 INFO:tasks.rados.rados.0.smithi022.stdout:2881: writing smithi02284960-48 from 2400256 to 3096576 tid 1 2024-09-21T09:41:17.590 INFO:tasks.rados.rados.0.smithi022.stdout:2881: writing smithi02284960-48 from 3096576 to 3497984 tid 2 2024-09-21T09:41:17.591 INFO:tasks.rados.rados.0.smithi022.stdout:2876: done (4 left) 2024-09-21T09:41:17.591 INFO:tasks.rados.rados.0.smithi022.stdout:2882: rollback oid 17 current snap is 287 2024-09-21T09:41:17.591 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 17 to 279 2024-09-21T09:41:17.591 INFO:tasks.rados.rados.0.smithi022.stdout:2878: finishing write tid 1 to smithi02284960-28 2024-09-21T09:41:17.591 INFO:tasks.rados.rados.0.smithi022.stdout:2878: finishing write tid 2 to smithi02284960-28 2024-09-21T09:41:17.591 INFO:tasks.rados.rados.0.smithi022.stdout:2878: finishing write tid 3 to smithi02284960-28 2024-09-21T09:41:17.591 INFO:tasks.rados.rados.0.smithi022.stdout:2878: finishing write tid 4 to smithi02284960-28 2024-09-21T09:41:17.591 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 28 v 1600 (ObjNum 923 snap 287 seq_num 923) dirty exists 2024-09-21T09:41:17.591 INFO:tasks.rados.rados.0.smithi022.stdout:2878: left oid 28 (ObjNum 923 snap 287 seq_num 923) 2024-09-21T09:41:17.591 INFO:tasks.rados.rados.0.smithi022.stdout:2879: finishing write tid 1 to smithi02284960-37 2024-09-21T09:41:17.591 INFO:tasks.rados.rados.0.smithi022.stdout:2879: finishing write tid 2 to smithi02284960-37 2024-09-21T09:41:17.591 INFO:tasks.rados.rados.0.smithi022.stdout:2879: finishing write tid 3 to smithi02284960-37 2024-09-21T09:41:17.592 INFO:tasks.rados.rados.0.smithi022.stdout:2878: done (4 left) 2024-09-21T09:41:17.592 INFO:tasks.rados.rados.0.smithi022.stdout:2883: snap_remove snap 275 2024-09-21T09:41:17.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:17 smithi022 ceph-mon[98615]: pgmap v576: 121 pgs: 121 active+clean; 248 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 20 MiB/s rd, 312 KiB/s wr, 7 op/s 2024-09-21T09:41:17.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:17 smithi022 ceph-mon[96736]: pgmap v576: 121 pgs: 121 active+clean; 248 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 20 MiB/s rd, 312 KiB/s wr, 7 op/s 2024-09-21T09:41:17.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:17 smithi064 ceph-mon[86709]: pgmap v576: 121 pgs: 121 active+clean; 248 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 20 MiB/s rd, 312 KiB/s wr, 7 op/s 2024-09-21T09:41:18.490 INFO:tasks.rados.rados.0.smithi022.stdout:2883: done (4 left) 2024-09-21T09:41:18.490 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 6 current snap is 287 2024-09-21T09:41:18.490 INFO:tasks.rados.rados.0.smithi022.stdout:2884: seq_num 926 ranges {0=712704,712704=401408} 2024-09-21T09:41:18.495 INFO:tasks.rados.rados.0.smithi022.stdout:2884: writing smithi02284960-6 from 0 to 712704 tid 1 2024-09-21T09:41:18.498 INFO:tasks.rados.rados.0.smithi022.stdout:2884: writing smithi02284960-6 from 712704 to 1114112 tid 2 2024-09-21T09:41:18.498 INFO:tasks.rados.rados.0.smithi022.stdout:2885: snap_create 2024-09-21T09:41:18.498 INFO:tasks.rados.rados.0.smithi022.stdout:2879: finishing write tid 4 to smithi02284960-37 2024-09-21T09:41:18.498 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 37 v 1699 (ObjNum 924 snap 287 seq_num 924) dirty exists 2024-09-21T09:41:18.498 INFO:tasks.rados.rados.0.smithi022.stdout:2879: left oid 37 (ObjNum 924 snap 287 seq_num 924) 2024-09-21T09:41:18.498 INFO:tasks.rados.rados.0.smithi022.stdout:2880: expect (ObjNum 884 snap 282 seq_num 884) 2024-09-21T09:41:18.747 INFO:tasks.rados.rados.0.smithi022.stdout:2881: finishing write tid 1 to smithi02284960-48 2024-09-21T09:41:18.747 INFO:tasks.rados.rados.0.smithi022.stdout:2881: finishing write tid 2 to smithi02284960-48 2024-09-21T09:41:18.747 INFO:tasks.rados.rados.0.smithi022.stdout:2881: finishing write tid 3 to smithi02284960-48 2024-09-21T09:41:18.747 INFO:tasks.rados.rados.0.smithi022.stdout:2882: finishing rollback tid 0 to smithi02284960-17 2024-09-21T09:41:18.747 INFO:tasks.rados.rados.0.smithi022.stdout:2882: finishing rollback tid 1 to smithi02284960-17 2024-09-21T09:41:18.747 INFO:tasks.rados.rados.0.smithi022.stdout:2881: finishing write tid 4 to smithi02284960-48 2024-09-21T09:41:18.747 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 48 v 1784 (ObjNum 925 snap 287 seq_num 925) dirty exists 2024-09-21T09:41:18.747 INFO:tasks.rados.rados.0.smithi022.stdout:2881: left oid 48 (ObjNum 925 snap 287 seq_num 925) 2024-09-21T09:41:18.747 INFO:tasks.rados.rados.0.smithi022.stdout:2882: finishing rollback tid 2 to smithi02284960-17 2024-09-21T09:41:18.747 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 17 v 1313 (ObjNum 842 snap 269 seq_num 842) dirty exists 2024-09-21T09:41:18.748 INFO:tasks.rados.rados.0.smithi022.stdout:2884: finishing write tid 1 to smithi02284960-6 2024-09-21T09:41:18.748 INFO:tasks.rados.rados.0.smithi022.stdout:2884: finishing write tid 2 to smithi02284960-6 2024-09-21T09:41:18.748 INFO:tasks.rados.rados.0.smithi022.stdout:2884: finishing write tid 3 to smithi02284960-6 2024-09-21T09:41:18.748 INFO:tasks.rados.rados.0.smithi022.stdout:2884: finishing write tid 4 to smithi02284960-6 2024-09-21T09:41:18.748 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 6 v 1787 (ObjNum 926 snap 287 seq_num 926) dirty exists 2024-09-21T09:41:18.748 INFO:tasks.rados.rados.0.smithi022.stdout:2884: left oid 6 (ObjNum 926 snap 287 seq_num 926) 2024-09-21T09:41:18.807 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:18 smithi022 ceph-mon[98615]: osdmap e646: 8 total, 8 up, 8 in 2024-09-21T09:41:18.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:18 smithi022 ceph-mon[96736]: osdmap e646: 8 total, 8 up, 8 in 2024-09-21T09:41:18.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:18 smithi064 ceph-mon[86709]: osdmap e646: 8 total, 8 up, 8 in 2024-09-21T09:41:19.044 INFO:tasks.rados.rados.0.smithi022.stdout:2879: done (5 left) 2024-09-21T09:41:19.044 INFO:tasks.rados.rados.0.smithi022.stdout:2880: done (4 left) 2024-09-21T09:41:19.044 INFO:tasks.rados.rados.0.smithi022.stdout:2881: done (3 left) 2024-09-21T09:41:19.044 INFO:tasks.rados.rados.0.smithi022.stdout:2882: done (2 left) 2024-09-21T09:41:19.044 INFO:tasks.rados.rados.0.smithi022.stdout:2884: done (1 left) 2024-09-21T09:41:19.044 INFO:tasks.rados.rados.0.smithi022.stdout:2885: done (0 left) 2024-09-21T09:41:19.044 INFO:tasks.rados.rados.0.smithi022.stdout:2886: setattr oid 17 current snap is 288 2024-09-21T09:41:19.045 INFO:tasks.rados.rados.0.smithi022.stdout:2887: read oid 6 snap -1 2024-09-21T09:41:19.045 INFO:tasks.rados.rados.0.smithi022.stdout:2887: expect (ObjNum 926 snap 287 seq_num 926) 2024-09-21T09:41:19.045 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 26 current snap is 288 2024-09-21T09:41:19.045 INFO:tasks.rados.rados.0.smithi022.stdout:2888: seq_num 927 ranges {1818624=729088,2547712=401408} 2024-09-21T09:41:19.059 INFO:tasks.rados.rados.0.smithi022.stdout:2888: writing smithi02284960-26 from 1818624 to 2547712 tid 1 2024-09-21T09:41:19.061 INFO:tasks.rados.rados.0.smithi022.stdout:2888: writing smithi02284960-26 from 2547712 to 2949120 tid 2 2024-09-21T09:41:19.061 INFO:tasks.rados.rados.0.smithi022.stdout:2889: snap_remove snap 286 2024-09-21T09:41:19.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:19 smithi022 ceph-mon[98615]: pgmap v578: 121 pgs: 121 active+clean; 250 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 20 MiB/s rd, 648 KiB/s wr, 12 op/s 2024-09-21T09:41:19.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:19 smithi022 ceph-mon[98615]: osdmap e647: 8 total, 8 up, 8 in 2024-09-21T09:41:19.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:19 smithi022 ceph-mon[98615]: osdmap e648: 8 total, 8 up, 8 in 2024-09-21T09:41:19.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:19 smithi022 ceph-mon[96736]: pgmap v578: 121 pgs: 121 active+clean; 250 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 20 MiB/s rd, 648 KiB/s wr, 12 op/s 2024-09-21T09:41:19.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:19 smithi022 ceph-mon[96736]: osdmap e647: 8 total, 8 up, 8 in 2024-09-21T09:41:19.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:19 smithi022 ceph-mon[96736]: osdmap e648: 8 total, 8 up, 8 in 2024-09-21T09:41:19.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:19 smithi064 ceph-mon[86709]: pgmap v578: 121 pgs: 121 active+clean; 250 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 20 MiB/s rd, 648 KiB/s wr, 12 op/s 2024-09-21T09:41:19.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:19 smithi064 ceph-mon[86709]: osdmap e647: 8 total, 8 up, 8 in 2024-09-21T09:41:19.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:19 smithi064 ceph-mon[86709]: osdmap e648: 8 total, 8 up, 8 in 2024-09-21T09:41:20.046 INFO:tasks.rados.rados.0.smithi022.stdout:2889: done (3 left) 2024-09-21T09:41:20.047 INFO:tasks.rados.rados.0.smithi022.stdout:2890: rollback oid 42 current snap is 288 2024-09-21T09:41:20.047 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 42 to 285 2024-09-21T09:41:20.047 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 17 v 1314 (ObjNum 842 snap 269 seq_num 842) dirty exists 2024-09-21T09:41:20.047 INFO:tasks.rados.rados.0.smithi022.stdout:2887: expect (ObjNum 926 snap 287 seq_num 926) 2024-09-21T09:41:20.081 INFO:tasks.rados.rados.0.smithi022.stdout:2888: finishing write tid 1 to smithi02284960-26 2024-09-21T09:41:20.081 INFO:tasks.rados.rados.0.smithi022.stdout:2888: finishing write tid 2 to smithi02284960-26 2024-09-21T09:41:20.081 INFO:tasks.rados.rados.0.smithi022.stdout:2888: finishing write tid 3 to smithi02284960-26 2024-09-21T09:41:20.081 INFO:tasks.rados.rados.0.smithi022.stdout:2888: finishing write tid 4 to smithi02284960-26 2024-09-21T09:41:20.081 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 26 v 1668 (ObjNum 927 snap 288 seq_num 927) dirty exists 2024-09-21T09:41:20.081 INFO:tasks.rados.rados.0.smithi022.stdout:2888: left oid 26 (ObjNum 927 snap 288 seq_num 927) 2024-09-21T09:41:20.081 INFO:tasks.rados.rados.0.smithi022.stdout:2886: done (3 left) 2024-09-21T09:41:20.081 INFO:tasks.rados.rados.0.smithi022.stdout:2887: done (2 left) 2024-09-21T09:41:20.081 INFO:tasks.rados.rados.0.smithi022.stdout:2888: done (1 left) 2024-09-21T09:41:20.082 INFO:tasks.rados.rados.0.smithi022.stdout:2891: read oid 13 snap -1 2024-09-21T09:41:20.082 INFO:tasks.rados.rados.0.smithi022.stdout:2891: expect (ObjNum 907 snap 284 seq_num 907) 2024-09-21T09:41:20.082 INFO:tasks.rados.rados.0.smithi022.stdout:2892: read oid 36 snap -1 2024-09-21T09:41:20.082 INFO:tasks.rados.rados.0.smithi022.stdout:2892: expect (ObjNum 919 snap 286 seq_num 919) 2024-09-21T09:41:20.082 INFO:tasks.rados.rados.0.smithi022.stdout:2893: delete oid 10 current snap is 288 2024-09-21T09:41:20.082 INFO:tasks.rados.rados.0.smithi022.stdout:2890: finishing rollback tid 0 to smithi02284960-42 2024-09-21T09:41:20.082 INFO:tasks.rados.rados.0.smithi022.stdout:2890: finishing rollback tid 1 to smithi02284960-42 2024-09-21T09:41:20.082 INFO:tasks.rados.rados.0.smithi022.stdout:2890: finishing rollback tid 2 to smithi02284960-42 2024-09-21T09:41:20.082 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 42 v 1671 (ObjNum 884 snap 282 seq_num 884) dirty exists 2024-09-21T09:41:20.085 INFO:tasks.rados.rados.0.smithi022.stdout:2890: done (3 left) 2024-09-21T09:41:20.085 INFO:tasks.rados.rados.0.smithi022.stdout:2893: done (2 left) 2024-09-21T09:41:20.085 INFO:tasks.rados.rados.0.smithi022.stdout:2894: rmattr oid 19 current snap is 288 2024-09-21T09:41:20.085 INFO:tasks.rados.rados.0.smithi022.stdout:2894: done (2 left) 2024-09-21T09:41:20.085 INFO:tasks.rados.rados.0.smithi022.stdout:2895: snap_remove snap 283 2024-09-21T09:41:20.691 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:41:20 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:41:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:41:21.061 INFO:tasks.rados.rados.0.smithi022.stdout:2895: done (2 left) 2024-09-21T09:41:21.061 INFO:tasks.rados.rados.0.smithi022.stdout:2896: rollback oid 9 current snap is 288 2024-09-21T09:41:21.061 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 9 to 264 2024-09-21T09:41:21.062 INFO:tasks.rados.rados.0.smithi022.stdout:2897: copy_from oid 33 from oid 49 current snap is 288 2024-09-21T09:41:21.062 INFO:tasks.rados.rados.0.smithi022.stdout:2898: read oid 34 snap -1 2024-09-21T09:41:21.062 INFO:tasks.rados.rados.0.smithi022.stdout:2898: expect (ObjNum 849 snap 273 seq_num 849) 2024-09-21T09:41:21.062 INFO:tasks.rados.rados.0.smithi022.stdout:2899: copy_from oid 24 from oid 43 current snap is 288 2024-09-21T09:41:21.062 INFO:tasks.rados.rados.0.smithi022.stdout:2900: read oid 39 snap -1 2024-09-21T09:41:21.062 INFO:tasks.rados.rados.0.smithi022.stdout:2900: expect (ObjNum 911 snap 284 seq_num 911) 2024-09-21T09:41:21.062 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 1 current snap is 288 2024-09-21T09:41:21.062 INFO:tasks.rados.rados.0.smithi022.stdout:2901: seq_num 929 ranges {6602752=761856,7364608=401408} 2024-09-21T09:41:21.099 INFO:tasks.rados.rados.0.smithi022.stdout:2901: writing smithi02284960-1 from 6602752 to 7364608 tid 1 2024-09-21T09:41:21.101 INFO:tasks.rados.rados.0.smithi022.stdout:2901: writing smithi02284960-1 from 7364608 to 7766016 tid 2 2024-09-21T09:41:21.101 INFO:tasks.rados.rados.0.smithi022.stdout:2892: expect (ObjNum 919 snap 286 seq_num 919) 2024-09-21T09:41:21.130 INFO:tasks.rados.rados.0.smithi022.stdout:2891: expect (ObjNum 907 snap 284 seq_num 907) 2024-09-21T09:41:21.153 INFO:tasks.rados.rados.0.smithi022.stdout:2900: expect (ObjNum 911 snap 284 seq_num 911) 2024-09-21T09:41:21.176 INFO:tasks.rados.rados.0.smithi022.stdout:2897: finishing copy_from to smithi02284960-33 2024-09-21T09:41:21.176 INFO:tasks.rados.rados.0.smithi022.stdout:2897: got expected ENOENT (src dne) 2024-09-21T09:41:21.176 INFO:tasks.rados.rados.0.smithi022.stdout:2897: finishing copy_from racing read to smithi02284960-33 2024-09-21T09:41:21.176 INFO:tasks.rados.rados.0.smithi022.stdout:2896: finishing rollback tid 0 to smithi02284960-9 2024-09-21T09:41:21.176 INFO:tasks.rados.rados.0.smithi022.stdout:2896: finishing rollback tid 1 to smithi02284960-9 2024-09-21T09:41:21.176 INFO:tasks.rados.rados.0.smithi022.stdout:2896: finishing rollback tid 2 to smithi02284960-9 2024-09-21T09:41:21.176 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 9 v 1660 (ObjNum 811 snap 259 seq_num 811) dirty exists 2024-09-21T09:41:21.176 INFO:tasks.rados.rados.0.smithi022.stdout:2898: expect (ObjNum 849 snap 273 seq_num 849) 2024-09-21T09:41:21.234 INFO:tasks.rados.rados.0.smithi022.stdout:2901: finishing write tid 1 to smithi02284960-1 2024-09-21T09:41:21.234 INFO:tasks.rados.rados.0.smithi022.stdout:2901: finishing write tid 2 to smithi02284960-1 2024-09-21T09:41:21.235 INFO:tasks.rados.rados.0.smithi022.stdout:2901: finishing write tid 3 to smithi02284960-1 2024-09-21T09:41:21.235 INFO:tasks.rados.rados.0.smithi022.stdout:2901: finishing write tid 4 to smithi02284960-1 2024-09-21T09:41:21.235 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 1 v 1655 (ObjNum 929 snap 288 seq_num 929) dirty exists 2024-09-21T09:41:21.235 INFO:tasks.rados.rados.0.smithi022.stdout:2901: left oid 1 (ObjNum 929 snap 288 seq_num 929) 2024-09-21T09:41:21.235 INFO:tasks.rados.rados.0.smithi022.stdout:2899: finishing copy_from racing read to smithi02284960-24 2024-09-21T09:41:21.235 INFO:tasks.rados.rados.0.smithi022.stdout:2891: done (7 left) 2024-09-21T09:41:21.235 INFO:tasks.rados.rados.0.smithi022.stdout:2892: done (6 left) 2024-09-21T09:41:21.235 INFO:tasks.rados.rados.0.smithi022.stdout:2896: done (5 left) 2024-09-21T09:41:21.235 INFO:tasks.rados.rados.0.smithi022.stdout:2897: done (4 left) 2024-09-21T09:41:21.235 INFO:tasks.rados.rados.0.smithi022.stdout:2898: done (3 left) 2024-09-21T09:41:21.235 INFO:tasks.rados.rados.0.smithi022.stdout:2900: done (2 left) 2024-09-21T09:41:21.235 INFO:tasks.rados.rados.0.smithi022.stdout:2901: done (1 left) 2024-09-21T09:41:21.235 INFO:tasks.rados.rados.0.smithi022.stdout:2902: setattr oid 45 current snap is 288 2024-09-21T09:41:21.236 INFO:tasks.rados.rados.0.smithi022.stdout:2899: finishing copy_from to smithi02284960-24 2024-09-21T09:41:21.236 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 24 v 1402 (ObjNum 817 snap 261 seq_num 817) dirty exists 2024-09-21T09:41:21.236 INFO:tasks.rados.rados.0.smithi022.stdout:2899: done (1 left) 2024-09-21T09:41:21.236 INFO:tasks.rados.rados.0.smithi022.stdout:2903: read oid 47 snap -1 2024-09-21T09:41:21.236 INFO:tasks.rados.rados.0.smithi022.stdout:2903: expect (ObjNum 893 snap 284 seq_num 893) 2024-09-21T09:41:21.236 INFO:tasks.rados.rados.0.smithi022.stdout:2904: rmattr oid 43 current snap is 288 2024-09-21T09:41:21.236 INFO:tasks.rados.rados.0.smithi022.stdout:2905: snap_create 2024-09-21T09:41:21.238 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 45 v 1415 (ObjNum 289 snap 0 seq_num 1) dirty exists 2024-09-21T09:41:21.239 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 43 v 1652 (ObjNum 817 snap 261 seq_num 817) dirty exists 2024-09-21T09:41:21.243 INFO:tasks.rados.rados.0.smithi022.stdout:2903: expect (ObjNum 893 snap 284 seq_num 893) 2024-09-21T09:41:21.307 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:21 smithi022 ceph-mon[98615]: osdmap e649: 8 total, 8 up, 8 in 2024-09-21T09:41:21.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:21 smithi022 ceph-mon[96736]: osdmap e649: 8 total, 8 up, 8 in 2024-09-21T09:41:21.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:21 smithi064 ceph-mon[86709]: osdmap e649: 8 total, 8 up, 8 in 2024-09-21T09:41:22.053 INFO:tasks.rados.rados.0.smithi022.stdout:2902: done (3 left) 2024-09-21T09:41:22.054 INFO:tasks.rados.rados.0.smithi022.stdout:2903: done (2 left) 2024-09-21T09:41:22.054 INFO:tasks.rados.rados.0.smithi022.stdout:2904: done (1 left) 2024-09-21T09:41:22.054 INFO:tasks.rados.rados.0.smithi022.stdout:2905: done (0 left) 2024-09-21T09:41:22.054 INFO:tasks.rados.rados.0.smithi022.stdout:2906: delete oid 33 current snap is 289 2024-09-21T09:41:22.060 INFO:tasks.rados.rados.0.smithi022.stdout:2906: done (0 left) 2024-09-21T09:41:22.060 INFO:tasks.rados.rados.0.smithi022.stdout:2907: snap_remove snap 270 2024-09-21T09:41:22.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:22 smithi022 ceph-mon[98615]: pgmap v582: 121 pgs: 121 active+clean; 253 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 817 KiB/s rd, 1.3 MiB/s wr, 11 op/s 2024-09-21T09:41:22.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:22 smithi022 ceph-mon[98615]: osdmap e650: 8 total, 8 up, 8 in 2024-09-21T09:41:22.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:22 smithi022 ceph-mon[96736]: pgmap v582: 121 pgs: 121 active+clean; 253 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 817 KiB/s rd, 1.3 MiB/s wr, 11 op/s 2024-09-21T09:41:22.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:22 smithi022 ceph-mon[96736]: osdmap e650: 8 total, 8 up, 8 in 2024-09-21T09:41:22.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:22 smithi064 ceph-mon[86709]: pgmap v582: 121 pgs: 121 active+clean; 253 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 817 KiB/s rd, 1.3 MiB/s wr, 11 op/s 2024-09-21T09:41:22.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:22 smithi064 ceph-mon[86709]: osdmap e650: 8 total, 8 up, 8 in 2024-09-21T09:41:23.071 INFO:tasks.rados.rados.0.smithi022.stdout:2907: done (0 left) 2024-09-21T09:41:23.071 INFO:tasks.rados.rados.0.smithi022.stdout:2908: snap_remove snap 288 2024-09-21T09:41:23.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:23 smithi064 ceph-mon[86709]: osdmap e651: 8 total, 8 up, 8 in 2024-09-21T09:41:23.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:23 smithi022 ceph-mon[96736]: osdmap e651: 8 total, 8 up, 8 in 2024-09-21T09:41:23.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:23 smithi022 ceph-mon[98615]: osdmap e651: 8 total, 8 up, 8 in 2024-09-21T09:41:24.072 INFO:tasks.rados.rados.0.smithi022.stdout:2908: done (0 left) 2024-09-21T09:41:24.072 INFO:tasks.rados.rados.0.smithi022.stdout:2909: read oid 40 snap -1 2024-09-21T09:41:24.072 INFO:tasks.rados.rados.0.smithi022.stdout:2909: expect deleted 2024-09-21T09:41:24.072 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 16 current snap is 289 2024-09-21T09:41:24.072 INFO:tasks.rados.rados.0.smithi022.stdout:2910: seq_num 931 ranges {0=802816,802816=401408} 2024-09-21T09:41:24.077 INFO:tasks.rados.rados.0.smithi022.stdout:2910: writing smithi02284960-16 from 0 to 802816 tid 1 2024-09-21T09:41:24.080 INFO:tasks.rados.rados.0.smithi022.stdout:2910: writing smithi02284960-16 from 802816 to 1204224 tid 2 2024-09-21T09:41:24.080 INFO:tasks.rados.rados.0.smithi022.stdout:2911: delete oid 9 current snap is 289 2024-09-21T09:41:24.084 INFO:tasks.rados.rados.0.smithi022.stdout:2909: done (2 left) 2024-09-21T09:41:24.084 INFO:tasks.rados.rados.0.smithi022.stdout:2911: done (1 left) 2024-09-21T09:41:24.084 INFO:tasks.rados.rados.0.smithi022.stdout:2912: rollback oid 33 current snap is 289 2024-09-21T09:41:24.084 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 33 to 281 2024-09-21T09:41:24.084 INFO:tasks.rados.rados.0.smithi022.stdout:2910: finishing write tid 1 to smithi02284960-16 2024-09-21T09:41:24.084 INFO:tasks.rados.rados.0.smithi022.stdout:2913: rollback oid 3 current snap is 289 2024-09-21T09:41:24.084 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 3 to 264 2024-09-21T09:41:24.085 INFO:tasks.rados.rados.0.smithi022.stdout:2914: copy_from oid 47 from oid 34 current snap is 289 2024-09-21T09:41:24.085 INFO:tasks.rados.rados.0.smithi022.stdout:2915: copy_from oid 13 from oid 17 current snap is 289 2024-09-21T09:41:24.085 INFO:tasks.rados.rados.0.smithi022.stdout:2916: snap_remove snap 284 2024-09-21T09:41:24.368 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:24 smithi022 ceph-mon[98615]: pgmap v585: 121 pgs: 121 active+clean; 272 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 4.8 MiB/s rd, 5.5 MiB/s wr, 17 op/s 2024-09-21T09:41:24.368 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:24 smithi022 ceph-mon[98615]: osdmap e652: 8 total, 8 up, 8 in 2024-09-21T09:41:24.368 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:24 smithi022 ceph-mon[96736]: pgmap v585: 121 pgs: 121 active+clean; 272 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 4.8 MiB/s rd, 5.5 MiB/s wr, 17 op/s 2024-09-21T09:41:24.368 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:24 smithi022 ceph-mon[96736]: osdmap e652: 8 total, 8 up, 8 in 2024-09-21T09:41:24.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:24 smithi064 ceph-mon[86709]: pgmap v585: 121 pgs: 121 active+clean; 272 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 4.8 MiB/s rd, 5.5 MiB/s wr, 17 op/s 2024-09-21T09:41:24.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:24 smithi064 ceph-mon[86709]: osdmap e652: 8 total, 8 up, 8 in 2024-09-21T09:41:24.823 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d65e801e-77fa-11ef-baf6-efdc52797490 -e sha1=62da54b4b83a96ca9976b7e1d0c8272a564a2208 -- bash -c 'ceph orch ps' 2024-09-21T09:41:25.079 INFO:tasks.rados.rados.0.smithi022.stdout:2916: done (5 left) 2024-09-21T09:41:25.080 INFO:tasks.rados.rados.0.smithi022.stdout:2917: delete oid 42 current snap is 289 2024-09-21T09:41:25.080 INFO:tasks.rados.rados.0.smithi022.stdout:2910: finishing write tid 2 to smithi02284960-16 2024-09-21T09:41:25.080 INFO:tasks.rados.rados.0.smithi022.stdout:2910: finishing write tid 3 to smithi02284960-16 2024-09-21T09:41:25.080 INFO:tasks.rados.rados.0.smithi022.stdout:2910: finishing write tid 4 to smithi02284960-16 2024-09-21T09:41:25.080 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 16 v 1514 (ObjNum 931 snap 289 seq_num 931) dirty exists 2024-09-21T09:41:25.080 INFO:tasks.rados.rados.0.smithi022.stdout:2910: left oid 16 (ObjNum 931 snap 289 seq_num 931) 2024-09-21T09:41:25.080 INFO:tasks.rados.rados.0.smithi022.stdout:2913: finishing rollback tid 0 to smithi02284960-3 2024-09-21T09:41:25.080 INFO:tasks.rados.rados.0.smithi022.stdout:2912: finishing rollback tid 1 to smithi02284960-33 2024-09-21T09:41:25.080 INFO:tasks.rados.rados.0.smithi022.stdout:2912: finishing rollback tid 2 to smithi02284960-33 2024-09-21T09:41:25.080 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 33 v 1819 (ObjNum 873 snap 280 seq_num 873) dirty exists 2024-09-21T09:41:25.080 INFO:tasks.rados.rados.0.smithi022.stdout:2913: finishing rollback tid 1 to smithi02284960-3 2024-09-21T09:41:25.080 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 3 v 618 (ObjNum 161 snap 0 seq_num 0) dirty dne 2024-09-21T09:41:25.080 INFO:tasks.rados.rados.0.smithi022.stdout:2914: finishing copy_from racing read to smithi02284960-47 2024-09-21T09:41:25.080 INFO:tasks.rados.rados.0.smithi022.stdout:2914: finishing copy_from to smithi02284960-47 2024-09-21T09:41:25.081 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 47 v 1422 (ObjNum 849 snap 273 seq_num 849) dirty exists 2024-09-21T09:41:25.081 INFO:tasks.rados.rados.0.smithi022.stdout:2915: finishing copy_from racing read to smithi02284960-13 2024-09-21T09:41:25.081 INFO:tasks.rados.rados.0.smithi022.stdout:2915: finishing copy_from to smithi02284960-13 2024-09-21T09:41:25.081 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 13 v 1314 (ObjNum 842 snap 269 seq_num 842) dirty exists 2024-09-21T09:41:25.091 INFO:tasks.rados.rados.0.smithi022.stdout:2910: done (5 left) 2024-09-21T09:41:25.091 INFO:tasks.rados.rados.0.smithi022.stdout:2912: done (4 left) 2024-09-21T09:41:25.092 INFO:tasks.rados.rados.0.smithi022.stdout:2913: done (3 left) 2024-09-21T09:41:25.092 INFO:tasks.rados.rados.0.smithi022.stdout:2914: done (2 left) 2024-09-21T09:41:25.092 INFO:tasks.rados.rados.0.smithi022.stdout:2915: done (1 left) 2024-09-21T09:41:25.092 INFO:tasks.rados.rados.0.smithi022.stdout:2917: done (0 left) 2024-09-21T09:41:25.092 INFO:tasks.rados.rados.0.smithi022.stdout:2918: delete oid 21 current snap is 289 2024-09-21T09:41:25.094 INFO:tasks.rados.rados.0.smithi022.stdout:2918: done (0 left) 2024-09-21T09:41:25.094 INFO:tasks.rados.rados.0.smithi022.stdout:2919: copy_from oid 13 from oid 17 current snap is 289 2024-09-21T09:41:25.096 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 26 current snap is 289 2024-09-21T09:41:25.096 INFO:tasks.rados.rados.0.smithi022.stdout:2920: seq_num 935 ranges {2949120=466944,3416064=720896,4136960=81920} 2024-09-21T09:41:25.114 INFO:tasks.rados.rados.0.smithi022.stdout:2920: writing smithi02284960-26 from 2949120 to 3416064 tid 1 2024-09-21T09:41:25.118 INFO:tasks.rados.rados.0.smithi022.stdout:2920: writing smithi02284960-26 from 3416064 to 4136960 tid 2 2024-09-21T09:41:25.118 INFO:tasks.rados.rados.0.smithi022.stdout:2920: writing smithi02284960-26 from 4136960 to 4218880 tid 3 2024-09-21T09:41:25.118 INFO:tasks.rados.rados.0.smithi022.stdout:2921: snap_create 2024-09-21T09:41:25.118 INFO:tasks.rados.rados.0.smithi022.stdout:2920: finishing write tid 1 to smithi02284960-26 2024-09-21T09:41:25.137 INFO:tasks.rados.rados.0.smithi022.stdout:2920: finishing write tid 2 to smithi02284960-26 2024-09-21T09:41:25.137 INFO:tasks.rados.rados.0.smithi022.stdout:2920: finishing write tid 3 to smithi02284960-26 2024-09-21T09:41:25.137 INFO:tasks.rados.rados.0.smithi022.stdout:2920: finishing write tid 4 to smithi02284960-26 2024-09-21T09:41:25.138 INFO:tasks.rados.rados.0.smithi022.stdout:2920: finishing write tid 5 to smithi02284960-26 2024-09-21T09:41:25.138 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 26 v 1676 (ObjNum 935 snap 289 seq_num 935) dirty exists 2024-09-21T09:41:25.138 INFO:tasks.rados.rados.0.smithi022.stdout:2920: left oid 26 (ObjNum 935 snap 289 seq_num 935) 2024-09-21T09:41:25.147 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:41:25.177 INFO:tasks.rados.rados.0.smithi022.stdout:2919: finishing copy_from racing read to smithi02284960-13 2024-09-21T09:41:25.179 INFO:tasks.rados.rados.0.smithi022.stdout:2919: finishing copy_from to smithi02284960-13 2024-09-21T09:41:25.179 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 13 v 1314 (ObjNum 842 snap 269 seq_num 842) dirty exists 2024-09-21T09:41:25.392 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:25 smithi022 ceph-mon[98615]: from='client.44483 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:25.392 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:25 smithi022 ceph-mon[98615]: osdmap e653: 8 total, 8 up, 8 in 2024-09-21T09:41:25.392 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:25 smithi022 ceph-mon[96736]: from='client.44483 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:25.392 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:25 smithi022 ceph-mon[96736]: osdmap e653: 8 total, 8 up, 8 in 2024-09-21T09:41:25.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:25 smithi064 ceph-mon[86709]: from='client.44483 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:25.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:25 smithi064 ceph-mon[86709]: osdmap e653: 8 total, 8 up, 8 in 2024-09-21T09:41:26.050 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:41:25 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:41:25] "GET /metrics HTTP/1.1" 200 36353 "" "Prometheus/2.51.0" 2024-09-21T09:41:26.082 INFO:tasks.rados.rados.0.smithi022.stdout:2919: done (2 left) 2024-09-21T09:41:26.082 INFO:tasks.rados.rados.0.smithi022.stdout:2920: done (1 left) 2024-09-21T09:41:26.083 INFO:tasks.rados.rados.0.smithi022.stdout:2921: done (0 left) 2024-09-21T09:41:26.083 INFO:tasks.rados.rados.0.smithi022.stdout:2922: copy_from oid 24 from oid 35 current snap is 290 2024-09-21T09:41:26.083 INFO:tasks.rados.rados.0.smithi022.stdout:2923: read oid 32 snap 285 2024-09-21T09:41:26.083 INFO:tasks.rados.rados.0.smithi022.stdout:2923: expect (ObjNum 868 snap 280 seq_num 868) 2024-09-21T09:41:26.083 INFO:tasks.rados.rados.0.smithi022.stdout:2924: read oid 38 snap -1 2024-09-21T09:41:26.083 INFO:tasks.rados.rados.0.smithi022.stdout:2924: expect (ObjNum 839 snap 269 seq_num 839) 2024-09-21T09:41:26.083 INFO:tasks.rados.rados.0.smithi022.stdout:2925: read oid 36 snap -1 2024-09-21T09:41:26.083 INFO:tasks.rados.rados.0.smithi022.stdout:2925: expect (ObjNum 919 snap 286 seq_num 919) 2024-09-21T09:41:26.083 INFO:tasks.rados.rados.0.smithi022.stdout:2926: snap_create 2024-09-21T09:41:26.088 INFO:tasks.rados.rados.0.smithi022.stdout:2923: expect (ObjNum 868 snap 280 seq_num 868) 2024-09-21T09:41:26.094 INFO:tasks.rados.rados.0.smithi022.stdout:2925: expect (ObjNum 919 snap 286 seq_num 919) 2024-09-21T09:41:26.122 INFO:tasks.rados.rados.0.smithi022.stdout:2924: expect (ObjNum 839 snap 269 seq_num 839) 2024-09-21T09:41:26.181 INFO:tasks.rados.rados.0.smithi022.stdout:2922: finishing copy_from racing read to smithi02284960-24 2024-09-21T09:41:26.181 INFO:tasks.rados.rados.0.smithi022.stdout:2922: finishing copy_from to smithi02284960-24 2024-09-21T09:41:26.181 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 24 v 1594 (ObjNum 877 snap 282 seq_num 877) dirty exists 2024-09-21T09:41:26.307 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:26 smithi022 ceph-mon[98615]: pgmap v588: 121 pgs: 121 active+clean; 268 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 6.2 MiB/s rd, 4.7 MiB/s wr, 19 op/s 2024-09-21T09:41:26.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:26 smithi022 ceph-mon[98615]: osdmap e654: 8 total, 8 up, 8 in 2024-09-21T09:41:26.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:26 smithi022 ceph-mon[96736]: pgmap v588: 121 pgs: 121 active+clean; 268 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 6.2 MiB/s rd, 4.7 MiB/s wr, 19 op/s 2024-09-21T09:41:26.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:26 smithi022 ceph-mon[96736]: osdmap e654: 8 total, 8 up, 8 in 2024-09-21T09:41:26.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:26 smithi064 ceph-mon[86709]: pgmap v588: 121 pgs: 121 active+clean; 268 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 6.2 MiB/s rd, 4.7 MiB/s wr, 19 op/s 2024-09-21T09:41:26.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:26 smithi064 ceph-mon[86709]: osdmap e654: 8 total, 8 up, 8 in 2024-09-21T09:41:27.087 INFO:tasks.rados.rados.0.smithi022.stdout:2922: done (4 left) 2024-09-21T09:41:27.090 INFO:tasks.rados.rados.0.smithi022.stdout:2923: done (3 left) 2024-09-21T09:41:27.090 INFO:tasks.rados.rados.0.smithi022.stdout:2924: done (2 left) 2024-09-21T09:41:27.090 INFO:tasks.rados.rados.0.smithi022.stdout:2925: done (1 left) 2024-09-21T09:41:27.090 INFO:tasks.rados.rados.0.smithi022.stdout:2926: done (0 left) 2024-09-21T09:41:27.090 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 49 current snap is 291 2024-09-21T09:41:27.090 INFO:tasks.rados.rados.0.smithi022.stdout:2927: seq_num 936 ranges {0=483328,483328=655360,1138688=147456} 2024-09-21T09:41:27.090 INFO:tasks.rados.rados.0.smithi022.stdout:2927: writing smithi02284960-49 from 0 to 483328 tid 1 2024-09-21T09:41:27.093 INFO:tasks.rados.rados.0.smithi022.stdout:2927: writing smithi02284960-49 from 483328 to 1138688 tid 2 2024-09-21T09:41:27.095 INFO:tasks.rados.rados.0.smithi022.stdout:2927: writing smithi02284960-49 from 1138688 to 1286144 tid 3 2024-09-21T09:41:27.095 INFO:tasks.rados.rados.0.smithi022.stdout:2928: copy_from oid 47 from oid 31 current snap is 291 2024-09-21T09:41:27.095 INFO:tasks.rados.rados.0.smithi022.stdout:2927: finishing write tid 1 to smithi02284960-49 2024-09-21T09:41:27.095 INFO:tasks.rados.rados.0.smithi022.stdout:2929: rollback oid 11 current snap is 291 2024-09-21T09:41:27.095 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 11 to 281 2024-09-21T09:41:27.095 INFO:tasks.rados.rados.0.smithi022.stdout:2930: setattr oid 8 current snap is 291 2024-09-21T09:41:27.096 INFO:tasks.rados.rados.0.smithi022.stdout:2931: snap_create 2024-09-21T09:41:27.099 INFO:tasks.rados.rados.0.smithi022.stdout:2927: finishing write tid 2 to smithi02284960-49 2024-09-21T09:41:27.101 INFO:tasks.rados.rados.0.smithi022.stdout:2927: finishing write tid 3 to smithi02284960-49 2024-09-21T09:41:27.101 INFO:tasks.rados.rados.0.smithi022.stdout:2927: finishing write tid 4 to smithi02284960-49 2024-09-21T09:41:27.101 INFO:tasks.rados.rados.0.smithi022.stdout:2929: finishing rollback tid 0 to smithi02284960-11 2024-09-21T09:41:27.102 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 8 v 1662 (ObjNum 897 snap 284 seq_num 897) dirty exists 2024-09-21T09:41:27.103 INFO:tasks.rados.rados.0.smithi022.stdout:2929: finishing rollback tid 1 to smithi02284960-11 2024-09-21T09:41:27.103 INFO:tasks.rados.rados.0.smithi022.stdout:2929: finishing rollback tid 2 to smithi02284960-11 2024-09-21T09:41:27.103 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 11 v 1658 (ObjNum 749 snap 227 seq_num 749) dirty exists 2024-09-21T09:41:27.108 INFO:tasks.rados.rados.0.smithi022.stdout:2927: finishing write tid 5 to smithi02284960-49 2024-09-21T09:41:27.108 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 49 v 1125 (ObjNum 936 snap 291 seq_num 936) dirty exists 2024-09-21T09:41:27.108 INFO:tasks.rados.rados.0.smithi022.stdout:2927: left oid 49 (ObjNum 936 snap 291 seq_num 936) 2024-09-21T09:41:27.114 INFO:tasks.rados.rados.0.smithi022.stdout:2928: finishing copy_from racing read to smithi02284960-47 2024-09-21T09:41:27.120 INFO:tasks.rados.rados.0.smithi022.stdout:2928: finishing copy_from to smithi02284960-47 2024-09-21T09:41:27.120 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 47 v 1510 (ObjNum 884 snap 282 seq_num 884) dirty exists 2024-09-21T09:41:27.125 INFO:teuthology.orchestra.run.smithi022.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T09:41:27.125 INFO:teuthology.orchestra.run.smithi022.stdout:alertmanager.a smithi022 *:9093,9094 running (8m) 71s ago 12m 22.0M - 0.27.0 11f11916f8cd caced80ba544 2024-09-21T09:41:27.125 INFO:teuthology.orchestra.run.smithi022.stdout:grafana.a smithi064 *:3000 running (35s) 29s ago 12m 46.3M - 10.4.8 5aad1d7cf004 0f60e823c29b 2024-09-21T09:41:27.126 INFO:teuthology.orchestra.run.smithi022.stdout:mds.a smithi022 running (75s) 71s ago 14m 11.0M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 88fab662525b 2024-09-21T09:41:27.126 INFO:teuthology.orchestra.run.smithi022.stdout:mds.b smithi064 running (57s) 29s ago 14m 12.5M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 e520bfb5f294 2024-09-21T09:41:27.126 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.x smithi064 *:8443,9283,8765 running (7m) 29s ago 17m 456M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 61abc61fea8d 2024-09-21T09:41:27.126 INFO:teuthology.orchestra.run.smithi022.stdout:mgr.y smithi022 *:8443,9283,8765 running (7m) 71s ago 19m 521M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 6c5bee78f09b 2024-09-21T09:41:27.126 INFO:teuthology.orchestra.run.smithi022.stdout:mon.a smithi022 running (6m) 71s ago 19m 69.2M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 0513395c784d 2024-09-21T09:41:27.126 INFO:teuthology.orchestra.run.smithi022.stdout:mon.b smithi064 running (6m) 29s ago 17m 58.6M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 9dab26b6a9e7 2024-09-21T09:41:27.126 INFO:teuthology.orchestra.run.smithi022.stdout:mon.c smithi022 running (6m) 71s ago 17m 55.9M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 c74b58b67bd6 2024-09-21T09:41:27.126 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.a smithi022 *:9100 running (8m) 71s ago 12m 11.5M - 1.7.0 72c9c2088986 593c6158f524 2024-09-21T09:41:27.126 INFO:teuthology.orchestra.run.smithi022.stdout:node-exporter.b smithi064 *:9100 running (7m) 29s ago 12m 12.6M - 1.7.0 72c9c2088986 a533420b0139 2024-09-21T09:41:27.126 INFO:teuthology.orchestra.run.smithi022.stdout:osd.0 smithi022 running (5m) 71s ago 17m 213M 2481M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 40ce23a2b569 2024-09-21T09:41:27.126 INFO:teuthology.orchestra.run.smithi022.stdout:osd.1 smithi022 running (4m) 71s ago 16m 200M 2481M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 7cd0c18d224c 2024-09-21T09:41:27.126 INFO:teuthology.orchestra.run.smithi022.stdout:osd.2 smithi022 running (4m) 71s ago 16m 208M 2481M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 b8c944306698 2024-09-21T09:41:27.126 INFO:teuthology.orchestra.run.smithi022.stdout:osd.3 smithi022 running (3m) 71s ago 16m 104M 2481M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 c90b27f61bb6 2024-09-21T09:41:27.127 INFO:teuthology.orchestra.run.smithi022.stdout:osd.4 smithi064 running (3m) 29s ago 15m 150M 2480M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 936099dbca14 2024-09-21T09:41:27.127 INFO:teuthology.orchestra.run.smithi022.stdout:osd.5 smithi064 running (2m) 29s ago 15m 193M 2480M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 b3225b090dba 2024-09-21T09:41:27.127 INFO:teuthology.orchestra.run.smithi022.stdout:osd.6 smithi064 running (2m) 29s ago 14m 117M 2480M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 c6a9a09637e5 2024-09-21T09:41:27.127 INFO:teuthology.orchestra.run.smithi022.stdout:osd.7 smithi064 running (92s) 29s ago 14m 119M 2480M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 6f82655605e3 2024-09-21T09:41:27.127 INFO:teuthology.orchestra.run.smithi022.stdout:prometheus.a smithi064 *:9095 running (7m) 29s ago 13m 56.6M - 2.51.0 1d3b7f56885b 80629ffa2179 2024-09-21T09:41:27.431 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:27 smithi022 ceph-mon[98615]: osdmap e655: 8 total, 8 up, 8 in 2024-09-21T09:41:27.431 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:27 smithi022 ceph-mon[96736]: osdmap e655: 8 total, 8 up, 8 in 2024-09-21T09:41:27.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:27 smithi064 ceph-mon[86709]: osdmap e655: 8 total, 8 up, 8 in 2024-09-21T09:41:27.750 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d65e801e-77fa-11ef-baf6-efdc52797490 -e sha1=62da54b4b83a96ca9976b7e1d0c8272a564a2208 -- bash -c 'ceph versions' 2024-09-21T09:41:28.008 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:41:28.091 INFO:tasks.rados.rados.0.smithi022.stdout:2927: done (4 left) 2024-09-21T09:41:28.092 INFO:tasks.rados.rados.0.smithi022.stdout:2928: done (3 left) 2024-09-21T09:41:28.092 INFO:tasks.rados.rados.0.smithi022.stdout:2929: done (2 left) 2024-09-21T09:41:28.092 INFO:tasks.rados.rados.0.smithi022.stdout:2930: done (1 left) 2024-09-21T09:41:28.092 INFO:tasks.rados.rados.0.smithi022.stdout:2931: done (0 left) 2024-09-21T09:41:28.092 INFO:tasks.rados.rados.0.smithi022.stdout:2932: read oid 18 snap -1 2024-09-21T09:41:28.092 INFO:tasks.rados.rados.0.smithi022.stdout:2932: expect deleted 2024-09-21T09:41:28.092 INFO:tasks.rados.rados.0.smithi022.stdout:2933: read oid 38 snap -1 2024-09-21T09:41:28.092 INFO:tasks.rados.rados.0.smithi022.stdout:2933: expect (ObjNum 839 snap 269 seq_num 839) 2024-09-21T09:41:28.092 INFO:tasks.rados.rados.0.smithi022.stdout:2934: setattr oid 33 current snap is 292 2024-09-21T09:41:28.093 INFO:tasks.rados.rados.0.smithi022.stdout:2935: rmattr oid 49 current snap is 292 2024-09-21T09:41:28.093 INFO:tasks.rados.rados.0.smithi022.stdout:2935: done (3 left) 2024-09-21T09:41:28.093 INFO:tasks.rados.rados.0.smithi022.stdout:2936: read oid 47 snap 289 2024-09-21T09:41:28.093 INFO:tasks.rados.rados.0.smithi022.stdout:2936: expect (ObjNum 849 snap 273 seq_num 849) 2024-09-21T09:41:28.093 INFO:tasks.rados.rados.0.smithi022.stdout:2937: delete oid 28 current snap is 292 2024-09-21T09:41:28.099 INFO:tasks.rados.rados.0.smithi022.stdout:2932: done (4 left) 2024-09-21T09:41:28.099 INFO:tasks.rados.rados.0.smithi022.stdout:2937: done (3 left) 2024-09-21T09:41:28.099 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 23 current snap is 292 2024-09-21T09:41:28.099 INFO:tasks.rados.rados.0.smithi022.stdout:2938: seq_num 938 ranges {0=516096,516096=417792,933888=385024} 2024-09-21T09:41:28.102 INFO:tasks.rados.rados.0.smithi022.stdout:2938: writing smithi02284960-23 from 0 to 516096 tid 1 2024-09-21T09:41:28.105 INFO:tasks.rados.rados.0.smithi022.stdout:2938: writing smithi02284960-23 from 516096 to 933888 tid 2 2024-09-21T09:41:28.108 INFO:tasks.rados.rados.0.smithi022.stdout:2938: writing smithi02284960-23 from 933888 to 1318912 tid 3 2024-09-21T09:41:28.108 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 20 current snap is 292 2024-09-21T09:41:28.108 INFO:tasks.rados.rados.0.smithi022.stdout:2939: seq_num 939 ranges {8585216=532480,9117696=704512,9822208=98304} 2024-09-21T09:41:28.158 INFO:tasks.rados.rados.0.smithi022.stdout:2939: writing smithi02284960-20 from 8585216 to 9117696 tid 1 2024-09-21T09:41:28.162 INFO:tasks.rados.rados.0.smithi022.stdout:2939: writing smithi02284960-20 from 9117696 to 9822208 tid 2 2024-09-21T09:41:28.163 INFO:tasks.rados.rados.0.smithi022.stdout:2939: writing smithi02284960-20 from 9822208 to 9920512 tid 3 2024-09-21T09:41:28.163 INFO:tasks.rados.rados.0.smithi022.stdout:2940: read oid 42 snap 291 2024-09-21T09:41:28.163 INFO:tasks.rados.rados.0.smithi022.stdout:2940: expect deleted 2024-09-21T09:41:28.163 INFO:tasks.rados.rados.0.smithi022.stdout:2941: snap_remove snap 287 2024-09-21T09:41:28.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:28 smithi022 ceph-mon[98615]: pgmap v591: 121 pgs: 121 active+clean; 293 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 4.5 MiB/s rd, 12 MiB/s wr, 19 op/s 2024-09-21T09:41:28.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:28 smithi022 ceph-mon[98615]: osdmap e656: 8 total, 8 up, 8 in 2024-09-21T09:41:28.427 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:28 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:41:28.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:28 smithi022 ceph-mon[96736]: pgmap v591: 121 pgs: 121 active+clean; 293 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 4.5 MiB/s rd, 12 MiB/s wr, 19 op/s 2024-09-21T09:41:28.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:28 smithi022 ceph-mon[96736]: osdmap e656: 8 total, 8 up, 8 in 2024-09-21T09:41:28.427 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:28 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:41:28.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:28 smithi064 ceph-mon[86709]: pgmap v591: 121 pgs: 121 active+clean; 293 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 4.5 MiB/s rd, 12 MiB/s wr, 19 op/s 2024-09-21T09:41:28.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:28 smithi064 ceph-mon[86709]: osdmap e656: 8 total, 8 up, 8 in 2024-09-21T09:41:28.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:28 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:41:29.048 INFO:tasks.rados.rados.0.smithi022.stdout:2941: done (6 left) 2024-09-21T09:41:29.048 INFO:tasks.rados.rados.0.smithi022.stdout:2942: delete oid 43 current snap is 292 2024-09-21T09:41:29.048 INFO:tasks.rados.rados.0.smithi022.stdout:2933: expect (ObjNum 839 snap 269 seq_num 839) 2024-09-21T09:41:29.116 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 33 v 1827 (ObjNum 873 snap 280 seq_num 873) dirty exists 2024-09-21T09:41:29.117 INFO:tasks.rados.rados.0.smithi022.stdout:2936: expect (ObjNum 849 snap 273 seq_num 849) 2024-09-21T09:41:29.175 INFO:tasks.rados.rados.0.smithi022.stdout:2938: finishing write tid 1 to smithi02284960-23 2024-09-21T09:41:29.176 INFO:tasks.rados.rados.0.smithi022.stdout:2938: finishing write tid 2 to smithi02284960-23 2024-09-21T09:41:29.176 INFO:tasks.rados.rados.0.smithi022.stdout:2938: finishing write tid 3 to smithi02284960-23 2024-09-21T09:41:29.176 INFO:tasks.rados.rados.0.smithi022.stdout:2938: finishing write tid 4 to smithi02284960-23 2024-09-21T09:41:29.176 INFO:tasks.rados.rados.0.smithi022.stdout:2938: finishing write tid 5 to smithi02284960-23 2024-09-21T09:41:29.176 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 23 v 1436 (ObjNum 938 snap 292 seq_num 938) dirty exists 2024-09-21T09:41:29.176 INFO:tasks.rados.rados.0.smithi022.stdout:2938: left oid 23 (ObjNum 938 snap 292 seq_num 938) 2024-09-21T09:41:29.176 INFO:tasks.rados.rados.0.smithi022.stdout:2939: finishing write tid 1 to smithi02284960-20 2024-09-21T09:41:29.176 INFO:tasks.rados.rados.0.smithi022.stdout:2933: done (6 left) 2024-09-21T09:41:29.176 INFO:tasks.rados.rados.0.smithi022.stdout:2934: done (5 left) 2024-09-21T09:41:29.176 INFO:tasks.rados.rados.0.smithi022.stdout:2936: done (4 left) 2024-09-21T09:41:29.176 INFO:tasks.rados.rados.0.smithi022.stdout:2938: done (3 left) 2024-09-21T09:41:29.176 INFO:tasks.rados.rados.0.smithi022.stdout:2942: done (2 left) 2024-09-21T09:41:29.176 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 41 current snap is 292 2024-09-21T09:41:29.176 INFO:tasks.rados.rados.0.smithi022.stdout:2939: finishing write tid 2 to smithi02284960-20 2024-09-21T09:41:29.176 INFO:tasks.rados.rados.0.smithi022.stdout:2939: finishing write tid 3 to smithi02284960-20 2024-09-21T09:41:29.177 INFO:tasks.rados.rados.0.smithi022.stdout:2939: finishing write tid 4 to smithi02284960-20 2024-09-21T09:41:29.177 INFO:tasks.rados.rados.0.smithi022.stdout:2943: seq_num 941 ranges {6602752=565248,7168000=475136,7643136=327680} 2024-09-21T09:41:29.214 INFO:tasks.rados.rados.0.smithi022.stdout:2943: writing smithi02284960-41 from 6602752 to 7168000 tid 1 2024-09-21T09:41:29.217 INFO:tasks.rados.rados.0.smithi022.stdout:2943: writing smithi02284960-41 from 7168000 to 7643136 tid 2 2024-09-21T09:41:29.219 INFO:tasks.rados.rados.0.smithi022.stdout:2943: writing smithi02284960-41 from 7643136 to 7970816 tid 3 2024-09-21T09:41:29.219 INFO:tasks.rados.rados.0.smithi022.stdout:2940: done (2 left) 2024-09-21T09:41:29.219 INFO:tasks.rados.rados.0.smithi022.stdout:2944: snap_create 2024-09-21T09:41:29.219 INFO:tasks.rados.rados.0.smithi022.stdout:2939: finishing write tid 5 to smithi02284960-20 2024-09-21T09:41:29.219 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 20 v 1832 (ObjNum 939 snap 292 seq_num 939) dirty exists 2024-09-21T09:41:29.219 INFO:tasks.rados.rados.0.smithi022.stdout:2939: left oid 20 (ObjNum 939 snap 292 seq_num 939) 2024-09-21T09:41:29.219 INFO:tasks.rados.rados.0.smithi022.stdout:2943: finishing write tid 1 to smithi02284960-41 2024-09-21T09:41:29.221 INFO:tasks.rados.rados.0.smithi022.stdout:2943: finishing write tid 2 to smithi02284960-41 2024-09-21T09:41:29.221 INFO:tasks.rados.rados.0.smithi022.stdout:2943: finishing write tid 3 to smithi02284960-41 2024-09-21T09:41:29.222 INFO:tasks.rados.rados.0.smithi022.stdout:2943: finishing write tid 4 to smithi02284960-41 2024-09-21T09:41:29.223 INFO:tasks.rados.rados.0.smithi022.stdout:2943: finishing write tid 5 to smithi02284960-41 2024-09-21T09:41:29.224 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 41 v 1657 (ObjNum 941 snap 292 seq_num 941) dirty exists 2024-09-21T09:41:29.224 INFO:tasks.rados.rados.0.smithi022.stdout:2943: left oid 41 (ObjNum 941 snap 292 seq_num 941) 2024-09-21T09:41:29.307 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:29 smithi022 ceph-mon[98615]: from='client.34560 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:29.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:29 smithi022 ceph-mon[98615]: osdmap e657: 8 total, 8 up, 8 in 2024-09-21T09:41:29.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:29 smithi022 ceph-mon[98615]: osdmap e658: 8 total, 8 up, 8 in 2024-09-21T09:41:29.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:29 smithi022 ceph-mon[96736]: from='client.34560 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:29.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:29 smithi022 ceph-mon[96736]: osdmap e657: 8 total, 8 up, 8 in 2024-09-21T09:41:29.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:29 smithi022 ceph-mon[96736]: osdmap e658: 8 total, 8 up, 8 in 2024-09-21T09:41:29.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:29 smithi064 ceph-mon[86709]: from='client.34560 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:29.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:29 smithi064 ceph-mon[86709]: osdmap e657: 8 total, 8 up, 8 in 2024-09-21T09:41:29.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:29 smithi064 ceph-mon[86709]: osdmap e658: 8 total, 8 up, 8 in 2024-09-21T09:41:30.098 INFO:tasks.rados.rados.0.smithi022.stdout:2939: done (2 left) 2024-09-21T09:41:30.098 INFO:tasks.rados.rados.0.smithi022.stdout:2943: done (1 left) 2024-09-21T09:41:30.098 INFO:tasks.rados.rados.0.smithi022.stdout:2944: done (0 left) 2024-09-21T09:41:30.098 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 13 current snap is 293 2024-09-21T09:41:30.098 INFO:tasks.rados.rados.0.smithi022.stdout:2945: seq_num 942 ranges {19423232=581632,20004864=761856,20766720=40960} 2024-09-21T09:41:30.207 INFO:tasks.rados.rados.0.smithi022.stdout:2945: writing smithi02284960-13 from 19423232 to 20004864 tid 1 2024-09-21T09:41:30.211 INFO:tasks.rados.rados.0.smithi022.stdout:2945: writing smithi02284960-13 from 20004864 to 20766720 tid 2 2024-09-21T09:41:30.211 INFO:tasks.rados.rados.0.smithi022.stdout:2945: writing smithi02284960-13 from 20766720 to 20807680 tid 3 2024-09-21T09:41:30.211 INFO:tasks.rados.rados.0.smithi022.stdout:2946: copy_from oid 41 from oid 46 current snap is 293 2024-09-21T09:41:30.212 INFO:tasks.rados.rados.0.smithi022.stdout:2947: snap_remove snap 289 2024-09-21T09:41:30.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:30 smithi022 ceph-mon[98615]: pgmap v594: 121 pgs: 121 active+clean; 294 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 6.0 MiB/s rd, 13 MiB/s wr, 18 op/s 2024-09-21T09:41:30.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:30 smithi022 ceph-mon[96736]: pgmap v594: 121 pgs: 121 active+clean; 294 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 6.0 MiB/s rd, 13 MiB/s wr, 18 op/s 2024-09-21T09:41:30.337 INFO:teuthology.orchestra.run.smithi022.stdout:{ 2024-09-21T09:41:30.337 INFO:teuthology.orchestra.run.smithi022.stdout: "mon": { 2024-09-21T09:41:30.338 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 3 2024-09-21T09:41:30.338 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-21T09:41:30.338 INFO:teuthology.orchestra.run.smithi022.stdout: "mgr": { 2024-09-21T09:41:30.338 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 2 2024-09-21T09:41:30.338 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-21T09:41:30.338 INFO:teuthology.orchestra.run.smithi022.stdout: "osd": { 2024-09-21T09:41:30.338 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 8 2024-09-21T09:41:30.338 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-21T09:41:30.338 INFO:teuthology.orchestra.run.smithi022.stdout: "mds": { 2024-09-21T09:41:30.338 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 2 2024-09-21T09:41:30.338 INFO:teuthology.orchestra.run.smithi022.stdout: }, 2024-09-21T09:41:30.338 INFO:teuthology.orchestra.run.smithi022.stdout: "overall": { 2024-09-21T09:41:30.338 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 15 2024-09-21T09:41:30.338 INFO:teuthology.orchestra.run.smithi022.stdout: } 2024-09-21T09:41:30.338 INFO:teuthology.orchestra.run.smithi022.stdout:} 2024-09-21T09:41:30.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:30 smithi064 ceph-mon[86709]: pgmap v594: 121 pgs: 121 active+clean; 294 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 6.0 MiB/s rd, 13 MiB/s wr, 18 op/s 2024-09-21T09:41:30.442 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:41:30 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:41:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:41:31.114 INFO:tasks.rados.rados.0.smithi022.stdout:2947: done (2 left) 2024-09-21T09:41:31.114 INFO:tasks.rados.rados.0.smithi022.stdout:2948: read oid 23 snap -1 2024-09-21T09:41:31.114 INFO:tasks.rados.rados.0.smithi022.stdout:2948: expect (ObjNum 938 snap 292 seq_num 938) 2024-09-21T09:41:31.114 INFO:tasks.rados.rados.0.smithi022.stdout:2949: rollback oid 28 current snap is 293 2024-09-21T09:41:31.114 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 28 to 279 2024-09-21T09:41:31.115 INFO:tasks.rados.rados.0.smithi022.stdout:2946: finishing copy_from racing read to smithi02284960-41 2024-09-21T09:41:31.115 INFO:tasks.rados.rados.0.smithi022.stdout:2945: finishing write tid 1 to smithi02284960-13 2024-09-21T09:41:31.115 INFO:tasks.rados.rados.0.smithi022.stdout:2946: finishing copy_from to smithi02284960-41 2024-09-21T09:41:31.115 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 41 v 1515 (ObjNum 865 snap 277 seq_num 865) dirty exists 2024-09-21T09:41:31.115 INFO:tasks.rados.rados.0.smithi022.stdout:2945: finishing write tid 2 to smithi02284960-13 2024-09-21T09:41:31.115 INFO:tasks.rados.rados.0.smithi022.stdout:2945: finishing write tid 3 to smithi02284960-13 2024-09-21T09:41:31.115 INFO:tasks.rados.rados.0.smithi022.stdout:2945: finishing write tid 4 to smithi02284960-13 2024-09-21T09:41:31.115 INFO:tasks.rados.rados.0.smithi022.stdout:2945: finishing write tid 5 to smithi02284960-13 2024-09-21T09:41:31.115 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 13 v 1680 (ObjNum 942 snap 293 seq_num 942) dirty exists 2024-09-21T09:41:31.115 INFO:tasks.rados.rados.0.smithi022.stdout:2945: left oid 13 (ObjNum 942 snap 293 seq_num 942) 2024-09-21T09:41:31.115 INFO:tasks.rados.rados.0.smithi022.stdout:2945: done (3 left) 2024-09-21T09:41:31.115 INFO:tasks.rados.rados.0.smithi022.stdout:2946: done (2 left) 2024-09-21T09:41:31.115 INFO:tasks.rados.rados.0.smithi022.stdout:2950: setattr oid 21 current snap is 293 2024-09-21T09:41:31.116 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 1 current snap is 293 2024-09-21T09:41:31.116 INFO:tasks.rados.rados.0.smithi022.stdout:2951: seq_num 943 ranges {7766016=598016,8364032=688128,9052160=114688} 2024-09-21T09:41:31.139 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d65e801e-77fa-11ef-baf6-efdc52797490 -e sha1=62da54b4b83a96ca9976b7e1d0c8272a564a2208 -- bash -c 'ceph versions | jq -e '"'"'.overall | length == 1'"'"'' 2024-09-21T09:41:31.163 INFO:tasks.rados.rados.0.smithi022.stdout:2951: writing smithi02284960-1 from 7766016 to 8364032 tid 1 2024-09-21T09:41:31.168 INFO:tasks.rados.rados.0.smithi022.stdout:2951: writing smithi02284960-1 from 8364032 to 9052160 tid 2 2024-09-21T09:41:31.169 INFO:tasks.rados.rados.0.smithi022.stdout:2951: writing smithi02284960-1 from 9052160 to 9166848 tid 3 2024-09-21T09:41:31.169 INFO:tasks.rados.rados.0.smithi022.stdout:2952: snap_remove snap 290 2024-09-21T09:41:31.169 INFO:tasks.rados.rados.0.smithi022.stdout:2949: finishing rollback tid 1 to smithi02284960-28 2024-09-21T09:41:31.169 INFO:tasks.rados.rados.0.smithi022.stdout:2949: finishing rollback tid 2 to smithi02284960-28 2024-09-21T09:41:31.169 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 28 v 1603 (ObjNum 841 snap 269 seq_num 841) dirty exists 2024-09-21T09:41:31.169 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 21 v 1541 (ObjNum 289 snap 0 seq_num 1693273218) dirty exists 2024-09-21T09:41:31.169 INFO:tasks.rados.rados.0.smithi022.stdout:2948: expect (ObjNum 938 snap 292 seq_num 938) 2024-09-21T09:41:31.209 INFO:tasks.rados.rados.0.smithi022.stdout:2951: finishing write tid 1 to smithi02284960-1 2024-09-21T09:41:31.209 INFO:tasks.rados.rados.0.smithi022.stdout:2951: finishing write tid 2 to smithi02284960-1 2024-09-21T09:41:31.209 INFO:tasks.rados.rados.0.smithi022.stdout:2951: finishing write tid 3 to smithi02284960-1 2024-09-21T09:41:31.209 INFO:tasks.rados.rados.0.smithi022.stdout:2951: finishing write tid 4 to smithi02284960-1 2024-09-21T09:41:31.406 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:41:31.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:31 smithi064 ceph-mon[86709]: osdmap e659: 8 total, 8 up, 8 in 2024-09-21T09:41:31.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:31 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/2185149322' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:41:31.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:31 smithi064 ceph-mon[86709]: pgmap v597: 121 pgs: 121 active+clean; 300 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 5.4 MiB/s rd, 2.1 MiB/s wr, 22 op/s 2024-09-21T09:41:31.528 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:31 smithi022 ceph-mon[98615]: osdmap e659: 8 total, 8 up, 8 in 2024-09-21T09:41:31.528 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:31 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/2185149322' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:41:31.528 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:31 smithi022 ceph-mon[98615]: pgmap v597: 121 pgs: 121 active+clean; 300 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 5.4 MiB/s rd, 2.1 MiB/s wr, 22 op/s 2024-09-21T09:41:31.528 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:31 smithi022 ceph-mon[96736]: osdmap e659: 8 total, 8 up, 8 in 2024-09-21T09:41:31.528 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:31 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/2185149322' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:41:31.528 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:31 smithi022 ceph-mon[96736]: pgmap v597: 121 pgs: 121 active+clean; 300 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 5.4 MiB/s rd, 2.1 MiB/s wr, 22 op/s 2024-09-21T09:41:32.117 INFO:tasks.rados.rados.0.smithi022.stdout:2948: done (4 left) 2024-09-21T09:41:32.117 INFO:tasks.rados.rados.0.smithi022.stdout:2949: done (3 left) 2024-09-21T09:41:32.117 INFO:tasks.rados.rados.0.smithi022.stdout:2950: done (2 left) 2024-09-21T09:41:32.118 INFO:tasks.rados.rados.0.smithi022.stdout:2952: done (1 left) 2024-09-21T09:41:32.118 INFO:tasks.rados.rados.0.smithi022.stdout:2953: delete oid 23 current snap is 293 2024-09-21T09:41:32.118 INFO:tasks.rados.rados.0.smithi022.stdout:2951: finishing write tid 5 to smithi02284960-1 2024-09-21T09:41:32.118 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 1 v 1662 (ObjNum 943 snap 293 seq_num 943) dirty exists 2024-09-21T09:41:32.118 INFO:tasks.rados.rados.0.smithi022.stdout:2951: left oid 1 (ObjNum 943 snap 293 seq_num 943) 2024-09-21T09:41:32.124 INFO:tasks.rados.rados.0.smithi022.stdout:2951: done (1 left) 2024-09-21T09:41:32.125 INFO:tasks.rados.rados.0.smithi022.stdout:2953: done (0 left) 2024-09-21T09:41:32.125 INFO:tasks.rados.rados.0.smithi022.stdout:2954: snap_create 2024-09-21T09:41:32.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:32 smithi022 ceph-mon[98615]: osdmap e660: 8 total, 8 up, 8 in 2024-09-21T09:41:32.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:32 smithi022 ceph-mon[96736]: osdmap e660: 8 total, 8 up, 8 in 2024-09-21T09:41:32.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:32 smithi064 ceph-mon[86709]: osdmap e660: 8 total, 8 up, 8 in 2024-09-21T09:41:33.124 INFO:tasks.rados.rados.0.smithi022.stdout:2954: done (0 left) 2024-09-21T09:41:33.125 INFO:tasks.rados.rados.0.smithi022.stdout:2955: read oid 43 snap 285 2024-09-21T09:41:33.125 INFO:tasks.rados.rados.0.smithi022.stdout:2955: expect (ObjNum 817 snap 261 seq_num 817) 2024-09-21T09:41:33.125 INFO:tasks.rados.rados.0.smithi022.stdout:2956: snap_create 2024-09-21T09:41:33.208 INFO:tasks.rados.rados.0.smithi022.stdout:2955: expect (ObjNum 817 snap 261 seq_num 817) 2024-09-21T09:41:33.307 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:33 smithi022 ceph-mon[98615]: osdmap e661: 8 total, 8 up, 8 in 2024-09-21T09:41:33.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:33 smithi022 ceph-mon[98615]: pgmap v600: 121 pgs: 121 active+clean; 298 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 1.9 MiB/s wr, 17 op/s 2024-09-21T09:41:33.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:33 smithi022 ceph-mon[96736]: osdmap e661: 8 total, 8 up, 8 in 2024-09-21T09:41:33.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:33 smithi022 ceph-mon[96736]: pgmap v600: 121 pgs: 121 active+clean; 298 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 1.9 MiB/s wr, 17 op/s 2024-09-21T09:41:33.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:33 smithi064 ceph-mon[86709]: osdmap e661: 8 total, 8 up, 8 in 2024-09-21T09:41:33.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:33 smithi064 ceph-mon[86709]: pgmap v600: 121 pgs: 121 active+clean; 298 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 1.9 MiB/s wr, 17 op/s 2024-09-21T09:41:33.467 INFO:teuthology.orchestra.run.smithi022.stdout:true 2024-09-21T09:41:34.118 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d65e801e-77fa-11ef-baf6-efdc52797490 -e sha1=62da54b4b83a96ca9976b7e1d0c8272a564a2208 -- bash -c 'ceph versions | jq -e '"'"'.overall | keys'"'"' | grep $sha1' 2024-09-21T09:41:34.422 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:41:34.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:34 smithi064 ceph-mon[86709]: osdmap e662: 8 total, 8 up, 8 in 2024-09-21T09:41:34.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:34 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/1121970115' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:41:34.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:34 smithi064 ceph-mon[86709]: osdmap e663: 8 total, 8 up, 8 in 2024-09-21T09:41:34.543 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:34 smithi022 ceph-mon[98615]: osdmap e662: 8 total, 8 up, 8 in 2024-09-21T09:41:34.543 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:34 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/1121970115' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:41:34.543 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:34 smithi022 ceph-mon[98615]: osdmap e663: 8 total, 8 up, 8 in 2024-09-21T09:41:34.543 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:34 smithi022 ceph-mon[96736]: osdmap e662: 8 total, 8 up, 8 in 2024-09-21T09:41:34.544 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:34 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/1121970115' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:41:34.544 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:34 smithi022 ceph-mon[96736]: osdmap e663: 8 total, 8 up, 8 in 2024-09-21T09:41:35.068 INFO:tasks.rados.rados.0.smithi022.stdout:2955: done (1 left) 2024-09-21T09:41:35.068 INFO:tasks.rados.rados.0.smithi022.stdout:2956: done (0 left) 2024-09-21T09:41:35.068 INFO:tasks.rados.rados.0.smithi022.stdout:2957: rmattr oid 44 current snap is 295 2024-09-21T09:41:35.068 INFO:tasks.rados.rados.0.smithi022.stdout:2957: done (0 left) 2024-09-21T09:41:35.068 INFO:tasks.rados.rados.0.smithi022.stdout:2958: snap_remove snap 291 2024-09-21T09:41:35.129 INFO:tasks.rados.rados.0.smithi022.stdout:2958: done (0 left) 2024-09-21T09:41:35.129 INFO:tasks.rados.rados.0.smithi022.stdout:2959: read oid 10 snap -1 2024-09-21T09:41:35.129 INFO:tasks.rados.rados.0.smithi022.stdout:2959: expect deleted 2024-09-21T09:41:35.129 INFO:tasks.rados.rados.0.smithi022.stdout:2960: copy_from oid 38 from oid 48 current snap is 295 2024-09-21T09:41:35.129 INFO:tasks.rados.rados.0.smithi022.stdout:2961: copy_from oid 2 from oid 9 current snap is 295 2024-09-21T09:41:35.131 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 4 current snap is 295 2024-09-21T09:41:35.131 INFO:tasks.rados.rados.0.smithi022.stdout:2962: seq_num 945 ranges {18997248=630784,19628032=458752,20086784=344064} 2024-09-21T09:41:35.236 INFO:tasks.rados.rados.0.smithi022.stdout:2962: writing smithi02284960-4 from 18997248 to 19628032 tid 1 2024-09-21T09:41:35.239 INFO:tasks.rados.rados.0.smithi022.stdout:2962: writing smithi02284960-4 from 19628032 to 20086784 tid 2 2024-09-21T09:41:35.241 INFO:tasks.rados.rados.0.smithi022.stdout:2962: writing smithi02284960-4 from 20086784 to 20430848 tid 3 2024-09-21T09:41:35.241 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 31 current snap is 295 2024-09-21T09:41:35.241 INFO:tasks.rados.rados.0.smithi022.stdout:2963: seq_num 946 ranges {3702784=647168,4349952=745472,5095424=57344} 2024-09-21T09:41:35.265 INFO:tasks.rados.rados.0.smithi022.stdout:2963: writing smithi02284960-31 from 3702784 to 4349952 tid 1 2024-09-21T09:41:35.269 INFO:tasks.rados.rados.0.smithi022.stdout:2963: writing smithi02284960-31 from 4349952 to 5095424 tid 2 2024-09-21T09:41:35.269 INFO:tasks.rados.rados.0.smithi022.stdout:2963: writing smithi02284960-31 from 5095424 to 5152768 tid 3 2024-09-21T09:41:35.269 INFO:tasks.rados.rados.0.smithi022.stdout:2964: snap_remove snap 279 2024-09-21T09:41:35.307 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:35 smithi022 ceph-mon[98615]: pgmap v603: 121 pgs: 121 active+clean; 302 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 966 KiB/s rd, 338 KiB/s wr, 8 op/s 2024-09-21T09:41:35.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:35 smithi022 ceph-mon[96736]: pgmap v603: 121 pgs: 121 active+clean; 302 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 966 KiB/s rd, 338 KiB/s wr, 8 op/s 2024-09-21T09:41:35.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:35 smithi064 ceph-mon[86709]: pgmap v603: 121 pgs: 121 active+clean; 302 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 966 KiB/s rd, 338 KiB/s wr, 8 op/s 2024-09-21T09:41:35.994 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:41:35 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:41:35] "GET /metrics HTTP/1.1" 200 36358 "" "Prometheus/2.51.0" 2024-09-21T09:41:36.139 INFO:tasks.rados.rados.0.smithi022.stdout:2964: done (5 left) 2024-09-21T09:41:36.139 INFO:tasks.rados.rados.0.smithi022.stdout:2965: delete oid 20 current snap is 295 2024-09-21T09:41:36.139 INFO:tasks.rados.rados.0.smithi022.stdout:2961: finishing copy_from to smithi02284960-2 2024-09-21T09:41:36.139 INFO:tasks.rados.rados.0.smithi022.stdout:2961: got expected ENOENT (src dne) 2024-09-21T09:41:36.140 INFO:tasks.rados.rados.0.smithi022.stdout:2961: finishing copy_from racing read to smithi02284960-2 2024-09-21T09:41:36.140 INFO:tasks.rados.rados.0.smithi022.stdout:2960: finishing copy_from racing read to smithi02284960-38 2024-09-21T09:41:36.140 INFO:tasks.rados.rados.0.smithi022.stdout:2960: finishing copy_from to smithi02284960-38 2024-09-21T09:41:36.140 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 38 v 1784 (ObjNum 925 snap 287 seq_num 925) dirty exists 2024-09-21T09:41:36.140 INFO:tasks.rados.rados.0.smithi022.stdout:2962: finishing write tid 1 to smithi02284960-4 2024-09-21T09:41:36.140 INFO:tasks.rados.rados.0.smithi022.stdout:2962: finishing write tid 2 to smithi02284960-4 2024-09-21T09:41:36.140 INFO:tasks.rados.rados.0.smithi022.stdout:2962: finishing write tid 3 to smithi02284960-4 2024-09-21T09:41:36.140 INFO:tasks.rados.rados.0.smithi022.stdout:2962: finishing write tid 4 to smithi02284960-4 2024-09-21T09:41:36.140 INFO:tasks.rados.rados.0.smithi022.stdout:2962: finishing write tid 5 to smithi02284960-4 2024-09-21T09:41:36.140 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 4 v 1666 (ObjNum 945 snap 295 seq_num 945) dirty exists 2024-09-21T09:41:36.140 INFO:tasks.rados.rados.0.smithi022.stdout:2962: left oid 4 (ObjNum 945 snap 295 seq_num 945) 2024-09-21T09:41:36.140 INFO:tasks.rados.rados.0.smithi022.stdout:2963: finishing write tid 1 to smithi02284960-31 2024-09-21T09:41:36.140 INFO:tasks.rados.rados.0.smithi022.stdout:2963: finishing write tid 2 to smithi02284960-31 2024-09-21T09:41:36.140 INFO:tasks.rados.rados.0.smithi022.stdout:2963: finishing write tid 3 to smithi02284960-31 2024-09-21T09:41:36.141 INFO:tasks.rados.rados.0.smithi022.stdout:2963: finishing write tid 4 to smithi02284960-31 2024-09-21T09:41:36.141 INFO:tasks.rados.rados.0.smithi022.stdout:2963: finishing write tid 5 to smithi02284960-31 2024-09-21T09:41:36.141 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 31 v 1736 (ObjNum 946 snap 295 seq_num 946) dirty exists 2024-09-21T09:41:36.141 INFO:tasks.rados.rados.0.smithi022.stdout:2963: left oid 31 (ObjNum 946 snap 295 seq_num 946) 2024-09-21T09:41:36.155 INFO:tasks.rados.rados.0.smithi022.stdout:2959: done (5 left) 2024-09-21T09:41:36.155 INFO:tasks.rados.rados.0.smithi022.stdout:2960: done (4 left) 2024-09-21T09:41:36.155 INFO:tasks.rados.rados.0.smithi022.stdout:2961: done (3 left) 2024-09-21T09:41:36.155 INFO:tasks.rados.rados.0.smithi022.stdout:2962: done (2 left) 2024-09-21T09:41:36.155 INFO:tasks.rados.rados.0.smithi022.stdout:2963: done (1 left) 2024-09-21T09:41:36.156 INFO:tasks.rados.rados.0.smithi022.stdout:2965: done (0 left) 2024-09-21T09:41:36.156 INFO:tasks.rados.rados.0.smithi022.stdout:2966: rollback oid 31 current snap is 295 2024-09-21T09:41:36.156 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 31 to 281 2024-09-21T09:41:36.156 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 29 current snap is 295 2024-09-21T09:41:36.156 INFO:tasks.rados.rados.0.smithi022.stdout:2967: seq_num 948 ranges {0=679936,679936=516096,1196032=286720} 2024-09-21T09:41:36.159 INFO:tasks.rados.rados.0.smithi022.stdout:2967: writing smithi02284960-29 from 0 to 679936 tid 1 2024-09-21T09:41:36.163 INFO:tasks.rados.rados.0.smithi022.stdout:2967: writing smithi02284960-29 from 679936 to 1196032 tid 2 2024-09-21T09:41:36.166 INFO:tasks.rados.rados.0.smithi022.stdout:2967: writing smithi02284960-29 from 1196032 to 1482752 tid 3 2024-09-21T09:41:36.166 INFO:tasks.rados.rados.0.smithi022.stdout:2967: finishing write tid 1 to smithi02284960-29 2024-09-21T09:41:36.166 INFO:tasks.rados.rados.0.smithi022.stdout:2966: finishing rollback tid 0 to smithi02284960-31 2024-09-21T09:41:36.166 INFO:tasks.rados.rados.0.smithi022.stdout:2966: finishing rollback tid 1 to smithi02284960-31 2024-09-21T09:41:36.166 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 31 v 1738 (ObjNum 161 snap 0 seq_num 0) dirty dne 2024-09-21T09:41:36.166 INFO:tasks.rados.rados.0.smithi022.stdout:2968: read oid 48 snap 294 2024-09-21T09:41:36.166 INFO:tasks.rados.rados.0.smithi022.stdout:2968: expect (ObjNum 925 snap 287 seq_num 925) 2024-09-21T09:41:36.166 INFO:tasks.rados.rados.0.smithi022.stdout:2966: done (2 left) 2024-09-21T09:41:36.167 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 9 current snap is 295 2024-09-21T09:41:36.167 INFO:tasks.rados.rados.0.smithi022.stdout:2969: seq_num 949 ranges {0=704512,704512=794624,1499136=8192} 2024-09-21T09:41:36.171 INFO:tasks.rados.rados.0.smithi022.stdout:2969: writing smithi02284960-9 from 0 to 704512 tid 1 2024-09-21T09:41:36.177 INFO:tasks.rados.rados.0.smithi022.stdout:2969: writing smithi02284960-9 from 704512 to 1499136 tid 2 2024-09-21T09:41:36.177 INFO:tasks.rados.rados.0.smithi022.stdout:2969: writing smithi02284960-9 from 1499136 to 1507328 tid 3 2024-09-21T09:41:36.177 INFO:tasks.rados.rados.0.smithi022.stdout:2970: copy_from oid 40 from oid 36 current snap is 295 2024-09-21T09:41:36.177 INFO:tasks.rados.rados.0.smithi022.stdout:2967: finishing write tid 2 to smithi02284960-29 2024-09-21T09:41:36.177 INFO:tasks.rados.rados.0.smithi022.stdout:2967: finishing write tid 3 to smithi02284960-29 2024-09-21T09:41:36.177 INFO:tasks.rados.rados.0.smithi022.stdout:2967: finishing write tid 4 to smithi02284960-29 2024-09-21T09:41:36.177 INFO:tasks.rados.rados.0.smithi022.stdout:2967: finishing write tid 5 to smithi02284960-29 2024-09-21T09:41:36.177 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 29 v 1788 (ObjNum 948 snap 295 seq_num 948) dirty exists 2024-09-21T09:41:36.177 INFO:tasks.rados.rados.0.smithi022.stdout:2967: left oid 29 (ObjNum 948 snap 295 seq_num 948) 2024-09-21T09:41:36.177 INFO:tasks.rados.rados.0.smithi022.stdout:2967: done (3 left) 2024-09-21T09:41:36.177 INFO:tasks.rados.rados.0.smithi022.stdout:2971: delete oid 32 current snap is 295 2024-09-21T09:41:36.178 INFO:tasks.rados.rados.0.smithi022.stdout:2969: finishing write tid 1 to smithi02284960-9 2024-09-21T09:41:36.181 INFO:tasks.rados.rados.0.smithi022.stdout:2968: expect (ObjNum 925 snap 287 seq_num 925) 2024-09-21T09:41:36.307 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:36 smithi022 ceph-mon[98615]: osdmap e664: 8 total, 8 up, 8 in 2024-09-21T09:41:36.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:36 smithi022 ceph-mon[96736]: osdmap e664: 8 total, 8 up, 8 in 2024-09-21T09:41:36.315 INFO:tasks.rados.rados.0.smithi022.stdout:2969: finishing write tid 2 to smithi02284960-9 2024-09-21T09:41:36.315 INFO:tasks.rados.rados.0.smithi022.stdout:2969: finishing write tid 3 to smithi02284960-9 2024-09-21T09:41:36.315 INFO:tasks.rados.rados.0.smithi022.stdout:2969: finishing write tid 4 to smithi02284960-9 2024-09-21T09:41:36.315 INFO:tasks.rados.rados.0.smithi022.stdout:2970: finishing copy_from racing read to smithi02284960-40 2024-09-21T09:41:36.315 INFO:tasks.rados.rados.0.smithi022.stdout:2969: finishing write tid 5 to smithi02284960-9 2024-09-21T09:41:36.315 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 9 v 1670 (ObjNum 949 snap 295 seq_num 949) dirty exists 2024-09-21T09:41:36.315 INFO:tasks.rados.rados.0.smithi022.stdout:2969: left oid 9 (ObjNum 949 snap 295 seq_num 949) 2024-09-21T09:41:36.315 INFO:tasks.rados.rados.0.smithi022.stdout:2970: finishing copy_from to smithi02284960-40 2024-09-21T09:41:36.315 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 40 v 1414 (ObjNum 919 snap 286 seq_num 919) dirty exists 2024-09-21T09:41:36.316 INFO:tasks.rados.rados.0.smithi022.stdout:2968: done (3 left) 2024-09-21T09:41:36.316 INFO:tasks.rados.rados.0.smithi022.stdout:2969: done (2 left) 2024-09-21T09:41:36.316 INFO:tasks.rados.rados.0.smithi022.stdout:2970: done (1 left) 2024-09-21T09:41:36.316 INFO:tasks.rados.rados.0.smithi022.stdout:2971: done (0 left) 2024-09-21T09:41:36.316 INFO:tasks.rados.rados.0.smithi022.stdout:2972: read oid 30 snap -1 2024-09-21T09:41:36.316 INFO:tasks.rados.rados.0.smithi022.stdout:2972: expect (ObjNum 590 snap 176 seq_num 590) 2024-09-21T09:41:36.316 INFO:tasks.rados.rados.0.smithi022.stdout:2973: rollback oid 11 current snap is 295 2024-09-21T09:41:36.316 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 11 to 285 2024-09-21T09:41:36.316 INFO:tasks.rados.rados.0.smithi022.stdout:2974: read oid 36 snap -1 2024-09-21T09:41:36.316 INFO:tasks.rados.rados.0.smithi022.stdout:2974: expect (ObjNum 919 snap 286 seq_num 919) 2024-09-21T09:41:36.316 INFO:tasks.rados.rados.0.smithi022.stdout:2975: read oid 26 snap -1 2024-09-21T09:41:36.316 INFO:tasks.rados.rados.0.smithi022.stdout:2975: expect (ObjNum 935 snap 289 seq_num 935) 2024-09-21T09:41:36.316 INFO:tasks.rados.rados.0.smithi022.stdout:2976: read oid 3 snap -1 2024-09-21T09:41:36.316 INFO:tasks.rados.rados.0.smithi022.stdout:2976: expect deleted 2024-09-21T09:41:36.316 INFO:tasks.rados.rados.0.smithi022.stdout:2977: rollback oid 39 current snap is 295 2024-09-21T09:41:36.316 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 39 to 293 2024-09-21T09:41:36.317 INFO:tasks.rados.rados.0.smithi022.stdout:2978: delete oid 9 current snap is 295 2024-09-21T09:41:36.318 INFO:tasks.rados.rados.0.smithi022.stdout:2976: done (6 left) 2024-09-21T09:41:36.318 INFO:tasks.rados.rados.0.smithi022.stdout:2978: done (5 left) 2024-09-21T09:41:36.318 INFO:tasks.rados.rados.0.smithi022.stdout:2979: read oid 10 snap -1 2024-09-21T09:41:36.318 INFO:tasks.rados.rados.0.smithi022.stdout:2979: expect deleted 2024-09-21T09:41:36.318 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 16 current snap is 295 2024-09-21T09:41:36.318 INFO:tasks.rados.rados.0.smithi022.stdout:2980: seq_num 952 ranges {1204224=753664,1957888=499712,2457600=303104} 2024-09-21T09:41:36.329 INFO:tasks.rados.rados.0.smithi022.stdout:2980: writing smithi02284960-16 from 1204224 to 1957888 tid 1 2024-09-21T09:41:36.332 INFO:tasks.rados.rados.0.smithi022.stdout:2980: writing smithi02284960-16 from 1957888 to 2457600 tid 2 2024-09-21T09:41:36.334 INFO:tasks.rados.rados.0.smithi022.stdout:2980: writing smithi02284960-16 from 2457600 to 2760704 tid 3 2024-09-21T09:41:36.334 INFO:tasks.rados.rados.0.smithi022.stdout:2981: read oid 33 snap -1 2024-09-21T09:41:36.334 INFO:tasks.rados.rados.0.smithi022.stdout:2981: expect (ObjNum 873 snap 280 seq_num 873) 2024-09-21T09:41:36.334 INFO:tasks.rados.rados.0.smithi022.stdout:2982: snap_create 2024-09-21T09:41:36.335 INFO:tasks.rados.rados.0.smithi022.stdout:2974: expect (ObjNum 919 snap 286 seq_num 919) 2024-09-21T09:41:36.364 INFO:tasks.rados.rados.0.smithi022.stdout:2972: expect (ObjNum 590 snap 176 seq_num 590) 2024-09-21T09:41:36.373 INFO:tasks.rados.rados.0.smithi022.stdout:2977: finishing rollback tid 0 to smithi02284960-39 2024-09-21T09:41:36.373 INFO:tasks.rados.rados.0.smithi022.stdout:2973: finishing rollback tid 0 to smithi02284960-11 2024-09-21T09:41:36.373 INFO:tasks.rados.rados.0.smithi022.stdout:2977: finishing rollback tid 1 to smithi02284960-39 2024-09-21T09:41:36.373 INFO:tasks.rados.rados.0.smithi022.stdout:2977: finishing rollback tid 2 to smithi02284960-39 2024-09-21T09:41:36.373 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 39 v 1317 (ObjNum 911 snap 284 seq_num 911) dirty exists 2024-09-21T09:41:36.373 INFO:tasks.rados.rados.0.smithi022.stdout:2973: finishing rollback tid 1 to smithi02284960-11 2024-09-21T09:41:36.373 INFO:tasks.rados.rados.0.smithi022.stdout:2973: finishing rollback tid 2 to smithi02284960-11 2024-09-21T09:41:36.373 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 11 v 1665 (ObjNum 749 snap 227 seq_num 749) dirty exists 2024-09-21T09:41:36.374 INFO:tasks.rados.rados.0.smithi022.stdout:2980: finishing write tid 1 to smithi02284960-16 2024-09-21T09:41:36.374 INFO:tasks.rados.rados.0.smithi022.stdout:2975: expect (ObjNum 935 snap 289 seq_num 935) 2024-09-21T09:41:36.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:36 smithi064 ceph-mon[86709]: osdmap e664: 8 total, 8 up, 8 in 2024-09-21T09:41:36.442 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:41:36 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=infra.usagestats t=2024-09-21T09:41:36.063089771Z level=info msg="Usage stats are ready to report" 2024-09-21T09:41:36.579 INFO:tasks.rados.rados.0.smithi022.stdout:2980: finishing write tid 2 to smithi02284960-16 2024-09-21T09:41:36.579 INFO:tasks.rados.rados.0.smithi022.stdout:2980: finishing write tid 3 to smithi02284960-16 2024-09-21T09:41:36.579 INFO:tasks.rados.rados.0.smithi022.stdout:2980: finishing write tid 4 to smithi02284960-16 2024-09-21T09:41:36.579 INFO:tasks.rados.rados.0.smithi022.stdout:2980: finishing write tid 5 to smithi02284960-16 2024-09-21T09:41:36.580 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 16 v 1518 (ObjNum 952 snap 295 seq_num 952) dirty exists 2024-09-21T09:41:36.580 INFO:tasks.rados.rados.0.smithi022.stdout:2980: left oid 16 (ObjNum 952 snap 295 seq_num 952) 2024-09-21T09:41:36.580 INFO:tasks.rados.rados.0.smithi022.stdout:2981: expect (ObjNum 873 snap 280 seq_num 873) 2024-09-21T09:41:36.721 INFO:teuthology.orchestra.run.smithi022.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)" 2024-09-21T09:41:37.141 INFO:tasks.rados.rados.0.smithi022.stdout:2972: done (8 left) 2024-09-21T09:41:37.141 INFO:tasks.rados.rados.0.smithi022.stdout:2973: done (7 left) 2024-09-21T09:41:37.141 INFO:tasks.rados.rados.0.smithi022.stdout:2974: done (6 left) 2024-09-21T09:41:37.141 INFO:tasks.rados.rados.0.smithi022.stdout:2975: done (5 left) 2024-09-21T09:41:37.141 INFO:tasks.rados.rados.0.smithi022.stdout:2977: done (4 left) 2024-09-21T09:41:37.141 INFO:tasks.rados.rados.0.smithi022.stdout:2979: done (3 left) 2024-09-21T09:41:37.141 INFO:tasks.rados.rados.0.smithi022.stdout:2980: done (2 left) 2024-09-21T09:41:37.141 INFO:tasks.rados.rados.0.smithi022.stdout:2981: done (1 left) 2024-09-21T09:41:37.141 INFO:tasks.rados.rados.0.smithi022.stdout:2982: done (0 left) 2024-09-21T09:41:37.141 INFO:tasks.rados.rados.0.smithi022.stdout:2983: delete oid 11 current snap is 296 2024-09-21T09:41:37.146 INFO:tasks.rados.rados.0.smithi022.stdout:2983: done (0 left) 2024-09-21T09:41:37.146 INFO:tasks.rados.rados.0.smithi022.stdout:2984: copy_from oid 34 from oid 29 current snap is 296 2024-09-21T09:41:37.146 INFO:tasks.rados.rados.0.smithi022.stdout:2985: delete oid 14 current snap is 296 2024-09-21T09:41:37.150 INFO:tasks.rados.rados.0.smithi022.stdout:2985: done (1 left) 2024-09-21T09:41:37.150 INFO:tasks.rados.rados.0.smithi022.stdout:2986: setattr oid 36 current snap is 296 2024-09-21T09:41:37.153 INFO:tasks.rados.rados.0.smithi022.stdout:2987: rollback oid 10 current snap is 296 2024-09-21T09:41:37.153 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 10 to 285 2024-09-21T09:41:37.153 INFO:tasks.rados.rados.0.smithi022.stdout:2988: copy_from oid 46 from oid 49 current snap is 296 2024-09-21T09:41:37.153 INFO:tasks.rados.rados.0.smithi022.stdout:2989: rollback oid 24 current snap is 296 2024-09-21T09:41:37.153 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 24 to 264 2024-09-21T09:41:37.154 INFO:tasks.rados.rados.0.smithi022.stdout:2990: read oid 15 snap -1 2024-09-21T09:41:37.154 INFO:tasks.rados.rados.0.smithi022.stdout:2990: expect (ObjNum 921 snap 287 seq_num 921) 2024-09-21T09:41:37.154 INFO:tasks.rados.rados.0.smithi022.stdout:2991: delete oid 1 current snap is 296 2024-09-21T09:41:37.157 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 36 v 1416 (ObjNum 919 snap 286 seq_num 919) dirty exists 2024-09-21T09:41:37.160 INFO:tasks.rados.rados.0.smithi022.stdout:2986: done (6 left) 2024-09-21T09:41:37.160 INFO:tasks.rados.rados.0.smithi022.stdout:2991: done (5 left) 2024-09-21T09:41:37.160 INFO:tasks.rados.rados.0.smithi022.stdout:2992: setattr oid 38 current snap is 296 2024-09-21T09:41:37.160 INFO:tasks.rados.rados.0.smithi022.stdout:2984: finishing copy_from racing read to smithi02284960-34 2024-09-21T09:41:37.161 INFO:tasks.rados.rados.0.smithi022.stdout:2984: finishing copy_from to smithi02284960-34 2024-09-21T09:41:37.161 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 34 v 1788 (ObjNum 948 snap 295 seq_num 948) dirty exists 2024-09-21T09:41:37.162 INFO:tasks.rados.rados.0.smithi022.stdout:2984: done (5 left) 2024-09-21T09:41:37.162 INFO:tasks.rados.rados.0.smithi022.stdout:2993: snap_remove snap 292 2024-09-21T09:41:37.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:37 smithi022 ceph-mon[98615]: osdmap e665: 8 total, 8 up, 8 in 2024-09-21T09:41:37.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:37 smithi022 ceph-mon[98615]: pgmap v606: 121 pgs: 121 active+clean; 287 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 718 KiB/s wr, 10 op/s 2024-09-21T09:41:37.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:37 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/226880981' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:41:37.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:37 smithi022 ceph-mon[96736]: osdmap e665: 8 total, 8 up, 8 in 2024-09-21T09:41:37.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:37 smithi022 ceph-mon[96736]: pgmap v606: 121 pgs: 121 active+clean; 287 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 718 KiB/s wr, 10 op/s 2024-09-21T09:41:37.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:37 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/226880981' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:41:37.426 INFO:teuthology.task.sequential:In sequential, running task print... 2024-09-21T09:41:37.427 INFO:teuthology.task.print:**** done end upgrade, wait... 2024-09-21T09:41:37.427 DEBUG:teuthology.parallel:result is None 2024-09-21T09:41:37.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:37 smithi064 ceph-mon[86709]: osdmap e665: 8 total, 8 up, 8 in 2024-09-21T09:41:37.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:37 smithi064 ceph-mon[86709]: pgmap v606: 121 pgs: 121 active+clean; 287 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 718 KiB/s wr, 10 op/s 2024-09-21T09:41:37.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:37 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/226880981' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:41:38.148 INFO:tasks.rados.rados.0.smithi022.stdout:2987: finishing rollback tid 1 to smithi02284960-10 2024-09-21T09:41:38.148 INFO:tasks.rados.rados.0.smithi022.stdout:2987: finishing rollback tid 2 to smithi02284960-10 2024-09-21T09:41:38.148 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 10 v 1659 (ObjNum 880 snap 282 seq_num 880) dirty exists 2024-09-21T09:41:38.148 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 38 v 1789 (ObjNum 925 snap 287 seq_num 925) dirty exists 2024-09-21T09:41:38.148 INFO:tasks.rados.rados.0.smithi022.stdout:2989: finishing rollback tid 0 to smithi02284960-24 2024-09-21T09:41:38.149 INFO:tasks.rados.rados.0.smithi022.stdout:2989: finishing rollback tid 1 to smithi02284960-24 2024-09-21T09:41:38.149 INFO:tasks.rados.rados.0.smithi022.stdout:2989: finishing rollback tid 2 to smithi02284960-24 2024-09-21T09:41:38.149 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 24 v 1750 (ObjNum 783 snap 246 seq_num 783) dirty exists 2024-09-21T09:41:38.149 INFO:tasks.rados.rados.0.smithi022.stdout:2988: finishing copy_from racing read to smithi02284960-46 2024-09-21T09:41:38.149 INFO:tasks.rados.rados.0.smithi022.stdout:2988: finishing copy_from to smithi02284960-46 2024-09-21T09:41:38.149 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 46 v 1125 (ObjNum 936 snap 291 seq_num 936) dirty exists 2024-09-21T09:41:38.149 INFO:tasks.rados.rados.0.smithi022.stdout:2990: expect (ObjNum 921 snap 287 seq_num 921) 2024-09-21T09:41:38.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:38 smithi064 ceph-mon[86709]: osdmap e666: 8 total, 8 up, 8 in 2024-09-21T09:41:38.557 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:38 smithi022 ceph-mon[98615]: osdmap e666: 8 total, 8 up, 8 in 2024-09-21T09:41:38.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:38 smithi022 ceph-mon[96736]: osdmap e666: 8 total, 8 up, 8 in 2024-09-21T09:41:38.618 INFO:tasks.rados.rados.0.smithi022.stdout:2987: done (5 left) 2024-09-21T09:41:38.618 INFO:tasks.rados.rados.0.smithi022.stdout:2988: done (4 left) 2024-09-21T09:41:38.618 INFO:tasks.rados.rados.0.smithi022.stdout:2989: done (3 left) 2024-09-21T09:41:38.618 INFO:tasks.rados.rados.0.smithi022.stdout:2990: done (2 left) 2024-09-21T09:41:38.618 INFO:tasks.rados.rados.0.smithi022.stdout:2992: done (1 left) 2024-09-21T09:41:38.618 INFO:tasks.rados.rados.0.smithi022.stdout:2993: done (0 left) 2024-09-21T09:41:38.619 INFO:tasks.rados.rados.0.smithi022.stdout:2994: read oid 50 snap 295 2024-09-21T09:41:38.619 INFO:tasks.rados.rados.0.smithi022.stdout:2994: expect (ObjNum 867 snap 280 seq_num 867) 2024-09-21T09:41:38.619 INFO:tasks.rados.rados.0.smithi022.stdout:2995: copy_from oid 12 from oid 46 current snap is 296 2024-09-21T09:41:38.619 INFO:tasks.rados.rados.0.smithi022.stdout:2996: read oid 31 snap -1 2024-09-21T09:41:38.619 INFO:tasks.rados.rados.0.smithi022.stdout:2996: expect deleted 2024-09-21T09:41:38.619 INFO:tasks.rados.rados.0.smithi022.stdout:2997: snap_remove snap 293 2024-09-21T09:41:39.049 INFO:tasks.rados.rados.0.smithi022.stdout:2997: done (3 left) 2024-09-21T09:41:39.049 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 21 current snap is 296 2024-09-21T09:41:39.049 INFO:tasks.rados.rados.0.smithi022.stdout:2998: seq_num 956 ranges {0=417792,417792=434176,851968=770048} 2024-09-21T09:41:39.052 INFO:tasks.rados.rados.0.smithi022.stdout:2998: writing smithi02284960-21 from 0 to 417792 tid 1 2024-09-21T09:41:39.054 INFO:tasks.rados.rados.0.smithi022.stdout:2998: writing smithi02284960-21 from 417792 to 851968 tid 2 2024-09-21T09:41:39.059 INFO:tasks.rados.rados.0.smithi022.stdout:2998: writing smithi02284960-21 from 851968 to 1622016 tid 3 2024-09-21T09:41:39.059 INFO:tasks.rados.rados.0.smithi022.stdout:2999: rollback oid 3 current snap is 296 2024-09-21T09:41:39.059 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 3 to 285 2024-09-21T09:41:39.059 INFO:tasks.rados.rados.0.smithi022.stdout:2995: finishing copy_from racing read to smithi02284960-12 2024-09-21T09:41:39.059 INFO:tasks.rados.rados.0.smithi022.stdout:2995: finishing copy_from to smithi02284960-12 2024-09-21T09:41:39.059 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 12 v 1125 (ObjNum 936 snap 291 seq_num 936) dirty exists 2024-09-21T09:41:39.060 INFO:tasks.rados.rados.0.smithi022.stdout:2994: expect (ObjNum 867 snap 280 seq_num 867) 2024-09-21T09:41:39.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:39 smithi064 ceph-mon[86709]: osdmap e667: 8 total, 8 up, 8 in 2024-09-21T09:41:39.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:39 smithi064 ceph-mon[86709]: pgmap v609: 121 pgs: 121 active+clean; 290 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 2.2 MiB/s wr, 17 op/s 2024-09-21T09:41:39.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:39 smithi064 ceph-mon[86709]: osdmap e668: 8 total, 8 up, 8 in 2024-09-21T09:41:39.557 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:39 smithi022 ceph-mon[96736]: osdmap e667: 8 total, 8 up, 8 in 2024-09-21T09:41:39.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:39 smithi022 ceph-mon[96736]: pgmap v609: 121 pgs: 121 active+clean; 290 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 2.2 MiB/s wr, 17 op/s 2024-09-21T09:41:39.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:39 smithi022 ceph-mon[96736]: osdmap e668: 8 total, 8 up, 8 in 2024-09-21T09:41:39.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:39 smithi022 ceph-mon[98615]: osdmap e667: 8 total, 8 up, 8 in 2024-09-21T09:41:39.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:39 smithi022 ceph-mon[98615]: pgmap v609: 121 pgs: 121 active+clean; 290 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 2.2 MiB/s wr, 17 op/s 2024-09-21T09:41:39.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:39 smithi022 ceph-mon[98615]: osdmap e668: 8 total, 8 up, 8 in 2024-09-21T09:41:40.691 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:41:40 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:41:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:41:41.175 INFO:tasks.rados.rados.0.smithi022.stdout:2998: finishing write tid 1 to smithi02284960-21 2024-09-21T09:41:41.176 INFO:tasks.rados.rados.0.smithi022.stdout:2998: finishing write tid 2 to smithi02284960-21 2024-09-21T09:41:41.176 INFO:tasks.rados.rados.0.smithi022.stdout:2998: finishing write tid 3 to smithi02284960-21 2024-09-21T09:41:41.176 INFO:tasks.rados.rados.0.smithi022.stdout:2998: finishing write tid 4 to smithi02284960-21 2024-09-21T09:41:41.176 INFO:tasks.rados.rados.0.smithi022.stdout:2999: finishing rollback tid 1 to smithi02284960-3 2024-09-21T09:41:41.176 INFO:tasks.rados.rados.0.smithi022.stdout:2999: finishing rollback tid 2 to smithi02284960-3 2024-09-21T09:41:41.176 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 3 v 620 (ObjNum 910 snap 284 seq_num 910) dirty exists 2024-09-21T09:41:41.176 INFO:tasks.rados.rados.0.smithi022.stdout:2998: finishing write tid 5 to smithi02284960-21 2024-09-21T09:41:41.176 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 21 v 1792 (ObjNum 956 snap 296 seq_num 956) dirty exists 2024-09-21T09:41:41.176 INFO:tasks.rados.rados.0.smithi022.stdout:2998: left oid 21 (ObjNum 956 snap 296 seq_num 956) 2024-09-21T09:41:41.176 INFO:tasks.rados.rados.0.smithi022.stdout:2994: done (4 left) 2024-09-21T09:41:41.176 INFO:tasks.rados.rados.0.smithi022.stdout:2995: done (3 left) 2024-09-21T09:41:41.176 INFO:tasks.rados.rados.0.smithi022.stdout:2996: done (2 left) 2024-09-21T09:41:41.176 INFO:tasks.rados.rados.0.smithi022.stdout:2998: done (1 left) 2024-09-21T09:41:41.176 INFO:tasks.rados.rados.0.smithi022.stdout:2999: done (0 left) 2024-09-21T09:41:41.177 INFO:tasks.rados.rados.0.smithi022.stdout:3000: delete oid 24 current snap is 296 2024-09-21T09:41:41.178 INFO:tasks.rados.rados.0.smithi022.stdout:3000: done (0 left) 2024-09-21T09:41:41.178 INFO:tasks.rados.rados.0.smithi022.stdout:3001: read oid 35 snap 264 2024-09-21T09:41:41.178 INFO:tasks.rados.rados.0.smithi022.stdout:3001: expect (ObjNum 825 snap 264 seq_num 825) 2024-09-21T09:41:41.178 INFO:tasks.rados.rados.0.smithi022.stdout:3002: delete oid 45 current snap is 296 2024-09-21T09:41:41.182 INFO:tasks.rados.rados.0.smithi022.stdout:3002: done (1 left) 2024-09-21T09:41:41.182 INFO:tasks.rados.rados.0.smithi022.stdout:3003: snap_remove snap 294 2024-09-21T09:41:41.491 INFO:tasks.rados.rados.0.smithi022.stdout:3003: done (1 left) 2024-09-21T09:41:41.491 INFO:tasks.rados.rados.0.smithi022.stdout:3004: read oid 13 snap 285 2024-09-21T09:41:41.491 INFO:tasks.rados.rados.0.smithi022.stdout:3004: expect (ObjNum 907 snap 284 seq_num 907) 2024-09-21T09:41:41.491 INFO:tasks.rados.rados.0.smithi022.stdout:3005: copy_from oid 28 from oid 31 current snap is 296 2024-09-21T09:41:41.491 INFO:tasks.rados.rados.0.smithi022.stdout:3006: rollback oid 27 current snap is 296 2024-09-21T09:41:41.492 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 27 to 281 2024-09-21T09:41:41.492 INFO:tasks.rados.rados.0.smithi022.stdout:3007: read oid 10 snap -1 2024-09-21T09:41:41.492 INFO:tasks.rados.rados.0.smithi022.stdout:3007: expect (ObjNum 880 snap 282 seq_num 880) 2024-09-21T09:41:41.492 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 30 current snap is 296 2024-09-21T09:41:41.492 INFO:tasks.rados.rados.0.smithi022.stdout:3008: seq_num 959 ranges {286720=466944,753664=540672,1294336=557056,1851392=106496} 2024-09-21T09:41:41.494 INFO:tasks.rados.rados.0.smithi022.stdout:3008: writing smithi02284960-30 from 286720 to 753664 tid 1 2024-09-21T09:41:41.498 INFO:tasks.rados.rados.0.smithi022.stdout:3008: writing smithi02284960-30 from 753664 to 1294336 tid 2 2024-09-21T09:41:41.502 INFO:tasks.rados.rados.0.smithi022.stdout:3008: writing smithi02284960-30 from 1294336 to 1851392 tid 3 2024-09-21T09:41:41.503 INFO:tasks.rados.rados.0.smithi022.stdout:3008: writing smithi02284960-30 from 1851392 to 1957888 tid 4 2024-09-21T09:41:41.503 INFO:tasks.rados.rados.0.smithi022.stdout:3009: rollback oid 42 current snap is 296 2024-09-21T09:41:41.503 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 42 to 264 2024-09-21T09:41:41.503 INFO:tasks.rados.rados.0.smithi022.stdout:3001: expect (ObjNum 825 snap 264 seq_num 825) 2024-09-21T09:41:41.625 INFO:tasks.rados.rados.0.smithi022.stdout:3005: finishing copy_from to smithi02284960-28 2024-09-21T09:41:41.625 INFO:tasks.rados.rados.0.smithi022.stdout:3005: got expected ENOENT (src dne) 2024-09-21T09:41:41.626 INFO:tasks.rados.rados.0.smithi022.stdout:3005: finishing copy_from racing read to smithi02284960-28 2024-09-21T09:41:41.626 INFO:tasks.rados.rados.0.smithi022.stdout:3008: finishing write tid 1 to smithi02284960-30 2024-09-21T09:41:41.626 INFO:tasks.rados.rados.0.smithi022.stdout:3008: finishing write tid 2 to smithi02284960-30 2024-09-21T09:41:41.626 INFO:tasks.rados.rados.0.smithi022.stdout:3004: expect (ObjNum 907 snap 284 seq_num 907) 2024-09-21T09:41:41.648 INFO:tasks.rados.rados.0.smithi022.stdout:3006: finishing rollback tid 1 to smithi02284960-27 2024-09-21T09:41:41.648 INFO:tasks.rados.rados.0.smithi022.stdout:3008: finishing write tid 3 to smithi02284960-30 2024-09-21T09:41:41.648 INFO:tasks.rados.rados.0.smithi022.stdout:3008: finishing write tid 4 to smithi02284960-30 2024-09-21T09:41:41.648 INFO:tasks.rados.rados.0.smithi022.stdout:3006: finishing rollback tid 2 to smithi02284960-27 2024-09-21T09:41:41.648 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 27 v 1682 (ObjNum 773 snap 242 seq_num 773) dirty exists 2024-09-21T09:41:41.648 INFO:tasks.rados.rados.0.smithi022.stdout:3009: finishing rollback tid 1 to smithi02284960-42 2024-09-21T09:41:41.648 INFO:tasks.rados.rados.0.smithi022.stdout:3008: finishing write tid 5 to smithi02284960-30 2024-09-21T09:41:41.649 INFO:tasks.rados.rados.0.smithi022.stdout:3009: finishing rollback tid 2 to smithi02284960-42 2024-09-21T09:41:41.649 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 42 v 1684 (ObjNum 2193 snap 0 seq_num 3570040736) dirty exists 2024-09-21T09:41:41.649 INFO:tasks.rados.rados.0.smithi022.stdout:3007: expect (ObjNum 880 snap 282 seq_num 880) 2024-09-21T09:41:41.795 INFO:tasks.rados.rados.0.smithi022.stdout:3008: finishing write tid 6 to smithi02284960-30 2024-09-21T09:41:41.796 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 30 v 1322 (ObjNum 959 snap 296 seq_num 959) dirty exists 2024-09-21T09:41:41.796 INFO:tasks.rados.rados.0.smithi022.stdout:3008: left oid 30 (ObjNum 959 snap 296 seq_num 959) 2024-09-21T09:41:41.796 INFO:tasks.rados.rados.0.smithi022.stdout:3001: done (6 left) 2024-09-21T09:41:41.796 INFO:tasks.rados.rados.0.smithi022.stdout:3004: done (5 left) 2024-09-21T09:41:41.796 INFO:tasks.rados.rados.0.smithi022.stdout:3005: done (4 left) 2024-09-21T09:41:41.797 INFO:tasks.rados.rados.0.smithi022.stdout:3006: done (3 left) 2024-09-21T09:41:41.797 INFO:tasks.rados.rados.0.smithi022.stdout:3007: done (2 left) 2024-09-21T09:41:41.797 INFO:tasks.rados.rados.0.smithi022.stdout:3008: done (1 left) 2024-09-21T09:41:41.797 INFO:tasks.rados.rados.0.smithi022.stdout:3009: done (0 left) 2024-09-21T09:41:41.797 INFO:tasks.rados.rados.0.smithi022.stdout:3010: read oid 35 snap -1 2024-09-21T09:41:41.797 INFO:tasks.rados.rados.0.smithi022.stdout:3010: expect (ObjNum 877 snap 282 seq_num 877) 2024-09-21T09:41:41.797 INFO:tasks.rados.rados.0.smithi022.stdout:3011: copy_from oid 34 from oid 3 current snap is 296 2024-09-21T09:41:41.798 INFO:tasks.rados.rados.0.smithi022.stdout:3012: snap_remove snap 285 2024-09-21T09:41:41.807 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:41 smithi022 ceph-mon[98615]: pgmap v611: 121 pgs: 121 active+clean; 294 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 5.7 MiB/s rd, 3.3 MiB/s wr, 22 op/s 2024-09-21T09:41:41.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:41 smithi022 ceph-mon[96736]: pgmap v611: 121 pgs: 121 active+clean; 294 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 5.7 MiB/s rd, 3.3 MiB/s wr, 22 op/s 2024-09-21T09:41:41.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:41 smithi064 ceph-mon[86709]: pgmap v611: 121 pgs: 121 active+clean; 294 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 5.7 MiB/s rd, 3.3 MiB/s wr, 22 op/s 2024-09-21T09:41:42.496 INFO:tasks.rados.rados.0.smithi022.stdout:3012: done (2 left) 2024-09-21T09:41:42.496 INFO:tasks.rados.rados.0.smithi022.stdout:3013: rollback oid 14 current snap is 296 2024-09-21T09:41:42.496 INFO:tasks.rados.rados.0.smithi022.stdout:3011: finishing copy_from racing read to smithi02284960-34 2024-09-21T09:41:42.496 INFO:tasks.rados.rados.0.smithi022.stdout:3011: finishing copy_from to smithi02284960-34 2024-09-21T09:41:42.496 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 34 v 620 (ObjNum 910 snap 284 seq_num 910) dirty exists 2024-09-21T09:41:42.496 INFO:tasks.rados.rados.0.smithi022.stdout:3010: expect (ObjNum 877 snap 282 seq_num 877) 2024-09-21T09:41:42.807 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:42 smithi022 ceph-mon[98615]: osdmap e669: 8 total, 8 up, 8 in 2024-09-21T09:41:42.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:42 smithi022 ceph-mon[96736]: osdmap e669: 8 total, 8 up, 8 in 2024-09-21T09:41:42.941 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 14 to 264 2024-09-21T09:41:42.941 INFO:tasks.rados.rados.0.smithi022.stdout:3010: done (2 left) 2024-09-21T09:41:42.941 INFO:tasks.rados.rados.0.smithi022.stdout:3011: done (1 left) 2024-09-21T09:41:42.941 INFO:tasks.rados.rados.0.smithi022.stdout:3014: rollback oid 27 current snap is 296 2024-09-21T09:41:42.941 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 27 to 295 2024-09-21T09:41:42.941 INFO:tasks.rados.rados.0.smithi022.stdout:3015: rollback oid 29 current snap is 296 2024-09-21T09:41:42.942 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 29 to 264 2024-09-21T09:41:42.942 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 38 current snap is 296 2024-09-21T09:41:42.942 INFO:tasks.rados.rados.0.smithi022.stdout:3016: seq_num 960 ranges {3497984=483328,3981312=425984,4407296=761856,5169152=16384} 2024-09-21T09:41:42.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:42 smithi064 ceph-mon[86709]: osdmap e669: 8 total, 8 up, 8 in 2024-09-21T09:41:42.963 INFO:tasks.rados.rados.0.smithi022.stdout:3016: writing smithi02284960-38 from 3497984 to 3981312 tid 1 2024-09-21T09:41:42.965 INFO:tasks.rados.rados.0.smithi022.stdout:3016: writing smithi02284960-38 from 3981312 to 4407296 tid 2 2024-09-21T09:41:42.970 INFO:tasks.rados.rados.0.smithi022.stdout:3016: writing smithi02284960-38 from 4407296 to 5169152 tid 3 2024-09-21T09:41:42.970 INFO:tasks.rados.rados.0.smithi022.stdout:3016: writing smithi02284960-38 from 5169152 to 5185536 tid 4 2024-09-21T09:41:42.970 INFO:tasks.rados.rados.0.smithi022.stdout:3017: read oid 49 snap -1 2024-09-21T09:41:42.970 INFO:tasks.rados.rados.0.smithi022.stdout:3017: expect (ObjNum 936 snap 291 seq_num 936) 2024-09-21T09:41:42.970 INFO:tasks.rados.rados.0.smithi022.stdout:3018: snap_remove snap 281 2024-09-21T09:41:43.498 INFO:tasks.rados.rados.0.smithi022.stdout:3018: done (5 left) 2024-09-21T09:41:43.498 INFO:tasks.rados.rados.0.smithi022.stdout:3019: read oid 21 snap -1 2024-09-21T09:41:43.498 INFO:tasks.rados.rados.0.smithi022.stdout:3019: expect (ObjNum 956 snap 296 seq_num 956) 2024-09-21T09:41:43.498 INFO:tasks.rados.rados.0.smithi022.stdout:3020: delete oid 19 current snap is 296 2024-09-21T09:41:43.498 INFO:tasks.rados.rados.0.smithi022.stdout:3013: finishing rollback tid 1 to smithi02284960-14 2024-09-21T09:41:43.499 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 14 v 1126 (ObjNum 161 snap 0 seq_num 0) dirty dne 2024-09-21T09:41:43.499 INFO:tasks.rados.rados.0.smithi022.stdout:3015: finishing rollback tid 0 to smithi02284960-29 2024-09-21T09:41:43.499 INFO:tasks.rados.rados.0.smithi022.stdout:3014: finishing rollback tid 0 to smithi02284960-27 2024-09-21T09:41:43.499 INFO:tasks.rados.rados.0.smithi022.stdout:3015: finishing rollback tid 1 to smithi02284960-29 2024-09-21T09:41:43.499 INFO:tasks.rados.rados.0.smithi022.stdout:3014: finishing rollback tid 1 to smithi02284960-27 2024-09-21T09:41:43.499 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 27 v 1686 (ObjNum 129 snap 0 seq_num 2956539409) dirty dne 2024-09-21T09:41:43.499 INFO:tasks.rados.rados.0.smithi022.stdout:3015: finishing rollback tid 2 to smithi02284960-29 2024-09-21T09:41:43.499 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 29 v 1792 (ObjNum 827 snap 264 seq_num 827) dirty exists 2024-09-21T09:41:43.499 INFO:tasks.rados.rados.0.smithi022.stdout:3016: finishing write tid 1 to smithi02284960-38 2024-09-21T09:41:43.499 INFO:tasks.rados.rados.0.smithi022.stdout:3016: finishing write tid 2 to smithi02284960-38 2024-09-21T09:41:43.499 INFO:tasks.rados.rados.0.smithi022.stdout:3016: finishing write tid 3 to smithi02284960-38 2024-09-21T09:41:43.499 INFO:tasks.rados.rados.0.smithi022.stdout:3016: finishing write tid 4 to smithi02284960-38 2024-09-21T09:41:43.499 INFO:tasks.rados.rados.0.smithi022.stdout:3016: finishing write tid 5 to smithi02284960-38 2024-09-21T09:41:43.499 INFO:tasks.rados.rados.0.smithi022.stdout:3016: finishing write tid 6 to smithi02284960-38 2024-09-21T09:41:43.499 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 38 v 1797 (ObjNum 960 snap 296 seq_num 960) dirty exists 2024-09-21T09:41:43.500 INFO:tasks.rados.rados.0.smithi022.stdout:3016: left oid 38 (ObjNum 960 snap 296 seq_num 960) 2024-09-21T09:41:43.500 INFO:tasks.rados.rados.0.smithi022.stdout:3017: expect (ObjNum 936 snap 291 seq_num 936) 2024-09-21T09:41:43.545 INFO:tasks.rados.rados.0.smithi022.stdout:3019: expect (ObjNum 956 snap 296 seq_num 956) 2024-09-21T09:41:43.589 INFO:tasks.rados.rados.0.smithi022.stdout:3013: done (6 left) 2024-09-21T09:41:43.589 INFO:tasks.rados.rados.0.smithi022.stdout:3014: done (5 left) 2024-09-21T09:41:43.589 INFO:tasks.rados.rados.0.smithi022.stdout:3015: done (4 left) 2024-09-21T09:41:43.589 INFO:tasks.rados.rados.0.smithi022.stdout:3016: done (3 left) 2024-09-21T09:41:43.589 INFO:tasks.rados.rados.0.smithi022.stdout:3017: done (2 left) 2024-09-21T09:41:43.590 INFO:tasks.rados.rados.0.smithi022.stdout:3019: done (1 left) 2024-09-21T09:41:43.590 INFO:tasks.rados.rados.0.smithi022.stdout:3020: done (0 left) 2024-09-21T09:41:43.590 INFO:tasks.rados.rados.0.smithi022.stdout:3021: snap_remove snap 264 2024-09-21T09:41:43.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:43 smithi022 ceph-mon[98615]: pgmap v613: 121 pgs: 121 active+clean; 294 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 2.6 MiB/s wr, 20 op/s 2024-09-21T09:41:43.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:43 smithi022 ceph-mon[98615]: osdmap e670: 8 total, 8 up, 8 in 2024-09-21T09:41:43.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:43 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:41:43.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:43 smithi022 ceph-mon[96736]: pgmap v613: 121 pgs: 121 active+clean; 294 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 2.6 MiB/s wr, 20 op/s 2024-09-21T09:41:43.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:43 smithi022 ceph-mon[96736]: osdmap e670: 8 total, 8 up, 8 in 2024-09-21T09:41:43.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:43 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:41:43.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:43 smithi064 ceph-mon[86709]: pgmap v613: 121 pgs: 121 active+clean; 294 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 2.6 MiB/s wr, 20 op/s 2024-09-21T09:41:43.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:43 smithi064 ceph-mon[86709]: osdmap e670: 8 total, 8 up, 8 in 2024-09-21T09:41:43.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:43 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:41:44.052 INFO:tasks.rados.rados.0.smithi022.stdout:3021: done (0 left) 2024-09-21T09:41:44.052 INFO:tasks.rados.rados.0.smithi022.stdout:3022: rollback oid 37 current snap is 296 2024-09-21T09:41:44.053 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 37 to 295 2024-09-21T09:41:44.053 INFO:tasks.rados.rados.0.smithi022.stdout:3023: copy_from oid 8 from oid 36 current snap is 296 2024-09-21T09:41:44.053 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 31 current snap is 296 2024-09-21T09:41:44.053 INFO:tasks.rados.rados.0.smithi022.stdout:3024: seq_num 962 ranges {0=516096,516096=589824,1105920=614400} 2024-09-21T09:41:44.056 INFO:tasks.rados.rados.0.smithi022.stdout:3024: writing smithi02284960-31 from 0 to 516096 tid 1 2024-09-21T09:41:44.059 INFO:tasks.rados.rados.0.smithi022.stdout:3024: writing smithi02284960-31 from 516096 to 1105920 tid 2 2024-09-21T09:41:44.064 INFO:tasks.rados.rados.0.smithi022.stdout:3024: writing smithi02284960-31 from 1105920 to 1720320 tid 3 2024-09-21T09:41:44.064 INFO:tasks.rados.rados.0.smithi022.stdout:3025: rmattr oid 17 current snap is 296 2024-09-21T09:41:44.064 INFO:tasks.rados.rados.0.smithi022.stdout:3026: delete oid 45 current snap is 296 2024-09-21T09:41:44.073 INFO:tasks.rados.rados.0.smithi022.stdout:3022: finishing rollback tid 0 to smithi02284960-37 2024-09-21T09:41:44.073 INFO:tasks.rados.rados.0.smithi022.stdout:3022: finishing rollback tid 1 to smithi02284960-37 2024-09-21T09:41:44.075 INFO:tasks.rados.rados.0.smithi022.stdout:3022: finishing rollback tid 2 to smithi02284960-37 2024-09-21T09:41:44.075 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 37 v 1794 (ObjNum 924 snap 287 seq_num 924) dirty exists 2024-09-21T09:41:44.075 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 17 v 1323 (ObjNum 842 snap 269 seq_num 842) dirty exists 2024-09-21T09:41:44.076 INFO:tasks.rados.rados.0.smithi022.stdout:3022: done (4 left) 2024-09-21T09:41:44.076 INFO:tasks.rados.rados.0.smithi022.stdout:3025: done (3 left) 2024-09-21T09:41:44.076 INFO:tasks.rados.rados.0.smithi022.stdout:3026: done (2 left) 2024-09-21T09:41:44.076 INFO:tasks.rados.rados.0.smithi022.stdout:3027: snap_remove snap 295 2024-09-21T09:41:44.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:44 smithi022 ceph-mon[98615]: osdmap e671: 8 total, 8 up, 8 in 2024-09-21T09:41:44.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:44 smithi022 ceph-mon[98615]: osdmap e672: 8 total, 8 up, 8 in 2024-09-21T09:41:44.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:44 smithi022 ceph-mon[96736]: osdmap e671: 8 total, 8 up, 8 in 2024-09-21T09:41:44.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:44 smithi022 ceph-mon[96736]: osdmap e672: 8 total, 8 up, 8 in 2024-09-21T09:41:44.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:44 smithi064 ceph-mon[86709]: osdmap e671: 8 total, 8 up, 8 in 2024-09-21T09:41:44.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:44 smithi064 ceph-mon[86709]: osdmap e672: 8 total, 8 up, 8 in 2024-09-21T09:41:45.056 INFO:tasks.rados.rados.0.smithi022.stdout:3027: done (2 left) 2024-09-21T09:41:45.056 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 27 current snap is 296 2024-09-21T09:41:45.056 INFO:tasks.rados.rados.0.smithi022.stdout:3028: seq_num 964 ranges {0=548864,548864=761856,1310720=442368} 2024-09-21T09:41:45.060 INFO:tasks.rados.rados.0.smithi022.stdout:3028: writing smithi02284960-27 from 0 to 548864 tid 1 2024-09-21T09:41:45.065 INFO:tasks.rados.rados.0.smithi022.stdout:3028: writing smithi02284960-27 from 548864 to 1310720 tid 2 2024-09-21T09:41:45.070 INFO:tasks.rados.rados.0.smithi022.stdout:3028: writing smithi02284960-27 from 1310720 to 1753088 tid 3 2024-09-21T09:41:45.070 INFO:tasks.rados.rados.0.smithi022.stdout:3029: rmattr oid 47 current snap is 296 2024-09-21T09:41:45.070 INFO:tasks.rados.rados.0.smithi022.stdout:3024: finishing write tid 1 to smithi02284960-31 2024-09-21T09:41:45.070 INFO:tasks.rados.rados.0.smithi022.stdout:3024: finishing write tid 2 to smithi02284960-31 2024-09-21T09:41:45.070 INFO:tasks.rados.rados.0.smithi022.stdout:3024: finishing write tid 3 to smithi02284960-31 2024-09-21T09:41:45.070 INFO:tasks.rados.rados.0.smithi022.stdout:3024: finishing write tid 4 to smithi02284960-31 2024-09-21T09:41:45.070 INFO:tasks.rados.rados.0.smithi022.stdout:3023: finishing copy_from racing read to smithi02284960-8 2024-09-21T09:41:45.070 INFO:tasks.rados.rados.0.smithi022.stdout:3024: finishing write tid 5 to smithi02284960-31 2024-09-21T09:41:45.070 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 31 v 1798 (ObjNum 962 snap 296 seq_num 962) dirty exists 2024-09-21T09:41:45.070 INFO:tasks.rados.rados.0.smithi022.stdout:3024: left oid 31 (ObjNum 962 snap 296 seq_num 962) 2024-09-21T09:41:45.070 INFO:tasks.rados.rados.0.smithi022.stdout:3023: finishing copy_from to smithi02284960-8 2024-09-21T09:41:45.070 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 8 v 1416 (ObjNum 919 snap 286 seq_num 919) dirty exists 2024-09-21T09:41:45.071 INFO:tasks.rados.rados.0.smithi022.stdout:3023: done (3 left) 2024-09-21T09:41:45.071 INFO:tasks.rados.rados.0.smithi022.stdout:3024: done (2 left) 2024-09-21T09:41:45.071 INFO:tasks.rados.rados.0.smithi022.stdout:3030: rollback oid 30 current snap is 296 2024-09-21T09:41:45.071 INFO:tasks.rados.rados.0.smithi022.stdout:3030: done (2 left) 2024-09-21T09:41:45.071 INFO:tasks.rados.rados.0.smithi022.stdout:3031: delete oid 2 current snap is 296 2024-09-21T09:41:45.071 INFO:tasks.rados.rados.0.smithi022.stdout:3028: finishing write tid 1 to smithi02284960-27 2024-09-21T09:41:45.073 INFO:tasks.rados.rados.0.smithi022.stdout:3031: done (2 left) 2024-09-21T09:41:45.073 INFO:tasks.rados.rados.0.smithi022.stdout:3032: rollback oid 43 current snap is 296 2024-09-21T09:41:45.073 INFO:tasks.rados.rados.0.smithi022.stdout:3032: done (2 left) 2024-09-21T09:41:45.073 INFO:tasks.rados.rados.0.smithi022.stdout:3033: copy_from oid 28 from oid 29 current snap is 296 2024-09-21T09:41:45.073 INFO:tasks.rados.rados.0.smithi022.stdout:3034: delete oid 7 current snap is 296 2024-09-21T09:41:45.074 INFO:tasks.rados.rados.0.smithi022.stdout:3028: finishing write tid 2 to smithi02284960-27 2024-09-21T09:41:45.075 INFO:tasks.rados.rados.0.smithi022.stdout:3028: finishing write tid 3 to smithi02284960-27 2024-09-21T09:41:45.075 INFO:tasks.rados.rados.0.smithi022.stdout:3028: finishing write tid 4 to smithi02284960-27 2024-09-21T09:41:45.077 INFO:tasks.rados.rados.0.smithi022.stdout:3028: finishing write tid 5 to smithi02284960-27 2024-09-21T09:41:45.077 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 27 v 1690 (ObjNum 964 snap 296 seq_num 964) dirty exists 2024-09-21T09:41:45.077 INFO:tasks.rados.rados.0.smithi022.stdout:3028: left oid 27 (ObjNum 964 snap 296 seq_num 964) 2024-09-21T09:41:45.077 INFO:tasks.rados.rados.0.smithi022.stdout:3028: done (3 left) 2024-09-21T09:41:45.078 INFO:tasks.rados.rados.0.smithi022.stdout:3034: done (2 left) 2024-09-21T09:41:45.078 INFO:tasks.rados.rados.0.smithi022.stdout:3035: copy_from oid 23 from oid 43 current snap is 296 2024-09-21T09:41:45.078 INFO:tasks.rados.rados.0.smithi022.stdout:3036: delete oid 37 current snap is 296 2024-09-21T09:41:45.078 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 47 v 1668 (ObjNum 884 snap 282 seq_num 884) dirty exists 2024-09-21T09:41:45.081 INFO:tasks.rados.rados.0.smithi022.stdout:3035: finishing copy_from to smithi02284960-23 2024-09-21T09:41:45.081 INFO:tasks.rados.rados.0.smithi022.stdout:3035: got expected ENOENT (src dne) 2024-09-21T09:41:45.081 INFO:tasks.rados.rados.0.smithi022.stdout:3029: done (3 left) 2024-09-21T09:41:45.082 INFO:tasks.rados.rados.0.smithi022.stdout:3036: done (2 left) 2024-09-21T09:41:45.082 INFO:tasks.rados.rados.0.smithi022.stdout:3037: read oid 35 snap -1 2024-09-21T09:41:45.082 INFO:tasks.rados.rados.0.smithi022.stdout:3037: expect (ObjNum 877 snap 282 seq_num 877) 2024-09-21T09:41:45.082 INFO:tasks.rados.rados.0.smithi022.stdout:3038: rmattr oid 8 current snap is 296 2024-09-21T09:41:45.082 INFO:tasks.rados.rados.0.smithi022.stdout:3039: setattr oid 24 current snap is 296 2024-09-21T09:41:45.082 INFO:tasks.rados.rados.0.smithi022.stdout:3035: finishing copy_from racing read to smithi02284960-23 2024-09-21T09:41:45.084 INFO:tasks.rados.rados.0.smithi022.stdout:3035: done (4 left) 2024-09-21T09:41:45.084 INFO:tasks.rados.rados.0.smithi022.stdout:3040: setattr oid 4 current snap is 296 2024-09-21T09:41:45.087 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 8 v 1672 (ObjNum 919 snap 286 seq_num 919) dirty exists 2024-09-21T09:41:45.087 INFO:tasks.rados.rados.0.smithi022.stdout:3038: done (4 left) 2024-09-21T09:41:45.087 INFO:tasks.rados.rados.0.smithi022.stdout:3041: read oid 44 snap -1 2024-09-21T09:41:45.087 INFO:tasks.rados.rados.0.smithi022.stdout:3041: expect (ObjNum 886 snap 282 seq_num 886) 2024-09-21T09:41:45.087 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 20 current snap is 296 2024-09-21T09:41:45.087 INFO:tasks.rados.rados.0.smithi022.stdout:3042: seq_num 968 ranges {0=614400,614400=745472,1359872=458752} 2024-09-21T09:41:45.091 INFO:tasks.rados.rados.0.smithi022.stdout:3042: writing smithi02284960-20 from 0 to 614400 tid 1 2024-09-21T09:41:45.096 INFO:tasks.rados.rados.0.smithi022.stdout:3042: writing smithi02284960-20 from 614400 to 1359872 tid 2 2024-09-21T09:41:45.100 INFO:tasks.rados.rados.0.smithi022.stdout:3042: writing smithi02284960-20 from 1359872 to 1818624 tid 3 2024-09-21T09:41:45.100 INFO:tasks.rados.rados.0.smithi022.stdout:3043: copy_from oid 9 from oid 50 current snap is 296 2024-09-21T09:41:45.100 INFO:tasks.rados.rados.0.smithi022.stdout:3044: delete oid 43 current snap is 296 2024-09-21T09:41:45.100 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 24 v 1808 (ObjNum 785 snap 0 seq_num 3412041696) dirty exists 2024-09-21T09:41:45.100 INFO:tasks.rados.rados.0.smithi022.stdout:3033: finishing copy_from racing read to smithi02284960-28 2024-09-21T09:41:45.100 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 4 v 1673 (ObjNum 945 snap 295 seq_num 945) dirty exists 2024-09-21T09:41:45.100 INFO:tasks.rados.rados.0.smithi022.stdout:3041: expect (ObjNum 886 snap 282 seq_num 886) 2024-09-21T09:41:45.116 INFO:tasks.rados.rados.0.smithi022.stdout:3042: finishing write tid 1 to smithi02284960-20 2024-09-21T09:41:45.116 INFO:tasks.rados.rados.0.smithi022.stdout:3042: finishing write tid 2 to smithi02284960-20 2024-09-21T09:41:45.116 INFO:tasks.rados.rados.0.smithi022.stdout:3042: finishing write tid 3 to smithi02284960-20 2024-09-21T09:41:45.116 INFO:tasks.rados.rados.0.smithi022.stdout:3042: finishing write tid 4 to smithi02284960-20 2024-09-21T09:41:45.116 INFO:tasks.rados.rados.0.smithi022.stdout:3037: expect (ObjNum 877 snap 282 seq_num 877) 2024-09-21T09:41:45.552 INFO:tasks.rados.rados.0.smithi022.stdout:3033: finishing copy_from to smithi02284960-28 2024-09-21T09:41:45.552 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 28 v 1792 (ObjNum 827 snap 264 seq_num 827) dirty exists 2024-09-21T09:41:45.552 INFO:tasks.rados.rados.0.smithi022.stdout:3042: finishing write tid 5 to smithi02284960-20 2024-09-21T09:41:45.552 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 20 v 1907 (ObjNum 968 snap 296 seq_num 968) dirty exists 2024-09-21T09:41:45.552 INFO:tasks.rados.rados.0.smithi022.stdout:3042: left oid 20 (ObjNum 968 snap 296 seq_num 968) 2024-09-21T09:41:45.552 INFO:tasks.rados.rados.0.smithi022.stdout:3033: done (7 left) 2024-09-21T09:41:45.552 INFO:tasks.rados.rados.0.smithi022.stdout:3037: done (6 left) 2024-09-21T09:41:45.553 INFO:tasks.rados.rados.0.smithi022.stdout:3039: done (5 left) 2024-09-21T09:41:45.553 INFO:tasks.rados.rados.0.smithi022.stdout:3040: done (4 left) 2024-09-21T09:41:45.553 INFO:tasks.rados.rados.0.smithi022.stdout:3041: done (3 left) 2024-09-21T09:41:45.553 INFO:tasks.rados.rados.0.smithi022.stdout:3042: done (2 left) 2024-09-21T09:41:45.553 INFO:tasks.rados.rados.0.smithi022.stdout:3044: done (1 left) 2024-09-21T09:41:45.553 INFO:tasks.rados.rados.0.smithi022.stdout:3045: copy_from oid 35 from oid 48 current snap is 296 2024-09-21T09:41:45.553 INFO:tasks.rados.rados.0.smithi022.stdout:3043: finishing copy_from racing read to smithi02284960-9 2024-09-21T09:41:45.553 INFO:tasks.rados.rados.0.smithi022.stdout:3043: finishing copy_from to smithi02284960-9 2024-09-21T09:41:45.553 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 9 v 1404 (ObjNum 867 snap 280 seq_num 867) dirty exists 2024-09-21T09:41:45.553 INFO:tasks.rados.rados.0.smithi022.stdout:3043: done (1 left) 2024-09-21T09:41:45.553 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 26 current snap is 296 2024-09-21T09:41:45.553 INFO:tasks.rados.rados.0.smithi022.stdout:3046: seq_num 970 ranges {4218880=655360,4874240=516096,5390336=688128} 2024-09-21T09:41:45.578 INFO:tasks.rados.rados.0.smithi022.stdout:3046: writing smithi02284960-26 from 4218880 to 4874240 tid 1 2024-09-21T09:41:45.581 INFO:tasks.rados.rados.0.smithi022.stdout:3046: writing smithi02284960-26 from 4874240 to 5390336 tid 2 2024-09-21T09:41:45.585 INFO:tasks.rados.rados.0.smithi022.stdout:3046: writing smithi02284960-26 from 5390336 to 6078464 tid 3 2024-09-21T09:41:45.585 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 11 current snap is 296 2024-09-21T09:41:45.585 INFO:tasks.rados.rados.0.smithi022.stdout:3045: finishing copy_from racing read to smithi02284960-35 2024-09-21T09:41:45.585 INFO:tasks.rados.rados.0.smithi022.stdout:3045: finishing copy_from to smithi02284960-35 2024-09-21T09:41:45.585 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 35 v 1784 (ObjNum 925 snap 287 seq_num 925) dirty exists 2024-09-21T09:41:45.585 INFO:tasks.rados.rados.0.smithi022.stdout:3046: finishing write tid 1 to smithi02284960-26 2024-09-21T09:41:45.585 INFO:tasks.rados.rados.0.smithi022.stdout:3046: finishing write tid 2 to smithi02284960-26 2024-09-21T09:41:45.586 INFO:tasks.rados.rados.0.smithi022.stdout:3047: seq_num 971 ranges {0=671744,671744=802816,1474560=401408} 2024-09-21T09:41:45.588 INFO:tasks.rados.rados.0.smithi022.stdout:3047: writing smithi02284960-11 from 0 to 671744 tid 1 2024-09-21T09:41:45.593 INFO:tasks.rados.rados.0.smithi022.stdout:3047: writing smithi02284960-11 from 671744 to 1474560 tid 2 2024-09-21T09:41:45.595 INFO:tasks.rados.rados.0.smithi022.stdout:3047: writing smithi02284960-11 from 1474560 to 1875968 tid 3 2024-09-21T09:41:45.595 INFO:tasks.rados.rados.0.smithi022.stdout:3045: done (2 left) 2024-09-21T09:41:45.596 INFO:tasks.rados.rados.0.smithi022.stdout:3048: copy_from oid 32 from oid 47 current snap is 296 2024-09-21T09:41:45.596 INFO:tasks.rados.rados.0.smithi022.stdout:3046: finishing write tid 3 to smithi02284960-26 2024-09-21T09:41:45.596 INFO:tasks.rados.rados.0.smithi022.stdout:3046: finishing write tid 4 to smithi02284960-26 2024-09-21T09:41:45.596 INFO:tasks.rados.rados.0.smithi022.stdout:3046: finishing write tid 5 to smithi02284960-26 2024-09-21T09:41:45.596 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 26 v 1694 (ObjNum 970 snap 296 seq_num 970) dirty exists 2024-09-21T09:41:45.596 INFO:tasks.rados.rados.0.smithi022.stdout:3046: left oid 26 (ObjNum 970 snap 296 seq_num 970) 2024-09-21T09:41:45.596 INFO:tasks.rados.rados.0.smithi022.stdout:3047: finishing write tid 1 to smithi02284960-11 2024-09-21T09:41:45.596 INFO:tasks.rados.rados.0.smithi022.stdout:3046: done (2 left) 2024-09-21T09:41:45.596 INFO:tasks.rados.rados.0.smithi022.stdout:3049: rollback oid 24 current snap is 296 2024-09-21T09:41:45.596 INFO:tasks.rados.rados.0.smithi022.stdout:3049: done (2 left) 2024-09-21T09:41:45.596 INFO:tasks.rados.rados.0.smithi022.stdout:3050: snap_create 2024-09-21T09:41:45.598 INFO:tasks.rados.rados.0.smithi022.stdout:3047: finishing write tid 2 to smithi02284960-11 2024-09-21T09:41:45.598 INFO:tasks.rados.rados.0.smithi022.stdout:3047: finishing write tid 3 to smithi02284960-11 2024-09-21T09:41:45.598 INFO:tasks.rados.rados.0.smithi022.stdout:3047: finishing write tid 4 to smithi02284960-11 2024-09-21T09:41:45.603 INFO:tasks.rados.rados.0.smithi022.stdout:3047: finishing write tid 5 to smithi02284960-11 2024-09-21T09:41:45.604 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 11 v 1672 (ObjNum 971 snap 296 seq_num 971) dirty exists 2024-09-21T09:41:45.604 INFO:tasks.rados.rados.0.smithi022.stdout:3047: left oid 11 (ObjNum 971 snap 296 seq_num 971) 2024-09-21T09:41:45.612 INFO:tasks.rados.rados.0.smithi022.stdout:3048: finishing copy_from racing read to smithi02284960-32 2024-09-21T09:41:45.613 INFO:tasks.rados.rados.0.smithi022.stdout:3048: finishing copy_from to smithi02284960-32 2024-09-21T09:41:45.613 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 32 v 1668 (ObjNum 884 snap 282 seq_num 884) dirty exists 2024-09-21T09:41:46.058 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:41:45 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:41:45] "GET /metrics HTTP/1.1" 200 36352 "" "Prometheus/2.51.0" 2024-09-21T09:41:46.059 INFO:tasks.rados.rados.0.smithi022.stdout:3047: done (2 left) 2024-09-21T09:41:46.059 INFO:tasks.rados.rados.0.smithi022.stdout:3048: done (1 left) 2024-09-21T09:41:46.059 INFO:tasks.rados.rados.0.smithi022.stdout:3050: done (0 left) 2024-09-21T09:41:46.059 INFO:tasks.rados.rados.0.smithi022.stdout:3051: read oid 20 snap -1 2024-09-21T09:41:46.059 INFO:tasks.rados.rados.0.smithi022.stdout:3051: expect (ObjNum 968 snap 296 seq_num 968) 2024-09-21T09:41:46.059 INFO:tasks.rados.rados.0.smithi022.stdout:3052: rollback oid 8 current snap is 297 2024-09-21T09:41:46.059 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 8 to 296 2024-09-21T09:41:46.059 INFO:tasks.rados.rados.0.smithi022.stdout:3053: copy_from oid 27 from oid 4 current snap is 297 2024-09-21T09:41:46.059 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 24 current snap is 297 2024-09-21T09:41:46.059 INFO:tasks.rados.rados.0.smithi022.stdout:3054: seq_num 972 ranges {0=688128,688128=688128,1376256=516096} 2024-09-21T09:41:46.063 INFO:tasks.rados.rados.0.smithi022.stdout:3054: writing smithi02284960-24 from 0 to 688128 tid 1 2024-09-21T09:41:46.068 INFO:tasks.rados.rados.0.smithi022.stdout:3054: writing smithi02284960-24 from 688128 to 1376256 tid 2 2024-09-21T09:41:46.071 INFO:tasks.rados.rados.0.smithi022.stdout:3054: writing smithi02284960-24 from 1376256 to 1892352 tid 3 2024-09-21T09:41:46.071 INFO:tasks.rados.rados.0.smithi022.stdout:3055: snap_create 2024-09-21T09:41:46.072 INFO:tasks.rados.rados.0.smithi022.stdout:3052: finishing rollback tid 0 to smithi02284960-8 2024-09-21T09:41:46.072 INFO:tasks.rados.rados.0.smithi022.stdout:3051: expect (ObjNum 968 snap 296 seq_num 968) 2024-09-21T09:41:46.123 INFO:tasks.rados.rados.0.smithi022.stdout:3052: finishing rollback tid 1 to smithi02284960-8 2024-09-21T09:41:46.123 INFO:tasks.rados.rados.0.smithi022.stdout:3054: finishing write tid 1 to smithi02284960-24 2024-09-21T09:41:46.123 INFO:tasks.rados.rados.0.smithi022.stdout:3054: finishing write tid 2 to smithi02284960-24 2024-09-21T09:41:46.123 INFO:tasks.rados.rados.0.smithi022.stdout:3054: finishing write tid 3 to smithi02284960-24 2024-09-21T09:41:46.123 INFO:tasks.rados.rados.0.smithi022.stdout:3054: finishing write tid 4 to smithi02284960-24 2024-09-21T09:41:46.123 INFO:tasks.rados.rados.0.smithi022.stdout:3052: finishing rollback tid 2 to smithi02284960-8 2024-09-21T09:41:46.123 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 8 v 1676 (ObjNum 919 snap 286 seq_num 919) dirty exists 2024-09-21T09:41:46.124 INFO:tasks.rados.rados.0.smithi022.stdout:3054: finishing write tid 5 to smithi02284960-24 2024-09-21T09:41:46.124 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 24 v 1813 (ObjNum 972 snap 297 seq_num 972) dirty exists 2024-09-21T09:41:46.124 INFO:tasks.rados.rados.0.smithi022.stdout:3054: left oid 24 (ObjNum 972 snap 297 seq_num 972) 2024-09-21T09:41:46.154 INFO:tasks.rados.rados.0.smithi022.stdout:3053: finishing copy_from racing read to smithi02284960-27 2024-09-21T09:41:46.157 INFO:tasks.rados.rados.0.smithi022.stdout:3053: finishing copy_from to smithi02284960-27 2024-09-21T09:41:46.157 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 27 v 1673 (ObjNum 945 snap 295 seq_num 945) dirty exists 2024-09-21T09:41:46.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:46 smithi064 ceph-mon[86709]: pgmap v617: 121 pgs: 121 active+clean; 294 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 26 MiB/s rd, 1.3 MiB/s wr, 12 op/s 2024-09-21T09:41:46.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:46 smithi064 ceph-mon[86709]: osdmap e673: 8 total, 8 up, 8 in 2024-09-21T09:41:46.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:46 smithi022 ceph-mon[98615]: pgmap v617: 121 pgs: 121 active+clean; 294 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 26 MiB/s rd, 1.3 MiB/s wr, 12 op/s 2024-09-21T09:41:46.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:46 smithi022 ceph-mon[98615]: osdmap e673: 8 total, 8 up, 8 in 2024-09-21T09:41:46.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:46 smithi022 ceph-mon[96736]: pgmap v617: 121 pgs: 121 active+clean; 294 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 26 MiB/s rd, 1.3 MiB/s wr, 12 op/s 2024-09-21T09:41:46.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:46 smithi022 ceph-mon[96736]: osdmap e673: 8 total, 8 up, 8 in 2024-09-21T09:41:47.072 INFO:tasks.rados.rados.0.smithi022.stdout:3051: done (4 left) 2024-09-21T09:41:47.072 INFO:tasks.rados.rados.0.smithi022.stdout:3052: done (3 left) 2024-09-21T09:41:47.072 INFO:tasks.rados.rados.0.smithi022.stdout:3053: done (2 left) 2024-09-21T09:41:47.072 INFO:tasks.rados.rados.0.smithi022.stdout:3054: done (1 left) 2024-09-21T09:41:47.072 INFO:tasks.rados.rados.0.smithi022.stdout:3055: done (0 left) 2024-09-21T09:41:47.072 INFO:tasks.rados.rados.0.smithi022.stdout:3056: rmattr oid 43 current snap is 298 2024-09-21T09:41:47.072 INFO:tasks.rados.rados.0.smithi022.stdout:3056: done (0 left) 2024-09-21T09:41:47.072 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 30 current snap is 298 2024-09-21T09:41:47.073 INFO:tasks.rados.rados.0.smithi022.stdout:3057: seq_num 973 ranges {1957888=704512,2662400=573440,3235840=507904,3743744=122880} 2024-09-21T09:41:47.086 INFO:tasks.rados.rados.0.smithi022.stdout:3057: writing smithi02284960-30 from 1957888 to 2662400 tid 1 2024-09-21T09:41:47.090 INFO:tasks.rados.rados.0.smithi022.stdout:3057: writing smithi02284960-30 from 2662400 to 3235840 tid 2 2024-09-21T09:41:47.093 INFO:tasks.rados.rados.0.smithi022.stdout:3057: writing smithi02284960-30 from 3235840 to 3743744 tid 3 2024-09-21T09:41:47.095 INFO:tasks.rados.rados.0.smithi022.stdout:3057: writing smithi02284960-30 from 3743744 to 3866624 tid 4 2024-09-21T09:41:47.095 INFO:tasks.rados.rados.0.smithi022.stdout:3058: read oid 44 snap -1 2024-09-21T09:41:47.095 INFO:tasks.rados.rados.0.smithi022.stdout:3058: expect (ObjNum 886 snap 282 seq_num 886) 2024-09-21T09:41:47.095 INFO:tasks.rados.rados.0.smithi022.stdout:3059: copy_from oid 31 from oid 28 current snap is 298 2024-09-21T09:41:47.095 INFO:tasks.rados.rados.0.smithi022.stdout:3057: finishing write tid 1 to smithi02284960-30 2024-09-21T09:41:47.095 INFO:tasks.rados.rados.0.smithi022.stdout:3060: copy_from oid 14 from oid 35 current snap is 298 2024-09-21T09:41:47.095 INFO:tasks.rados.rados.0.smithi022.stdout:3061: snap_remove snap 296 2024-09-21T09:41:47.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:47 smithi064 ceph-mon[86709]: osdmap e674: 8 total, 8 up, 8 in 2024-09-21T09:41:47.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:47 smithi022 ceph-mon[98615]: osdmap e674: 8 total, 8 up, 8 in 2024-09-21T09:41:47.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:47 smithi022 ceph-mon[96736]: osdmap e674: 8 total, 8 up, 8 in 2024-09-21T09:41:48.079 INFO:tasks.rados.rados.0.smithi022.stdout:3061: done (4 left) 2024-09-21T09:41:48.079 INFO:tasks.rados.rados.0.smithi022.stdout:3062: rmattr oid 1 current snap is 298 2024-09-21T09:41:48.080 INFO:tasks.rados.rados.0.smithi022.stdout:3062: done (4 left) 2024-09-21T09:41:48.080 INFO:tasks.rados.rados.0.smithi022.stdout:3063: snap_remove snap 297 2024-09-21T09:41:48.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:48 smithi064 ceph-mon[86709]: pgmap v620: 121 pgs: 121 active+clean; 190 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 21 MiB/s rd, 7.8 MiB/s wr, 45 op/s 2024-09-21T09:41:48.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:48 smithi064 ceph-mon[86709]: osdmap e675: 8 total, 8 up, 8 in 2024-09-21T09:41:48.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:48 smithi022 ceph-mon[96736]: pgmap v620: 121 pgs: 121 active+clean; 190 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 21 MiB/s rd, 7.8 MiB/s wr, 45 op/s 2024-09-21T09:41:48.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:48 smithi022 ceph-mon[96736]: osdmap e675: 8 total, 8 up, 8 in 2024-09-21T09:41:48.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:48 smithi022 ceph-mon[98615]: pgmap v620: 121 pgs: 121 active+clean; 190 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 21 MiB/s rd, 7.8 MiB/s wr, 45 op/s 2024-09-21T09:41:48.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:48 smithi022 ceph-mon[98615]: osdmap e675: 8 total, 8 up, 8 in 2024-09-21T09:41:49.053 INFO:tasks.rados.rados.0.smithi022.stdout:3057: finishing write tid 2 to smithi02284960-30 2024-09-21T09:41:49.053 INFO:tasks.rados.rados.0.smithi022.stdout:3057: finishing write tid 3 to smithi02284960-30 2024-09-21T09:41:49.053 INFO:tasks.rados.rados.0.smithi022.stdout:3057: finishing write tid 4 to smithi02284960-30 2024-09-21T09:41:49.053 INFO:tasks.rados.rados.0.smithi022.stdout:3057: finishing write tid 5 to smithi02284960-30 2024-09-21T09:41:49.054 INFO:tasks.rados.rados.0.smithi022.stdout:3057: finishing write tid 6 to smithi02284960-30 2024-09-21T09:41:49.054 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 30 v 1328 (ObjNum 973 snap 298 seq_num 973) dirty exists 2024-09-21T09:41:49.054 INFO:tasks.rados.rados.0.smithi022.stdout:3057: left oid 30 (ObjNum 973 snap 298 seq_num 973) 2024-09-21T09:41:49.054 INFO:tasks.rados.rados.0.smithi022.stdout:3058: expect (ObjNum 886 snap 282 seq_num 886) 2024-09-21T09:41:49.066 INFO:tasks.rados.rados.0.smithi022.stdout:3059: finishing copy_from racing read to smithi02284960-31 2024-09-21T09:41:49.066 INFO:tasks.rados.rados.0.smithi022.stdout:3059: finishing copy_from to smithi02284960-31 2024-09-21T09:41:49.066 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 31 v 1792 (ObjNum 827 snap 264 seq_num 827) dirty exists 2024-09-21T09:41:49.066 INFO:tasks.rados.rados.0.smithi022.stdout:3060: finishing copy_from racing read to smithi02284960-14 2024-09-21T09:41:49.066 INFO:tasks.rados.rados.0.smithi022.stdout:3060: finishing copy_from to smithi02284960-14 2024-09-21T09:41:49.066 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 14 v 1784 (ObjNum 925 snap 287 seq_num 925) dirty exists 2024-09-21T09:41:49.066 INFO:tasks.rados.rados.0.smithi022.stdout:3057: done (4 left) 2024-09-21T09:41:49.066 INFO:tasks.rados.rados.0.smithi022.stdout:3058: done (3 left) 2024-09-21T09:41:49.066 INFO:tasks.rados.rados.0.smithi022.stdout:3059: done (2 left) 2024-09-21T09:41:49.067 INFO:tasks.rados.rados.0.smithi022.stdout:3060: done (1 left) 2024-09-21T09:41:49.067 INFO:tasks.rados.rados.0.smithi022.stdout:3063: done (0 left) 2024-09-21T09:41:49.067 INFO:tasks.rados.rados.0.smithi022.stdout:3064: rollback oid 46 current snap is 298 2024-09-21T09:41:49.067 INFO:tasks.rados.rados.0.smithi022.stdout:3064: done (0 left) 2024-09-21T09:41:49.067 INFO:tasks.rados.rados.0.smithi022.stdout:3065: read oid 31 snap -1 2024-09-21T09:41:49.067 INFO:tasks.rados.rados.0.smithi022.stdout:3065: expect (ObjNum 827 snap 264 seq_num 827) 2024-09-21T09:41:49.067 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 16 current snap is 298 2024-09-21T09:41:49.067 INFO:tasks.rados.rados.0.smithi022.stdout:3066: seq_num 974 ranges {2760704=720896,3481600=507904,3989504=671744,4661248=24576} 2024-09-21T09:41:49.085 INFO:tasks.rados.rados.0.smithi022.stdout:3066: writing smithi02284960-16 from 2760704 to 3481600 tid 1 2024-09-21T09:41:49.088 INFO:tasks.rados.rados.0.smithi022.stdout:3066: writing smithi02284960-16 from 3481600 to 3989504 tid 2 2024-09-21T09:41:49.093 INFO:tasks.rados.rados.0.smithi022.stdout:3066: writing smithi02284960-16 from 3989504 to 4661248 tid 3 2024-09-21T09:41:49.093 INFO:tasks.rados.rados.0.smithi022.stdout:3066: writing smithi02284960-16 from 4661248 to 4685824 tid 4 2024-09-21T09:41:49.093 INFO:tasks.rados.rados.0.smithi022.stdout:3067: copy_from oid 21 from oid 26 current snap is 298 2024-09-21T09:41:49.093 INFO:tasks.rados.rados.0.smithi022.stdout:3065: expect (ObjNum 827 snap 264 seq_num 827) 2024-09-21T09:41:49.148 INFO:tasks.rados.rados.0.smithi022.stdout:3066: finishing write tid 1 to smithi02284960-16 2024-09-21T09:41:49.148 INFO:tasks.rados.rados.0.smithi022.stdout:3066: finishing write tid 2 to smithi02284960-16 2024-09-21T09:41:49.149 INFO:tasks.rados.rados.0.smithi022.stdout:3065: done (2 left) 2024-09-21T09:41:49.149 INFO:tasks.rados.rados.0.smithi022.stdout:3068: read oid 1 snap -1 2024-09-21T09:41:49.149 INFO:tasks.rados.rados.0.smithi022.stdout:3068: expect deleted 2024-09-21T09:41:49.149 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 42 current snap is 298 2024-09-21T09:41:49.149 INFO:tasks.rados.rados.0.smithi022.stdout:3066: finishing write tid 3 to smithi02284960-16 2024-09-21T09:41:49.149 INFO:tasks.rados.rados.0.smithi022.stdout:3066: finishing write tid 4 to smithi02284960-16 2024-09-21T09:41:49.149 INFO:tasks.rados.rados.0.smithi022.stdout:3066: finishing write tid 5 to smithi02284960-16 2024-09-21T09:41:49.149 INFO:tasks.rados.rados.0.smithi022.stdout:3066: finishing write tid 6 to smithi02284960-16 2024-09-21T09:41:49.149 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 16 v 1523 (ObjNum 974 snap 298 seq_num 974) dirty exists 2024-09-21T09:41:49.149 INFO:tasks.rados.rados.0.smithi022.stdout:3066: left oid 16 (ObjNum 974 snap 298 seq_num 974) 2024-09-21T09:41:49.149 INFO:tasks.rados.rados.0.smithi022.stdout:3067: finishing copy_from racing read to smithi02284960-21 2024-09-21T09:41:49.149 INFO:tasks.rados.rados.0.smithi022.stdout:3067: finishing copy_from to smithi02284960-21 2024-09-21T09:41:49.149 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 21 v 1694 (ObjNum 970 snap 296 seq_num 970) dirty exists 2024-09-21T09:41:49.150 INFO:tasks.rados.rados.0.smithi022.stdout:3069: seq_num 975 ranges {0=737280,737280=786432,1523712=417792} 2024-09-21T09:41:49.152 INFO:tasks.rados.rados.0.smithi022.stdout:3069: writing smithi02284960-42 from 0 to 737280 tid 1 2024-09-21T09:41:49.156 INFO:tasks.rados.rados.0.smithi022.stdout:3069: writing smithi02284960-42 from 737280 to 1523712 tid 2 2024-09-21T09:41:49.159 INFO:tasks.rados.rados.0.smithi022.stdout:3069: writing smithi02284960-42 from 1523712 to 1941504 tid 3 2024-09-21T09:41:49.159 INFO:tasks.rados.rados.0.smithi022.stdout:3066: done (3 left) 2024-09-21T09:41:49.159 INFO:tasks.rados.rados.0.smithi022.stdout:3067: done (2 left) 2024-09-21T09:41:49.159 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 40 current snap is 298 2024-09-21T09:41:49.159 INFO:tasks.rados.rados.0.smithi022.stdout:3070: seq_num 976 ranges {999424=753664,1753088=671744,2424832=532480} 2024-09-21T09:41:49.168 INFO:tasks.rados.rados.0.smithi022.stdout:3070: writing smithi02284960-40 from 999424 to 1753088 tid 1 2024-09-21T09:41:49.172 INFO:tasks.rados.rados.0.smithi022.stdout:3070: writing smithi02284960-40 from 1753088 to 2424832 tid 2 2024-09-21T09:41:49.176 INFO:tasks.rados.rados.0.smithi022.stdout:3070: writing smithi02284960-40 from 2424832 to 2957312 tid 3 2024-09-21T09:41:49.176 INFO:tasks.rados.rados.0.smithi022.stdout:3071: snap_create 2024-09-21T09:41:49.176 INFO:tasks.rados.rados.0.smithi022.stdout:3069: finishing write tid 1 to smithi02284960-42 2024-09-21T09:41:49.176 INFO:tasks.rados.rados.0.smithi022.stdout:3069: finishing write tid 2 to smithi02284960-42 2024-09-21T09:41:49.176 INFO:tasks.rados.rados.0.smithi022.stdout:3069: finishing write tid 3 to smithi02284960-42 2024-09-21T09:41:49.176 INFO:tasks.rados.rados.0.smithi022.stdout:3069: finishing write tid 4 to smithi02284960-42 2024-09-21T09:41:49.176 INFO:tasks.rados.rados.0.smithi022.stdout:3069: finishing write tid 5 to smithi02284960-42 2024-09-21T09:41:49.176 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 42 v 1698 (ObjNum 975 snap 298 seq_num 975) dirty exists 2024-09-21T09:41:49.176 INFO:tasks.rados.rados.0.smithi022.stdout:3069: left oid 42 (ObjNum 975 snap 298 seq_num 975) 2024-09-21T09:41:49.176 INFO:tasks.rados.rados.0.smithi022.stdout:3070: finishing write tid 1 to smithi02284960-40 2024-09-21T09:41:49.176 INFO:tasks.rados.rados.0.smithi022.stdout:3070: finishing write tid 2 to smithi02284960-40 2024-09-21T09:41:49.178 INFO:tasks.rados.rados.0.smithi022.stdout:3070: finishing write tid 3 to smithi02284960-40 2024-09-21T09:41:49.178 INFO:tasks.rados.rados.0.smithi022.stdout:3070: finishing write tid 4 to smithi02284960-40 2024-09-21T09:41:49.181 INFO:tasks.rados.rados.0.smithi022.stdout:3070: finishing write tid 5 to smithi02284960-40 2024-09-21T09:41:49.181 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 40 v 1663 (ObjNum 976 snap 298 seq_num 976) dirty exists 2024-09-21T09:41:49.181 INFO:tasks.rados.rados.0.smithi022.stdout:3070: left oid 40 (ObjNum 976 snap 298 seq_num 976) 2024-09-21T09:41:49.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:49 smithi064 ceph-mon[86709]: osdmap e676: 8 total, 8 up, 8 in 2024-09-21T09:41:49.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:49 smithi064 ceph-mon[86709]: osdmap e677: 8 total, 8 up, 8 in 2024-09-21T09:41:49.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:49 smithi022 ceph-mon[98615]: osdmap e676: 8 total, 8 up, 8 in 2024-09-21T09:41:49.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:49 smithi022 ceph-mon[98615]: osdmap e677: 8 total, 8 up, 8 in 2024-09-21T09:41:49.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:49 smithi022 ceph-mon[96736]: osdmap e676: 8 total, 8 up, 8 in 2024-09-21T09:41:49.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:49 smithi022 ceph-mon[96736]: osdmap e677: 8 total, 8 up, 8 in 2024-09-21T09:41:50.087 INFO:tasks.rados.rados.0.smithi022.stdout:3068: done (3 left) 2024-09-21T09:41:50.088 INFO:tasks.rados.rados.0.smithi022.stdout:3069: done (2 left) 2024-09-21T09:41:50.088 INFO:tasks.rados.rados.0.smithi022.stdout:3070: done (1 left) 2024-09-21T09:41:50.088 INFO:tasks.rados.rados.0.smithi022.stdout:3071: done (0 left) 2024-09-21T09:41:50.088 INFO:tasks.rados.rados.0.smithi022.stdout:3072: read oid 28 snap -1 2024-09-21T09:41:50.088 INFO:tasks.rados.rados.0.smithi022.stdout:3072: expect (ObjNum 827 snap 264 seq_num 827) 2024-09-21T09:41:50.088 INFO:tasks.rados.rados.0.smithi022.stdout:3073: read oid 20 snap -1 2024-09-21T09:41:50.088 INFO:tasks.rados.rados.0.smithi022.stdout:3073: expect (ObjNum 968 snap 296 seq_num 968) 2024-09-21T09:41:50.088 INFO:tasks.rados.rados.0.smithi022.stdout:3074: snap_remove snap 298 2024-09-21T09:41:50.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:50 smithi064 ceph-mon[86709]: pgmap v623: 121 pgs: 121 active+clean; 157 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 8.9 MiB/s rd, 8.8 MiB/s wr, 41 op/s 2024-09-21T09:41:50.442 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:41:50 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:41:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:41:50.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:50 smithi022 ceph-mon[98615]: pgmap v623: 121 pgs: 121 active+clean; 157 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 8.9 MiB/s rd, 8.8 MiB/s wr, 41 op/s 2024-09-21T09:41:50.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:50 smithi022 ceph-mon[96736]: pgmap v623: 121 pgs: 121 active+clean; 157 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 8.9 MiB/s rd, 8.8 MiB/s wr, 41 op/s 2024-09-21T09:41:51.100 INFO:tasks.rados.rados.0.smithi022.stdout:3074: done (2 left) 2024-09-21T09:41:51.100 INFO:tasks.rados.rados.0.smithi022.stdout:3075: snap_create 2024-09-21T09:41:51.100 INFO:tasks.rados.rados.0.smithi022.stdout:3072: expect (ObjNum 827 snap 264 seq_num 827) 2024-09-21T09:41:51.152 INFO:tasks.rados.rados.0.smithi022.stdout:3073: expect (ObjNum 968 snap 296 seq_num 968) 2024-09-21T09:41:51.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:51 smithi064 ceph-mon[86709]: osdmap e678: 8 total, 8 up, 8 in 2024-09-21T09:41:51.557 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:51 smithi022 ceph-mon[98615]: osdmap e678: 8 total, 8 up, 8 in 2024-09-21T09:41:51.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:51 smithi022 ceph-mon[96736]: osdmap e678: 8 total, 8 up, 8 in 2024-09-21T09:41:52.102 INFO:tasks.rados.rados.0.smithi022.stdout:3072: done (2 left) 2024-09-21T09:41:52.102 INFO:tasks.rados.rados.0.smithi022.stdout:3073: done (1 left) 2024-09-21T09:41:52.102 INFO:tasks.rados.rados.0.smithi022.stdout:3075: done (0 left) 2024-09-21T09:41:52.102 INFO:tasks.rados.rados.0.smithi022.stdout:3076: read oid 39 snap -1 2024-09-21T09:41:52.102 INFO:tasks.rados.rados.0.smithi022.stdout:3076: expect (ObjNum 911 snap 284 seq_num 911) 2024-09-21T09:41:52.102 INFO:tasks.rados.rados.0.smithi022.stdout:3077: delete oid 30 current snap is 300 2024-09-21T09:41:52.113 INFO:tasks.rados.rados.0.smithi022.stdout:3076: expect (ObjNum 911 snap 284 seq_num 911) 2024-09-21T09:41:52.136 INFO:tasks.rados.rados.0.smithi022.stdout:3076: done (1 left) 2024-09-21T09:41:52.137 INFO:tasks.rados.rados.0.smithi022.stdout:3077: done (0 left) 2024-09-21T09:41:52.137 INFO:tasks.rados.rados.0.smithi022.stdout:3078: setattr oid 41 current snap is 300 2024-09-21T09:41:52.138 INFO:tasks.rados.rados.0.smithi022.stdout:3079: setattr oid 7 current snap is 300 2024-09-21T09:41:52.141 INFO:tasks.rados.rados.0.smithi022.stdout:3080: read oid 4 snap 299 2024-09-21T09:41:52.141 INFO:tasks.rados.rados.0.smithi022.stdout:3080: expect (ObjNum 945 snap 295 seq_num 945) 2024-09-21T09:41:52.141 INFO:tasks.rados.rados.0.smithi022.stdout:3081: read oid 17 snap -1 2024-09-21T09:41:52.141 INFO:tasks.rados.rados.0.smithi022.stdout:3081: expect (ObjNum 842 snap 269 seq_num 842) 2024-09-21T09:41:52.191 INFO:tasks.rados.rados.0.smithi022.stdout:3082: delete oid 48 current snap is 300 2024-09-21T09:41:52.191 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 41 v 1664 (ObjNum 865 snap 277 seq_num 865) dirty exists 2024-09-21T09:41:52.191 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 7 v 1909 (ObjNum 4913 snap 0 seq_num 3570242528) dirty exists 2024-09-21T09:41:52.203 INFO:tasks.rados.rados.0.smithi022.stdout:3078: done (4 left) 2024-09-21T09:41:52.203 INFO:tasks.rados.rados.0.smithi022.stdout:3079: done (3 left) 2024-09-21T09:41:52.203 INFO:tasks.rados.rados.0.smithi022.stdout:3082: done (2 left) 2024-09-21T09:41:52.203 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 9 current snap is 300 2024-09-21T09:41:52.204 INFO:tasks.rados.rados.0.smithi022.stdout:3083: seq_num 979 ranges {16457728=802816,17260544=729088,17989632=458752,18448384=16384} 2024-09-21T09:41:52.298 INFO:tasks.rados.rados.0.smithi022.stdout:3083: writing smithi02284960-9 from 16457728 to 17260544 tid 1 2024-09-21T09:41:52.302 INFO:tasks.rados.rados.0.smithi022.stdout:3083: writing smithi02284960-9 from 17260544 to 17989632 tid 2 2024-09-21T09:41:52.305 INFO:tasks.rados.rados.0.smithi022.stdout:3083: writing smithi02284960-9 from 17989632 to 18448384 tid 3 2024-09-21T09:41:52.306 INFO:tasks.rados.rados.0.smithi022.stdout:3083: writing smithi02284960-9 from 18448384 to 18464768 tid 4 2024-09-21T09:41:52.306 INFO:tasks.rados.rados.0.smithi022.stdout:3084: snap_create 2024-09-21T09:41:52.306 INFO:tasks.rados.rados.0.smithi022.stdout:3080: expect (ObjNum 945 snap 295 seq_num 945) 2024-09-21T09:41:52.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:52 smithi064 ceph-mon[86709]: pgmap v626: 121 pgs: 121 active+clean; 166 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 8.9 MiB/s rd, 4.4 MiB/s wr, 12 op/s 2024-09-21T09:41:52.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:52 smithi064 ceph-mon[86709]: osdmap e679: 8 total, 8 up, 8 in 2024-09-21T09:41:52.557 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:52 smithi022 ceph-mon[98615]: pgmap v626: 121 pgs: 121 active+clean; 166 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 8.9 MiB/s rd, 4.4 MiB/s wr, 12 op/s 2024-09-21T09:41:52.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:52 smithi022 ceph-mon[98615]: osdmap e679: 8 total, 8 up, 8 in 2024-09-21T09:41:52.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:52 smithi022 ceph-mon[96736]: pgmap v626: 121 pgs: 121 active+clean; 166 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 8.9 MiB/s rd, 4.4 MiB/s wr, 12 op/s 2024-09-21T09:41:52.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:52 smithi022 ceph-mon[96736]: osdmap e679: 8 total, 8 up, 8 in 2024-09-21T09:41:53.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:53 smithi064 ceph-mon[86709]: osdmap e680: 8 total, 8 up, 8 in 2024-09-21T09:41:53.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:53 smithi064 ceph-mon[86709]: pgmap v629: 121 pgs: 121 active+clean; 188 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 4.3 MiB/s rd, 8.5 MiB/s wr, 12 op/s 2024-09-21T09:41:53.557 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:53 smithi022 ceph-mon[98615]: osdmap e680: 8 total, 8 up, 8 in 2024-09-21T09:41:53.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:53 smithi022 ceph-mon[98615]: pgmap v629: 121 pgs: 121 active+clean; 188 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 4.3 MiB/s rd, 8.5 MiB/s wr, 12 op/s 2024-09-21T09:41:53.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:53 smithi022 ceph-mon[96736]: osdmap e680: 8 total, 8 up, 8 in 2024-09-21T09:41:53.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:53 smithi022 ceph-mon[96736]: pgmap v629: 121 pgs: 121 active+clean; 188 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 4.3 MiB/s rd, 8.5 MiB/s wr, 12 op/s 2024-09-21T09:41:54.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:54 smithi064 ceph-mon[86709]: osdmap e681: 8 total, 8 up, 8 in 2024-09-21T09:41:54.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:54 smithi064 ceph-mon[86709]: osdmap e682: 8 total, 8 up, 8 in 2024-09-21T09:41:54.458 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:54 smithi022 ceph-mon[98615]: osdmap e681: 8 total, 8 up, 8 in 2024-09-21T09:41:54.459 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:54 smithi022 ceph-mon[98615]: osdmap e682: 8 total, 8 up, 8 in 2024-09-21T09:41:54.459 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:54 smithi022 ceph-mon[96736]: osdmap e681: 8 total, 8 up, 8 in 2024-09-21T09:41:54.459 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:54 smithi022 ceph-mon[96736]: osdmap e682: 8 total, 8 up, 8 in 2024-09-21T09:41:55.137 INFO:tasks.rados.rados.0.smithi022.stdout:3081: expect (ObjNum 842 snap 269 seq_num 842) 2024-09-21T09:41:55.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:55 smithi064 ceph-mon[86709]: pgmap v632: 121 pgs: 121 active+clean; 192 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.7 MiB/s rd, 7.7 MiB/s wr, 6 op/s 2024-09-21T09:41:55.474 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:55 smithi022 ceph-mon[98615]: pgmap v632: 121 pgs: 121 active+clean; 192 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.7 MiB/s rd, 7.7 MiB/s wr, 6 op/s 2024-09-21T09:41:55.474 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:55 smithi022 ceph-mon[96736]: pgmap v632: 121 pgs: 121 active+clean; 192 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.7 MiB/s rd, 7.7 MiB/s wr, 6 op/s 2024-09-21T09:41:55.728 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:41:55 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:41:55] "GET /metrics HTTP/1.1" 200 36352 "" "Prometheus/2.51.0" 2024-09-21T09:41:57.181 INFO:tasks.rados.rados.0.smithi022.stdout:3083: finishing write tid 1 to smithi02284960-9 2024-09-21T09:41:57.182 INFO:tasks.rados.rados.0.smithi022.stdout:3083: finishing write tid 2 to smithi02284960-9 2024-09-21T09:41:57.182 INFO:tasks.rados.rados.0.smithi022.stdout:3083: finishing write tid 3 to smithi02284960-9 2024-09-21T09:41:57.182 INFO:tasks.rados.rados.0.smithi022.stdout:3083: finishing write tid 4 to smithi02284960-9 2024-09-21T09:41:57.182 INFO:tasks.rados.rados.0.smithi022.stdout:3083: finishing write tid 5 to smithi02284960-9 2024-09-21T09:41:57.182 INFO:tasks.rados.rados.0.smithi022.stdout:3083: finishing write tid 6 to smithi02284960-9 2024-09-21T09:41:57.182 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 9 v 1681 (ObjNum 979 snap 300 seq_num 979) dirty exists 2024-09-21T09:41:57.182 INFO:tasks.rados.rados.0.smithi022.stdout:3083: left oid 9 (ObjNum 979 snap 300 seq_num 979) 2024-09-21T09:41:57.182 INFO:tasks.rados.rados.0.smithi022.stdout:3080: done (3 left) 2024-09-21T09:41:57.182 INFO:tasks.rados.rados.0.smithi022.stdout:3081: done (2 left) 2024-09-21T09:41:57.182 INFO:tasks.rados.rados.0.smithi022.stdout:3083: done (1 left) 2024-09-21T09:41:57.182 INFO:tasks.rados.rados.0.smithi022.stdout:3084: done (0 left) 2024-09-21T09:41:57.182 INFO:tasks.rados.rados.0.smithi022.stdout:3085: rollback oid 21 current snap is 301 2024-09-21T09:41:57.183 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 21 to 300 2024-09-21T09:41:57.183 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 12 current snap is 301 2024-09-21T09:41:57.183 INFO:tasks.rados.rados.0.smithi022.stdout:3086: seq_num 980 ranges {1286144=417792,1703936=614400,2318336=663552,2981888=327680} 2024-09-21T09:41:57.190 INFO:tasks.rados.rados.0.smithi022.stdout:3086: writing smithi02284960-12 from 1286144 to 1703936 tid 1 2024-09-21T09:41:57.194 INFO:tasks.rados.rados.0.smithi022.stdout:3086: writing smithi02284960-12 from 1703936 to 2318336 tid 2 2024-09-21T09:41:57.197 INFO:tasks.rados.rados.0.smithi022.stdout:3086: writing smithi02284960-12 from 2318336 to 2981888 tid 3 2024-09-21T09:41:57.200 INFO:tasks.rados.rados.0.smithi022.stdout:3086: writing smithi02284960-12 from 2981888 to 3309568 tid 4 2024-09-21T09:41:57.200 INFO:tasks.rados.rados.0.smithi022.stdout:3087: delete oid 13 current snap is 301 2024-09-21T09:41:57.200 INFO:tasks.rados.rados.0.smithi022.stdout:3085: finishing rollback tid 0 to smithi02284960-21 2024-09-21T09:41:57.200 INFO:tasks.rados.rados.0.smithi022.stdout:3085: finishing rollback tid 1 to smithi02284960-21 2024-09-21T09:41:57.200 INFO:tasks.rados.rados.0.smithi022.stdout:3085: finishing rollback tid 2 to smithi02284960-21 2024-09-21T09:41:57.200 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 21 v 1795 (ObjNum 970 snap 296 seq_num 970) dirty exists 2024-09-21T09:41:57.200 INFO:tasks.rados.rados.0.smithi022.stdout:3086: finishing write tid 1 to smithi02284960-12 2024-09-21T09:41:57.200 INFO:tasks.rados.rados.0.smithi022.stdout:3086: finishing write tid 2 to smithi02284960-12 2024-09-21T09:41:57.203 INFO:tasks.rados.rados.0.smithi022.stdout:3086: finishing write tid 3 to smithi02284960-12 2024-09-21T09:41:57.203 INFO:tasks.rados.rados.0.smithi022.stdout:3086: finishing write tid 4 to smithi02284960-12 2024-09-21T09:41:57.204 INFO:tasks.rados.rados.0.smithi022.stdout:3086: finishing write tid 5 to smithi02284960-12 2024-09-21T09:41:57.205 INFO:tasks.rados.rados.0.smithi022.stdout:3085: done (2 left) 2024-09-21T09:41:57.205 INFO:tasks.rados.rados.0.smithi022.stdout:3087: done (1 left) 2024-09-21T09:41:57.205 INFO:tasks.rados.rados.0.smithi022.stdout:3088: rmattr oid 43 current snap is 301 2024-09-21T09:41:57.205 INFO:tasks.rados.rados.0.smithi022.stdout:3088: done (1 left) 2024-09-21T09:41:57.205 INFO:tasks.rados.rados.0.smithi022.stdout:3089: rollback oid 14 current snap is 301 2024-09-21T09:41:57.205 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 14 to 299 2024-09-21T09:41:57.205 INFO:tasks.rados.rados.0.smithi022.stdout:3090: read oid 39 snap -1 2024-09-21T09:41:57.205 INFO:tasks.rados.rados.0.smithi022.stdout:3090: expect (ObjNum 911 snap 284 seq_num 911) 2024-09-21T09:41:57.205 INFO:tasks.rados.rados.0.smithi022.stdout:3091: snap_create 2024-09-21T09:41:57.207 INFO:tasks.rados.rados.0.smithi022.stdout:3086: finishing write tid 6 to smithi02284960-12 2024-09-21T09:41:57.207 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 12 v 1827 (ObjNum 980 snap 301 seq_num 980) dirty exists 2024-09-21T09:41:57.208 INFO:tasks.rados.rados.0.smithi022.stdout:3086: left oid 12 (ObjNum 980 snap 301 seq_num 980) 2024-09-21T09:41:57.208 INFO:tasks.rados.rados.0.smithi022.stdout:3089: finishing rollback tid 0 to smithi02284960-14 2024-09-21T09:41:57.208 INFO:tasks.rados.rados.0.smithi022.stdout:3089: finishing rollback tid 1 to smithi02284960-14 2024-09-21T09:41:57.209 INFO:tasks.rados.rados.0.smithi022.stdout:3089: finishing rollback tid 2 to smithi02284960-14 2024-09-21T09:41:57.209 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 14 v 1787 (ObjNum 925 snap 287 seq_num 925) dirty exists 2024-09-21T09:41:57.209 INFO:tasks.rados.rados.0.smithi022.stdout:3090: expect (ObjNum 911 snap 284 seq_num 911) 2024-09-21T09:41:57.491 INFO:tasks.rados.rados.0.smithi022.stdout:3086: done (3 left) 2024-09-21T09:41:57.491 INFO:tasks.rados.rados.0.smithi022.stdout:3089: done (2 left) 2024-09-21T09:41:57.491 INFO:tasks.rados.rados.0.smithi022.stdout:3090: done (1 left) 2024-09-21T09:41:57.491 INFO:tasks.rados.rados.0.smithi022.stdout:3091: done (0 left) 2024-09-21T09:41:57.492 INFO:tasks.rados.rados.0.smithi022.stdout:3092: rmattr oid 4 current snap is 302 2024-09-21T09:41:57.492 INFO:tasks.rados.rados.0.smithi022.stdout:3093: snap_create 2024-09-21T09:41:57.496 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 4 v 1682 (ObjNum 945 snap 295 seq_num 945) dirty exists 2024-09-21T09:41:57.807 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:57 smithi022 ceph-mon[98615]: pgmap v633: 121 pgs: 121 active+clean; 194 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 24 MiB/s rd, 5.8 MiB/s wr, 14 op/s 2024-09-21T09:41:57.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:57 smithi022 ceph-mon[96736]: pgmap v633: 121 pgs: 121 active+clean; 194 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 24 MiB/s rd, 5.8 MiB/s wr, 14 op/s 2024-09-21T09:41:57.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:57 smithi064 ceph-mon[86709]: pgmap v633: 121 pgs: 121 active+clean; 194 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 24 MiB/s rd, 5.8 MiB/s wr, 14 op/s 2024-09-21T09:41:58.498 INFO:tasks.rados.rados.0.smithi022.stdout:3092: done (1 left) 2024-09-21T09:41:58.498 INFO:tasks.rados.rados.0.smithi022.stdout:3093: done (0 left) 2024-09-21T09:41:58.498 INFO:tasks.rados.rados.0.smithi022.stdout:3094: snap_create 2024-09-21T09:41:58.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:58 smithi022 ceph-mon[98615]: osdmap e683: 8 total, 8 up, 8 in 2024-09-21T09:41:58.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:58 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:41:58.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:58 smithi022 ceph-mon[96736]: osdmap e683: 8 total, 8 up, 8 in 2024-09-21T09:41:58.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:58 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:41:58.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:58 smithi064 ceph-mon[86709]: osdmap e683: 8 total, 8 up, 8 in 2024-09-21T09:41:58.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:58 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:41:59.056 INFO:tasks.rados.rados.0.smithi022.stdout:3094: done (0 left) 2024-09-21T09:41:59.056 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 15 current snap is 304 2024-09-21T09:41:59.056 INFO:tasks.rados.rados.0.smithi022.stdout:3095: seq_num 982 ranges {6914048=450560,7364608=425984,7790592=679936,8470528=466944,8937472=32768} 2024-09-21T09:41:59.096 INFO:tasks.rados.rados.0.smithi022.stdout:3095: writing smithi02284960-15 from 6914048 to 7364608 tid 1 2024-09-21T09:41:59.098 INFO:tasks.rados.rados.0.smithi022.stdout:3095: writing smithi02284960-15 from 7364608 to 7790592 tid 2 2024-09-21T09:41:59.102 INFO:tasks.rados.rados.0.smithi022.stdout:3095: writing smithi02284960-15 from 7790592 to 8470528 tid 3 2024-09-21T09:41:59.105 INFO:tasks.rados.rados.0.smithi022.stdout:3095: writing smithi02284960-15 from 8470528 to 8937472 tid 4 2024-09-21T09:41:59.105 INFO:tasks.rados.rados.0.smithi022.stdout:3095: writing smithi02284960-15 from 8937472 to 8970240 tid 5 2024-09-21T09:41:59.105 INFO:tasks.rados.rados.0.smithi022.stdout:3096: rollback oid 3 current snap is 304 2024-09-21T09:41:59.106 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 3 to 300 2024-09-21T09:41:59.106 INFO:tasks.rados.rados.0.smithi022.stdout:3097: read oid 50 snap 302 2024-09-21T09:41:59.106 INFO:tasks.rados.rados.0.smithi022.stdout:3097: expect (ObjNum 867 snap 280 seq_num 867) 2024-09-21T09:41:59.106 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 43 current snap is 304 2024-09-21T09:41:59.106 INFO:tasks.rados.rados.0.smithi022.stdout:3098: seq_num 983 ranges {0=466944,466944=712704,1179648=434176,1613824=458752} 2024-09-21T09:41:59.108 INFO:tasks.rados.rados.0.smithi022.stdout:3098: writing smithi02284960-43 from 0 to 466944 tid 1 2024-09-21T09:41:59.112 INFO:tasks.rados.rados.0.smithi022.stdout:3098: writing smithi02284960-43 from 466944 to 1179648 tid 2 2024-09-21T09:41:59.115 INFO:tasks.rados.rados.0.smithi022.stdout:3098: writing smithi02284960-43 from 1179648 to 1613824 tid 3 2024-09-21T09:41:59.119 INFO:tasks.rados.rados.0.smithi022.stdout:3098: writing smithi02284960-43 from 1613824 to 2072576 tid 4 2024-09-21T09:41:59.119 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 34 current snap is 304 2024-09-21T09:41:59.119 INFO:tasks.rados.rados.0.smithi022.stdout:3095: finishing write tid 1 to smithi02284960-15 2024-09-21T09:41:59.119 INFO:tasks.rados.rados.0.smithi022.stdout:3099: seq_num 984 ranges {843776=483328,1327104=598016,1925120=647168,2572288=360448} 2024-09-21T09:41:59.127 INFO:tasks.rados.rados.0.smithi022.stdout:3099: writing smithi02284960-34 from 843776 to 1327104 tid 1 2024-09-21T09:41:59.131 INFO:tasks.rados.rados.0.smithi022.stdout:3099: writing smithi02284960-34 from 1327104 to 1925120 tid 2 2024-09-21T09:41:59.135 INFO:tasks.rados.rados.0.smithi022.stdout:3099: writing smithi02284960-34 from 1925120 to 2572288 tid 3 2024-09-21T09:41:59.138 INFO:tasks.rados.rados.0.smithi022.stdout:3099: writing smithi02284960-34 from 2572288 to 2932736 tid 4 2024-09-21T09:41:59.139 INFO:tasks.rados.rados.0.smithi022.stdout:3100: copy_from oid 5 from oid 10 current snap is 304 2024-09-21T09:41:59.139 INFO:tasks.rados.rados.0.smithi022.stdout:3101: read oid 4 snap 301 2024-09-21T09:41:59.139 INFO:tasks.rados.rados.0.smithi022.stdout:3101: expect (ObjNum 945 snap 295 seq_num 945) 2024-09-21T09:41:59.142 INFO:tasks.rados.rados.0.smithi022.stdout:3102: snap_remove snap 303 2024-09-21T09:41:59.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:59 smithi022 ceph-mon[98615]: pgmap v635: 121 pgs: 121 active+clean; 194 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 20 MiB/s rd, 1.3 MiB/s wr, 11 op/s 2024-09-21T09:41:59.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:59 smithi022 ceph-mon[98615]: osdmap e684: 8 total, 8 up, 8 in 2024-09-21T09:41:59.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:41:59 smithi022 ceph-mon[98615]: osdmap e685: 8 total, 8 up, 8 in 2024-09-21T09:41:59.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:59 smithi022 ceph-mon[96736]: pgmap v635: 121 pgs: 121 active+clean; 194 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 20 MiB/s rd, 1.3 MiB/s wr, 11 op/s 2024-09-21T09:41:59.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:59 smithi022 ceph-mon[96736]: osdmap e684: 8 total, 8 up, 8 in 2024-09-21T09:41:59.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:41:59 smithi022 ceph-mon[96736]: osdmap e685: 8 total, 8 up, 8 in 2024-09-21T09:41:59.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:59 smithi064 ceph-mon[86709]: pgmap v635: 121 pgs: 121 active+clean; 194 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 20 MiB/s rd, 1.3 MiB/s wr, 11 op/s 2024-09-21T09:41:59.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:59 smithi064 ceph-mon[86709]: osdmap e684: 8 total, 8 up, 8 in 2024-09-21T09:41:59.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:41:59 smithi064 ceph-mon[86709]: osdmap e685: 8 total, 8 up, 8 in 2024-09-21T09:42:00.058 INFO:tasks.rados.rados.0.smithi022.stdout:3102: done (7 left) 2024-09-21T09:42:00.058 INFO:tasks.rados.rados.0.smithi022.stdout:3103: rollback oid 27 current snap is 304 2024-09-21T09:42:00.058 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 27 to 302 2024-09-21T09:42:00.058 INFO:tasks.rados.rados.0.smithi022.stdout:3095: finishing write tid 2 to smithi02284960-15 2024-09-21T09:42:00.058 INFO:tasks.rados.rados.0.smithi022.stdout:3096: finishing rollback tid 0 to smithi02284960-3 2024-09-21T09:42:00.058 INFO:tasks.rados.rados.0.smithi022.stdout:3095: finishing write tid 3 to smithi02284960-15 2024-09-21T09:42:00.058 INFO:tasks.rados.rados.0.smithi022.stdout:3095: finishing write tid 4 to smithi02284960-15 2024-09-21T09:42:00.059 INFO:tasks.rados.rados.0.smithi022.stdout:3095: finishing write tid 5 to smithi02284960-15 2024-09-21T09:42:00.059 INFO:tasks.rados.rados.0.smithi022.stdout:3096: finishing rollback tid 1 to smithi02284960-3 2024-09-21T09:42:00.059 INFO:tasks.rados.rados.0.smithi022.stdout:3095: finishing write tid 6 to smithi02284960-15 2024-09-21T09:42:00.059 INFO:tasks.rados.rados.0.smithi022.stdout:3096: finishing rollback tid 2 to smithi02284960-3 2024-09-21T09:42:00.059 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 3 v 623 (ObjNum 910 snap 284 seq_num 910) dirty exists 2024-09-21T09:42:00.059 INFO:tasks.rados.rados.0.smithi022.stdout:3095: finishing write tid 7 to smithi02284960-15 2024-09-21T09:42:00.059 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 15 v 1834 (ObjNum 982 snap 304 seq_num 982) dirty exists 2024-09-21T09:42:00.059 INFO:tasks.rados.rados.0.smithi022.stdout:3095: left oid 15 (ObjNum 982 snap 304 seq_num 982) 2024-09-21T09:42:00.060 INFO:tasks.rados.rados.0.smithi022.stdout:3099: finishing write tid 1 to smithi02284960-34 2024-09-21T09:42:00.060 INFO:tasks.rados.rados.0.smithi022.stdout:3098: finishing write tid 1 to smithi02284960-43 2024-09-21T09:42:00.060 INFO:tasks.rados.rados.0.smithi022.stdout:3098: finishing write tid 2 to smithi02284960-43 2024-09-21T09:42:00.060 INFO:tasks.rados.rados.0.smithi022.stdout:3098: finishing write tid 3 to smithi02284960-43 2024-09-21T09:42:00.060 INFO:tasks.rados.rados.0.smithi022.stdout:3098: finishing write tid 4 to smithi02284960-43 2024-09-21T09:42:00.060 INFO:tasks.rados.rados.0.smithi022.stdout:3098: finishing write tid 5 to smithi02284960-43 2024-09-21T09:42:00.060 INFO:tasks.rados.rados.0.smithi022.stdout:3098: finishing write tid 6 to smithi02284960-43 2024-09-21T09:42:00.060 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 43 v 1669 (ObjNum 983 snap 304 seq_num 983) dirty exists 2024-09-21T09:42:00.060 INFO:tasks.rados.rados.0.smithi022.stdout:3098: left oid 43 (ObjNum 983 snap 304 seq_num 983) 2024-09-21T09:42:00.060 INFO:tasks.rados.rados.0.smithi022.stdout:3099: finishing write tid 2 to smithi02284960-34 2024-09-21T09:42:00.060 INFO:tasks.rados.rados.0.smithi022.stdout:3099: finishing write tid 3 to smithi02284960-34 2024-09-21T09:42:00.060 INFO:tasks.rados.rados.0.smithi022.stdout:3099: finishing write tid 4 to smithi02284960-34 2024-09-21T09:42:00.060 INFO:tasks.rados.rados.0.smithi022.stdout:3099: finishing write tid 5 to smithi02284960-34 2024-09-21T09:42:00.060 INFO:tasks.rados.rados.0.smithi022.stdout:3099: finishing write tid 6 to smithi02284960-34 2024-09-21T09:42:00.061 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 34 v 1800 (ObjNum 984 snap 304 seq_num 984) dirty exists 2024-09-21T09:42:00.061 INFO:tasks.rados.rados.0.smithi022.stdout:3099: left oid 34 (ObjNum 984 snap 304 seq_num 984) 2024-09-21T09:42:00.061 INFO:tasks.rados.rados.0.smithi022.stdout:3100: finishing copy_from racing read to smithi02284960-5 2024-09-21T09:42:00.061 INFO:tasks.rados.rados.0.smithi022.stdout:3097: expect (ObjNum 867 snap 280 seq_num 867) 2024-09-21T09:42:00.691 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:42:00 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:42:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:42:01.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:42:01 smithi022 ceph-mon[98615]: osdmap e686: 8 total, 8 up, 8 in 2024-09-21T09:42:01.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:42:01 smithi022 ceph-mon[96736]: osdmap e686: 8 total, 8 up, 8 in 2024-09-21T09:42:01.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:42:01 smithi064 ceph-mon[86709]: osdmap e686: 8 total, 8 up, 8 in 2024-09-21T09:42:02.178 INFO:tasks.rados.rados.0.smithi022.stdout:3100: finishing copy_from to smithi02284960-5 2024-09-21T09:42:02.178 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 5 v 1659 (ObjNum 880 snap 282 seq_num 880) dirty exists 2024-09-21T09:42:02.178 INFO:tasks.rados.rados.0.smithi022.stdout:3101: expect (ObjNum 945 snap 295 seq_num 945) 2024-09-21T09:42:02.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:42:02 smithi064 ceph-mon[86709]: pgmap v639: 121 pgs: 121 active+clean; 199 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 1.2 MiB/s wr, 9 op/s 2024-09-21T09:42:02.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:42:02 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:42:02.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:42:02 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:42:02.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:42:02 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:42:02.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:42:02 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:42:02.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:42:02 smithi022 ceph-mon[98615]: pgmap v639: 121 pgs: 121 active+clean; 199 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 1.2 MiB/s wr, 9 op/s 2024-09-21T09:42:02.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:42:02 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:42:02.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:42:02 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:42:02.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:42:02 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:42:02.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:42:02 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:42:02.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:42:02 smithi022 ceph-mon[96736]: pgmap v639: 121 pgs: 121 active+clean; 199 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 1.2 MiB/s wr, 9 op/s 2024-09-21T09:42:02.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:42:02 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:42:02.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:42:02 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:42:02.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:42:02 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:42:02.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:42:02 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:42:04.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:42:04 smithi064 ceph-mon[86709]: pgmap v640: 121 pgs: 121 active+clean; 202 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 1.7 MiB/s wr, 11 op/s 2024-09-21T09:42:04.557 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:42:04 smithi022 ceph-mon[98615]: pgmap v640: 121 pgs: 121 active+clean; 202 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 1.7 MiB/s wr, 11 op/s 2024-09-21T09:42:04.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:42:04 smithi022 ceph-mon[96736]: pgmap v640: 121 pgs: 121 active+clean; 202 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 1.7 MiB/s wr, 11 op/s 2024-09-21T09:42:05.007 INFO:tasks.rados.rados.0.smithi022.stdout:3103: finishing rollback tid 0 to smithi02284960-27 2024-09-21T09:42:05.007 INFO:tasks.rados.rados.0.smithi022.stdout:3103: finishing rollback tid 1 to smithi02284960-27 2024-09-21T09:42:05.007 INFO:tasks.rados.rados.0.smithi022.stdout:3103: finishing rollback tid 2 to smithi02284960-27 2024-09-21T09:42:05.007 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 27 v 1702 (ObjNum 945 snap 295 seq_num 945) dirty exists 2024-09-21T09:42:05.007 INFO:tasks.rados.rados.0.smithi022.stdout:3095: done (7 left) 2024-09-21T09:42:05.007 INFO:tasks.rados.rados.0.smithi022.stdout:3096: done (6 left) 2024-09-21T09:42:05.007 INFO:tasks.rados.rados.0.smithi022.stdout:3097: done (5 left) 2024-09-21T09:42:05.007 INFO:tasks.rados.rados.0.smithi022.stdout:3098: done (4 left) 2024-09-21T09:42:05.007 INFO:tasks.rados.rados.0.smithi022.stdout:3099: done (3 left) 2024-09-21T09:42:05.007 INFO:tasks.rados.rados.0.smithi022.stdout:3100: done (2 left) 2024-09-21T09:42:05.007 INFO:tasks.rados.rados.0.smithi022.stdout:3101: done (1 left) 2024-09-21T09:42:05.008 INFO:tasks.rados.rados.0.smithi022.stdout:3103: done (0 left) 2024-09-21T09:42:05.008 INFO:tasks.rados.rados.0.smithi022.stdout:3104: read oid 32 snap -1 2024-09-21T09:42:05.008 INFO:tasks.rados.rados.0.smithi022.stdout:3104: expect (ObjNum 884 snap 282 seq_num 884) 2024-09-21T09:42:05.008 INFO:tasks.rados.rados.0.smithi022.stdout:3105: rollback oid 11 current snap is 304 2024-09-21T09:42:05.008 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 11 to 302 2024-09-21T09:42:05.008 INFO:tasks.rados.rados.0.smithi022.stdout:3106: rmattr oid 37 current snap is 304 2024-09-21T09:42:05.008 INFO:tasks.rados.rados.0.smithi022.stdout:3106: done (2 left) 2024-09-21T09:42:05.008 INFO:tasks.rados.rados.0.smithi022.stdout:3107: snap_remove snap 299 2024-09-21T09:42:05.068 INFO:tasks.rados.rados.0.smithi022.stdout:3107: done (2 left) 2024-09-21T09:42:05.068 INFO:tasks.rados.rados.0.smithi022.stdout:3108: snap_remove snap 301 2024-09-21T09:42:06.058 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:42:05 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:42:05] "GET /metrics HTTP/1.1" 200 36357 "" "Prometheus/2.51.0" 2024-09-21T09:42:06.072 INFO:tasks.rados.rados.0.smithi022.stdout:3108: done (2 left) 2024-09-21T09:42:06.072 INFO:tasks.rados.rados.0.smithi022.stdout:3109: read oid 35 snap 300 2024-09-21T09:42:06.072 INFO:tasks.rados.rados.0.smithi022.stdout:3109: expect (ObjNum 925 snap 287 seq_num 925) 2024-09-21T09:42:06.072 INFO:tasks.rados.rados.0.smithi022.stdout:3110: rollback oid 41 current snap is 304 2024-09-21T09:42:06.072 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 41 to 300 2024-09-21T09:42:06.072 INFO:tasks.rados.rados.0.smithi022.stdout:3105: finishing rollback tid 0 to smithi02284960-11 2024-09-21T09:42:06.072 INFO:tasks.rados.rados.0.smithi022.stdout:3105: finishing rollback tid 1 to smithi02284960-11 2024-09-21T09:42:06.072 INFO:tasks.rados.rados.0.smithi022.stdout:3105: finishing rollback tid 2 to smithi02284960-11 2024-09-21T09:42:06.072 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 11 v 1675 (ObjNum 971 snap 296 seq_num 971) dirty exists 2024-09-21T09:42:06.073 INFO:tasks.rados.rados.0.smithi022.stdout:3104: expect (ObjNum 884 snap 282 seq_num 884) 2024-09-21T09:42:06.318 INFO:tasks.rados.rados.0.smithi022.stdout:3110: finishing rollback tid 0 to smithi02284960-41 2024-09-21T09:42:06.318 INFO:tasks.rados.rados.0.smithi022.stdout:3110: finishing rollback tid 1 to smithi02284960-41 2024-09-21T09:42:06.318 INFO:tasks.rados.rados.0.smithi022.stdout:3110: finishing rollback tid 2 to smithi02284960-41 2024-09-21T09:42:06.318 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 41 v 1672 (ObjNum 865 snap 277 seq_num 865) dirty exists 2024-09-21T09:42:06.319 INFO:tasks.rados.rados.0.smithi022.stdout:3104: done (3 left) 2024-09-21T09:42:06.319 INFO:tasks.rados.rados.0.smithi022.stdout:3105: done (2 left) 2024-09-21T09:42:06.319 INFO:tasks.rados.rados.0.smithi022.stdout:3110: done (1 left) 2024-09-21T09:42:06.319 INFO:tasks.rados.rados.0.smithi022.stdout:3111: snap_create 2024-09-21T09:42:06.319 INFO:tasks.rados.rados.0.smithi022.stdout:3109: expect (ObjNum 925 snap 287 seq_num 925) 2024-09-21T09:42:06.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:42:06 smithi064 ceph-mon[86709]: pgmap v641: 121 pgs: 121 active+clean; 204 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 18 MiB/s rd, 1.7 MiB/s wr, 10 op/s 2024-09-21T09:42:06.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:42:06 smithi064 ceph-mon[86709]: osdmap e687: 8 total, 8 up, 8 in 2024-09-21T09:42:06.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:42:06 smithi022 ceph-mon[98615]: pgmap v641: 121 pgs: 121 active+clean; 204 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 18 MiB/s rd, 1.7 MiB/s wr, 10 op/s 2024-09-21T09:42:06.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:42:06 smithi022 ceph-mon[98615]: osdmap e687: 8 total, 8 up, 8 in 2024-09-21T09:42:06.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:42:06 smithi022 ceph-mon[96736]: pgmap v641: 121 pgs: 121 active+clean; 204 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 18 MiB/s rd, 1.7 MiB/s wr, 10 op/s 2024-09-21T09:42:06.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:42:06 smithi022 ceph-mon[96736]: osdmap e687: 8 total, 8 up, 8 in 2024-09-21T09:42:07.080 INFO:tasks.rados.rados.0.smithi022.stdout:3109: done (1 left) 2024-09-21T09:42:07.081 INFO:tasks.rados.rados.0.smithi022.stdout:3111: done (0 left) 2024-09-21T09:42:07.081 INFO:tasks.rados.rados.0.smithi022.stdout:3112: read oid 13 snap -1 2024-09-21T09:42:07.081 INFO:tasks.rados.rados.0.smithi022.stdout:3112: expect deleted 2024-09-21T09:42:07.081 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 14 current snap is 305 2024-09-21T09:42:07.081 INFO:tasks.rados.rados.0.smithi022.stdout:3113: seq_num 985 ranges {3497984=499712,3997696=483328,4481024=450560,4931584=581632,5513216=90112} 2024-09-21T09:42:07.102 INFO:tasks.rados.rados.0.smithi022.stdout:3113: writing smithi02284960-14 from 3497984 to 3997696 tid 1 2024-09-21T09:42:07.105 INFO:tasks.rados.rados.0.smithi022.stdout:3113: writing smithi02284960-14 from 3997696 to 4481024 tid 2 2024-09-21T09:42:07.107 INFO:tasks.rados.rados.0.smithi022.stdout:3113: writing smithi02284960-14 from 4481024 to 4931584 tid 3 2024-09-21T09:42:07.110 INFO:tasks.rados.rados.0.smithi022.stdout:3113: writing smithi02284960-14 from 4931584 to 5513216 tid 4 2024-09-21T09:42:07.111 INFO:tasks.rados.rados.0.smithi022.stdout:3113: writing smithi02284960-14 from 5513216 to 5603328 tid 5 2024-09-21T09:42:07.111 INFO:tasks.rados.rados.0.smithi022.stdout:3114: rmattr oid 44 current snap is 305 2024-09-21T09:42:07.111 INFO:tasks.rados.rados.0.smithi022.stdout:3114: done (2 left) 2024-09-21T09:42:07.111 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 45 current snap is 305 2024-09-21T09:42:07.112 INFO:tasks.rados.rados.0.smithi022.stdout:3115: seq_num 986 ranges {0=516096,516096=770048,1286144=655360,1941504=180224} 2024-09-21T09:42:07.114 INFO:tasks.rados.rados.0.smithi022.stdout:3115: writing smithi02284960-45 from 0 to 516096 tid 1 2024-09-21T09:42:07.119 INFO:tasks.rados.rados.0.smithi022.stdout:3115: writing smithi02284960-45 from 516096 to 1286144 tid 2 2024-09-21T09:42:07.122 INFO:tasks.rados.rados.0.smithi022.stdout:3115: writing smithi02284960-45 from 1286144 to 1941504 tid 3 2024-09-21T09:42:07.124 INFO:tasks.rados.rados.0.smithi022.stdout:3115: writing smithi02284960-45 from 1941504 to 2121728 tid 4 2024-09-21T09:42:07.124 INFO:tasks.rados.rados.0.smithi022.stdout:3116: rollback oid 47 current snap is 305 2024-09-21T09:42:07.124 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 47 to 304 2024-09-21T09:42:07.124 INFO:tasks.rados.rados.0.smithi022.stdout:3113: finishing write tid 1 to smithi02284960-14 2024-09-21T09:42:07.125 INFO:tasks.rados.rados.0.smithi022.stdout:3113: finishing write tid 2 to smithi02284960-14 2024-09-21T09:42:07.125 INFO:tasks.rados.rados.0.smithi022.stdout:3113: finishing write tid 3 to smithi02284960-14 2024-09-21T09:42:07.125 INFO:tasks.rados.rados.0.smithi022.stdout:3113: finishing write tid 4 to smithi02284960-14 2024-09-21T09:42:07.125 INFO:tasks.rados.rados.0.smithi022.stdout:3113: finishing write tid 5 to smithi02284960-14 2024-09-21T09:42:07.125 INFO:tasks.rados.rados.0.smithi022.stdout:3113: finishing write tid 6 to smithi02284960-14 2024-09-21T09:42:07.125 INFO:tasks.rados.rados.0.smithi022.stdout:3113: finishing write tid 7 to smithi02284960-14 2024-09-21T09:42:07.125 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 14 v 1793 (ObjNum 985 snap 305 seq_num 985) dirty exists 2024-09-21T09:42:07.125 INFO:tasks.rados.rados.0.smithi022.stdout:3113: left oid 14 (ObjNum 985 snap 305 seq_num 985) 2024-09-21T09:42:07.125 INFO:tasks.rados.rados.0.smithi022.stdout:3115: finishing write tid 1 to smithi02284960-45 2024-09-21T09:42:07.125 INFO:tasks.rados.rados.0.smithi022.stdout:3115: finishing write tid 2 to smithi02284960-45 2024-09-21T09:42:07.125 INFO:tasks.rados.rados.0.smithi022.stdout:3112: done (3 left) 2024-09-21T09:42:07.125 INFO:tasks.rados.rados.0.smithi022.stdout:3113: done (2 left) 2024-09-21T09:42:07.125 INFO:tasks.rados.rados.0.smithi022.stdout:3117: read oid 24 snap -1 2024-09-21T09:42:07.125 INFO:tasks.rados.rados.0.smithi022.stdout:3117: expect (ObjNum 972 snap 297 seq_num 972) 2024-09-21T09:42:07.125 INFO:tasks.rados.rados.0.smithi022.stdout:3118: delete oid 13 current snap is 305 2024-09-21T09:42:07.127 INFO:tasks.rados.rados.0.smithi022.stdout:3115: finishing write tid 3 to smithi02284960-45 2024-09-21T09:42:07.127 INFO:tasks.rados.rados.0.smithi022.stdout:3115: finishing write tid 4 to smithi02284960-45 2024-09-21T09:42:07.128 INFO:tasks.rados.rados.0.smithi022.stdout:3115: finishing write tid 5 to smithi02284960-45 2024-09-21T09:42:07.129 INFO:tasks.rados.rados.0.smithi022.stdout:3115: finishing write tid 6 to smithi02284960-45 2024-09-21T09:42:07.129 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 45 v 1422 (ObjNum 986 snap 305 seq_num 986) dirty exists 2024-09-21T09:42:07.129 INFO:tasks.rados.rados.0.smithi022.stdout:3115: left oid 45 (ObjNum 986 snap 305 seq_num 986) 2024-09-21T09:42:07.130 INFO:tasks.rados.rados.0.smithi022.stdout:3116: finishing rollback tid 0 to smithi02284960-47 2024-09-21T09:42:07.130 INFO:tasks.rados.rados.0.smithi022.stdout:3115: done (3 left) 2024-09-21T09:42:07.130 INFO:tasks.rados.rados.0.smithi022.stdout:3118: done (2 left) 2024-09-21T09:42:07.131 INFO:tasks.rados.rados.0.smithi022.stdout:3119: read oid 31 snap -1 2024-09-21T09:42:07.131 INFO:tasks.rados.rados.0.smithi022.stdout:3119: expect (ObjNum 827 snap 264 seq_num 827) 2024-09-21T09:42:07.131 INFO:tasks.rados.rados.0.smithi022.stdout:3120: rollback oid 25 current snap is 305 2024-09-21T09:42:07.131 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 25 to 302 2024-09-21T09:42:07.131 INFO:tasks.rados.rados.0.smithi022.stdout:3121: read oid 22 snap -1 2024-09-21T09:42:07.131 INFO:tasks.rados.rados.0.smithi022.stdout:3121: expect (ObjNum 914 snap 284 seq_num 914) 2024-09-21T09:42:07.131 INFO:tasks.rados.rados.0.smithi022.stdout:3122: rmattr oid 9 current snap is 305 2024-09-21T09:42:07.131 INFO:tasks.rados.rados.0.smithi022.stdout:3123: rmattr oid 28 current snap is 305 2024-09-21T09:42:07.131 INFO:tasks.rados.rados.0.smithi022.stdout:3124: copy_from oid 40 from oid 48 current snap is 305 2024-09-21T09:42:07.131 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 8 current snap is 305 2024-09-21T09:42:07.131 INFO:tasks.rados.rados.0.smithi022.stdout:3125: seq_num 988 ranges {999424=557056,1556480=532480,2088960=622592,2711552=450560} 2024-09-21T09:42:07.139 INFO:tasks.rados.rados.0.smithi022.stdout:3125: writing smithi02284960-8 from 999424 to 1556480 tid 1 2024-09-21T09:42:07.142 INFO:tasks.rados.rados.0.smithi022.stdout:3125: writing smithi02284960-8 from 1556480 to 2088960 tid 2 2024-09-21T09:42:07.145 INFO:tasks.rados.rados.0.smithi022.stdout:3125: writing smithi02284960-8 from 2088960 to 2711552 tid 3 2024-09-21T09:42:07.149 INFO:tasks.rados.rados.0.smithi022.stdout:3125: writing smithi02284960-8 from 2711552 to 3162112 tid 4 2024-09-21T09:42:07.149 INFO:tasks.rados.rados.0.smithi022.stdout:3126: read oid 32 snap -1 2024-09-21T09:42:07.149 INFO:tasks.rados.rados.0.smithi022.stdout:3126: expect (ObjNum 884 snap 282 seq_num 884) 2024-09-21T09:42:07.149 INFO:tasks.rados.rados.0.smithi022.stdout:3127: snap_remove snap 300 2024-09-21T09:42:07.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:42:07 smithi064 ceph-mon[86709]: osdmap e688: 8 total, 8 up, 8 in 2024-09-21T09:42:07.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:42:07 smithi022 ceph-mon[98615]: osdmap e688: 8 total, 8 up, 8 in 2024-09-21T09:42:07.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:42:07 smithi022 ceph-mon[96736]: osdmap e688: 8 total, 8 up, 8 in 2024-09-21T09:42:08.089 INFO:tasks.rados.rados.0.smithi022.stdout:3127: done (10 left) 2024-09-21T09:42:08.090 INFO:tasks.rados.rados.0.smithi022.stdout:3128: delete oid 26 current snap is 305 2024-09-21T09:42:08.090 INFO:tasks.rados.rados.0.smithi022.stdout:3120: finishing rollback tid 1 to smithi02284960-25 2024-09-21T09:42:08.090 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 25 v 0 (ObjNum 161 snap 0 seq_num 1) dirty dne 2024-09-21T09:42:08.090 INFO:tasks.rados.rados.0.smithi022.stdout:3124: finishing copy_from to smithi02284960-40 2024-09-21T09:42:08.090 INFO:tasks.rados.rados.0.smithi022.stdout:3124: got expected ENOENT (src dne) 2024-09-21T09:42:08.090 INFO:tasks.rados.rados.0.smithi022.stdout:3124: finishing copy_from racing read to smithi02284960-40 2024-09-21T09:42:08.090 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 28 v 1793 (ObjNum 827 snap 264 seq_num 827) dirty exists 2024-09-21T09:42:08.090 INFO:tasks.rados.rados.0.smithi022.stdout:3116: finishing rollback tid 1 to smithi02284960-47 2024-09-21T09:42:08.090 INFO:tasks.rados.rados.0.smithi022.stdout:3116: finishing rollback tid 2 to smithi02284960-47 2024-09-21T09:42:08.090 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 47 v 1678 (ObjNum 884 snap 282 seq_num 884) dirty exists 2024-09-21T09:42:08.090 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 9 v 1683 (ObjNum 979 snap 300 seq_num 979) dirty exists 2024-09-21T09:42:08.090 INFO:tasks.rados.rados.0.smithi022.stdout:3117: expect (ObjNum 972 snap 297 seq_num 972) 2024-09-21T09:42:08.148 INFO:tasks.rados.rados.0.smithi022.stdout:3121: expect (ObjNum 914 snap 284 seq_num 914) 2024-09-21T09:42:08.200 INFO:tasks.rados.rados.0.smithi022.stdout:3125: finishing write tid 1 to smithi02284960-8 2024-09-21T09:42:08.200 INFO:tasks.rados.rados.0.smithi022.stdout:3125: finishing write tid 2 to smithi02284960-8 2024-09-21T09:42:08.200 INFO:tasks.rados.rados.0.smithi022.stdout:3116: done (10 left) 2024-09-21T09:42:08.200 INFO:tasks.rados.rados.0.smithi022.stdout:3117: done (9 left) 2024-09-21T09:42:08.200 INFO:tasks.rados.rados.0.smithi022.stdout:3120: done (8 left) 2024-09-21T09:42:08.200 INFO:tasks.rados.rados.0.smithi022.stdout:3121: done (7 left) 2024-09-21T09:42:08.200 INFO:tasks.rados.rados.0.smithi022.stdout:3122: done (6 left) 2024-09-21T09:42:08.200 INFO:tasks.rados.rados.0.smithi022.stdout:3123: done (5 left) 2024-09-21T09:42:08.200 INFO:tasks.rados.rados.0.smithi022.stdout:3124: done (4 left) 2024-09-21T09:42:08.200 INFO:tasks.rados.rados.0.smithi022.stdout:3128: done (3 left) 2024-09-21T09:42:08.201 INFO:tasks.rados.rados.0.smithi022.stdout:3129: copy_from oid 12 from oid 11 current snap is 305 2024-09-21T09:42:08.201 INFO:tasks.rados.rados.0.smithi022.stdout:3119: expect (ObjNum 827 snap 264 seq_num 827) 2024-09-21T09:42:08.251 INFO:tasks.rados.rados.0.smithi022.stdout:3125: finishing write tid 3 to smithi02284960-8 2024-09-21T09:42:08.251 INFO:tasks.rados.rados.0.smithi022.stdout:3125: finishing write tid 4 to smithi02284960-8 2024-09-21T09:42:08.251 INFO:tasks.rados.rados.0.smithi022.stdout:3125: finishing write tid 5 to smithi02284960-8 2024-09-21T09:42:08.251 INFO:tasks.rados.rados.0.smithi022.stdout:3125: finishing write tid 6 to smithi02284960-8 2024-09-21T09:42:08.252 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 8 v 1688 (ObjNum 988 snap 305 seq_num 988) dirty exists 2024-09-21T09:42:08.252 INFO:tasks.rados.rados.0.smithi022.stdout:3125: left oid 8 (ObjNum 988 snap 305 seq_num 988) 2024-09-21T09:42:08.252 INFO:tasks.rados.rados.0.smithi022.stdout:3126: expect (ObjNum 884 snap 282 seq_num 884) 2024-09-21T09:42:08.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:42:08 smithi064 ceph-mon[86709]: pgmap v644: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 197 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 19 MiB/s rd, 1.6 MiB/s wr, 11 op/s 2024-09-21T09:42:08.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:42:08 smithi064 ceph-mon[86709]: osdmap e689: 8 total, 8 up, 8 in 2024-09-21T09:42:08.490 INFO:tasks.rados.rados.0.smithi022.stdout:3129: finishing copy_from racing read to smithi02284960-12 2024-09-21T09:42:08.490 INFO:tasks.rados.rados.0.smithi022.stdout:3129: finishing copy_from to smithi02284960-12 2024-09-21T09:42:08.490 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 12 v 1675 (ObjNum 971 snap 296 seq_num 971) dirty exists 2024-09-21T09:42:08.490 INFO:tasks.rados.rados.0.smithi022.stdout:3119: done (3 left) 2024-09-21T09:42:08.490 INFO:tasks.rados.rados.0.smithi022.stdout:3125: done (2 left) 2024-09-21T09:42:08.491 INFO:tasks.rados.rados.0.smithi022.stdout:3126: done (1 left) 2024-09-21T09:42:08.491 INFO:tasks.rados.rados.0.smithi022.stdout:3129: done (0 left) 2024-09-21T09:42:08.491 INFO:tasks.rados.rados.0.smithi022.stdout:3130: delete oid 10 current snap is 305 2024-09-21T09:42:08.492 INFO:tasks.rados.rados.0.smithi022.stdout:3130: done (0 left) 2024-09-21T09:42:08.492 INFO:tasks.rados.rados.0.smithi022.stdout:3131: snap_remove snap 302 2024-09-21T09:42:08.557 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:42:08 smithi022 ceph-mon[96736]: pgmap v644: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 197 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 19 MiB/s rd, 1.6 MiB/s wr, 11 op/s 2024-09-21T09:42:08.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:42:08 smithi022 ceph-mon[96736]: osdmap e689: 8 total, 8 up, 8 in 2024-09-21T09:42:08.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:42:08 smithi022 ceph-mon[98615]: pgmap v644: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 197 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 19 MiB/s rd, 1.6 MiB/s wr, 11 op/s 2024-09-21T09:42:08.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:42:08 smithi022 ceph-mon[98615]: osdmap e689: 8 total, 8 up, 8 in 2024-09-21T09:42:09.059 INFO:tasks.rados.rados.0.smithi022.stdout:3131: done (0 left) 2024-09-21T09:42:09.059 INFO:tasks.rados.rados.0.smithi022.stdout:3132: read oid 39 snap -1 2024-09-21T09:42:09.059 INFO:tasks.rados.rados.0.smithi022.stdout:3132: expect (ObjNum 911 snap 284 seq_num 911) 2024-09-21T09:42:09.059 INFO:tasks.rados.rados.0.smithi022.stdout:3133: setattr oid 34 current snap is 305 2024-09-21T09:42:09.064 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 38 current snap is 305 2024-09-21T09:42:09.064 INFO:tasks.rados.rados.0.smithi022.stdout:3134: seq_num 991 ranges {5185536=606208,5791744=638976,6430720=794624,7225344=172032} 2024-09-21T09:42:09.094 INFO:tasks.rados.rados.0.smithi022.stdout:3134: writing smithi02284960-38 from 5185536 to 5791744 tid 1 2024-09-21T09:42:09.098 INFO:tasks.rados.rados.0.smithi022.stdout:3134: writing smithi02284960-38 from 5791744 to 6430720 tid 2 2024-09-21T09:42:09.103 INFO:tasks.rados.rados.0.smithi022.stdout:3134: writing smithi02284960-38 from 6430720 to 7225344 tid 3 2024-09-21T09:42:09.104 INFO:tasks.rados.rados.0.smithi022.stdout:3134: writing smithi02284960-38 from 7225344 to 7397376 tid 4 2024-09-21T09:42:09.104 INFO:tasks.rados.rados.0.smithi022.stdout:3135: read oid 31 snap -1 2024-09-21T09:42:09.104 INFO:tasks.rados.rados.0.smithi022.stdout:3135: expect (ObjNum 827 snap 264 seq_num 827) 2024-09-21T09:42:09.104 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 21 current snap is 305 2024-09-21T09:42:09.104 INFO:tasks.rados.rados.0.smithi022.stdout:3136: seq_num 992 ranges {6078464=622592,6701056=524288,7225344=606208,7831552=475136} 2024-09-21T09:42:09.142 INFO:tasks.rados.rados.0.smithi022.stdout:3136: writing smithi02284960-21 from 6078464 to 6701056 tid 1 2024-09-21T09:42:09.145 INFO:tasks.rados.rados.0.smithi022.stdout:3136: writing smithi02284960-21 from 6701056 to 7225344 tid 2 2024-09-21T09:42:09.149 INFO:tasks.rados.rados.0.smithi022.stdout:3136: writing smithi02284960-21 from 7225344 to 7831552 tid 3 2024-09-21T09:42:09.152 INFO:tasks.rados.rados.0.smithi022.stdout:3136: writing smithi02284960-21 from 7831552 to 8306688 tid 4 2024-09-21T09:42:09.152 INFO:tasks.rados.rados.0.smithi022.stdout:3137: delete oid 10 current snap is 305 2024-09-21T09:42:09.152 INFO:tasks.rados.rados.0.smithi022.stdout:3132: expect (ObjNum 911 snap 284 seq_num 911) 2024-09-21T09:42:09.176 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 34 v 1801 (ObjNum 984 snap 304 seq_num 984) dirty exists 2024-09-21T09:42:09.176 INFO:tasks.rados.rados.0.smithi022.stdout:3134: finishing write tid 1 to smithi02284960-38 2024-09-21T09:42:09.176 INFO:tasks.rados.rados.0.smithi022.stdout:3134: finishing write tid 2 to smithi02284960-38 2024-09-21T09:42:09.176 INFO:tasks.rados.rados.0.smithi022.stdout:3134: finishing write tid 3 to smithi02284960-38 2024-09-21T09:42:09.176 INFO:tasks.rados.rados.0.smithi022.stdout:3134: finishing write tid 4 to smithi02284960-38 2024-09-21T09:42:09.176 INFO:tasks.rados.rados.0.smithi022.stdout:3134: finishing write tid 5 to smithi02284960-38 2024-09-21T09:42:09.176 INFO:tasks.rados.rados.0.smithi022.stdout:3134: finishing write tid 6 to smithi02284960-38 2024-09-21T09:42:09.176 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 38 v 1802 (ObjNum 991 snap 305 seq_num 991) dirty exists 2024-09-21T09:42:09.177 INFO:tasks.rados.rados.0.smithi022.stdout:3134: left oid 38 (ObjNum 991 snap 305 seq_num 991) 2024-09-21T09:42:09.177 INFO:tasks.rados.rados.0.smithi022.stdout:3135: expect (ObjNum 827 snap 264 seq_num 827) 2024-09-21T09:42:09.228 INFO:tasks.rados.rados.0.smithi022.stdout:3136: finishing write tid 1 to smithi02284960-21 2024-09-21T09:42:09.228 INFO:tasks.rados.rados.0.smithi022.stdout:3136: finishing write tid 2 to smithi02284960-21 2024-09-21T09:42:09.228 INFO:tasks.rados.rados.0.smithi022.stdout:3136: finishing write tid 3 to smithi02284960-21 2024-09-21T09:42:09.228 INFO:tasks.rados.rados.0.smithi022.stdout:3136: finishing write tid 4 to smithi02284960-21 2024-09-21T09:42:09.228 INFO:tasks.rados.rados.0.smithi022.stdout:3136: finishing write tid 5 to smithi02284960-21 2024-09-21T09:42:09.228 INFO:tasks.rados.rados.0.smithi022.stdout:3136: finishing write tid 6 to smithi02284960-21 2024-09-21T09:42:09.228 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 21 v 1806 (ObjNum 992 snap 305 seq_num 992) dirty exists 2024-09-21T09:42:09.228 INFO:tasks.rados.rados.0.smithi022.stdout:3136: left oid 21 (ObjNum 992 snap 305 seq_num 992) 2024-09-21T09:42:09.228 INFO:tasks.rados.rados.0.smithi022.stdout:3132: done (5 left) 2024-09-21T09:42:09.228 INFO:tasks.rados.rados.0.smithi022.stdout:3133: done (4 left) 2024-09-21T09:42:09.228 INFO:tasks.rados.rados.0.smithi022.stdout:3134: done (3 left) 2024-09-21T09:42:09.229 INFO:tasks.rados.rados.0.smithi022.stdout:3135: done (2 left) 2024-09-21T09:42:09.229 INFO:tasks.rados.rados.0.smithi022.stdout:3136: done (1 left) 2024-09-21T09:42:09.229 INFO:tasks.rados.rados.0.smithi022.stdout:3137: done (0 left) 2024-09-21T09:42:09.229 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 34 current snap is 305 2024-09-21T09:42:09.229 INFO:tasks.rados.rados.0.smithi022.stdout:3138: seq_num 994 ranges {2932736=655360,3588096=688128,4276224=622592,4898816=294912} 2024-09-21T09:42:09.247 INFO:tasks.rados.rados.0.smithi022.stdout:3138: writing smithi02284960-34 from 2932736 to 3588096 tid 1 2024-09-21T09:42:09.251 INFO:tasks.rados.rados.0.smithi022.stdout:3138: writing smithi02284960-34 from 3588096 to 4276224 tid 2 2024-09-21T09:42:09.254 INFO:tasks.rados.rados.0.smithi022.stdout:3138: writing smithi02284960-34 from 4276224 to 4898816 tid 3 2024-09-21T09:42:09.256 INFO:tasks.rados.rados.0.smithi022.stdout:3138: writing smithi02284960-34 from 4898816 to 5193728 tid 4 2024-09-21T09:42:09.257 INFO:tasks.rados.rados.0.smithi022.stdout:3139: read oid 41 snap 304 2024-09-21T09:42:09.257 INFO:tasks.rados.rados.0.smithi022.stdout:3139: expect (ObjNum 865 snap 277 seq_num 865) 2024-09-21T09:42:09.257 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 24 current snap is 305 2024-09-21T09:42:09.257 INFO:tasks.rados.rados.0.smithi022.stdout:3140: seq_num 995 ranges {1892352=671744,2564096=573440,3137536=778240,3915776=253952} 2024-09-21T09:42:09.270 INFO:tasks.rados.rados.0.smithi022.stdout:3140: writing smithi02284960-24 from 1892352 to 2564096 tid 1 2024-09-21T09:42:09.274 INFO:tasks.rados.rados.0.smithi022.stdout:3140: writing smithi02284960-24 from 2564096 to 3137536 tid 2 2024-09-21T09:42:09.278 INFO:tasks.rados.rados.0.smithi022.stdout:3140: writing smithi02284960-24 from 3137536 to 3915776 tid 3 2024-09-21T09:42:09.280 INFO:tasks.rados.rados.0.smithi022.stdout:3140: writing smithi02284960-24 from 3915776 to 4169728 tid 4 2024-09-21T09:42:09.280 INFO:tasks.rados.rados.0.smithi022.stdout:3141: read oid 45 snap 304 2024-09-21T09:42:09.280 INFO:tasks.rados.rados.0.smithi022.stdout:3141: expect deleted 2024-09-21T09:42:09.280 INFO:tasks.rados.rados.0.smithi022.stdout:3138: finishing write tid 1 to smithi02284960-34 2024-09-21T09:42:09.280 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 15 current snap is 305 2024-09-21T09:42:09.280 INFO:tasks.rados.rados.0.smithi022.stdout:3138: finishing write tid 2 to smithi02284960-34 2024-09-21T09:42:09.280 INFO:tasks.rados.rados.0.smithi022.stdout:3138: finishing write tid 3 to smithi02284960-34 2024-09-21T09:42:09.280 INFO:tasks.rados.rados.0.smithi022.stdout:3142: seq_num 996 ranges {8970240=688128,9658368=507904,10166272=581632,10747904=516096} 2024-09-21T09:42:09.332 INFO:tasks.rados.rados.0.smithi022.stdout:3142: writing smithi02284960-15 from 8970240 to 9658368 tid 1 2024-09-21T09:42:09.335 INFO:tasks.rados.rados.0.smithi022.stdout:3142: writing smithi02284960-15 from 9658368 to 10166272 tid 2 2024-09-21T09:42:09.338 INFO:tasks.rados.rados.0.smithi022.stdout:3142: writing smithi02284960-15 from 10166272 to 10747904 tid 3 2024-09-21T09:42:09.342 INFO:tasks.rados.rados.0.smithi022.stdout:3142: writing smithi02284960-15 from 10747904 to 11264000 tid 4 2024-09-21T09:42:09.342 INFO:tasks.rados.rados.0.smithi022.stdout:3143: rollback oid 35 current snap is 305 2024-09-21T09:42:09.342 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 35 to 304 2024-09-21T09:42:09.342 INFO:tasks.rados.rados.0.smithi022.stdout:3138: finishing write tid 4 to smithi02284960-34 2024-09-21T09:42:09.342 INFO:tasks.rados.rados.0.smithi022.stdout:3138: finishing write tid 5 to smithi02284960-34 2024-09-21T09:42:09.342 INFO:tasks.rados.rados.0.smithi022.stdout:3138: finishing write tid 6 to smithi02284960-34 2024-09-21T09:42:09.342 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 34 v 1811 (ObjNum 994 snap 305 seq_num 994) dirty exists 2024-09-21T09:42:09.342 INFO:tasks.rados.rados.0.smithi022.stdout:3138: left oid 34 (ObjNum 994 snap 305 seq_num 994) 2024-09-21T09:42:09.342 INFO:tasks.rados.rados.0.smithi022.stdout:3139: expect (ObjNum 865 snap 277 seq_num 865) 2024-09-21T09:42:09.348 INFO:tasks.rados.rados.0.smithi022.stdout:3140: finishing write tid 1 to smithi02284960-24 2024-09-21T09:42:09.348 INFO:tasks.rados.rados.0.smithi022.stdout:3140: finishing write tid 2 to smithi02284960-24 2024-09-21T09:42:09.348 INFO:tasks.rados.rados.0.smithi022.stdout:3138: done (5 left) 2024-09-21T09:42:09.348 INFO:tasks.rados.rados.0.smithi022.stdout:3139: done (4 left) 2024-09-21T09:42:09.348 INFO:tasks.rados.rados.0.smithi022.stdout:3144: rollback oid 46 current snap is 305 2024-09-21T09:42:09.348 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 46 to 304 2024-09-21T09:42:09.348 INFO:tasks.rados.rados.0.smithi022.stdout:3140: finishing write tid 3 to smithi02284960-24 2024-09-21T09:42:09.349 INFO:tasks.rados.rados.0.smithi022.stdout:3140: finishing write tid 4 to smithi02284960-24 2024-09-21T09:42:09.349 INFO:tasks.rados.rados.0.smithi022.stdout:3140: finishing write tid 5 to smithi02284960-24 2024-09-21T09:42:09.349 INFO:tasks.rados.rados.0.smithi022.stdout:3140: finishing write tid 6 to smithi02284960-24 2024-09-21T09:42:09.349 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 24 v 1858 (ObjNum 995 snap 305 seq_num 995) dirty exists 2024-09-21T09:42:09.349 INFO:tasks.rados.rados.0.smithi022.stdout:3140: left oid 24 (ObjNum 995 snap 305 seq_num 995) 2024-09-21T09:42:09.349 INFO:tasks.rados.rados.0.smithi022.stdout:3142: finishing write tid 1 to smithi02284960-15 2024-09-21T09:42:09.349 INFO:tasks.rados.rados.0.smithi022.stdout:3142: finishing write tid 2 to smithi02284960-15 2024-09-21T09:42:09.349 INFO:tasks.rados.rados.0.smithi022.stdout:3142: finishing write tid 3 to smithi02284960-15 2024-09-21T09:42:09.349 INFO:tasks.rados.rados.0.smithi022.stdout:3143: finishing rollback tid 0 to smithi02284960-35 2024-09-21T09:42:09.349 INFO:tasks.rados.rados.0.smithi022.stdout:3143: finishing rollback tid 1 to smithi02284960-35 2024-09-21T09:42:09.349 INFO:tasks.rados.rados.0.smithi022.stdout:3142: finishing write tid 4 to smithi02284960-15 2024-09-21T09:42:09.349 INFO:tasks.rados.rados.0.smithi022.stdout:3143: finishing rollback tid 2 to smithi02284960-35 2024-09-21T09:42:09.349 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 35 v 1796 (ObjNum 925 snap 287 seq_num 925) dirty exists 2024-09-21T09:42:09.349 INFO:tasks.rados.rados.0.smithi022.stdout:3140: done (4 left) 2024-09-21T09:42:09.349 INFO:tasks.rados.rados.0.smithi022.stdout:3141: done (3 left) 2024-09-21T09:42:09.350 INFO:tasks.rados.rados.0.smithi022.stdout:3143: done (2 left) 2024-09-21T09:42:09.350 INFO:tasks.rados.rados.0.smithi022.stdout:3145: snap_create 2024-09-21T09:42:09.350 INFO:tasks.rados.rados.0.smithi022.stdout:3142: finishing write tid 5 to smithi02284960-15 2024-09-21T09:42:09.350 INFO:tasks.rados.rados.0.smithi022.stdout:3142: finishing write tid 6 to smithi02284960-15 2024-09-21T09:42:09.350 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 15 v 1864 (ObjNum 996 snap 305 seq_num 996) dirty exists 2024-09-21T09:42:09.350 INFO:tasks.rados.rados.0.smithi022.stdout:3142: left oid 15 (ObjNum 996 snap 305 seq_num 996) 2024-09-21T09:42:09.350 INFO:tasks.rados.rados.0.smithi022.stdout:3144: finishing rollback tid 0 to smithi02284960-46 2024-09-21T09:42:09.351 INFO:tasks.rados.rados.0.smithi022.stdout:3144: finishing rollback tid 1 to smithi02284960-46 2024-09-21T09:42:09.351 INFO:tasks.rados.rados.0.smithi022.stdout:3144: finishing rollback tid 2 to smithi02284960-46 2024-09-21T09:42:09.351 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 46 v 1681 (ObjNum 936 snap 291 seq_num 936) dirty exists 2024-09-21T09:42:09.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:42:09 smithi064 ceph-mon[86709]: osdmap e690: 8 total, 8 up, 8 in 2024-09-21T09:42:09.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:42:09 smithi064 ceph-mon[86709]: osdmap e691: 8 total, 8 up, 8 in 2024-09-21T09:42:09.557 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:42:09 smithi022 ceph-mon[98615]: osdmap e690: 8 total, 8 up, 8 in 2024-09-21T09:42:09.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:42:09 smithi022 ceph-mon[98615]: osdmap e691: 8 total, 8 up, 8 in 2024-09-21T09:42:09.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:42:09 smithi022 ceph-mon[96736]: osdmap e690: 8 total, 8 up, 8 in 2024-09-21T09:42:09.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:42:09 smithi022 ceph-mon[96736]: osdmap e691: 8 total, 8 up, 8 in 2024-09-21T09:42:10.096 INFO:tasks.rados.rados.0.smithi022.stdout:3142: done (2 left) 2024-09-21T09:42:10.096 INFO:tasks.rados.rados.0.smithi022.stdout:3144: done (1 left) 2024-09-21T09:42:10.096 INFO:tasks.rados.rados.0.smithi022.stdout:3145: done (0 left) 2024-09-21T09:42:10.096 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 30 current snap is 306 2024-09-21T09:42:10.096 INFO:tasks.rados.rados.0.smithi022.stdout:3146: seq_num 997 ranges {0=704512,704512=794624,1499136=794624,2293760=16384} 2024-09-21T09:42:10.099 INFO:tasks.rados.rados.0.smithi022.stdout:3146: writing smithi02284960-30 from 0 to 704512 tid 1 2024-09-21T09:42:10.104 INFO:tasks.rados.rados.0.smithi022.stdout:3146: writing smithi02284960-30 from 704512 to 1499136 tid 2 2024-09-21T09:42:10.109 INFO:tasks.rados.rados.0.smithi022.stdout:3146: writing smithi02284960-30 from 1499136 to 2293760 tid 3 2024-09-21T09:42:10.109 INFO:tasks.rados.rados.0.smithi022.stdout:3146: writing smithi02284960-30 from 2293760 to 2310144 tid 4 2024-09-21T09:42:10.109 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 38 current snap is 306 2024-09-21T09:42:10.109 INFO:tasks.rados.rados.0.smithi022.stdout:3147: seq_num 998 ranges {7397376=720896,8118272=679936,8798208=598016,9396224=327680} 2024-09-21T09:42:10.153 INFO:tasks.rados.rados.0.smithi022.stdout:3147: writing smithi02284960-38 from 7397376 to 8118272 tid 1 2024-09-21T09:42:10.157 INFO:tasks.rados.rados.0.smithi022.stdout:3147: writing smithi02284960-38 from 8118272 to 8798208 tid 2 2024-09-21T09:42:10.161 INFO:tasks.rados.rados.0.smithi022.stdout:3147: writing smithi02284960-38 from 8798208 to 9396224 tid 3 2024-09-21T09:42:10.163 INFO:tasks.rados.rados.0.smithi022.stdout:3147: writing smithi02284960-38 from 9396224 to 9723904 tid 4 2024-09-21T09:42:10.163 INFO:tasks.rados.rados.0.smithi022.stdout:3148: snap_create 2024-09-21T09:42:10.164 INFO:tasks.rados.rados.0.smithi022.stdout:3146: finishing write tid 1 to smithi02284960-30 2024-09-21T09:42:10.164 INFO:tasks.rados.rados.0.smithi022.stdout:3146: finishing write tid 2 to smithi02284960-30 2024-09-21T09:42:10.164 INFO:tasks.rados.rados.0.smithi022.stdout:3146: finishing write tid 3 to smithi02284960-30 2024-09-21T09:42:10.164 INFO:tasks.rados.rados.0.smithi022.stdout:3146: finishing write tid 4 to smithi02284960-30 2024-09-21T09:42:10.164 INFO:tasks.rados.rados.0.smithi022.stdout:3146: finishing write tid 5 to smithi02284960-30 2024-09-21T09:42:10.164 INFO:tasks.rados.rados.0.smithi022.stdout:3146: finishing write tid 6 to smithi02284960-30 2024-09-21T09:42:10.164 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 30 v 1334 (ObjNum 997 snap 306 seq_num 997) dirty exists 2024-09-21T09:42:10.164 INFO:tasks.rados.rados.0.smithi022.stdout:3146: left oid 30 (ObjNum 997 snap 306 seq_num 997) 2024-09-21T09:42:10.164 INFO:tasks.rados.rados.0.smithi022.stdout:3147: finishing write tid 1 to smithi02284960-38 2024-09-21T09:42:10.164 INFO:tasks.rados.rados.0.smithi022.stdout:3147: finishing write tid 2 to smithi02284960-38 2024-09-21T09:42:10.165 INFO:tasks.rados.rados.0.smithi022.stdout:3147: finishing write tid 3 to smithi02284960-38 2024-09-21T09:42:10.166 INFO:tasks.rados.rados.0.smithi022.stdout:3147: finishing write tid 4 to smithi02284960-38 2024-09-21T09:42:10.166 INFO:tasks.rados.rados.0.smithi022.stdout:3147: finishing write tid 5 to smithi02284960-38 2024-09-21T09:42:10.167 INFO:tasks.rados.rados.0.smithi022.stdout:3147: finishing write tid 6 to smithi02284960-38 2024-09-21T09:42:10.167 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 38 v 1807 (ObjNum 998 snap 306 seq_num 998) dirty exists 2024-09-21T09:42:10.167 INFO:tasks.rados.rados.0.smithi022.stdout:3147: left oid 38 (ObjNum 998 snap 306 seq_num 998) 2024-09-21T09:42:10.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:42:10 smithi064 ceph-mon[86709]: pgmap v647: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 201 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 5.2 MiB/s rd, 1.0 MiB/s wr, 7 op/s 2024-09-21T09:42:10.442 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:42:10 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:42:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:42:10.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:42:10 smithi022 ceph-mon[96736]: pgmap v647: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 201 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 5.2 MiB/s rd, 1.0 MiB/s wr, 7 op/s 2024-09-21T09:42:10.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:42:10 smithi022 ceph-mon[98615]: pgmap v647: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 201 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 5.2 MiB/s rd, 1.0 MiB/s wr, 7 op/s 2024-09-21T09:42:11.110 INFO:tasks.rados.rados.0.smithi022.stdout:3146: done (2 left) 2024-09-21T09:42:11.110 INFO:tasks.rados.rados.0.smithi022.stdout:3147: done (1 left) 2024-09-21T09:42:11.110 INFO:tasks.rados.rados.0.smithi022.stdout:3148: done (0 left) 2024-09-21T09:42:11.110 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 31 current snap is 307 2024-09-21T09:42:11.111 INFO:tasks.rados.rados.0.smithi022.stdout:3149: seq_num 999 ranges {1859584=737280,2596864=565248,3162112=761856,3923968=278528} 2024-09-21T09:42:11.124 INFO:tasks.rados.rados.0.smithi022.stdout:3149: writing smithi02284960-31 from 1859584 to 2596864 tid 1 2024-09-21T09:42:11.127 INFO:tasks.rados.rados.0.smithi022.stdout:3149: writing smithi02284960-31 from 2596864 to 3162112 tid 2 2024-09-21T09:42:11.132 INFO:tasks.rados.rados.0.smithi022.stdout:3149: writing smithi02284960-31 from 3162112 to 3923968 tid 3 2024-09-21T09:42:11.134 INFO:tasks.rados.rados.0.smithi022.stdout:3149: writing smithi02284960-31 from 3923968 to 4202496 tid 4 2024-09-21T09:42:11.134 INFO:tasks.rados.rados.0.smithi022.stdout:3150: rmattr oid 35 current snap is 307 2024-09-21T09:42:11.134 INFO:tasks.rados.rados.0.smithi022.stdout:3150: done (1 left) 2024-09-21T09:42:11.134 INFO:tasks.rados.rados.0.smithi022.stdout:3151: read oid 29 snap 306 2024-09-21T09:42:11.134 INFO:tasks.rados.rados.0.smithi022.stdout:3151: expect (ObjNum 827 snap 264 seq_num 827) 2024-09-21T09:42:11.134 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 49 current snap is 307 2024-09-21T09:42:11.134 INFO:tasks.rados.rados.0.smithi022.stdout:3149: finishing write tid 1 to smithi02284960-31 2024-09-21T09:42:11.134 INFO:tasks.rados.rados.0.smithi022.stdout:3152: seq_num 1000 ranges {1286144=753664,2039808=442368,2482176=565248,3047424=458752,3506176=139264} 2024-09-21T09:42:11.145 INFO:tasks.rados.rados.0.smithi022.stdout:3152: writing smithi02284960-49 from 1286144 to 2039808 tid 1 2024-09-21T09:42:11.148 INFO:tasks.rados.rados.0.smithi022.stdout:3152: writing smithi02284960-49 from 2039808 to 2482176 tid 2 2024-09-21T09:42:11.151 INFO:tasks.rados.rados.0.smithi022.stdout:3152: writing smithi02284960-49 from 2482176 to 3047424 tid 3 2024-09-21T09:42:11.154 INFO:tasks.rados.rados.0.smithi022.stdout:3152: writing smithi02284960-49 from 3047424 to 3506176 tid 4 2024-09-21T09:42:11.155 INFO:tasks.rados.rados.0.smithi022.stdout:3152: writing smithi02284960-49 from 3506176 to 3645440 tid 5 2024-09-21T09:42:11.155 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 6 current snap is 307 2024-09-21T09:42:11.155 INFO:tasks.rados.rados.0.smithi022.stdout:3149: finishing write tid 2 to smithi02284960-31 2024-09-21T09:42:11.156 INFO:tasks.rados.rados.0.smithi022.stdout:3149: finishing write tid 3 to smithi02284960-31 2024-09-21T09:42:11.156 INFO:tasks.rados.rados.0.smithi022.stdout:3149: finishing write tid 4 to smithi02284960-31 2024-09-21T09:42:11.156 INFO:tasks.rados.rados.0.smithi022.stdout:3149: finishing write tid 5 to smithi02284960-31 2024-09-21T09:42:11.156 INFO:tasks.rados.rados.0.smithi022.stdout:3149: finishing write tid 6 to smithi02284960-31 2024-09-21T09:42:11.156 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 31 v 1870 (ObjNum 999 snap 307 seq_num 999) dirty exists 2024-09-21T09:42:11.156 INFO:tasks.rados.rados.0.smithi022.stdout:3149: left oid 31 (ObjNum 999 snap 307 seq_num 999) 2024-09-21T09:42:11.156 INFO:tasks.rados.rados.0.smithi022.stdout:3151: expect (ObjNum 827 snap 264 seq_num 827) 2024-09-21T09:42:11.207 INFO:tasks.rados.rados.0.smithi022.stdout:3152: finishing write tid 1 to smithi02284960-49 2024-09-21T09:42:11.207 INFO:tasks.rados.rados.0.smithi022.stdout:3152: finishing write tid 2 to smithi02284960-49 2024-09-21T09:42:11.207 INFO:tasks.rados.rados.0.smithi022.stdout:3152: finishing write tid 3 to smithi02284960-49 2024-09-21T09:42:11.207 INFO:tasks.rados.rados.0.smithi022.stdout:3152: finishing write tid 4 to smithi02284960-49 2024-09-21T09:42:11.207 INFO:tasks.rados.rados.0.smithi022.stdout:3152: finishing write tid 5 to smithi02284960-49 2024-09-21T09:42:11.207 INFO:tasks.rados.rados.0.smithi022.stdout:3152: finishing write tid 6 to smithi02284960-49 2024-09-21T09:42:11.207 INFO:tasks.rados.rados.0.smithi022.stdout:3152: finishing write tid 7 to smithi02284960-49 2024-09-21T09:42:11.208 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 49 v 1799 (ObjNum 1000 snap 307 seq_num 1000) dirty exists 2024-09-21T09:42:11.208 INFO:tasks.rados.rados.0.smithi022.stdout:3152: left oid 49 (ObjNum 1000 snap 307 seq_num 1000) 2024-09-21T09:42:11.208 INFO:tasks.rados.rados.0.smithi022.stdout:3153: seq_num 1001 ranges {1114112=770048,1884160=729088,2613248=770048,3383296=106496} 2024-09-21T09:42:11.217 INFO:tasks.rados.rados.0.smithi022.stdout:3153: writing smithi02284960-6 from 1114112 to 1884160 tid 1 2024-09-21T09:42:11.221 INFO:tasks.rados.rados.0.smithi022.stdout:3153: writing smithi02284960-6 from 1884160 to 2613248 tid 2 2024-09-21T09:42:11.226 INFO:tasks.rados.rados.0.smithi022.stdout:3153: writing smithi02284960-6 from 2613248 to 3383296 tid 3 2024-09-21T09:42:11.228 INFO:tasks.rados.rados.0.smithi022.stdout:3153: writing smithi02284960-6 from 3383296 to 3489792 tid 4 2024-09-21T09:42:11.228 INFO:tasks.rados.rados.0.smithi022.stdout:3149: done (3 left) 2024-09-21T09:42:11.228 INFO:tasks.rados.rados.0.smithi022.stdout:3151: done (2 left) 2024-09-21T09:42:11.228 INFO:tasks.rados.rados.0.smithi022.stdout:3152: done (1 left) 2024-09-21T09:42:11.228 INFO:tasks.rados.rados.0.smithi022.stdout:3154: copy_from oid 41 from oid 14 current snap is 307 2024-09-21T09:42:11.228 INFO:tasks.rados.rados.0.smithi022.stdout:3153: finishing write tid 1 to smithi02284960-6 2024-09-21T09:42:11.228 INFO:tasks.rados.rados.0.smithi022.stdout:3153: finishing write tid 2 to smithi02284960-6 2024-09-21T09:42:11.228 INFO:tasks.rados.rados.0.smithi022.stdout:3155: copy_from oid 7 from oid 2 current snap is 307 2024-09-21T09:42:11.228 INFO:tasks.rados.rados.0.smithi022.stdout:3156: copy_from oid 4 from oid 38 current snap is 307 2024-09-21T09:42:11.228 INFO:tasks.rados.rados.0.smithi022.stdout:3157: read oid 10 snap -1 2024-09-21T09:42:11.228 INFO:tasks.rados.rados.0.smithi022.stdout:3157: expect deleted 2024-09-21T09:42:11.229 INFO:tasks.rados.rados.0.smithi022.stdout:3158: rollback oid 25 current snap is 307 2024-09-21T09:42:11.229 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 25 to 305 2024-09-21T09:42:11.229 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 44 current snap is 307 2024-09-21T09:42:11.229 INFO:tasks.rados.rados.0.smithi022.stdout:3159: seq_num 1002 ranges {442368=786432,1228800=614400,1843200=581632,2424832=409600} 2024-09-21T09:42:11.235 INFO:tasks.rados.rados.0.smithi022.stdout:3159: writing smithi02284960-44 from 442368 to 1228800 tid 1 2024-09-21T09:42:11.239 INFO:tasks.rados.rados.0.smithi022.stdout:3159: writing smithi02284960-44 from 1228800 to 1843200 tid 2 2024-09-21T09:42:11.243 INFO:tasks.rados.rados.0.smithi022.stdout:3159: writing smithi02284960-44 from 1843200 to 2424832 tid 3 2024-09-21T09:42:11.245 INFO:tasks.rados.rados.0.smithi022.stdout:3159: writing smithi02284960-44 from 2424832 to 2834432 tid 4 2024-09-21T09:42:11.245 INFO:tasks.rados.rados.0.smithi022.stdout:3160: read oid 31 snap -1 2024-09-21T09:42:11.245 INFO:tasks.rados.rados.0.smithi022.stdout:3160: expect (ObjNum 999 snap 307 seq_num 999) 2024-09-21T09:42:11.246 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 26 current snap is 307 2024-09-21T09:42:11.246 INFO:tasks.rados.rados.0.smithi022.stdout:3161: seq_num 1003 ranges {0=802816,802816=499712,1302528=737280,2039808=368640} 2024-09-21T09:42:11.250 INFO:tasks.rados.rados.0.smithi022.stdout:3161: writing smithi02284960-26 from 0 to 802816 tid 1 2024-09-21T09:42:11.253 INFO:tasks.rados.rados.0.smithi022.stdout:3161: writing smithi02284960-26 from 802816 to 1302528 tid 2 2024-09-21T09:42:11.258 INFO:tasks.rados.rados.0.smithi022.stdout:3161: writing smithi02284960-26 from 1302528 to 2039808 tid 3 2024-09-21T09:42:11.262 INFO:tasks.rados.rados.0.smithi022.stdout:3161: writing smithi02284960-26 from 2039808 to 2408448 tid 4 2024-09-21T09:42:11.262 INFO:tasks.rados.rados.0.smithi022.stdout:3162: copy_from oid 12 from oid 30 current snap is 307 2024-09-21T09:42:11.262 INFO:tasks.rados.rados.0.smithi022.stdout:3155: finishing copy_from to smithi02284960-7 2024-09-21T09:42:11.262 INFO:tasks.rados.rados.0.smithi022.stdout:3155: got expected ENOENT (src dne) 2024-09-21T09:42:11.262 INFO:tasks.rados.rados.0.smithi022.stdout:3155: finishing copy_from racing read to smithi02284960-7 2024-09-21T09:42:11.262 INFO:tasks.rados.rados.0.smithi022.stdout:3153: finishing write tid 3 to smithi02284960-6 2024-09-21T09:42:11.262 INFO:tasks.rados.rados.0.smithi022.stdout:3155: done (9 left) 2024-09-21T09:42:11.262 INFO:tasks.rados.rados.0.smithi022.stdout:3157: done (8 left) 2024-09-21T09:42:11.262 INFO:tasks.rados.rados.0.smithi022.stdout:3163: rmattr oid 19 current snap is 307 2024-09-21T09:42:11.262 INFO:tasks.rados.rados.0.smithi022.stdout:3163: done (8 left) 2024-09-21T09:42:11.262 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 5 current snap is 307 2024-09-21T09:42:11.262 INFO:tasks.rados.rados.0.smithi022.stdout:3153: finishing write tid 4 to smithi02284960-6 2024-09-21T09:42:11.262 INFO:tasks.rados.rados.0.smithi022.stdout:3153: finishing write tid 5 to smithi02284960-6 2024-09-21T09:42:11.263 INFO:tasks.rados.rados.0.smithi022.stdout:3158: finishing rollback tid 1 to smithi02284960-25 2024-09-21T09:42:11.263 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 25 v 0 (ObjNum 33 snap 0 seq_num 3570242336) dirty dne 2024-09-21T09:42:11.263 INFO:tasks.rados.rados.0.smithi022.stdout:3153: finishing write tid 6 to smithi02284960-6 2024-09-21T09:42:11.263 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 6 v 1919 (ObjNum 1001 snap 307 seq_num 1001) dirty exists 2024-09-21T09:42:11.263 INFO:tasks.rados.rados.0.smithi022.stdout:3153: left oid 6 (ObjNum 1001 snap 307 seq_num 1001) 2024-09-21T09:42:11.263 INFO:tasks.rados.rados.0.smithi022.stdout:3159: finishing write tid 1 to smithi02284960-44 2024-09-21T09:42:11.263 INFO:tasks.rados.rados.0.smithi022.stdout:3159: finishing write tid 2 to smithi02284960-44 2024-09-21T09:42:11.263 INFO:tasks.rados.rados.0.smithi022.stdout:3159: finishing write tid 3 to smithi02284960-44 2024-09-21T09:42:11.263 INFO:tasks.rados.rados.0.smithi022.stdout:3159: finishing write tid 4 to smithi02284960-44 2024-09-21T09:42:11.263 INFO:tasks.rados.rados.0.smithi022.stdout:3154: finishing copy_from racing read to smithi02284960-41 2024-09-21T09:42:11.263 INFO:tasks.rados.rados.0.smithi022.stdout:3164: seq_num 1004 ranges {2752512=16384} 2024-09-21T09:42:11.278 INFO:tasks.rados.rados.0.smithi022.stdout:3164: writing smithi02284960-5 from 2752512 to 2768896 tid 1 2024-09-21T09:42:11.278 INFO:tasks.rados.rados.0.smithi022.stdout:3153: done (8 left) 2024-09-21T09:42:11.278 INFO:tasks.rados.rados.0.smithi022.stdout:3158: done (7 left) 2024-09-21T09:42:11.278 INFO:tasks.rados.rados.0.smithi022.stdout:3165: snap_create 2024-09-21T09:42:11.278 INFO:tasks.rados.rados.0.smithi022.stdout:3156: finishing copy_from racing read to smithi02284960-4 2024-09-21T09:42:11.278 INFO:tasks.rados.rados.0.smithi022.stdout:3161: finishing write tid 1 to smithi02284960-26 2024-09-21T09:42:11.278 INFO:tasks.rados.rados.0.smithi022.stdout:3159: finishing write tid 5 to smithi02284960-44 2024-09-21T09:42:11.278 INFO:tasks.rados.rados.0.smithi022.stdout:3154: finishing copy_from to smithi02284960-41 2024-09-21T09:42:11.278 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 41 v 1793 (ObjNum 985 snap 305 seq_num 985) dirty exists 2024-09-21T09:42:11.278 INFO:tasks.rados.rados.0.smithi022.stdout:3159: finishing write tid 6 to smithi02284960-44 2024-09-21T09:42:11.278 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 44 v 1686 (ObjNum 1002 snap 307 seq_num 1002) dirty exists 2024-09-21T09:42:11.278 INFO:tasks.rados.rados.0.smithi022.stdout:3159: left oid 44 (ObjNum 1002 snap 307 seq_num 1002) 2024-09-21T09:42:11.278 INFO:tasks.rados.rados.0.smithi022.stdout:3156: finishing copy_from to smithi02284960-4 2024-09-21T09:42:11.279 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 4 v 1807 (ObjNum 998 snap 306 seq_num 998) dirty exists 2024-09-21T09:42:11.279 INFO:tasks.rados.rados.0.smithi022.stdout:3160: expect (ObjNum 999 snap 307 seq_num 999) 2024-09-21T09:42:11.421 INFO:tasks.rados.rados.0.smithi022.stdout:3161: finishing write tid 2 to smithi02284960-26 2024-09-21T09:42:11.421 INFO:tasks.rados.rados.0.smithi022.stdout:3161: finishing write tid 3 to smithi02284960-26 2024-09-21T09:42:11.421 INFO:tasks.rados.rados.0.smithi022.stdout:3161: finishing write tid 4 to smithi02284960-26 2024-09-21T09:42:11.421 INFO:tasks.rados.rados.0.smithi022.stdout:3161: finishing write tid 5 to smithi02284960-26 2024-09-21T09:42:11.421 INFO:tasks.rados.rados.0.smithi022.stdout:3161: finishing write tid 6 to smithi02284960-26 2024-09-21T09:42:11.421 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 26 v 1708 (ObjNum 1003 snap 307 seq_num 1003) dirty exists 2024-09-21T09:42:11.421 INFO:tasks.rados.rados.0.smithi022.stdout:3161: left oid 26 (ObjNum 1003 snap 307 seq_num 1003) 2024-09-21T09:42:11.421 INFO:tasks.rados.rados.0.smithi022.stdout:3164: finishing write tid 1 to smithi02284960-5 2024-09-21T09:42:11.421 INFO:tasks.rados.rados.0.smithi022.stdout:3164: finishing write tid 2 to smithi02284960-5 2024-09-21T09:42:11.421 INFO:tasks.rados.rados.0.smithi022.stdout:3162: finishing copy_from racing read to smithi02284960-12 2024-09-21T09:42:11.422 INFO:tasks.rados.rados.0.smithi022.stdout:3164: finishing write tid 3 to smithi02284960-5 2024-09-21T09:42:11.422 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 5 v 1809 (ObjNum 1004 snap 307 seq_num 1004) dirty exists 2024-09-21T09:42:11.422 INFO:tasks.rados.rados.0.smithi022.stdout:3164: left oid 5 (ObjNum 1004 snap 307 seq_num 1004) 2024-09-21T09:42:11.422 INFO:tasks.rados.rados.0.smithi022.stdout:3162: finishing copy_from to smithi02284960-12 2024-09-21T09:42:11.422 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 12 v 1334 (ObjNum 997 snap 306 seq_num 997) dirty exists 2024-09-21T09:42:11.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:42:11 smithi064 ceph-mon[86709]: osdmap e692: 8 total, 8 up, 8 in 2024-09-21T09:42:11.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:42:11 smithi022 ceph-mon[96736]: osdmap e692: 8 total, 8 up, 8 in 2024-09-21T09:42:11.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:42:11 smithi022 ceph-mon[98615]: osdmap e692: 8 total, 8 up, 8 in 2024-09-21T09:42:12.117 INFO:tasks.rados.rados.0.smithi022.stdout:3154: done (7 left) 2024-09-21T09:42:12.117 INFO:tasks.rados.rados.0.smithi022.stdout:3156: done (6 left) 2024-09-21T09:42:12.117 INFO:tasks.rados.rados.0.smithi022.stdout:3159: done (5 left) 2024-09-21T09:42:12.118 INFO:tasks.rados.rados.0.smithi022.stdout:3160: done (4 left) 2024-09-21T09:42:12.118 INFO:tasks.rados.rados.0.smithi022.stdout:3161: done (3 left) 2024-09-21T09:42:12.118 INFO:tasks.rados.rados.0.smithi022.stdout:3162: done (2 left) 2024-09-21T09:42:12.118 INFO:tasks.rados.rados.0.smithi022.stdout:3164: done (1 left) 2024-09-21T09:42:12.118 INFO:tasks.rados.rados.0.smithi022.stdout:3165: done (0 left) 2024-09-21T09:42:12.118 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 39 current snap is 308 2024-09-21T09:42:12.118 INFO:tasks.rados.rados.0.smithi022.stdout:3166: seq_num 1005 ranges {868352=32768} 2024-09-21T09:42:12.122 INFO:tasks.rados.rados.0.smithi022.stdout:3166: writing smithi02284960-39 from 868352 to 901120 tid 1 2024-09-21T09:42:12.122 INFO:tasks.rados.rados.0.smithi022.stdout:3167: read oid 17 snap -1 2024-09-21T09:42:12.122 INFO:tasks.rados.rados.0.smithi022.stdout:3167: expect (ObjNum 842 snap 269 seq_num 842) 2024-09-21T09:42:12.122 INFO:tasks.rados.rados.0.smithi022.stdout:3168: snap_create 2024-09-21T09:42:12.131 INFO:tasks.rados.rados.0.smithi022.stdout:3166: finishing write tid 1 to smithi02284960-39 2024-09-21T09:42:12.131 INFO:tasks.rados.rados.0.smithi022.stdout:3166: finishing write tid 2 to smithi02284960-39 2024-09-21T09:42:12.173 INFO:tasks.rados.rados.0.smithi022.stdout:3166: finishing write tid 3 to smithi02284960-39 2024-09-21T09:42:12.173 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 39 v 1336 (ObjNum 1005 snap 308 seq_num 1005) dirty exists 2024-09-21T09:42:12.173 INFO:tasks.rados.rados.0.smithi022.stdout:3166: left oid 39 (ObjNum 1005 snap 308 seq_num 1005) 2024-09-21T09:42:12.183 INFO:tasks.rados.rados.0.smithi022.stdout:3167: expect (ObjNum 842 snap 269 seq_num 842) 2024-09-21T09:42:12.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:42:12 smithi064 ceph-mon[86709]: pgmap v650: 121 pgs: 121 active+clean; 202 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 5.3 MiB/s rd, 1.5 MiB/s wr, 16 op/s 2024-09-21T09:42:12.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:42:12 smithi064 ceph-mon[86709]: osdmap e693: 8 total, 8 up, 8 in 2024-09-21T09:42:12.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:42:12 smithi022 ceph-mon[98615]: pgmap v650: 121 pgs: 121 active+clean; 202 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 5.3 MiB/s rd, 1.5 MiB/s wr, 16 op/s 2024-09-21T09:42:12.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:42:12 smithi022 ceph-mon[98615]: osdmap e693: 8 total, 8 up, 8 in 2024-09-21T09:42:12.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:42:12 smithi022 ceph-mon[96736]: pgmap v650: 121 pgs: 121 active+clean; 202 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 5.3 MiB/s rd, 1.5 MiB/s wr, 16 op/s 2024-09-21T09:42:12.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:42:12 smithi022 ceph-mon[96736]: osdmap e693: 8 total, 8 up, 8 in 2024-09-21T09:42:13.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:42:13 smithi064 ceph-mon[86709]: osdmap e694: 8 total, 8 up, 8 in 2024-09-21T09:42:13.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:42:13 smithi064 ceph-mon[86709]: pgmap v653: 121 pgs: 121 active+clean; 195 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 8.6 MiB/s rd, 3.7 MiB/s wr, 23 op/s 2024-09-21T09:42:13.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:42:13 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:42:13.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:42:13 smithi022 ceph-mon[98615]: osdmap e694: 8 total, 8 up, 8 in 2024-09-21T09:42:13.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:42:13 smithi022 ceph-mon[98615]: pgmap v653: 121 pgs: 121 active+clean; 195 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 8.6 MiB/s rd, 3.7 MiB/s wr, 23 op/s 2024-09-21T09:42:13.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:42:13 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:42:13.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:42:13 smithi022 ceph-mon[96736]: osdmap e694: 8 total, 8 up, 8 in 2024-09-21T09:42:13.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:42:13 smithi022 ceph-mon[96736]: pgmap v653: 121 pgs: 121 active+clean; 195 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 8.6 MiB/s rd, 3.7 MiB/s wr, 23 op/s 2024-09-21T09:42:13.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:42:13 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:42:14.228 INFO:tasks.rados.rados.0.smithi022.stdout:3166: done (2 left) 2024-09-21T09:42:14.228 INFO:tasks.rados.rados.0.smithi022.stdout:3167: done (1 left) 2024-09-21T09:42:14.228 INFO:tasks.rados.rados.0.smithi022.stdout:3168: done (0 left) 2024-09-21T09:42:14.228 INFO:tasks.rados.rados.0.smithi022.stdout:3169: read oid 5 snap -1 2024-09-21T09:42:14.228 INFO:tasks.rados.rados.0.smithi022.stdout:3169: expect (ObjNum 1004 snap 307 seq_num 1004) 2024-09-21T09:42:14.228 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 17 current snap is 309 2024-09-21T09:42:14.228 INFO:tasks.rados.rados.0.smithi022.stdout:3170: seq_num 1006 ranges {19423232=49152} 2024-09-21T09:42:14.334 INFO:tasks.rados.rados.0.smithi022.stdout:3170: writing smithi02284960-17 from 19423232 to 19472384 tid 1 2024-09-21T09:42:14.335 INFO:tasks.rados.rados.0.smithi022.stdout:3171: copy_from oid 41 from oid 43 current snap is 309 2024-09-21T09:42:14.335 INFO:tasks.rados.rados.0.smithi022.stdout:3169: expect (ObjNum 1004 snap 307 seq_num 1004) 2024-09-21T09:42:14.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:42:14 smithi064 ceph-mon[86709]: osdmap e695: 8 total, 8 up, 8 in 2024-09-21T09:42:14.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:42:14 smithi064 ceph-mon[86709]: osdmap e696: 8 total, 8 up, 8 in 2024-09-21T09:42:14.528 INFO:tasks.rados.rados.0.smithi022.stdout:3170: finishing write tid 1 to smithi02284960-17 2024-09-21T09:42:14.528 INFO:tasks.rados.rados.0.smithi022.stdout:3170: finishing write tid 2 to smithi02284960-17 2024-09-21T09:42:14.528 INFO:tasks.rados.rados.0.smithi022.stdout:3170: finishing write tid 3 to smithi02284960-17 2024-09-21T09:42:14.528 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 17 v 1338 (ObjNum 1006 snap 309 seq_num 1006) dirty exists 2024-09-21T09:42:14.528 INFO:tasks.rados.rados.0.smithi022.stdout:3170: left oid 17 (ObjNum 1006 snap 309 seq_num 1006) 2024-09-21T09:42:14.528 INFO:tasks.rados.rados.0.smithi022.stdout:3171: finishing copy_from racing read to smithi02284960-41 2024-09-21T09:42:14.528 INFO:tasks.rados.rados.0.smithi022.stdout:3171: finishing copy_from to smithi02284960-41 2024-09-21T09:42:14.528 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 41 v 1669 (ObjNum 983 snap 304 seq_num 983) dirty exists 2024-09-21T09:42:14.528 INFO:tasks.rados.rados.0.smithi022.stdout:3169: done (2 left) 2024-09-21T09:42:14.529 INFO:tasks.rados.rados.0.smithi022.stdout:3170: done (1 left) 2024-09-21T09:42:14.529 INFO:tasks.rados.rados.0.smithi022.stdout:3171: done (0 left) 2024-09-21T09:42:14.529 INFO:tasks.rados.rados.0.smithi022.stdout:3172: read oid 45 snap -1 2024-09-21T09:42:14.529 INFO:tasks.rados.rados.0.smithi022.stdout:3172: expect (ObjNum 986 snap 305 seq_num 986) 2024-09-21T09:42:14.529 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 18 current snap is 309 2024-09-21T09:42:14.529 INFO:tasks.rados.rados.0.smithi022.stdout:3173: seq_num 1007 ranges {0=65536} 2024-09-21T09:42:14.529 INFO:tasks.rados.rados.0.smithi022.stdout:3173: writing smithi02284960-18 from 0 to 65536 tid 1 2024-09-21T09:42:14.529 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 10 current snap is 309 2024-09-21T09:42:14.529 INFO:tasks.rados.rados.0.smithi022.stdout:3174: seq_num 1008 ranges {0=90112} 2024-09-21T09:42:14.531 INFO:tasks.rados.rados.0.smithi022.stdout:3174: writing smithi02284960-10 from 0 to 90112 tid 1 2024-09-21T09:42:14.531 INFO:tasks.rados.rados.0.smithi022.stdout:3175: rollback oid 8 current snap is 309 2024-09-21T09:42:14.531 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 8 to 305 2024-09-21T09:42:14.531 INFO:tasks.rados.rados.0.smithi022.stdout:3176: read oid 5 snap -1 2024-09-21T09:42:14.531 INFO:tasks.rados.rados.0.smithi022.stdout:3176: expect (ObjNum 1004 snap 307 seq_num 1004) 2024-09-21T09:42:14.531 INFO:tasks.rados.rados.0.smithi022.stdout:3177: read oid 24 snap 308 2024-09-21T09:42:14.531 INFO:tasks.rados.rados.0.smithi022.stdout:3177: expect (ObjNum 995 snap 305 seq_num 995) 2024-09-21T09:42:14.531 INFO:tasks.rados.rados.0.smithi022.stdout:3178: read oid 19 snap 307 2024-09-21T09:42:14.532 INFO:tasks.rados.rados.0.smithi022.stdout:3178: expect deleted 2024-09-21T09:42:14.532 INFO:tasks.rados.rados.0.smithi022.stdout:3179: read oid 38 snap -1 2024-09-21T09:42:14.532 INFO:tasks.rados.rados.0.smithi022.stdout:3179: expect (ObjNum 998 snap 306 seq_num 998) 2024-09-21T09:42:14.532 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 1 current snap is 309 2024-09-21T09:42:14.532 INFO:tasks.rados.rados.0.smithi022.stdout:3180: seq_num 1009 ranges {0=106496} 2024-09-21T09:42:14.532 INFO:tasks.rados.rados.0.smithi022.stdout:3180: writing smithi02284960-1 from 0 to 106496 tid 1 2024-09-21T09:42:14.532 INFO:tasks.rados.rados.0.smithi022.stdout:3181: rollback oid 46 current snap is 309 2024-09-21T09:42:14.532 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 46 to 308 2024-09-21T09:42:14.532 INFO:tasks.rados.rados.0.smithi022.stdout:3182: read oid 2 snap -1 2024-09-21T09:42:14.532 INFO:tasks.rados.rados.0.smithi022.stdout:3182: expect deleted 2024-09-21T09:42:14.532 INFO:tasks.rados.rados.0.smithi022.stdout:3183: setattr oid 47 current snap is 309 2024-09-21T09:42:14.533 INFO:tasks.rados.rados.0.smithi022.stdout:3182: done (11 left) 2024-09-21T09:42:14.533 INFO:tasks.rados.rados.0.smithi022.stdout:3184: rmattr oid 20 current snap is 309 2024-09-21T09:42:14.533 INFO:tasks.rados.rados.0.smithi022.stdout:3184: done (11 left) 2024-09-21T09:42:14.533 INFO:tasks.rados.rados.0.smithi022.stdout:3185: read oid 14 snap 305 2024-09-21T09:42:14.534 INFO:tasks.rados.rados.0.smithi022.stdout:3185: expect (ObjNum 985 snap 305 seq_num 985) 2024-09-21T09:42:14.534 INFO:tasks.rados.rados.0.smithi022.stdout:3186: read oid 31 snap -1 2024-09-21T09:42:14.534 INFO:tasks.rados.rados.0.smithi022.stdout:3186: expect (ObjNum 999 snap 307 seq_num 999) 2024-09-21T09:42:14.534 INFO:tasks.rados.rados.0.smithi022.stdout:3187: snap_remove snap 306 2024-09-21T09:42:14.557 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:42:14 smithi022 ceph-mon[98615]: osdmap e695: 8 total, 8 up, 8 in 2024-09-21T09:42:14.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:42:14 smithi022 ceph-mon[98615]: osdmap e696: 8 total, 8 up, 8 in 2024-09-21T09:42:14.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:42:14 smithi022 ceph-mon[96736]: osdmap e695: 8 total, 8 up, 8 in 2024-09-21T09:42:14.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:42:14 smithi022 ceph-mon[96736]: osdmap e696: 8 total, 8 up, 8 in 2024-09-21T09:42:15.129 INFO:tasks.rados.rados.0.smithi022.stdout:3173: finishing write tid 1 to smithi02284960-18 2024-09-21T09:42:15.129 INFO:tasks.rados.rados.0.smithi022.stdout:3173: finishing write tid 2 to smithi02284960-18 2024-09-21T09:42:15.129 INFO:tasks.rados.rados.0.smithi022.stdout:3173: finishing write tid 3 to smithi02284960-18 2024-09-21T09:42:15.129 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 18 v 1525 (ObjNum 1007 snap 309 seq_num 1007) dirty exists 2024-09-21T09:42:15.129 INFO:tasks.rados.rados.0.smithi022.stdout:3173: left oid 18 (ObjNum 1007 snap 309 seq_num 1007) 2024-09-21T09:42:15.129 INFO:tasks.rados.rados.0.smithi022.stdout:3174: finishing write tid 1 to smithi02284960-10 2024-09-21T09:42:15.129 INFO:tasks.rados.rados.0.smithi022.stdout:3174: finishing write tid 2 to smithi02284960-10 2024-09-21T09:42:15.129 INFO:tasks.rados.rados.0.smithi022.stdout:3175: finishing rollback tid 0 to smithi02284960-8 2024-09-21T09:42:15.129 INFO:tasks.rados.rados.0.smithi022.stdout:3175: finishing rollback tid 1 to smithi02284960-8 2024-09-21T09:42:15.129 INFO:tasks.rados.rados.0.smithi022.stdout:3175: finishing rollback tid 2 to smithi02284960-8 2024-09-21T09:42:15.129 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 8 v 1812 (ObjNum 988 snap 305 seq_num 988) dirty exists 2024-09-21T09:42:15.129 INFO:tasks.rados.rados.0.smithi022.stdout:3174: finishing write tid 3 to smithi02284960-10 2024-09-21T09:42:15.130 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 10 v 1795 (ObjNum 1008 snap 309 seq_num 1008) dirty exists 2024-09-21T09:42:15.130 INFO:tasks.rados.rados.0.smithi022.stdout:3174: left oid 10 (ObjNum 1008 snap 309 seq_num 1008) 2024-09-21T09:42:15.130 INFO:tasks.rados.rados.0.smithi022.stdout:3172: expect (ObjNum 986 snap 305 seq_num 986) 2024-09-21T09:42:15.190 INFO:tasks.rados.rados.0.smithi022.stdout:3180: finishing write tid 1 to smithi02284960-1 2024-09-21T09:42:15.190 INFO:tasks.rados.rados.0.smithi022.stdout:3180: finishing write tid 2 to smithi02284960-1 2024-09-21T09:42:15.190 INFO:tasks.rados.rados.0.smithi022.stdout:3181: finishing rollback tid 0 to smithi02284960-46 2024-09-21T09:42:15.190 INFO:tasks.rados.rados.0.smithi022.stdout:3181: finishing rollback tid 1 to smithi02284960-46 2024-09-21T09:42:15.190 INFO:tasks.rados.rados.0.smithi022.stdout:3180: finishing write tid 3 to smithi02284960-1 2024-09-21T09:42:15.191 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 1 v 1688 (ObjNum 1009 snap 309 seq_num 1009) dirty exists 2024-09-21T09:42:15.191 INFO:tasks.rados.rados.0.smithi022.stdout:3180: left oid 1 (ObjNum 1009 snap 309 seq_num 1009) 2024-09-21T09:42:15.191 INFO:tasks.rados.rados.0.smithi022.stdout:3181: finishing rollback tid 2 to smithi02284960-46 2024-09-21T09:42:15.191 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 46 v 1691 (ObjNum 936 snap 291 seq_num 936) dirty exists 2024-09-21T09:42:15.191 INFO:tasks.rados.rados.0.smithi022.stdout:3176: expect (ObjNum 1004 snap 307 seq_num 1004) 2024-09-21T09:42:15.383 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 47 v 1692 (ObjNum 884 snap 282 seq_num 884) dirty exists 2024-09-21T09:42:15.383 INFO:tasks.rados.rados.0.smithi022.stdout:3185: expect (ObjNum 985 snap 305 seq_num 985) 2024-09-21T09:42:15.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:42:15 smithi064 ceph-mon[86709]: pgmap v656: 121 pgs: 121 active+clean; 204 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 7.2 MiB/s rd, 5.4 MiB/s wr, 26 op/s 2024-09-21T09:42:15.557 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:42:15 smithi022 ceph-mon[98615]: pgmap v656: 121 pgs: 121 active+clean; 204 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 7.2 MiB/s rd, 5.4 MiB/s wr, 26 op/s 2024-09-21T09:42:15.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:42:15 smithi022 ceph-mon[96736]: pgmap v656: 121 pgs: 121 active+clean; 204 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 7.2 MiB/s rd, 5.4 MiB/s wr, 26 op/s 2024-09-21T09:42:15.625 INFO:tasks.rados.rados.0.smithi022.stdout:3179: expect (ObjNum 998 snap 306 seq_num 998) 2024-09-21T09:42:16.058 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:42:15 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:42:15] "GET /metrics HTTP/1.1" 200 36357 "" "Prometheus/2.51.0" 2024-09-21T09:42:16.184 INFO:tasks.rados.rados.0.smithi022.stdout:3177: expect (ObjNum 995 snap 305 seq_num 995) 2024-09-21T09:42:16.326 INFO:tasks.rados.rados.0.smithi022.stdout:3186: expect (ObjNum 999 snap 307 seq_num 999) 2024-09-21T09:42:16.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:42:16 smithi064 ceph-mon[86709]: osdmap e697: 8 total, 8 up, 8 in 2024-09-21T09:42:16.470 INFO:tasks.rados.rados.0.smithi022.stdout:3172: done (13 left) 2024-09-21T09:42:16.470 INFO:tasks.rados.rados.0.smithi022.stdout:3173: done (12 left) 2024-09-21T09:42:16.470 INFO:tasks.rados.rados.0.smithi022.stdout:3174: done (11 left) 2024-09-21T09:42:16.470 INFO:tasks.rados.rados.0.smithi022.stdout:3175: done (10 left) 2024-09-21T09:42:16.470 INFO:tasks.rados.rados.0.smithi022.stdout:3176: done (9 left) 2024-09-21T09:42:16.470 INFO:tasks.rados.rados.0.smithi022.stdout:3177: done (8 left) 2024-09-21T09:42:16.470 INFO:tasks.rados.rados.0.smithi022.stdout:3178: done (7 left) 2024-09-21T09:42:16.470 INFO:tasks.rados.rados.0.smithi022.stdout:3179: done (6 left) 2024-09-21T09:42:16.470 INFO:tasks.rados.rados.0.smithi022.stdout:3180: done (5 left) 2024-09-21T09:42:16.470 INFO:tasks.rados.rados.0.smithi022.stdout:3181: done (4 left) 2024-09-21T09:42:16.470 INFO:tasks.rados.rados.0.smithi022.stdout:3183: done (3 left) 2024-09-21T09:42:16.470 INFO:tasks.rados.rados.0.smithi022.stdout:3185: done (2 left) 2024-09-21T09:42:16.470 INFO:tasks.rados.rados.0.smithi022.stdout:3186: done (1 left) 2024-09-21T09:42:16.470 INFO:tasks.rados.rados.0.smithi022.stdout:3187: done (0 left) 2024-09-21T09:42:16.471 INFO:tasks.rados.rados.0.smithi022.stdout:3188: copy_from oid 40 from oid 12 current snap is 309 2024-09-21T09:42:16.471 INFO:tasks.rados.rados.0.smithi022.stdout:3189: snap_create 2024-09-21T09:42:16.482 INFO:tasks.rados.rados.0.smithi022.stdout:3188: finishing copy_from racing read to smithi02284960-40 2024-09-21T09:42:16.482 INFO:tasks.rados.rados.0.smithi022.stdout:3188: finishing copy_from to smithi02284960-40 2024-09-21T09:42:16.482 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 40 v 1334 (ObjNum 997 snap 306 seq_num 997) dirty exists 2024-09-21T09:42:16.557 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:42:16 smithi022 ceph-mon[98615]: osdmap e697: 8 total, 8 up, 8 in 2024-09-21T09:42:16.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:42:16 smithi022 ceph-mon[96736]: osdmap e697: 8 total, 8 up, 8 in 2024-09-21T09:42:17.135 INFO:tasks.rados.rados.0.smithi022.stdout:3188: done (1 left) 2024-09-21T09:42:17.135 INFO:tasks.rados.rados.0.smithi022.stdout:3189: done (0 left) 2024-09-21T09:42:17.135 INFO:tasks.rados.rados.0.smithi022.stdout:3190: snap_create 2024-09-21T09:42:17.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:42:17 smithi064 ceph-mon[86709]: pgmap v658: 121 pgs: 121 active+clean; 230 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 26 MiB/s rd, 8.5 MiB/s wr, 34 op/s 2024-09-21T09:42:17.557 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:42:17 smithi022 ceph-mon[96736]: pgmap v658: 121 pgs: 121 active+clean; 230 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 26 MiB/s rd, 8.5 MiB/s wr, 34 op/s 2024-09-21T09:42:17.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:42:17 smithi022 ceph-mon[98615]: pgmap v658: 121 pgs: 121 active+clean; 230 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 26 MiB/s rd, 8.5 MiB/s wr, 34 op/s 2024-09-21T09:42:18.137 INFO:tasks.rados.rados.0.smithi022.stdout:3190: done (0 left) 2024-09-21T09:42:18.137 INFO:tasks.rados.rados.0.smithi022.stdout:3191: snap_create 2024-09-21T09:42:18.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:42:18 smithi064 ceph-mon[86709]: osdmap e698: 8 total, 8 up, 8 in 2024-09-21T09:42:18.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:42:18 smithi022 ceph-mon[98615]: osdmap e698: 8 total, 8 up, 8 in 2024-09-21T09:42:18.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:42:18 smithi022 ceph-mon[96736]: osdmap e698: 8 total, 8 up, 8 in 2024-09-21T09:42:19.061 INFO:tasks.rados.rados.0.smithi022.stdout:3191: done (0 left) 2024-09-21T09:42:19.062 INFO:tasks.rados.rados.0.smithi022.stdout:3192: rmattr oid 31 current snap is 312 2024-09-21T09:42:19.062 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 37 current snap is 312 2024-09-21T09:42:19.062 INFO:tasks.rados.rados.0.smithi022.stdout:3193: seq_num 1010 ranges {0=122880} 2024-09-21T09:42:19.062 INFO:tasks.rados.rados.0.smithi022.stdout:3193: writing smithi02284960-37 from 0 to 122880 tid 1 2024-09-21T09:42:19.063 INFO:tasks.rados.rados.0.smithi022.stdout:3194: copy_from oid 2 from oid 39 current snap is 312 2024-09-21T09:42:19.063 INFO:tasks.rados.rados.0.smithi022.stdout:3195: rollback oid 44 current snap is 312 2024-09-21T09:42:19.063 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 44 to 309 2024-09-21T09:42:19.063 INFO:tasks.rados.rados.0.smithi022.stdout:3196: read oid 4 snap -1 2024-09-21T09:42:19.063 INFO:tasks.rados.rados.0.smithi022.stdout:3196: expect (ObjNum 998 snap 306 seq_num 998) 2024-09-21T09:42:19.063 INFO:tasks.rados.rados.0.smithi022.stdout:3197: delete oid 19 current snap is 312 2024-09-21T09:42:19.068 INFO:tasks.rados.rados.0.smithi022.stdout:3195: finishing rollback tid 0 to smithi02284960-44 2024-09-21T09:42:19.068 INFO:tasks.rados.rados.0.smithi022.stdout:3195: finishing rollback tid 1 to smithi02284960-44 2024-09-21T09:42:19.069 INFO:tasks.rados.rados.0.smithi022.stdout:3195: finishing rollback tid 2 to smithi02284960-44 2024-09-21T09:42:19.069 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 44 v 1695 (ObjNum 1002 snap 307 seq_num 1002) dirty exists 2024-09-21T09:42:19.072 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 31 v 1878 (ObjNum 999 snap 307 seq_num 999) dirty exists 2024-09-21T09:42:19.072 INFO:tasks.rados.rados.0.smithi022.stdout:3193: finishing write tid 1 to smithi02284960-37 2024-09-21T09:42:19.072 INFO:tasks.rados.rados.0.smithi022.stdout:3193: finishing write tid 2 to smithi02284960-37 2024-09-21T09:42:19.074 INFO:tasks.rados.rados.0.smithi022.stdout:3192: done (5 left) 2024-09-21T09:42:19.074 INFO:tasks.rados.rados.0.smithi022.stdout:3195: done (4 left) 2024-09-21T09:42:19.074 INFO:tasks.rados.rados.0.smithi022.stdout:3197: done (3 left) 2024-09-21T09:42:19.074 INFO:tasks.rados.rados.0.smithi022.stdout:3198: read oid 23 snap 309 2024-09-21T09:42:19.074 INFO:tasks.rados.rados.0.smithi022.stdout:3198: expect deleted 2024-09-21T09:42:19.075 INFO:tasks.rados.rados.0.smithi022.stdout:3199: copy_from oid 47 from oid 3 current snap is 312 2024-09-21T09:42:19.075 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 30 current snap is 312 2024-09-21T09:42:19.075 INFO:tasks.rados.rados.0.smithi022.stdout:3200: seq_num 1012 ranges {2310144=155648} 2024-09-21T09:42:19.088 INFO:tasks.rados.rados.0.smithi022.stdout:3200: writing smithi02284960-30 from 2310144 to 2465792 tid 1 2024-09-21T09:42:19.089 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 48 current snap is 312 2024-09-21T09:42:19.089 INFO:tasks.rados.rados.0.smithi022.stdout:3201: seq_num 1013 ranges {0=172032} 2024-09-21T09:42:19.090 INFO:tasks.rados.rados.0.smithi022.stdout:3201: writing smithi02284960-48 from 0 to 172032 tid 1 2024-09-21T09:42:19.090 INFO:tasks.rados.rados.0.smithi022.stdout:3202: copy_from oid 25 from oid 36 current snap is 312 2024-09-21T09:42:19.090 INFO:tasks.rados.rados.0.smithi022.stdout:3193: finishing write tid 3 to smithi02284960-37 2024-09-21T09:42:19.090 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 37 v 1880 (ObjNum 1010 snap 312 seq_num 1010) dirty exists 2024-09-21T09:42:19.090 INFO:tasks.rados.rados.0.smithi022.stdout:3193: left oid 37 (ObjNum 1010 snap 312 seq_num 1010) 2024-09-21T09:42:19.091 INFO:tasks.rados.rados.0.smithi022.stdout:3194: finishing copy_from racing read to smithi02284960-2 2024-09-21T09:42:19.091 INFO:tasks.rados.rados.0.smithi022.stdout:3194: finishing copy_from to smithi02284960-2 2024-09-21T09:42:19.091 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 2 v 1336 (ObjNum 1005 snap 308 seq_num 1005) dirty exists 2024-09-21T09:42:19.091 INFO:tasks.rados.rados.0.smithi022.stdout:3199: finishing copy_from racing read to smithi02284960-47 2024-09-21T09:42:19.091 INFO:tasks.rados.rados.0.smithi022.stdout:3193: done (7 left) 2024-09-21T09:42:19.091 INFO:tasks.rados.rados.0.smithi022.stdout:3194: done (6 left) 2024-09-21T09:42:19.091 INFO:tasks.rados.rados.0.smithi022.stdout:3198: done (5 left) 2024-09-21T09:42:19.091 INFO:tasks.rados.rados.0.smithi022.stdout:3203: snap_create 2024-09-21T09:42:19.092 INFO:tasks.rados.rados.0.smithi022.stdout:3199: finishing copy_from to smithi02284960-47 2024-09-21T09:42:19.092 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 47 v 623 (ObjNum 910 snap 284 seq_num 910) dirty exists 2024-09-21T09:42:19.093 INFO:tasks.rados.rados.0.smithi022.stdout:3200: finishing write tid 1 to smithi02284960-30 2024-09-21T09:42:19.093 INFO:tasks.rados.rados.0.smithi022.stdout:3200: finishing write tid 2 to smithi02284960-30 2024-09-21T09:42:19.096 INFO:tasks.rados.rados.0.smithi022.stdout:3200: finishing write tid 3 to smithi02284960-30 2024-09-21T09:42:19.096 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 30 v 1340 (ObjNum 1012 snap 312 seq_num 1012) dirty exists 2024-09-21T09:42:19.096 INFO:tasks.rados.rados.0.smithi022.stdout:3200: left oid 30 (ObjNum 1012 snap 312 seq_num 1012) 2024-09-21T09:42:19.099 INFO:tasks.rados.rados.0.smithi022.stdout:3201: finishing write tid 1 to smithi02284960-48 2024-09-21T09:42:19.099 INFO:tasks.rados.rados.0.smithi022.stdout:3201: finishing write tid 2 to smithi02284960-48 2024-09-21T09:42:19.100 INFO:tasks.rados.rados.0.smithi022.stdout:3201: finishing write tid 3 to smithi02284960-48 2024-09-21T09:42:19.100 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 48 v 1923 (ObjNum 1013 snap 312 seq_num 1013) dirty exists 2024-09-21T09:42:19.100 INFO:tasks.rados.rados.0.smithi022.stdout:3201: left oid 48 (ObjNum 1013 snap 312 seq_num 1013) 2024-09-21T09:42:19.101 INFO:tasks.rados.rados.0.smithi022.stdout:3202: finishing copy_from racing read to smithi02284960-25 2024-09-21T09:42:19.102 INFO:tasks.rados.rados.0.smithi022.stdout:3202: finishing copy_from to smithi02284960-25 2024-09-21T09:42:19.102 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 25 v 1416 (ObjNum 919 snap 286 seq_num 919) dirty exists 2024-09-21T09:42:19.107 INFO:tasks.rados.rados.0.smithi022.stdout:3196: expect (ObjNum 998 snap 306 seq_num 998) 2024-09-21T09:42:19.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:42:19 smithi064 ceph-mon[86709]: osdmap e699: 8 total, 8 up, 8 in 2024-09-21T09:42:19.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:42:19 smithi064 ceph-mon[86709]: pgmap v661: 121 pgs: 121 active+clean; 230 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 36 MiB/s rd, 5.9 MiB/s wr, 23 op/s 2024-09-21T09:42:19.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:42:19 smithi064 ceph-mon[86709]: osdmap e700: 8 total, 8 up, 8 in 2024-09-21T09:42:19.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:42:19 smithi022 ceph-mon[98615]: osdmap e699: 8 total, 8 up, 8 in 2024-09-21T09:42:19.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:42:19 smithi022 ceph-mon[98615]: pgmap v661: 121 pgs: 121 active+clean; 230 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 36 MiB/s rd, 5.9 MiB/s wr, 23 op/s 2024-09-21T09:42:19.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:42:19 smithi022 ceph-mon[98615]: osdmap e700: 8 total, 8 up, 8 in 2024-09-21T09:42:19.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:42:19 smithi022 ceph-mon[96736]: osdmap e699: 8 total, 8 up, 8 in 2024-09-21T09:42:19.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:42:19 smithi022 ceph-mon[96736]: pgmap v661: 121 pgs: 121 active+clean; 230 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 36 MiB/s rd, 5.9 MiB/s wr, 23 op/s 2024-09-21T09:42:19.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:42:19 smithi022 ceph-mon[96736]: osdmap e700: 8 total, 8 up, 8 in 2024-09-21T09:42:20.062 INFO:tasks.rados.rados.0.smithi022.stdout:3196: done (5 left) 2024-09-21T09:42:20.062 INFO:tasks.rados.rados.0.smithi022.stdout:3199: done (4 left) 2024-09-21T09:42:20.062 INFO:tasks.rados.rados.0.smithi022.stdout:3200: done (3 left) 2024-09-21T09:42:20.062 INFO:tasks.rados.rados.0.smithi022.stdout:3201: done (2 left) 2024-09-21T09:42:20.063 INFO:tasks.rados.rados.0.smithi022.stdout:3202: done (1 left) 2024-09-21T09:42:20.063 INFO:tasks.rados.rados.0.smithi022.stdout:3203: done (0 left) 2024-09-21T09:42:20.063 INFO:tasks.rados.rados.0.smithi022.stdout:3204: read oid 38 snap 311 2024-09-21T09:42:20.063 INFO:tasks.rados.rados.0.smithi022.stdout:3204: expect (ObjNum 998 snap 306 seq_num 998) 2024-09-21T09:42:20.063 INFO:tasks.rados.rados.0.smithi022.stdout:3205: read oid 37 snap 311 2024-09-21T09:42:20.063 INFO:tasks.rados.rados.0.smithi022.stdout:3205: expect deleted 2024-09-21T09:42:20.063 INFO:tasks.rados.rados.0.smithi022.stdout:3206: snap_create 2024-09-21T09:42:20.093 INFO:tasks.rados.rados.0.smithi022.stdout:3204: expect (ObjNum 998 snap 306 seq_num 998) 2024-09-21T09:42:20.692 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:42:20 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:42:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:42:21.067 INFO:tasks.rados.rados.0.smithi022.stdout:3204: done (2 left) 2024-09-21T09:42:21.067 INFO:tasks.rados.rados.0.smithi022.stdout:3205: done (1 left) 2024-09-21T09:42:21.067 INFO:tasks.rados.rados.0.smithi022.stdout:3206: done (0 left) 2024-09-21T09:42:21.067 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 45 current snap is 314 2024-09-21T09:42:21.067 INFO:tasks.rados.rados.0.smithi022.stdout:3207: seq_num 1014 ranges {2121728=188416} 2024-09-21T09:42:21.079 INFO:tasks.rados.rados.0.smithi022.stdout:3207: writing smithi02284960-45 from 2121728 to 2310144 tid 1 2024-09-21T09:42:21.079 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 1 current snap is 314 2024-09-21T09:42:21.080 INFO:tasks.rados.rados.0.smithi022.stdout:3208: seq_num 1015 ranges {106496=204800} 2024-09-21T09:42:21.081 INFO:tasks.rados.rados.0.smithi022.stdout:3208: writing smithi02284960-1 from 106496 to 311296 tid 1 2024-09-21T09:42:21.081 INFO:tasks.rados.rados.0.smithi022.stdout:3209: snap_remove snap 304 2024-09-21T09:42:21.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:42:21 smithi064 ceph-mon[86709]: osdmap e701: 8 total, 8 up, 8 in 2024-09-21T09:42:21.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:42:21 smithi022 ceph-mon[98615]: osdmap e701: 8 total, 8 up, 8 in 2024-09-21T09:42:21.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:42:21 smithi022 ceph-mon[96736]: osdmap e701: 8 total, 8 up, 8 in 2024-09-21T09:42:22.075 INFO:tasks.rados.rados.0.smithi022.stdout:3209: done (2 left) 2024-09-21T09:42:22.075 INFO:tasks.rados.rados.0.smithi022.stdout:3210: read oid 9 snap 311 2024-09-21T09:42:22.075 INFO:tasks.rados.rados.0.smithi022.stdout:3210: expect (ObjNum 979 snap 300 seq_num 979) 2024-09-21T09:42:22.075 INFO:tasks.rados.rados.0.smithi022.stdout:3211: snap_remove snap 309 2024-09-21T09:42:22.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:42:22 smithi064 ceph-mon[86709]: pgmap v664: 121 pgs: 121 active+clean; 234 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 1.0 MiB/s wr, 6 op/s 2024-09-21T09:42:22.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:42:22 smithi064 ceph-mon[86709]: osdmap e702: 8 total, 8 up, 8 in 2024-09-21T09:42:22.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:42:22 smithi022 ceph-mon[98615]: pgmap v664: 121 pgs: 121 active+clean; 234 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 1.0 MiB/s wr, 6 op/s 2024-09-21T09:42:22.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:42:22 smithi022 ceph-mon[98615]: osdmap e702: 8 total, 8 up, 8 in 2024-09-21T09:42:22.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:42:22 smithi022 ceph-mon[96736]: pgmap v664: 121 pgs: 121 active+clean; 234 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 1.0 MiB/s wr, 6 op/s 2024-09-21T09:42:22.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:42:22 smithi022 ceph-mon[96736]: osdmap e702: 8 total, 8 up, 8 in 2024-09-21T09:42:23.080 INFO:tasks.rados.rados.0.smithi022.stdout:3211: done (3 left) 2024-09-21T09:42:23.081 INFO:tasks.rados.rados.0.smithi022.stdout:3212: read oid 30 snap -1 2024-09-21T09:42:23.081 INFO:tasks.rados.rados.0.smithi022.stdout:3212: expect (ObjNum 1012 snap 312 seq_num 1012) 2024-09-21T09:42:23.081 INFO:tasks.rados.rados.0.smithi022.stdout:3213: read oid 12 snap -1 2024-09-21T09:42:23.081 INFO:tasks.rados.rados.0.smithi022.stdout:3213: expect (ObjNum 997 snap 306 seq_num 997) 2024-09-21T09:42:23.081 INFO:tasks.rados.rados.0.smithi022.stdout:3214: setattr oid 2 current snap is 314 2024-09-21T09:42:23.081 INFO:tasks.rados.rados.0.smithi022.stdout:3207: finishing write tid 1 to smithi02284960-45 2024-09-21T09:42:23.081 INFO:tasks.rados.rados.0.smithi022.stdout:3207: finishing write tid 2 to smithi02284960-45 2024-09-21T09:42:23.081 INFO:tasks.rados.rados.0.smithi022.stdout:3207: finishing write tid 3 to smithi02284960-45 2024-09-21T09:42:23.081 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 45 v 1424 (ObjNum 1014 snap 314 seq_num 1014) dirty exists 2024-09-21T09:42:23.082 INFO:tasks.rados.rados.0.smithi022.stdout:3207: left oid 45 (ObjNum 1014 snap 314 seq_num 1014) 2024-09-21T09:42:23.082 INFO:tasks.rados.rados.0.smithi022.stdout:3208: finishing write tid 1 to smithi02284960-1 2024-09-21T09:42:23.082 INFO:tasks.rados.rados.0.smithi022.stdout:3208: finishing write tid 2 to smithi02284960-1 2024-09-21T09:42:23.082 INFO:tasks.rados.rados.0.smithi022.stdout:3208: finishing write tid 3 to smithi02284960-1 2024-09-21T09:42:23.082 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 1 v 1697 (ObjNum 1015 snap 314 seq_num 1015) dirty exists 2024-09-21T09:42:23.082 INFO:tasks.rados.rados.0.smithi022.stdout:3208: left oid 1 (ObjNum 1015 snap 314 seq_num 1015) 2024-09-21T09:42:23.082 INFO:tasks.rados.rados.0.smithi022.stdout:3210: expect (ObjNum 979 snap 300 seq_num 979) 2024-09-21T09:42:23.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:42:23 smithi064 ceph-mon[86709]: osdmap e703: 8 total, 8 up, 8 in 2024-09-21T09:42:23.557 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:42:23 smithi022 ceph-mon[98615]: osdmap e703: 8 total, 8 up, 8 in 2024-09-21T09:42:23.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:42:23 smithi022 ceph-mon[96736]: osdmap e703: 8 total, 8 up, 8 in 2024-09-21T09:42:24.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:42:24 smithi064 ceph-mon[86709]: pgmap v667: 121 pgs: 121 active+clean; 234 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 7.0 MiB/s rd, 1.1 MiB/s wr, 6 op/s 2024-09-21T09:42:24.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:42:24 smithi064 ceph-mon[86709]: osdmap e704: 8 total, 8 up, 8 in 2024-09-21T09:42:24.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:42:24 smithi064 ceph-mon[86709]: osdmap e705: 8 total, 8 up, 8 in 2024-09-21T09:42:24.459 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:42:24 smithi022 ceph-mon[98615]: pgmap v667: 121 pgs: 121 active+clean; 234 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 7.0 MiB/s rd, 1.1 MiB/s wr, 6 op/s 2024-09-21T09:42:24.459 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:42:24 smithi022 ceph-mon[98615]: osdmap e704: 8 total, 8 up, 8 in 2024-09-21T09:42:24.459 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:42:24 smithi022 ceph-mon[98615]: osdmap e705: 8 total, 8 up, 8 in 2024-09-21T09:42:24.459 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:42:24 smithi022 ceph-mon[96736]: pgmap v667: 121 pgs: 121 active+clean; 234 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 7.0 MiB/s rd, 1.1 MiB/s wr, 6 op/s 2024-09-21T09:42:24.459 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:42:24 smithi022 ceph-mon[96736]: osdmap e704: 8 total, 8 up, 8 in 2024-09-21T09:42:24.459 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:42:24 smithi022 ceph-mon[96736]: osdmap e705: 8 total, 8 up, 8 in 2024-09-21T09:42:25.524 INFO:tasks.rados.rados.0.smithi022.stdout:3212: expect (ObjNum 1012 snap 312 seq_num 1012) 2024-09-21T09:42:25.626 INFO:tasks.rados.rados.0.smithi022.stdout:3213: expect (ObjNum 997 snap 306 seq_num 997) 2024-09-21T09:42:25.687 INFO:tasks.rados.rados.0.smithi022.stdout:3207: done (5 left) 2024-09-21T09:42:25.687 INFO:tasks.rados.rados.0.smithi022.stdout:3208: done (4 left) 2024-09-21T09:42:25.687 INFO:tasks.rados.rados.0.smithi022.stdout:3210: done (3 left) 2024-09-21T09:42:25.687 INFO:tasks.rados.rados.0.smithi022.stdout:3212: done (2 left) 2024-09-21T09:42:25.687 INFO:tasks.rados.rados.0.smithi022.stdout:3213: done (1 left) 2024-09-21T09:42:25.687 INFO:tasks.rados.rados.0.smithi022.stdout:3215: copy_from oid 27 from oid 10 current snap is 314 2024-09-21T09:42:25.687 INFO:tasks.rados.rados.0.smithi022.stdout:3216: delete oid 9 current snap is 314 2024-09-21T09:42:25.690 INFO:tasks.rados.rados.0.smithi022.stdout:3216: done (2 left) 2024-09-21T09:42:25.690 INFO:tasks.rados.rados.0.smithi022.stdout:3217: snap_create 2024-09-21T09:42:25.690 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 2 v 1812 (ObjNum 1005 snap 308 seq_num 1005) dirty exists 2024-09-21T09:42:25.692 INFO:tasks.rados.rados.0.smithi022.stdout:3215: finishing copy_from racing read to smithi02284960-27 2024-09-21T09:42:25.693 INFO:tasks.rados.rados.0.smithi022.stdout:3215: finishing copy_from to smithi02284960-27 2024-09-21T09:42:25.693 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 27 v 1795 (ObjNum 1008 snap 309 seq_num 1008) dirty exists 2024-09-21T09:42:25.735 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:42:25 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:42:25] "GET /metrics HTTP/1.1" 200 36357 "" "Prometheus/2.51.0" 2024-09-21T09:42:26.085 INFO:tasks.rados.rados.0.smithi022.stdout:3214: done (2 left) 2024-09-21T09:42:26.085 INFO:tasks.rados.rados.0.smithi022.stdout:3215: done (1 left) 2024-09-21T09:42:26.086 INFO:tasks.rados.rados.0.smithi022.stdout:3217: done (0 left) 2024-09-21T09:42:26.086 INFO:tasks.rados.rados.0.smithi022.stdout:3218: copy_from oid 14 from oid 19 current snap is 315 2024-09-21T09:42:26.086 INFO:tasks.rados.rados.0.smithi022.stdout:3219: rmattr oid 49 current snap is 315 2024-09-21T09:42:26.086 INFO:tasks.rados.rados.0.smithi022.stdout:3219: done (1 left) 2024-09-21T09:42:26.086 INFO:tasks.rados.rados.0.smithi022.stdout:3220: delete oid 29 current snap is 315 2024-09-21T09:42:26.090 INFO:tasks.rados.rados.0.smithi022.stdout:3220: done (1 left) 2024-09-21T09:42:26.090 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 39 current snap is 315 2024-09-21T09:42:26.090 INFO:tasks.rados.rados.0.smithi022.stdout:3221: seq_num 1018 ranges {901120=253952} 2024-09-21T09:42:26.096 INFO:tasks.rados.rados.0.smithi022.stdout:3221: writing smithi02284960-39 from 901120 to 1155072 tid 1 2024-09-21T09:42:26.096 INFO:tasks.rados.rados.0.smithi022.stdout:3222: delete oid 5 current snap is 315 2024-09-21T09:42:26.096 INFO:tasks.rados.rados.0.smithi022.stdout:3218: finishing copy_from to smithi02284960-14 2024-09-21T09:42:26.096 INFO:tasks.rados.rados.0.smithi022.stdout:3218: got expected ENOENT (src dne) 2024-09-21T09:42:26.096 INFO:tasks.rados.rados.0.smithi022.stdout:3218: finishing copy_from racing read to smithi02284960-14 2024-09-21T09:42:26.099 INFO:tasks.rados.rados.0.smithi022.stdout:3221: finishing write tid 1 to smithi02284960-39 2024-09-21T09:42:26.099 INFO:tasks.rados.rados.0.smithi022.stdout:3221: finishing write tid 2 to smithi02284960-39 2024-09-21T09:42:26.101 INFO:tasks.rados.rados.0.smithi022.stdout:3221: finishing write tid 3 to smithi02284960-39 2024-09-21T09:42:26.101 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 39 v 1342 (ObjNum 1018 snap 315 seq_num 1018) dirty exists 2024-09-21T09:42:26.101 INFO:tasks.rados.rados.0.smithi022.stdout:3221: left oid 39 (ObjNum 1018 snap 315 seq_num 1018) 2024-09-21T09:42:26.102 INFO:tasks.rados.rados.0.smithi022.stdout:3218: done (2 left) 2024-09-21T09:42:26.102 INFO:tasks.rados.rados.0.smithi022.stdout:3221: done (1 left) 2024-09-21T09:42:26.102 INFO:tasks.rados.rados.0.smithi022.stdout:3222: done (0 left) 2024-09-21T09:42:26.102 INFO:tasks.rados.rados.0.smithi022.stdout:3223: rmattr oid 30 current snap is 315 2024-09-21T09:42:26.102 INFO:tasks.rados.rados.0.smithi022.stdout:3223: done (0 left) 2024-09-21T09:42:26.103 INFO:tasks.rados.rados.0.smithi022.stdout:3224: read oid 26 snap 305 2024-09-21T09:42:26.103 INFO:tasks.rados.rados.0.smithi022.stdout:3224: expect deleted 2024-09-21T09:42:26.103 INFO:tasks.rados.rados.0.smithi022.stdout:3225: read oid 15 snap -1 2024-09-21T09:42:26.103 INFO:tasks.rados.rados.0.smithi022.stdout:3225: expect (ObjNum 996 snap 305 seq_num 996) 2024-09-21T09:42:26.103 INFO:tasks.rados.rados.0.smithi022.stdout:3226: rollback oid 43 current snap is 315 2024-09-21T09:42:26.103 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 43 to 307 2024-09-21T09:42:26.103 INFO:tasks.rados.rados.0.smithi022.stdout:3227: copy_from oid 38 from oid 11 current snap is 315 2024-09-21T09:42:26.103 INFO:tasks.rados.rados.0.smithi022.stdout:3228: read oid 47 snap 312 2024-09-21T09:42:26.103 INFO:tasks.rados.rados.0.smithi022.stdout:3228: expect (ObjNum 910 snap 284 seq_num 910) 2024-09-21T09:42:26.103 INFO:tasks.rados.rados.0.smithi022.stdout:3229: read oid 35 snap -1 2024-09-21T09:42:26.103 INFO:tasks.rados.rados.0.smithi022.stdout:3229: expect (ObjNum 925 snap 287 seq_num 925) 2024-09-21T09:42:26.103 INFO:tasks.rados.rados.0.smithi022.stdout:3230: snap_remove snap 310 2024-09-21T09:42:26.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:42:26 smithi064 ceph-mon[86709]: pgmap v670: 121 pgs: 121 active+clean; 235 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 7.0 MiB/s rd, 334 KiB/s wr, 4 op/s 2024-09-21T09:42:26.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:42:26 smithi022 ceph-mon[98615]: pgmap v670: 121 pgs: 121 active+clean; 235 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 7.0 MiB/s rd, 334 KiB/s wr, 4 op/s 2024-09-21T09:42:26.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:42:26 smithi022 ceph-mon[96736]: pgmap v670: 121 pgs: 121 active+clean; 235 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 7.0 MiB/s rd, 334 KiB/s wr, 4 op/s 2024-09-21T09:42:27.091 INFO:tasks.rados.rados.0.smithi022.stdout:3230: done (6 left) 2024-09-21T09:42:27.091 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 29 current snap is 315 2024-09-21T09:42:27.091 INFO:tasks.rados.rados.0.smithi022.stdout:3231: seq_num 1020 ranges {0=286720} 2024-09-21T09:42:27.093 INFO:tasks.rados.rados.0.smithi022.stdout:3231: writing smithi02284960-29 from 0 to 286720 tid 1 2024-09-21T09:42:27.093 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 23 current snap is 315 2024-09-21T09:42:27.093 INFO:tasks.rados.rados.0.smithi022.stdout:3228: expect (ObjNum 910 snap 284 seq_num 910) 2024-09-21T09:42:27.118 INFO:tasks.rados.rados.0.smithi022.stdout:3226: finishing rollback tid 0 to smithi02284960-43 2024-09-21T09:42:27.118 INFO:tasks.rados.rados.0.smithi022.stdout:3226: finishing rollback tid 1 to smithi02284960-43 2024-09-21T09:42:27.118 INFO:tasks.rados.rados.0.smithi022.stdout:3226: finishing rollback tid 2 to smithi02284960-43 2024-09-21T09:42:27.118 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 43 v 1798 (ObjNum 983 snap 304 seq_num 983) dirty exists 2024-09-21T09:42:27.118 INFO:tasks.rados.rados.0.smithi022.stdout:3227: finishing copy_from racing read to smithi02284960-38 2024-09-21T09:42:27.118 INFO:tasks.rados.rados.0.smithi022.stdout:3229: expect (ObjNum 925 snap 287 seq_num 925) 2024-09-21T09:42:27.251 INFO:tasks.rados.rados.0.smithi022.stdout:3227: finishing copy_from to smithi02284960-38 2024-09-21T09:42:27.251 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 38 v 1675 (ObjNum 971 snap 296 seq_num 971) dirty exists 2024-09-21T09:42:27.251 INFO:tasks.rados.rados.0.smithi022.stdout:3225: expect (ObjNum 996 snap 305 seq_num 996) 2024-09-21T09:42:27.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:42:27 smithi064 ceph-mon[86709]: osdmap e706: 8 total, 8 up, 8 in 2024-09-21T09:42:27.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:42:27 smithi022 ceph-mon[98615]: osdmap e706: 8 total, 8 up, 8 in 2024-09-21T09:42:27.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:42:27 smithi022 ceph-mon[96736]: osdmap e706: 8 total, 8 up, 8 in 2024-09-21T09:42:28.091 INFO:tasks.rados.rados.0.smithi022.stdout:3231: finishing write tid 1 to smithi02284960-29 2024-09-21T09:42:28.091 INFO:tasks.rados.rados.0.smithi022.stdout:3231: finishing write tid 2 to smithi02284960-29 2024-09-21T09:42:28.091 INFO:tasks.rados.rados.0.smithi022.stdout:3231: finishing write tid 3 to smithi02284960-29 2024-09-21T09:42:28.092 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 29 v 1810 (ObjNum 1020 snap 315 seq_num 1020) dirty exists 2024-09-21T09:42:28.092 INFO:tasks.rados.rados.0.smithi022.stdout:3231: left oid 29 (ObjNum 1020 snap 315 seq_num 1020) 2024-09-21T09:42:28.092 INFO:tasks.rados.rados.0.smithi022.stdout:3232: seq_num 1021 ranges {0=303104} 2024-09-21T09:42:28.094 INFO:tasks.rados.rados.0.smithi022.stdout:3232: writing smithi02284960-23 from 0 to 303104 tid 1 2024-09-21T09:42:28.094 INFO:tasks.rados.rados.0.smithi022.stdout:3224: done (7 left) 2024-09-21T09:42:28.094 INFO:tasks.rados.rados.0.smithi022.stdout:3225: done (6 left) 2024-09-21T09:42:28.094 INFO:tasks.rados.rados.0.smithi022.stdout:3226: done (5 left) 2024-09-21T09:42:28.094 INFO:tasks.rados.rados.0.smithi022.stdout:3227: done (4 left) 2024-09-21T09:42:28.094 INFO:tasks.rados.rados.0.smithi022.stdout:3228: done (3 left) 2024-09-21T09:42:28.094 INFO:tasks.rados.rados.0.smithi022.stdout:3229: done (2 left) 2024-09-21T09:42:28.094 INFO:tasks.rados.rados.0.smithi022.stdout:3231: done (1 left) 2024-09-21T09:42:28.094 INFO:tasks.rados.rados.0.smithi022.stdout:3233: rollback oid 45 current snap is 315 2024-09-21T09:42:28.094 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 45 to 308 2024-09-21T09:42:28.094 INFO:tasks.rados.rados.0.smithi022.stdout:3234: rollback oid 20 current snap is 315 2024-09-21T09:42:28.094 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 20 to 305 2024-09-21T09:42:28.095 INFO:tasks.rados.rados.0.smithi022.stdout:3235: rmattr oid 13 current snap is 315 2024-09-21T09:42:28.095 INFO:tasks.rados.rados.0.smithi022.stdout:3235: done (3 left) 2024-09-21T09:42:28.095 INFO:tasks.rados.rados.0.smithi022.stdout:3236: read oid 32 snap -1 2024-09-21T09:42:28.095 INFO:tasks.rados.rados.0.smithi022.stdout:3236: expect (ObjNum 884 snap 282 seq_num 884) 2024-09-21T09:42:28.095 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 40 current snap is 315 2024-09-21T09:42:28.095 INFO:tasks.rados.rados.0.smithi022.stdout:3237: seq_num 1022 ranges {2310144=319488} 2024-09-21T09:42:28.110 INFO:tasks.rados.rados.0.smithi022.stdout:3237: writing smithi02284960-40 from 2310144 to 2629632 tid 1 2024-09-21T09:42:28.110 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 16 current snap is 315 2024-09-21T09:42:28.110 INFO:tasks.rados.rados.0.smithi022.stdout:3232: finishing write tid 1 to smithi02284960-23 2024-09-21T09:42:28.110 INFO:tasks.rados.rados.0.smithi022.stdout:3238: seq_num 1023 ranges {4685824=335872} 2024-09-21T09:42:28.136 INFO:tasks.rados.rados.0.smithi022.stdout:3238: writing smithi02284960-16 from 4685824 to 5021696 tid 1 2024-09-21T09:42:28.136 INFO:tasks.rados.rados.0.smithi022.stdout:3239: rmattr oid 10 current snap is 315 2024-09-21T09:42:28.137 INFO:tasks.rados.rados.0.smithi022.stdout:3239: done (6 left) 2024-09-21T09:42:28.137 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 46 current snap is 315 2024-09-21T09:42:28.137 INFO:tasks.rados.rados.0.smithi022.stdout:3232: finishing write tid 2 to smithi02284960-23 2024-09-21T09:42:28.137 INFO:tasks.rados.rados.0.smithi022.stdout:3233: finishing rollback tid 0 to smithi02284960-45 2024-09-21T09:42:28.137 INFO:tasks.rados.rados.0.smithi022.stdout:3233: finishing rollback tid 1 to smithi02284960-45 2024-09-21T09:42:28.137 INFO:tasks.rados.rados.0.smithi022.stdout:3234: finishing rollback tid 0 to smithi02284960-20 2024-09-21T09:42:28.137 INFO:tasks.rados.rados.0.smithi022.stdout:3233: finishing rollback tid 2 to smithi02284960-45 2024-09-21T09:42:28.137 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 45 v 1427 (ObjNum 986 snap 305 seq_num 986) dirty exists 2024-09-21T09:42:28.137 INFO:tasks.rados.rados.0.smithi022.stdout:3232: finishing write tid 3 to smithi02284960-23 2024-09-21T09:42:28.137 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 23 v 1812 (ObjNum 1021 snap 315 seq_num 1021) dirty exists 2024-09-21T09:42:28.137 INFO:tasks.rados.rados.0.smithi022.stdout:3232: left oid 23 (ObjNum 1021 snap 315 seq_num 1021) 2024-09-21T09:42:28.137 INFO:tasks.rados.rados.0.smithi022.stdout:3234: finishing rollback tid 1 to smithi02284960-20 2024-09-21T09:42:28.137 INFO:tasks.rados.rados.0.smithi022.stdout:3234: finishing rollback tid 2 to smithi02284960-20 2024-09-21T09:42:28.137 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 20 v 1929 (ObjNum 968 snap 296 seq_num 968) dirty exists 2024-09-21T09:42:28.138 INFO:tasks.rados.rados.0.smithi022.stdout:3240: seq_num 1024 ranges {1286144=352256} 2024-09-21T09:42:28.145 INFO:tasks.rados.rados.0.smithi022.stdout:3240: writing smithi02284960-46 from 1286144 to 1638400 tid 1 2024-09-21T09:42:28.145 INFO:tasks.rados.rados.0.smithi022.stdout:3232: done (6 left) 2024-09-21T09:42:28.145 INFO:tasks.rados.rados.0.smithi022.stdout:3233: done (5 left) 2024-09-21T09:42:28.145 INFO:tasks.rados.rados.0.smithi022.stdout:3234: done (4 left) 2024-09-21T09:42:28.145 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 23 current snap is 315 2024-09-21T09:42:28.145 INFO:tasks.rados.rados.0.smithi022.stdout:3236: expect (ObjNum 884 snap 282 seq_num 884) 2024-09-21T09:42:28.384 INFO:tasks.rados.rados.0.smithi022.stdout:3237: finishing write tid 1 to smithi02284960-40 2024-09-21T09:42:28.384 INFO:tasks.rados.rados.0.smithi022.stdout:3237: finishing write tid 2 to smithi02284960-40 2024-09-21T09:42:28.384 INFO:tasks.rados.rados.0.smithi022.stdout:3237: finishing write tid 3 to smithi02284960-40 2024-09-21T09:42:28.384 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 40 v 1800 (ObjNum 1022 snap 315 seq_num 1022) dirty exists 2024-09-21T09:42:28.384 INFO:tasks.rados.rados.0.smithi022.stdout:3237: left oid 40 (ObjNum 1022 snap 315 seq_num 1022) 2024-09-21T09:42:28.384 INFO:tasks.rados.rados.0.smithi022.stdout:3241: seq_num 1025 ranges {303104=368640} 2024-09-21T09:42:28.388 INFO:tasks.rados.rados.0.smithi022.stdout:3241: writing smithi02284960-23 from 303104 to 671744 tid 1 2024-09-21T09:42:28.389 INFO:tasks.rados.rados.0.smithi022.stdout:3236: done (4 left) 2024-09-21T09:42:28.389 INFO:tasks.rados.rados.0.smithi022.stdout:3237: done (3 left) 2024-09-21T09:42:28.389 INFO:tasks.rados.rados.0.smithi022.stdout:3242: read oid 1 snap 307 2024-09-21T09:42:28.389 INFO:tasks.rados.rados.0.smithi022.stdout:3242: expect deleted 2024-09-21T09:42:28.389 INFO:tasks.rados.rados.0.smithi022.stdout:3243: copy_from oid 6 from oid 29 current snap is 315 2024-09-21T09:42:28.389 INFO:tasks.rados.rados.0.smithi022.stdout:3238: finishing write tid 1 to smithi02284960-16 2024-09-21T09:42:28.389 INFO:tasks.rados.rados.0.smithi022.stdout:3238: finishing write tid 2 to smithi02284960-16 2024-09-21T09:42:28.389 INFO:tasks.rados.rados.0.smithi022.stdout:3238: finishing write tid 3 to smithi02284960-16 2024-09-21T09:42:28.389 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 16 v 1527 (ObjNum 1023 snap 315 seq_num 1023) dirty exists 2024-09-21T09:42:28.389 INFO:tasks.rados.rados.0.smithi022.stdout:3238: left oid 16 (ObjNum 1023 snap 315 seq_num 1023) 2024-09-21T09:42:28.389 INFO:tasks.rados.rados.0.smithi022.stdout:3240: finishing write tid 1 to smithi02284960-46 2024-09-21T09:42:28.389 INFO:tasks.rados.rados.0.smithi022.stdout:3240: finishing write tid 2 to smithi02284960-46 2024-09-21T09:42:28.389 INFO:tasks.rados.rados.0.smithi022.stdout:3240: finishing write tid 3 to smithi02284960-46 2024-09-21T09:42:28.389 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 46 v 1699 (ObjNum 1024 snap 315 seq_num 1024) dirty exists 2024-09-21T09:42:28.390 INFO:tasks.rados.rados.0.smithi022.stdout:3240: left oid 46 (ObjNum 1024 snap 315 seq_num 1024) 2024-09-21T09:42:28.390 INFO:tasks.rados.rados.0.smithi022.stdout:3238: done (4 left) 2024-09-21T09:42:28.390 INFO:tasks.rados.rados.0.smithi022.stdout:3240: done (3 left) 2024-09-21T09:42:28.390 INFO:tasks.rados.rados.0.smithi022.stdout:3244: delete oid 11 current snap is 315 2024-09-21T09:42:28.391 INFO:tasks.rados.rados.0.smithi022.stdout:3242: done (3 left) 2024-09-21T09:42:28.391 INFO:tasks.rados.rados.0.smithi022.stdout:3244: done (2 left) 2024-09-21T09:42:28.391 INFO:tasks.rados.rados.0.smithi022.stdout:3245: snap_remove snap 312 2024-09-21T09:42:28.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:42:28 smithi064 ceph-mon[86709]: pgmap v672: 121 pgs: 121 active+clean; 224 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 29 MiB/s rd, 310 KiB/s wr, 6 op/s 2024-09-21T09:42:28.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:42:28 smithi064 ceph-mon[86709]: osdmap e707: 8 total, 8 up, 8 in 2024-09-21T09:42:28.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:42:28 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:42:28.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:42:28 smithi022 ceph-mon[98615]: pgmap v672: 121 pgs: 121 active+clean; 224 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 29 MiB/s rd, 310 KiB/s wr, 6 op/s 2024-09-21T09:42:28.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:42:28 smithi022 ceph-mon[98615]: osdmap e707: 8 total, 8 up, 8 in 2024-09-21T09:42:28.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:42:28 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:42:28.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:42:28 smithi022 ceph-mon[96736]: pgmap v672: 121 pgs: 121 active+clean; 224 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 29 MiB/s rd, 310 KiB/s wr, 6 op/s 2024-09-21T09:42:28.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:42:28 smithi022 ceph-mon[96736]: osdmap e707: 8 total, 8 up, 8 in 2024-09-21T09:42:28.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:42:28 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:42:29.064 INFO:tasks.rados.rados.0.smithi022.stdout:3245: done (2 left) 2024-09-21T09:42:29.064 INFO:tasks.rados.rados.0.smithi022.stdout:3246: read oid 21 snap -1 2024-09-21T09:42:29.064 INFO:tasks.rados.rados.0.smithi022.stdout:3246: expect (ObjNum 992 snap 305 seq_num 992) 2024-09-21T09:42:29.064 INFO:tasks.rados.rados.0.smithi022.stdout:3247: rollback oid 43 current snap is 315 2024-09-21T09:42:29.064 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 43 to 307 2024-09-21T09:42:29.065 INFO:tasks.rados.rados.0.smithi022.stdout:3248: read oid 37 snap 313 2024-09-21T09:42:29.065 INFO:tasks.rados.rados.0.smithi022.stdout:3248: expect (ObjNum 1010 snap 312 seq_num 1010) 2024-09-21T09:42:29.065 INFO:tasks.rados.rados.0.smithi022.stdout:3249: snap_remove snap 311 2024-09-21T09:42:30.073 INFO:tasks.rados.rados.0.smithi022.stdout:3249: done (5 left) 2024-09-21T09:42:30.073 INFO:tasks.rados.rados.0.smithi022.stdout:3250: copy_from oid 4 from oid 40 current snap is 315 2024-09-21T09:42:30.073 INFO:tasks.rados.rados.0.smithi022.stdout:3251: snap_remove snap 314 2024-09-21T09:42:30.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:42:30 smithi064 ceph-mon[86709]: pgmap v674: 121 pgs: 121 active+clean; 226 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 25 MiB/s rd, 667 KiB/s wr, 11 op/s 2024-09-21T09:42:30.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:42:30 smithi064 ceph-mon[86709]: osdmap e708: 8 total, 8 up, 8 in 2024-09-21T09:42:30.442 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:42:30 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:42:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:42:30.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:42:30 smithi022 ceph-mon[98615]: pgmap v674: 121 pgs: 121 active+clean; 226 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 25 MiB/s rd, 667 KiB/s wr, 11 op/s 2024-09-21T09:42:30.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:42:30 smithi022 ceph-mon[98615]: osdmap e708: 8 total, 8 up, 8 in 2024-09-21T09:42:30.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:42:30 smithi022 ceph-mon[96736]: pgmap v674: 121 pgs: 121 active+clean; 226 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 25 MiB/s rd, 667 KiB/s wr, 11 op/s 2024-09-21T09:42:30.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:42:30 smithi022 ceph-mon[96736]: osdmap e708: 8 total, 8 up, 8 in 2024-09-21T09:42:31.078 INFO:tasks.rados.rados.0.smithi022.stdout:3251: done (6 left) 2024-09-21T09:42:31.079 INFO:tasks.rados.rados.0.smithi022.stdout:3252: read oid 30 snap -1 2024-09-21T09:42:31.079 INFO:tasks.rados.rados.0.smithi022.stdout:3252: expect (ObjNum 1012 snap 312 seq_num 1012) 2024-09-21T09:42:31.079 INFO:tasks.rados.rados.0.smithi022.stdout:3253: delete oid 20 current snap is 315 2024-09-21T09:42:31.079 INFO:tasks.rados.rados.0.smithi022.stdout:3241: finishing write tid 1 to smithi02284960-23 2024-09-21T09:42:31.079 INFO:tasks.rados.rados.0.smithi022.stdout:3241: finishing write tid 2 to smithi02284960-23 2024-09-21T09:42:31.079 INFO:tasks.rados.rados.0.smithi022.stdout:3241: finishing write tid 3 to smithi02284960-23 2024-09-21T09:42:31.079 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 23 v 1814 (ObjNum 1025 snap 315 seq_num 1025) dirty exists 2024-09-21T09:42:31.079 INFO:tasks.rados.rados.0.smithi022.stdout:3241: left oid 23 (ObjNum 1025 snap 315 seq_num 1025) 2024-09-21T09:42:31.079 INFO:tasks.rados.rados.0.smithi022.stdout:3243: finishing copy_from racing read to smithi02284960-6 2024-09-21T09:42:31.079 INFO:tasks.rados.rados.0.smithi022.stdout:3243: finishing copy_from to smithi02284960-6 2024-09-21T09:42:31.079 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 6 v 1810 (ObjNum 1020 snap 315 seq_num 1020) dirty exists 2024-09-21T09:42:31.079 INFO:tasks.rados.rados.0.smithi022.stdout:3247: finishing rollback tid 0 to smithi02284960-43 2024-09-21T09:42:31.079 INFO:tasks.rados.rados.0.smithi022.stdout:3247: finishing rollback tid 1 to smithi02284960-43 2024-09-21T09:42:31.079 INFO:tasks.rados.rados.0.smithi022.stdout:3247: finishing rollback tid 2 to smithi02284960-43 2024-09-21T09:42:31.080 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 43 v 1803 (ObjNum 983 snap 304 seq_num 983) dirty exists 2024-09-21T09:42:31.080 INFO:tasks.rados.rados.0.smithi022.stdout:3248: expect (ObjNum 1010 snap 312 seq_num 1010) 2024-09-21T09:42:31.084 INFO:tasks.rados.rados.0.smithi022.stdout:3246: expect (ObjNum 992 snap 305 seq_num 992) 2024-09-21T09:42:31.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:42:31 smithi064 ceph-mon[86709]: osdmap e709: 8 total, 8 up, 8 in 2024-09-21T09:42:31.557 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:42:31 smithi022 ceph-mon[98615]: osdmap e709: 8 total, 8 up, 8 in 2024-09-21T09:42:31.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:42:31 smithi022 ceph-mon[96736]: osdmap e709: 8 total, 8 up, 8 in 2024-09-21T09:42:31.569 INFO:tasks.rados.rados.0.smithi022.stdout:3250: finishing copy_from racing read to smithi02284960-4 2024-09-21T09:42:31.569 INFO:tasks.rados.rados.0.smithi022.stdout:3250: finishing copy_from to smithi02284960-4 2024-09-21T09:42:31.569 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 4 v 1800 (ObjNum 1022 snap 315 seq_num 1022) dirty exists 2024-09-21T09:42:31.569 INFO:tasks.rados.rados.0.smithi022.stdout:3252: expect (ObjNum 1012 snap 312 seq_num 1012) 2024-09-21T09:42:31.671 INFO:tasks.rados.rados.0.smithi022.stdout:3241: done (7 left) 2024-09-21T09:42:31.671 INFO:tasks.rados.rados.0.smithi022.stdout:3243: done (6 left) 2024-09-21T09:42:31.671 INFO:tasks.rados.rados.0.smithi022.stdout:3246: done (5 left) 2024-09-21T09:42:31.671 INFO:tasks.rados.rados.0.smithi022.stdout:3247: done (4 left) 2024-09-21T09:42:31.671 INFO:tasks.rados.rados.0.smithi022.stdout:3248: done (3 left) 2024-09-21T09:42:31.671 INFO:tasks.rados.rados.0.smithi022.stdout:3250: done (2 left) 2024-09-21T09:42:31.671 INFO:tasks.rados.rados.0.smithi022.stdout:3252: done (1 left) 2024-09-21T09:42:31.671 INFO:tasks.rados.rados.0.smithi022.stdout:3253: done (0 left) 2024-09-21T09:42:31.672 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 4 current snap is 315 2024-09-21T09:42:31.672 INFO:tasks.rados.rados.0.smithi022.stdout:3254: seq_num 1028 ranges {2629632=425984} 2024-09-21T09:42:31.688 INFO:tasks.rados.rados.0.smithi022.stdout:3254: writing smithi02284960-4 from 2629632 to 3055616 tid 1 2024-09-21T09:42:31.688 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 40 current snap is 315 2024-09-21T09:42:31.688 INFO:tasks.rados.rados.0.smithi022.stdout:3255: seq_num 1029 ranges {2629632=442368} 2024-09-21T09:42:31.704 INFO:tasks.rados.rados.0.smithi022.stdout:3255: writing smithi02284960-40 from 2629632 to 3072000 tid 1 2024-09-21T09:42:31.705 INFO:tasks.rados.rados.0.smithi022.stdout:3256: copy_from oid 33 from oid 22 current snap is 315 2024-09-21T09:42:31.705 INFO:tasks.rados.rados.0.smithi022.stdout:3254: finishing write tid 1 to smithi02284960-4 2024-09-21T09:42:31.705 INFO:tasks.rados.rados.0.smithi022.stdout:3254: finishing write tid 2 to smithi02284960-4 2024-09-21T09:42:31.705 INFO:tasks.rados.rados.0.smithi022.stdout:3254: finishing write tid 3 to smithi02284960-4 2024-09-21T09:42:31.705 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 4 v 1816 (ObjNum 1028 snap 315 seq_num 1028) dirty exists 2024-09-21T09:42:31.705 INFO:tasks.rados.rados.0.smithi022.stdout:3254: left oid 4 (ObjNum 1028 snap 315 seq_num 1028) 2024-09-21T09:42:31.705 INFO:tasks.rados.rados.0.smithi022.stdout:3254: done (2 left) 2024-09-21T09:42:31.705 INFO:tasks.rados.rados.0.smithi022.stdout:3257: snap_remove snap 308 2024-09-21T09:42:32.080 INFO:tasks.rados.rados.0.smithi022.stdout:3257: done (2 left) 2024-09-21T09:42:32.080 INFO:tasks.rados.rados.0.smithi022.stdout:3258: rmattr oid 45 current snap is 315 2024-09-21T09:42:32.080 INFO:tasks.rados.rados.0.smithi022.stdout:3258: done (2 left) 2024-09-21T09:42:32.080 INFO:tasks.rados.rados.0.smithi022.stdout:3259: setattr oid 32 current snap is 315 2024-09-21T09:42:32.081 INFO:tasks.rados.rados.0.smithi022.stdout:3255: finishing write tid 1 to smithi02284960-40 2024-09-21T09:42:32.081 INFO:tasks.rados.rados.0.smithi022.stdout:3255: finishing write tid 2 to smithi02284960-40 2024-09-21T09:42:32.081 INFO:tasks.rados.rados.0.smithi022.stdout:3255: finishing write tid 3 to smithi02284960-40 2024-09-21T09:42:32.081 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 40 v 1805 (ObjNum 1029 snap 315 seq_num 1029) dirty exists 2024-09-21T09:42:32.081 INFO:tasks.rados.rados.0.smithi022.stdout:3255: left oid 40 (ObjNum 1029 snap 315 seq_num 1029) 2024-09-21T09:42:32.081 INFO:tasks.rados.rados.0.smithi022.stdout:3256: finishing copy_from racing read to smithi02284960-33 2024-09-21T09:42:32.081 INFO:tasks.rados.rados.0.smithi022.stdout:3256: finishing copy_from to smithi02284960-33 2024-09-21T09:42:32.081 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 33 v 1662 (ObjNum 914 snap 284 seq_num 914) dirty exists 2024-09-21T09:42:32.082 INFO:tasks.rados.rados.0.smithi022.stdout:3255: done (2 left) 2024-09-21T09:42:32.082 INFO:tasks.rados.rados.0.smithi022.stdout:3256: done (1 left) 2024-09-21T09:42:32.082 INFO:tasks.rados.rados.0.smithi022.stdout:3260: read oid 33 snap -1 2024-09-21T09:42:32.082 INFO:tasks.rados.rados.0.smithi022.stdout:3260: expect (ObjNum 914 snap 284 seq_num 914) 2024-09-21T09:42:32.082 INFO:tasks.rados.rados.0.smithi022.stdout:3261: snap_remove snap 305 2024-09-21T09:42:32.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:42:32 smithi064 ceph-mon[86709]: pgmap v677: 121 pgs: 121 active+clean; 230 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 1.4 MiB/s wr, 17 op/s 2024-09-21T09:42:32.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:42:32 smithi064 ceph-mon[86709]: osdmap e710: 8 total, 8 up, 8 in 2024-09-21T09:42:32.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:42:32 smithi022 ceph-mon[98615]: pgmap v677: 121 pgs: 121 active+clean; 230 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 1.4 MiB/s wr, 17 op/s 2024-09-21T09:42:32.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:42:32 smithi022 ceph-mon[98615]: osdmap e710: 8 total, 8 up, 8 in 2024-09-21T09:42:32.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:42:32 smithi022 ceph-mon[96736]: pgmap v677: 121 pgs: 121 active+clean; 230 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 1.4 MiB/s wr, 17 op/s 2024-09-21T09:42:32.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:42:32 smithi022 ceph-mon[96736]: osdmap e710: 8 total, 8 up, 8 in 2024-09-21T09:42:33.088 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 32 v 1948 (ObjNum 884 snap 282 seq_num 884) dirty exists 2024-09-21T09:42:33.088 INFO:tasks.rados.rados.0.smithi022.stdout:3260: expect (ObjNum 914 snap 284 seq_num 914) 2024-09-21T09:42:33.147 INFO:tasks.rados.rados.0.smithi022.stdout:3259: done (2 left) 2024-09-21T09:42:33.147 INFO:tasks.rados.rados.0.smithi022.stdout:3260: done (1 left) 2024-09-21T09:42:33.147 INFO:tasks.rados.rados.0.smithi022.stdout:3261: done (0 left) 2024-09-21T09:42:33.148 INFO:tasks.rados.rados.0.smithi022.stdout:3262: snap_remove snap 307 2024-09-21T09:42:33.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:42:33 smithi064 ceph-mon[86709]: osdmap e711: 8 total, 8 up, 8 in 2024-09-21T09:42:33.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:42:33 smithi022 ceph-mon[98615]: osdmap e711: 8 total, 8 up, 8 in 2024-09-21T09:42:33.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:42:33 smithi022 ceph-mon[96736]: osdmap e711: 8 total, 8 up, 8 in 2024-09-21T09:42:34.066 INFO:tasks.rados.rados.0.smithi022.stdout:3262: done (0 left) 2024-09-21T09:42:34.066 INFO:tasks.rados.rados.0.smithi022.stdout:3263: copy_from oid 5 from oid 40 current snap is 315 2024-09-21T09:42:34.066 INFO:tasks.rados.rados.0.smithi022.stdout:3264: setattr oid 39 current snap is 315 2024-09-21T09:42:34.068 INFO:tasks.rados.rados.0.smithi022.stdout:3265: snap_remove snap 313 2024-09-21T09:42:34.343 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:42:34 smithi064 ceph-mon[86709]: pgmap v680: 121 pgs: 121 active+clean; 230 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3.4 MiB/s rd, 1020 KiB/s wr, 9 op/s 2024-09-21T09:42:34.343 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:42:34 smithi064 ceph-mon[86709]: osdmap e712: 8 total, 8 up, 8 in 2024-09-21T09:42:34.343 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:42:34 smithi064 ceph-mon[86709]: osdmap e713: 8 total, 8 up, 8 in 2024-09-21T09:42:34.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:42:34 smithi022 ceph-mon[98615]: pgmap v680: 121 pgs: 121 active+clean; 230 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3.4 MiB/s rd, 1020 KiB/s wr, 9 op/s 2024-09-21T09:42:34.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:42:34 smithi022 ceph-mon[98615]: osdmap e712: 8 total, 8 up, 8 in 2024-09-21T09:42:34.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:42:34 smithi022 ceph-mon[98615]: osdmap e713: 8 total, 8 up, 8 in 2024-09-21T09:42:34.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:42:34 smithi022 ceph-mon[96736]: pgmap v680: 121 pgs: 121 active+clean; 230 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3.4 MiB/s rd, 1020 KiB/s wr, 9 op/s 2024-09-21T09:42:34.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:42:34 smithi022 ceph-mon[96736]: osdmap e712: 8 total, 8 up, 8 in 2024-09-21T09:42:34.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:42:34 smithi022 ceph-mon[96736]: osdmap e713: 8 total, 8 up, 8 in 2024-09-21T09:42:35.067 INFO:tasks.rados.rados.0.smithi022.stdout:3265: done (2 left) 2024-09-21T09:42:35.067 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 42 current snap is 315 2024-09-21T09:42:35.067 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 39 v 1343 (ObjNum 1018 snap 315 seq_num 1018) dirty exists 2024-09-21T09:42:35.067 INFO:tasks.rados.rados.0.smithi022.stdout:3263: finishing copy_from racing read to smithi02284960-5 2024-09-21T09:42:35.068 INFO:tasks.rados.rados.0.smithi022.stdout:3263: finishing copy_from to smithi02284960-5 2024-09-21T09:42:35.068 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 5 v 1805 (ObjNum 1029 snap 315 seq_num 1029) dirty exists 2024-09-21T09:42:35.068 INFO:tasks.rados.rados.0.smithi022.stdout:3266: seq_num 1030 ranges {1941504=458752} 2024-09-21T09:42:35.081 INFO:tasks.rados.rados.0.smithi022.stdout:3266: writing smithi02284960-42 from 1941504 to 2400256 tid 1 2024-09-21T09:42:35.081 INFO:tasks.rados.rados.0.smithi022.stdout:3263: done (2 left) 2024-09-21T09:42:35.081 INFO:tasks.rados.rados.0.smithi022.stdout:3264: done (1 left) 2024-09-21T09:42:35.081 INFO:tasks.rados.rados.0.smithi022.stdout:3267: rollback oid 28 current snap is 315 2024-09-21T09:42:35.081 INFO:tasks.rados.rados.0.smithi022.stdout:3267: done (1 left) 2024-09-21T09:42:35.081 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 48 current snap is 315 2024-09-21T09:42:35.081 INFO:tasks.rados.rados.0.smithi022.stdout:3268: seq_num 1031 ranges {172032=475136} 2024-09-21T09:42:35.084 INFO:tasks.rados.rados.0.smithi022.stdout:3268: writing smithi02284960-48 from 172032 to 647168 tid 1 2024-09-21T09:42:35.085 INFO:tasks.rados.rados.0.smithi022.stdout:3269: rollback oid 45 current snap is 315 2024-09-21T09:42:35.085 INFO:tasks.rados.rados.0.smithi022.stdout:3269: done (2 left) 2024-09-21T09:42:35.085 INFO:tasks.rados.rados.0.smithi022.stdout:3270: delete oid 24 current snap is 315 2024-09-21T09:42:35.091 INFO:tasks.rados.rados.0.smithi022.stdout:3270: done (2 left) 2024-09-21T09:42:35.091 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 36 current snap is 315 2024-09-21T09:42:35.091 INFO:tasks.rados.rados.0.smithi022.stdout:3271: seq_num 1033 ranges {999424=507904} 2024-09-21T09:42:35.100 INFO:tasks.rados.rados.0.smithi022.stdout:3271: writing smithi02284960-36 from 999424 to 1507328 tid 1 2024-09-21T09:42:35.100 INFO:tasks.rados.rados.0.smithi022.stdout:3272: delete oid 50 current snap is 315 2024-09-21T09:42:35.100 INFO:tasks.rados.rados.0.smithi022.stdout:3266: finishing write tid 1 to smithi02284960-42 2024-09-21T09:42:35.100 INFO:tasks.rados.rados.0.smithi022.stdout:3266: finishing write tid 2 to smithi02284960-42 2024-09-21T09:42:35.100 INFO:tasks.rados.rados.0.smithi022.stdout:3266: finishing write tid 3 to smithi02284960-42 2024-09-21T09:42:35.101 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 42 v 1797 (ObjNum 1030 snap 315 seq_num 1030) dirty exists 2024-09-21T09:42:35.101 INFO:tasks.rados.rados.0.smithi022.stdout:3266: left oid 42 (ObjNum 1030 snap 315 seq_num 1030) 2024-09-21T09:42:35.101 INFO:tasks.rados.rados.0.smithi022.stdout:3268: finishing write tid 1 to smithi02284960-48 2024-09-21T09:42:35.101 INFO:tasks.rados.rados.0.smithi022.stdout:3268: finishing write tid 2 to smithi02284960-48 2024-09-21T09:42:35.101 INFO:tasks.rados.rados.0.smithi022.stdout:3268: finishing write tid 3 to smithi02284960-48 2024-09-21T09:42:35.102 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 48 v 1972 (ObjNum 1031 snap 315 seq_num 1031) dirty exists 2024-09-21T09:42:35.102 INFO:tasks.rados.rados.0.smithi022.stdout:3268: left oid 48 (ObjNum 1031 snap 315 seq_num 1031) 2024-09-21T09:42:35.105 INFO:tasks.rados.rados.0.smithi022.stdout:3271: finishing write tid 1 to smithi02284960-36 2024-09-21T09:42:35.106 INFO:tasks.rados.rados.0.smithi022.stdout:3271: finishing write tid 2 to smithi02284960-36 2024-09-21T09:42:35.107 INFO:tasks.rados.rados.0.smithi022.stdout:3266: done (3 left) 2024-09-21T09:42:35.107 INFO:tasks.rados.rados.0.smithi022.stdout:3268: done (2 left) 2024-09-21T09:42:35.108 INFO:tasks.rados.rados.0.smithi022.stdout:3272: done (1 left) 2024-09-21T09:42:35.108 INFO:tasks.rados.rados.0.smithi022.stdout:3273: copy_from oid 34 from oid 3 current snap is 315 2024-09-21T09:42:35.108 INFO:tasks.rados.rados.0.smithi022.stdout:3274: copy_from oid 25 from oid 31 current snap is 315 2024-09-21T09:42:35.108 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 40 current snap is 315 2024-09-21T09:42:35.108 INFO:tasks.rados.rados.0.smithi022.stdout:3275: seq_num 1035 ranges {3072000=540672} 2024-09-21T09:42:35.128 INFO:tasks.rados.rados.0.smithi022.stdout:3275: writing smithi02284960-40 from 3072000 to 3612672 tid 1 2024-09-21T09:42:35.128 INFO:tasks.rados.rados.0.smithi022.stdout:3276: rollback oid 12 current snap is 315 2024-09-21T09:42:35.128 INFO:tasks.rados.rados.0.smithi022.stdout:3276: done (4 left) 2024-09-21T09:42:35.128 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 49 current snap is 315 2024-09-21T09:42:35.128 INFO:tasks.rados.rados.0.smithi022.stdout:3271: finishing write tid 3 to smithi02284960-36 2024-09-21T09:42:35.128 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 36 v 1429 (ObjNum 1033 snap 315 seq_num 1033) dirty exists 2024-09-21T09:42:35.129 INFO:tasks.rados.rados.0.smithi022.stdout:3271: left oid 36 (ObjNum 1033 snap 315 seq_num 1033) 2024-09-21T09:42:35.129 INFO:tasks.rados.rados.0.smithi022.stdout:3273: finishing copy_from racing read to smithi02284960-34 2024-09-21T09:42:35.129 INFO:tasks.rados.rados.0.smithi022.stdout:3273: finishing copy_from to smithi02284960-34 2024-09-21T09:42:35.129 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 34 v 623 (ObjNum 910 snap 284 seq_num 910) dirty exists 2024-09-21T09:42:35.129 INFO:tasks.rados.rados.0.smithi022.stdout:3274: finishing copy_from racing read to smithi02284960-25 2024-09-21T09:42:35.129 INFO:tasks.rados.rados.0.smithi022.stdout:3277: seq_num 1036 ranges {3645440=557056} 2024-09-21T09:42:35.152 INFO:tasks.rados.rados.0.smithi022.stdout:3277: writing smithi02284960-49 from 3645440 to 4202496 tid 1 2024-09-21T09:42:35.152 INFO:tasks.rados.rados.0.smithi022.stdout:3271: done (4 left) 2024-09-21T09:42:35.152 INFO:tasks.rados.rados.0.smithi022.stdout:3273: done (3 left) 2024-09-21T09:42:35.152 INFO:tasks.rados.rados.0.smithi022.stdout:3278: read oid 35 snap -1 2024-09-21T09:42:35.152 INFO:tasks.rados.rados.0.smithi022.stdout:3278: expect (ObjNum 925 snap 287 seq_num 925) 2024-09-21T09:42:35.152 INFO:tasks.rados.rados.0.smithi022.stdout:3279: copy_from oid 43 from oid 6 current snap is 315 2024-09-21T09:42:35.152 INFO:tasks.rados.rados.0.smithi022.stdout:3274: finishing copy_from to smithi02284960-25 2024-09-21T09:42:35.153 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 25 v 1878 (ObjNum 999 snap 307 seq_num 999) dirty exists 2024-09-21T09:42:35.153 INFO:tasks.rados.rados.0.smithi022.stdout:3275: finishing write tid 1 to smithi02284960-40 2024-09-21T09:42:35.153 INFO:tasks.rados.rados.0.smithi022.stdout:3275: finishing write tid 2 to smithi02284960-40 2024-09-21T09:42:35.153 INFO:tasks.rados.rados.0.smithi022.stdout:3275: finishing write tid 3 to smithi02284960-40 2024-09-21T09:42:35.153 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 40 v 1880 (ObjNum 1035 snap 315 seq_num 1035) dirty exists 2024-09-21T09:42:35.153 INFO:tasks.rados.rados.0.smithi022.stdout:3275: left oid 40 (ObjNum 1035 snap 315 seq_num 1035) 2024-09-21T09:42:35.153 INFO:tasks.rados.rados.0.smithi022.stdout:3274: done (4 left) 2024-09-21T09:42:35.153 INFO:tasks.rados.rados.0.smithi022.stdout:3275: done (3 left) 2024-09-21T09:42:35.153 INFO:tasks.rados.rados.0.smithi022.stdout:3280: rollback oid 23 current snap is 315 2024-09-21T09:42:35.153 INFO:tasks.rados.rados.0.smithi022.stdout:3280: done (3 left) 2024-09-21T09:42:35.153 INFO:tasks.rados.rados.0.smithi022.stdout:3281: rollback oid 22 current snap is 315 2024-09-21T09:42:35.153 INFO:tasks.rados.rados.0.smithi022.stdout:3281: done (3 left) 2024-09-21T09:42:35.153 INFO:tasks.rados.rados.0.smithi022.stdout:3282: rmattr oid 1 current snap is 315 2024-09-21T09:42:35.153 INFO:tasks.rados.rados.0.smithi022.stdout:3282: done (3 left) 2024-09-21T09:42:35.153 INFO:tasks.rados.rados.0.smithi022.stdout:3283: delete oid 25 current snap is 315 2024-09-21T09:42:35.154 INFO:tasks.rados.rados.0.smithi022.stdout:3283: done (3 left) 2024-09-21T09:42:35.154 INFO:tasks.rados.rados.0.smithi022.stdout:3284: read oid 12 snap -1 2024-09-21T09:42:35.154 INFO:tasks.rados.rados.0.smithi022.stdout:3284: expect (ObjNum 997 snap 306 seq_num 997) 2024-09-21T09:42:35.154 INFO:tasks.rados.rados.0.smithi022.stdout:3285: delete oid 10 current snap is 315 2024-09-21T09:42:35.155 INFO:tasks.rados.rados.0.smithi022.stdout:3277: finishing write tid 1 to smithi02284960-49 2024-09-21T09:42:35.155 INFO:tasks.rados.rados.0.smithi022.stdout:3277: finishing write tid 2 to smithi02284960-49 2024-09-21T09:42:35.158 INFO:tasks.rados.rados.0.smithi022.stdout:3279: finishing copy_from racing read to smithi02284960-43 2024-09-21T09:42:35.159 INFO:tasks.rados.rados.0.smithi022.stdout:3285: done (4 left) 2024-09-21T09:42:35.159 INFO:tasks.rados.rados.0.smithi022.stdout:3286: copy_from oid 21 from oid 5 current snap is 315 2024-09-21T09:42:35.159 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 30 current snap is 315 2024-09-21T09:42:35.159 INFO:tasks.rados.rados.0.smithi022.stdout:3287: seq_num 1039 ranges {2465792=606208} 2024-09-21T09:42:35.177 INFO:tasks.rados.rados.0.smithi022.stdout:3287: writing smithi02284960-30 from 2465792 to 3072000 tid 1 2024-09-21T09:42:35.177 INFO:tasks.rados.rados.0.smithi022.stdout:3288: read oid 9 snap -1 2024-09-21T09:42:35.177 INFO:tasks.rados.rados.0.smithi022.stdout:3288: expect deleted 2024-09-21T09:42:35.177 INFO:tasks.rados.rados.0.smithi022.stdout:3289: delete oid 31 current snap is 315 2024-09-21T09:42:35.177 INFO:tasks.rados.rados.0.smithi022.stdout:3279: finishing copy_from to smithi02284960-43 2024-09-21T09:42:35.177 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 43 v 1810 (ObjNum 1020 snap 315 seq_num 1020) dirty exists 2024-09-21T09:42:35.177 INFO:tasks.rados.rados.0.smithi022.stdout:3277: finishing write tid 3 to smithi02284960-49 2024-09-21T09:42:35.177 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 49 v 1801 (ObjNum 1036 snap 315 seq_num 1036) dirty exists 2024-09-21T09:42:35.177 INFO:tasks.rados.rados.0.smithi022.stdout:3277: left oid 49 (ObjNum 1036 snap 315 seq_num 1036) 2024-09-21T09:42:35.177 INFO:tasks.rados.rados.0.smithi022.stdout:3278: expect (ObjNum 925 snap 287 seq_num 925) 2024-09-21T09:42:35.310 INFO:tasks.rados.rados.0.smithi022.stdout:3284: expect (ObjNum 997 snap 306 seq_num 997) 2024-09-21T09:42:35.371 INFO:tasks.rados.rados.0.smithi022.stdout:3286: finishing copy_from racing read to smithi02284960-21 2024-09-21T09:42:35.371 INFO:tasks.rados.rados.0.smithi022.stdout:3287: finishing write tid 1 to smithi02284960-30 2024-09-21T09:42:35.371 INFO:tasks.rados.rados.0.smithi022.stdout:3287: finishing write tid 2 to smithi02284960-30 2024-09-21T09:42:35.371 INFO:tasks.rados.rados.0.smithi022.stdout:3286: finishing copy_from to smithi02284960-21 2024-09-21T09:42:35.371 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 21 v 1805 (ObjNum 1029 snap 315 seq_num 1029) dirty exists 2024-09-21T09:42:35.371 INFO:tasks.rados.rados.0.smithi022.stdout:3287: finishing write tid 3 to smithi02284960-30 2024-09-21T09:42:35.371 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 30 v 1345 (ObjNum 1039 snap 315 seq_num 1039) dirty exists 2024-09-21T09:42:35.372 INFO:tasks.rados.rados.0.smithi022.stdout:3287: left oid 30 (ObjNum 1039 snap 315 seq_num 1039) 2024-09-21T09:42:35.372 INFO:tasks.rados.rados.0.smithi022.stdout:3277: done (7 left) 2024-09-21T09:42:35.372 INFO:tasks.rados.rados.0.smithi022.stdout:3278: done (6 left) 2024-09-21T09:42:35.372 INFO:tasks.rados.rados.0.smithi022.stdout:3279: done (5 left) 2024-09-21T09:42:35.372 INFO:tasks.rados.rados.0.smithi022.stdout:3284: done (4 left) 2024-09-21T09:42:35.372 INFO:tasks.rados.rados.0.smithi022.stdout:3286: done (3 left) 2024-09-21T09:42:35.372 INFO:tasks.rados.rados.0.smithi022.stdout:3287: done (2 left) 2024-09-21T09:42:35.372 INFO:tasks.rados.rados.0.smithi022.stdout:3288: done (1 left) 2024-09-21T09:42:35.372 INFO:tasks.rados.rados.0.smithi022.stdout:3289: done (0 left) 2024-09-21T09:42:35.372 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 24 current snap is 315 2024-09-21T09:42:35.372 INFO:tasks.rados.rados.0.smithi022.stdout:3290: seq_num 1041 ranges {0=638976} 2024-09-21T09:42:35.374 INFO:tasks.rados.rados.0.smithi022.stdout:3290: writing smithi02284960-24 from 0 to 638976 tid 1 2024-09-21T09:42:35.374 INFO:tasks.rados.rados.0.smithi022.stdout:3291: rollback oid 13 current snap is 315 2024-09-21T09:42:35.374 INFO:tasks.rados.rados.0.smithi022.stdout:3291: done (1 left) 2024-09-21T09:42:35.374 INFO:tasks.rados.rados.0.smithi022.stdout:3292: copy_from oid 8 from oid 9 current snap is 315 2024-09-21T09:42:35.375 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 4 current snap is 315 2024-09-21T09:42:35.375 INFO:tasks.rados.rados.0.smithi022.stdout:3293: seq_num 1042 ranges {3055616=655360} 2024-09-21T09:42:35.395 INFO:tasks.rados.rados.0.smithi022.stdout:3293: writing smithi02284960-4 from 3055616 to 3710976 tid 1 2024-09-21T09:42:35.395 INFO:tasks.rados.rados.0.smithi022.stdout:3294: snap_create 2024-09-21T09:42:35.395 INFO:tasks.rados.rados.0.smithi022.stdout:3292: finishing copy_from to smithi02284960-8 2024-09-21T09:42:35.395 INFO:tasks.rados.rados.0.smithi022.stdout:3292: got expected ENOENT (src dne) 2024-09-21T09:42:35.395 INFO:tasks.rados.rados.0.smithi022.stdout:3292: finishing copy_from racing read to smithi02284960-8 2024-09-21T09:42:35.395 INFO:tasks.rados.rados.0.smithi022.stdout:3290: finishing write tid 1 to smithi02284960-24 2024-09-21T09:42:35.395 INFO:tasks.rados.rados.0.smithi022.stdout:3290: finishing write tid 2 to smithi02284960-24 2024-09-21T09:42:35.395 INFO:tasks.rados.rados.0.smithi022.stdout:3290: finishing write tid 3 to smithi02284960-24 2024-09-21T09:42:35.395 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 24 v 1907 (ObjNum 1041 snap 315 seq_num 1041) dirty exists 2024-09-21T09:42:35.395 INFO:tasks.rados.rados.0.smithi022.stdout:3290: left oid 24 (ObjNum 1041 snap 315 seq_num 1041) 2024-09-21T09:42:35.397 INFO:tasks.rados.rados.0.smithi022.stdout:3293: finishing write tid 1 to smithi02284960-4 2024-09-21T09:42:35.398 INFO:tasks.rados.rados.0.smithi022.stdout:3293: finishing write tid 2 to smithi02284960-4 2024-09-21T09:42:35.398 INFO:tasks.rados.rados.0.smithi022.stdout:3293: finishing write tid 3 to smithi02284960-4 2024-09-21T09:42:35.398 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 4 v 1818 (ObjNum 1042 snap 315 seq_num 1042) dirty exists 2024-09-21T09:42:35.399 INFO:tasks.rados.rados.0.smithi022.stdout:3293: left oid 4 (ObjNum 1042 snap 315 seq_num 1042) 2024-09-21T09:42:36.058 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:42:35 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:42:35] "GET /metrics HTTP/1.1" 200 36350 "" "Prometheus/2.51.0" 2024-09-21T09:42:36.090 INFO:tasks.rados.rados.0.smithi022.stdout:3290: done (3 left) 2024-09-21T09:42:36.090 INFO:tasks.rados.rados.0.smithi022.stdout:3292: done (2 left) 2024-09-21T09:42:36.090 INFO:tasks.rados.rados.0.smithi022.stdout:3293: done (1 left) 2024-09-21T09:42:36.090 INFO:tasks.rados.rados.0.smithi022.stdout:3294: done (0 left) 2024-09-21T09:42:36.090 INFO:tasks.rados.rados.0.smithi022.stdout:3295: delete oid 47 current snap is 316 2024-09-21T09:42:36.096 INFO:tasks.rados.rados.0.smithi022.stdout:3295: done (0 left) 2024-09-21T09:42:36.096 INFO:tasks.rados.rados.0.smithi022.stdout:3296: snap_create 2024-09-21T09:42:36.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:42:36 smithi064 ceph-mon[86709]: pgmap v683: 121 pgs: 121 active+clean; 230 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6.0 MiB/s rd, 152 KiB/s wr, 3 op/s 2024-09-21T09:42:36.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:42:36 smithi064 ceph-mon[86709]: osdmap e714: 8 total, 8 up, 8 in 2024-09-21T09:42:36.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:42:36 smithi022 ceph-mon[98615]: pgmap v683: 121 pgs: 121 active+clean; 230 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6.0 MiB/s rd, 152 KiB/s wr, 3 op/s 2024-09-21T09:42:36.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:42:36 smithi022 ceph-mon[98615]: osdmap e714: 8 total, 8 up, 8 in 2024-09-21T09:42:36.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:42:36 smithi022 ceph-mon[96736]: pgmap v683: 121 pgs: 121 active+clean; 230 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6.0 MiB/s rd, 152 KiB/s wr, 3 op/s 2024-09-21T09:42:36.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:42:36 smithi022 ceph-mon[96736]: osdmap e714: 8 total, 8 up, 8 in 2024-09-21T09:42:37.102 INFO:tasks.rados.rados.0.smithi022.stdout:3296: done (0 left) 2024-09-21T09:42:37.102 INFO:tasks.rados.rados.0.smithi022.stdout:3297: read oid 39 snap -1 2024-09-21T09:42:37.102 INFO:tasks.rados.rados.0.smithi022.stdout:3297: expect (ObjNum 1018 snap 315 seq_num 1018) 2024-09-21T09:42:37.102 INFO:tasks.rados.rados.0.smithi022.stdout:3298: read oid 43 snap -1 2024-09-21T09:42:37.102 INFO:tasks.rados.rados.0.smithi022.stdout:3298: expect (ObjNum 1020 snap 315 seq_num 1020) 2024-09-21T09:42:37.102 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 26 current snap is 317 2024-09-21T09:42:37.102 INFO:tasks.rados.rados.0.smithi022.stdout:3299: seq_num 1044 ranges {2408448=688128} 2024-09-21T09:42:37.121 INFO:tasks.rados.rados.0.smithi022.stdout:3299: writing smithi02284960-26 from 2408448 to 3096576 tid 1 2024-09-21T09:42:37.121 INFO:tasks.rados.rados.0.smithi022.stdout:3300: delete oid 4 current snap is 317 2024-09-21T09:42:37.122 INFO:tasks.rados.rados.0.smithi022.stdout:3298: expect (ObjNum 1020 snap 315 seq_num 1020) 2024-09-21T09:42:37.133 INFO:tasks.rados.rados.0.smithi022.stdout:3297: expect (ObjNum 1018 snap 315 seq_num 1018) 2024-09-21T09:42:37.196 INFO:tasks.rados.rados.0.smithi022.stdout:3299: finishing write tid 1 to smithi02284960-26 2024-09-21T09:42:37.196 INFO:tasks.rados.rados.0.smithi022.stdout:3299: finishing write tid 2 to smithi02284960-26 2024-09-21T09:42:37.196 INFO:tasks.rados.rados.0.smithi022.stdout:3299: finishing write tid 3 to smithi02284960-26 2024-09-21T09:42:37.196 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 26 v 1799 (ObjNum 1044 snap 317 seq_num 1044) dirty exists 2024-09-21T09:42:37.196 INFO:tasks.rados.rados.0.smithi022.stdout:3299: left oid 26 (ObjNum 1044 snap 317 seq_num 1044) 2024-09-21T09:42:37.196 INFO:tasks.rados.rados.0.smithi022.stdout:3297: done (3 left) 2024-09-21T09:42:37.196 INFO:tasks.rados.rados.0.smithi022.stdout:3298: done (2 left) 2024-09-21T09:42:37.196 INFO:tasks.rados.rados.0.smithi022.stdout:3299: done (1 left) 2024-09-21T09:42:37.197 INFO:tasks.rados.rados.0.smithi022.stdout:3300: done (0 left) 2024-09-21T09:42:37.197 INFO:tasks.rados.rados.0.smithi022.stdout:3301: setattr oid 33 current snap is 317 2024-09-21T09:42:37.197 INFO:tasks.rados.rados.0.smithi022.stdout:3302: delete oid 45 current snap is 317 2024-09-21T09:42:37.199 INFO:tasks.rados.rados.0.smithi022.stdout:3302: done (1 left) 2024-09-21T09:42:37.199 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 43 current snap is 317 2024-09-21T09:42:37.199 INFO:tasks.rados.rados.0.smithi022.stdout:3303: seq_num 1047 ranges {286720=745472} 2024-09-21T09:42:37.205 INFO:tasks.rados.rados.0.smithi022.stdout:3303: writing smithi02284960-43 from 286720 to 1032192 tid 1 2024-09-21T09:42:37.205 INFO:tasks.rados.rados.0.smithi022.stdout:3304: delete oid 14 current snap is 317 2024-09-21T09:42:37.205 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 33 v 1982 (ObjNum 914 snap 284 seq_num 914) dirty exists 2024-09-21T09:42:37.207 INFO:tasks.rados.rados.0.smithi022.stdout:3301: done (2 left) 2024-09-21T09:42:37.207 INFO:tasks.rados.rados.0.smithi022.stdout:3304: done (1 left) 2024-09-21T09:42:37.207 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 5 current snap is 317 2024-09-21T09:42:37.207 INFO:tasks.rados.rados.0.smithi022.stdout:3305: seq_num 1049 ranges {3072000=778240} 2024-09-21T09:42:37.228 INFO:tasks.rados.rados.0.smithi022.stdout:3305: writing smithi02284960-5 from 3072000 to 3850240 tid 1 2024-09-21T09:42:37.228 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 48 current snap is 317 2024-09-21T09:42:37.229 INFO:tasks.rados.rados.0.smithi022.stdout:3306: seq_num 1050 ranges {647168=794624} 2024-09-21T09:42:37.237 INFO:tasks.rados.rados.0.smithi022.stdout:3306: writing smithi02284960-48 from 647168 to 1441792 tid 1 2024-09-21T09:42:37.237 INFO:tasks.rados.rados.0.smithi022.stdout:3307: rollback oid 40 current snap is 317 2024-09-21T09:42:37.237 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 40 to 315 2024-09-21T09:42:37.237 INFO:tasks.rados.rados.0.smithi022.stdout:3303: finishing write tid 1 to smithi02284960-43 2024-09-21T09:42:37.237 INFO:tasks.rados.rados.0.smithi022.stdout:3303: finishing write tid 2 to smithi02284960-43 2024-09-21T09:42:37.237 INFO:tasks.rados.rados.0.smithi022.stdout:3308: delete oid 33 current snap is 317 2024-09-21T09:42:37.237 INFO:tasks.rados.rados.0.smithi022.stdout:3303: finishing write tid 3 to smithi02284960-43 2024-09-21T09:42:37.238 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 43 v 1884 (ObjNum 1047 snap 317 seq_num 1047) dirty exists 2024-09-21T09:42:37.238 INFO:tasks.rados.rados.0.smithi022.stdout:3303: left oid 43 (ObjNum 1047 snap 317 seq_num 1047) 2024-09-21T09:42:37.238 INFO:tasks.rados.rados.0.smithi022.stdout:3305: finishing write tid 1 to smithi02284960-5 2024-09-21T09:42:37.238 INFO:tasks.rados.rados.0.smithi022.stdout:3305: finishing write tid 2 to smithi02284960-5 2024-09-21T09:42:37.238 INFO:tasks.rados.rados.0.smithi022.stdout:3305: finishing write tid 3 to smithi02284960-5 2024-09-21T09:42:37.238 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 5 v 1821 (ObjNum 1049 snap 317 seq_num 1049) dirty exists 2024-09-21T09:42:37.238 INFO:tasks.rados.rados.0.smithi022.stdout:3305: left oid 5 (ObjNum 1049 snap 317 seq_num 1049) 2024-09-21T09:42:37.240 INFO:tasks.rados.rados.0.smithi022.stdout:3307: finishing rollback tid 0 to smithi02284960-40 2024-09-21T09:42:37.240 INFO:tasks.rados.rados.0.smithi022.stdout:3307: finishing rollback tid 1 to smithi02284960-40 2024-09-21T09:42:37.241 INFO:tasks.rados.rados.0.smithi022.stdout:3307: finishing rollback tid 2 to smithi02284960-40 2024-09-21T09:42:37.241 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 40 v 1887 (ObjNum 1035 snap 315 seq_num 1035) dirty exists 2024-09-21T09:42:37.241 INFO:tasks.rados.rados.0.smithi022.stdout:3306: finishing write tid 1 to smithi02284960-48 2024-09-21T09:42:37.242 INFO:tasks.rados.rados.0.smithi022.stdout:3306: finishing write tid 2 to smithi02284960-48 2024-09-21T09:42:37.242 INFO:tasks.rados.rados.0.smithi022.stdout:3306: finishing write tid 3 to smithi02284960-48 2024-09-21T09:42:37.243 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 48 v 1985 (ObjNum 1050 snap 317 seq_num 1050) dirty exists 2024-09-21T09:42:37.243 INFO:tasks.rados.rados.0.smithi022.stdout:3306: left oid 48 (ObjNum 1050 snap 317 seq_num 1050) 2024-09-21T09:42:37.243 INFO:tasks.rados.rados.0.smithi022.stdout:3303: done (4 left) 2024-09-21T09:42:37.243 INFO:tasks.rados.rados.0.smithi022.stdout:3305: done (3 left) 2024-09-21T09:42:37.243 INFO:tasks.rados.rados.0.smithi022.stdout:3306: done (2 left) 2024-09-21T09:42:37.243 INFO:tasks.rados.rados.0.smithi022.stdout:3307: done (1 left) 2024-09-21T09:42:37.243 INFO:tasks.rados.rados.0.smithi022.stdout:3308: done (0 left) 2024-09-21T09:42:37.243 INFO:tasks.rados.rados.0.smithi022.stdout:3309: delete oid 9 current snap is 317 2024-09-21T09:42:37.245 INFO:tasks.rados.rados.0.smithi022.stdout:3309: done (0 left) 2024-09-21T09:42:37.245 INFO:tasks.rados.rados.0.smithi022.stdout:3310: read oid 36 snap 315 2024-09-21T09:42:37.245 INFO:tasks.rados.rados.0.smithi022.stdout:3310: expect (ObjNum 1033 snap 315 seq_num 1033) 2024-09-21T09:42:37.245 INFO:tasks.rados.rados.0.smithi022.stdout:3311: rollback oid 37 current snap is 317 2024-09-21T09:42:37.245 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 37 to 315 2024-09-21T09:42:37.245 INFO:tasks.rados.rados.0.smithi022.stdout:3312: snap_remove snap 316 2024-09-21T09:42:37.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:42:37 smithi064 ceph-mon[86709]: osdmap e715: 8 total, 8 up, 8 in 2024-09-21T09:42:37.557 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:42:37 smithi022 ceph-mon[98615]: osdmap e715: 8 total, 8 up, 8 in 2024-09-21T09:42:37.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:42:37 smithi022 ceph-mon[96736]: osdmap e715: 8 total, 8 up, 8 in 2024-09-21T09:42:38.107 INFO:tasks.rados.rados.0.smithi022.stdout:3312: done (2 left) 2024-09-21T09:42:38.107 INFO:tasks.rados.rados.0.smithi022.stdout:3313: rmattr oid 14 current snap is 317 2024-09-21T09:42:38.107 INFO:tasks.rados.rados.0.smithi022.stdout:3313: done (2 left) 2024-09-21T09:42:38.107 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 13 current snap is 317 2024-09-21T09:42:38.107 INFO:tasks.rados.rados.0.smithi022.stdout:3314: seq_num 1053 ranges {0=442368,442368=401408} 2024-09-21T09:42:38.109 INFO:tasks.rados.rados.0.smithi022.stdout:3314: writing smithi02284960-13 from 0 to 442368 tid 1 2024-09-21T09:42:38.112 INFO:tasks.rados.rados.0.smithi022.stdout:3314: writing smithi02284960-13 from 442368 to 843776 tid 2 2024-09-21T09:42:38.112 INFO:tasks.rados.rados.0.smithi022.stdout:3315: copy_from oid 24 from oid 25 current snap is 317 2024-09-21T09:42:38.112 INFO:tasks.rados.rados.0.smithi022.stdout:3311: finishing rollback tid 0 to smithi02284960-37 2024-09-21T09:42:38.112 INFO:tasks.rados.rados.0.smithi022.stdout:3311: finishing rollback tid 1 to smithi02284960-37 2024-09-21T09:42:38.112 INFO:tasks.rados.rados.0.smithi022.stdout:3316: copy_from oid 11 from oid 19 current snap is 317 2024-09-21T09:42:38.112 INFO:tasks.rados.rados.0.smithi022.stdout:3311: finishing rollback tid 2 to smithi02284960-37 2024-09-21T09:42:38.112 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 37 v 1911 (ObjNum 1010 snap 312 seq_num 1010) dirty exists 2024-09-21T09:42:38.112 INFO:tasks.rados.rados.0.smithi022.stdout:3310: expect (ObjNum 1033 snap 315 seq_num 1033) 2024-09-21T09:42:38.176 INFO:tasks.rados.rados.0.smithi022.stdout:3316: finishing copy_from to smithi02284960-11 2024-09-21T09:42:38.176 INFO:tasks.rados.rados.0.smithi022.stdout:3316: got expected ENOENT (src dne) 2024-09-21T09:42:38.176 INFO:tasks.rados.rados.0.smithi022.stdout:3316: finishing copy_from racing read to smithi02284960-11 2024-09-21T09:42:38.176 INFO:tasks.rados.rados.0.smithi022.stdout:3314: finishing write tid 1 to smithi02284960-13 2024-09-21T09:42:38.177 INFO:tasks.rados.rados.0.smithi022.stdout:3315: finishing copy_from to smithi02284960-24 2024-09-21T09:42:38.177 INFO:tasks.rados.rados.0.smithi022.stdout:3315: got expected ENOENT (src dne) 2024-09-21T09:42:38.177 INFO:tasks.rados.rados.0.smithi022.stdout:3310: done (4 left) 2024-09-21T09:42:38.177 INFO:tasks.rados.rados.0.smithi022.stdout:3311: done (3 left) 2024-09-21T09:42:38.177 INFO:tasks.rados.rados.0.smithi022.stdout:3316: done (2 left) 2024-09-21T09:42:38.177 INFO:tasks.rados.rados.0.smithi022.stdout:3317: setattr oid 48 current snap is 317 2024-09-21T09:42:38.177 INFO:tasks.rados.rados.0.smithi022.stdout:3315: finishing copy_from racing read to smithi02284960-24 2024-09-21T09:42:38.177 INFO:tasks.rados.rados.0.smithi022.stdout:3314: finishing write tid 2 to smithi02284960-13 2024-09-21T09:42:38.177 INFO:tasks.rados.rados.0.smithi022.stdout:3314: finishing write tid 3 to smithi02284960-13 2024-09-21T09:42:38.177 INFO:tasks.rados.rados.0.smithi022.stdout:3314: finishing write tid 4 to smithi02284960-13 2024-09-21T09:42:38.177 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 13 v 1802 (ObjNum 1053 snap 317 seq_num 1053) dirty exists 2024-09-21T09:42:38.177 INFO:tasks.rados.rados.0.smithi022.stdout:3314: left oid 13 (ObjNum 1053 snap 317 seq_num 1053) 2024-09-21T09:42:38.178 INFO:tasks.rados.rados.0.smithi022.stdout:3314: done (2 left) 2024-09-21T09:42:38.178 INFO:tasks.rados.rados.0.smithi022.stdout:3315: done (1 left) 2024-09-21T09:42:38.178 INFO:tasks.rados.rados.0.smithi022.stdout:3318: delete oid 7 current snap is 317 2024-09-21T09:42:38.181 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 48 v 1991 (ObjNum 1050 snap 317 seq_num 1050) dirty exists 2024-09-21T09:42:38.181 INFO:tasks.rados.rados.0.smithi022.stdout:3317: done (1 left) 2024-09-21T09:42:38.182 INFO:tasks.rados.rados.0.smithi022.stdout:3318: done (0 left) 2024-09-21T09:42:38.182 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 30 current snap is 317 2024-09-21T09:42:38.182 INFO:tasks.rados.rados.0.smithi022.stdout:3319: seq_num 1055 ranges {3072000=475136,3547136=401408} 2024-09-21T09:42:38.201 INFO:tasks.rados.rados.0.smithi022.stdout:3319: writing smithi02284960-30 from 3072000 to 3547136 tid 1 2024-09-21T09:42:38.203 INFO:tasks.rados.rados.0.smithi022.stdout:3319: writing smithi02284960-30 from 3547136 to 3948544 tid 2 2024-09-21T09:42:38.203 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 7 current snap is 317 2024-09-21T09:42:38.203 INFO:tasks.rados.rados.0.smithi022.stdout:3320: seq_num 1056 ranges {0=491520,491520=401408} 2024-09-21T09:42:38.206 INFO:tasks.rados.rados.0.smithi022.stdout:3320: writing smithi02284960-7 from 0 to 491520 tid 1 2024-09-21T09:42:38.209 INFO:tasks.rados.rados.0.smithi022.stdout:3320: writing smithi02284960-7 from 491520 to 892928 tid 2 2024-09-21T09:42:38.209 INFO:tasks.rados.rados.0.smithi022.stdout:3321: setattr oid 36 current snap is 317 2024-09-21T09:42:38.209 INFO:tasks.rados.rados.0.smithi022.stdout:3319: finishing write tid 1 to smithi02284960-30 2024-09-21T09:42:38.209 INFO:tasks.rados.rados.0.smithi022.stdout:3319: finishing write tid 2 to smithi02284960-30 2024-09-21T09:42:38.209 INFO:tasks.rados.rados.0.smithi022.stdout:3319: finishing write tid 3 to smithi02284960-30 2024-09-21T09:42:38.210 INFO:tasks.rados.rados.0.smithi022.stdout:3319: finishing write tid 4 to smithi02284960-30 2024-09-21T09:42:38.210 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 30 v 1348 (ObjNum 1055 snap 317 seq_num 1055) dirty exists 2024-09-21T09:42:38.210 INFO:tasks.rados.rados.0.smithi022.stdout:3319: left oid 30 (ObjNum 1055 snap 317 seq_num 1055) 2024-09-21T09:42:38.211 INFO:tasks.rados.rados.0.smithi022.stdout:3320: finishing write tid 1 to smithi02284960-7 2024-09-21T09:42:38.211 INFO:tasks.rados.rados.0.smithi022.stdout:3319: done (2 left) 2024-09-21T09:42:38.211 INFO:tasks.rados.rados.0.smithi022.stdout:3322: rollback oid 14 current snap is 317 2024-09-21T09:42:38.211 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 14 to 315 2024-09-21T09:42:38.212 INFO:tasks.rados.rados.0.smithi022.stdout:3323: setattr oid 22 current snap is 317 2024-09-21T09:42:38.212 INFO:tasks.rados.rados.0.smithi022.stdout:3320: finishing write tid 2 to smithi02284960-7 2024-09-21T09:42:38.213 INFO:tasks.rados.rados.0.smithi022.stdout:3320: finishing write tid 3 to smithi02284960-7 2024-09-21T09:42:38.214 INFO:tasks.rados.rados.0.smithi022.stdout:3322: finishing rollback tid 1 to smithi02284960-14 2024-09-21T09:42:38.214 INFO:tasks.rados.rados.0.smithi022.stdout:3324: snap_create 2024-09-21T09:42:38.214 INFO:tasks.rados.rados.0.smithi022.stdout:3322: finishing rollback tid 2 to smithi02284960-14 2024-09-21T09:42:38.214 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 14 v 1804 (ObjNum 985 snap 305 seq_num 985) dirty exists 2024-09-21T09:42:38.215 INFO:tasks.rados.rados.0.smithi022.stdout:3320: finishing write tid 4 to smithi02284960-7 2024-09-21T09:42:38.215 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 7 v 1996 (ObjNum 1056 snap 317 seq_num 1056) dirty exists 2024-09-21T09:42:38.215 INFO:tasks.rados.rados.0.smithi022.stdout:3320: left oid 7 (ObjNum 1056 snap 317 seq_num 1056) 2024-09-21T09:42:38.215 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 36 v 1432 (ObjNum 1033 snap 315 seq_num 1033) dirty exists 2024-09-21T09:42:38.217 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 22 v 1803 (ObjNum 914 snap 284 seq_num 914) dirty exists 2024-09-21T09:42:38.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:42:38 smithi064 ceph-mon[86709]: pgmap v686: 121 pgs: 121 active+clean; 140 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 2.9 MiB/s wr, 22 op/s 2024-09-21T09:42:38.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:42:38 smithi064 ceph-mon[86709]: osdmap e716: 8 total, 8 up, 8 in 2024-09-21T09:42:38.557 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:42:38 smithi022 ceph-mon[98615]: pgmap v686: 121 pgs: 121 active+clean; 140 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 2.9 MiB/s wr, 22 op/s 2024-09-21T09:42:38.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:42:38 smithi022 ceph-mon[98615]: osdmap e716: 8 total, 8 up, 8 in 2024-09-21T09:42:38.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:42:38 smithi022 ceph-mon[96736]: pgmap v686: 121 pgs: 121 active+clean; 140 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 2.9 MiB/s wr, 22 op/s 2024-09-21T09:42:38.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:42:38 smithi022 ceph-mon[96736]: osdmap e716: 8 total, 8 up, 8 in 2024-09-21T09:42:39.068 INFO:tasks.rados.rados.0.smithi022.stdout:3320: done (4 left) 2024-09-21T09:42:39.068 INFO:tasks.rados.rados.0.smithi022.stdout:3321: done (3 left) 2024-09-21T09:42:39.068 INFO:tasks.rados.rados.0.smithi022.stdout:3322: done (2 left) 2024-09-21T09:42:39.068 INFO:tasks.rados.rados.0.smithi022.stdout:3323: done (1 left) 2024-09-21T09:42:39.068 INFO:tasks.rados.rados.0.smithi022.stdout:3324: done (0 left) 2024-09-21T09:42:39.068 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 45 current snap is 318 2024-09-21T09:42:39.069 INFO:tasks.rados.rados.0.smithi022.stdout:3325: seq_num 1057 ranges {0=507904,507904=401408} 2024-09-21T09:42:39.071 INFO:tasks.rados.rados.0.smithi022.stdout:3325: writing smithi02284960-45 from 0 to 507904 tid 1 2024-09-21T09:42:39.073 INFO:tasks.rados.rados.0.smithi022.stdout:3325: writing smithi02284960-45 from 507904 to 909312 tid 2 2024-09-21T09:42:39.073 INFO:tasks.rados.rados.0.smithi022.stdout:3326: read oid 41 snap 317 2024-09-21T09:42:39.073 INFO:tasks.rados.rados.0.smithi022.stdout:3326: expect (ObjNum 983 snap 304 seq_num 983) 2024-09-21T09:42:39.074 INFO:tasks.rados.rados.0.smithi022.stdout:3327: delete oid 6 current snap is 318 2024-09-21T09:42:39.076 INFO:tasks.rados.rados.0.smithi022.stdout:3325: finishing write tid 1 to smithi02284960-45 2024-09-21T09:42:39.077 INFO:tasks.rados.rados.0.smithi022.stdout:3325: finishing write tid 2 to smithi02284960-45 2024-09-21T09:42:39.078 INFO:tasks.rados.rados.0.smithi022.stdout:3325: finishing write tid 3 to smithi02284960-45 2024-09-21T09:42:39.080 INFO:tasks.rados.rados.0.smithi022.stdout:3327: done (2 left) 2024-09-21T09:42:39.080 INFO:tasks.rados.rados.0.smithi022.stdout:3328: copy_from oid 5 from oid 32 current snap is 318 2024-09-21T09:42:39.080 INFO:tasks.rados.rados.0.smithi022.stdout:3325: finishing write tid 4 to smithi02284960-45 2024-09-21T09:42:39.081 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 45 v 1435 (ObjNum 1057 snap 318 seq_num 1057) dirty exists 2024-09-21T09:42:39.081 INFO:tasks.rados.rados.0.smithi022.stdout:3325: left oid 45 (ObjNum 1057 snap 318 seq_num 1057) 2024-09-21T09:42:39.081 INFO:tasks.rados.rados.0.smithi022.stdout:3325: done (2 left) 2024-09-21T09:42:39.081 INFO:tasks.rados.rados.0.smithi022.stdout:3329: setattr oid 15 current snap is 318 2024-09-21T09:42:39.083 INFO:tasks.rados.rados.0.smithi022.stdout:3330: read oid 38 snap 315 2024-09-21T09:42:39.083 INFO:tasks.rados.rados.0.smithi022.stdout:3330: expect (ObjNum 971 snap 296 seq_num 971) 2024-09-21T09:42:39.083 INFO:tasks.rados.rados.0.smithi022.stdout:3331: setattr oid 19 current snap is 318 2024-09-21T09:42:39.086 INFO:tasks.rados.rados.0.smithi022.stdout:3332: read oid 37 snap 317 2024-09-21T09:42:39.087 INFO:tasks.rados.rados.0.smithi022.stdout:3332: expect (ObjNum 1010 snap 312 seq_num 1010) 2024-09-21T09:42:39.087 INFO:tasks.rados.rados.0.smithi022.stdout:3333: delete oid 10 current snap is 318 2024-09-21T09:42:39.088 INFO:tasks.rados.rados.0.smithi022.stdout:3326: expect (ObjNum 983 snap 304 seq_num 983) 2024-09-21T09:42:39.148 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 15 v 1915 (ObjNum 996 snap 305 seq_num 996) dirty exists 2024-09-21T09:42:39.148 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 19 v 1916 (ObjNum 65 snap 0 seq_num 3569398000) dirty exists 2024-09-21T09:42:39.149 INFO:tasks.rados.rados.0.smithi022.stdout:3332: expect (ObjNum 1010 snap 312 seq_num 1010) 2024-09-21T09:42:39.152 INFO:tasks.rados.rados.0.smithi022.stdout:3330: expect (ObjNum 971 snap 296 seq_num 971) 2024-09-21T09:42:39.201 INFO:tasks.rados.rados.0.smithi022.stdout:3328: finishing copy_from racing read to smithi02284960-5 2024-09-21T09:42:39.201 INFO:tasks.rados.rados.0.smithi022.stdout:3328: finishing copy_from to smithi02284960-5 2024-09-21T09:42:39.201 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 5 v 1948 (ObjNum 884 snap 282 seq_num 884) dirty exists 2024-09-21T09:42:39.201 INFO:tasks.rados.rados.0.smithi022.stdout:3326: done (6 left) 2024-09-21T09:42:39.201 INFO:tasks.rados.rados.0.smithi022.stdout:3328: done (5 left) 2024-09-21T09:42:39.202 INFO:tasks.rados.rados.0.smithi022.stdout:3329: done (4 left) 2024-09-21T09:42:39.202 INFO:tasks.rados.rados.0.smithi022.stdout:3330: done (3 left) 2024-09-21T09:42:39.202 INFO:tasks.rados.rados.0.smithi022.stdout:3331: done (2 left) 2024-09-21T09:42:39.202 INFO:tasks.rados.rados.0.smithi022.stdout:3332: done (1 left) 2024-09-21T09:42:39.202 INFO:tasks.rados.rados.0.smithi022.stdout:3333: done (0 left) 2024-09-21T09:42:39.202 INFO:tasks.rados.rados.0.smithi022.stdout:3334: read oid 6 snap -1 2024-09-21T09:42:39.202 INFO:tasks.rados.rados.0.smithi022.stdout:3334: expect deleted 2024-09-21T09:42:39.202 INFO:tasks.rados.rados.0.smithi022.stdout:3335: rollback oid 47 current snap is 318 2024-09-21T09:42:39.202 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 47 to 315 2024-09-21T09:42:39.202 INFO:tasks.rados.rados.0.smithi022.stdout:3336: snap_create 2024-09-21T09:42:39.204 INFO:tasks.rados.rados.0.smithi022.stdout:3335: finishing rollback tid 1 to smithi02284960-47 2024-09-21T09:42:39.204 INFO:tasks.rados.rados.0.smithi022.stdout:3335: finishing rollback tid 2 to smithi02284960-47 2024-09-21T09:42:39.204 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 47 v 1747 (ObjNum 910 snap 284 seq_num 910) dirty exists 2024-09-21T09:42:39.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:42:39 smithi064 ceph-mon[86709]: osdmap e717: 8 total, 8 up, 8 in 2024-09-21T09:42:39.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:42:39 smithi064 ceph-mon[86709]: osdmap e718: 8 total, 8 up, 8 in 2024-09-21T09:42:39.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:42:39 smithi022 ceph-mon[98615]: osdmap e717: 8 total, 8 up, 8 in 2024-09-21T09:42:39.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:42:39 smithi022 ceph-mon[98615]: osdmap e718: 8 total, 8 up, 8 in 2024-09-21T09:42:39.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:42:39 smithi022 ceph-mon[96736]: osdmap e717: 8 total, 8 up, 8 in 2024-09-21T09:42:39.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:42:39 smithi022 ceph-mon[96736]: osdmap e718: 8 total, 8 up, 8 in 2024-09-21T09:42:40.116 INFO:tasks.rados.rados.0.smithi022.stdout:3334: done (2 left) 2024-09-21T09:42:40.116 INFO:tasks.rados.rados.0.smithi022.stdout:3335: done (1 left) 2024-09-21T09:42:40.116 INFO:tasks.rados.rados.0.smithi022.stdout:3336: done (0 left) 2024-09-21T09:42:40.116 INFO:tasks.rados.rados.0.smithi022.stdout:3337: rollback oid 24 current snap is 319 2024-09-21T09:42:40.116 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 24 to 318 2024-09-21T09:42:40.116 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 22 current snap is 319 2024-09-21T09:42:40.116 INFO:tasks.rados.rados.0.smithi022.stdout:3338: seq_num 1060 ranges {1564672=557056,2121728=401408} 2024-09-21T09:42:40.127 INFO:tasks.rados.rados.0.smithi022.stdout:3338: writing smithi02284960-22 from 1564672 to 2121728 tid 1 2024-09-21T09:42:40.130 INFO:tasks.rados.rados.0.smithi022.stdout:3338: writing smithi02284960-22 from 2121728 to 2523136 tid 2 2024-09-21T09:42:40.130 INFO:tasks.rados.rados.0.smithi022.stdout:3339: delete oid 47 current snap is 319 2024-09-21T09:42:40.130 INFO:tasks.rados.rados.0.smithi022.stdout:3337: finishing rollback tid 0 to smithi02284960-24 2024-09-21T09:42:40.130 INFO:tasks.rados.rados.0.smithi022.stdout:3337: finishing rollback tid 1 to smithi02284960-24 2024-09-21T09:42:40.130 INFO:tasks.rados.rados.0.smithi022.stdout:3337: finishing rollback tid 2 to smithi02284960-24 2024-09-21T09:42:40.131 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 24 v 1920 (ObjNum 1041 snap 315 seq_num 1041) dirty exists 2024-09-21T09:42:40.133 INFO:tasks.rados.rados.0.smithi022.stdout:3337: done (2 left) 2024-09-21T09:42:40.133 INFO:tasks.rados.rados.0.smithi022.stdout:3339: done (1 left) 2024-09-21T09:42:40.133 INFO:tasks.rados.rados.0.smithi022.stdout:3340: copy_from oid 42 from oid 30 current snap is 319 2024-09-21T09:42:40.134 INFO:tasks.rados.rados.0.smithi022.stdout:3341: setattr oid 16 current snap is 319 2024-09-21T09:42:40.135 INFO:tasks.rados.rados.0.smithi022.stdout:3342: rollback oid 43 current snap is 319 2024-09-21T09:42:40.135 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 43 to 317 2024-09-21T09:42:40.135 INFO:tasks.rados.rados.0.smithi022.stdout:3343: rmattr oid 25 current snap is 319 2024-09-21T09:42:40.135 INFO:tasks.rados.rados.0.smithi022.stdout:3343: done (4 left) 2024-09-21T09:42:40.135 INFO:tasks.rados.rados.0.smithi022.stdout:3344: rollback oid 15 current snap is 319 2024-09-21T09:42:40.136 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 15 to 315 2024-09-21T09:42:40.136 INFO:tasks.rados.rados.0.smithi022.stdout:3345: copy_from oid 46 from oid 9 current snap is 319 2024-09-21T09:42:40.136 INFO:tasks.rados.rados.0.smithi022.stdout:3346: rollback oid 21 current snap is 319 2024-09-21T09:42:40.136 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 21 to 315 2024-09-21T09:42:40.136 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 11 current snap is 319 2024-09-21T09:42:40.136 INFO:tasks.rados.rados.0.smithi022.stdout:3347: seq_num 1062 ranges {0=589824,589824=401408} 2024-09-21T09:42:40.139 INFO:tasks.rados.rados.0.smithi022.stdout:3347: writing smithi02284960-11 from 0 to 589824 tid 1 2024-09-21T09:42:40.146 INFO:tasks.rados.rados.0.smithi022.stdout:3347: writing smithi02284960-11 from 589824 to 991232 tid 2 2024-09-21T09:42:40.146 INFO:tasks.rados.rados.0.smithi022.stdout:3348: read oid 23 snap 315 2024-09-21T09:42:40.146 INFO:tasks.rados.rados.0.smithi022.stdout:3348: expect (ObjNum 1025 snap 315 seq_num 1025) 2024-09-21T09:42:40.146 INFO:tasks.rados.rados.0.smithi022.stdout:3349: read oid 44 snap -1 2024-09-21T09:42:40.147 INFO:tasks.rados.rados.0.smithi022.stdout:3349: expect (ObjNum 1002 snap 307 seq_num 1002) 2024-09-21T09:42:40.147 INFO:tasks.rados.rados.0.smithi022.stdout:3350: delete oid 34 current snap is 319 2024-09-21T09:42:40.147 INFO:tasks.rados.rados.0.smithi022.stdout:3345: finishing copy_from to smithi02284960-46 2024-09-21T09:42:40.147 INFO:tasks.rados.rados.0.smithi022.stdout:3345: got expected ENOENT (src dne) 2024-09-21T09:42:40.147 INFO:tasks.rados.rados.0.smithi022.stdout:3338: finishing write tid 1 to smithi02284960-22 2024-09-21T09:42:40.147 INFO:tasks.rados.rados.0.smithi022.stdout:3345: finishing copy_from racing read to smithi02284960-46 2024-09-21T09:42:40.148 INFO:tasks.rados.rados.0.smithi022.stdout:3338: finishing write tid 2 to smithi02284960-22 2024-09-21T09:42:40.148 INFO:tasks.rados.rados.0.smithi022.stdout:3338: finishing write tid 3 to smithi02284960-22 2024-09-21T09:42:40.148 INFO:tasks.rados.rados.0.smithi022.stdout:3338: finishing write tid 4 to smithi02284960-22 2024-09-21T09:42:40.148 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 22 v 1806 (ObjNum 1060 snap 319 seq_num 1060) dirty exists 2024-09-21T09:42:40.148 INFO:tasks.rados.rados.0.smithi022.stdout:3338: left oid 22 (ObjNum 1060 snap 319 seq_num 1060) 2024-09-21T09:42:40.148 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 16 v 1528 (ObjNum 1023 snap 315 seq_num 1023) dirty exists 2024-09-21T09:42:40.148 INFO:tasks.rados.rados.0.smithi022.stdout:3342: finishing rollback tid 0 to smithi02284960-43 2024-09-21T09:42:40.148 INFO:tasks.rados.rados.0.smithi022.stdout:3344: finishing rollback tid 0 to smithi02284960-15 2024-09-21T09:42:40.148 INFO:tasks.rados.rados.0.smithi022.stdout:3342: finishing rollback tid 1 to smithi02284960-43 2024-09-21T09:42:40.148 INFO:tasks.rados.rados.0.smithi022.stdout:3346: finishing rollback tid 0 to smithi02284960-21 2024-09-21T09:42:40.148 INFO:tasks.rados.rados.0.smithi022.stdout:3344: finishing rollback tid 1 to smithi02284960-15 2024-09-21T09:42:40.148 INFO:tasks.rados.rados.0.smithi022.stdout:3342: finishing rollback tid 2 to smithi02284960-43 2024-09-21T09:42:40.148 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 43 v 1890 (ObjNum 1047 snap 317 seq_num 1047) dirty exists 2024-09-21T09:42:40.148 INFO:tasks.rados.rados.0.smithi022.stdout:3344: finishing rollback tid 2 to smithi02284960-15 2024-09-21T09:42:40.148 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 15 v 1924 (ObjNum 996 snap 305 seq_num 996) dirty exists 2024-09-21T09:42:40.149 INFO:tasks.rados.rados.0.smithi022.stdout:3346: finishing rollback tid 1 to smithi02284960-21 2024-09-21T09:42:40.149 INFO:tasks.rados.rados.0.smithi022.stdout:3346: finishing rollback tid 2 to smithi02284960-21 2024-09-21T09:42:40.149 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 21 v 1815 (ObjNum 1029 snap 315 seq_num 1029) dirty exists 2024-09-21T09:42:40.149 INFO:tasks.rados.rados.0.smithi022.stdout:3338: done (10 left) 2024-09-21T09:42:40.149 INFO:tasks.rados.rados.0.smithi022.stdout:3341: done (9 left) 2024-09-21T09:42:40.150 INFO:tasks.rados.rados.0.smithi022.stdout:3342: done (8 left) 2024-09-21T09:42:40.150 INFO:tasks.rados.rados.0.smithi022.stdout:3344: done (7 left) 2024-09-21T09:42:40.150 INFO:tasks.rados.rados.0.smithi022.stdout:3345: done (6 left) 2024-09-21T09:42:40.150 INFO:tasks.rados.rados.0.smithi022.stdout:3346: done (5 left) 2024-09-21T09:42:40.150 INFO:tasks.rados.rados.0.smithi022.stdout:3350: done (4 left) 2024-09-21T09:42:40.150 INFO:tasks.rados.rados.0.smithi022.stdout:3351: delete oid 43 current snap is 319 2024-09-21T09:42:40.150 INFO:tasks.rados.rados.0.smithi022.stdout:3347: finishing write tid 1 to smithi02284960-11 2024-09-21T09:42:40.150 INFO:tasks.rados.rados.0.smithi022.stdout:3348: expect (ObjNum 1025 snap 315 seq_num 1025) 2024-09-21T09:42:40.173 INFO:tasks.rados.rados.0.smithi022.stdout:3347: finishing write tid 2 to smithi02284960-11 2024-09-21T09:42:40.173 INFO:tasks.rados.rados.0.smithi022.stdout:3347: finishing write tid 3 to smithi02284960-11 2024-09-21T09:42:40.173 INFO:tasks.rados.rados.0.smithi022.stdout:3347: finishing write tid 4 to smithi02284960-11 2024-09-21T09:42:40.173 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 11 v 1752 (ObjNum 1062 snap 319 seq_num 1062) dirty exists 2024-09-21T09:42:40.173 INFO:tasks.rados.rados.0.smithi022.stdout:3347: left oid 11 (ObjNum 1062 snap 319 seq_num 1062) 2024-09-21T09:42:40.173 INFO:tasks.rados.rados.0.smithi022.stdout:3349: expect (ObjNum 1002 snap 307 seq_num 1002) 2024-09-21T09:42:40.256 INFO:tasks.rados.rados.0.smithi022.stdout:3340: finishing copy_from racing read to smithi02284960-42 2024-09-21T09:42:40.256 INFO:tasks.rados.rados.0.smithi022.stdout:3340: finishing copy_from to smithi02284960-42 2024-09-21T09:42:40.256 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 42 v 1348 (ObjNum 1055 snap 317 seq_num 1055) dirty exists 2024-09-21T09:42:40.256 INFO:tasks.rados.rados.0.smithi022.stdout:3340: done (4 left) 2024-09-21T09:42:40.256 INFO:tasks.rados.rados.0.smithi022.stdout:3347: done (3 left) 2024-09-21T09:42:40.256 INFO:tasks.rados.rados.0.smithi022.stdout:3348: done (2 left) 2024-09-21T09:42:40.256 INFO:tasks.rados.rados.0.smithi022.stdout:3349: done (1 left) 2024-09-21T09:42:40.256 INFO:tasks.rados.rados.0.smithi022.stdout:3351: done (0 left) 2024-09-21T09:42:40.256 INFO:tasks.rados.rados.0.smithi022.stdout:3352: read oid 45 snap -1 2024-09-21T09:42:40.256 INFO:tasks.rados.rados.0.smithi022.stdout:3352: expect (ObjNum 1057 snap 318 seq_num 1057) 2024-09-21T09:42:40.256 INFO:tasks.rados.rados.0.smithi022.stdout:3353: copy_from oid 44 from oid 31 current snap is 319 2024-09-21T09:42:40.256 INFO:tasks.rados.rados.0.smithi022.stdout:3354: rollback oid 4 current snap is 319 2024-09-21T09:42:40.256 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 4 to 317 2024-09-21T09:42:40.257 INFO:tasks.rados.rados.0.smithi022.stdout:3355: snap_create 2024-09-21T09:42:40.257 INFO:tasks.rados.rados.0.smithi022.stdout:3353: finishing copy_from to smithi02284960-44 2024-09-21T09:42:40.257 INFO:tasks.rados.rados.0.smithi022.stdout:3353: got expected ENOENT (src dne) 2024-09-21T09:42:40.257 INFO:tasks.rados.rados.0.smithi022.stdout:3353: finishing copy_from racing read to smithi02284960-44 2024-09-21T09:42:40.257 INFO:tasks.rados.rados.0.smithi022.stdout:3354: finishing rollback tid 1 to smithi02284960-4 2024-09-21T09:42:40.258 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 4 v 1819 (ObjNum 161 snap 0 seq_num 3569895536) dirty dne 2024-09-21T09:42:40.259 INFO:tasks.rados.rados.0.smithi022.stdout:3352: expect (ObjNum 1057 snap 318 seq_num 1057) 2024-09-21T09:42:40.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:42:40 smithi064 ceph-mon[86709]: pgmap v689: 121 pgs: 121 active+clean; 105 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 8.1 MiB/s rd, 4.0 MiB/s wr, 25 op/s 2024-09-21T09:42:40.442 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:42:40 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:42:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:42:40.557 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:42:40 smithi022 ceph-mon[98615]: pgmap v689: 121 pgs: 121 active+clean; 105 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 8.1 MiB/s rd, 4.0 MiB/s wr, 25 op/s 2024-09-21T09:42:40.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:42:40 smithi022 ceph-mon[96736]: pgmap v689: 121 pgs: 121 active+clean; 105 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 8.1 MiB/s rd, 4.0 MiB/s wr, 25 op/s 2024-09-21T09:42:41.122 INFO:tasks.rados.rados.0.smithi022.stdout:3352: done (3 left) 2024-09-21T09:42:41.122 INFO:tasks.rados.rados.0.smithi022.stdout:3353: done (2 left) 2024-09-21T09:42:41.122 INFO:tasks.rados.rados.0.smithi022.stdout:3354: done (1 left) 2024-09-21T09:42:41.122 INFO:tasks.rados.rados.0.smithi022.stdout:3355: done (0 left) 2024-09-21T09:42:41.122 INFO:tasks.rados.rados.0.smithi022.stdout:3356: snap_create 2024-09-21T09:42:41.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:42:41 smithi064 ceph-mon[86709]: osdmap e719: 8 total, 8 up, 8 in 2024-09-21T09:42:41.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:42:41 smithi064 ceph-mon[86709]: pgmap v692: 121 pgs: 121 active+clean; 109 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 7.3 MiB/s rd, 3.4 MiB/s wr, 34 op/s 2024-09-21T09:42:41.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:42:41 smithi022 ceph-mon[96736]: osdmap e719: 8 total, 8 up, 8 in 2024-09-21T09:42:41.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:42:41 smithi022 ceph-mon[96736]: pgmap v692: 121 pgs: 121 active+clean; 109 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 7.3 MiB/s rd, 3.4 MiB/s wr, 34 op/s 2024-09-21T09:42:41.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:42:41 smithi022 ceph-mon[98615]: osdmap e719: 8 total, 8 up, 8 in 2024-09-21T09:42:41.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:42:41 smithi022 ceph-mon[98615]: pgmap v692: 121 pgs: 121 active+clean; 109 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 7.3 MiB/s rd, 3.4 MiB/s wr, 34 op/s 2024-09-21T09:42:42.123 INFO:tasks.rados.rados.0.smithi022.stdout:3356: done (0 left) 2024-09-21T09:42:42.123 INFO:tasks.rados.rados.0.smithi022.stdout:3357: delete oid 22 current snap is 321 2024-09-21T09:42:42.131 INFO:tasks.rados.rados.0.smithi022.stdout:3357: done (0 left) 2024-09-21T09:42:42.131 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 47 current snap is 321 2024-09-21T09:42:42.131 INFO:tasks.rados.rados.0.smithi022.stdout:3358: seq_num 1066 ranges {0=663552,663552=401408} 2024-09-21T09:42:42.135 INFO:tasks.rados.rados.0.smithi022.stdout:3358: writing smithi02284960-47 from 0 to 663552 tid 1 2024-09-21T09:42:42.137 INFO:tasks.rados.rados.0.smithi022.stdout:3358: writing smithi02284960-47 from 663552 to 1064960 tid 2 2024-09-21T09:42:42.138 INFO:tasks.rados.rados.0.smithi022.stdout:3359: setattr oid 7 current snap is 321 2024-09-21T09:42:42.139 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 2 current snap is 321 2024-09-21T09:42:42.139 INFO:tasks.rados.rados.0.smithi022.stdout:3360: seq_num 1067 ranges {901120=679936,1581056=401408} 2024-09-21T09:42:42.148 INFO:tasks.rados.rados.0.smithi022.stdout:3360: writing smithi02284960-2 from 901120 to 1581056 tid 1 2024-09-21T09:42:42.150 INFO:tasks.rados.rados.0.smithi022.stdout:3360: writing smithi02284960-2 from 1581056 to 1982464 tid 2 2024-09-21T09:42:42.150 INFO:tasks.rados.rados.0.smithi022.stdout:3361: rmattr oid 28 current snap is 321 2024-09-21T09:42:42.151 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 7 v 2000 (ObjNum 1056 snap 317 seq_num 1056) dirty exists 2024-09-21T09:42:42.151 INFO:tasks.rados.rados.0.smithi022.stdout:3359: done (3 left) 2024-09-21T09:42:42.151 INFO:tasks.rados.rados.0.smithi022.stdout:3362: snap_create 2024-09-21T09:42:42.151 INFO:tasks.rados.rados.0.smithi022.stdout:3358: finishing write tid 1 to smithi02284960-47 2024-09-21T09:42:42.151 INFO:tasks.rados.rados.0.smithi022.stdout:3358: finishing write tid 2 to smithi02284960-47 2024-09-21T09:42:42.151 INFO:tasks.rados.rados.0.smithi022.stdout:3358: finishing write tid 3 to smithi02284960-47 2024-09-21T09:42:42.151 INFO:tasks.rados.rados.0.smithi022.stdout:3358: finishing write tid 4 to smithi02284960-47 2024-09-21T09:42:42.151 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 47 v 1755 (ObjNum 1066 snap 321 seq_num 1066) dirty exists 2024-09-21T09:42:42.151 INFO:tasks.rados.rados.0.smithi022.stdout:3358: left oid 47 (ObjNum 1066 snap 321 seq_num 1066) 2024-09-21T09:42:42.153 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 28 v 1797 (ObjNum 827 snap 264 seq_num 827) dirty exists 2024-09-21T09:42:42.154 INFO:tasks.rados.rados.0.smithi022.stdout:3360: finishing write tid 1 to smithi02284960-2 2024-09-21T09:42:42.154 INFO:tasks.rados.rados.0.smithi022.stdout:3360: finishing write tid 2 to smithi02284960-2 2024-09-21T09:42:42.155 INFO:tasks.rados.rados.0.smithi022.stdout:3360: finishing write tid 3 to smithi02284960-2 2024-09-21T09:42:42.156 INFO:tasks.rados.rados.0.smithi022.stdout:3360: finishing write tid 4 to smithi02284960-2 2024-09-21T09:42:42.157 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 2 v 1819 (ObjNum 1067 snap 321 seq_num 1067) dirty exists 2024-09-21T09:42:42.157 INFO:tasks.rados.rados.0.smithi022.stdout:3360: left oid 2 (ObjNum 1067 snap 321 seq_num 1067) 2024-09-21T09:42:42.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:42:42 smithi064 ceph-mon[86709]: osdmap e720: 8 total, 8 up, 8 in 2024-09-21T09:42:42.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:42:42 smithi022 ceph-mon[98615]: osdmap e720: 8 total, 8 up, 8 in 2024-09-21T09:42:42.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:42:42 smithi022 ceph-mon[96736]: osdmap e720: 8 total, 8 up, 8 in 2024-09-21T09:42:43.136 INFO:tasks.rados.rados.0.smithi022.stdout:3358: done (3 left) 2024-09-21T09:42:43.136 INFO:tasks.rados.rados.0.smithi022.stdout:3360: done (2 left) 2024-09-21T09:42:43.136 INFO:tasks.rados.rados.0.smithi022.stdout:3361: done (1 left) 2024-09-21T09:42:43.136 INFO:tasks.rados.rados.0.smithi022.stdout:3362: done (0 left) 2024-09-21T09:42:43.136 INFO:tasks.rados.rados.0.smithi022.stdout:3363: copy_from oid 26 from oid 49 current snap is 322 2024-09-21T09:42:43.136 INFO:tasks.rados.rados.0.smithi022.stdout:3364: setattr oid 24 current snap is 322 2024-09-21T09:42:43.138 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 16 current snap is 322 2024-09-21T09:42:43.138 INFO:tasks.rados.rados.0.smithi022.stdout:3365: seq_num 1068 ranges {5021696=696320,5718016=401408} 2024-09-21T09:42:43.170 INFO:tasks.rados.rados.0.smithi022.stdout:3365: writing smithi02284960-16 from 5021696 to 5718016 tid 1 2024-09-21T09:42:43.172 INFO:tasks.rados.rados.0.smithi022.stdout:3365: writing smithi02284960-16 from 5718016 to 6119424 tid 2 2024-09-21T09:42:43.172 INFO:tasks.rados.rados.0.smithi022.stdout:3366: delete oid 41 current snap is 322 2024-09-21T09:42:43.172 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 24 v 1926 (ObjNum 1041 snap 315 seq_num 1041) dirty exists 2024-09-21T09:42:43.172 INFO:tasks.rados.rados.0.smithi022.stdout:3363: finishing copy_from racing read to smithi02284960-26 2024-09-21T09:42:43.172 INFO:tasks.rados.rados.0.smithi022.stdout:3363: finishing copy_from to smithi02284960-26 2024-09-21T09:42:43.173 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 26 v 1801 (ObjNum 1036 snap 315 seq_num 1036) dirty exists 2024-09-21T09:42:43.175 INFO:tasks.rados.rados.0.smithi022.stdout:3363: done (3 left) 2024-09-21T09:42:43.175 INFO:tasks.rados.rados.0.smithi022.stdout:3364: done (2 left) 2024-09-21T09:42:43.175 INFO:tasks.rados.rados.0.smithi022.stdout:3366: done (1 left) 2024-09-21T09:42:43.175 INFO:tasks.rados.rados.0.smithi022.stdout:3367: rmattr oid 10 current snap is 322 2024-09-21T09:42:43.175 INFO:tasks.rados.rados.0.smithi022.stdout:3367: done (1 left) 2024-09-21T09:42:43.176 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 5 current snap is 322 2024-09-21T09:42:43.176 INFO:tasks.rados.rados.0.smithi022.stdout:3368: seq_num 1070 ranges {3702784=729088,4431872=401408} 2024-09-21T09:42:43.199 INFO:tasks.rados.rados.0.smithi022.stdout:3368: writing smithi02284960-5 from 3702784 to 4431872 tid 1 2024-09-21T09:42:43.202 INFO:tasks.rados.rados.0.smithi022.stdout:3368: writing smithi02284960-5 from 4431872 to 4833280 tid 2 2024-09-21T09:42:43.202 INFO:tasks.rados.rados.0.smithi022.stdout:3369: snap_create 2024-09-21T09:42:43.202 INFO:tasks.rados.rados.0.smithi022.stdout:3365: finishing write tid 1 to smithi02284960-16 2024-09-21T09:42:43.202 INFO:tasks.rados.rados.0.smithi022.stdout:3365: finishing write tid 2 to smithi02284960-16 2024-09-21T09:42:43.202 INFO:tasks.rados.rados.0.smithi022.stdout:3365: finishing write tid 3 to smithi02284960-16 2024-09-21T09:42:43.203 INFO:tasks.rados.rados.0.smithi022.stdout:3365: finishing write tid 4 to smithi02284960-16 2024-09-21T09:42:43.203 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 16 v 1531 (ObjNum 1068 snap 322 seq_num 1068) dirty exists 2024-09-21T09:42:43.203 INFO:tasks.rados.rados.0.smithi022.stdout:3365: left oid 16 (ObjNum 1068 snap 322 seq_num 1068) 2024-09-21T09:42:43.206 INFO:tasks.rados.rados.0.smithi022.stdout:3368: finishing write tid 1 to smithi02284960-5 2024-09-21T09:42:43.206 INFO:tasks.rados.rados.0.smithi022.stdout:3368: finishing write tid 2 to smithi02284960-5 2024-09-21T09:42:43.206 INFO:tasks.rados.rados.0.smithi022.stdout:3368: finishing write tid 3 to smithi02284960-5 2024-09-21T09:42:43.207 INFO:tasks.rados.rados.0.smithi022.stdout:3368: finishing write tid 4 to smithi02284960-5 2024-09-21T09:42:43.208 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 5 v 1951 (ObjNum 1070 snap 322 seq_num 1070) dirty exists 2024-09-21T09:42:43.208 INFO:tasks.rados.rados.0.smithi022.stdout:3368: left oid 5 (ObjNum 1070 snap 322 seq_num 1070) 2024-09-21T09:42:43.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:42:43 smithi064 ceph-mon[86709]: osdmap e721: 8 total, 8 up, 8 in 2024-09-21T09:42:43.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:42:43 smithi064 ceph-mon[86709]: pgmap v695: 121 pgs: 121 active+clean; 121 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 4.6 MiB/s rd, 4.0 MiB/s wr, 52 op/s 2024-09-21T09:42:43.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:42:43 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:42:43.557 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:42:43 smithi022 ceph-mon[98615]: osdmap e721: 8 total, 8 up, 8 in 2024-09-21T09:42:43.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:42:43 smithi022 ceph-mon[98615]: pgmap v695: 121 pgs: 121 active+clean; 121 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 4.6 MiB/s rd, 4.0 MiB/s wr, 52 op/s 2024-09-21T09:42:43.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:42:43 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:42:43.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:42:43 smithi022 ceph-mon[96736]: osdmap e721: 8 total, 8 up, 8 in 2024-09-21T09:42:43.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:42:43 smithi022 ceph-mon[96736]: pgmap v695: 121 pgs: 121 active+clean; 121 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 4.6 MiB/s rd, 4.0 MiB/s wr, 52 op/s 2024-09-21T09:42:43.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:42:43 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:42:44.069 INFO:tasks.rados.rados.0.smithi022.stdout:3365: done (2 left) 2024-09-21T09:42:44.069 INFO:tasks.rados.rados.0.smithi022.stdout:3368: done (1 left) 2024-09-21T09:42:44.069 INFO:tasks.rados.rados.0.smithi022.stdout:3369: done (0 left) 2024-09-21T09:42:44.069 INFO:tasks.rados.rados.0.smithi022.stdout:3370: delete oid 25 current snap is 323 2024-09-21T09:42:44.072 INFO:tasks.rados.rados.0.smithi022.stdout:3370: done (0 left) 2024-09-21T09:42:44.072 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 45 current snap is 323 2024-09-21T09:42:44.072 INFO:tasks.rados.rados.0.smithi022.stdout:3371: seq_num 1072 ranges {909312=761856,1671168=401408} 2024-09-21T09:42:44.082 INFO:tasks.rados.rados.0.smithi022.stdout:3371: writing smithi02284960-45 from 909312 to 1671168 tid 1 2024-09-21T09:42:44.085 INFO:tasks.rados.rados.0.smithi022.stdout:3371: writing smithi02284960-45 from 1671168 to 2072576 tid 2 2024-09-21T09:42:44.085 INFO:tasks.rados.rados.0.smithi022.stdout:3372: read oid 17 snap -1 2024-09-21T09:42:44.086 INFO:tasks.rados.rados.0.smithi022.stdout:3372: expect (ObjNum 1006 snap 309 seq_num 1006) 2024-09-21T09:42:44.086 INFO:tasks.rados.rados.0.smithi022.stdout:3373: snap_create 2024-09-21T09:42:44.088 INFO:tasks.rados.rados.0.smithi022.stdout:3371: finishing write tid 1 to smithi02284960-45 2024-09-21T09:42:44.089 INFO:tasks.rados.rados.0.smithi022.stdout:3371: finishing write tid 2 to smithi02284960-45 2024-09-21T09:42:44.089 INFO:tasks.rados.rados.0.smithi022.stdout:3371: finishing write tid 3 to smithi02284960-45 2024-09-21T09:42:44.090 INFO:tasks.rados.rados.0.smithi022.stdout:3371: finishing write tid 4 to smithi02284960-45 2024-09-21T09:42:44.090 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 45 v 1438 (ObjNum 1072 snap 323 seq_num 1072) dirty exists 2024-09-21T09:42:44.090 INFO:tasks.rados.rados.0.smithi022.stdout:3371: left oid 45 (ObjNum 1072 snap 323 seq_num 1072) 2024-09-21T09:42:44.149 INFO:tasks.rados.rados.0.smithi022.stdout:3372: expect (ObjNum 1006 snap 309 seq_num 1006) 2024-09-21T09:42:44.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:42:44 smithi064 ceph-mon[86709]: osdmap e722: 8 total, 8 up, 8 in 2024-09-21T09:42:44.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:42:44 smithi064 ceph-mon[86709]: osdmap e723: 8 total, 8 up, 8 in 2024-09-21T09:42:44.557 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:42:44 smithi022 ceph-mon[98615]: osdmap e722: 8 total, 8 up, 8 in 2024-09-21T09:42:44.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:42:44 smithi022 ceph-mon[98615]: osdmap e723: 8 total, 8 up, 8 in 2024-09-21T09:42:44.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:42:44 smithi022 ceph-mon[96736]: osdmap e722: 8 total, 8 up, 8 in 2024-09-21T09:42:44.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:42:44 smithi022 ceph-mon[96736]: osdmap e723: 8 total, 8 up, 8 in 2024-09-21T09:42:46.057 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:42:45 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:42:45] "GET /metrics HTTP/1.1" 200 36353 "" "Prometheus/2.51.0" 2024-09-21T09:42:46.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:42:46 smithi064 ceph-mon[86709]: pgmap v698: 121 pgs: 121 active+clean; 123 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.4 MiB/s rd, 2.2 MiB/s wr, 34 op/s 2024-09-21T09:42:46.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:42:46 smithi064 ceph-mon[86709]: osdmap e724: 8 total, 8 up, 8 in 2024-09-21T09:42:46.509 INFO:tasks.rados.rados.0.smithi022.stdout:3371: done (2 left) 2024-09-21T09:42:46.509 INFO:tasks.rados.rados.0.smithi022.stdout:3372: done (1 left) 2024-09-21T09:42:46.509 INFO:tasks.rados.rados.0.smithi022.stdout:3373: done (0 left) 2024-09-21T09:42:46.509 INFO:tasks.rados.rados.0.smithi022.stdout:3374: read oid 25 snap -1 2024-09-21T09:42:46.510 INFO:tasks.rados.rados.0.smithi022.stdout:3374: expect deleted 2024-09-21T09:42:46.510 INFO:tasks.rados.rados.0.smithi022.stdout:3375: read oid 33 snap -1 2024-09-21T09:42:46.510 INFO:tasks.rados.rados.0.smithi022.stdout:3375: expect deleted 2024-09-21T09:42:46.510 INFO:tasks.rados.rados.0.smithi022.stdout:3376: read oid 27 snap -1 2024-09-21T09:42:46.510 INFO:tasks.rados.rados.0.smithi022.stdout:3376: expect (ObjNum 1008 snap 309 seq_num 1008) 2024-09-21T09:42:46.510 INFO:tasks.rados.rados.0.smithi022.stdout:3377: read oid 18 snap -1 2024-09-21T09:42:46.510 INFO:tasks.rados.rados.0.smithi022.stdout:3377: expect (ObjNum 1007 snap 309 seq_num 1007) 2024-09-21T09:42:46.510 INFO:tasks.rados.rados.0.smithi022.stdout:3378: read oid 16 snap 323 2024-09-21T09:42:46.510 INFO:tasks.rados.rados.0.smithi022.stdout:3378: expect (ObjNum 1068 snap 322 seq_num 1068) 2024-09-21T09:42:46.510 INFO:tasks.rados.rados.0.smithi022.stdout:3379: snap_create 2024-09-21T09:42:46.512 INFO:tasks.rados.rados.0.smithi022.stdout:3376: expect (ObjNum 1008 snap 309 seq_num 1008) 2024-09-21T09:42:46.514 INFO:tasks.rados.rados.0.smithi022.stdout:3377: expect (ObjNum 1007 snap 309 seq_num 1007) 2024-09-21T09:42:46.528 INFO:tasks.rados.rados.0.smithi022.stdout:3378: expect (ObjNum 1068 snap 322 seq_num 1068) 2024-09-21T09:42:46.557 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:42:46 smithi022 ceph-mon[96736]: pgmap v698: 121 pgs: 121 active+clean; 123 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.4 MiB/s rd, 2.2 MiB/s wr, 34 op/s 2024-09-21T09:42:46.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:42:46 smithi022 ceph-mon[96736]: osdmap e724: 8 total, 8 up, 8 in 2024-09-21T09:42:46.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:42:46 smithi022 ceph-mon[98615]: pgmap v698: 121 pgs: 121 active+clean; 123 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.4 MiB/s rd, 2.2 MiB/s wr, 34 op/s 2024-09-21T09:42:46.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:42:46 smithi022 ceph-mon[98615]: osdmap e724: 8 total, 8 up, 8 in 2024-09-21T09:42:47.075 INFO:tasks.rados.rados.0.smithi022.stdout:3374: done (5 left) 2024-09-21T09:42:47.075 INFO:tasks.rados.rados.0.smithi022.stdout:3375: done (4 left) 2024-09-21T09:42:47.075 INFO:tasks.rados.rados.0.smithi022.stdout:3376: done (3 left) 2024-09-21T09:42:47.075 INFO:tasks.rados.rados.0.smithi022.stdout:3377: done (2 left) 2024-09-21T09:42:47.075 INFO:tasks.rados.rados.0.smithi022.stdout:3378: done (1 left) 2024-09-21T09:42:47.075 INFO:tasks.rados.rados.0.smithi022.stdout:3379: done (0 left) 2024-09-21T09:42:47.075 INFO:tasks.rados.rados.0.smithi022.stdout:3380: copy_from oid 20 from oid 30 current snap is 325 2024-09-21T09:42:47.076 INFO:tasks.rados.rados.0.smithi022.stdout:3381: setattr oid 7 current snap is 325 2024-09-21T09:42:47.077 INFO:tasks.rados.rados.0.smithi022.stdout:3382: copy_from oid 15 from oid 11 current snap is 325 2024-09-21T09:42:47.077 INFO:tasks.rados.rados.0.smithi022.stdout:3383: delete oid 25 current snap is 325 2024-09-21T09:42:47.080 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 7 v 2002 (ObjNum 1056 snap 317 seq_num 1056) dirty exists 2024-09-21T09:42:47.083 INFO:tasks.rados.rados.0.smithi022.stdout:3381: done (3 left) 2024-09-21T09:42:47.083 INFO:tasks.rados.rados.0.smithi022.stdout:3383: done (2 left) 2024-09-21T09:42:47.083 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 18 current snap is 325 2024-09-21T09:42:47.083 INFO:tasks.rados.rados.0.smithi022.stdout:3384: seq_num 1074 ranges {65536=794624,860160=401408} 2024-09-21T09:42:47.088 INFO:tasks.rados.rados.0.smithi022.stdout:3384: writing smithi02284960-18 from 65536 to 860160 tid 1 2024-09-21T09:42:47.091 INFO:tasks.rados.rados.0.smithi022.stdout:3384: writing smithi02284960-18 from 860160 to 1261568 tid 2 2024-09-21T09:42:47.092 INFO:tasks.rados.rados.0.smithi022.stdout:3385: copy_from oid 22 from oid 5 current snap is 325 2024-09-21T09:42:47.092 INFO:tasks.rados.rados.0.smithi022.stdout:3382: finishing copy_from racing read to smithi02284960-15 2024-09-21T09:42:47.092 INFO:tasks.rados.rados.0.smithi022.stdout:3382: finishing copy_from to smithi02284960-15 2024-09-21T09:42:47.092 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 15 v 1752 (ObjNum 1062 snap 319 seq_num 1062) dirty exists 2024-09-21T09:42:47.092 INFO:tasks.rados.rados.0.smithi022.stdout:3382: done (3 left) 2024-09-21T09:42:47.092 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 35 current snap is 325 2024-09-21T09:42:47.092 INFO:tasks.rados.rados.0.smithi022.stdout:3386: seq_num 1075 ranges {3497984=409600,3907584=671744,4579328=131072} 2024-09-21T09:42:47.114 INFO:tasks.rados.rados.0.smithi022.stdout:3386: writing smithi02284960-35 from 3497984 to 3907584 tid 1 2024-09-21T09:42:47.118 INFO:tasks.rados.rados.0.smithi022.stdout:3386: writing smithi02284960-35 from 3907584 to 4579328 tid 2 2024-09-21T09:42:47.120 INFO:tasks.rados.rados.0.smithi022.stdout:3386: writing smithi02284960-35 from 4579328 to 4710400 tid 3 2024-09-21T09:42:47.120 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 7 current snap is 325 2024-09-21T09:42:47.120 INFO:tasks.rados.rados.0.smithi022.stdout:3384: finishing write tid 1 to smithi02284960-18 2024-09-21T09:42:47.120 INFO:tasks.rados.rados.0.smithi022.stdout:3380: finishing copy_from racing read to smithi02284960-20 2024-09-21T09:42:47.120 INFO:tasks.rados.rados.0.smithi022.stdout:3384: finishing write tid 2 to smithi02284960-18 2024-09-21T09:42:47.120 INFO:tasks.rados.rados.0.smithi022.stdout:3384: finishing write tid 3 to smithi02284960-18 2024-09-21T09:42:47.120 INFO:tasks.rados.rados.0.smithi022.stdout:3380: finishing copy_from to smithi02284960-20 2024-09-21T09:42:47.120 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 20 v 1348 (ObjNum 1055 snap 317 seq_num 1055) dirty exists 2024-09-21T09:42:47.120 INFO:tasks.rados.rados.0.smithi022.stdout:3384: finishing write tid 4 to smithi02284960-18 2024-09-21T09:42:47.120 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 18 v 1534 (ObjNum 1074 snap 325 seq_num 1074) dirty exists 2024-09-21T09:42:47.120 INFO:tasks.rados.rados.0.smithi022.stdout:3384: left oid 18 (ObjNum 1074 snap 325 seq_num 1074) 2024-09-21T09:42:47.120 INFO:tasks.rados.rados.0.smithi022.stdout:3385: finishing copy_from racing read to smithi02284960-22 2024-09-21T09:42:47.120 INFO:tasks.rados.rados.0.smithi022.stdout:3385: finishing copy_from to smithi02284960-22 2024-09-21T09:42:47.120 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 22 v 1951 (ObjNum 1070 snap 322 seq_num 1070) dirty exists 2024-09-21T09:42:47.121 INFO:tasks.rados.rados.0.smithi022.stdout:3387: seq_num 1076 ranges {892928=425984,1318912=548864,1867776=253952} 2024-09-21T09:42:47.127 INFO:tasks.rados.rados.0.smithi022.stdout:3387: writing smithi02284960-7 from 892928 to 1318912 tid 1 2024-09-21T09:42:47.130 INFO:tasks.rados.rados.0.smithi022.stdout:3387: writing smithi02284960-7 from 1318912 to 1867776 tid 2 2024-09-21T09:42:47.132 INFO:tasks.rados.rados.0.smithi022.stdout:3387: writing smithi02284960-7 from 1867776 to 2121728 tid 3 2024-09-21T09:42:47.133 INFO:tasks.rados.rados.0.smithi022.stdout:3380: done (4 left) 2024-09-21T09:42:47.133 INFO:tasks.rados.rados.0.smithi022.stdout:3384: done (3 left) 2024-09-21T09:42:47.133 INFO:tasks.rados.rados.0.smithi022.stdout:3385: done (2 left) 2024-09-21T09:42:47.133 INFO:tasks.rados.rados.0.smithi022.stdout:3388: rmattr oid 4 current snap is 325 2024-09-21T09:42:47.133 INFO:tasks.rados.rados.0.smithi022.stdout:3388: done (2 left) 2024-09-21T09:42:47.133 INFO:tasks.rados.rados.0.smithi022.stdout:3389: snap_create 2024-09-21T09:42:47.133 INFO:tasks.rados.rados.0.smithi022.stdout:3386: finishing write tid 1 to smithi02284960-35 2024-09-21T09:42:47.133 INFO:tasks.rados.rados.0.smithi022.stdout:3386: finishing write tid 2 to smithi02284960-35 2024-09-21T09:42:47.133 INFO:tasks.rados.rados.0.smithi022.stdout:3386: finishing write tid 3 to smithi02284960-35 2024-09-21T09:42:47.133 INFO:tasks.rados.rados.0.smithi022.stdout:3386: finishing write tid 4 to smithi02284960-35 2024-09-21T09:42:47.133 INFO:tasks.rados.rados.0.smithi022.stdout:3386: finishing write tid 5 to smithi02284960-35 2024-09-21T09:42:47.133 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 35 v 1801 (ObjNum 1075 snap 325 seq_num 1075) dirty exists 2024-09-21T09:42:47.133 INFO:tasks.rados.rados.0.smithi022.stdout:3386: left oid 35 (ObjNum 1075 snap 325 seq_num 1075) 2024-09-21T09:42:47.133 INFO:tasks.rados.rados.0.smithi022.stdout:3387: finishing write tid 1 to smithi02284960-7 2024-09-21T09:42:47.136 INFO:tasks.rados.rados.0.smithi022.stdout:3387: finishing write tid 2 to smithi02284960-7 2024-09-21T09:42:47.136 INFO:tasks.rados.rados.0.smithi022.stdout:3387: finishing write tid 3 to smithi02284960-7 2024-09-21T09:42:47.136 INFO:tasks.rados.rados.0.smithi022.stdout:3387: finishing write tid 4 to smithi02284960-7 2024-09-21T09:42:47.137 INFO:tasks.rados.rados.0.smithi022.stdout:3387: finishing write tid 5 to smithi02284960-7 2024-09-21T09:42:47.138 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 7 v 2007 (ObjNum 1076 snap 325 seq_num 1076) dirty exists 2024-09-21T09:42:47.138 INFO:tasks.rados.rados.0.smithi022.stdout:3387: left oid 7 (ObjNum 1076 snap 325 seq_num 1076) 2024-09-21T09:42:48.081 INFO:tasks.rados.rados.0.smithi022.stdout:3386: done (2 left) 2024-09-21T09:42:48.081 INFO:tasks.rados.rados.0.smithi022.stdout:3387: done (1 left) 2024-09-21T09:42:48.081 INFO:tasks.rados.rados.0.smithi022.stdout:3389: done (0 left) 2024-09-21T09:42:48.081 INFO:tasks.rados.rados.0.smithi022.stdout:3390: delete oid 7 current snap is 326 2024-09-21T09:42:48.087 INFO:tasks.rados.rados.0.smithi022.stdout:3390: done (0 left) 2024-09-21T09:42:48.087 INFO:tasks.rados.rados.0.smithi022.stdout:3391: snap_remove snap 325 2024-09-21T09:42:48.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:42:48 smithi064 ceph-mon[86709]: pgmap v700: 121 pgs: 121 active+clean; 130 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 2.1 MiB/s wr, 25 op/s 2024-09-21T09:42:48.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:42:48 smithi064 ceph-mon[86709]: osdmap e725: 8 total, 8 up, 8 in 2024-09-21T09:42:48.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:42:48 smithi022 ceph-mon[98615]: pgmap v700: 121 pgs: 121 active+clean; 130 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 2.1 MiB/s wr, 25 op/s 2024-09-21T09:42:48.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:42:48 smithi022 ceph-mon[98615]: osdmap e725: 8 total, 8 up, 8 in 2024-09-21T09:42:48.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:42:48 smithi022 ceph-mon[96736]: pgmap v700: 121 pgs: 121 active+clean; 130 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 2.1 MiB/s wr, 25 op/s 2024-09-21T09:42:48.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:42:48 smithi022 ceph-mon[96736]: osdmap e725: 8 total, 8 up, 8 in 2024-09-21T09:42:49.072 INFO:tasks.rados.rados.0.smithi022.stdout:3391: done (0 left) 2024-09-21T09:42:49.072 INFO:tasks.rados.rados.0.smithi022.stdout:3392: read oid 20 snap -1 2024-09-21T09:42:49.072 INFO:tasks.rados.rados.0.smithi022.stdout:3392: expect (ObjNum 1055 snap 317 seq_num 1055) 2024-09-21T09:42:49.072 INFO:tasks.rados.rados.0.smithi022.stdout:3393: read oid 12 snap -1 2024-09-21T09:42:49.072 INFO:tasks.rados.rados.0.smithi022.stdout:3393: expect (ObjNum 997 snap 306 seq_num 997) 2024-09-21T09:42:49.072 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 3 current snap is 326 2024-09-21T09:42:49.073 INFO:tasks.rados.rados.0.smithi022.stdout:3394: seq_num 1078 ranges {843776=458752,1302528=720896,2023424=81920} 2024-09-21T09:42:49.080 INFO:tasks.rados.rados.0.smithi022.stdout:3394: writing smithi02284960-3 from 843776 to 1302528 tid 1 2024-09-21T09:42:49.086 INFO:tasks.rados.rados.0.smithi022.stdout:3394: writing smithi02284960-3 from 1302528 to 2023424 tid 2 2024-09-21T09:42:49.087 INFO:tasks.rados.rados.0.smithi022.stdout:3394: writing smithi02284960-3 from 2023424 to 2105344 tid 3 2024-09-21T09:42:49.087 INFO:tasks.rados.rados.0.smithi022.stdout:3395: read oid 24 snap 324 2024-09-21T09:42:49.088 INFO:tasks.rados.rados.0.smithi022.stdout:3395: expect (ObjNum 1041 snap 315 seq_num 1041) 2024-09-21T09:42:49.088 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 16 current snap is 326 2024-09-21T09:42:49.088 INFO:tasks.rados.rados.0.smithi022.stdout:3396: seq_num 1079 ranges {6119424=475136,6594560=606208,7200768=196608} 2024-09-21T09:42:49.124 INFO:tasks.rados.rados.0.smithi022.stdout:3396: writing smithi02284960-16 from 6119424 to 6594560 tid 1 2024-09-21T09:42:49.128 INFO:tasks.rados.rados.0.smithi022.stdout:3396: writing smithi02284960-16 from 6594560 to 7200768 tid 2 2024-09-21T09:42:49.130 INFO:tasks.rados.rados.0.smithi022.stdout:3396: writing smithi02284960-16 from 7200768 to 7397376 tid 3 2024-09-21T09:42:49.130 INFO:tasks.rados.rados.0.smithi022.stdout:3397: read oid 31 snap -1 2024-09-21T09:42:49.130 INFO:tasks.rados.rados.0.smithi022.stdout:3397: expect deleted 2024-09-21T09:42:49.130 INFO:tasks.rados.rados.0.smithi022.stdout:3398: delete oid 38 current snap is 326 2024-09-21T09:42:49.130 INFO:tasks.rados.rados.0.smithi022.stdout:3393: expect (ObjNum 997 snap 306 seq_num 997) 2024-09-21T09:42:49.193 INFO:tasks.rados.rados.0.smithi022.stdout:3394: finishing write tid 1 to smithi02284960-3 2024-09-21T09:42:49.193 INFO:tasks.rados.rados.0.smithi022.stdout:3392: expect (ObjNum 1055 snap 317 seq_num 1055) 2024-09-21T09:42:49.423 INFO:tasks.rados.rados.0.smithi022.stdout:3394: finishing write tid 2 to smithi02284960-3 2024-09-21T09:42:49.423 INFO:tasks.rados.rados.0.smithi022.stdout:3395: expect (ObjNum 1041 snap 315 seq_num 1041) 2024-09-21T09:42:49.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:42:49 smithi064 ceph-mon[86709]: osdmap e726: 8 total, 8 up, 8 in 2024-09-21T09:42:49.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:42:49 smithi064 ceph-mon[86709]: osdmap e727: 8 total, 8 up, 8 in 2024-09-21T09:42:49.444 INFO:tasks.rados.rados.0.smithi022.stdout:3394: finishing write tid 3 to smithi02284960-3 2024-09-21T09:42:49.444 INFO:tasks.rados.rados.0.smithi022.stdout:3394: finishing write tid 4 to smithi02284960-3 2024-09-21T09:42:49.444 INFO:tasks.rados.rados.0.smithi022.stdout:3392: done (6 left) 2024-09-21T09:42:49.444 INFO:tasks.rados.rados.0.smithi022.stdout:3393: done (5 left) 2024-09-21T09:42:49.444 INFO:tasks.rados.rados.0.smithi022.stdout:3395: done (4 left) 2024-09-21T09:42:49.444 INFO:tasks.rados.rados.0.smithi022.stdout:3398: done (3 left) 2024-09-21T09:42:49.444 INFO:tasks.rados.rados.0.smithi022.stdout:3399: read oid 40 snap 317 2024-09-21T09:42:49.444 INFO:tasks.rados.rados.0.smithi022.stdout:3399: expect (ObjNum 1035 snap 315 seq_num 1035) 2024-09-21T09:42:49.444 INFO:tasks.rados.rados.0.smithi022.stdout:3400: read oid 22 snap -1 2024-09-21T09:42:49.444 INFO:tasks.rados.rados.0.smithi022.stdout:3400: expect (ObjNum 1070 snap 322 seq_num 1070) 2024-09-21T09:42:49.445 INFO:tasks.rados.rados.0.smithi022.stdout:3401: snap_remove snap 324 2024-09-21T09:42:49.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:42:49 smithi022 ceph-mon[96736]: osdmap e726: 8 total, 8 up, 8 in 2024-09-21T09:42:49.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:42:49 smithi022 ceph-mon[96736]: osdmap e727: 8 total, 8 up, 8 in 2024-09-21T09:42:49.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:42:49 smithi022 ceph-mon[98615]: osdmap e726: 8 total, 8 up, 8 in 2024-09-21T09:42:49.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:42:49 smithi022 ceph-mon[98615]: osdmap e727: 8 total, 8 up, 8 in 2024-09-21T09:42:50.095 INFO:tasks.rados.rados.0.smithi022.stdout:3401: done (5 left) 2024-09-21T09:42:50.095 INFO:tasks.rados.rados.0.smithi022.stdout:3402: snap_remove snap 320 2024-09-21T09:42:50.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:42:50 smithi064 ceph-mon[86709]: pgmap v703: 121 pgs: 121 active+clean; 132 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 2.2 MiB/s wr, 16 op/s 2024-09-21T09:42:50.442 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:42:50 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:42:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:42:50.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:42:50 smithi022 ceph-mon[98615]: pgmap v703: 121 pgs: 121 active+clean; 132 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 2.2 MiB/s wr, 16 op/s 2024-09-21T09:42:50.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:42:50 smithi022 ceph-mon[96736]: pgmap v703: 121 pgs: 121 active+clean; 132 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 2.2 MiB/s wr, 16 op/s 2024-09-21T09:42:51.101 INFO:tasks.rados.rados.0.smithi022.stdout:3402: done (5 left) 2024-09-21T09:42:51.101 INFO:tasks.rados.rados.0.smithi022.stdout:3403: snap_remove snap 322 2024-09-21T09:42:51.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:42:51 smithi064 ceph-mon[86709]: osdmap e728: 8 total, 8 up, 8 in 2024-09-21T09:42:51.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:42:51 smithi022 ceph-mon[98615]: osdmap e728: 8 total, 8 up, 8 in 2024-09-21T09:42:51.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:42:51 smithi022 ceph-mon[96736]: osdmap e728: 8 total, 8 up, 8 in 2024-09-21T09:42:52.102 INFO:tasks.rados.rados.0.smithi022.stdout:3403: done (5 left) 2024-09-21T09:42:52.102 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 37 current snap is 326 2024-09-21T09:42:52.102 INFO:tasks.rados.rados.0.smithi022.stdout:3404: seq_num 1081 ranges {122880=507904,630784=425984,1056768=376832} 2024-09-21T09:42:52.106 INFO:tasks.rados.rados.0.smithi022.stdout:3404: writing smithi02284960-37 from 122880 to 630784 tid 1 2024-09-21T09:42:52.109 INFO:tasks.rados.rados.0.smithi022.stdout:3404: writing smithi02284960-37 from 630784 to 1056768 tid 2 2024-09-21T09:42:52.112 INFO:tasks.rados.rados.0.smithi022.stdout:3404: writing smithi02284960-37 from 1056768 to 1433600 tid 3 2024-09-21T09:42:52.112 INFO:tasks.rados.rados.0.smithi022.stdout:3405: snap_create 2024-09-21T09:42:52.112 INFO:tasks.rados.rados.0.smithi022.stdout:3394: finishing write tid 5 to smithi02284960-3 2024-09-21T09:42:52.112 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 3 v 627 (ObjNum 1078 snap 326 seq_num 1078) dirty exists 2024-09-21T09:42:52.112 INFO:tasks.rados.rados.0.smithi022.stdout:3394: left oid 3 (ObjNum 1078 snap 326 seq_num 1078) 2024-09-21T09:42:52.112 INFO:tasks.rados.rados.0.smithi022.stdout:3396: finishing write tid 1 to smithi02284960-16 2024-09-21T09:42:52.112 INFO:tasks.rados.rados.0.smithi022.stdout:3396: finishing write tid 2 to smithi02284960-16 2024-09-21T09:42:52.112 INFO:tasks.rados.rados.0.smithi022.stdout:3396: finishing write tid 3 to smithi02284960-16 2024-09-21T09:42:52.112 INFO:tasks.rados.rados.0.smithi022.stdout:3396: finishing write tid 4 to smithi02284960-16 2024-09-21T09:42:52.113 INFO:tasks.rados.rados.0.smithi022.stdout:3396: finishing write tid 5 to smithi02284960-16 2024-09-21T09:42:52.113 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 16 v 1538 (ObjNum 1079 snap 326 seq_num 1079) dirty exists 2024-09-21T09:42:52.113 INFO:tasks.rados.rados.0.smithi022.stdout:3396: left oid 16 (ObjNum 1079 snap 326 seq_num 1079) 2024-09-21T09:42:52.113 INFO:tasks.rados.rados.0.smithi022.stdout:3399: expect (ObjNum 1035 snap 315 seq_num 1035) 2024-09-21T09:42:52.340 INFO:tasks.rados.rados.0.smithi022.stdout:3400: expect (ObjNum 1070 snap 322 seq_num 1070) 2024-09-21T09:42:52.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:42:52 smithi064 ceph-mon[86709]: pgmap v706: 121 pgs: 121 active+clean; 138 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 9.9 MiB/s rd, 2.4 MiB/s wr, 15 op/s 2024-09-21T09:42:52.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:42:52 smithi064 ceph-mon[86709]: osdmap e729: 8 total, 8 up, 8 in 2024-09-21T09:42:52.557 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:42:52 smithi022 ceph-mon[98615]: pgmap v706: 121 pgs: 121 active+clean; 138 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 9.9 MiB/s rd, 2.4 MiB/s wr, 15 op/s 2024-09-21T09:42:52.557 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:42:52 smithi022 ceph-mon[98615]: osdmap e729: 8 total, 8 up, 8 in 2024-09-21T09:42:52.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:42:52 smithi022 ceph-mon[96736]: pgmap v706: 121 pgs: 121 active+clean; 138 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 9.9 MiB/s rd, 2.4 MiB/s wr, 15 op/s 2024-09-21T09:42:52.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:42:52 smithi022 ceph-mon[96736]: osdmap e729: 8 total, 8 up, 8 in 2024-09-21T09:42:52.672 INFO:tasks.rados.rados.0.smithi022.stdout:3404: finishing write tid 1 to smithi02284960-37 2024-09-21T09:42:52.672 INFO:tasks.rados.rados.0.smithi022.stdout:3404: finishing write tid 2 to smithi02284960-37 2024-09-21T09:42:52.672 INFO:tasks.rados.rados.0.smithi022.stdout:3404: finishing write tid 3 to smithi02284960-37 2024-09-21T09:42:52.672 INFO:tasks.rados.rados.0.smithi022.stdout:3404: finishing write tid 4 to smithi02284960-37 2024-09-21T09:42:52.672 INFO:tasks.rados.rados.0.smithi022.stdout:3404: finishing write tid 5 to smithi02284960-37 2024-09-21T09:42:52.672 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 37 v 1939 (ObjNum 1081 snap 326 seq_num 1081) dirty exists 2024-09-21T09:42:52.672 INFO:tasks.rados.rados.0.smithi022.stdout:3404: left oid 37 (ObjNum 1081 snap 326 seq_num 1081) 2024-09-21T09:42:53.112 INFO:tasks.rados.rados.0.smithi022.stdout:3394: done (6 left) 2024-09-21T09:42:53.112 INFO:tasks.rados.rados.0.smithi022.stdout:3396: done (5 left) 2024-09-21T09:42:53.112 INFO:tasks.rados.rados.0.smithi022.stdout:3397: done (4 left) 2024-09-21T09:42:53.112 INFO:tasks.rados.rados.0.smithi022.stdout:3399: done (3 left) 2024-09-21T09:42:53.112 INFO:tasks.rados.rados.0.smithi022.stdout:3400: done (2 left) 2024-09-21T09:42:53.112 INFO:tasks.rados.rados.0.smithi022.stdout:3404: done (1 left) 2024-09-21T09:42:53.113 INFO:tasks.rados.rados.0.smithi022.stdout:3405: done (0 left) 2024-09-21T09:42:53.113 INFO:tasks.rados.rados.0.smithi022.stdout:3406: rmattr oid 43 current snap is 327 2024-09-21T09:42:53.113 INFO:tasks.rados.rados.0.smithi022.stdout:3406: done (0 left) 2024-09-21T09:42:53.113 INFO:tasks.rados.rados.0.smithi022.stdout:3407: read oid 26 snap 326 2024-09-21T09:42:53.113 INFO:tasks.rados.rados.0.smithi022.stdout:3407: expect (ObjNum 1036 snap 315 seq_num 1036) 2024-09-21T09:42:53.113 INFO:tasks.rados.rados.0.smithi022.stdout:3408: delete oid 33 current snap is 327 2024-09-21T09:42:53.117 INFO:tasks.rados.rados.0.smithi022.stdout:3408: done (1 left) 2024-09-21T09:42:53.117 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 23 current snap is 327 2024-09-21T09:42:53.117 INFO:tasks.rados.rados.0.smithi022.stdout:3409: seq_num 1083 ranges {671744=540672,1212416=589824,1802240=212992} 2024-09-21T09:42:53.123 INFO:tasks.rados.rados.0.smithi022.stdout:3409: writing smithi02284960-23 from 671744 to 1212416 tid 1 2024-09-21T09:42:53.127 INFO:tasks.rados.rados.0.smithi022.stdout:3409: writing smithi02284960-23 from 1212416 to 1802240 tid 2 2024-09-21T09:42:53.128 INFO:tasks.rados.rados.0.smithi022.stdout:3409: writing smithi02284960-23 from 1802240 to 2015232 tid 3 2024-09-21T09:42:53.129 INFO:tasks.rados.rados.0.smithi022.stdout:3410: copy_from oid 8 from oid 30 current snap is 327 2024-09-21T09:42:53.129 INFO:tasks.rados.rados.0.smithi022.stdout:3409: finishing write tid 1 to smithi02284960-23 2024-09-21T09:42:53.129 INFO:tasks.rados.rados.0.smithi022.stdout:3411: delete oid 48 current snap is 327 2024-09-21T09:42:53.134 INFO:tasks.rados.rados.0.smithi022.stdout:3409: finishing write tid 2 to smithi02284960-23 2024-09-21T09:42:53.134 INFO:tasks.rados.rados.0.smithi022.stdout:3409: finishing write tid 3 to smithi02284960-23 2024-09-21T09:42:53.134 INFO:tasks.rados.rados.0.smithi022.stdout:3409: finishing write tid 4 to smithi02284960-23 2024-09-21T09:42:53.134 INFO:tasks.rados.rados.0.smithi022.stdout:3411: done (3 left) 2024-09-21T09:42:53.134 INFO:tasks.rados.rados.0.smithi022.stdout:3412: read oid 42 snap 326 2024-09-21T09:42:53.134 INFO:tasks.rados.rados.0.smithi022.stdout:3412: expect (ObjNum 1055 snap 317 seq_num 1055) 2024-09-21T09:42:53.134 INFO:tasks.rados.rados.0.smithi022.stdout:3413: snap_remove snap 318 2024-09-21T09:42:53.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:42:53 smithi064 ceph-mon[86709]: osdmap e730: 8 total, 8 up, 8 in 2024-09-21T09:42:53.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:42:53 smithi022 ceph-mon[98615]: osdmap e730: 8 total, 8 up, 8 in 2024-09-21T09:42:53.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:42:53 smithi022 ceph-mon[96736]: osdmap e730: 8 total, 8 up, 8 in 2024-09-21T09:42:54.118 INFO:tasks.rados.rados.0.smithi022.stdout:3413: done (4 left) 2024-09-21T09:42:54.118 INFO:tasks.rados.rados.0.smithi022.stdout:3414: snap_create 2024-09-21T09:42:54.118 INFO:tasks.rados.rados.0.smithi022.stdout:3409: finishing write tid 5 to smithi02284960-23 2024-09-21T09:42:54.118 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 23 v 1819 (ObjNum 1083 snap 327 seq_num 1083) dirty exists 2024-09-21T09:42:54.118 INFO:tasks.rados.rados.0.smithi022.stdout:3409: left oid 23 (ObjNum 1083 snap 327 seq_num 1083) 2024-09-21T09:42:54.118 INFO:tasks.rados.rados.0.smithi022.stdout:3407: expect (ObjNum 1036 snap 315 seq_num 1036) 2024-09-21T09:42:54.314 INFO:tasks.rados.rados.0.smithi022.stdout:3410: finishing copy_from racing read to smithi02284960-8 2024-09-21T09:42:54.314 INFO:tasks.rados.rados.0.smithi022.stdout:3410: finishing copy_from to smithi02284960-8 2024-09-21T09:42:54.314 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 8 v 1348 (ObjNum 1055 snap 317 seq_num 1055) dirty exists 2024-09-21T09:42:54.315 INFO:tasks.rados.rados.0.smithi022.stdout:3412: expect (ObjNum 1055 snap 317 seq_num 1055) 2024-09-21T09:42:54.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:42:54 smithi064 ceph-mon[86709]: pgmap v709: 121 pgs: 121 active+clean; 144 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 3.2 MiB/s wr, 18 op/s 2024-09-21T09:42:54.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:42:54 smithi064 ceph-mon[86709]: osdmap e731: 8 total, 8 up, 8 in 2024-09-21T09:42:54.459 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:42:54 smithi022 ceph-mon[98615]: pgmap v709: 121 pgs: 121 active+clean; 144 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 3.2 MiB/s wr, 18 op/s 2024-09-21T09:42:54.459 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:42:54 smithi022 ceph-mon[98615]: osdmap e731: 8 total, 8 up, 8 in 2024-09-21T09:42:54.459 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:42:54 smithi022 ceph-mon[96736]: pgmap v709: 121 pgs: 121 active+clean; 144 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 3.2 MiB/s wr, 18 op/s 2024-09-21T09:42:54.459 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:42:54 smithi022 ceph-mon[96736]: osdmap e731: 8 total, 8 up, 8 in 2024-09-21T09:42:55.123 INFO:tasks.rados.rados.0.smithi022.stdout:3407: done (4 left) 2024-09-21T09:42:55.123 INFO:tasks.rados.rados.0.smithi022.stdout:3409: done (3 left) 2024-09-21T09:42:55.123 INFO:tasks.rados.rados.0.smithi022.stdout:3410: done (2 left) 2024-09-21T09:42:55.123 INFO:tasks.rados.rados.0.smithi022.stdout:3412: done (1 left) 2024-09-21T09:42:55.123 INFO:tasks.rados.rados.0.smithi022.stdout:3414: done (0 left) 2024-09-21T09:42:55.123 INFO:tasks.rados.rados.0.smithi022.stdout:3415: read oid 37 snap -1 2024-09-21T09:42:55.123 INFO:tasks.rados.rados.0.smithi022.stdout:3415: expect (ObjNum 1081 snap 326 seq_num 1081) 2024-09-21T09:42:55.123 INFO:tasks.rados.rados.0.smithi022.stdout:3416: rmattr oid 21 current snap is 328 2024-09-21T09:42:55.123 INFO:tasks.rados.rados.0.smithi022.stdout:3416: done (1 left) 2024-09-21T09:42:55.124 INFO:tasks.rados.rados.0.smithi022.stdout:3417: snap_create 2024-09-21T09:42:55.133 INFO:tasks.rados.rados.0.smithi022.stdout:3415: expect (ObjNum 1081 snap 326 seq_num 1081) 2024-09-21T09:42:55.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:42:55 smithi064 ceph-mon[86709]: osdmap e732: 8 total, 8 up, 8 in 2024-09-21T09:42:55.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:42:55 smithi064 ceph-mon[86709]: pgmap v712: 121 pgs: 121 active+clean; 146 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5.7 MiB/s rd, 2.0 MiB/s wr, 7 op/s 2024-09-21T09:42:55.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:42:55 smithi022 ceph-mon[98615]: osdmap e732: 8 total, 8 up, 8 in 2024-09-21T09:42:55.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:42:55 smithi022 ceph-mon[98615]: pgmap v712: 121 pgs: 121 active+clean; 146 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5.7 MiB/s rd, 2.0 MiB/s wr, 7 op/s 2024-09-21T09:42:55.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:42:55 smithi022 ceph-mon[96736]: osdmap e732: 8 total, 8 up, 8 in 2024-09-21T09:42:55.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:42:55 smithi022 ceph-mon[96736]: pgmap v712: 121 pgs: 121 active+clean; 146 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5.7 MiB/s rd, 2.0 MiB/s wr, 7 op/s 2024-09-21T09:42:56.058 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:42:55 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:42:55] "GET /metrics HTTP/1.1" 200 36353 "" "Prometheus/2.51.0" 2024-09-21T09:42:56.124 INFO:tasks.rados.rados.0.smithi022.stdout:3415: done (1 left) 2024-09-21T09:42:56.124 INFO:tasks.rados.rados.0.smithi022.stdout:3417: done (0 left) 2024-09-21T09:42:56.124 INFO:tasks.rados.rados.0.smithi022.stdout:3418: read oid 43 snap -1 2024-09-21T09:42:56.124 INFO:tasks.rados.rados.0.smithi022.stdout:3418: expect deleted 2024-09-21T09:42:56.124 INFO:tasks.rados.rados.0.smithi022.stdout:3419: snap_create 2024-09-21T09:42:56.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:42:56 smithi064 ceph-mon[86709]: osdmap e733: 8 total, 8 up, 8 in 2024-09-21T09:42:56.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:42:56 smithi022 ceph-mon[98615]: osdmap e733: 8 total, 8 up, 8 in 2024-09-21T09:42:56.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:42:56 smithi022 ceph-mon[96736]: osdmap e733: 8 total, 8 up, 8 in 2024-09-21T09:42:57.136 INFO:tasks.rados.rados.0.smithi022.stdout:3418: done (1 left) 2024-09-21T09:42:57.136 INFO:tasks.rados.rados.0.smithi022.stdout:3419: done (0 left) 2024-09-21T09:42:57.136 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 7 current snap is 330 2024-09-21T09:42:57.136 INFO:tasks.rados.rados.0.smithi022.stdout:3420: seq_num 1085 ranges {0=581632,581632=761856,1343488=40960} 2024-09-21T09:42:57.139 INFO:tasks.rados.rados.0.smithi022.stdout:3420: writing smithi02284960-7 from 0 to 581632 tid 1 2024-09-21T09:42:57.143 INFO:tasks.rados.rados.0.smithi022.stdout:3420: writing smithi02284960-7 from 581632 to 1343488 tid 2 2024-09-21T09:42:57.144 INFO:tasks.rados.rados.0.smithi022.stdout:3420: writing smithi02284960-7 from 1343488 to 1384448 tid 3 2024-09-21T09:42:57.144 INFO:tasks.rados.rados.0.smithi022.stdout:3421: delete oid 47 current snap is 330 2024-09-21T09:42:57.150 INFO:tasks.rados.rados.0.smithi022.stdout:3420: finishing write tid 1 to smithi02284960-7 2024-09-21T09:42:57.150 INFO:tasks.rados.rados.0.smithi022.stdout:3420: finishing write tid 2 to smithi02284960-7 2024-09-21T09:42:57.150 INFO:tasks.rados.rados.0.smithi022.stdout:3421: done (1 left) 2024-09-21T09:42:57.150 INFO:tasks.rados.rados.0.smithi022.stdout:3422: read oid 21 snap -1 2024-09-21T09:42:57.150 INFO:tasks.rados.rados.0.smithi022.stdout:3422: expect (ObjNum 1029 snap 315 seq_num 1029) 2024-09-21T09:42:57.150 INFO:tasks.rados.rados.0.smithi022.stdout:3423: rollback oid 4 current snap is 330 2024-09-21T09:42:57.150 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 4 to 327 2024-09-21T09:42:57.150 INFO:tasks.rados.rados.0.smithi022.stdout:3420: finishing write tid 3 to smithi02284960-7 2024-09-21T09:42:57.150 INFO:tasks.rados.rados.0.smithi022.stdout:3424: read oid 11 snap -1 2024-09-21T09:42:57.150 INFO:tasks.rados.rados.0.smithi022.stdout:3424: expect (ObjNum 1062 snap 319 seq_num 1062) 2024-09-21T09:42:57.150 INFO:tasks.rados.rados.0.smithi022.stdout:3425: copy_from oid 45 from oid 40 current snap is 330 2024-09-21T09:42:57.150 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 26 current snap is 330 2024-09-21T09:42:57.151 INFO:tasks.rados.rados.0.smithi022.stdout:3426: seq_num 1087 ranges {4202496=614400,4816896=532480,5349376=270336} 2024-09-21T09:42:57.177 INFO:tasks.rados.rados.0.smithi022.stdout:3426: writing smithi02284960-26 from 4202496 to 4816896 tid 1 2024-09-21T09:42:57.180 INFO:tasks.rados.rados.0.smithi022.stdout:3426: writing smithi02284960-26 from 4816896 to 5349376 tid 2 2024-09-21T09:42:57.182 INFO:tasks.rados.rados.0.smithi022.stdout:3426: writing smithi02284960-26 from 5349376 to 5619712 tid 3 2024-09-21T09:42:57.182 INFO:tasks.rados.rados.0.smithi022.stdout:3427: copy_from oid 49 from oid 14 current snap is 330 2024-09-21T09:42:57.182 INFO:tasks.rados.rados.0.smithi022.stdout:3423: finishing rollback tid 1 to smithi02284960-4 2024-09-21T09:42:57.182 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 4 v 1819 (ObjNum 65 snap 0 seq_num 1347902061) dirty dne 2024-09-21T09:42:57.182 INFO:tasks.rados.rados.0.smithi022.stdout:3423: done (6 left) 2024-09-21T09:42:57.182 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 30 current snap is 330 2024-09-21T09:42:57.182 INFO:tasks.rados.rados.0.smithi022.stdout:3420: finishing write tid 4 to smithi02284960-7 2024-09-21T09:42:57.182 INFO:tasks.rados.rados.0.smithi022.stdout:3424: expect (ObjNum 1062 snap 319 seq_num 1062) 2024-09-21T09:42:57.208 INFO:tasks.rados.rados.0.smithi022.stdout:3420: finishing write tid 5 to smithi02284960-7 2024-09-21T09:42:57.208 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 7 v 2028 (ObjNum 1085 snap 330 seq_num 1085) dirty exists 2024-09-21T09:42:57.208 INFO:tasks.rados.rados.0.smithi022.stdout:3420: left oid 7 (ObjNum 1085 snap 330 seq_num 1085) 2024-09-21T09:42:57.208 INFO:tasks.rados.rados.0.smithi022.stdout:3428: seq_num 1088 ranges {3948544=630784,4579328=466944,5046272=335872} 2024-09-21T09:42:57.233 INFO:tasks.rados.rados.0.smithi022.stdout:3428: writing smithi02284960-30 from 3948544 to 4579328 tid 1 2024-09-21T09:42:57.236 INFO:tasks.rados.rados.0.smithi022.stdout:3428: writing smithi02284960-30 from 4579328 to 5046272 tid 2 2024-09-21T09:42:57.238 INFO:tasks.rados.rados.0.smithi022.stdout:3428: writing smithi02284960-30 from 5046272 to 5382144 tid 3 2024-09-21T09:42:57.239 INFO:tasks.rados.rados.0.smithi022.stdout:3420: done (6 left) 2024-09-21T09:42:57.239 INFO:tasks.rados.rados.0.smithi022.stdout:3424: done (5 left) 2024-09-21T09:42:57.239 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 31 current snap is 330 2024-09-21T09:42:57.239 INFO:tasks.rados.rados.0.smithi022.stdout:3422: expect (ObjNum 1029 snap 315 seq_num 1029) 2024-09-21T09:42:57.401 INFO:tasks.rados.rados.0.smithi022.stdout:3425: finishing copy_from racing read to smithi02284960-45 2024-09-21T09:42:57.401 INFO:tasks.rados.rados.0.smithi022.stdout:3425: finishing copy_from to smithi02284960-45 2024-09-21T09:42:57.401 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 45 v 1887 (ObjNum 1035 snap 315 seq_num 1035) dirty exists 2024-09-21T09:42:57.401 INFO:tasks.rados.rados.0.smithi022.stdout:3429: seq_num 1089 ranges {0=647168,647168=745472,1392640=57344} 2024-09-21T09:42:57.404 INFO:tasks.rados.rados.0.smithi022.stdout:3429: writing smithi02284960-31 from 0 to 647168 tid 1 2024-09-21T09:42:57.408 INFO:tasks.rados.rados.0.smithi022.stdout:3429: writing smithi02284960-31 from 647168 to 1392640 tid 2 2024-09-21T09:42:57.410 INFO:tasks.rados.rados.0.smithi022.stdout:3429: writing smithi02284960-31 from 1392640 to 1449984 tid 3 2024-09-21T09:42:57.410 INFO:tasks.rados.rados.0.smithi022.stdout:3422: done (5 left) 2024-09-21T09:42:57.411 INFO:tasks.rados.rados.0.smithi022.stdout:3425: done (4 left) 2024-09-21T09:42:57.411 INFO:tasks.rados.rados.0.smithi022.stdout:3430: rollback oid 22 current snap is 330 2024-09-21T09:42:57.411 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 22 to 327 2024-09-21T09:42:57.411 INFO:tasks.rados.rados.0.smithi022.stdout:3426: finishing write tid 1 to smithi02284960-26 2024-09-21T09:42:57.411 INFO:tasks.rados.rados.0.smithi022.stdout:3426: finishing write tid 2 to smithi02284960-26 2024-09-21T09:42:57.411 INFO:tasks.rados.rados.0.smithi022.stdout:3426: finishing write tid 3 to smithi02284960-26 2024-09-21T09:42:57.411 INFO:tasks.rados.rados.0.smithi022.stdout:3426: finishing write tid 4 to smithi02284960-26 2024-09-21T09:42:57.411 INFO:tasks.rados.rados.0.smithi022.stdout:3426: finishing write tid 5 to smithi02284960-26 2024-09-21T09:42:57.411 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 26 v 1955 (ObjNum 1087 snap 330 seq_num 1087) dirty exists 2024-09-21T09:42:57.411 INFO:tasks.rados.rados.0.smithi022.stdout:3426: left oid 26 (ObjNum 1087 snap 330 seq_num 1087) 2024-09-21T09:42:57.411 INFO:tasks.rados.rados.0.smithi022.stdout:3427: finishing copy_from racing read to smithi02284960-49 2024-09-21T09:42:57.411 INFO:tasks.rados.rados.0.smithi022.stdout:3427: finishing copy_from to smithi02284960-49 2024-09-21T09:42:57.411 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 49 v 1804 (ObjNum 985 snap 305 seq_num 985) dirty exists 2024-09-21T09:42:57.412 INFO:tasks.rados.rados.0.smithi022.stdout:3428: finishing write tid 1 to smithi02284960-30 2024-09-21T09:42:57.412 INFO:tasks.rados.rados.0.smithi022.stdout:3426: done (4 left) 2024-09-21T09:42:57.412 INFO:tasks.rados.rados.0.smithi022.stdout:3427: done (3 left) 2024-09-21T09:42:57.412 INFO:tasks.rados.rados.0.smithi022.stdout:3431: copy_from oid 24 from oid 16 current snap is 330 2024-09-21T09:42:57.412 INFO:tasks.rados.rados.0.smithi022.stdout:3428: finishing write tid 2 to smithi02284960-30 2024-09-21T09:42:57.412 INFO:tasks.rados.rados.0.smithi022.stdout:3428: finishing write tid 3 to smithi02284960-30 2024-09-21T09:42:57.412 INFO:tasks.rados.rados.0.smithi022.stdout:3428: finishing write tid 4 to smithi02284960-30 2024-09-21T09:42:57.412 INFO:tasks.rados.rados.0.smithi022.stdout:3428: finishing write tid 5 to smithi02284960-30 2024-09-21T09:42:57.412 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 30 v 1352 (ObjNum 1088 snap 330 seq_num 1088) dirty exists 2024-09-21T09:42:57.412 INFO:tasks.rados.rados.0.smithi022.stdout:3428: left oid 30 (ObjNum 1088 snap 330 seq_num 1088) 2024-09-21T09:42:57.412 INFO:tasks.rados.rados.0.smithi022.stdout:3429: finishing write tid 1 to smithi02284960-31 2024-09-21T09:42:57.412 INFO:tasks.rados.rados.0.smithi022.stdout:3428: done (3 left) 2024-09-21T09:42:57.412 INFO:tasks.rados.rados.0.smithi022.stdout:3432: snap_create 2024-09-21T09:42:57.415 INFO:tasks.rados.rados.0.smithi022.stdout:3430: finishing rollback tid 0 to smithi02284960-22 2024-09-21T09:42:57.415 INFO:tasks.rados.rados.0.smithi022.stdout:3430: finishing rollback tid 1 to smithi02284960-22 2024-09-21T09:42:57.415 INFO:tasks.rados.rados.0.smithi022.stdout:3429: finishing write tid 2 to smithi02284960-31 2024-09-21T09:42:57.416 INFO:tasks.rados.rados.0.smithi022.stdout:3429: finishing write tid 3 to smithi02284960-31 2024-09-21T09:42:57.416 INFO:tasks.rados.rados.0.smithi022.stdout:3430: finishing rollback tid 2 to smithi02284960-22 2024-09-21T09:42:57.416 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 22 v 1958 (ObjNum 1070 snap 322 seq_num 1070) dirty exists 2024-09-21T09:42:57.416 INFO:tasks.rados.rados.0.smithi022.stdout:3429: finishing write tid 4 to smithi02284960-31 2024-09-21T09:42:57.417 INFO:tasks.rados.rados.0.smithi022.stdout:3429: finishing write tid 5 to smithi02284960-31 2024-09-21T09:42:57.418 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 31 v 1951 (ObjNum 1089 snap 330 seq_num 1089) dirty exists 2024-09-21T09:42:57.418 INFO:tasks.rados.rados.0.smithi022.stdout:3429: left oid 31 (ObjNum 1089 snap 330 seq_num 1089) 2024-09-21T09:42:57.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:42:57 smithi064 ceph-mon[86709]: osdmap e734: 8 total, 8 up, 8 in 2024-09-21T09:42:57.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:42:57 smithi064 ceph-mon[86709]: pgmap v715: 121 pgs: 121 active+clean; 151 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6.9 MiB/s rd, 1.9 MiB/s wr, 7 op/s 2024-09-21T09:42:57.448 INFO:tasks.rados.rados.0.smithi022.stdout:3431: finishing copy_from racing read to smithi02284960-24 2024-09-21T09:42:57.453 INFO:tasks.rados.rados.0.smithi022.stdout:3431: finishing copy_from to smithi02284960-24 2024-09-21T09:42:57.453 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 24 v 1538 (ObjNum 1079 snap 326 seq_num 1079) dirty exists 2024-09-21T09:42:57.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:42:57 smithi022 ceph-mon[98615]: osdmap e734: 8 total, 8 up, 8 in 2024-09-21T09:42:57.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:42:57 smithi022 ceph-mon[98615]: pgmap v715: 121 pgs: 121 active+clean; 151 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6.9 MiB/s rd, 1.9 MiB/s wr, 7 op/s 2024-09-21T09:42:57.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:42:57 smithi022 ceph-mon[96736]: osdmap e734: 8 total, 8 up, 8 in 2024-09-21T09:42:57.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:42:57 smithi022 ceph-mon[96736]: pgmap v715: 121 pgs: 121 active+clean; 151 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6.9 MiB/s rd, 1.9 MiB/s wr, 7 op/s 2024-09-21T09:42:58.137 INFO:tasks.rados.rados.0.smithi022.stdout:3429: done (3 left) 2024-09-21T09:42:58.137 INFO:tasks.rados.rados.0.smithi022.stdout:3430: done (2 left) 2024-09-21T09:42:58.137 INFO:tasks.rados.rados.0.smithi022.stdout:3431: done (1 left) 2024-09-21T09:42:58.137 INFO:tasks.rados.rados.0.smithi022.stdout:3432: done (0 left) 2024-09-21T09:42:58.137 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 2 current snap is 331 2024-09-21T09:42:58.137 INFO:tasks.rados.rados.0.smithi022.stdout:3433: seq_num 1090 ranges {1982464=663552,2646016=630784,3276800=172032} 2024-09-21T09:42:58.151 INFO:tasks.rados.rados.0.smithi022.stdout:3433: writing smithi02284960-2 from 1982464 to 2646016 tid 1 2024-09-21T09:42:58.155 INFO:tasks.rados.rados.0.smithi022.stdout:3433: writing smithi02284960-2 from 2646016 to 3276800 tid 2 2024-09-21T09:42:58.156 INFO:tasks.rados.rados.0.smithi022.stdout:3433: writing smithi02284960-2 from 3276800 to 3448832 tid 3 2024-09-21T09:42:58.156 INFO:tasks.rados.rados.0.smithi022.stdout:3434: setattr oid 41 current snap is 331 2024-09-21T09:42:58.157 INFO:tasks.rados.rados.0.smithi022.stdout:3435: read oid 44 snap -1 2024-09-21T09:42:58.158 INFO:tasks.rados.rados.0.smithi022.stdout:3435: expect (ObjNum 1002 snap 307 seq_num 1002) 2024-09-21T09:42:58.158 INFO:tasks.rados.rados.0.smithi022.stdout:3436: read oid 6 snap -1 2024-09-21T09:42:58.158 INFO:tasks.rados.rados.0.smithi022.stdout:3436: expect deleted 2024-09-21T09:42:58.158 INFO:tasks.rados.rados.0.smithi022.stdout:3437: read oid 35 snap -1 2024-09-21T09:42:58.158 INFO:tasks.rados.rados.0.smithi022.stdout:3437: expect (ObjNum 1075 snap 325 seq_num 1075) 2024-09-21T09:42:58.158 INFO:tasks.rados.rados.0.smithi022.stdout:3438: delete oid 45 current snap is 331 2024-09-21T09:42:58.163 INFO:tasks.rados.rados.0.smithi022.stdout:3433: finishing write tid 1 to smithi02284960-2 2024-09-21T09:42:58.163 INFO:tasks.rados.rados.0.smithi022.stdout:3433: finishing write tid 2 to smithi02284960-2 2024-09-21T09:42:58.165 INFO:tasks.rados.rados.0.smithi022.stdout:3438: done (5 left) 2024-09-21T09:42:58.165 INFO:tasks.rados.rados.0.smithi022.stdout:3439: read oid 48 snap -1 2024-09-21T09:42:58.165 INFO:tasks.rados.rados.0.smithi022.stdout:3439: expect deleted 2024-09-21T09:42:58.165 INFO:tasks.rados.rados.0.smithi022.stdout:3440: rollback oid 46 current snap is 331 2024-09-21T09:42:58.165 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 46 to 329 2024-09-21T09:42:58.165 INFO:tasks.rados.rados.0.smithi022.stdout:3441: snap_create 2024-09-21T09:42:58.167 INFO:tasks.rados.rados.0.smithi022.stdout:3433: finishing write tid 3 to smithi02284960-2 2024-09-21T09:42:58.167 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 41 v 1893 (ObjNum 193 snap 0 seq_num 3569891232) dirty exists 2024-09-21T09:42:58.167 INFO:tasks.rados.rados.0.smithi022.stdout:3433: finishing write tid 4 to smithi02284960-2 2024-09-21T09:42:58.170 INFO:tasks.rados.rados.0.smithi022.stdout:3433: finishing write tid 5 to smithi02284960-2 2024-09-21T09:42:58.170 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 2 v 1823 (ObjNum 1090 snap 331 seq_num 1090) dirty exists 2024-09-21T09:42:58.170 INFO:tasks.rados.rados.0.smithi022.stdout:3433: left oid 2 (ObjNum 1090 snap 331 seq_num 1090) 2024-09-21T09:42:58.171 INFO:tasks.rados.rados.0.smithi022.stdout:3440: finishing rollback tid 0 to smithi02284960-46 2024-09-21T09:42:58.176 INFO:tasks.rados.rados.0.smithi022.stdout:3440: finishing rollback tid 1 to smithi02284960-46 2024-09-21T09:42:58.176 INFO:tasks.rados.rados.0.smithi022.stdout:3440: finishing rollback tid 2 to smithi02284960-46 2024-09-21T09:42:58.176 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 46 v 1763 (ObjNum 1024 snap 315 seq_num 1024) dirty exists 2024-09-21T09:42:58.179 INFO:tasks.rados.rados.0.smithi022.stdout:3437: expect (ObjNum 1075 snap 325 seq_num 1075) 2024-09-21T09:42:58.400 INFO:tasks.rados.rados.0.smithi022.stdout:3435: expect (ObjNum 1002 snap 307 seq_num 1002) 2024-09-21T09:42:58.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:42:58 smithi064 ceph-mon[86709]: osdmap e735: 8 total, 8 up, 8 in 2024-09-21T09:42:58.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:42:58 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:42:58.557 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:42:58 smithi022 ceph-mon[98615]: osdmap e735: 8 total, 8 up, 8 in 2024-09-21T09:42:58.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:42:58 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:42:58.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:42:58 smithi022 ceph-mon[96736]: osdmap e735: 8 total, 8 up, 8 in 2024-09-21T09:42:58.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:42:58 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:42:59.074 INFO:tasks.rados.rados.0.smithi022.stdout:3433: done (7 left) 2024-09-21T09:42:59.074 INFO:tasks.rados.rados.0.smithi022.stdout:3434: done (6 left) 2024-09-21T09:42:59.074 INFO:tasks.rados.rados.0.smithi022.stdout:3435: done (5 left) 2024-09-21T09:42:59.074 INFO:tasks.rados.rados.0.smithi022.stdout:3436: done (4 left) 2024-09-21T09:42:59.074 INFO:tasks.rados.rados.0.smithi022.stdout:3437: done (3 left) 2024-09-21T09:42:59.074 INFO:tasks.rados.rados.0.smithi022.stdout:3439: done (2 left) 2024-09-21T09:42:59.074 INFO:tasks.rados.rados.0.smithi022.stdout:3440: done (1 left) 2024-09-21T09:42:59.074 INFO:tasks.rados.rados.0.smithi022.stdout:3441: done (0 left) 2024-09-21T09:42:59.074 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 4 current snap is 332 2024-09-21T09:42:59.075 INFO:tasks.rados.rados.0.smithi022.stdout:3442: seq_num 1092 ranges {0=696320,696320=802816} 2024-09-21T09:42:59.078 INFO:tasks.rados.rados.0.smithi022.stdout:3442: writing smithi02284960-4 from 0 to 696320 tid 1 2024-09-21T09:42:59.083 INFO:tasks.rados.rados.0.smithi022.stdout:3442: writing smithi02284960-4 from 696320 to 1499136 tid 2 2024-09-21T09:42:59.083 INFO:tasks.rados.rados.0.smithi022.stdout:3443: copy_from oid 17 from oid 13 current snap is 332 2024-09-21T09:42:59.083 INFO:tasks.rados.rados.0.smithi022.stdout:3444: snap_remove snap 331 2024-09-21T09:42:59.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:42:59 smithi064 ceph-mon[86709]: osdmap e736: 8 total, 8 up, 8 in 2024-09-21T09:42:59.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:42:59 smithi064 ceph-mon[86709]: pgmap v718: 121 pgs: 121 active+clean; 161 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 9.1 MiB/s rd, 3.8 MiB/s wr, 8 op/s 2024-09-21T09:42:59.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:42:59 smithi064 ceph-mon[86709]: osdmap e737: 8 total, 8 up, 8 in 2024-09-21T09:42:59.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:42:59 smithi022 ceph-mon[98615]: osdmap e736: 8 total, 8 up, 8 in 2024-09-21T09:42:59.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:42:59 smithi022 ceph-mon[98615]: pgmap v718: 121 pgs: 121 active+clean; 161 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 9.1 MiB/s rd, 3.8 MiB/s wr, 8 op/s 2024-09-21T09:42:59.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:42:59 smithi022 ceph-mon[98615]: osdmap e737: 8 total, 8 up, 8 in 2024-09-21T09:42:59.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:42:59 smithi022 ceph-mon[96736]: osdmap e736: 8 total, 8 up, 8 in 2024-09-21T09:42:59.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:42:59 smithi022 ceph-mon[96736]: pgmap v718: 121 pgs: 121 active+clean; 161 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 9.1 MiB/s rd, 3.8 MiB/s wr, 8 op/s 2024-09-21T09:42:59.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:42:59 smithi022 ceph-mon[96736]: osdmap e737: 8 total, 8 up, 8 in 2024-09-21T09:43:00.075 INFO:tasks.rados.rados.0.smithi022.stdout:3444: done (2 left) 2024-09-21T09:43:00.075 INFO:tasks.rados.rados.0.smithi022.stdout:3445: delete oid 42 current snap is 332 2024-09-21T09:43:00.076 INFO:tasks.rados.rados.0.smithi022.stdout:3442: finishing write tid 1 to smithi02284960-4 2024-09-21T09:43:00.076 INFO:tasks.rados.rados.0.smithi022.stdout:3442: finishing write tid 2 to smithi02284960-4 2024-09-21T09:43:00.076 INFO:tasks.rados.rados.0.smithi022.stdout:3442: finishing write tid 3 to smithi02284960-4 2024-09-21T09:43:00.076 INFO:tasks.rados.rados.0.smithi022.stdout:3442: finishing write tid 4 to smithi02284960-4 2024-09-21T09:43:00.076 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 4 v 1954 (ObjNum 1092 snap 332 seq_num 1092) dirty exists 2024-09-21T09:43:00.076 INFO:tasks.rados.rados.0.smithi022.stdout:3442: left oid 4 (ObjNum 1092 snap 332 seq_num 1092) 2024-09-21T09:43:00.076 INFO:tasks.rados.rados.0.smithi022.stdout:3443: finishing copy_from racing read to smithi02284960-17 2024-09-21T09:43:00.076 INFO:tasks.rados.rados.0.smithi022.stdout:3443: finishing copy_from to smithi02284960-17 2024-09-21T09:43:00.076 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 17 v 1802 (ObjNum 1053 snap 317 seq_num 1053) dirty exists 2024-09-21T09:43:00.081 INFO:tasks.rados.rados.0.smithi022.stdout:3442: done (2 left) 2024-09-21T09:43:00.081 INFO:tasks.rados.rados.0.smithi022.stdout:3443: done (1 left) 2024-09-21T09:43:00.082 INFO:tasks.rados.rados.0.smithi022.stdout:3445: done (0 left) 2024-09-21T09:43:00.082 INFO:tasks.rados.rados.0.smithi022.stdout:3446: setattr oid 12 current snap is 332 2024-09-21T09:43:00.083 INFO:tasks.rados.rados.0.smithi022.stdout:3447: read oid 5 snap -1 2024-09-21T09:43:00.083 INFO:tasks.rados.rados.0.smithi022.stdout:3447: expect (ObjNum 1070 snap 322 seq_num 1070) 2024-09-21T09:43:00.083 INFO:tasks.rados.rados.0.smithi022.stdout:3448: read oid 33 snap 327 2024-09-21T09:43:00.083 INFO:tasks.rados.rados.0.smithi022.stdout:3448: expect deleted 2024-09-21T09:43:00.083 INFO:tasks.rados.rados.0.smithi022.stdout:3449: copy_from oid 10 from oid 41 current snap is 332 2024-09-21T09:43:00.083 INFO:tasks.rados.rados.0.smithi022.stdout:3450: read oid 47 snap 327 2024-09-21T09:43:00.083 INFO:tasks.rados.rados.0.smithi022.stdout:3450: expect (ObjNum 1066 snap 321 seq_num 1066) 2024-09-21T09:43:00.083 INFO:tasks.rados.rados.0.smithi022.stdout:3451: rollback oid 3 current snap is 332 2024-09-21T09:43:00.083 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 3 to 329 2024-09-21T09:43:00.083 INFO:tasks.rados.rados.0.smithi022.stdout:3452: rmattr oid 36 current snap is 332 2024-09-21T09:43:00.083 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 17 current snap is 332 2024-09-21T09:43:00.084 INFO:tasks.rados.rados.0.smithi022.stdout:3453: seq_num 1094 ranges {843776=729088,1572864=573440,2146304=229376} 2024-09-21T09:43:00.092 INFO:tasks.rados.rados.0.smithi022.stdout:3453: writing smithi02284960-17 from 843776 to 1572864 tid 1 2024-09-21T09:43:00.096 INFO:tasks.rados.rados.0.smithi022.stdout:3453: writing smithi02284960-17 from 1572864 to 2146304 tid 2 2024-09-21T09:43:00.097 INFO:tasks.rados.rados.0.smithi022.stdout:3453: writing smithi02284960-17 from 2146304 to 2375680 tid 3 2024-09-21T09:43:00.099 INFO:tasks.rados.rados.0.smithi022.stdout:3454: snap_create 2024-09-21T09:43:00.099 INFO:tasks.rados.rados.0.smithi022.stdout:3449: finishing copy_from racing read to smithi02284960-10 2024-09-21T09:43:00.099 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 36 v 1889 (ObjNum 1033 snap 315 seq_num 1033) dirty exists 2024-09-21T09:43:00.099 INFO:tasks.rados.rados.0.smithi022.stdout:3449: finishing copy_from to smithi02284960-10 2024-09-21T09:43:00.099 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 10 v 1893 (ObjNum 49 snap 0 seq_num 1953066355) dirty exists 2024-09-21T09:43:00.099 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 12 v 1955 (ObjNum 997 snap 306 seq_num 997) dirty exists 2024-09-21T09:43:00.099 INFO:tasks.rados.rados.0.smithi022.stdout:3451: finishing rollback tid 0 to smithi02284960-3 2024-09-21T09:43:00.099 INFO:tasks.rados.rados.0.smithi022.stdout:3451: finishing rollback tid 1 to smithi02284960-3 2024-09-21T09:43:00.099 INFO:tasks.rados.rados.0.smithi022.stdout:3451: finishing rollback tid 2 to smithi02284960-3 2024-09-21T09:43:00.099 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 3 v 630 (ObjNum 1078 snap 326 seq_num 1078) dirty exists 2024-09-21T09:43:00.100 INFO:tasks.rados.rados.0.smithi022.stdout:3450: expect (ObjNum 1066 snap 321 seq_num 1066) 2024-09-21T09:43:00.129 INFO:tasks.rados.rados.0.smithi022.stdout:3453: finishing write tid 1 to smithi02284960-17 2024-09-21T09:43:00.129 INFO:tasks.rados.rados.0.smithi022.stdout:3453: finishing write tid 2 to smithi02284960-17 2024-09-21T09:43:00.129 INFO:tasks.rados.rados.0.smithi022.stdout:3453: finishing write tid 3 to smithi02284960-17 2024-09-21T09:43:00.129 INFO:tasks.rados.rados.0.smithi022.stdout:3453: finishing write tid 4 to smithi02284960-17 2024-09-21T09:43:00.129 INFO:tasks.rados.rados.0.smithi022.stdout:3453: finishing write tid 5 to smithi02284960-17 2024-09-21T09:43:00.129 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 17 v 1806 (ObjNum 1094 snap 332 seq_num 1094) dirty exists 2024-09-21T09:43:00.129 INFO:tasks.rados.rados.0.smithi022.stdout:3453: left oid 17 (ObjNum 1094 snap 332 seq_num 1094) 2024-09-21T09:43:00.130 INFO:tasks.rados.rados.0.smithi022.stdout:3447: expect (ObjNum 1070 snap 322 seq_num 1070) 2024-09-21T09:43:00.692 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:43:00 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:43:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:43:01.080 INFO:tasks.rados.rados.0.smithi022.stdout:3446: done (8 left) 2024-09-21T09:43:01.080 INFO:tasks.rados.rados.0.smithi022.stdout:3447: done (7 left) 2024-09-21T09:43:01.080 INFO:tasks.rados.rados.0.smithi022.stdout:3448: done (6 left) 2024-09-21T09:43:01.080 INFO:tasks.rados.rados.0.smithi022.stdout:3449: done (5 left) 2024-09-21T09:43:01.080 INFO:tasks.rados.rados.0.smithi022.stdout:3450: done (4 left) 2024-09-21T09:43:01.081 INFO:tasks.rados.rados.0.smithi022.stdout:3451: done (3 left) 2024-09-21T09:43:01.081 INFO:tasks.rados.rados.0.smithi022.stdout:3452: done (2 left) 2024-09-21T09:43:01.081 INFO:tasks.rados.rados.0.smithi022.stdout:3453: done (1 left) 2024-09-21T09:43:01.081 INFO:tasks.rados.rados.0.smithi022.stdout:3454: done (0 left) 2024-09-21T09:43:01.081 INFO:tasks.rados.rados.0.smithi022.stdout:3455: copy_from oid 19 from oid 27 current snap is 333 2024-09-21T09:43:01.081 INFO:tasks.rados.rados.0.smithi022.stdout:3456: snap_remove snap 315 2024-09-21T09:43:01.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:43:01 smithi064 ceph-mon[86709]: osdmap e738: 8 total, 8 up, 8 in 2024-09-21T09:43:01.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:43:01 smithi022 ceph-mon[96736]: osdmap e738: 8 total, 8 up, 8 in 2024-09-21T09:43:01.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:43:01 smithi022 ceph-mon[98615]: osdmap e738: 8 total, 8 up, 8 in 2024-09-21T09:43:02.081 INFO:tasks.rados.rados.0.smithi022.stdout:3456: done (1 left) 2024-09-21T09:43:02.082 INFO:tasks.rados.rados.0.smithi022.stdout:3457: read oid 21 snap 327 2024-09-21T09:43:02.082 INFO:tasks.rados.rados.0.smithi022.stdout:3457: expect (ObjNum 1029 snap 315 seq_num 1029) 2024-09-21T09:43:02.082 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 7 current snap is 333 2024-09-21T09:43:02.082 INFO:tasks.rados.rados.0.smithi022.stdout:3458: seq_num 1095 ranges {1384448=745472,2129920=499712,2629632=303104} 2024-09-21T09:43:02.095 INFO:tasks.rados.rados.0.smithi022.stdout:3458: writing smithi02284960-7 from 1384448 to 2129920 tid 1 2024-09-21T09:43:02.098 INFO:tasks.rados.rados.0.smithi022.stdout:3458: writing smithi02284960-7 from 2129920 to 2629632 tid 2 2024-09-21T09:43:02.101 INFO:tasks.rados.rados.0.smithi022.stdout:3458: writing smithi02284960-7 from 2629632 to 2932736 tid 3 2024-09-21T09:43:02.101 INFO:tasks.rados.rados.0.smithi022.stdout:3455: finishing copy_from racing read to smithi02284960-19 2024-09-21T09:43:02.101 INFO:tasks.rados.rados.0.smithi022.stdout:3459: read oid 38 snap 327 2024-09-21T09:43:02.101 INFO:tasks.rados.rados.0.smithi022.stdout:3459: expect deleted 2024-09-21T09:43:02.101 INFO:tasks.rados.rados.0.smithi022.stdout:3460: delete oid 1 current snap is 333 2024-09-21T09:43:02.101 INFO:tasks.rados.rados.0.smithi022.stdout:3455: finishing copy_from to smithi02284960-19 2024-09-21T09:43:02.101 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 19 v 1795 (ObjNum 1008 snap 309 seq_num 1008) dirty exists 2024-09-21T09:43:02.103 INFO:tasks.rados.rados.0.smithi022.stdout:3457: expect (ObjNum 1029 snap 315 seq_num 1029) 2024-09-21T09:43:02.275 INFO:tasks.rados.rados.0.smithi022.stdout:3458: finishing write tid 1 to smithi02284960-7 2024-09-21T09:43:02.275 INFO:tasks.rados.rados.0.smithi022.stdout:3458: finishing write tid 2 to smithi02284960-7 2024-09-21T09:43:02.275 INFO:tasks.rados.rados.0.smithi022.stdout:3458: finishing write tid 3 to smithi02284960-7 2024-09-21T09:43:02.275 INFO:tasks.rados.rados.0.smithi022.stdout:3458: finishing write tid 4 to smithi02284960-7 2024-09-21T09:43:02.275 INFO:tasks.rados.rados.0.smithi022.stdout:3458: finishing write tid 5 to smithi02284960-7 2024-09-21T09:43:02.275 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 7 v 2033 (ObjNum 1095 snap 333 seq_num 1095) dirty exists 2024-09-21T09:43:02.276 INFO:tasks.rados.rados.0.smithi022.stdout:3458: left oid 7 (ObjNum 1095 snap 333 seq_num 1095) 2024-09-21T09:43:02.276 INFO:tasks.rados.rados.0.smithi022.stdout:3455: done (4 left) 2024-09-21T09:43:02.276 INFO:tasks.rados.rados.0.smithi022.stdout:3457: done (3 left) 2024-09-21T09:43:02.276 INFO:tasks.rados.rados.0.smithi022.stdout:3458: done (2 left) 2024-09-21T09:43:02.276 INFO:tasks.rados.rados.0.smithi022.stdout:3459: done (1 left) 2024-09-21T09:43:02.276 INFO:tasks.rados.rados.0.smithi022.stdout:3460: done (0 left) 2024-09-21T09:43:02.276 INFO:tasks.rados.rados.0.smithi022.stdout:3461: read oid 27 snap -1 2024-09-21T09:43:02.276 INFO:tasks.rados.rados.0.smithi022.stdout:3461: expect (ObjNum 1008 snap 309 seq_num 1008) 2024-09-21T09:43:02.276 INFO:tasks.rados.rados.0.smithi022.stdout:3462: snap_remove snap 319 2024-09-21T09:43:02.441 INFO:journalctl@ceph.osd.5.smithi064.stdout:Sep 21 09:43:02 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-5[96798]: 2024-09-21T09:43:02.095+0000 7fe939b2d640 -1 snap_mapper.get_next_objects_to_trim::New Clone-Objects were added to Snap 194 after trimming was started 2024-09-21T09:43:02.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:43:02 smithi064 ceph-mon[86709]: pgmap v721: 121 pgs: 121 active+clean; 164 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 9.2 MiB/s rd, 3.2 MiB/s wr, 15 op/s 2024-09-21T09:43:02.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:43:02 smithi064 ceph-mon[86709]: osdmap e739: 8 total, 8 up, 8 in 2024-09-21T09:43:02.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:43:02 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:43:02.557 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:43:02 smithi022 ceph-mon[98615]: pgmap v721: 121 pgs: 121 active+clean; 164 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 9.2 MiB/s rd, 3.2 MiB/s wr, 15 op/s 2024-09-21T09:43:02.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:43:02 smithi022 ceph-mon[98615]: osdmap e739: 8 total, 8 up, 8 in 2024-09-21T09:43:02.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:43:02 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:43:02.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:43:02 smithi022 ceph-mon[96736]: pgmap v721: 121 pgs: 121 active+clean; 164 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 9.2 MiB/s rd, 3.2 MiB/s wr, 15 op/s 2024-09-21T09:43:02.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:43:02 smithi022 ceph-mon[96736]: osdmap e739: 8 total, 8 up, 8 in 2024-09-21T09:43:02.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:43:02 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:43:03.256 INFO:tasks.rados.rados.0.smithi022.stdout:3462: done (1 left) 2024-09-21T09:43:03.256 INFO:tasks.rados.rados.0.smithi022.stdout:3463: snap_remove snap 327 2024-09-21T09:43:03.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:43:03 smithi064 ceph-mon[86709]: osdmap e740: 8 total, 8 up, 8 in 2024-09-21T09:43:03.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:43:03 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:43:03.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:43:03 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:43:03.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:43:03 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:43:03.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:43:03 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:43:03.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:43:03 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:43:03.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:43:03 smithi022 ceph-mon[98615]: osdmap e740: 8 total, 8 up, 8 in 2024-09-21T09:43:03.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:43:03 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:43:03.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:43:03 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:43:03.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:43:03 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:43:03.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:43:03 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:43:03.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:43:03 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:43:03.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:43:03 smithi022 ceph-mon[96736]: osdmap e740: 8 total, 8 up, 8 in 2024-09-21T09:43:03.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:43:03 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:43:03.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:43:03 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:43:03.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:43:03 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:43:03.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:43:03 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:43:03.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:43:03 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:43:04.075 INFO:tasks.rados.rados.0.smithi022.stdout:3463: done (1 left) 2024-09-21T09:43:04.075 INFO:tasks.rados.rados.0.smithi022.stdout:3464: snap_create 2024-09-21T09:43:04.075 INFO:tasks.rados.rados.0.smithi022.stdout:3461: expect (ObjNum 1008 snap 309 seq_num 1008) 2024-09-21T09:43:04.343 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:43:04 smithi064 ceph-mon[86709]: pgmap v724: 121 pgs: 121 active+clean; 172 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 7.0 MiB/s rd, 4.1 MiB/s wr, 25 op/s 2024-09-21T09:43:04.343 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:43:04 smithi064 ceph-mon[86709]: osdmap e741: 8 total, 8 up, 8 in 2024-09-21T09:43:04.343 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:43:04 smithi064 ceph-mon[86709]: osdmap e742: 8 total, 8 up, 8 in 2024-09-21T09:43:04.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:43:04 smithi022 ceph-mon[98615]: pgmap v724: 121 pgs: 121 active+clean; 172 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 7.0 MiB/s rd, 4.1 MiB/s wr, 25 op/s 2024-09-21T09:43:04.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:43:04 smithi022 ceph-mon[98615]: osdmap e741: 8 total, 8 up, 8 in 2024-09-21T09:43:04.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:43:04 smithi022 ceph-mon[98615]: osdmap e742: 8 total, 8 up, 8 in 2024-09-21T09:43:04.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:43:04 smithi022 ceph-mon[96736]: pgmap v724: 121 pgs: 121 active+clean; 172 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 7.0 MiB/s rd, 4.1 MiB/s wr, 25 op/s 2024-09-21T09:43:04.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:43:04 smithi022 ceph-mon[96736]: osdmap e741: 8 total, 8 up, 8 in 2024-09-21T09:43:04.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:43:04 smithi022 ceph-mon[96736]: osdmap e742: 8 total, 8 up, 8 in 2024-09-21T09:43:05.077 INFO:tasks.rados.rados.0.smithi022.stdout:3461: done (1 left) 2024-09-21T09:43:05.077 INFO:tasks.rados.rados.0.smithi022.stdout:3464: done (0 left) 2024-09-21T09:43:05.077 INFO:tasks.rados.rados.0.smithi022.stdout:3465: delete oid 7 current snap is 334 2024-09-21T09:43:05.080 INFO:tasks.rados.rados.0.smithi022.stdout:3465: done (0 left) 2024-09-21T09:43:05.080 INFO:tasks.rados.rados.0.smithi022.stdout:3466: setattr oid 17 current snap is 334 2024-09-21T09:43:05.082 INFO:tasks.rados.rados.0.smithi022.stdout:3467: snap_remove snap 329 2024-09-21T09:43:06.058 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:43:05 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:43:05] "GET /metrics HTTP/1.1" 200 36354 "" "Prometheus/2.51.0" 2024-09-21T09:43:06.082 INFO:tasks.rados.rados.0.smithi022.stdout:3467: done (1 left) 2024-09-21T09:43:06.082 INFO:tasks.rados.rados.0.smithi022.stdout:3468: snap_create 2024-09-21T09:43:06.082 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 17 v 1807 (ObjNum 1094 snap 332 seq_num 1094) dirty exists 2024-09-21T09:43:06.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:43:06 smithi064 ceph-mon[86709]: pgmap v727: 121 pgs: 121 active+clean; 170 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5.6 MiB/s rd, 3.4 MiB/s wr, 18 op/s 2024-09-21T09:43:06.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:43:06 smithi064 ceph-mon[86709]: osdmap e743: 8 total, 8 up, 8 in 2024-09-21T09:43:06.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:43:06 smithi022 ceph-mon[98615]: pgmap v727: 121 pgs: 121 active+clean; 170 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5.6 MiB/s rd, 3.4 MiB/s wr, 18 op/s 2024-09-21T09:43:06.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:43:06 smithi022 ceph-mon[98615]: osdmap e743: 8 total, 8 up, 8 in 2024-09-21T09:43:06.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:43:06 smithi022 ceph-mon[96736]: pgmap v727: 121 pgs: 121 active+clean; 170 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5.6 MiB/s rd, 3.4 MiB/s wr, 18 op/s 2024-09-21T09:43:06.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:43:06 smithi022 ceph-mon[96736]: osdmap e743: 8 total, 8 up, 8 in 2024-09-21T09:43:07.087 INFO:tasks.rados.rados.0.smithi022.stdout:3466: done (1 left) 2024-09-21T09:43:07.087 INFO:tasks.rados.rados.0.smithi022.stdout:3468: done (0 left) 2024-09-21T09:43:07.087 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 46 current snap is 335 2024-09-21T09:43:07.087 INFO:tasks.rados.rados.0.smithi022.stdout:3469: seq_num 1098 ranges {1638400=794624,2433024=557056,2990080=245760} 2024-09-21T09:43:07.100 INFO:tasks.rados.rados.0.smithi022.stdout:3469: writing smithi02284960-46 from 1638400 to 2433024 tid 1 2024-09-21T09:43:07.103 INFO:tasks.rados.rados.0.smithi022.stdout:3469: writing smithi02284960-46 from 2433024 to 2990080 tid 2 2024-09-21T09:43:07.105 INFO:tasks.rados.rados.0.smithi022.stdout:3469: writing smithi02284960-46 from 2990080 to 3235840 tid 3 2024-09-21T09:43:07.105 INFO:tasks.rados.rados.0.smithi022.stdout:3470: read oid 11 snap -1 2024-09-21T09:43:07.105 INFO:tasks.rados.rados.0.smithi022.stdout:3470: expect (ObjNum 1062 snap 319 seq_num 1062) 2024-09-21T09:43:07.105 INFO:tasks.rados.rados.0.smithi022.stdout:3471: setattr oid 38 current snap is 335 2024-09-21T09:43:07.106 INFO:tasks.rados.rados.0.smithi022.stdout:3472: snap_remove snap 333 2024-09-21T09:43:07.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:43:07 smithi064 ceph-mon[86709]: osdmap e744: 8 total, 8 up, 8 in 2024-09-21T09:43:07.557 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:43:07 smithi022 ceph-mon[98615]: osdmap e744: 8 total, 8 up, 8 in 2024-09-21T09:43:07.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:43:07 smithi022 ceph-mon[96736]: osdmap e744: 8 total, 8 up, 8 in 2024-09-21T09:43:08.095 INFO:tasks.rados.rados.0.smithi022.stdout:3472: done (3 left) 2024-09-21T09:43:08.096 INFO:tasks.rados.rados.0.smithi022.stdout:3473: snap_remove snap 334 2024-09-21T09:43:08.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:43:08 smithi064 ceph-mon[86709]: pgmap v730: 121 pgs: 121 active+clean; 164 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5.6 MiB/s rd, 378 KiB/s wr, 14 op/s 2024-09-21T09:43:08.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:43:08 smithi064 ceph-mon[86709]: osdmap e745: 8 total, 8 up, 8 in 2024-09-21T09:43:08.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:43:08 smithi022 ceph-mon[98615]: pgmap v730: 121 pgs: 121 active+clean; 164 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5.6 MiB/s rd, 378 KiB/s wr, 14 op/s 2024-09-21T09:43:08.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:43:08 smithi022 ceph-mon[98615]: osdmap e745: 8 total, 8 up, 8 in 2024-09-21T09:43:08.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:43:08 smithi022 ceph-mon[96736]: pgmap v730: 121 pgs: 121 active+clean; 164 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5.6 MiB/s rd, 378 KiB/s wr, 14 op/s 2024-09-21T09:43:08.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:43:08 smithi022 ceph-mon[96736]: osdmap e745: 8 total, 8 up, 8 in 2024-09-21T09:43:09.078 INFO:tasks.rados.rados.0.smithi022.stdout:3469: finishing write tid 1 to smithi02284960-46 2024-09-21T09:43:09.078 INFO:tasks.rados.rados.0.smithi022.stdout:3473: done (3 left) 2024-09-21T09:43:09.078 INFO:tasks.rados.rados.0.smithi022.stdout:3474: snap_create 2024-09-21T09:43:09.078 INFO:tasks.rados.rados.0.smithi022.stdout:3469: finishing write tid 2 to smithi02284960-46 2024-09-21T09:43:09.078 INFO:tasks.rados.rados.0.smithi022.stdout:3469: finishing write tid 3 to smithi02284960-46 2024-09-21T09:43:09.078 INFO:tasks.rados.rados.0.smithi022.stdout:3469: finishing write tid 4 to smithi02284960-46 2024-09-21T09:43:09.078 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 38 v 1820 (ObjNum 161 snap 0 seq_num 3569549136) dirty exists 2024-09-21T09:43:09.078 INFO:tasks.rados.rados.0.smithi022.stdout:3469: finishing write tid 5 to smithi02284960-46 2024-09-21T09:43:09.078 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 46 v 1790 (ObjNum 1098 snap 335 seq_num 1098) dirty exists 2024-09-21T09:43:09.078 INFO:tasks.rados.rados.0.smithi022.stdout:3469: left oid 46 (ObjNum 1098 snap 335 seq_num 1098) 2024-09-21T09:43:09.078 INFO:tasks.rados.rados.0.smithi022.stdout:3470: expect (ObjNum 1062 snap 319 seq_num 1062) 2024-09-21T09:43:09.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:43:09 smithi064 ceph-mon[86709]: osdmap e746: 8 total, 8 up, 8 in 2024-09-21T09:43:09.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:43:09 smithi064 ceph-mon[86709]: osdmap e747: 8 total, 8 up, 8 in 2024-09-21T09:43:09.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:43:09 smithi022 ceph-mon[98615]: osdmap e746: 8 total, 8 up, 8 in 2024-09-21T09:43:09.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:43:09 smithi022 ceph-mon[98615]: osdmap e747: 8 total, 8 up, 8 in 2024-09-21T09:43:09.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:43:09 smithi022 ceph-mon[96736]: osdmap e746: 8 total, 8 up, 8 in 2024-09-21T09:43:09.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:43:09 smithi022 ceph-mon[96736]: osdmap e747: 8 total, 8 up, 8 in 2024-09-21T09:43:10.079 INFO:tasks.rados.rados.0.smithi022.stdout:3469: done (3 left) 2024-09-21T09:43:10.079 INFO:tasks.rados.rados.0.smithi022.stdout:3470: done (2 left) 2024-09-21T09:43:10.079 INFO:tasks.rados.rados.0.smithi022.stdout:3471: done (1 left) 2024-09-21T09:43:10.079 INFO:tasks.rados.rados.0.smithi022.stdout:3474: done (0 left) 2024-09-21T09:43:10.079 INFO:tasks.rados.rados.0.smithi022.stdout:3475: rmattr oid 20 current snap is 336 2024-09-21T09:43:10.079 INFO:tasks.rados.rados.0.smithi022.stdout:3475: done (0 left) 2024-09-21T09:43:10.079 INFO:tasks.rados.rados.0.smithi022.stdout:3476: rollback oid 4 current snap is 336 2024-09-21T09:43:10.079 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 4 to 317 2024-09-21T09:43:10.079 INFO:tasks.rados.rados.0.smithi022.stdout:3477: rmattr oid 11 current snap is 336 2024-09-21T09:43:10.080 INFO:tasks.rados.rados.0.smithi022.stdout:3477: done (1 left) 2024-09-21T09:43:10.080 INFO:tasks.rados.rados.0.smithi022.stdout:3478: read oid 23 snap 323 2024-09-21T09:43:10.080 INFO:tasks.rados.rados.0.smithi022.stdout:3478: expect (ObjNum 1025 snap 315 seq_num 1025) 2024-09-21T09:43:10.080 INFO:tasks.rados.rados.0.smithi022.stdout:3479: read oid 22 snap 317 2024-09-21T09:43:10.080 INFO:tasks.rados.rados.0.smithi022.stdout:3479: expect (ObjNum 914 snap 284 seq_num 914) 2024-09-21T09:43:10.080 INFO:tasks.rados.rados.0.smithi022.stdout:3480: copy_from oid 30 from oid 8 current snap is 336 2024-09-21T09:43:10.080 INFO:tasks.rados.rados.0.smithi022.stdout:3481: read oid 14 snap 328 2024-09-21T09:43:10.080 INFO:tasks.rados.rados.0.smithi022.stdout:3481: expect (ObjNum 985 snap 305 seq_num 985) 2024-09-21T09:43:10.080 INFO:tasks.rados.rados.0.smithi022.stdout:3482: snap_remove snap 332 2024-09-21T09:43:10.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:43:10 smithi064 ceph-mon[86709]: pgmap v733: 121 pgs: 121 active+clean; 164 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 66 KiB/s rd, 378 KiB/s wr, 13 op/s 2024-09-21T09:43:10.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:43:10 smithi064 ceph-mon[86709]: osdmap e748: 8 total, 8 up, 8 in 2024-09-21T09:43:10.442 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:43:10 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:43:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:43:10.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:43:10 smithi022 ceph-mon[98615]: pgmap v733: 121 pgs: 121 active+clean; 164 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 66 KiB/s rd, 378 KiB/s wr, 13 op/s 2024-09-21T09:43:10.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:43:10 smithi022 ceph-mon[98615]: osdmap e748: 8 total, 8 up, 8 in 2024-09-21T09:43:10.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:43:10 smithi022 ceph-mon[96736]: pgmap v733: 121 pgs: 121 active+clean; 164 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 66 KiB/s rd, 378 KiB/s wr, 13 op/s 2024-09-21T09:43:10.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:43:10 smithi022 ceph-mon[96736]: osdmap e748: 8 total, 8 up, 8 in 2024-09-21T09:43:11.079 INFO:tasks.rados.rados.0.smithi022.stdout:3482: done (5 left) 2024-09-21T09:43:11.080 INFO:tasks.rados.rados.0.smithi022.stdout:3483: snap_remove snap 335 2024-09-21T09:43:12.081 INFO:tasks.rados.rados.0.smithi022.stdout:3483: done (5 left) 2024-09-21T09:43:12.082 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 2 current snap is 336 2024-09-21T09:43:12.082 INFO:tasks.rados.rados.0.smithi022.stdout:3484: seq_num 1099 ranges {3448832=409600,3858432=442368,4300800=720896,5021696=40960} 2024-09-21T09:43:12.102 INFO:tasks.rados.rados.0.smithi022.stdout:3484: writing smithi02284960-2 from 3448832 to 3858432 tid 1 2024-09-21T09:43:12.105 INFO:tasks.rados.rados.0.smithi022.stdout:3484: writing smithi02284960-2 from 3858432 to 4300800 tid 2 2024-09-21T09:43:12.109 INFO:tasks.rados.rados.0.smithi022.stdout:3484: writing smithi02284960-2 from 4300800 to 5021696 tid 3 2024-09-21T09:43:12.110 INFO:tasks.rados.rados.0.smithi022.stdout:3484: writing smithi02284960-2 from 5021696 to 5062656 tid 4 2024-09-21T09:43:12.110 INFO:tasks.rados.rados.0.smithi022.stdout:3485: copy_from oid 17 from oid 27 current snap is 336 2024-09-21T09:43:12.110 INFO:tasks.rados.rados.0.smithi022.stdout:3478: expect (ObjNum 1025 snap 315 seq_num 1025) 2024-09-21T09:43:12.134 INFO:tasks.rados.rados.0.smithi022.stdout:3476: finishing rollback tid 0 to smithi02284960-4 2024-09-21T09:43:12.135 INFO:tasks.rados.rados.0.smithi022.stdout:3476: finishing rollback tid 1 to smithi02284960-4 2024-09-21T09:43:12.135 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 4 v 1956 (ObjNum 129 snap 0 seq_num 3570745648) dirty dne 2024-09-21T09:43:12.135 INFO:tasks.rados.rados.0.smithi022.stdout:3479: expect (ObjNum 914 snap 284 seq_num 914) 2024-09-21T09:43:12.189 INFO:tasks.rados.rados.0.smithi022.stdout:3480: finishing copy_from racing read to smithi02284960-30 2024-09-21T09:43:12.189 INFO:tasks.rados.rados.0.smithi022.stdout:3480: finishing copy_from to smithi02284960-30 2024-09-21T09:43:12.189 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 30 v 1348 (ObjNum 1055 snap 317 seq_num 1055) dirty exists 2024-09-21T09:43:12.189 INFO:tasks.rados.rados.0.smithi022.stdout:3481: expect (ObjNum 985 snap 305 seq_num 985) 2024-09-21T09:43:12.432 INFO:tasks.rados.rados.0.smithi022.stdout:3484: finishing write tid 1 to smithi02284960-2 2024-09-21T09:43:12.432 INFO:tasks.rados.rados.0.smithi022.stdout:3484: finishing write tid 2 to smithi02284960-2 2024-09-21T09:43:12.432 INFO:tasks.rados.rados.0.smithi022.stdout:3484: finishing write tid 3 to smithi02284960-2 2024-09-21T09:43:12.433 INFO:tasks.rados.rados.0.smithi022.stdout:3484: finishing write tid 4 to smithi02284960-2 2024-09-21T09:43:12.433 INFO:tasks.rados.rados.0.smithi022.stdout:3484: finishing write tid 5 to smithi02284960-2 2024-09-21T09:43:12.433 INFO:tasks.rados.rados.0.smithi022.stdout:3485: finishing copy_from racing read to smithi02284960-17 2024-09-21T09:43:12.433 INFO:tasks.rados.rados.0.smithi022.stdout:3485: finishing copy_from to smithi02284960-17 2024-09-21T09:43:12.433 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 17 v 1795 (ObjNum 1008 snap 309 seq_num 1008) dirty exists 2024-09-21T09:43:12.433 INFO:tasks.rados.rados.0.smithi022.stdout:3484: finishing write tid 6 to smithi02284960-2 2024-09-21T09:43:12.433 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 2 v 1828 (ObjNum 1099 snap 336 seq_num 1099) dirty exists 2024-09-21T09:43:12.433 INFO:tasks.rados.rados.0.smithi022.stdout:3484: left oid 2 (ObjNum 1099 snap 336 seq_num 1099) 2024-09-21T09:43:12.433 INFO:tasks.rados.rados.0.smithi022.stdout:3476: done (6 left) 2024-09-21T09:43:12.433 INFO:tasks.rados.rados.0.smithi022.stdout:3478: done (5 left) 2024-09-21T09:43:12.433 INFO:tasks.rados.rados.0.smithi022.stdout:3479: done (4 left) 2024-09-21T09:43:12.433 INFO:tasks.rados.rados.0.smithi022.stdout:3480: done (3 left) 2024-09-21T09:43:12.433 INFO:tasks.rados.rados.0.smithi022.stdout:3481: done (2 left) 2024-09-21T09:43:12.434 INFO:tasks.rados.rados.0.smithi022.stdout:3484: done (1 left) 2024-09-21T09:43:12.434 INFO:tasks.rados.rados.0.smithi022.stdout:3485: done (0 left) 2024-09-21T09:43:12.434 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 1 current snap is 336 2024-09-21T09:43:12.434 INFO:tasks.rados.rados.0.smithi022.stdout:3486: seq_num 1100 ranges {0=425984,425984=729088,1155072=475136} 2024-09-21T09:43:12.434 INFO:tasks.rados.rados.0.smithi022.stdout:3486: writing smithi02284960-1 from 0 to 425984 tid 1 2024-09-21T09:43:12.439 INFO:tasks.rados.rados.0.smithi022.stdout:3486: writing smithi02284960-1 from 425984 to 1155072 tid 2 2024-09-21T09:43:12.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:43:12 smithi064 ceph-mon[86709]: pgmap v736: 121 pgs: 121 active+clean; 165 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 726 KiB/s rd, 390 KiB/s wr, 7 op/s 2024-09-21T09:43:12.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:43:12 smithi064 ceph-mon[86709]: osdmap e749: 8 total, 8 up, 8 in 2024-09-21T09:43:12.442 INFO:tasks.rados.rados.0.smithi022.stdout:3486: writing smithi02284960-1 from 1155072 to 1630208 tid 3 2024-09-21T09:43:12.442 INFO:tasks.rados.rados.0.smithi022.stdout:3487: rollback oid 40 current snap is 336 2024-09-21T09:43:12.442 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 40 to 317 2024-09-21T09:43:12.442 INFO:tasks.rados.rados.0.smithi022.stdout:3486: finishing write tid 1 to smithi02284960-1 2024-09-21T09:43:12.442 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 4 current snap is 336 2024-09-21T09:43:12.442 INFO:tasks.rados.rados.0.smithi022.stdout:3488: seq_num 1101 ranges {0=442368,442368=606208,1048576=598016} 2024-09-21T09:43:12.444 INFO:tasks.rados.rados.0.smithi022.stdout:3488: writing smithi02284960-4 from 0 to 442368 tid 1 2024-09-21T09:43:12.448 INFO:tasks.rados.rados.0.smithi022.stdout:3488: writing smithi02284960-4 from 442368 to 1048576 tid 2 2024-09-21T09:43:12.452 INFO:tasks.rados.rados.0.smithi022.stdout:3488: writing smithi02284960-4 from 1048576 to 1646592 tid 3 2024-09-21T09:43:12.453 INFO:tasks.rados.rados.0.smithi022.stdout:3489: snap_create 2024-09-21T09:43:12.453 INFO:tasks.rados.rados.0.smithi022.stdout:3486: finishing write tid 2 to smithi02284960-1 2024-09-21T09:43:12.453 INFO:tasks.rados.rados.0.smithi022.stdout:3486: finishing write tid 3 to smithi02284960-1 2024-09-21T09:43:12.453 INFO:tasks.rados.rados.0.smithi022.stdout:3486: finishing write tid 4 to smithi02284960-1 2024-09-21T09:43:12.453 INFO:tasks.rados.rados.0.smithi022.stdout:3486: finishing write tid 5 to smithi02284960-1 2024-09-21T09:43:12.453 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 1 v 1802 (ObjNum 1100 snap 336 seq_num 1100) dirty exists 2024-09-21T09:43:12.453 INFO:tasks.rados.rados.0.smithi022.stdout:3486: left oid 1 (ObjNum 1100 snap 336 seq_num 1100) 2024-09-21T09:43:12.453 INFO:tasks.rados.rados.0.smithi022.stdout:3488: finishing write tid 1 to smithi02284960-4 2024-09-21T09:43:12.453 INFO:tasks.rados.rados.0.smithi022.stdout:3487: finishing rollback tid 0 to smithi02284960-40 2024-09-21T09:43:12.453 INFO:tasks.rados.rados.0.smithi022.stdout:3487: finishing rollback tid 1 to smithi02284960-40 2024-09-21T09:43:12.453 INFO:tasks.rados.rados.0.smithi022.stdout:3487: finishing rollback tid 2 to smithi02284960-40 2024-09-21T09:43:12.453 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 40 v 1896 (ObjNum 1035 snap 315 seq_num 1035) dirty exists 2024-09-21T09:43:12.453 INFO:tasks.rados.rados.0.smithi022.stdout:3488: finishing write tid 2 to smithi02284960-4 2024-09-21T09:43:12.455 INFO:tasks.rados.rados.0.smithi022.stdout:3488: finishing write tid 3 to smithi02284960-4 2024-09-21T09:43:12.455 INFO:tasks.rados.rados.0.smithi022.stdout:3488: finishing write tid 4 to smithi02284960-4 2024-09-21T09:43:12.456 INFO:tasks.rados.rados.0.smithi022.stdout:3488: finishing write tid 5 to smithi02284960-4 2024-09-21T09:43:12.456 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 4 v 1960 (ObjNum 1101 snap 336 seq_num 1101) dirty exists 2024-09-21T09:43:12.456 INFO:tasks.rados.rados.0.smithi022.stdout:3488: left oid 4 (ObjNum 1101 snap 336 seq_num 1101) 2024-09-21T09:43:12.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:43:12 smithi022 ceph-mon[98615]: pgmap v736: 121 pgs: 121 active+clean; 165 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 726 KiB/s rd, 390 KiB/s wr, 7 op/s 2024-09-21T09:43:12.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:43:12 smithi022 ceph-mon[98615]: osdmap e749: 8 total, 8 up, 8 in 2024-09-21T09:43:12.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:43:12 smithi022 ceph-mon[96736]: pgmap v736: 121 pgs: 121 active+clean; 165 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 726 KiB/s rd, 390 KiB/s wr, 7 op/s 2024-09-21T09:43:12.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:43:12 smithi022 ceph-mon[96736]: osdmap e749: 8 total, 8 up, 8 in 2024-09-21T09:43:13.094 INFO:tasks.rados.rados.0.smithi022.stdout:3486: done (3 left) 2024-09-21T09:43:13.094 INFO:tasks.rados.rados.0.smithi022.stdout:3487: done (2 left) 2024-09-21T09:43:13.094 INFO:tasks.rados.rados.0.smithi022.stdout:3488: done (1 left) 2024-09-21T09:43:13.095 INFO:tasks.rados.rados.0.smithi022.stdout:3489: done (0 left) 2024-09-21T09:43:13.095 INFO:tasks.rados.rados.0.smithi022.stdout:3490: rmattr oid 26 current snap is 337 2024-09-21T09:43:13.095 INFO:tasks.rados.rados.0.smithi022.stdout:3490: done (0 left) 2024-09-21T09:43:13.095 INFO:tasks.rados.rados.0.smithi022.stdout:3491: rollback oid 26 current snap is 337 2024-09-21T09:43:13.095 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 26 to 326 2024-09-21T09:43:13.095 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 9 current snap is 337 2024-09-21T09:43:13.095 INFO:tasks.rados.rados.0.smithi022.stdout:3492: seq_num 1102 ranges {0=458752,458752=491520,950272=491520,1441792=221184} 2024-09-21T09:43:13.096 INFO:tasks.rados.rados.0.smithi022.stdout:3492: writing smithi02284960-9 from 0 to 458752 tid 1 2024-09-21T09:43:13.100 INFO:tasks.rados.rados.0.smithi022.stdout:3492: writing smithi02284960-9 from 458752 to 950272 tid 2 2024-09-21T09:43:13.103 INFO:tasks.rados.rados.0.smithi022.stdout:3492: writing smithi02284960-9 from 950272 to 1441792 tid 3 2024-09-21T09:43:13.105 INFO:tasks.rados.rados.0.smithi022.stdout:3492: writing smithi02284960-9 from 1441792 to 1662976 tid 4 2024-09-21T09:43:13.105 INFO:tasks.rados.rados.0.smithi022.stdout:3493: snap_remove snap 330 2024-09-21T09:43:13.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:43:13 smithi064 ceph-mon[86709]: osdmap e750: 8 total, 8 up, 8 in 2024-09-21T09:43:13.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:43:13 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:43:13.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:43:13 smithi022 ceph-mon[98615]: osdmap e750: 8 total, 8 up, 8 in 2024-09-21T09:43:13.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:43:13 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:43:13.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:43:13 smithi022 ceph-mon[96736]: osdmap e750: 8 total, 8 up, 8 in 2024-09-21T09:43:13.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:43:13 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:43:14.083 INFO:tasks.rados.rados.0.smithi022.stdout:3491: finishing rollback tid 0 to smithi02284960-26 2024-09-21T09:43:14.083 INFO:tasks.rados.rados.0.smithi022.stdout:3491: finishing rollback tid 1 to smithi02284960-26 2024-09-21T09:43:14.083 INFO:tasks.rados.rados.0.smithi022.stdout:3491: finishing rollback tid 2 to smithi02284960-26 2024-09-21T09:43:14.083 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 26 v 1962 (ObjNum 1036 snap 315 seq_num 1036) dirty exists 2024-09-21T09:43:14.083 INFO:tasks.rados.rados.0.smithi022.stdout:3492: finishing write tid 1 to smithi02284960-9 2024-09-21T09:43:14.083 INFO:tasks.rados.rados.0.smithi022.stdout:3492: finishing write tid 2 to smithi02284960-9 2024-09-21T09:43:14.083 INFO:tasks.rados.rados.0.smithi022.stdout:3492: finishing write tid 3 to smithi02284960-9 2024-09-21T09:43:14.083 INFO:tasks.rados.rados.0.smithi022.stdout:3492: finishing write tid 4 to smithi02284960-9 2024-09-21T09:43:14.083 INFO:tasks.rados.rados.0.smithi022.stdout:3492: finishing write tid 5 to smithi02284960-9 2024-09-21T09:43:14.083 INFO:tasks.rados.rados.0.smithi022.stdout:3492: finishing write tid 6 to smithi02284960-9 2024-09-21T09:43:14.083 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 9 v 1965 (ObjNum 1102 snap 337 seq_num 1102) dirty exists 2024-09-21T09:43:14.084 INFO:tasks.rados.rados.0.smithi022.stdout:3492: left oid 9 (ObjNum 1102 snap 337 seq_num 1102) 2024-09-21T09:43:14.084 INFO:tasks.rados.rados.0.smithi022.stdout:3491: done (2 left) 2024-09-21T09:43:14.084 INFO:tasks.rados.rados.0.smithi022.stdout:3492: done (1 left) 2024-09-21T09:43:14.084 INFO:tasks.rados.rados.0.smithi022.stdout:3493: done (0 left) 2024-09-21T09:43:14.084 INFO:tasks.rados.rados.0.smithi022.stdout:3494: setattr oid 27 current snap is 337 2024-09-21T09:43:14.085 INFO:tasks.rados.rados.0.smithi022.stdout:3495: read oid 44 snap -1 2024-09-21T09:43:14.085 INFO:tasks.rados.rados.0.smithi022.stdout:3495: expect (ObjNum 1002 snap 307 seq_num 1002) 2024-09-21T09:43:14.086 INFO:tasks.rados.rados.0.smithi022.stdout:3496: read oid 50 snap -1 2024-09-21T09:43:14.086 INFO:tasks.rados.rados.0.smithi022.stdout:3496: expect deleted 2024-09-21T09:43:14.086 INFO:tasks.rados.rados.0.smithi022.stdout:3497: copy_from oid 42 from oid 11 current snap is 337 2024-09-21T09:43:14.086 INFO:tasks.rados.rados.0.smithi022.stdout:3498: rmattr oid 19 current snap is 337 2024-09-21T09:43:14.086 INFO:tasks.rados.rados.0.smithi022.stdout:3498: done (4 left) 2024-09-21T09:43:14.086 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 12 current snap is 337 2024-09-21T09:43:14.086 INFO:tasks.rados.rados.0.smithi022.stdout:3499: seq_num 1103 ranges {2310144=475136,2785280=425984,3211264=696320,3907584=81920} 2024-09-21T09:43:14.102 INFO:tasks.rados.rados.0.smithi022.stdout:3499: writing smithi02284960-12 from 2310144 to 2785280 tid 1 2024-09-21T09:43:14.106 INFO:tasks.rados.rados.0.smithi022.stdout:3499: writing smithi02284960-12 from 2785280 to 3211264 tid 2 2024-09-21T09:43:14.111 INFO:tasks.rados.rados.0.smithi022.stdout:3499: writing smithi02284960-12 from 3211264 to 3907584 tid 3 2024-09-21T09:43:14.112 INFO:tasks.rados.rados.0.smithi022.stdout:3499: writing smithi02284960-12 from 3907584 to 3989504 tid 4 2024-09-21T09:43:14.112 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 28 current snap is 337 2024-09-21T09:43:14.112 INFO:tasks.rados.rados.0.smithi022.stdout:3500: seq_num 1104 ranges {1859584=499712,2359296=712704,3072000=491520} 2024-09-21T09:43:14.125 INFO:tasks.rados.rados.0.smithi022.stdout:3500: writing smithi02284960-28 from 1859584 to 2359296 tid 1 2024-09-21T09:43:14.130 INFO:tasks.rados.rados.0.smithi022.stdout:3500: writing smithi02284960-28 from 2359296 to 3072000 tid 2 2024-09-21T09:43:14.133 INFO:tasks.rados.rados.0.smithi022.stdout:3500: writing smithi02284960-28 from 3072000 to 3563520 tid 3 2024-09-21T09:43:14.133 INFO:tasks.rados.rados.0.smithi022.stdout:3501: snap_remove snap 323 2024-09-21T09:43:14.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:43:14 smithi064 ceph-mon[86709]: pgmap v739: 121 pgs: 121 active+clean; 168 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5.9 MiB/s rd, 1.3 MiB/s wr, 5 op/s 2024-09-21T09:43:14.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:43:14 smithi064 ceph-mon[86709]: osdmap e751: 8 total, 8 up, 8 in 2024-09-21T09:43:14.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:43:14 smithi064 ceph-mon[86709]: osdmap e752: 8 total, 8 up, 8 in 2024-09-21T09:43:14.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:43:14 smithi022 ceph-mon[98615]: pgmap v739: 121 pgs: 121 active+clean; 168 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5.9 MiB/s rd, 1.3 MiB/s wr, 5 op/s 2024-09-21T09:43:14.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:43:14 smithi022 ceph-mon[98615]: osdmap e751: 8 total, 8 up, 8 in 2024-09-21T09:43:14.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:43:14 smithi022 ceph-mon[98615]: osdmap e752: 8 total, 8 up, 8 in 2024-09-21T09:43:14.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:43:14 smithi022 ceph-mon[96736]: pgmap v739: 121 pgs: 121 active+clean; 168 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5.9 MiB/s rd, 1.3 MiB/s wr, 5 op/s 2024-09-21T09:43:14.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:43:14 smithi022 ceph-mon[96736]: osdmap e751: 8 total, 8 up, 8 in 2024-09-21T09:43:14.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:43:14 smithi022 ceph-mon[96736]: osdmap e752: 8 total, 8 up, 8 in 2024-09-21T09:43:15.107 INFO:tasks.rados.rados.0.smithi022.stdout:3501: done (6 left) 2024-09-21T09:43:15.108 INFO:tasks.rados.rados.0.smithi022.stdout:3502: snap_create 2024-09-21T09:43:15.108 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 27 v 1963 (ObjNum 1008 snap 309 seq_num 1008) dirty exists 2024-09-21T09:43:15.108 INFO:tasks.rados.rados.0.smithi022.stdout:3495: expect (ObjNum 1002 snap 307 seq_num 1002) 2024-09-21T09:43:15.200 INFO:tasks.rados.rados.0.smithi022.stdout:3499: finishing write tid 1 to smithi02284960-12 2024-09-21T09:43:15.200 INFO:tasks.rados.rados.0.smithi022.stdout:3499: finishing write tid 2 to smithi02284960-12 2024-09-21T09:43:15.200 INFO:tasks.rados.rados.0.smithi022.stdout:3497: finishing copy_from racing read to smithi02284960-42 2024-09-21T09:43:15.200 INFO:tasks.rados.rados.0.smithi022.stdout:3497: finishing copy_from to smithi02284960-42 2024-09-21T09:43:15.200 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 42 v 1752 (ObjNum 1062 snap 319 seq_num 1062) dirty exists 2024-09-21T09:43:15.200 INFO:tasks.rados.rados.0.smithi022.stdout:3499: finishing write tid 3 to smithi02284960-12 2024-09-21T09:43:15.200 INFO:tasks.rados.rados.0.smithi022.stdout:3499: finishing write tid 4 to smithi02284960-12 2024-09-21T09:43:15.200 INFO:tasks.rados.rados.0.smithi022.stdout:3499: finishing write tid 5 to smithi02284960-12 2024-09-21T09:43:15.200 INFO:tasks.rados.rados.0.smithi022.stdout:3499: finishing write tid 6 to smithi02284960-12 2024-09-21T09:43:15.200 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 12 v 1999 (ObjNum 1103 snap 337 seq_num 1103) dirty exists 2024-09-21T09:43:15.200 INFO:tasks.rados.rados.0.smithi022.stdout:3499: left oid 12 (ObjNum 1103 snap 337 seq_num 1103) 2024-09-21T09:43:15.200 INFO:tasks.rados.rados.0.smithi022.stdout:3500: finishing write tid 1 to smithi02284960-28 2024-09-21T09:43:15.200 INFO:tasks.rados.rados.0.smithi022.stdout:3500: finishing write tid 2 to smithi02284960-28 2024-09-21T09:43:15.201 INFO:tasks.rados.rados.0.smithi022.stdout:3500: finishing write tid 3 to smithi02284960-28 2024-09-21T09:43:15.201 INFO:tasks.rados.rados.0.smithi022.stdout:3500: finishing write tid 4 to smithi02284960-28 2024-09-21T09:43:15.201 INFO:tasks.rados.rados.0.smithi022.stdout:3500: finishing write tid 5 to smithi02284960-28 2024-09-21T09:43:15.201 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 28 v 1805 (ObjNum 1104 snap 337 seq_num 1104) dirty exists 2024-09-21T09:43:15.201 INFO:tasks.rados.rados.0.smithi022.stdout:3500: left oid 28 (ObjNum 1104 snap 337 seq_num 1104) 2024-09-21T09:43:16.058 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:43:15 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:43:15] "GET /metrics HTTP/1.1" 200 36359 "" "Prometheus/2.51.0" 2024-09-21T09:43:16.109 INFO:tasks.rados.rados.0.smithi022.stdout:3494: done (6 left) 2024-09-21T09:43:16.109 INFO:tasks.rados.rados.0.smithi022.stdout:3495: done (5 left) 2024-09-21T09:43:16.109 INFO:tasks.rados.rados.0.smithi022.stdout:3496: done (4 left) 2024-09-21T09:43:16.109 INFO:tasks.rados.rados.0.smithi022.stdout:3497: done (3 left) 2024-09-21T09:43:16.109 INFO:tasks.rados.rados.0.smithi022.stdout:3499: done (2 left) 2024-09-21T09:43:16.109 INFO:tasks.rados.rados.0.smithi022.stdout:3500: done (1 left) 2024-09-21T09:43:16.109 INFO:tasks.rados.rados.0.smithi022.stdout:3502: done (0 left) 2024-09-21T09:43:16.109 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 17 current snap is 338 2024-09-21T09:43:16.110 INFO:tasks.rados.rados.0.smithi022.stdout:3503: seq_num 1105 ranges {90112=516096,606208=598016,1204224=606208} 2024-09-21T09:43:16.112 INFO:tasks.rados.rados.0.smithi022.stdout:3503: writing smithi02284960-17 from 90112 to 606208 tid 1 2024-09-21T09:43:16.116 INFO:tasks.rados.rados.0.smithi022.stdout:3503: writing smithi02284960-17 from 606208 to 1204224 tid 2 2024-09-21T09:43:16.120 INFO:tasks.rados.rados.0.smithi022.stdout:3503: writing smithi02284960-17 from 1204224 to 1810432 tid 3 2024-09-21T09:43:16.120 INFO:tasks.rados.rados.0.smithi022.stdout:3504: read oid 7 snap -1 2024-09-21T09:43:16.120 INFO:tasks.rados.rados.0.smithi022.stdout:3504: expect deleted 2024-09-21T09:43:16.120 INFO:tasks.rados.rados.0.smithi022.stdout:3505: snap_remove snap 337 2024-09-21T09:43:16.120 INFO:tasks.rados.rados.0.smithi022.stdout:3503: finishing write tid 1 to smithi02284960-17 2024-09-21T09:43:16.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:43:16 smithi064 ceph-mon[86709]: pgmap v742: 121 pgs: 121 active+clean; 170 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5.6 MiB/s rd, 1.3 MiB/s wr, 4 op/s 2024-09-21T09:43:16.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:43:16 smithi064 ceph-mon[86709]: osdmap e753: 8 total, 8 up, 8 in 2024-09-21T09:43:16.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:43:16 smithi022 ceph-mon[98615]: pgmap v742: 121 pgs: 121 active+clean; 170 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5.6 MiB/s rd, 1.3 MiB/s wr, 4 op/s 2024-09-21T09:43:16.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:43:16 smithi022 ceph-mon[98615]: osdmap e753: 8 total, 8 up, 8 in 2024-09-21T09:43:16.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:43:16 smithi022 ceph-mon[96736]: pgmap v742: 121 pgs: 121 active+clean; 170 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5.6 MiB/s rd, 1.3 MiB/s wr, 4 op/s 2024-09-21T09:43:16.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:43:16 smithi022 ceph-mon[96736]: osdmap e753: 8 total, 8 up, 8 in 2024-09-21T09:43:17.120 INFO:tasks.rados.rados.0.smithi022.stdout:3505: done (2 left) 2024-09-21T09:43:17.120 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 16 current snap is 338 2024-09-21T09:43:17.120 INFO:tasks.rados.rados.0.smithi022.stdout:3503: finishing write tid 2 to smithi02284960-17 2024-09-21T09:43:17.121 INFO:tasks.rados.rados.0.smithi022.stdout:3506: seq_num 1106 ranges {7397376=532480,7929856=483328,8413184=475136,8888320=245760} 2024-09-21T09:43:17.163 INFO:tasks.rados.rados.0.smithi022.stdout:3506: writing smithi02284960-16 from 7397376 to 7929856 tid 1 2024-09-21T09:43:17.166 INFO:tasks.rados.rados.0.smithi022.stdout:3506: writing smithi02284960-16 from 7929856 to 8413184 tid 2 2024-09-21T09:43:17.170 INFO:tasks.rados.rados.0.smithi022.stdout:3506: writing smithi02284960-16 from 8413184 to 8888320 tid 3 2024-09-21T09:43:17.171 INFO:tasks.rados.rados.0.smithi022.stdout:3506: writing smithi02284960-16 from 8888320 to 9134080 tid 4 2024-09-21T09:43:17.171 INFO:tasks.rados.rados.0.smithi022.stdout:3507: rmattr oid 29 current snap is 338 2024-09-21T09:43:17.172 INFO:tasks.rados.rados.0.smithi022.stdout:3507: done (3 left) 2024-09-21T09:43:17.172 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 10 current snap is 338 2024-09-21T09:43:17.172 INFO:tasks.rados.rados.0.smithi022.stdout:3508: seq_num 1107 ranges {0=548864,548864=770048,1318912=434176} 2024-09-21T09:43:17.174 INFO:tasks.rados.rados.0.smithi022.stdout:3508: writing smithi02284960-10 from 0 to 548864 tid 1 2024-09-21T09:43:17.179 INFO:tasks.rados.rados.0.smithi022.stdout:3508: writing smithi02284960-10 from 548864 to 1318912 tid 2 2024-09-21T09:43:17.183 INFO:tasks.rados.rados.0.smithi022.stdout:3508: writing smithi02284960-10 from 1318912 to 1753088 tid 3 2024-09-21T09:43:17.183 INFO:tasks.rados.rados.0.smithi022.stdout:3509: delete oid 19 current snap is 338 2024-09-21T09:43:17.183 INFO:tasks.rados.rados.0.smithi022.stdout:3503: finishing write tid 3 to smithi02284960-17 2024-09-21T09:43:17.183 INFO:tasks.rados.rados.0.smithi022.stdout:3503: finishing write tid 4 to smithi02284960-17 2024-09-21T09:43:17.183 INFO:tasks.rados.rados.0.smithi022.stdout:3503: finishing write tid 5 to smithi02284960-17 2024-09-21T09:43:17.183 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 17 v 1811 (ObjNum 1105 snap 338 seq_num 1105) dirty exists 2024-09-21T09:43:17.183 INFO:tasks.rados.rados.0.smithi022.stdout:3503: left oid 17 (ObjNum 1105 snap 338 seq_num 1105) 2024-09-21T09:43:17.183 INFO:tasks.rados.rados.0.smithi022.stdout:3506: finishing write tid 1 to smithi02284960-16 2024-09-21T09:43:17.183 INFO:tasks.rados.rados.0.smithi022.stdout:3506: finishing write tid 2 to smithi02284960-16 2024-09-21T09:43:17.183 INFO:tasks.rados.rados.0.smithi022.stdout:3506: finishing write tid 3 to smithi02284960-16 2024-09-21T09:43:17.183 INFO:tasks.rados.rados.0.smithi022.stdout:3506: finishing write tid 4 to smithi02284960-16 2024-09-21T09:43:17.184 INFO:tasks.rados.rados.0.smithi022.stdout:3506: finishing write tid 5 to smithi02284960-16 2024-09-21T09:43:17.184 INFO:tasks.rados.rados.0.smithi022.stdout:3506: finishing write tid 6 to smithi02284960-16 2024-09-21T09:43:17.184 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 16 v 1543 (ObjNum 1106 snap 338 seq_num 1106) dirty exists 2024-09-21T09:43:17.184 INFO:tasks.rados.rados.0.smithi022.stdout:3506: left oid 16 (ObjNum 1106 snap 338 seq_num 1106) 2024-09-21T09:43:17.185 INFO:tasks.rados.rados.0.smithi022.stdout:3508: finishing write tid 1 to smithi02284960-10 2024-09-21T09:43:17.189 INFO:tasks.rados.rados.0.smithi022.stdout:3503: done (4 left) 2024-09-21T09:43:17.189 INFO:tasks.rados.rados.0.smithi022.stdout:3504: done (3 left) 2024-09-21T09:43:17.189 INFO:tasks.rados.rados.0.smithi022.stdout:3506: done (2 left) 2024-09-21T09:43:17.189 INFO:tasks.rados.rados.0.smithi022.stdout:3509: done (1 left) 2024-09-21T09:43:17.189 INFO:tasks.rados.rados.0.smithi022.stdout:3510: copy_from oid 35 from oid 7 current snap is 338 2024-09-21T09:43:17.189 INFO:tasks.rados.rados.0.smithi022.stdout:3511: rmattr oid 43 current snap is 338 2024-09-21T09:43:17.189 INFO:tasks.rados.rados.0.smithi022.stdout:3511: done (2 left) 2024-09-21T09:43:17.189 INFO:tasks.rados.rados.0.smithi022.stdout:3512: snap_remove snap 321 2024-09-21T09:43:17.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:43:17 smithi064 ceph-mon[86709]: osdmap e754: 8 total, 8 up, 8 in 2024-09-21T09:43:17.557 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:43:17 smithi022 ceph-mon[98615]: osdmap e754: 8 total, 8 up, 8 in 2024-09-21T09:43:17.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:43:17 smithi022 ceph-mon[96736]: osdmap e754: 8 total, 8 up, 8 in 2024-09-21T09:43:18.127 INFO:tasks.rados.rados.0.smithi022.stdout:3512: done (2 left) 2024-09-21T09:43:18.128 INFO:tasks.rados.rados.0.smithi022.stdout:3513: setattr oid 27 current snap is 338 2024-09-21T09:43:18.128 INFO:tasks.rados.rados.0.smithi022.stdout:3508: finishing write tid 2 to smithi02284960-10 2024-09-21T09:43:18.128 INFO:tasks.rados.rados.0.smithi022.stdout:3508: finishing write tid 3 to smithi02284960-10 2024-09-21T09:43:18.128 INFO:tasks.rados.rados.0.smithi022.stdout:3508: finishing write tid 4 to smithi02284960-10 2024-09-21T09:43:18.128 INFO:tasks.rados.rados.0.smithi022.stdout:3510: finishing copy_from to smithi02284960-35 2024-09-21T09:43:18.128 INFO:tasks.rados.rados.0.smithi022.stdout:3510: got expected ENOENT (src dne) 2024-09-21T09:43:18.128 INFO:tasks.rados.rados.0.smithi022.stdout:3510: finishing copy_from racing read to smithi02284960-35 2024-09-21T09:43:18.128 INFO:tasks.rados.rados.0.smithi022.stdout:3508: finishing write tid 5 to smithi02284960-10 2024-09-21T09:43:18.128 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 10 v 1900 (ObjNum 1107 snap 338 seq_num 1107) dirty exists 2024-09-21T09:43:18.128 INFO:tasks.rados.rados.0.smithi022.stdout:3508: left oid 10 (ObjNum 1107 snap 338 seq_num 1107) 2024-09-21T09:43:18.130 INFO:tasks.rados.rados.0.smithi022.stdout:3508: done (2 left) 2024-09-21T09:43:18.130 INFO:tasks.rados.rados.0.smithi022.stdout:3510: done (1 left) 2024-09-21T09:43:18.130 INFO:tasks.rados.rados.0.smithi022.stdout:3514: copy_from oid 30 from oid 9 current snap is 338 2024-09-21T09:43:18.130 INFO:tasks.rados.rados.0.smithi022.stdout:3515: read oid 44 snap -1 2024-09-21T09:43:18.130 INFO:tasks.rados.rados.0.smithi022.stdout:3515: expect (ObjNum 1002 snap 307 seq_num 1002) 2024-09-21T09:43:18.131 INFO:tasks.rados.rados.0.smithi022.stdout:3516: read oid 49 snap 326 2024-09-21T09:43:18.131 INFO:tasks.rados.rados.0.smithi022.stdout:3516: expect (ObjNum 1036 snap 315 seq_num 1036) 2024-09-21T09:43:18.131 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 3 current snap is 338 2024-09-21T09:43:18.131 INFO:tasks.rados.rados.0.smithi022.stdout:3517: seq_num 1109 ranges {2105344=581632,2686976=532480,3219456=647168,3866624=24576} 2024-09-21T09:43:18.148 INFO:tasks.rados.rados.0.smithi022.stdout:3517: writing smithi02284960-3 from 2105344 to 2686976 tid 1 2024-09-21T09:43:18.152 INFO:tasks.rados.rados.0.smithi022.stdout:3517: writing smithi02284960-3 from 2686976 to 3219456 tid 2 2024-09-21T09:43:18.157 INFO:tasks.rados.rados.0.smithi022.stdout:3517: writing smithi02284960-3 from 3219456 to 3866624 tid 3 2024-09-21T09:43:18.157 INFO:tasks.rados.rados.0.smithi022.stdout:3517: writing smithi02284960-3 from 3866624 to 3891200 tid 4 2024-09-21T09:43:18.157 INFO:tasks.rados.rados.0.smithi022.stdout:3518: snap_remove snap 328 2024-09-21T09:43:18.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:43:18 smithi064 ceph-mon[86709]: pgmap v745: 121 pgs: 121 active+clean; 171 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.2 MiB/s wr, 15 op/s 2024-09-21T09:43:18.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:43:18 smithi064 ceph-mon[86709]: osdmap e755: 8 total, 8 up, 8 in 2024-09-21T09:43:18.442 INFO:journalctl@ceph.osd.4.smithi064.stdout:Sep 21 09:43:18 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-4[92249]: 2024-09-21T09:43:18.156+0000 7ff3157ec640 -1 snap_mapper.get_next_objects_to_trim::New Clone-Objects were added to Snap 19a after trimming was started 2024-09-21T09:43:18.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:43:18 smithi022 ceph-mon[98615]: pgmap v745: 121 pgs: 121 active+clean; 171 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.2 MiB/s wr, 15 op/s 2024-09-21T09:43:18.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:43:18 smithi022 ceph-mon[98615]: osdmap e755: 8 total, 8 up, 8 in 2024-09-21T09:43:18.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:43:18 smithi022 ceph-mon[96736]: pgmap v745: 121 pgs: 121 active+clean; 171 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.2 MiB/s wr, 15 op/s 2024-09-21T09:43:18.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:43:18 smithi022 ceph-mon[96736]: osdmap e755: 8 total, 8 up, 8 in 2024-09-21T09:43:19.084 INFO:tasks.rados.rados.0.smithi022.stdout:3518: done (5 left) 2024-09-21T09:43:19.084 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 34 current snap is 338 2024-09-21T09:43:19.084 INFO:tasks.rados.rados.0.smithi022.stdout:3519: seq_num 1110 ranges {0=598016,598016=466944,1064960=450560,1515520=286720} 2024-09-21T09:43:19.088 INFO:tasks.rados.rados.0.smithi022.stdout:3519: writing smithi02284960-34 from 0 to 598016 tid 1 2024-09-21T09:43:19.091 INFO:tasks.rados.rados.0.smithi022.stdout:3519: writing smithi02284960-34 from 598016 to 1064960 tid 2 2024-09-21T09:43:19.094 INFO:tasks.rados.rados.0.smithi022.stdout:3519: writing smithi02284960-34 from 1064960 to 1515520 tid 3 2024-09-21T09:43:19.096 INFO:tasks.rados.rados.0.smithi022.stdout:3519: writing smithi02284960-34 from 1515520 to 1802240 tid 4 2024-09-21T09:43:19.096 INFO:tasks.rados.rados.0.smithi022.stdout:3520: snap_create 2024-09-21T09:43:19.096 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 27 v 1964 (ObjNum 1008 snap 309 seq_num 1008) dirty exists 2024-09-21T09:43:19.096 INFO:tasks.rados.rados.0.smithi022.stdout:3515: expect (ObjNum 1002 snap 307 seq_num 1002) 2024-09-21T09:43:19.179 INFO:tasks.rados.rados.0.smithi022.stdout:3517: finishing write tid 1 to smithi02284960-3 2024-09-21T09:43:19.179 INFO:tasks.rados.rados.0.smithi022.stdout:3517: finishing write tid 2 to smithi02284960-3 2024-09-21T09:43:19.179 INFO:tasks.rados.rados.0.smithi022.stdout:3514: finishing copy_from racing read to smithi02284960-30 2024-09-21T09:43:19.179 INFO:tasks.rados.rados.0.smithi022.stdout:3514: finishing copy_from to smithi02284960-30 2024-09-21T09:43:19.179 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 30 v 1965 (ObjNum 1102 snap 337 seq_num 1102) dirty exists 2024-09-21T09:43:19.179 INFO:tasks.rados.rados.0.smithi022.stdout:3516: expect (ObjNum 1036 snap 315 seq_num 1036) 2024-09-21T09:43:19.370 INFO:tasks.rados.rados.0.smithi022.stdout:3517: finishing write tid 3 to smithi02284960-3 2024-09-21T09:43:19.370 INFO:tasks.rados.rados.0.smithi022.stdout:3517: finishing write tid 4 to smithi02284960-3 2024-09-21T09:43:19.370 INFO:tasks.rados.rados.0.smithi022.stdout:3517: finishing write tid 5 to smithi02284960-3 2024-09-21T09:43:19.371 INFO:tasks.rados.rados.0.smithi022.stdout:3517: finishing write tid 6 to smithi02284960-3 2024-09-21T09:43:19.371 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 3 v 635 (ObjNum 1109 snap 338 seq_num 1109) dirty exists 2024-09-21T09:43:19.371 INFO:tasks.rados.rados.0.smithi022.stdout:3517: left oid 3 (ObjNum 1109 snap 338 seq_num 1109) 2024-09-21T09:43:19.371 INFO:tasks.rados.rados.0.smithi022.stdout:3519: finishing write tid 1 to smithi02284960-34 2024-09-21T09:43:19.371 INFO:tasks.rados.rados.0.smithi022.stdout:3519: finishing write tid 2 to smithi02284960-34 2024-09-21T09:43:19.371 INFO:tasks.rados.rados.0.smithi022.stdout:3519: finishing write tid 3 to smithi02284960-34 2024-09-21T09:43:19.371 INFO:tasks.rados.rados.0.smithi022.stdout:3519: finishing write tid 4 to smithi02284960-34 2024-09-21T09:43:19.371 INFO:tasks.rados.rados.0.smithi022.stdout:3519: finishing write tid 5 to smithi02284960-34 2024-09-21T09:43:19.371 INFO:tasks.rados.rados.0.smithi022.stdout:3519: finishing write tid 6 to smithi02284960-34 2024-09-21T09:43:19.371 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 34 v 1833 (ObjNum 1110 snap 338 seq_num 1110) dirty exists 2024-09-21T09:43:19.371 INFO:tasks.rados.rados.0.smithi022.stdout:3519: left oid 34 (ObjNum 1110 snap 338 seq_num 1110) 2024-09-21T09:43:19.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:43:19 smithi064 ceph-mon[86709]: osdmap e756: 8 total, 8 up, 8 in 2024-09-21T09:43:19.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:43:19 smithi064 ceph-mon[86709]: pgmap v748: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 171 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 2.0 MiB/s rd, 2.7 MiB/s wr, 16 op/s 2024-09-21T09:43:19.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:43:19 smithi064 ceph-mon[86709]: osdmap e757: 8 total, 8 up, 8 in 2024-09-21T09:43:19.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:43:19 smithi022 ceph-mon[98615]: osdmap e756: 8 total, 8 up, 8 in 2024-09-21T09:43:19.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:43:19 smithi022 ceph-mon[98615]: pgmap v748: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 171 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 2.0 MiB/s rd, 2.7 MiB/s wr, 16 op/s 2024-09-21T09:43:19.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:43:19 smithi022 ceph-mon[98615]: osdmap e757: 8 total, 8 up, 8 in 2024-09-21T09:43:19.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:43:19 smithi022 ceph-mon[96736]: osdmap e756: 8 total, 8 up, 8 in 2024-09-21T09:43:19.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:43:19 smithi022 ceph-mon[96736]: pgmap v748: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 171 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 2.0 MiB/s rd, 2.7 MiB/s wr, 16 op/s 2024-09-21T09:43:19.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:43:19 smithi022 ceph-mon[96736]: osdmap e757: 8 total, 8 up, 8 in 2024-09-21T09:43:20.085 INFO:tasks.rados.rados.0.smithi022.stdout:3513: done (6 left) 2024-09-21T09:43:20.085 INFO:tasks.rados.rados.0.smithi022.stdout:3514: done (5 left) 2024-09-21T09:43:20.085 INFO:tasks.rados.rados.0.smithi022.stdout:3515: done (4 left) 2024-09-21T09:43:20.085 INFO:tasks.rados.rados.0.smithi022.stdout:3516: done (3 left) 2024-09-21T09:43:20.085 INFO:tasks.rados.rados.0.smithi022.stdout:3517: done (2 left) 2024-09-21T09:43:20.086 INFO:tasks.rados.rados.0.smithi022.stdout:3519: done (1 left) 2024-09-21T09:43:20.086 INFO:tasks.rados.rados.0.smithi022.stdout:3520: done (0 left) 2024-09-21T09:43:20.086 INFO:tasks.rados.rados.0.smithi022.stdout:3521: read oid 3 snap -1 2024-09-21T09:43:20.086 INFO:tasks.rados.rados.0.smithi022.stdout:3521: expect (ObjNum 1109 snap 338 seq_num 1109) 2024-09-21T09:43:20.086 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 22 current snap is 339 2024-09-21T09:43:20.086 INFO:tasks.rados.rados.0.smithi022.stdout:3522: seq_num 1111 ranges {4833280=614400,5447680=753664,6201344=450560} 2024-09-21T09:43:20.115 INFO:tasks.rados.rados.0.smithi022.stdout:3522: writing smithi02284960-22 from 4833280 to 5447680 tid 1 2024-09-21T09:43:20.119 INFO:tasks.rados.rados.0.smithi022.stdout:3522: writing smithi02284960-22 from 5447680 to 6201344 tid 2 2024-09-21T09:43:20.122 INFO:tasks.rados.rados.0.smithi022.stdout:3522: writing smithi02284960-22 from 6201344 to 6651904 tid 3 2024-09-21T09:43:20.122 INFO:tasks.rados.rados.0.smithi022.stdout:3523: read oid 37 snap -1 2024-09-21T09:43:20.122 INFO:tasks.rados.rados.0.smithi022.stdout:3523: expect (ObjNum 1081 snap 326 seq_num 1081) 2024-09-21T09:43:20.123 INFO:tasks.rados.rados.0.smithi022.stdout:3524: read oid 40 snap -1 2024-09-21T09:43:20.123 INFO:tasks.rados.rados.0.smithi022.stdout:3524: expect (ObjNum 1035 snap 315 seq_num 1035) 2024-09-21T09:43:20.123 INFO:tasks.rados.rados.0.smithi022.stdout:3525: snap_remove snap 317 2024-09-21T09:43:20.691 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:43:20 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:43:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:43:21.094 INFO:tasks.rados.rados.0.smithi022.stdout:3525: done (4 left) 2024-09-21T09:43:21.094 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 2 current snap is 339 2024-09-21T09:43:21.094 INFO:tasks.rados.rados.0.smithi022.stdout:3526: seq_num 1112 ranges {5062656=630784,5693440=638976,6332416=466944,6799360=98304} 2024-09-21T09:43:21.125 INFO:tasks.rados.rados.0.smithi022.stdout:3526: writing smithi02284960-2 from 5062656 to 5693440 tid 1 2024-09-21T09:43:21.129 INFO:tasks.rados.rados.0.smithi022.stdout:3526: writing smithi02284960-2 from 5693440 to 6332416 tid 2 2024-09-21T09:43:21.132 INFO:tasks.rados.rados.0.smithi022.stdout:3526: writing smithi02284960-2 from 6332416 to 6799360 tid 3 2024-09-21T09:43:21.133 INFO:tasks.rados.rados.0.smithi022.stdout:3526: writing smithi02284960-2 from 6799360 to 6897664 tid 4 2024-09-21T09:43:21.133 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 10 current snap is 339 2024-09-21T09:43:21.133 INFO:tasks.rados.rados.0.smithi022.stdout:3527: seq_num 1113 ranges {1753088=647168,2400256=524288,2924544=622592,3547136=57344} 2024-09-21T09:43:21.146 INFO:tasks.rados.rados.0.smithi022.stdout:3527: writing smithi02284960-10 from 1753088 to 2400256 tid 1 2024-09-21T09:43:21.149 INFO:tasks.rados.rados.0.smithi022.stdout:3527: writing smithi02284960-10 from 2400256 to 2924544 tid 2 2024-09-21T09:43:21.152 INFO:tasks.rados.rados.0.smithi022.stdout:3527: writing smithi02284960-10 from 2924544 to 3547136 tid 3 2024-09-21T09:43:21.153 INFO:tasks.rados.rados.0.smithi022.stdout:3527: writing smithi02284960-10 from 3547136 to 3604480 tid 4 2024-09-21T09:43:21.153 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 18 current snap is 339 2024-09-21T09:43:21.153 INFO:tasks.rados.rados.0.smithi022.stdout:3521: expect (ObjNum 1109 snap 338 seq_num 1109) 2024-09-21T09:43:21.304 INFO:tasks.rados.rados.0.smithi022.stdout:3522: finishing write tid 1 to smithi02284960-22 2024-09-21T09:43:21.304 INFO:tasks.rados.rados.0.smithi022.stdout:3522: finishing write tid 2 to smithi02284960-22 2024-09-21T09:43:21.304 INFO:tasks.rados.rados.0.smithi022.stdout:3523: expect (ObjNum 1081 snap 326 seq_num 1081) 2024-09-21T09:43:21.349 INFO:tasks.rados.rados.0.smithi022.stdout:3522: finishing write tid 3 to smithi02284960-22 2024-09-21T09:43:21.349 INFO:tasks.rados.rados.0.smithi022.stdout:3522: finishing write tid 4 to smithi02284960-22 2024-09-21T09:43:21.349 INFO:tasks.rados.rados.0.smithi022.stdout:3522: finishing write tid 5 to smithi02284960-22 2024-09-21T09:43:21.349 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 22 v 1968 (ObjNum 1111 snap 339 seq_num 1111) dirty exists 2024-09-21T09:43:21.349 INFO:tasks.rados.rados.0.smithi022.stdout:3522: left oid 22 (ObjNum 1111 snap 339 seq_num 1111) 2024-09-21T09:43:21.349 INFO:tasks.rados.rados.0.smithi022.stdout:3524: expect (ObjNum 1035 snap 315 seq_num 1035) 2024-09-21T09:43:21.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:43:21 smithi064 ceph-mon[86709]: osdmap e758: 8 total, 8 up, 8 in 2024-09-21T09:43:21.557 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:43:21 smithi022 ceph-mon[98615]: osdmap e758: 8 total, 8 up, 8 in 2024-09-21T09:43:21.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:43:21 smithi022 ceph-mon[96736]: osdmap e758: 8 total, 8 up, 8 in 2024-09-21T09:43:21.574 INFO:tasks.rados.rados.0.smithi022.stdout:3526: finishing write tid 1 to smithi02284960-2 2024-09-21T09:43:21.574 INFO:tasks.rados.rados.0.smithi022.stdout:3526: finishing write tid 2 to smithi02284960-2 2024-09-21T09:43:21.574 INFO:tasks.rados.rados.0.smithi022.stdout:3526: finishing write tid 3 to smithi02284960-2 2024-09-21T09:43:21.574 INFO:tasks.rados.rados.0.smithi022.stdout:3526: finishing write tid 4 to smithi02284960-2 2024-09-21T09:43:21.574 INFO:tasks.rados.rados.0.smithi022.stdout:3526: finishing write tid 5 to smithi02284960-2 2024-09-21T09:43:21.574 INFO:tasks.rados.rados.0.smithi022.stdout:3526: finishing write tid 6 to smithi02284960-2 2024-09-21T09:43:21.575 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 2 v 1838 (ObjNum 1112 snap 339 seq_num 1112) dirty exists 2024-09-21T09:43:21.575 INFO:tasks.rados.rados.0.smithi022.stdout:3526: left oid 2 (ObjNum 1112 snap 339 seq_num 1112) 2024-09-21T09:43:21.575 INFO:tasks.rados.rados.0.smithi022.stdout:3527: finishing write tid 1 to smithi02284960-10 2024-09-21T09:43:21.575 INFO:tasks.rados.rados.0.smithi022.stdout:3527: finishing write tid 2 to smithi02284960-10 2024-09-21T09:43:21.575 INFO:tasks.rados.rados.0.smithi022.stdout:3527: finishing write tid 3 to smithi02284960-10 2024-09-21T09:43:21.575 INFO:tasks.rados.rados.0.smithi022.stdout:3528: seq_num 1114 ranges {1261568=663552,1925120=802816,2727936=401408} 2024-09-21T09:43:21.584 INFO:tasks.rados.rados.0.smithi022.stdout:3528: writing smithi02284960-18 from 1261568 to 1925120 tid 1 2024-09-21T09:43:21.589 INFO:tasks.rados.rados.0.smithi022.stdout:3528: writing smithi02284960-18 from 1925120 to 2727936 tid 2 2024-09-21T09:43:21.592 INFO:tasks.rados.rados.0.smithi022.stdout:3528: writing smithi02284960-18 from 2727936 to 3129344 tid 3 2024-09-21T09:43:21.592 INFO:tasks.rados.rados.0.smithi022.stdout:3521: done (6 left) 2024-09-21T09:43:21.592 INFO:tasks.rados.rados.0.smithi022.stdout:3522: done (5 left) 2024-09-21T09:43:21.592 INFO:tasks.rados.rados.0.smithi022.stdout:3523: done (4 left) 2024-09-21T09:43:21.592 INFO:tasks.rados.rados.0.smithi022.stdout:3524: done (3 left) 2024-09-21T09:43:21.593 INFO:tasks.rados.rados.0.smithi022.stdout:3526: done (2 left) 2024-09-21T09:43:21.593 INFO:tasks.rados.rados.0.smithi022.stdout:3529: snap_remove snap 338 2024-09-21T09:43:22.095 INFO:tasks.rados.rados.0.smithi022.stdout:3529: done (2 left) 2024-09-21T09:43:22.095 INFO:tasks.rados.rados.0.smithi022.stdout:3530: snap_remove snap 326 2024-09-21T09:43:22.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:43:22 smithi064 ceph-mon[86709]: pgmap v751: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 174 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 4.6 MiB/s rd, 1.7 MiB/s wr, 7 op/s 2024-09-21T09:43:22.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:43:22 smithi064 ceph-mon[86709]: osdmap e759: 8 total, 8 up, 8 in 2024-09-21T09:43:22.557 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:43:22 smithi022 ceph-mon[98615]: pgmap v751: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 174 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 4.6 MiB/s rd, 1.7 MiB/s wr, 7 op/s 2024-09-21T09:43:22.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:43:22 smithi022 ceph-mon[98615]: osdmap e759: 8 total, 8 up, 8 in 2024-09-21T09:43:22.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:43:22 smithi022 ceph-mon[96736]: pgmap v751: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 174 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 4.6 MiB/s rd, 1.7 MiB/s wr, 7 op/s 2024-09-21T09:43:22.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:43:22 smithi022 ceph-mon[96736]: osdmap e759: 8 total, 8 up, 8 in 2024-09-21T09:43:23.101 INFO:tasks.rados.rados.0.smithi022.stdout:3530: done (2 left) 2024-09-21T09:43:23.102 INFO:tasks.rados.rados.0.smithi022.stdout:3531: snap_remove snap 336 2024-09-21T09:43:23.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:43:23 smithi064 ceph-mon[86709]: osdmap e760: 8 total, 8 up, 8 in 2024-09-21T09:43:23.557 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:43:23 smithi022 ceph-mon[98615]: osdmap e760: 8 total, 8 up, 8 in 2024-09-21T09:43:23.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:43:23 smithi022 ceph-mon[96736]: osdmap e760: 8 total, 8 up, 8 in 2024-09-21T09:43:24.088 INFO:tasks.rados.rados.0.smithi022.stdout:3531: done (2 left) 2024-09-21T09:43:24.088 INFO:tasks.rados.rados.0.smithi022.stdout:3532: read oid 30 snap -1 2024-09-21T09:43:24.088 INFO:tasks.rados.rados.0.smithi022.stdout:3532: expect (ObjNum 1102 snap 337 seq_num 1102) 2024-09-21T09:43:24.088 INFO:tasks.rados.rados.0.smithi022.stdout:3527: finishing write tid 4 to smithi02284960-10 2024-09-21T09:43:24.088 INFO:tasks.rados.rados.0.smithi022.stdout:3527: finishing write tid 5 to smithi02284960-10 2024-09-21T09:43:24.088 INFO:tasks.rados.rados.0.smithi022.stdout:3527: finishing write tid 6 to smithi02284960-10 2024-09-21T09:43:24.088 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 10 v 1905 (ObjNum 1113 snap 339 seq_num 1113) dirty exists 2024-09-21T09:43:24.088 INFO:tasks.rados.rados.0.smithi022.stdout:3527: left oid 10 (ObjNum 1113 snap 339 seq_num 1113) 2024-09-21T09:43:24.089 INFO:tasks.rados.rados.0.smithi022.stdout:3528: finishing write tid 1 to smithi02284960-18 2024-09-21T09:43:24.089 INFO:tasks.rados.rados.0.smithi022.stdout:3528: finishing write tid 2 to smithi02284960-18 2024-09-21T09:43:24.089 INFO:tasks.rados.rados.0.smithi022.stdout:3528: finishing write tid 3 to smithi02284960-18 2024-09-21T09:43:24.089 INFO:tasks.rados.rados.0.smithi022.stdout:3528: finishing write tid 4 to smithi02284960-18 2024-09-21T09:43:24.089 INFO:tasks.rados.rados.0.smithi022.stdout:3528: finishing write tid 5 to smithi02284960-18 2024-09-21T09:43:24.089 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 18 v 1547 (ObjNum 1114 snap 339 seq_num 1114) dirty exists 2024-09-21T09:43:24.089 INFO:tasks.rados.rados.0.smithi022.stdout:3528: left oid 18 (ObjNum 1114 snap 339 seq_num 1114) 2024-09-21T09:43:24.089 INFO:tasks.rados.rados.0.smithi022.stdout:3533: read oid 34 snap -1 2024-09-21T09:43:24.089 INFO:tasks.rados.rados.0.smithi022.stdout:3533: expect (ObjNum 1110 snap 338 seq_num 1110) 2024-09-21T09:43:24.090 INFO:tasks.rados.rados.0.smithi022.stdout:3527: done (3 left) 2024-09-21T09:43:24.090 INFO:tasks.rados.rados.0.smithi022.stdout:3528: done (2 left) 2024-09-21T09:43:24.090 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 5 current snap is 339 2024-09-21T09:43:24.090 INFO:tasks.rados.rados.0.smithi022.stdout:3534: seq_num 1115 ranges {4833280=679936,5513216=688128,6201344=516096} 2024-09-21T09:43:24.120 INFO:tasks.rados.rados.0.smithi022.stdout:3534: writing smithi02284960-5 from 4833280 to 5513216 tid 1 2024-09-21T09:43:24.124 INFO:tasks.rados.rados.0.smithi022.stdout:3534: writing smithi02284960-5 from 5513216 to 6201344 tid 2 2024-09-21T09:43:24.127 INFO:tasks.rados.rados.0.smithi022.stdout:3534: writing smithi02284960-5 from 6201344 to 6717440 tid 3 2024-09-21T09:43:24.127 INFO:tasks.rados.rados.0.smithi022.stdout:3535: delete oid 50 current snap is 339 2024-09-21T09:43:24.127 INFO:tasks.rados.rados.0.smithi022.stdout:3532: expect (ObjNum 1102 snap 337 seq_num 1102) 2024-09-21T09:43:24.173 INFO:tasks.rados.rados.0.smithi022.stdout:3533: expect (ObjNum 1110 snap 338 seq_num 1110) 2024-09-21T09:43:24.221 INFO:tasks.rados.rados.0.smithi022.stdout:3534: finishing write tid 1 to smithi02284960-5 2024-09-21T09:43:24.221 INFO:tasks.rados.rados.0.smithi022.stdout:3534: finishing write tid 2 to smithi02284960-5 2024-09-21T09:43:24.221 INFO:tasks.rados.rados.0.smithi022.stdout:3534: finishing write tid 3 to smithi02284960-5 2024-09-21T09:43:24.221 INFO:tasks.rados.rados.0.smithi022.stdout:3534: finishing write tid 4 to smithi02284960-5 2024-09-21T09:43:24.221 INFO:tasks.rados.rados.0.smithi022.stdout:3534: finishing write tid 5 to smithi02284960-5 2024-09-21T09:43:24.221 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 5 v 1969 (ObjNum 1115 snap 339 seq_num 1115) dirty exists 2024-09-21T09:43:24.221 INFO:tasks.rados.rados.0.smithi022.stdout:3534: left oid 5 (ObjNum 1115 snap 339 seq_num 1115) 2024-09-21T09:43:24.221 INFO:tasks.rados.rados.0.smithi022.stdout:3532: done (3 left) 2024-09-21T09:43:24.221 INFO:tasks.rados.rados.0.smithi022.stdout:3533: done (2 left) 2024-09-21T09:43:24.221 INFO:tasks.rados.rados.0.smithi022.stdout:3534: done (1 left) 2024-09-21T09:43:24.221 INFO:tasks.rados.rados.0.smithi022.stdout:3535: done (0 left) 2024-09-21T09:43:24.221 INFO:tasks.rados.rados.0.smithi022.stdout:3536: read oid 10 snap -1 2024-09-21T09:43:24.222 INFO:tasks.rados.rados.0.smithi022.stdout:3536: expect (ObjNum 1113 snap 339 seq_num 1113) 2024-09-21T09:43:24.222 INFO:tasks.rados.rados.0.smithi022.stdout:3537: snap_create 2024-09-21T09:43:24.237 INFO:tasks.rados.rados.0.smithi022.stdout:3536: expect (ObjNum 1113 snap 339 seq_num 1113) 2024-09-21T09:43:24.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:43:24 smithi064 ceph-mon[86709]: pgmap v754: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 179 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 8.6 MiB/s rd, 2.1 MiB/s wr, 14 op/s 2024-09-21T09:43:24.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:43:24 smithi064 ceph-mon[86709]: osdmap e761: 8 total, 8 up, 8 in 2024-09-21T09:43:24.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:43:24 smithi064 ceph-mon[86709]: osdmap e762: 8 total, 8 up, 8 in 2024-09-21T09:43:24.459 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:43:24 smithi022 ceph-mon[96736]: pgmap v754: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 179 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 8.6 MiB/s rd, 2.1 MiB/s wr, 14 op/s 2024-09-21T09:43:24.459 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:43:24 smithi022 ceph-mon[96736]: osdmap e761: 8 total, 8 up, 8 in 2024-09-21T09:43:24.459 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:43:24 smithi022 ceph-mon[96736]: osdmap e762: 8 total, 8 up, 8 in 2024-09-21T09:43:24.459 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:43:24 smithi022 ceph-mon[98615]: pgmap v754: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 179 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 8.6 MiB/s rd, 2.1 MiB/s wr, 14 op/s 2024-09-21T09:43:24.460 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:43:24 smithi022 ceph-mon[98615]: osdmap e761: 8 total, 8 up, 8 in 2024-09-21T09:43:24.460 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:43:24 smithi022 ceph-mon[98615]: osdmap e762: 8 total, 8 up, 8 in 2024-09-21T09:43:25.107 INFO:tasks.rados.rados.0.smithi022.stdout:3536: done (1 left) 2024-09-21T09:43:25.107 INFO:tasks.rados.rados.0.smithi022.stdout:3537: done (0 left) 2024-09-21T09:43:25.107 INFO:tasks.rados.rados.0.smithi022.stdout:3538: snap_create 2024-09-21T09:43:26.058 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:43:25 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:43:25] "GET /metrics HTTP/1.1" 200 36359 "" "Prometheus/2.51.0" 2024-09-21T09:43:26.110 INFO:tasks.rados.rados.0.smithi022.stdout:3538: done (0 left) 2024-09-21T09:43:26.110 INFO:tasks.rados.rados.0.smithi022.stdout:3539: read oid 11 snap -1 2024-09-21T09:43:26.110 INFO:tasks.rados.rados.0.smithi022.stdout:3539: expect (ObjNum 1062 snap 319 seq_num 1062) 2024-09-21T09:43:26.110 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 28 current snap is 341 2024-09-21T09:43:26.110 INFO:tasks.rados.rados.0.smithi022.stdout:3540: seq_num 1117 ranges {3563520=712704,4276224=458752,4734976=655360,5390336=90112} 2024-09-21T09:43:26.133 INFO:tasks.rados.rados.0.smithi022.stdout:3540: writing smithi02284960-28 from 3563520 to 4276224 tid 1 2024-09-21T09:43:26.136 INFO:tasks.rados.rados.0.smithi022.stdout:3540: writing smithi02284960-28 from 4276224 to 4734976 tid 2 2024-09-21T09:43:26.140 INFO:tasks.rados.rados.0.smithi022.stdout:3540: writing smithi02284960-28 from 4734976 to 5390336 tid 3 2024-09-21T09:43:26.141 INFO:tasks.rados.rados.0.smithi022.stdout:3540: writing smithi02284960-28 from 5390336 to 5480448 tid 4 2024-09-21T09:43:26.141 INFO:tasks.rados.rados.0.smithi022.stdout:3541: delete oid 46 current snap is 341 2024-09-21T09:43:26.141 INFO:tasks.rados.rados.0.smithi022.stdout:3539: expect (ObjNum 1062 snap 319 seq_num 1062) 2024-09-21T09:43:26.170 INFO:tasks.rados.rados.0.smithi022.stdout:3540: finishing write tid 1 to smithi02284960-28 2024-09-21T09:43:26.171 INFO:tasks.rados.rados.0.smithi022.stdout:3540: finishing write tid 2 to smithi02284960-28 2024-09-21T09:43:26.171 INFO:tasks.rados.rados.0.smithi022.stdout:3540: finishing write tid 3 to smithi02284960-28 2024-09-21T09:43:26.171 INFO:tasks.rados.rados.0.smithi022.stdout:3540: finishing write tid 4 to smithi02284960-28 2024-09-21T09:43:26.171 INFO:tasks.rados.rados.0.smithi022.stdout:3540: finishing write tid 5 to smithi02284960-28 2024-09-21T09:43:26.171 INFO:tasks.rados.rados.0.smithi022.stdout:3540: finishing write tid 6 to smithi02284960-28 2024-09-21T09:43:26.171 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 28 v 1810 (ObjNum 1117 snap 341 seq_num 1117) dirty exists 2024-09-21T09:43:26.171 INFO:tasks.rados.rados.0.smithi022.stdout:3540: left oid 28 (ObjNum 1117 snap 341 seq_num 1117) 2024-09-21T09:43:26.172 INFO:tasks.rados.rados.0.smithi022.stdout:3539: done (2 left) 2024-09-21T09:43:26.172 INFO:tasks.rados.rados.0.smithi022.stdout:3540: done (1 left) 2024-09-21T09:43:26.172 INFO:tasks.rados.rados.0.smithi022.stdout:3541: done (0 left) 2024-09-21T09:43:26.172 INFO:tasks.rados.rados.0.smithi022.stdout:3542: read oid 9 snap -1 2024-09-21T09:43:26.172 INFO:tasks.rados.rados.0.smithi022.stdout:3542: expect (ObjNum 1102 snap 337 seq_num 1102) 2024-09-21T09:43:26.172 INFO:tasks.rados.rados.0.smithi022.stdout:3543: rollback oid 49 current snap is 341 2024-09-21T09:43:26.172 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 49 to 339 2024-09-21T09:43:26.173 INFO:tasks.rados.rados.0.smithi022.stdout:3544: rmattr oid 38 current snap is 341 2024-09-21T09:43:26.173 INFO:tasks.rados.rados.0.smithi022.stdout:3545: read oid 41 snap -1 2024-09-21T09:43:26.173 INFO:tasks.rados.rados.0.smithi022.stdout:3545: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-21T09:43:26.173 INFO:tasks.rados.rados.0.smithi022.stdout:3546: delete oid 47 current snap is 341 2024-09-21T09:43:26.173 INFO:tasks.rados.rados.0.smithi022.stdout:3545: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-21T09:43:26.174 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 38 v 1821 (ObjNum 289 snap 0 seq_num 1693273420) dirty exists 2024-09-21T09:43:26.175 INFO:tasks.rados.rados.0.smithi022.stdout:3543: finishing rollback tid 0 to smithi02284960-49 2024-09-21T09:43:26.175 INFO:tasks.rados.rados.0.smithi022.stdout:3543: finishing rollback tid 1 to smithi02284960-49 2024-09-21T09:43:26.175 INFO:tasks.rados.rados.0.smithi022.stdout:3544: done (4 left) 2024-09-21T09:43:26.175 INFO:tasks.rados.rados.0.smithi022.stdout:3545: done (3 left) 2024-09-21T09:43:26.175 INFO:tasks.rados.rados.0.smithi022.stdout:3546: done (2 left) 2024-09-21T09:43:26.175 INFO:tasks.rados.rados.0.smithi022.stdout:3547: read oid 25 snap -1 2024-09-21T09:43:26.175 INFO:tasks.rados.rados.0.smithi022.stdout:3547: expect deleted 2024-09-21T09:43:26.175 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 41 current snap is 341 2024-09-21T09:43:26.175 INFO:tasks.rados.rados.0.smithi022.stdout:3543: finishing rollback tid 2 to smithi02284960-49 2024-09-21T09:43:26.176 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 49 v 1807 (ObjNum 985 snap 305 seq_num 985) dirty exists 2024-09-21T09:43:26.176 INFO:tasks.rados.rados.0.smithi022.stdout:3548: seq_num 1120 ranges {0=761856,761856=557056,1318912=425984,1744896=221184} 2024-09-21T09:43:26.178 INFO:tasks.rados.rados.0.smithi022.stdout:3548: writing smithi02284960-41 from 0 to 761856 tid 1 2024-09-21T09:43:26.181 INFO:tasks.rados.rados.0.smithi022.stdout:3548: writing smithi02284960-41 from 761856 to 1318912 tid 2 2024-09-21T09:43:26.184 INFO:tasks.rados.rados.0.smithi022.stdout:3548: writing smithi02284960-41 from 1318912 to 1744896 tid 3 2024-09-21T09:43:26.186 INFO:tasks.rados.rados.0.smithi022.stdout:3548: writing smithi02284960-41 from 1744896 to 1966080 tid 4 2024-09-21T09:43:26.186 INFO:tasks.rados.rados.0.smithi022.stdout:3543: done (3 left) 2024-09-21T09:43:26.186 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 5 current snap is 341 2024-09-21T09:43:26.187 INFO:tasks.rados.rados.0.smithi022.stdout:3549: seq_num 1121 ranges {6717440=778240,7495680=442368,7938048=638976,8577024=122880} 2024-09-21T09:43:26.227 INFO:tasks.rados.rados.0.smithi022.stdout:3549: writing smithi02284960-5 from 6717440 to 7495680 tid 1 2024-09-21T09:43:26.229 INFO:tasks.rados.rados.0.smithi022.stdout:3549: writing smithi02284960-5 from 7495680 to 7938048 tid 2 2024-09-21T09:43:26.233 INFO:tasks.rados.rados.0.smithi022.stdout:3549: writing smithi02284960-5 from 7938048 to 8577024 tid 3 2024-09-21T09:43:26.235 INFO:tasks.rados.rados.0.smithi022.stdout:3549: writing smithi02284960-5 from 8577024 to 8699904 tid 4 2024-09-21T09:43:26.235 INFO:tasks.rados.rados.0.smithi022.stdout:3550: read oid 42 snap 339 2024-09-21T09:43:26.235 INFO:tasks.rados.rados.0.smithi022.stdout:3550: expect (ObjNum 1062 snap 319 seq_num 1062) 2024-09-21T09:43:26.235 INFO:tasks.rados.rados.0.smithi022.stdout:3551: snap_create 2024-09-21T09:43:26.235 INFO:tasks.rados.rados.0.smithi022.stdout:3542: expect (ObjNum 1102 snap 337 seq_num 1102) 2024-09-21T09:43:26.280 INFO:tasks.rados.rados.0.smithi022.stdout:3548: finishing write tid 1 to smithi02284960-41 2024-09-21T09:43:26.280 INFO:tasks.rados.rados.0.smithi022.stdout:3548: finishing write tid 2 to smithi02284960-41 2024-09-21T09:43:26.280 INFO:tasks.rados.rados.0.smithi022.stdout:3548: finishing write tid 3 to smithi02284960-41 2024-09-21T09:43:26.280 INFO:tasks.rados.rados.0.smithi022.stdout:3548: finishing write tid 4 to smithi02284960-41 2024-09-21T09:43:26.280 INFO:tasks.rados.rados.0.smithi022.stdout:3548: finishing write tid 5 to smithi02284960-41 2024-09-21T09:43:26.280 INFO:tasks.rados.rados.0.smithi022.stdout:3548: finishing write tid 6 to smithi02284960-41 2024-09-21T09:43:26.280 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 41 v 1910 (ObjNum 1120 snap 341 seq_num 1120) dirty exists 2024-09-21T09:43:26.280 INFO:tasks.rados.rados.0.smithi022.stdout:3548: left oid 41 (ObjNum 1120 snap 341 seq_num 1120) 2024-09-21T09:43:26.281 INFO:tasks.rados.rados.0.smithi022.stdout:3549: finishing write tid 1 to smithi02284960-5 2024-09-21T09:43:26.281 INFO:tasks.rados.rados.0.smithi022.stdout:3549: finishing write tid 2 to smithi02284960-5 2024-09-21T09:43:26.281 INFO:tasks.rados.rados.0.smithi022.stdout:3549: finishing write tid 3 to smithi02284960-5 2024-09-21T09:43:26.281 INFO:tasks.rados.rados.0.smithi022.stdout:3549: finishing write tid 4 to smithi02284960-5 2024-09-21T09:43:26.281 INFO:tasks.rados.rados.0.smithi022.stdout:3549: finishing write tid 5 to smithi02284960-5 2024-09-21T09:43:26.281 INFO:tasks.rados.rados.0.smithi022.stdout:3549: finishing write tid 6 to smithi02284960-5 2024-09-21T09:43:26.281 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 5 v 1974 (ObjNum 1121 snap 341 seq_num 1121) dirty exists 2024-09-21T09:43:26.281 INFO:tasks.rados.rados.0.smithi022.stdout:3549: left oid 5 (ObjNum 1121 snap 341 seq_num 1121) 2024-09-21T09:43:26.281 INFO:tasks.rados.rados.0.smithi022.stdout:3550: expect (ObjNum 1062 snap 319 seq_num 1062) 2024-09-21T09:43:26.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:43:26 smithi064 ceph-mon[86709]: pgmap v757: 121 pgs: 121 active+clean; 180 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 6.8 MiB/s rd, 2.1 MiB/s wr, 14 op/s 2024-09-21T09:43:26.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:43:26 smithi064 ceph-mon[86709]: osdmap e763: 8 total, 8 up, 8 in 2024-09-21T09:43:26.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:43:26 smithi022 ceph-mon[98615]: pgmap v757: 121 pgs: 121 active+clean; 180 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 6.8 MiB/s rd, 2.1 MiB/s wr, 14 op/s 2024-09-21T09:43:26.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:43:26 smithi022 ceph-mon[98615]: osdmap e763: 8 total, 8 up, 8 in 2024-09-21T09:43:26.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:43:26 smithi022 ceph-mon[96736]: pgmap v757: 121 pgs: 121 active+clean; 180 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 6.8 MiB/s rd, 2.1 MiB/s wr, 14 op/s 2024-09-21T09:43:26.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:43:26 smithi022 ceph-mon[96736]: osdmap e763: 8 total, 8 up, 8 in 2024-09-21T09:43:27.121 INFO:tasks.rados.rados.0.smithi022.stdout:3542: done (5 left) 2024-09-21T09:43:27.121 INFO:tasks.rados.rados.0.smithi022.stdout:3547: done (4 left) 2024-09-21T09:43:27.121 INFO:tasks.rados.rados.0.smithi022.stdout:3548: done (3 left) 2024-09-21T09:43:27.121 INFO:tasks.rados.rados.0.smithi022.stdout:3549: done (2 left) 2024-09-21T09:43:27.121 INFO:tasks.rados.rados.0.smithi022.stdout:3550: done (1 left) 2024-09-21T09:43:27.121 INFO:tasks.rados.rados.0.smithi022.stdout:3551: done (0 left) 2024-09-21T09:43:27.121 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 20 current snap is 342 2024-09-21T09:43:27.121 INFO:tasks.rados.rados.0.smithi022.stdout:3552: seq_num 1122 ranges {3948544=794624,4743168=729088,5472256=475136} 2024-09-21T09:43:27.146 INFO:tasks.rados.rados.0.smithi022.stdout:3552: writing smithi02284960-20 from 3948544 to 4743168 tid 1 2024-09-21T09:43:27.151 INFO:tasks.rados.rados.0.smithi022.stdout:3552: writing smithi02284960-20 from 4743168 to 5472256 tid 2 2024-09-21T09:43:27.154 INFO:tasks.rados.rados.0.smithi022.stdout:3552: writing smithi02284960-20 from 5472256 to 5947392 tid 3 2024-09-21T09:43:27.154 INFO:tasks.rados.rados.0.smithi022.stdout:3553: read oid 7 snap -1 2024-09-21T09:43:27.154 INFO:tasks.rados.rados.0.smithi022.stdout:3553: expect deleted 2024-09-21T09:43:27.154 INFO:tasks.rados.rados.0.smithi022.stdout:3554: rollback oid 37 current snap is 342 2024-09-21T09:43:27.155 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 37 to 339 2024-09-21T09:43:27.155 INFO:tasks.rados.rados.0.smithi022.stdout:3555: snap_create 2024-09-21T09:43:27.155 INFO:tasks.rados.rados.0.smithi022.stdout:3552: finishing write tid 1 to smithi02284960-20 2024-09-21T09:43:27.158 INFO:tasks.rados.rados.0.smithi022.stdout:3552: finishing write tid 2 to smithi02284960-20 2024-09-21T09:43:27.158 INFO:tasks.rados.rados.0.smithi022.stdout:3552: finishing write tid 3 to smithi02284960-20 2024-09-21T09:43:27.159 INFO:tasks.rados.rados.0.smithi022.stdout:3552: finishing write tid 4 to smithi02284960-20 2024-09-21T09:43:27.161 INFO:tasks.rados.rados.0.smithi022.stdout:3552: finishing write tid 5 to smithi02284960-20 2024-09-21T09:43:27.161 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 20 v 2074 (ObjNum 1122 snap 342 seq_num 1122) dirty exists 2024-09-21T09:43:27.161 INFO:tasks.rados.rados.0.smithi022.stdout:3552: left oid 20 (ObjNum 1122 snap 342 seq_num 1122) 2024-09-21T09:43:27.162 INFO:tasks.rados.rados.0.smithi022.stdout:3554: finishing rollback tid 0 to smithi02284960-37 2024-09-21T09:43:27.162 INFO:tasks.rados.rados.0.smithi022.stdout:3554: finishing rollback tid 1 to smithi02284960-37 2024-09-21T09:43:27.162 INFO:tasks.rados.rados.0.smithi022.stdout:3554: finishing rollback tid 2 to smithi02284960-37 2024-09-21T09:43:27.162 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 37 v 2049 (ObjNum 1081 snap 326 seq_num 1081) dirty exists 2024-09-21T09:43:27.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:43:27 smithi064 ceph-mon[86709]: osdmap e764: 8 total, 8 up, 8 in 2024-09-21T09:43:27.557 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:43:27 smithi022 ceph-mon[98615]: osdmap e764: 8 total, 8 up, 8 in 2024-09-21T09:43:27.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:43:27 smithi022 ceph-mon[96736]: osdmap e764: 8 total, 8 up, 8 in 2024-09-21T09:43:28.128 INFO:tasks.rados.rados.0.smithi022.stdout:3552: done (3 left) 2024-09-21T09:43:28.128 INFO:tasks.rados.rados.0.smithi022.stdout:3553: done (2 left) 2024-09-21T09:43:28.128 INFO:tasks.rados.rados.0.smithi022.stdout:3554: done (1 left) 2024-09-21T09:43:28.128 INFO:tasks.rados.rados.0.smithi022.stdout:3555: done (0 left) 2024-09-21T09:43:28.128 INFO:tasks.rados.rados.0.smithi022.stdout:3556: snap_create 2024-09-21T09:43:28.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:43:28 smithi064 ceph-mon[86709]: pgmap v760: 121 pgs: 121 active+clean; 122 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 6.6 MiB/s rd, 2.2 MiB/s wr, 9 op/s 2024-09-21T09:43:28.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:43:28 smithi064 ceph-mon[86709]: osdmap e765: 8 total, 8 up, 8 in 2024-09-21T09:43:28.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:43:28 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:43:28.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:43:28 smithi022 ceph-mon[98615]: pgmap v760: 121 pgs: 121 active+clean; 122 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 6.6 MiB/s rd, 2.2 MiB/s wr, 9 op/s 2024-09-21T09:43:28.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:43:28 smithi022 ceph-mon[98615]: osdmap e765: 8 total, 8 up, 8 in 2024-09-21T09:43:28.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:43:28 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:43:28.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:43:28 smithi022 ceph-mon[96736]: pgmap v760: 121 pgs: 121 active+clean; 122 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 6.6 MiB/s rd, 2.2 MiB/s wr, 9 op/s 2024-09-21T09:43:28.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:43:28 smithi022 ceph-mon[96736]: osdmap e765: 8 total, 8 up, 8 in 2024-09-21T09:43:28.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:43:28 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:43:29.087 INFO:tasks.rados.rados.0.smithi022.stdout:3556: done (0 left) 2024-09-21T09:43:29.087 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 8 current snap is 344 2024-09-21T09:43:29.087 INFO:tasks.rados.rados.0.smithi022.stdout:3557: seq_num 1123 ranges {3948544=409600,4358144=614400,4972544=598016,5570560=393216} 2024-09-21T09:43:29.111 INFO:tasks.rados.rados.0.smithi022.stdout:3557: writing smithi02284960-8 from 3948544 to 4358144 tid 1 2024-09-21T09:43:29.114 INFO:tasks.rados.rados.0.smithi022.stdout:3557: writing smithi02284960-8 from 4358144 to 4972544 tid 2 2024-09-21T09:43:29.118 INFO:tasks.rados.rados.0.smithi022.stdout:3557: writing smithi02284960-8 from 4972544 to 5570560 tid 3 2024-09-21T09:43:29.121 INFO:tasks.rados.rados.0.smithi022.stdout:3557: writing smithi02284960-8 from 5570560 to 5963776 tid 4 2024-09-21T09:43:29.121 INFO:tasks.rados.rados.0.smithi022.stdout:3558: rollback oid 37 current snap is 344 2024-09-21T09:43:29.121 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 37 to 339 2024-09-21T09:43:29.121 INFO:tasks.rados.rados.0.smithi022.stdout:3557: finishing write tid 1 to smithi02284960-8 2024-09-21T09:43:29.121 INFO:tasks.rados.rados.0.smithi022.stdout:3557: finishing write tid 2 to smithi02284960-8 2024-09-21T09:43:29.121 INFO:tasks.rados.rados.0.smithi022.stdout:3559: snap_create 2024-09-21T09:43:29.124 INFO:tasks.rados.rados.0.smithi022.stdout:3557: finishing write tid 3 to smithi02284960-8 2024-09-21T09:43:29.124 INFO:tasks.rados.rados.0.smithi022.stdout:3557: finishing write tid 4 to smithi02284960-8 2024-09-21T09:43:29.124 INFO:tasks.rados.rados.0.smithi022.stdout:3557: finishing write tid 5 to smithi02284960-8 2024-09-21T09:43:29.126 INFO:tasks.rados.rados.0.smithi022.stdout:3557: finishing write tid 6 to smithi02284960-8 2024-09-21T09:43:29.126 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 8 v 1979 (ObjNum 1123 snap 344 seq_num 1123) dirty exists 2024-09-21T09:43:29.126 INFO:tasks.rados.rados.0.smithi022.stdout:3557: left oid 8 (ObjNum 1123 snap 344 seq_num 1123) 2024-09-21T09:43:29.128 INFO:tasks.rados.rados.0.smithi022.stdout:3558: finishing rollback tid 0 to smithi02284960-37 2024-09-21T09:43:29.128 INFO:tasks.rados.rados.0.smithi022.stdout:3558: finishing rollback tid 1 to smithi02284960-37 2024-09-21T09:43:29.128 INFO:tasks.rados.rados.0.smithi022.stdout:3558: finishing rollback tid 2 to smithi02284960-37 2024-09-21T09:43:29.128 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 37 v 2053 (ObjNum 1081 snap 326 seq_num 1081) dirty exists 2024-09-21T09:43:29.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:43:29 smithi064 ceph-mon[86709]: osdmap e766: 8 total, 8 up, 8 in 2024-09-21T09:43:29.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:43:29 smithi064 ceph-mon[86709]: pgmap v763: 121 pgs: 121 active+clean; 118 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5.1 MiB/s rd, 1.8 MiB/s wr, 11 op/s 2024-09-21T09:43:29.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:43:29 smithi064 ceph-mon[86709]: osdmap e767: 8 total, 8 up, 8 in 2024-09-21T09:43:29.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:43:29 smithi022 ceph-mon[98615]: osdmap e766: 8 total, 8 up, 8 in 2024-09-21T09:43:29.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:43:29 smithi022 ceph-mon[98615]: pgmap v763: 121 pgs: 121 active+clean; 118 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5.1 MiB/s rd, 1.8 MiB/s wr, 11 op/s 2024-09-21T09:43:29.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:43:29 smithi022 ceph-mon[98615]: osdmap e767: 8 total, 8 up, 8 in 2024-09-21T09:43:29.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:43:29 smithi022 ceph-mon[96736]: osdmap e766: 8 total, 8 up, 8 in 2024-09-21T09:43:29.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:43:29 smithi022 ceph-mon[96736]: pgmap v763: 121 pgs: 121 active+clean; 118 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5.1 MiB/s rd, 1.8 MiB/s wr, 11 op/s 2024-09-21T09:43:29.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:43:29 smithi022 ceph-mon[96736]: osdmap e767: 8 total, 8 up, 8 in 2024-09-21T09:43:30.088 INFO:tasks.rados.rados.0.smithi022.stdout:3557: done (2 left) 2024-09-21T09:43:30.089 INFO:tasks.rados.rados.0.smithi022.stdout:3558: done (1 left) 2024-09-21T09:43:30.089 INFO:tasks.rados.rados.0.smithi022.stdout:3559: done (0 left) 2024-09-21T09:43:30.089 INFO:tasks.rados.rados.0.smithi022.stdout:3560: setattr oid 17 current snap is 345 2024-09-21T09:43:30.090 INFO:tasks.rados.rados.0.smithi022.stdout:3561: snap_create 2024-09-21T09:43:30.094 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 17 v 1966 (ObjNum 1105 snap 338 seq_num 1105) dirty exists 2024-09-21T09:43:30.691 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:43:30 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:43:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:43:31.094 INFO:tasks.rados.rados.0.smithi022.stdout:3560: done (1 left) 2024-09-21T09:43:31.094 INFO:tasks.rados.rados.0.smithi022.stdout:3561: done (0 left) 2024-09-21T09:43:31.094 INFO:tasks.rados.rados.0.smithi022.stdout:3562: snap_create 2024-09-21T09:43:31.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:43:31 smithi064 ceph-mon[86709]: osdmap e768: 8 total, 8 up, 8 in 2024-09-21T09:43:31.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:43:31 smithi022 ceph-mon[98615]: osdmap e768: 8 total, 8 up, 8 in 2024-09-21T09:43:31.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:43:31 smithi022 ceph-mon[96736]: osdmap e768: 8 total, 8 up, 8 in 2024-09-21T09:43:32.096 INFO:tasks.rados.rados.0.smithi022.stdout:3562: done (0 left) 2024-09-21T09:43:32.096 INFO:tasks.rados.rados.0.smithi022.stdout:3563: read oid 29 snap -1 2024-09-21T09:43:32.096 INFO:tasks.rados.rados.0.smithi022.stdout:3563: expect (ObjNum 1020 snap 315 seq_num 1020) 2024-09-21T09:43:32.096 INFO:tasks.rados.rados.0.smithi022.stdout:3564: read oid 23 snap 344 2024-09-21T09:43:32.096 INFO:tasks.rados.rados.0.smithi022.stdout:3564: expect (ObjNum 1083 snap 327 seq_num 1083) 2024-09-21T09:43:32.096 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 1 current snap is 347 2024-09-21T09:43:32.096 INFO:tasks.rados.rados.0.smithi022.stdout:3565: seq_num 1124 ranges {1630208=434176,2064384=499712,2564096=409600,2973696=663552,3637248=32768} 2024-09-21T09:43:32.107 INFO:tasks.rados.rados.0.smithi022.stdout:3565: writing smithi02284960-1 from 1630208 to 2064384 tid 1 2024-09-21T09:43:32.110 INFO:tasks.rados.rados.0.smithi022.stdout:3565: writing smithi02284960-1 from 2064384 to 2564096 tid 2 2024-09-21T09:43:32.113 INFO:tasks.rados.rados.0.smithi022.stdout:3565: writing smithi02284960-1 from 2564096 to 2973696 tid 3 2024-09-21T09:43:32.117 INFO:tasks.rados.rados.0.smithi022.stdout:3565: writing smithi02284960-1 from 2973696 to 3637248 tid 4 2024-09-21T09:43:32.117 INFO:tasks.rados.rados.0.smithi022.stdout:3565: writing smithi02284960-1 from 3637248 to 3670016 tid 5 2024-09-21T09:43:32.117 INFO:tasks.rados.rados.0.smithi022.stdout:3566: delete oid 37 current snap is 347 2024-09-21T09:43:32.117 INFO:tasks.rados.rados.0.smithi022.stdout:3563: expect (ObjNum 1020 snap 315 seq_num 1020) 2024-09-21T09:43:32.126 INFO:tasks.rados.rados.0.smithi022.stdout:3564: expect (ObjNum 1083 snap 327 seq_num 1083) 2024-09-21T09:43:32.195 INFO:tasks.rados.rados.0.smithi022.stdout:3565: finishing write tid 1 to smithi02284960-1 2024-09-21T09:43:32.195 INFO:tasks.rados.rados.0.smithi022.stdout:3565: finishing write tid 2 to smithi02284960-1 2024-09-21T09:43:32.195 INFO:tasks.rados.rados.0.smithi022.stdout:3565: finishing write tid 3 to smithi02284960-1 2024-09-21T09:43:32.196 INFO:tasks.rados.rados.0.smithi022.stdout:3565: finishing write tid 4 to smithi02284960-1 2024-09-21T09:43:32.196 INFO:tasks.rados.rados.0.smithi022.stdout:3565: finishing write tid 5 to smithi02284960-1 2024-09-21T09:43:32.196 INFO:tasks.rados.rados.0.smithi022.stdout:3565: finishing write tid 6 to smithi02284960-1 2024-09-21T09:43:32.196 INFO:tasks.rados.rados.0.smithi022.stdout:3565: finishing write tid 7 to smithi02284960-1 2024-09-21T09:43:32.196 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 1 v 1844 (ObjNum 1124 snap 347 seq_num 1124) dirty exists 2024-09-21T09:43:32.196 INFO:tasks.rados.rados.0.smithi022.stdout:3565: left oid 1 (ObjNum 1124 snap 347 seq_num 1124) 2024-09-21T09:43:32.196 INFO:tasks.rados.rados.0.smithi022.stdout:3563: done (3 left) 2024-09-21T09:43:32.196 INFO:tasks.rados.rados.0.smithi022.stdout:3564: done (2 left) 2024-09-21T09:43:32.196 INFO:tasks.rados.rados.0.smithi022.stdout:3565: done (1 left) 2024-09-21T09:43:32.196 INFO:tasks.rados.rados.0.smithi022.stdout:3566: done (0 left) 2024-09-21T09:43:32.196 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 44 current snap is 347 2024-09-21T09:43:32.196 INFO:tasks.rados.rados.0.smithi022.stdout:3567: seq_num 1126 ranges {2834432=466944,3301376=720896,4022272=778240,4800512=106496} 2024-09-21T09:43:32.213 INFO:tasks.rados.rados.0.smithi022.stdout:3567: writing smithi02284960-44 from 2834432 to 3301376 tid 1 2024-09-21T09:43:32.217 INFO:tasks.rados.rados.0.smithi022.stdout:3567: writing smithi02284960-44 from 3301376 to 4022272 tid 2 2024-09-21T09:43:32.222 INFO:tasks.rados.rados.0.smithi022.stdout:3567: writing smithi02284960-44 from 4022272 to 4800512 tid 3 2024-09-21T09:43:32.223 INFO:tasks.rados.rados.0.smithi022.stdout:3567: writing smithi02284960-44 from 4800512 to 4907008 tid 4 2024-09-21T09:43:32.223 INFO:tasks.rados.rados.0.smithi022.stdout:3568: snap_create 2024-09-21T09:43:32.223 INFO:tasks.rados.rados.0.smithi022.stdout:3567: finishing write tid 1 to smithi02284960-44 2024-09-21T09:43:32.223 INFO:tasks.rados.rados.0.smithi022.stdout:3567: finishing write tid 2 to smithi02284960-44 2024-09-21T09:43:32.226 INFO:tasks.rados.rados.0.smithi022.stdout:3567: finishing write tid 3 to smithi02284960-44 2024-09-21T09:43:32.226 INFO:tasks.rados.rados.0.smithi022.stdout:3567: finishing write tid 4 to smithi02284960-44 2024-09-21T09:43:32.226 INFO:tasks.rados.rados.0.smithi022.stdout:3567: finishing write tid 5 to smithi02284960-44 2024-09-21T09:43:32.227 INFO:tasks.rados.rados.0.smithi022.stdout:3567: finishing write tid 6 to smithi02284960-44 2024-09-21T09:43:32.227 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 44 v 1850 (ObjNum 1126 snap 347 seq_num 1126) dirty exists 2024-09-21T09:43:32.227 INFO:tasks.rados.rados.0.smithi022.stdout:3567: left oid 44 (ObjNum 1126 snap 347 seq_num 1126) 2024-09-21T09:43:32.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:43:32 smithi064 ceph-mon[86709]: pgmap v766: 121 pgs: 121 active+clean; 126 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 3.2 MiB/s rd, 2.4 MiB/s wr, 14 op/s 2024-09-21T09:43:32.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:43:32 smithi064 ceph-mon[86709]: osdmap e769: 8 total, 8 up, 8 in 2024-09-21T09:43:32.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:43:32 smithi022 ceph-mon[98615]: pgmap v766: 121 pgs: 121 active+clean; 126 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 3.2 MiB/s rd, 2.4 MiB/s wr, 14 op/s 2024-09-21T09:43:32.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:43:32 smithi022 ceph-mon[98615]: osdmap e769: 8 total, 8 up, 8 in 2024-09-21T09:43:32.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:43:32 smithi022 ceph-mon[96736]: pgmap v766: 121 pgs: 121 active+clean; 126 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 3.2 MiB/s rd, 2.4 MiB/s wr, 14 op/s 2024-09-21T09:43:32.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:43:32 smithi022 ceph-mon[96736]: osdmap e769: 8 total, 8 up, 8 in 2024-09-21T09:43:33.107 INFO:tasks.rados.rados.0.smithi022.stdout:3567: done (1 left) 2024-09-21T09:43:33.107 INFO:tasks.rados.rados.0.smithi022.stdout:3568: done (0 left) 2024-09-21T09:43:33.107 INFO:tasks.rados.rados.0.smithi022.stdout:3569: read oid 24 snap -1 2024-09-21T09:43:33.107 INFO:tasks.rados.rados.0.smithi022.stdout:3569: expect (ObjNum 1079 snap 326 seq_num 1079) 2024-09-21T09:43:33.107 INFO:tasks.rados.rados.0.smithi022.stdout:3570: copy_from oid 39 from oid 28 current snap is 348 2024-09-21T09:43:33.107 INFO:tasks.rados.rados.0.smithi022.stdout:3571: setattr oid 6 current snap is 348 2024-09-21T09:43:33.108 INFO:tasks.rados.rados.0.smithi022.stdout:3572: snap_remove snap 344 2024-09-21T09:43:33.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:43:33 smithi064 ceph-mon[86709]: osdmap e770: 8 total, 8 up, 8 in 2024-09-21T09:43:33.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:43:33 smithi022 ceph-mon[98615]: osdmap e770: 8 total, 8 up, 8 in 2024-09-21T09:43:33.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:43:33 smithi022 ceph-mon[96736]: osdmap e770: 8 total, 8 up, 8 in 2024-09-21T09:43:34.089 INFO:tasks.rados.rados.0.smithi022.stdout:3572: done (3 left) 2024-09-21T09:43:34.089 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 2 current snap is 348 2024-09-21T09:43:34.089 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 6 v 2075 (ObjNum 81 snap 0 seq_num 1634623575) dirty exists 2024-09-21T09:43:34.089 INFO:tasks.rados.rados.0.smithi022.stdout:3570: finishing copy_from racing read to smithi02284960-39 2024-09-21T09:43:34.089 INFO:tasks.rados.rados.0.smithi022.stdout:3570: finishing copy_from to smithi02284960-39 2024-09-21T09:43:34.089 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 39 v 1810 (ObjNum 1117 snap 341 seq_num 1117) dirty exists 2024-09-21T09:43:34.089 INFO:tasks.rados.rados.0.smithi022.stdout:3569: expect (ObjNum 1079 snap 326 seq_num 1079) 2024-09-21T09:43:34.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:43:34 smithi064 ceph-mon[86709]: pgmap v769: 121 pgs: 121 active+clean; 126 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 2.6 MiB/s rd, 1.9 MiB/s wr, 16 op/s 2024-09-21T09:43:34.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:43:34 smithi064 ceph-mon[86709]: osdmap e771: 8 total, 8 up, 8 in 2024-09-21T09:43:34.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:43:34 smithi064 ceph-mon[86709]: osdmap e772: 8 total, 8 up, 8 in 2024-09-21T09:43:34.557 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:43:34 smithi022 ceph-mon[98615]: pgmap v769: 121 pgs: 121 active+clean; 126 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 2.6 MiB/s rd, 1.9 MiB/s wr, 16 op/s 2024-09-21T09:43:34.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:43:34 smithi022 ceph-mon[98615]: osdmap e771: 8 total, 8 up, 8 in 2024-09-21T09:43:34.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:43:34 smithi022 ceph-mon[98615]: osdmap e772: 8 total, 8 up, 8 in 2024-09-21T09:43:34.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:43:34 smithi022 ceph-mon[96736]: pgmap v769: 121 pgs: 121 active+clean; 126 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 2.6 MiB/s rd, 1.9 MiB/s wr, 16 op/s 2024-09-21T09:43:34.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:43:34 smithi022 ceph-mon[96736]: osdmap e771: 8 total, 8 up, 8 in 2024-09-21T09:43:34.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:43:34 smithi022 ceph-mon[96736]: osdmap e772: 8 total, 8 up, 8 in 2024-09-21T09:43:34.613 INFO:tasks.rados.rados.0.smithi022.stdout:3573: seq_num 1127 ranges {6897664=483328,7380992=598016,7979008=581632,8560640=425984} 2024-09-21T09:43:34.653 INFO:tasks.rados.rados.0.smithi022.stdout:3573: writing smithi02284960-2 from 6897664 to 7380992 tid 1 2024-09-21T09:43:34.657 INFO:tasks.rados.rados.0.smithi022.stdout:3573: writing smithi02284960-2 from 7380992 to 7979008 tid 2 2024-09-21T09:43:34.660 INFO:tasks.rados.rados.0.smithi022.stdout:3573: writing smithi02284960-2 from 7979008 to 8560640 tid 3 2024-09-21T09:43:34.663 INFO:tasks.rados.rados.0.smithi022.stdout:3573: writing smithi02284960-2 from 8560640 to 8986624 tid 4 2024-09-21T09:43:34.663 INFO:tasks.rados.rados.0.smithi022.stdout:3569: done (3 left) 2024-09-21T09:43:34.663 INFO:tasks.rados.rados.0.smithi022.stdout:3570: done (2 left) 2024-09-21T09:43:34.663 INFO:tasks.rados.rados.0.smithi022.stdout:3571: done (1 left) 2024-09-21T09:43:34.663 INFO:tasks.rados.rados.0.smithi022.stdout:3574: snap_remove snap 343 2024-09-21T09:43:35.112 INFO:tasks.rados.rados.0.smithi022.stdout:3574: done (1 left) 2024-09-21T09:43:35.112 INFO:tasks.rados.rados.0.smithi022.stdout:3575: read oid 14 snap 340 2024-09-21T09:43:35.112 INFO:tasks.rados.rados.0.smithi022.stdout:3575: expect (ObjNum 985 snap 305 seq_num 985) 2024-09-21T09:43:35.112 INFO:tasks.rados.rados.0.smithi022.stdout:3576: copy_from oid 15 from oid 1 current snap is 348 2024-09-21T09:43:35.112 INFO:tasks.rados.rados.0.smithi022.stdout:3573: finishing write tid 1 to smithi02284960-2 2024-09-21T09:43:35.112 INFO:tasks.rados.rados.0.smithi022.stdout:3573: finishing write tid 2 to smithi02284960-2 2024-09-21T09:43:35.112 INFO:tasks.rados.rados.0.smithi022.stdout:3573: finishing write tid 3 to smithi02284960-2 2024-09-21T09:43:35.112 INFO:tasks.rados.rados.0.smithi022.stdout:3573: finishing write tid 4 to smithi02284960-2 2024-09-21T09:43:35.112 INFO:tasks.rados.rados.0.smithi022.stdout:3573: finishing write tid 5 to smithi02284960-2 2024-09-21T09:43:35.112 INFO:tasks.rados.rados.0.smithi022.stdout:3573: finishing write tid 6 to smithi02284960-2 2024-09-21T09:43:35.112 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 2 v 1843 (ObjNum 1127 snap 348 seq_num 1127) dirty exists 2024-09-21T09:43:35.113 INFO:tasks.rados.rados.0.smithi022.stdout:3573: left oid 2 (ObjNum 1127 snap 348 seq_num 1127) 2024-09-21T09:43:35.113 INFO:tasks.rados.rados.0.smithi022.stdout:3573: done (2 left) 2024-09-21T09:43:35.113 INFO:tasks.rados.rados.0.smithi022.stdout:3577: setattr oid 48 current snap is 348 2024-09-21T09:43:35.113 INFO:tasks.rados.rados.0.smithi022.stdout:3578: snap_create 2024-09-21T09:43:35.119 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 48 v 2076 (ObjNum 81 snap 0 seq_num 1901946195) dirty exists 2024-09-21T09:43:35.138 INFO:tasks.rados.rados.0.smithi022.stdout:3576: finishing copy_from racing read to smithi02284960-15 2024-09-21T09:43:35.140 INFO:tasks.rados.rados.0.smithi022.stdout:3576: finishing copy_from to smithi02284960-15 2024-09-21T09:43:35.141 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 15 v 1844 (ObjNum 1124 snap 347 seq_num 1124) dirty exists 2024-09-21T09:43:35.142 INFO:tasks.rados.rados.0.smithi022.stdout:3575: expect (ObjNum 985 snap 305 seq_num 985) 2024-09-21T09:43:36.058 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:43:35 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:43:35] "GET /metrics HTTP/1.1" 200 36354 "" "Prometheus/2.51.0" 2024-09-21T09:43:36.115 INFO:tasks.rados.rados.0.smithi022.stdout:3575: done (3 left) 2024-09-21T09:43:36.115 INFO:tasks.rados.rados.0.smithi022.stdout:3576: done (2 left) 2024-09-21T09:43:36.115 INFO:tasks.rados.rados.0.smithi022.stdout:3577: done (1 left) 2024-09-21T09:43:36.115 INFO:tasks.rados.rados.0.smithi022.stdout:3578: done (0 left) 2024-09-21T09:43:36.115 INFO:tasks.rados.rados.0.smithi022.stdout:3579: delete oid 34 current snap is 349 2024-09-21T09:43:36.122 INFO:tasks.rados.rados.0.smithi022.stdout:3579: done (0 left) 2024-09-21T09:43:36.122 INFO:tasks.rados.rados.0.smithi022.stdout:3580: rmattr oid 6 current snap is 349 2024-09-21T09:43:36.122 INFO:tasks.rados.rados.0.smithi022.stdout:3581: read oid 37 snap -1 2024-09-21T09:43:36.122 INFO:tasks.rados.rados.0.smithi022.stdout:3581: expect deleted 2024-09-21T09:43:36.122 INFO:tasks.rados.rados.0.smithi022.stdout:3582: delete oid 32 current snap is 349 2024-09-21T09:43:36.128 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 6 v 2078 (ObjNum 33 snap 0 seq_num 3569538688) dirty exists 2024-09-21T09:43:36.130 INFO:tasks.rados.rados.0.smithi022.stdout:3580: done (2 left) 2024-09-21T09:43:36.130 INFO:tasks.rados.rados.0.smithi022.stdout:3581: done (1 left) 2024-09-21T09:43:36.130 INFO:tasks.rados.rados.0.smithi022.stdout:3582: done (0 left) 2024-09-21T09:43:36.130 INFO:tasks.rados.rados.0.smithi022.stdout:3583: rollback oid 23 current snap is 349 2024-09-21T09:43:36.130 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 23 to 346 2024-09-21T09:43:36.130 INFO:tasks.rados.rados.0.smithi022.stdout:3584: read oid 4 snap 340 2024-09-21T09:43:36.130 INFO:tasks.rados.rados.0.smithi022.stdout:3584: expect (ObjNum 1101 snap 336 seq_num 1101) 2024-09-21T09:43:36.130 INFO:tasks.rados.rados.0.smithi022.stdout:3585: snap_remove snap 348 2024-09-21T09:43:36.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:43:36 smithi064 ceph-mon[86709]: pgmap v772: 121 pgs: 121 active+clean; 126 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 2.4 MiB/s rd, 0 B/s wr, 8 op/s 2024-09-21T09:43:36.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:43:36 smithi064 ceph-mon[86709]: osdmap e773: 8 total, 8 up, 8 in 2024-09-21T09:43:36.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:43:36 smithi022 ceph-mon[98615]: pgmap v772: 121 pgs: 121 active+clean; 126 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 2.4 MiB/s rd, 0 B/s wr, 8 op/s 2024-09-21T09:43:36.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:43:36 smithi022 ceph-mon[98615]: osdmap e773: 8 total, 8 up, 8 in 2024-09-21T09:43:36.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:43:36 smithi022 ceph-mon[96736]: pgmap v772: 121 pgs: 121 active+clean; 126 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 2.4 MiB/s rd, 0 B/s wr, 8 op/s 2024-09-21T09:43:36.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:43:36 smithi022 ceph-mon[96736]: osdmap e773: 8 total, 8 up, 8 in 2024-09-21T09:43:37.125 INFO:tasks.rados.rados.0.smithi022.stdout:3585: done (2 left) 2024-09-21T09:43:37.125 INFO:tasks.rados.rados.0.smithi022.stdout:3586: snap_remove snap 341 2024-09-21T09:43:37.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:43:37 smithi064 ceph-mon[86709]: osdmap e774: 8 total, 8 up, 8 in 2024-09-21T09:43:37.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:43:37 smithi022 ceph-mon[98615]: osdmap e774: 8 total, 8 up, 8 in 2024-09-21T09:43:37.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:43:37 smithi022 ceph-mon[96736]: osdmap e774: 8 total, 8 up, 8 in 2024-09-21T09:43:38.127 INFO:tasks.rados.rados.0.smithi022.stdout:3586: done (2 left) 2024-09-21T09:43:38.127 INFO:tasks.rados.rados.0.smithi022.stdout:3587: rollback oid 14 current snap is 349 2024-09-21T09:43:38.127 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 14 to 342 2024-09-21T09:43:38.127 INFO:tasks.rados.rados.0.smithi022.stdout:3583: finishing rollback tid 0 to smithi02284960-23 2024-09-21T09:43:38.127 INFO:tasks.rados.rados.0.smithi022.stdout:3583: finishing rollback tid 1 to smithi02284960-23 2024-09-21T09:43:38.127 INFO:tasks.rados.rados.0.smithi022.stdout:3583: finishing rollback tid 2 to smithi02284960-23 2024-09-21T09:43:38.127 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 23 v 1824 (ObjNum 1083 snap 327 seq_num 1083) dirty exists 2024-09-21T09:43:38.127 INFO:tasks.rados.rados.0.smithi022.stdout:3584: expect (ObjNum 1101 snap 336 seq_num 1101) 2024-09-21T09:43:38.174 INFO:tasks.rados.rados.0.smithi022.stdout:3587: finishing rollback tid 0 to smithi02284960-14 2024-09-21T09:43:38.175 INFO:tasks.rados.rados.0.smithi022.stdout:3587: finishing rollback tid 1 to smithi02284960-14 2024-09-21T09:43:38.175 INFO:tasks.rados.rados.0.smithi022.stdout:3587: finishing rollback tid 2 to smithi02284960-14 2024-09-21T09:43:38.175 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 14 v 1810 (ObjNum 985 snap 305 seq_num 985) dirty exists 2024-09-21T09:43:38.175 INFO:tasks.rados.rados.0.smithi022.stdout:3583: done (2 left) 2024-09-21T09:43:38.175 INFO:tasks.rados.rados.0.smithi022.stdout:3584: done (1 left) 2024-09-21T09:43:38.175 INFO:tasks.rados.rados.0.smithi022.stdout:3587: done (0 left) 2024-09-21T09:43:38.175 INFO:tasks.rados.rados.0.smithi022.stdout:3588: snap_remove snap 339 2024-09-21T09:43:38.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:43:38 smithi064 ceph-mon[86709]: pgmap v775: 121 pgs: 121 active+clean; 138 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 7.0 MiB/s rd, 3.2 MiB/s wr, 17 op/s 2024-09-21T09:43:38.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:43:38 smithi064 ceph-mon[86709]: osdmap e775: 8 total, 8 up, 8 in 2024-09-21T09:43:38.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:43:38 smithi022 ceph-mon[98615]: pgmap v775: 121 pgs: 121 active+clean; 138 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 7.0 MiB/s rd, 3.2 MiB/s wr, 17 op/s 2024-09-21T09:43:38.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:43:38 smithi022 ceph-mon[98615]: osdmap e775: 8 total, 8 up, 8 in 2024-09-21T09:43:38.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:43:38 smithi022 ceph-mon[96736]: pgmap v775: 121 pgs: 121 active+clean; 138 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 7.0 MiB/s rd, 3.2 MiB/s wr, 17 op/s 2024-09-21T09:43:38.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:43:38 smithi022 ceph-mon[96736]: osdmap e775: 8 total, 8 up, 8 in 2024-09-21T09:43:39.144 INFO:tasks.rados.rados.0.smithi022.stdout:3588: done (0 left) 2024-09-21T09:43:39.144 INFO:tasks.rados.rados.0.smithi022.stdout:3589: copy_from oid 22 from oid 26 current snap is 349 2024-09-21T09:43:39.145 INFO:tasks.rados.rados.0.smithi022.stdout:3590: read oid 48 snap -1 2024-09-21T09:43:39.145 INFO:tasks.rados.rados.0.smithi022.stdout:3590: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-21T09:43:39.145 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 5 current snap is 349 2024-09-21T09:43:39.145 INFO:tasks.rados.rados.0.smithi022.stdout:3591: seq_num 1130 ranges {8699904=532480,9232384=655360,9887744=753664,10641408=196608} 2024-09-21T09:43:39.196 INFO:tasks.rados.rados.0.smithi022.stdout:3591: writing smithi02284960-5 from 8699904 to 9232384 tid 1 2024-09-21T09:43:39.200 INFO:tasks.rados.rados.0.smithi022.stdout:3591: writing smithi02284960-5 from 9232384 to 9887744 tid 2 2024-09-21T09:43:39.205 INFO:tasks.rados.rados.0.smithi022.stdout:3591: writing smithi02284960-5 from 9887744 to 10641408 tid 3 2024-09-21T09:43:39.207 INFO:tasks.rados.rados.0.smithi022.stdout:3591: writing smithi02284960-5 from 10641408 to 10838016 tid 4 2024-09-21T09:43:39.207 INFO:tasks.rados.rados.0.smithi022.stdout:3592: read oid 37 snap -1 2024-09-21T09:43:39.207 INFO:tasks.rados.rados.0.smithi022.stdout:3592: expect deleted 2024-09-21T09:43:39.207 INFO:tasks.rados.rados.0.smithi022.stdout:3593: rmattr oid 27 current snap is 349 2024-09-21T09:43:39.207 INFO:tasks.rados.rados.0.smithi022.stdout:3590: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-21T09:43:39.211 INFO:tasks.rados.rados.0.smithi022.stdout:3589: finishing copy_from racing read to smithi02284960-22 2024-09-21T09:43:39.211 INFO:tasks.rados.rados.0.smithi022.stdout:3589: finishing copy_from to smithi02284960-22 2024-09-21T09:43:39.211 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 22 v 1962 (ObjNum 1036 snap 315 seq_num 1036) dirty exists 2024-09-21T09:43:39.211 INFO:tasks.rados.rados.0.smithi022.stdout:3591: finishing write tid 1 to smithi02284960-5 2024-09-21T09:43:39.211 INFO:tasks.rados.rados.0.smithi022.stdout:3589: done (4 left) 2024-09-21T09:43:39.211 INFO:tasks.rados.rados.0.smithi022.stdout:3590: done (3 left) 2024-09-21T09:43:39.211 INFO:tasks.rados.rados.0.smithi022.stdout:3594: read oid 32 snap -1 2024-09-21T09:43:39.211 INFO:tasks.rados.rados.0.smithi022.stdout:3594: expect deleted 2024-09-21T09:43:39.211 INFO:tasks.rados.rados.0.smithi022.stdout:3595: snap_remove snap 340 2024-09-21T09:43:39.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:43:39 smithi064 ceph-mon[86709]: osdmap e776: 8 total, 8 up, 8 in 2024-09-21T09:43:39.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:43:39 smithi064 ceph-mon[86709]: pgmap v778: 121 pgs: 121 active+clean; 140 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 9.3 MiB/s rd, 3.7 MiB/s wr, 18 op/s 2024-09-21T09:43:39.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:43:39 smithi022 ceph-mon[98615]: osdmap e776: 8 total, 8 up, 8 in 2024-09-21T09:43:39.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:43:39 smithi022 ceph-mon[98615]: pgmap v778: 121 pgs: 121 active+clean; 140 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 9.3 MiB/s rd, 3.7 MiB/s wr, 18 op/s 2024-09-21T09:43:39.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:43:39 smithi022 ceph-mon[96736]: osdmap e776: 8 total, 8 up, 8 in 2024-09-21T09:43:39.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:43:39 smithi022 ceph-mon[96736]: pgmap v778: 121 pgs: 121 active+clean; 140 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 9.3 MiB/s rd, 3.7 MiB/s wr, 18 op/s 2024-09-21T09:43:40.150 INFO:tasks.rados.rados.0.smithi022.stdout:3595: done (4 left) 2024-09-21T09:43:40.150 INFO:tasks.rados.rados.0.smithi022.stdout:3596: read oid 9 snap -1 2024-09-21T09:43:40.150 INFO:tasks.rados.rados.0.smithi022.stdout:3596: expect (ObjNum 1102 snap 337 seq_num 1102) 2024-09-21T09:43:40.150 INFO:tasks.rados.rados.0.smithi022.stdout:3597: delete oid 7 current snap is 349 2024-09-21T09:43:40.150 INFO:tasks.rados.rados.0.smithi022.stdout:3591: finishing write tid 2 to smithi02284960-5 2024-09-21T09:43:40.150 INFO:tasks.rados.rados.0.smithi022.stdout:3591: finishing write tid 3 to smithi02284960-5 2024-09-21T09:43:40.150 INFO:tasks.rados.rados.0.smithi022.stdout:3591: finishing write tid 4 to smithi02284960-5 2024-09-21T09:43:40.150 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 27 v 1969 (ObjNum 1008 snap 309 seq_num 1008) dirty exists 2024-09-21T09:43:40.150 INFO:tasks.rados.rados.0.smithi022.stdout:3591: finishing write tid 5 to smithi02284960-5 2024-09-21T09:43:40.150 INFO:tasks.rados.rados.0.smithi022.stdout:3591: finishing write tid 6 to smithi02284960-5 2024-09-21T09:43:40.151 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 5 v 1984 (ObjNum 1130 snap 349 seq_num 1130) dirty exists 2024-09-21T09:43:40.151 INFO:tasks.rados.rados.0.smithi022.stdout:3591: left oid 5 (ObjNum 1130 snap 349 seq_num 1130) 2024-09-21T09:43:40.164 INFO:tasks.rados.rados.0.smithi022.stdout:3591: done (5 left) 2024-09-21T09:43:40.164 INFO:tasks.rados.rados.0.smithi022.stdout:3592: done (4 left) 2024-09-21T09:43:40.164 INFO:tasks.rados.rados.0.smithi022.stdout:3593: done (3 left) 2024-09-21T09:43:40.165 INFO:tasks.rados.rados.0.smithi022.stdout:3594: done (2 left) 2024-09-21T09:43:40.165 INFO:tasks.rados.rados.0.smithi022.stdout:3597: done (1 left) 2024-09-21T09:43:40.165 INFO:tasks.rados.rados.0.smithi022.stdout:3598: snap_remove snap 342 2024-09-21T09:43:40.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:43:40 smithi064 ceph-mon[86709]: osdmap e777: 8 total, 8 up, 8 in 2024-09-21T09:43:40.442 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:43:40 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:43:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:43:40.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:43:40 smithi022 ceph-mon[98615]: osdmap e777: 8 total, 8 up, 8 in 2024-09-21T09:43:40.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:43:40 smithi022 ceph-mon[96736]: osdmap e777: 8 total, 8 up, 8 in 2024-09-21T09:43:41.155 INFO:tasks.rados.rados.0.smithi022.stdout:3598: done (1 left) 2024-09-21T09:43:41.155 INFO:tasks.rados.rados.0.smithi022.stdout:3599: read oid 10 snap -1 2024-09-21T09:43:41.155 INFO:tasks.rados.rados.0.smithi022.stdout:3599: expect (ObjNum 1113 snap 339 seq_num 1113) 2024-09-21T09:43:41.155 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 38 current snap is 349 2024-09-21T09:43:41.155 INFO:tasks.rados.rados.0.smithi022.stdout:3600: seq_num 1132 ranges {0=565248,565248=425984,991232=770048,1761280=409600} 2024-09-21T09:43:41.158 INFO:tasks.rados.rados.0.smithi022.stdout:3600: writing smithi02284960-38 from 0 to 565248 tid 1 2024-09-21T09:43:41.161 INFO:tasks.rados.rados.0.smithi022.stdout:3600: writing smithi02284960-38 from 565248 to 991232 tid 2 2024-09-21T09:43:41.166 INFO:tasks.rados.rados.0.smithi022.stdout:3600: writing smithi02284960-38 from 991232 to 1761280 tid 3 2024-09-21T09:43:41.169 INFO:tasks.rados.rados.0.smithi022.stdout:3600: writing smithi02284960-38 from 1761280 to 2170880 tid 4 2024-09-21T09:43:41.169 INFO:tasks.rados.rados.0.smithi022.stdout:3601: read oid 3 snap -1 2024-09-21T09:43:41.169 INFO:tasks.rados.rados.0.smithi022.stdout:3601: expect (ObjNum 1109 snap 338 seq_num 1109) 2024-09-21T09:43:41.169 INFO:tasks.rados.rados.0.smithi022.stdout:3602: snap_remove snap 347 2024-09-21T09:43:41.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:43:41 smithi064 ceph-mon[86709]: osdmap e778: 8 total, 8 up, 8 in 2024-09-21T09:43:41.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:43:41 smithi064 ceph-mon[86709]: pgmap v781: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 139 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 6.4 MiB/s rd, 1.0 MiB/s wr, 9 op/s 2024-09-21T09:43:41.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:43:41 smithi022 ceph-mon[98615]: osdmap e778: 8 total, 8 up, 8 in 2024-09-21T09:43:41.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:43:41 smithi022 ceph-mon[98615]: pgmap v781: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 139 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 6.4 MiB/s rd, 1.0 MiB/s wr, 9 op/s 2024-09-21T09:43:41.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:43:41 smithi022 ceph-mon[96736]: osdmap e778: 8 total, 8 up, 8 in 2024-09-21T09:43:41.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:43:41 smithi022 ceph-mon[96736]: pgmap v781: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 139 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 6.4 MiB/s rd, 1.0 MiB/s wr, 9 op/s 2024-09-21T09:43:42.160 INFO:tasks.rados.rados.0.smithi022.stdout:3602: done (4 left) 2024-09-21T09:43:42.161 INFO:tasks.rados.rados.0.smithi022.stdout:3603: read oid 46 snap 345 2024-09-21T09:43:42.161 INFO:tasks.rados.rados.0.smithi022.stdout:3603: expect deleted 2024-09-21T09:43:42.161 INFO:tasks.rados.rados.0.smithi022.stdout:3604: rollback oid 11 current snap is 349 2024-09-21T09:43:42.161 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 11 to 346 2024-09-21T09:43:42.161 INFO:tasks.rados.rados.0.smithi022.stdout:3605: delete oid 19 current snap is 349 2024-09-21T09:43:42.161 INFO:tasks.rados.rados.0.smithi022.stdout:3596: expect (ObjNum 1102 snap 337 seq_num 1102) 2024-09-21T09:43:42.209 INFO:tasks.rados.rados.0.smithi022.stdout:3600: finishing write tid 1 to smithi02284960-38 2024-09-21T09:43:42.209 INFO:tasks.rados.rados.0.smithi022.stdout:3600: finishing write tid 2 to smithi02284960-38 2024-09-21T09:43:42.209 INFO:tasks.rados.rados.0.smithi022.stdout:3600: finishing write tid 3 to smithi02284960-38 2024-09-21T09:43:42.209 INFO:tasks.rados.rados.0.smithi022.stdout:3599: expect (ObjNum 1113 snap 339 seq_num 1113) 2024-09-21T09:43:42.334 INFO:tasks.rados.rados.0.smithi022.stdout:3600: finishing write tid 4 to smithi02284960-38 2024-09-21T09:43:42.334 INFO:tasks.rados.rados.0.smithi022.stdout:3600: finishing write tid 5 to smithi02284960-38 2024-09-21T09:43:42.334 INFO:tasks.rados.rados.0.smithi022.stdout:3600: finishing write tid 6 to smithi02284960-38 2024-09-21T09:43:42.334 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 38 v 1829 (ObjNum 1132 snap 349 seq_num 1132) dirty exists 2024-09-21T09:43:42.334 INFO:tasks.rados.rados.0.smithi022.stdout:3600: left oid 38 (ObjNum 1132 snap 349 seq_num 1132) 2024-09-21T09:43:42.334 INFO:tasks.rados.rados.0.smithi022.stdout:3601: expect (ObjNum 1109 snap 338 seq_num 1109) 2024-09-21T09:43:42.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:43:42 smithi064 ceph-mon[86709]: osdmap e779: 8 total, 8 up, 8 in 2024-09-21T09:43:42.484 INFO:tasks.rados.rados.0.smithi022.stdout:3604: finishing rollback tid 0 to smithi02284960-11 2024-09-21T09:43:42.484 INFO:tasks.rados.rados.0.smithi022.stdout:3604: finishing rollback tid 1 to smithi02284960-11 2024-09-21T09:43:42.484 INFO:tasks.rados.rados.0.smithi022.stdout:3604: finishing rollback tid 2 to smithi02284960-11 2024-09-21T09:43:42.484 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 11 v 1882 (ObjNum 1062 snap 319 seq_num 1062) dirty exists 2024-09-21T09:43:42.484 INFO:tasks.rados.rados.0.smithi022.stdout:3596: done (6 left) 2024-09-21T09:43:42.485 INFO:tasks.rados.rados.0.smithi022.stdout:3599: done (5 left) 2024-09-21T09:43:42.485 INFO:tasks.rados.rados.0.smithi022.stdout:3600: done (4 left) 2024-09-21T09:43:42.485 INFO:tasks.rados.rados.0.smithi022.stdout:3601: done (3 left) 2024-09-21T09:43:42.485 INFO:tasks.rados.rados.0.smithi022.stdout:3603: done (2 left) 2024-09-21T09:43:42.485 INFO:tasks.rados.rados.0.smithi022.stdout:3604: done (1 left) 2024-09-21T09:43:42.485 INFO:tasks.rados.rados.0.smithi022.stdout:3605: done (0 left) 2024-09-21T09:43:42.485 INFO:tasks.rados.rados.0.smithi022.stdout:3606: delete oid 10 current snap is 349 2024-09-21T09:43:42.489 INFO:tasks.rados.rados.0.smithi022.stdout:3606: done (0 left) 2024-09-21T09:43:42.489 INFO:tasks.rados.rados.0.smithi022.stdout:3607: rollback oid 11 current snap is 349 2024-09-21T09:43:42.489 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 11 to 346 2024-09-21T09:43:42.489 INFO:tasks.rados.rados.0.smithi022.stdout:3608: rollback oid 20 current snap is 349 2024-09-21T09:43:42.489 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 20 to 345 2024-09-21T09:43:42.489 INFO:tasks.rados.rados.0.smithi022.stdout:3609: rollback oid 50 current snap is 349 2024-09-21T09:43:42.489 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 50 to 345 2024-09-21T09:43:42.489 INFO:tasks.rados.rados.0.smithi022.stdout:3610: delete oid 14 current snap is 349 2024-09-21T09:43:42.490 INFO:tasks.rados.rados.0.smithi022.stdout:3610: done (3 left) 2024-09-21T09:43:42.490 INFO:tasks.rados.rados.0.smithi022.stdout:3611: delete oid 38 current snap is 349 2024-09-21T09:43:42.490 INFO:tasks.rados.rados.0.smithi022.stdout:3609: finishing rollback tid 1 to smithi02284960-50 2024-09-21T09:43:42.490 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 50 v 0 (ObjNum 129 snap 0 seq_num 3569588368) dirty dne 2024-09-21T09:43:42.493 INFO:tasks.rados.rados.0.smithi022.stdout:3607: finishing rollback tid 0 to smithi02284960-11 2024-09-21T09:43:42.493 INFO:tasks.rados.rados.0.smithi022.stdout:3608: finishing rollback tid 0 to smithi02284960-20 2024-09-21T09:43:42.493 INFO:tasks.rados.rados.0.smithi022.stdout:3607: finishing rollback tid 1 to smithi02284960-11 2024-09-21T09:43:42.494 INFO:tasks.rados.rados.0.smithi022.stdout:3607: finishing rollback tid 2 to smithi02284960-11 2024-09-21T09:43:42.494 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 11 v 1885 (ObjNum 1062 snap 319 seq_num 1062) dirty exists 2024-09-21T09:43:42.494 INFO:tasks.rados.rados.0.smithi022.stdout:3608: finishing rollback tid 1 to smithi02284960-20 2024-09-21T09:43:42.495 INFO:tasks.rados.rados.0.smithi022.stdout:3608: finishing rollback tid 2 to smithi02284960-20 2024-09-21T09:43:42.495 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 20 v 2105 (ObjNum 1122 snap 342 seq_num 1122) dirty exists 2024-09-21T09:43:42.497 INFO:tasks.rados.rados.0.smithi022.stdout:3607: done (3 left) 2024-09-21T09:43:42.497 INFO:tasks.rados.rados.0.smithi022.stdout:3608: done (2 left) 2024-09-21T09:43:42.497 INFO:tasks.rados.rados.0.smithi022.stdout:3609: done (1 left) 2024-09-21T09:43:42.498 INFO:tasks.rados.rados.0.smithi022.stdout:3611: done (0 left) 2024-09-21T09:43:42.498 INFO:tasks.rados.rados.0.smithi022.stdout:3612: read oid 30 snap 345 2024-09-21T09:43:42.498 INFO:tasks.rados.rados.0.smithi022.stdout:3612: expect (ObjNum 1102 snap 337 seq_num 1102) 2024-09-21T09:43:42.498 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 35 current snap is 349 2024-09-21T09:43:42.498 INFO:tasks.rados.rados.0.smithi022.stdout:3613: seq_num 1137 ranges {4710400=647168,5357568=696320,6053888=557056,6610944=352256} 2024-09-21T09:43:42.526 INFO:tasks.rados.rados.0.smithi022.stdout:3613: writing smithi02284960-35 from 4710400 to 5357568 tid 1 2024-09-21T09:43:42.530 INFO:tasks.rados.rados.0.smithi022.stdout:3613: writing smithi02284960-35 from 5357568 to 6053888 tid 2 2024-09-21T09:43:42.533 INFO:tasks.rados.rados.0.smithi022.stdout:3613: writing smithi02284960-35 from 6053888 to 6610944 tid 3 2024-09-21T09:43:42.536 INFO:tasks.rados.rados.0.smithi022.stdout:3613: writing smithi02284960-35 from 6610944 to 6963200 tid 4 2024-09-21T09:43:42.536 INFO:tasks.rados.rados.0.smithi022.stdout:3614: copy_from oid 6 from oid 11 current snap is 349 2024-09-21T09:43:42.536 INFO:tasks.rados.rados.0.smithi022.stdout:3612: expect (ObjNum 1102 snap 337 seq_num 1102) 2024-09-21T09:43:42.557 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:43:42 smithi022 ceph-mon[98615]: osdmap e779: 8 total, 8 up, 8 in 2024-09-21T09:43:42.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:43:42 smithi022 ceph-mon[96736]: osdmap e779: 8 total, 8 up, 8 in 2024-09-21T09:43:42.581 INFO:tasks.rados.rados.0.smithi022.stdout:3613: finishing write tid 1 to smithi02284960-35 2024-09-21T09:43:42.581 INFO:tasks.rados.rados.0.smithi022.stdout:3613: finishing write tid 2 to smithi02284960-35 2024-09-21T09:43:42.581 INFO:tasks.rados.rados.0.smithi022.stdout:3613: finishing write tid 3 to smithi02284960-35 2024-09-21T09:43:42.581 INFO:tasks.rados.rados.0.smithi022.stdout:3613: finishing write tid 4 to smithi02284960-35 2024-09-21T09:43:42.581 INFO:tasks.rados.rados.0.smithi022.stdout:3613: finishing write tid 5 to smithi02284960-35 2024-09-21T09:43:42.581 INFO:tasks.rados.rados.0.smithi022.stdout:3613: finishing write tid 6 to smithi02284960-35 2024-09-21T09:43:42.581 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 35 v 1815 (ObjNum 1137 snap 349 seq_num 1137) dirty exists 2024-09-21T09:43:42.581 INFO:tasks.rados.rados.0.smithi022.stdout:3613: left oid 35 (ObjNum 1137 snap 349 seq_num 1137) 2024-09-21T09:43:42.581 INFO:tasks.rados.rados.0.smithi022.stdout:3614: finishing copy_from racing read to smithi02284960-6 2024-09-21T09:43:42.581 INFO:tasks.rados.rados.0.smithi022.stdout:3614: finishing copy_from to smithi02284960-6 2024-09-21T09:43:42.582 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 6 v 1885 (ObjNum 1062 snap 319 seq_num 1062) dirty exists 2024-09-21T09:43:42.582 INFO:tasks.rados.rados.0.smithi022.stdout:3612: done (2 left) 2024-09-21T09:43:42.582 INFO:tasks.rados.rados.0.smithi022.stdout:3613: done (1 left) 2024-09-21T09:43:42.582 INFO:tasks.rados.rados.0.smithi022.stdout:3614: done (0 left) 2024-09-21T09:43:42.582 INFO:tasks.rados.rados.0.smithi022.stdout:3615: snap_create 2024-09-21T09:43:43.164 INFO:tasks.rados.rados.0.smithi022.stdout:3615: done (0 left) 2024-09-21T09:43:43.164 INFO:tasks.rados.rados.0.smithi022.stdout:3616: delete oid 26 current snap is 350 2024-09-21T09:43:43.170 INFO:tasks.rados.rados.0.smithi022.stdout:3616: done (0 left) 2024-09-21T09:43:43.171 INFO:tasks.rados.rados.0.smithi022.stdout:3617: snap_remove snap 349 2024-09-21T09:43:43.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:43:43 smithi064 ceph-mon[86709]: osdmap e780: 8 total, 8 up, 8 in 2024-09-21T09:43:43.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:43:43 smithi064 ceph-mon[86709]: pgmap v784: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 143 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2.4 MiB/s rd, 1.5 MiB/s wr, 10 op/s 2024-09-21T09:43:43.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:43:43 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:43:43.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:43:43 smithi022 ceph-mon[98615]: osdmap e780: 8 total, 8 up, 8 in 2024-09-21T09:43:43.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:43:43 smithi022 ceph-mon[98615]: pgmap v784: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 143 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2.4 MiB/s rd, 1.5 MiB/s wr, 10 op/s 2024-09-21T09:43:43.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:43:43 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:43:43.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:43:43 smithi022 ceph-mon[96736]: osdmap e780: 8 total, 8 up, 8 in 2024-09-21T09:43:43.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:43:43 smithi022 ceph-mon[96736]: pgmap v784: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 143 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2.4 MiB/s rd, 1.5 MiB/s wr, 10 op/s 2024-09-21T09:43:43.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:43:43 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:43:44.098 INFO:tasks.rados.rados.0.smithi022.stdout:3617: done (0 left) 2024-09-21T09:43:44.098 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 1 current snap is 350 2024-09-21T09:43:44.098 INFO:tasks.rados.rados.0.smithi022.stdout:3618: seq_num 1139 ranges {3670016=679936,4349952=458752,4808704=524288,5332992=540672,5873664=81920} 2024-09-21T09:43:44.122 INFO:tasks.rados.rados.0.smithi022.stdout:3618: writing smithi02284960-1 from 3670016 to 4349952 tid 1 2024-09-21T09:43:44.124 INFO:tasks.rados.rados.0.smithi022.stdout:3618: writing smithi02284960-1 from 4349952 to 4808704 tid 2 2024-09-21T09:43:44.128 INFO:tasks.rados.rados.0.smithi022.stdout:3618: writing smithi02284960-1 from 4808704 to 5332992 tid 3 2024-09-21T09:43:44.131 INFO:tasks.rados.rados.0.smithi022.stdout:3618: writing smithi02284960-1 from 5332992 to 5873664 tid 4 2024-09-21T09:43:44.132 INFO:tasks.rados.rados.0.smithi022.stdout:3618: writing smithi02284960-1 from 5873664 to 5955584 tid 5 2024-09-21T09:43:44.132 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 31 current snap is 350 2024-09-21T09:43:44.133 INFO:tasks.rados.rados.0.smithi022.stdout:3619: seq_num 1140 ranges {1449984=696320,2146304=745472,2891776=729088,3620864=131072} 2024-09-21T09:43:44.144 INFO:tasks.rados.rados.0.smithi022.stdout:3619: writing smithi02284960-31 from 1449984 to 2146304 tid 1 2024-09-21T09:43:44.148 INFO:tasks.rados.rados.0.smithi022.stdout:3619: writing smithi02284960-31 from 2146304 to 2891776 tid 2 2024-09-21T09:43:44.153 INFO:tasks.rados.rados.0.smithi022.stdout:3619: writing smithi02284960-31 from 2891776 to 3620864 tid 3 2024-09-21T09:43:44.154 INFO:tasks.rados.rados.0.smithi022.stdout:3619: writing smithi02284960-31 from 3620864 to 3751936 tid 4 2024-09-21T09:43:44.154 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 38 current snap is 350 2024-09-21T09:43:44.154 INFO:tasks.rados.rados.0.smithi022.stdout:3620: seq_num 1141 ranges {0=712704,712704=679936,1392640=540672,1933312=385024} 2024-09-21T09:43:44.158 INFO:tasks.rados.rados.0.smithi022.stdout:3620: writing smithi02284960-38 from 0 to 712704 tid 1 2024-09-21T09:43:44.163 INFO:tasks.rados.rados.0.smithi022.stdout:3620: writing smithi02284960-38 from 712704 to 1392640 tid 2 2024-09-21T09:43:44.166 INFO:tasks.rados.rados.0.smithi022.stdout:3620: writing smithi02284960-38 from 1392640 to 1933312 tid 3 2024-09-21T09:43:44.169 INFO:tasks.rados.rados.0.smithi022.stdout:3620: writing smithi02284960-38 from 1933312 to 2318336 tid 4 2024-09-21T09:43:44.169 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 44 current snap is 350 2024-09-21T09:43:44.169 INFO:tasks.rados.rados.0.smithi022.stdout:3618: finishing write tid 1 to smithi02284960-1 2024-09-21T09:43:44.169 INFO:tasks.rados.rados.0.smithi022.stdout:3618: finishing write tid 2 to smithi02284960-1 2024-09-21T09:43:44.169 INFO:tasks.rados.rados.0.smithi022.stdout:3618: finishing write tid 3 to smithi02284960-1 2024-09-21T09:43:44.169 INFO:tasks.rados.rados.0.smithi022.stdout:3618: finishing write tid 4 to smithi02284960-1 2024-09-21T09:43:44.170 INFO:tasks.rados.rados.0.smithi022.stdout:3618: finishing write tid 5 to smithi02284960-1 2024-09-21T09:43:44.170 INFO:tasks.rados.rados.0.smithi022.stdout:3618: finishing write tid 6 to smithi02284960-1 2024-09-21T09:43:44.170 INFO:tasks.rados.rados.0.smithi022.stdout:3618: finishing write tid 7 to smithi02284960-1 2024-09-21T09:43:44.170 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 1 v 1891 (ObjNum 1139 snap 350 seq_num 1139) dirty exists 2024-09-21T09:43:44.170 INFO:tasks.rados.rados.0.smithi022.stdout:3618: left oid 1 (ObjNum 1139 snap 350 seq_num 1139) 2024-09-21T09:43:44.170 INFO:tasks.rados.rados.0.smithi022.stdout:3619: finishing write tid 1 to smithi02284960-31 2024-09-21T09:43:44.170 INFO:tasks.rados.rados.0.smithi022.stdout:3619: finishing write tid 2 to smithi02284960-31 2024-09-21T09:43:44.170 INFO:tasks.rados.rados.0.smithi022.stdout:3621: seq_num 1142 ranges {4907008=729088,5636096=565248,6201344=696320,6897664=344064} 2024-09-21T09:43:44.199 INFO:tasks.rados.rados.0.smithi022.stdout:3621: writing smithi02284960-44 from 4907008 to 5636096 tid 1 2024-09-21T09:43:44.202 INFO:tasks.rados.rados.0.smithi022.stdout:3621: writing smithi02284960-44 from 5636096 to 6201344 tid 2 2024-09-21T09:43:44.206 INFO:tasks.rados.rados.0.smithi022.stdout:3621: writing smithi02284960-44 from 6201344 to 6897664 tid 3 2024-09-21T09:43:44.209 INFO:tasks.rados.rados.0.smithi022.stdout:3621: writing smithi02284960-44 from 6897664 to 7241728 tid 4 2024-09-21T09:43:44.209 INFO:tasks.rados.rados.0.smithi022.stdout:3618: done (3 left) 2024-09-21T09:43:44.209 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 29 current snap is 350 2024-09-21T09:43:44.209 INFO:tasks.rados.rados.0.smithi022.stdout:3619: finishing write tid 3 to smithi02284960-31 2024-09-21T09:43:44.209 INFO:tasks.rados.rados.0.smithi022.stdout:3619: finishing write tid 4 to smithi02284960-31 2024-09-21T09:43:44.209 INFO:tasks.rados.rados.0.smithi022.stdout:3619: finishing write tid 5 to smithi02284960-31 2024-09-21T09:43:44.209 INFO:tasks.rados.rados.0.smithi022.stdout:3619: finishing write tid 6 to smithi02284960-31 2024-09-21T09:43:44.209 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 31 v 2086 (ObjNum 1140 snap 350 seq_num 1140) dirty exists 2024-09-21T09:43:44.209 INFO:tasks.rados.rados.0.smithi022.stdout:3619: left oid 31 (ObjNum 1140 snap 350 seq_num 1140) 2024-09-21T09:43:44.210 INFO:tasks.rados.rados.0.smithi022.stdout:3622: seq_num 1143 ranges {286720=753664,1040384=450560,1490944=507904,1998848=589824,2588672=57344} 2024-09-21T09:43:44.214 INFO:tasks.rados.rados.0.smithi022.stdout:3622: writing smithi02284960-29 from 286720 to 1040384 tid 1 2024-09-21T09:43:44.217 INFO:tasks.rados.rados.0.smithi022.stdout:3622: writing smithi02284960-29 from 1040384 to 1490944 tid 2 2024-09-21T09:43:44.220 INFO:tasks.rados.rados.0.smithi022.stdout:3622: writing smithi02284960-29 from 1490944 to 1998848 tid 3 2024-09-21T09:43:44.223 INFO:tasks.rados.rados.0.smithi022.stdout:3622: writing smithi02284960-29 from 1998848 to 2588672 tid 4 2024-09-21T09:43:44.226 INFO:tasks.rados.rados.0.smithi022.stdout:3622: writing smithi02284960-29 from 2588672 to 2646016 tid 5 2024-09-21T09:43:44.226 INFO:tasks.rados.rados.0.smithi022.stdout:3619: done (3 left) 2024-09-21T09:43:44.226 INFO:tasks.rados.rados.0.smithi022.stdout:3623: rmattr oid 19 current snap is 350 2024-09-21T09:43:44.226 INFO:tasks.rados.rados.0.smithi022.stdout:3623: done (3 left) 2024-09-21T09:43:44.226 INFO:tasks.rados.rados.0.smithi022.stdout:3624: rollback oid 27 current snap is 350 2024-09-21T09:43:44.226 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 27 to 345 2024-09-21T09:43:44.226 INFO:tasks.rados.rados.0.smithi022.stdout:3620: finishing write tid 1 to smithi02284960-38 2024-09-21T09:43:44.226 INFO:tasks.rados.rados.0.smithi022.stdout:3620: finishing write tid 2 to smithi02284960-38 2024-09-21T09:43:44.226 INFO:tasks.rados.rados.0.smithi022.stdout:3620: finishing write tid 3 to smithi02284960-38 2024-09-21T09:43:44.226 INFO:tasks.rados.rados.0.smithi022.stdout:3620: finishing write tid 4 to smithi02284960-38 2024-09-21T09:43:44.226 INFO:tasks.rados.rados.0.smithi022.stdout:3620: finishing write tid 5 to smithi02284960-38 2024-09-21T09:43:44.226 INFO:tasks.rados.rados.0.smithi022.stdout:3620: finishing write tid 6 to smithi02284960-38 2024-09-21T09:43:44.226 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 38 v 1835 (ObjNum 1141 snap 350 seq_num 1141) dirty exists 2024-09-21T09:43:44.227 INFO:tasks.rados.rados.0.smithi022.stdout:3620: left oid 38 (ObjNum 1141 snap 350 seq_num 1141) 2024-09-21T09:43:44.227 INFO:tasks.rados.rados.0.smithi022.stdout:3621: finishing write tid 1 to smithi02284960-44 2024-09-21T09:43:44.227 INFO:tasks.rados.rados.0.smithi022.stdout:3620: done (3 left) 2024-09-21T09:43:44.227 INFO:tasks.rados.rados.0.smithi022.stdout:3625: rmattr oid 2 current snap is 350 2024-09-21T09:43:44.227 INFO:tasks.rados.rados.0.smithi022.stdout:3621: finishing write tid 2 to smithi02284960-44 2024-09-21T09:43:44.227 INFO:tasks.rados.rados.0.smithi022.stdout:3621: finishing write tid 3 to smithi02284960-44 2024-09-21T09:43:44.227 INFO:tasks.rados.rados.0.smithi022.stdout:3621: finishing write tid 4 to smithi02284960-44 2024-09-21T09:43:44.227 INFO:tasks.rados.rados.0.smithi022.stdout:3621: finishing write tid 5 to smithi02284960-44 2024-09-21T09:43:44.227 INFO:tasks.rados.rados.0.smithi022.stdout:3621: finishing write tid 6 to smithi02284960-44 2024-09-21T09:43:44.227 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 44 v 1896 (ObjNum 1142 snap 350 seq_num 1142) dirty exists 2024-09-21T09:43:44.227 INFO:tasks.rados.rados.0.smithi022.stdout:3621: left oid 44 (ObjNum 1142 snap 350 seq_num 1142) 2024-09-21T09:43:44.227 INFO:tasks.rados.rados.0.smithi022.stdout:3622: finishing write tid 1 to smithi02284960-29 2024-09-21T09:43:44.228 INFO:tasks.rados.rados.0.smithi022.stdout:3622: finishing write tid 2 to smithi02284960-29 2024-09-21T09:43:44.228 INFO:tasks.rados.rados.0.smithi022.stdout:3621: done (3 left) 2024-09-21T09:43:44.228 INFO:tasks.rados.rados.0.smithi022.stdout:3626: rmattr oid 42 current snap is 350 2024-09-21T09:43:44.228 INFO:tasks.rados.rados.0.smithi022.stdout:3626: done (3 left) 2024-09-21T09:43:44.228 INFO:tasks.rados.rados.0.smithi022.stdout:3622: finishing write tid 3 to smithi02284960-29 2024-09-21T09:43:44.228 INFO:tasks.rados.rados.0.smithi022.stdout:3627: read oid 24 snap -1 2024-09-21T09:43:44.228 INFO:tasks.rados.rados.0.smithi022.stdout:3627: expect (ObjNum 1079 snap 326 seq_num 1079) 2024-09-21T09:43:44.228 INFO:tasks.rados.rados.0.smithi022.stdout:3628: delete oid 40 current snap is 350 2024-09-21T09:43:44.228 INFO:tasks.rados.rados.0.smithi022.stdout:3622: finishing write tid 4 to smithi02284960-29 2024-09-21T09:43:44.228 INFO:tasks.rados.rados.0.smithi022.stdout:3622: finishing write tid 5 to smithi02284960-29 2024-09-21T09:43:44.229 INFO:tasks.rados.rados.0.smithi022.stdout:3622: finishing write tid 6 to smithi02284960-29 2024-09-21T09:43:44.230 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 2 v 1845 (ObjNum 1127 snap 348 seq_num 1127) dirty exists 2024-09-21T09:43:44.230 INFO:tasks.rados.rados.0.smithi022.stdout:3625: done (4 left) 2024-09-21T09:43:44.230 INFO:tasks.rados.rados.0.smithi022.stdout:3628: done (3 left) 2024-09-21T09:43:44.230 INFO:tasks.rados.rados.0.smithi022.stdout:3629: read oid 39 snap -1 2024-09-21T09:43:44.230 INFO:tasks.rados.rados.0.smithi022.stdout:3629: expect (ObjNum 1117 snap 341 seq_num 1117) 2024-09-21T09:43:44.230 INFO:tasks.rados.rados.0.smithi022.stdout:3630: copy_from oid 45 from oid 34 current snap is 350 2024-09-21T09:43:44.230 INFO:tasks.rados.rados.0.smithi022.stdout:3631: delete oid 36 current snap is 350 2024-09-21T09:43:44.231 INFO:tasks.rados.rados.0.smithi022.stdout:3622: finishing write tid 7 to smithi02284960-29 2024-09-21T09:43:44.231 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 29 v 1841 (ObjNum 1143 snap 350 seq_num 1143) dirty exists 2024-09-21T09:43:44.231 INFO:tasks.rados.rados.0.smithi022.stdout:3622: left oid 29 (ObjNum 1143 snap 350 seq_num 1143) 2024-09-21T09:43:44.231 INFO:tasks.rados.rados.0.smithi022.stdout:3624: finishing rollback tid 0 to smithi02284960-27 2024-09-21T09:43:44.231 INFO:tasks.rados.rados.0.smithi022.stdout:3624: finishing rollback tid 1 to smithi02284960-27 2024-09-21T09:43:44.233 INFO:tasks.rados.rados.0.smithi022.stdout:3630: finishing copy_from to smithi02284960-45 2024-09-21T09:43:44.233 INFO:tasks.rados.rados.0.smithi022.stdout:3630: got expected ENOENT (src dne) 2024-09-21T09:43:44.233 INFO:tasks.rados.rados.0.smithi022.stdout:3624: finishing rollback tid 2 to smithi02284960-27 2024-09-21T09:43:44.233 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 27 v 1973 (ObjNum 1008 snap 309 seq_num 1008) dirty exists 2024-09-21T09:43:44.233 INFO:tasks.rados.rados.0.smithi022.stdout:3630: finishing copy_from racing read to smithi02284960-45 2024-09-21T09:43:44.234 INFO:tasks.rados.rados.0.smithi022.stdout:3622: done (5 left) 2024-09-21T09:43:44.234 INFO:tasks.rados.rados.0.smithi022.stdout:3624: done (4 left) 2024-09-21T09:43:44.234 INFO:tasks.rados.rados.0.smithi022.stdout:3630: done (3 left) 2024-09-21T09:43:44.235 INFO:tasks.rados.rados.0.smithi022.stdout:3631: done (2 left) 2024-09-21T09:43:44.235 INFO:tasks.rados.rados.0.smithi022.stdout:3632: rollback oid 48 current snap is 350 2024-09-21T09:43:44.235 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 48 to 345 2024-09-21T09:43:44.235 INFO:tasks.rados.rados.0.smithi022.stdout:3633: rollback oid 17 current snap is 350 2024-09-21T09:43:44.235 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 17 to 345 2024-09-21T09:43:44.235 INFO:tasks.rados.rados.0.smithi022.stdout:3634: delete oid 4 current snap is 350 2024-09-21T09:43:44.240 INFO:tasks.rados.rados.0.smithi022.stdout:3634: done (4 left) 2024-09-21T09:43:44.241 INFO:tasks.rados.rados.0.smithi022.stdout:3635: copy_from oid 38 from oid 34 current snap is 350 2024-09-21T09:43:44.241 INFO:tasks.rados.rados.0.smithi022.stdout:3632: finishing rollback tid 0 to smithi02284960-48 2024-09-21T09:43:44.241 INFO:tasks.rados.rados.0.smithi022.stdout:3632: finishing rollback tid 1 to smithi02284960-48 2024-09-21T09:43:44.241 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 48 v 2108 (ObjNum 49 snap 0 seq_num 1953066355) dirty dne 2024-09-21T09:43:44.241 INFO:tasks.rados.rados.0.smithi022.stdout:3632: done (4 left) 2024-09-21T09:43:44.241 INFO:tasks.rados.rados.0.smithi022.stdout:3636: snap_create 2024-09-21T09:43:44.243 INFO:tasks.rados.rados.0.smithi022.stdout:3635: finishing copy_from to smithi02284960-38 2024-09-21T09:43:44.243 INFO:tasks.rados.rados.0.smithi022.stdout:3635: got expected ENOENT (src dne) 2024-09-21T09:43:44.243 INFO:tasks.rados.rados.0.smithi022.stdout:3635: finishing copy_from racing read to smithi02284960-38 2024-09-21T09:43:44.255 INFO:tasks.rados.rados.0.smithi022.stdout:3629: expect (ObjNum 1117 snap 341 seq_num 1117) 2024-09-21T09:43:44.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:43:44 smithi064 ceph-mon[86709]: osdmap e781: 8 total, 8 up, 8 in 2024-09-21T09:43:44.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:43:44 smithi064 ceph-mon[86709]: osdmap e782: 8 total, 8 up, 8 in 2024-09-21T09:43:44.489 INFO:tasks.rados.rados.0.smithi022.stdout:3633: finishing rollback tid 0 to smithi02284960-17 2024-09-21T09:43:44.489 INFO:tasks.rados.rados.0.smithi022.stdout:3633: finishing rollback tid 1 to smithi02284960-17 2024-09-21T09:43:44.489 INFO:tasks.rados.rados.0.smithi022.stdout:3633: finishing rollback tid 2 to smithi02284960-17 2024-09-21T09:43:44.489 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 17 v 1969 (ObjNum 1105 snap 338 seq_num 1105) dirty exists 2024-09-21T09:43:44.489 INFO:tasks.rados.rados.0.smithi022.stdout:3627: expect (ObjNum 1079 snap 326 seq_num 1079) 2024-09-21T09:43:44.557 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:43:44 smithi022 ceph-mon[98615]: osdmap e781: 8 total, 8 up, 8 in 2024-09-21T09:43:44.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:43:44 smithi022 ceph-mon[98615]: osdmap e782: 8 total, 8 up, 8 in 2024-09-21T09:43:44.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:43:44 smithi022 ceph-mon[96736]: osdmap e781: 8 total, 8 up, 8 in 2024-09-21T09:43:44.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:43:44 smithi022 ceph-mon[96736]: osdmap e782: 8 total, 8 up, 8 in 2024-09-21T09:43:45.169 INFO:tasks.rados.rados.0.smithi022.stdout:3627: done (4 left) 2024-09-21T09:43:45.169 INFO:tasks.rados.rados.0.smithi022.stdout:3629: done (3 left) 2024-09-21T09:43:45.169 INFO:tasks.rados.rados.0.smithi022.stdout:3633: done (2 left) 2024-09-21T09:43:45.169 INFO:tasks.rados.rados.0.smithi022.stdout:3635: done (1 left) 2024-09-21T09:43:45.169 INFO:tasks.rados.rados.0.smithi022.stdout:3636: done (0 left) 2024-09-21T09:43:45.170 INFO:tasks.rados.rados.0.smithi022.stdout:3637: delete oid 13 current snap is 351 2024-09-21T09:43:45.175 INFO:tasks.rados.rados.0.smithi022.stdout:3637: done (0 left) 2024-09-21T09:43:45.175 INFO:tasks.rados.rados.0.smithi022.stdout:3638: delete oid 2 current snap is 351 2024-09-21T09:43:45.181 INFO:tasks.rados.rados.0.smithi022.stdout:3638: done (0 left) 2024-09-21T09:43:45.181 INFO:tasks.rados.rados.0.smithi022.stdout:3639: snap_create 2024-09-21T09:43:45.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:43:45 smithi064 ceph-mon[86709]: pgmap v787: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 146 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.1 MiB/s wr, 8 op/s 2024-09-21T09:43:45.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:43:45 smithi022 ceph-mon[98615]: pgmap v787: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 146 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.1 MiB/s wr, 8 op/s 2024-09-21T09:43:45.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:43:45 smithi022 ceph-mon[96736]: pgmap v787: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 146 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.1 MiB/s wr, 8 op/s 2024-09-21T09:43:46.058 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:43:45 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:43:45] "GET /metrics HTTP/1.1" 200 36351 "" "Prometheus/2.51.0" 2024-09-21T09:43:46.176 INFO:tasks.rados.rados.0.smithi022.stdout:3639: done (0 left) 2024-09-21T09:43:46.176 INFO:tasks.rados.rados.0.smithi022.stdout:3640: read oid 49 snap -1 2024-09-21T09:43:46.176 INFO:tasks.rados.rados.0.smithi022.stdout:3640: expect (ObjNum 985 snap 305 seq_num 985) 2024-09-21T09:43:46.176 INFO:tasks.rados.rados.0.smithi022.stdout:3641: snap_remove snap 351 2024-09-21T09:43:46.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:43:46 smithi064 ceph-mon[86709]: osdmap e783: 8 total, 8 up, 8 in 2024-09-21T09:43:46.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:43:46 smithi022 ceph-mon[98615]: osdmap e783: 8 total, 8 up, 8 in 2024-09-21T09:43:46.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:43:46 smithi022 ceph-mon[96736]: osdmap e783: 8 total, 8 up, 8 in 2024-09-21T09:43:47.182 INFO:tasks.rados.rados.0.smithi022.stdout:3641: done (1 left) 2024-09-21T09:43:47.182 INFO:tasks.rados.rados.0.smithi022.stdout:3642: snap_remove snap 350 2024-09-21T09:43:47.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:43:47 smithi064 ceph-mon[86709]: osdmap e784: 8 total, 8 up, 8 in 2024-09-21T09:43:47.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:43:47 smithi064 ceph-mon[86709]: pgmap v790: 121 pgs: 121 active+clean; 153 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 2.9 MiB/s wr, 18 op/s 2024-09-21T09:43:47.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:43:47 smithi022 ceph-mon[98615]: osdmap e784: 8 total, 8 up, 8 in 2024-09-21T09:43:47.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:43:47 smithi022 ceph-mon[98615]: pgmap v790: 121 pgs: 121 active+clean; 153 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 2.9 MiB/s wr, 18 op/s 2024-09-21T09:43:47.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:43:47 smithi022 ceph-mon[96736]: osdmap e784: 8 total, 8 up, 8 in 2024-09-21T09:43:47.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:43:47 smithi022 ceph-mon[96736]: pgmap v790: 121 pgs: 121 active+clean; 153 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 2.9 MiB/s wr, 18 op/s 2024-09-21T09:43:48.184 INFO:tasks.rados.rados.0.smithi022.stdout:3642: done (1 left) 2024-09-21T09:43:48.184 INFO:tasks.rados.rados.0.smithi022.stdout:3643: delete oid 14 current snap is 352 2024-09-21T09:43:48.184 INFO:tasks.rados.rados.0.smithi022.stdout:3640: expect (ObjNum 985 snap 305 seq_num 985) 2024-09-21T09:43:48.431 INFO:tasks.rados.rados.0.smithi022.stdout:3640: done (1 left) 2024-09-21T09:43:48.431 INFO:tasks.rados.rados.0.smithi022.stdout:3643: done (0 left) 2024-09-21T09:43:48.432 INFO:tasks.rados.rados.0.smithi022.stdout:3644: rollback oid 19 current snap is 352 2024-09-21T09:43:48.432 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 19 to 346 2024-09-21T09:43:48.432 INFO:tasks.rados.rados.0.smithi022.stdout:3645: copy_from oid 18 from oid 42 current snap is 352 2024-09-21T09:43:48.432 INFO:tasks.rados.rados.0.smithi022.stdout:3646: read oid 30 snap 345 2024-09-21T09:43:48.432 INFO:tasks.rados.rados.0.smithi022.stdout:3646: expect (ObjNum 1102 snap 337 seq_num 1102) 2024-09-21T09:43:48.432 INFO:tasks.rados.rados.0.smithi022.stdout:3647: read oid 10 snap -1 2024-09-21T09:43:48.432 INFO:tasks.rados.rados.0.smithi022.stdout:3647: expect deleted 2024-09-21T09:43:48.432 INFO:tasks.rados.rados.0.smithi022.stdout:3648: delete oid 21 current snap is 352 2024-09-21T09:43:48.432 INFO:tasks.rados.rados.0.smithi022.stdout:3644: finishing rollback tid 1 to smithi02284960-19 2024-09-21T09:43:48.432 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 19 v 0 (ObjNum 97 snap 0 seq_num 3569632656) dirty dne 2024-09-21T09:43:48.434 INFO:tasks.rados.rados.0.smithi022.stdout:3644: done (4 left) 2024-09-21T09:43:48.434 INFO:tasks.rados.rados.0.smithi022.stdout:3647: done (3 left) 2024-09-21T09:43:48.434 INFO:tasks.rados.rados.0.smithi022.stdout:3648: done (2 left) 2024-09-21T09:43:48.434 INFO:tasks.rados.rados.0.smithi022.stdout:3649: rmattr oid 10 current snap is 352 2024-09-21T09:43:48.434 INFO:tasks.rados.rados.0.smithi022.stdout:3649: done (2 left) 2024-09-21T09:43:48.434 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 39 current snap is 352 2024-09-21T09:43:48.434 INFO:tasks.rados.rados.0.smithi022.stdout:3650: seq_num 1151 ranges {5480448=81920} 2024-09-21T09:43:48.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:43:48 smithi064 ceph-mon[86709]: osdmap e785: 8 total, 8 up, 8 in 2024-09-21T09:43:48.464 INFO:tasks.rados.rados.0.smithi022.stdout:3650: writing smithi02284960-39 from 5480448 to 5562368 tid 1 2024-09-21T09:43:48.465 INFO:tasks.rados.rados.0.smithi022.stdout:3651: rollback oid 20 current snap is 352 2024-09-21T09:43:48.465 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 20 to 345 2024-09-21T09:43:48.465 INFO:tasks.rados.rados.0.smithi022.stdout:3646: expect (ObjNum 1102 snap 337 seq_num 1102) 2024-09-21T09:43:48.511 INFO:tasks.rados.rados.0.smithi022.stdout:3645: finishing copy_from racing read to smithi02284960-18 2024-09-21T09:43:48.512 INFO:tasks.rados.rados.0.smithi022.stdout:3645: finishing copy_from to smithi02284960-18 2024-09-21T09:43:48.512 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 18 v 1752 (ObjNum 1062 snap 319 seq_num 1062) dirty exists 2024-09-21T09:43:48.512 INFO:tasks.rados.rados.0.smithi022.stdout:3650: finishing write tid 1 to smithi02284960-39 2024-09-21T09:43:48.512 INFO:tasks.rados.rados.0.smithi022.stdout:3645: done (3 left) 2024-09-21T09:43:48.512 INFO:tasks.rados.rados.0.smithi022.stdout:3646: done (2 left) 2024-09-21T09:43:48.512 INFO:tasks.rados.rados.0.smithi022.stdout:3652: rmattr oid 6 current snap is 352 2024-09-21T09:43:48.512 INFO:tasks.rados.rados.0.smithi022.stdout:3652: done (2 left) 2024-09-21T09:43:48.512 INFO:tasks.rados.rados.0.smithi022.stdout:3653: read oid 17 snap -1 2024-09-21T09:43:48.512 INFO:tasks.rados.rados.0.smithi022.stdout:3653: expect (ObjNum 1105 snap 338 seq_num 1105) 2024-09-21T09:43:48.512 INFO:tasks.rados.rados.0.smithi022.stdout:3650: finishing write tid 2 to smithi02284960-39 2024-09-21T09:43:48.513 INFO:tasks.rados.rados.0.smithi022.stdout:3650: finishing write tid 3 to smithi02284960-39 2024-09-21T09:43:48.513 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 39 v 1971 (ObjNum 1151 snap 352 seq_num 1151) dirty exists 2024-09-21T09:43:48.513 INFO:tasks.rados.rados.0.smithi022.stdout:3650: left oid 39 (ObjNum 1151 snap 352 seq_num 1151) 2024-09-21T09:43:48.513 INFO:tasks.rados.rados.0.smithi022.stdout:3651: finishing rollback tid 0 to smithi02284960-20 2024-09-21T09:43:48.513 INFO:tasks.rados.rados.0.smithi022.stdout:3651: finishing rollback tid 1 to smithi02284960-20 2024-09-21T09:43:48.513 INFO:tasks.rados.rados.0.smithi022.stdout:3651: finishing rollback tid 2 to smithi02284960-20 2024-09-21T09:43:48.513 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 20 v 2111 (ObjNum 1122 snap 342 seq_num 1122) dirty exists 2024-09-21T09:43:48.513 INFO:tasks.rados.rados.0.smithi022.stdout:3654: read oid 5 snap -1 2024-09-21T09:43:48.513 INFO:tasks.rados.rados.0.smithi022.stdout:3654: expect (ObjNum 1130 snap 349 seq_num 1130) 2024-09-21T09:43:48.513 INFO:tasks.rados.rados.0.smithi022.stdout:3650: done (3 left) 2024-09-21T09:43:48.513 INFO:tasks.rados.rados.0.smithi022.stdout:3651: done (2 left) 2024-09-21T09:43:48.513 INFO:tasks.rados.rados.0.smithi022.stdout:3655: rmattr oid 27 current snap is 352 2024-09-21T09:43:48.513 INFO:tasks.rados.rados.0.smithi022.stdout:3656: read oid 9 snap 345 2024-09-21T09:43:48.514 INFO:tasks.rados.rados.0.smithi022.stdout:3656: expect (ObjNum 1102 snap 337 seq_num 1102) 2024-09-21T09:43:48.514 INFO:tasks.rados.rados.0.smithi022.stdout:3657: copy_from oid 15 from oid 44 current snap is 352 2024-09-21T09:43:48.514 INFO:tasks.rados.rados.0.smithi022.stdout:3658: snap_create 2024-09-21T09:43:48.515 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 27 v 1975 (ObjNum 1008 snap 309 seq_num 1008) dirty exists 2024-09-21T09:43:48.517 INFO:tasks.rados.rados.0.smithi022.stdout:3653: expect (ObjNum 1105 snap 338 seq_num 1105) 2024-09-21T09:43:48.557 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:43:48 smithi022 ceph-mon[98615]: osdmap e785: 8 total, 8 up, 8 in 2024-09-21T09:43:48.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:43:48 smithi022 ceph-mon[96736]: osdmap e785: 8 total, 8 up, 8 in 2024-09-21T09:43:48.576 INFO:tasks.rados.rados.0.smithi022.stdout:3657: finishing copy_from racing read to smithi02284960-15 2024-09-21T09:43:48.576 INFO:tasks.rados.rados.0.smithi022.stdout:3654: expect (ObjNum 1130 snap 349 seq_num 1130) 2024-09-21T09:43:49.395 INFO:tasks.rados.rados.0.smithi022.stdout:3656: expect (ObjNum 1102 snap 337 seq_num 1102) 2024-09-21T09:43:49.440 INFO:tasks.rados.rados.0.smithi022.stdout:3657: finishing copy_from to smithi02284960-15 2024-09-21T09:43:49.440 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 15 v 1896 (ObjNum 1142 snap 350 seq_num 1142) dirty exists 2024-09-21T09:43:49.440 INFO:tasks.rados.rados.0.smithi022.stdout:3653: done (5 left) 2024-09-21T09:43:49.440 INFO:tasks.rados.rados.0.smithi022.stdout:3654: done (4 left) 2024-09-21T09:43:49.440 INFO:tasks.rados.rados.0.smithi022.stdout:3655: done (3 left) 2024-09-21T09:43:49.440 INFO:tasks.rados.rados.0.smithi022.stdout:3656: done (2 left) 2024-09-21T09:43:49.440 INFO:tasks.rados.rados.0.smithi022.stdout:3657: done (1 left) 2024-09-21T09:43:49.440 INFO:tasks.rados.rados.0.smithi022.stdout:3658: done (0 left) 2024-09-21T09:43:49.440 INFO:tasks.rados.rados.0.smithi022.stdout:3659: snap_create 2024-09-21T09:43:49.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:43:49 smithi064 ceph-mon[86709]: osdmap e786: 8 total, 8 up, 8 in 2024-09-21T09:43:49.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:43:49 smithi064 ceph-mon[86709]: pgmap v793: 121 pgs: 121 active+clean; 156 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 3.0 MiB/s wr, 22 op/s 2024-09-21T09:43:49.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:43:49 smithi064 ceph-mon[86709]: osdmap e787: 8 total, 8 up, 8 in 2024-09-21T09:43:49.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:43:49 smithi022 ceph-mon[98615]: osdmap e786: 8 total, 8 up, 8 in 2024-09-21T09:43:49.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:43:49 smithi022 ceph-mon[98615]: pgmap v793: 121 pgs: 121 active+clean; 156 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 3.0 MiB/s wr, 22 op/s 2024-09-21T09:43:49.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:43:49 smithi022 ceph-mon[98615]: osdmap e787: 8 total, 8 up, 8 in 2024-09-21T09:43:49.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:43:49 smithi022 ceph-mon[96736]: osdmap e786: 8 total, 8 up, 8 in 2024-09-21T09:43:49.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:43:49 smithi022 ceph-mon[96736]: pgmap v793: 121 pgs: 121 active+clean; 156 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 3.0 MiB/s wr, 22 op/s 2024-09-21T09:43:49.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:43:49 smithi022 ceph-mon[96736]: osdmap e787: 8 total, 8 up, 8 in 2024-09-21T09:43:50.197 INFO:tasks.rados.rados.0.smithi022.stdout:3659: done (0 left) 2024-09-21T09:43:50.197 INFO:tasks.rados.rados.0.smithi022.stdout:3660: rollback oid 9 current snap is 354 2024-09-21T09:43:50.197 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 9 to 346 2024-09-21T09:43:50.197 INFO:tasks.rados.rados.0.smithi022.stdout:3661: copy_from oid 11 from oid 31 current snap is 354 2024-09-21T09:43:50.197 INFO:tasks.rados.rados.0.smithi022.stdout:3662: read oid 13 snap -1 2024-09-21T09:43:50.197 INFO:tasks.rados.rados.0.smithi022.stdout:3662: expect deleted 2024-09-21T09:43:50.197 INFO:tasks.rados.rados.0.smithi022.stdout:3663: snap_create 2024-09-21T09:43:50.202 INFO:tasks.rados.rados.0.smithi022.stdout:3660: finishing rollback tid 0 to smithi02284960-9 2024-09-21T09:43:50.202 INFO:tasks.rados.rados.0.smithi022.stdout:3660: finishing rollback tid 1 to smithi02284960-9 2024-09-21T09:43:50.202 INFO:tasks.rados.rados.0.smithi022.stdout:3660: finishing rollback tid 2 to smithi02284960-9 2024-09-21T09:43:50.203 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 9 v 1988 (ObjNum 1102 snap 337 seq_num 1102) dirty exists 2024-09-21T09:43:50.218 INFO:tasks.rados.rados.0.smithi022.stdout:3661: finishing copy_from racing read to smithi02284960-11 2024-09-21T09:43:50.231 INFO:tasks.rados.rados.0.smithi022.stdout:3661: finishing copy_from to smithi02284960-11 2024-09-21T09:43:50.231 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 11 v 2086 (ObjNum 1140 snap 350 seq_num 1140) dirty exists 2024-09-21T09:43:50.692 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:43:50 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:43:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:43:51.202 INFO:tasks.rados.rados.0.smithi022.stdout:3660: done (3 left) 2024-09-21T09:43:51.202 INFO:tasks.rados.rados.0.smithi022.stdout:3661: done (2 left) 2024-09-21T09:43:51.202 INFO:tasks.rados.rados.0.smithi022.stdout:3662: done (1 left) 2024-09-21T09:43:51.202 INFO:tasks.rados.rados.0.smithi022.stdout:3663: done (0 left) 2024-09-21T09:43:51.202 INFO:tasks.rados.rados.0.smithi022.stdout:3664: rollback oid 20 current snap is 355 2024-09-21T09:43:51.202 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 20 to 354 2024-09-21T09:43:51.202 INFO:tasks.rados.rados.0.smithi022.stdout:3665: rmattr oid 23 current snap is 355 2024-09-21T09:43:51.202 INFO:tasks.rados.rados.0.smithi022.stdout:3665: done (1 left) 2024-09-21T09:43:51.202 INFO:tasks.rados.rados.0.smithi022.stdout:3666: rollback oid 31 current snap is 355 2024-09-21T09:43:51.203 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 31 to 354 2024-09-21T09:43:51.203 INFO:tasks.rados.rados.0.smithi022.stdout:3667: delete oid 39 current snap is 355 2024-09-21T09:43:51.207 INFO:tasks.rados.rados.0.smithi022.stdout:3664: finishing rollback tid 0 to smithi02284960-20 2024-09-21T09:43:51.208 INFO:tasks.rados.rados.0.smithi022.stdout:3664: finishing rollback tid 1 to smithi02284960-20 2024-09-21T09:43:51.208 INFO:tasks.rados.rados.0.smithi022.stdout:3667: done (2 left) 2024-09-21T09:43:51.208 INFO:tasks.rados.rados.0.smithi022.stdout:3668: delete oid 11 current snap is 355 2024-09-21T09:43:51.208 INFO:tasks.rados.rados.0.smithi022.stdout:3664: finishing rollback tid 2 to smithi02284960-20 2024-09-21T09:43:51.208 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 20 v 2115 (ObjNum 1122 snap 342 seq_num 1122) dirty exists 2024-09-21T09:43:51.211 INFO:tasks.rados.rados.0.smithi022.stdout:3666: finishing rollback tid 0 to smithi02284960-31 2024-09-21T09:43:51.211 INFO:tasks.rados.rados.0.smithi022.stdout:3664: done (2 left) 2024-09-21T09:43:51.211 INFO:tasks.rados.rados.0.smithi022.stdout:3668: done (1 left) 2024-09-21T09:43:51.212 INFO:tasks.rados.rados.0.smithi022.stdout:3669: snap_remove snap 345 2024-09-21T09:43:51.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:43:51 smithi022 ceph-mon[98615]: osdmap e788: 8 total, 8 up, 8 in 2024-09-21T09:43:51.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:43:51 smithi022 ceph-mon[98615]: pgmap v796: 121 pgs: 121 active+clean; 156 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 1.4 MiB/s wr, 11 op/s 2024-09-21T09:43:51.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:43:51 smithi022 ceph-mon[96736]: osdmap e788: 8 total, 8 up, 8 in 2024-09-21T09:43:51.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:43:51 smithi022 ceph-mon[96736]: pgmap v796: 121 pgs: 121 active+clean; 156 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 1.4 MiB/s wr, 11 op/s 2024-09-21T09:43:51.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:43:51 smithi064 ceph-mon[86709]: osdmap e788: 8 total, 8 up, 8 in 2024-09-21T09:43:51.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:43:51 smithi064 ceph-mon[86709]: pgmap v796: 121 pgs: 121 active+clean; 156 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 1.4 MiB/s wr, 11 op/s 2024-09-21T09:43:52.208 INFO:tasks.rados.rados.0.smithi022.stdout:3666: finishing rollback tid 1 to smithi02284960-31 2024-09-21T09:43:52.208 INFO:tasks.rados.rados.0.smithi022.stdout:3666: finishing rollback tid 2 to smithi02284960-31 2024-09-21T09:43:52.208 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 31 v 2091 (ObjNum 1140 snap 350 seq_num 1140) dirty exists 2024-09-21T09:43:52.208 INFO:tasks.rados.rados.0.smithi022.stdout:3666: done (1 left) 2024-09-21T09:43:52.208 INFO:tasks.rados.rados.0.smithi022.stdout:3669: done (0 left) 2024-09-21T09:43:52.208 INFO:tasks.rados.rados.0.smithi022.stdout:3670: delete oid 17 current snap is 355 2024-09-21T09:43:52.214 INFO:tasks.rados.rados.0.smithi022.stdout:3670: done (0 left) 2024-09-21T09:43:52.214 INFO:tasks.rados.rados.0.smithi022.stdout:3671: snap_create 2024-09-21T09:43:52.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:43:52 smithi022 ceph-mon[96736]: osdmap e789: 8 total, 8 up, 8 in 2024-09-21T09:43:52.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:43:52 smithi022 ceph-mon[98615]: osdmap e789: 8 total, 8 up, 8 in 2024-09-21T09:43:52.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:43:52 smithi064 ceph-mon[86709]: osdmap e789: 8 total, 8 up, 8 in 2024-09-21T09:43:53.213 INFO:tasks.rados.rados.0.smithi022.stdout:3671: done (0 left) 2024-09-21T09:43:53.213 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 8 current snap is 356 2024-09-21T09:43:53.213 INFO:tasks.rados.rados.0.smithi022.stdout:3672: seq_num 1155 ranges {5963776=147456} 2024-09-21T09:43:53.246 INFO:tasks.rados.rados.0.smithi022.stdout:3672: writing smithi02284960-8 from 5963776 to 6111232 tid 1 2024-09-21T09:43:53.246 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 29 current snap is 356 2024-09-21T09:43:53.246 INFO:tasks.rados.rados.0.smithi022.stdout:3673: seq_num 1156 ranges {2646016=163840} 2024-09-21T09:43:53.262 INFO:tasks.rados.rados.0.smithi022.stdout:3673: writing smithi02284960-29 from 2646016 to 2809856 tid 1 2024-09-21T09:43:53.262 INFO:tasks.rados.rados.0.smithi022.stdout:3674: copy_from oid 12 from oid 17 current snap is 356 2024-09-21T09:43:53.262 INFO:tasks.rados.rados.0.smithi022.stdout:3672: finishing write tid 1 to smithi02284960-8 2024-09-21T09:43:53.262 INFO:tasks.rados.rados.0.smithi022.stdout:3672: finishing write tid 2 to smithi02284960-8 2024-09-21T09:43:53.262 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 10 current snap is 356 2024-09-21T09:43:53.262 INFO:tasks.rados.rados.0.smithi022.stdout:3675: seq_num 1157 ranges {0=180224} 2024-09-21T09:43:53.263 INFO:tasks.rados.rados.0.smithi022.stdout:3675: writing smithi02284960-10 from 0 to 180224 tid 1 2024-09-21T09:43:53.263 INFO:tasks.rados.rados.0.smithi022.stdout:3676: snap_remove snap 352 2024-09-21T09:43:53.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:43:53 smithi022 ceph-mon[98615]: osdmap e790: 8 total, 8 up, 8 in 2024-09-21T09:43:53.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:43:53 smithi022 ceph-mon[98615]: pgmap v799: 121 pgs: 121 active+clean; 160 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 2.0 MiB/s wr, 9 op/s 2024-09-21T09:43:53.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:43:53 smithi022 ceph-mon[96736]: osdmap e790: 8 total, 8 up, 8 in 2024-09-21T09:43:53.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:43:53 smithi022 ceph-mon[96736]: pgmap v799: 121 pgs: 121 active+clean; 160 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 2.0 MiB/s wr, 9 op/s 2024-09-21T09:43:53.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:43:53 smithi064 ceph-mon[86709]: osdmap e790: 8 total, 8 up, 8 in 2024-09-21T09:43:53.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:43:53 smithi064 ceph-mon[86709]: pgmap v799: 121 pgs: 121 active+clean; 160 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 2.0 MiB/s wr, 9 op/s 2024-09-21T09:43:54.100 INFO:tasks.rados.rados.0.smithi022.stdout:3676: done (4 left) 2024-09-21T09:43:54.100 INFO:tasks.rados.rados.0.smithi022.stdout:3677: rollback oid 26 current snap is 356 2024-09-21T09:43:54.100 INFO:tasks.rados.rados.0.smithi022.stdout:3672: finishing write tid 3 to smithi02284960-8 2024-09-21T09:43:54.100 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 8 v 1990 (ObjNum 1155 snap 356 seq_num 1155) dirty exists 2024-09-21T09:43:54.101 INFO:tasks.rados.rados.0.smithi022.stdout:3672: left oid 8 (ObjNum 1155 snap 356 seq_num 1155) 2024-09-21T09:43:54.101 INFO:tasks.rados.rados.0.smithi022.stdout:3674: finishing copy_from to smithi02284960-12 2024-09-21T09:43:54.101 INFO:tasks.rados.rados.0.smithi022.stdout:3674: got expected ENOENT (src dne) 2024-09-21T09:43:54.101 INFO:tasks.rados.rados.0.smithi022.stdout:3674: finishing copy_from racing read to smithi02284960-12 2024-09-21T09:43:54.101 INFO:tasks.rados.rados.0.smithi022.stdout:3673: finishing write tid 1 to smithi02284960-29 2024-09-21T09:43:54.101 INFO:tasks.rados.rados.0.smithi022.stdout:3673: finishing write tid 2 to smithi02284960-29 2024-09-21T09:43:54.101 INFO:tasks.rados.rados.0.smithi022.stdout:3673: finishing write tid 3 to smithi02284960-29 2024-09-21T09:43:54.101 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 29 v 1843 (ObjNum 1156 snap 356 seq_num 1156) dirty exists 2024-09-21T09:43:54.101 INFO:tasks.rados.rados.0.smithi022.stdout:3673: left oid 29 (ObjNum 1156 snap 356 seq_num 1156) 2024-09-21T09:43:54.101 INFO:tasks.rados.rados.0.smithi022.stdout:3675: finishing write tid 1 to smithi02284960-10 2024-09-21T09:43:54.101 INFO:tasks.rados.rados.0.smithi022.stdout:3675: finishing write tid 2 to smithi02284960-10 2024-09-21T09:43:54.101 INFO:tasks.rados.rados.0.smithi022.stdout:3675: finishing write tid 3 to smithi02284960-10 2024-09-21T09:43:54.101 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 10 v 1914 (ObjNum 1157 snap 356 seq_num 1157) dirty exists 2024-09-21T09:43:54.101 INFO:tasks.rados.rados.0.smithi022.stdout:3675: left oid 10 (ObjNum 1157 snap 356 seq_num 1157) 2024-09-21T09:43:54.102 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 26 to 346 2024-09-21T09:43:54.102 INFO:tasks.rados.rados.0.smithi022.stdout:3672: done (4 left) 2024-09-21T09:43:54.102 INFO:tasks.rados.rados.0.smithi022.stdout:3673: done (3 left) 2024-09-21T09:43:54.102 INFO:tasks.rados.rados.0.smithi022.stdout:3674: done (2 left) 2024-09-21T09:43:54.102 INFO:tasks.rados.rados.0.smithi022.stdout:3675: done (1 left) 2024-09-21T09:43:54.102 INFO:tasks.rados.rados.0.smithi022.stdout:3678: snap_remove snap 354 2024-09-21T09:43:54.459 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:43:54 smithi022 ceph-mon[98615]: osdmap e791: 8 total, 8 up, 8 in 2024-09-21T09:43:54.459 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:43:54 smithi022 ceph-mon[98615]: osdmap e792: 8 total, 8 up, 8 in 2024-09-21T09:43:54.460 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:43:54 smithi022 ceph-mon[96736]: osdmap e791: 8 total, 8 up, 8 in 2024-09-21T09:43:54.460 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:43:54 smithi022 ceph-mon[96736]: osdmap e792: 8 total, 8 up, 8 in 2024-09-21T09:43:54.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:43:54 smithi064 ceph-mon[86709]: osdmap e791: 8 total, 8 up, 8 in 2024-09-21T09:43:54.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:43:54 smithi064 ceph-mon[86709]: osdmap e792: 8 total, 8 up, 8 in 2024-09-21T09:43:55.102 INFO:tasks.rados.rados.0.smithi022.stdout:3678: done (1 left) 2024-09-21T09:43:55.102 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 4 current snap is 356 2024-09-21T09:43:55.102 INFO:tasks.rados.rados.0.smithi022.stdout:3677: finishing rollback tid 1 to smithi02284960-26 2024-09-21T09:43:55.102 INFO:tasks.rados.rados.0.smithi022.stdout:3677: finishing rollback tid 2 to smithi02284960-26 2024-09-21T09:43:55.102 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 26 v 1977 (ObjNum 1036 snap 315 seq_num 1036) dirty exists 2024-09-21T09:43:55.102 INFO:tasks.rados.rados.0.smithi022.stdout:3679: seq_num 1158 ranges {0=196608} 2024-09-21T09:43:55.104 INFO:tasks.rados.rados.0.smithi022.stdout:3679: writing smithi02284960-4 from 0 to 196608 tid 1 2024-09-21T09:43:55.104 INFO:tasks.rados.rados.0.smithi022.stdout:3677: done (1 left) 2024-09-21T09:43:55.104 INFO:tasks.rados.rados.0.smithi022.stdout:3680: read oid 44 snap -1 2024-09-21T09:43:55.104 INFO:tasks.rados.rados.0.smithi022.stdout:3680: expect (ObjNum 1142 snap 350 seq_num 1142) 2024-09-21T09:43:55.104 INFO:tasks.rados.rados.0.smithi022.stdout:3681: read oid 6 snap -1 2024-09-21T09:43:55.104 INFO:tasks.rados.rados.0.smithi022.stdout:3681: expect (ObjNum 1062 snap 319 seq_num 1062) 2024-09-21T09:43:55.104 INFO:tasks.rados.rados.0.smithi022.stdout:3682: read oid 34 snap 355 2024-09-21T09:43:55.104 INFO:tasks.rados.rados.0.smithi022.stdout:3682: expect deleted 2024-09-21T09:43:55.105 INFO:tasks.rados.rados.0.smithi022.stdout:3683: delete oid 11 current snap is 356 2024-09-21T09:43:55.114 INFO:tasks.rados.rados.0.smithi022.stdout:3679: finishing write tid 1 to smithi02284960-4 2024-09-21T09:43:55.115 INFO:tasks.rados.rados.0.smithi022.stdout:3679: finishing write tid 2 to smithi02284960-4 2024-09-21T09:43:55.116 INFO:tasks.rados.rados.0.smithi022.stdout:3681: expect (ObjNum 1062 snap 319 seq_num 1062) 2024-09-21T09:43:55.145 INFO:tasks.rados.rados.0.smithi022.stdout:3679: finishing write tid 3 to smithi02284960-4 2024-09-21T09:43:55.145 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 4 v 1992 (ObjNum 1158 snap 356 seq_num 1158) dirty exists 2024-09-21T09:43:55.145 INFO:tasks.rados.rados.0.smithi022.stdout:3679: left oid 4 (ObjNum 1158 snap 356 seq_num 1158) 2024-09-21T09:43:55.146 INFO:tasks.rados.rados.0.smithi022.stdout:3679: done (4 left) 2024-09-21T09:43:55.146 INFO:tasks.rados.rados.0.smithi022.stdout:3681: done (3 left) 2024-09-21T09:43:55.146 INFO:tasks.rados.rados.0.smithi022.stdout:3682: done (2 left) 2024-09-21T09:43:55.146 INFO:tasks.rados.rados.0.smithi022.stdout:3683: done (1 left) 2024-09-21T09:43:55.146 INFO:tasks.rados.rados.0.smithi022.stdout:3684: copy_from oid 1 from oid 41 current snap is 356 2024-09-21T09:43:55.146 INFO:tasks.rados.rados.0.smithi022.stdout:3680: expect (ObjNum 1142 snap 350 seq_num 1142) 2024-09-21T09:43:55.473 INFO:tasks.rados.rados.0.smithi022.stdout:3684: finishing copy_from racing read to smithi02284960-1 2024-09-21T09:43:55.473 INFO:tasks.rados.rados.0.smithi022.stdout:3684: finishing copy_from to smithi02284960-1 2024-09-21T09:43:55.473 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 1 v 1910 (ObjNum 1120 snap 341 seq_num 1120) dirty exists 2024-09-21T09:43:55.473 INFO:tasks.rados.rados.0.smithi022.stdout:3680: done (1 left) 2024-09-21T09:43:55.473 INFO:tasks.rados.rados.0.smithi022.stdout:3684: done (0 left) 2024-09-21T09:43:55.473 INFO:tasks.rados.rados.0.smithi022.stdout:3685: snap_remove snap 353 2024-09-21T09:43:56.058 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:43:55 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:43:55] "GET /metrics HTTP/1.1" 200 36351 "" "Prometheus/2.51.0" 2024-09-21T09:43:56.217 INFO:tasks.rados.rados.0.smithi022.stdout:3685: done (0 left) 2024-09-21T09:43:56.217 INFO:tasks.rados.rados.0.smithi022.stdout:3686: snap_remove snap 355 2024-09-21T09:43:56.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:43:56 smithi064 ceph-mon[86709]: pgmap v802: 121 pgs: 121 active+clean; 160 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2.5 MiB/s rd, 1.7 MiB/s wr, 7 op/s 2024-09-21T09:43:56.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:43:56 smithi064 ceph-mon[86709]: osdmap e793: 8 total, 8 up, 8 in 2024-09-21T09:43:56.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:43:56 smithi022 ceph-mon[98615]: pgmap v802: 121 pgs: 121 active+clean; 160 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2.5 MiB/s rd, 1.7 MiB/s wr, 7 op/s 2024-09-21T09:43:56.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:43:56 smithi022 ceph-mon[98615]: osdmap e793: 8 total, 8 up, 8 in 2024-09-21T09:43:56.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:43:56 smithi022 ceph-mon[96736]: pgmap v802: 121 pgs: 121 active+clean; 160 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2.5 MiB/s rd, 1.7 MiB/s wr, 7 op/s 2024-09-21T09:43:56.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:43:56 smithi022 ceph-mon[96736]: osdmap e793: 8 total, 8 up, 8 in 2024-09-21T09:43:57.223 INFO:tasks.rados.rados.0.smithi022.stdout:3686: done (0 left) 2024-09-21T09:43:57.223 INFO:tasks.rados.rados.0.smithi022.stdout:3687: rollback oid 42 current snap is 356 2024-09-21T09:43:57.223 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 42 to 346 2024-09-21T09:43:57.223 INFO:tasks.rados.rados.0.smithi022.stdout:3688: copy_from oid 23 from oid 29 current snap is 356 2024-09-21T09:43:57.223 INFO:tasks.rados.rados.0.smithi022.stdout:3689: read oid 17 snap 346 2024-09-21T09:43:57.224 INFO:tasks.rados.rados.0.smithi022.stdout:3689: expect (ObjNum 1105 snap 338 seq_num 1105) 2024-09-21T09:43:57.224 INFO:tasks.rados.rados.0.smithi022.stdout:3690: copy_from oid 9 from oid 6 current snap is 356 2024-09-21T09:43:57.224 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 31 current snap is 356 2024-09-21T09:43:57.224 INFO:tasks.rados.rados.0.smithi022.stdout:3691: seq_num 1160 ranges {3751936=229376} 2024-09-21T09:43:57.246 INFO:tasks.rados.rados.0.smithi022.stdout:3691: writing smithi02284960-31 from 3751936 to 3981312 tid 1 2024-09-21T09:43:57.246 INFO:tasks.rados.rados.0.smithi022.stdout:3692: snap_create 2024-09-21T09:43:57.246 INFO:tasks.rados.rados.0.smithi022.stdout:3687: finishing rollback tid 0 to smithi02284960-42 2024-09-21T09:43:57.246 INFO:tasks.rados.rados.0.smithi022.stdout:3689: expect (ObjNum 1105 snap 338 seq_num 1105) 2024-09-21T09:43:57.298 INFO:tasks.rados.rados.0.smithi022.stdout:3687: finishing rollback tid 1 to smithi02284960-42 2024-09-21T09:43:57.298 INFO:tasks.rados.rados.0.smithi022.stdout:3687: finishing rollback tid 2 to smithi02284960-42 2024-09-21T09:43:57.298 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 42 v 1980 (ObjNum 1062 snap 319 seq_num 1062) dirty exists 2024-09-21T09:43:57.298 INFO:tasks.rados.rados.0.smithi022.stdout:3690: finishing copy_from racing read to smithi02284960-9 2024-09-21T09:43:57.298 INFO:tasks.rados.rados.0.smithi022.stdout:3690: finishing copy_from to smithi02284960-9 2024-09-21T09:43:57.298 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 9 v 1885 (ObjNum 1062 snap 319 seq_num 1062) dirty exists 2024-09-21T09:43:57.298 INFO:tasks.rados.rados.0.smithi022.stdout:3688: finishing copy_from racing read to smithi02284960-23 2024-09-21T09:43:57.299 INFO:tasks.rados.rados.0.smithi022.stdout:3688: finishing copy_from to smithi02284960-23 2024-09-21T09:43:57.299 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 23 v 1843 (ObjNum 1156 snap 356 seq_num 1156) dirty exists 2024-09-21T09:43:57.299 INFO:tasks.rados.rados.0.smithi022.stdout:3691: finishing write tid 1 to smithi02284960-31 2024-09-21T09:43:57.299 INFO:tasks.rados.rados.0.smithi022.stdout:3691: finishing write tid 2 to smithi02284960-31 2024-09-21T09:43:57.299 INFO:tasks.rados.rados.0.smithi022.stdout:3691: finishing write tid 3 to smithi02284960-31 2024-09-21T09:43:57.299 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 31 v 2105 (ObjNum 1160 snap 356 seq_num 1160) dirty exists 2024-09-21T09:43:57.299 INFO:tasks.rados.rados.0.smithi022.stdout:3691: left oid 31 (ObjNum 1160 snap 356 seq_num 1160) 2024-09-21T09:43:57.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:43:57 smithi022 ceph-mon[96736]: osdmap e794: 8 total, 8 up, 8 in 2024-09-21T09:43:57.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:43:57 smithi022 ceph-mon[96736]: pgmap v805: 121 pgs: 121 active+clean; 166 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 5.9 MiB/s rd, 1.5 MiB/s wr, 8 op/s 2024-09-21T09:43:57.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:43:57 smithi022 ceph-mon[98615]: osdmap e794: 8 total, 8 up, 8 in 2024-09-21T09:43:57.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:43:57 smithi022 ceph-mon[98615]: pgmap v805: 121 pgs: 121 active+clean; 166 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 5.9 MiB/s rd, 1.5 MiB/s wr, 8 op/s 2024-09-21T09:43:57.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:43:57 smithi064 ceph-mon[86709]: osdmap e794: 8 total, 8 up, 8 in 2024-09-21T09:43:57.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:43:57 smithi064 ceph-mon[86709]: pgmap v805: 121 pgs: 121 active+clean; 166 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 5.9 MiB/s rd, 1.5 MiB/s wr, 8 op/s 2024-09-21T09:43:58.228 INFO:tasks.rados.rados.0.smithi022.stdout:3687: done (5 left) 2024-09-21T09:43:58.228 INFO:tasks.rados.rados.0.smithi022.stdout:3688: done (4 left) 2024-09-21T09:43:58.228 INFO:tasks.rados.rados.0.smithi022.stdout:3689: done (3 left) 2024-09-21T09:43:58.228 INFO:tasks.rados.rados.0.smithi022.stdout:3690: done (2 left) 2024-09-21T09:43:58.228 INFO:tasks.rados.rados.0.smithi022.stdout:3691: done (1 left) 2024-09-21T09:43:58.228 INFO:tasks.rados.rados.0.smithi022.stdout:3692: done (0 left) 2024-09-21T09:43:58.228 INFO:tasks.rados.rados.0.smithi022.stdout:3693: read oid 10 snap -1 2024-09-21T09:43:58.228 INFO:tasks.rados.rados.0.smithi022.stdout:3693: expect (ObjNum 1157 snap 356 seq_num 1157) 2024-09-21T09:43:58.228 INFO:tasks.rados.rados.0.smithi022.stdout:3694: snap_create 2024-09-21T09:43:58.230 INFO:tasks.rados.rados.0.smithi022.stdout:3693: expect (ObjNum 1157 snap 356 seq_num 1157) 2024-09-21T09:43:58.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:43:58 smithi022 ceph-mon[98615]: osdmap e795: 8 total, 8 up, 8 in 2024-09-21T09:43:58.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:43:58 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:43:58.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:43:58 smithi022 ceph-mon[96736]: osdmap e795: 8 total, 8 up, 8 in 2024-09-21T09:43:58.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:43:58 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:43:58.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:43:58 smithi064 ceph-mon[86709]: osdmap e795: 8 total, 8 up, 8 in 2024-09-21T09:43:58.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:43:58 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:43:59.101 INFO:tasks.rados.rados.0.smithi022.stdout:3693: done (1 left) 2024-09-21T09:43:59.102 INFO:tasks.rados.rados.0.smithi022.stdout:3694: done (0 left) 2024-09-21T09:43:59.102 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 42 current snap is 358 2024-09-21T09:43:59.102 INFO:tasks.rados.rados.0.smithi022.stdout:3695: seq_num 1161 ranges {991232=245760} 2024-09-21T09:43:59.108 INFO:tasks.rados.rados.0.smithi022.stdout:3695: writing smithi02284960-42 from 991232 to 1236992 tid 1 2024-09-21T09:43:59.108 INFO:tasks.rados.rados.0.smithi022.stdout:3696: setattr oid 12 current snap is 358 2024-09-21T09:43:59.109 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 25 current snap is 358 2024-09-21T09:43:59.109 INFO:tasks.rados.rados.0.smithi022.stdout:3697: seq_num 1162 ranges {0=270336} 2024-09-21T09:43:59.111 INFO:tasks.rados.rados.0.smithi022.stdout:3697: writing smithi02284960-25 from 0 to 270336 tid 1 2024-09-21T09:43:59.111 INFO:tasks.rados.rados.0.smithi022.stdout:3698: read oid 23 snap -1 2024-09-21T09:43:59.111 INFO:tasks.rados.rados.0.smithi022.stdout:3698: expect (ObjNum 1156 snap 356 seq_num 1156) 2024-09-21T09:43:59.111 INFO:tasks.rados.rados.0.smithi022.stdout:3699: delete oid 17 current snap is 358 2024-09-21T09:43:59.114 INFO:tasks.rados.rados.0.smithi022.stdout:3699: done (4 left) 2024-09-21T09:43:59.114 INFO:tasks.rados.rados.0.smithi022.stdout:3700: setattr oid 30 current snap is 358 2024-09-21T09:43:59.115 INFO:tasks.rados.rados.0.smithi022.stdout:3701: delete oid 43 current snap is 358 2024-09-21T09:43:59.118 INFO:tasks.rados.rados.0.smithi022.stdout:3695: finishing write tid 1 to smithi02284960-42 2024-09-21T09:43:59.119 INFO:tasks.rados.rados.0.smithi022.stdout:3695: finishing write tid 2 to smithi02284960-42 2024-09-21T09:43:59.119 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 12 v 2107 (ObjNum 1103 snap 337 seq_num 1103) dirty exists 2024-09-21T09:43:59.120 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 30 v 1974 (ObjNum 1102 snap 337 seq_num 1102) dirty exists 2024-09-21T09:43:59.121 INFO:tasks.rados.rados.0.smithi022.stdout:3695: finishing write tid 3 to smithi02284960-42 2024-09-21T09:43:59.121 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 42 v 1982 (ObjNum 1161 snap 358 seq_num 1161) dirty exists 2024-09-21T09:43:59.121 INFO:tasks.rados.rados.0.smithi022.stdout:3695: left oid 42 (ObjNum 1161 snap 358 seq_num 1161) 2024-09-21T09:43:59.121 INFO:tasks.rados.rados.0.smithi022.stdout:3697: finishing write tid 1 to smithi02284960-25 2024-09-21T09:43:59.121 INFO:tasks.rados.rados.0.smithi022.stdout:3697: finishing write tid 2 to smithi02284960-25 2024-09-21T09:43:59.123 INFO:tasks.rados.rados.0.smithi022.stdout:3697: finishing write tid 3 to smithi02284960-25 2024-09-21T09:43:59.123 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 25 v 1916 (ObjNum 1162 snap 358 seq_num 1162) dirty exists 2024-09-21T09:43:59.123 INFO:tasks.rados.rados.0.smithi022.stdout:3697: left oid 25 (ObjNum 1162 snap 358 seq_num 1162) 2024-09-21T09:43:59.123 INFO:tasks.rados.rados.0.smithi022.stdout:3695: done (5 left) 2024-09-21T09:43:59.123 INFO:tasks.rados.rados.0.smithi022.stdout:3696: done (4 left) 2024-09-21T09:43:59.123 INFO:tasks.rados.rados.0.smithi022.stdout:3697: done (3 left) 2024-09-21T09:43:59.123 INFO:tasks.rados.rados.0.smithi022.stdout:3700: done (2 left) 2024-09-21T09:43:59.124 INFO:tasks.rados.rados.0.smithi022.stdout:3701: done (1 left) 2024-09-21T09:43:59.124 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 29 current snap is 358 2024-09-21T09:43:59.124 INFO:tasks.rados.rados.0.smithi022.stdout:3702: seq_num 1165 ranges {2809856=319488} 2024-09-21T09:43:59.140 INFO:tasks.rados.rados.0.smithi022.stdout:3702: writing smithi02284960-29 from 2809856 to 3129344 tid 1 2024-09-21T09:43:59.140 INFO:tasks.rados.rados.0.smithi022.stdout:3703: snap_remove snap 356 2024-09-21T09:43:59.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:43:59 smithi022 ceph-mon[98615]: osdmap e796: 8 total, 8 up, 8 in 2024-09-21T09:43:59.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:43:59 smithi022 ceph-mon[98615]: pgmap v808: 121 pgs: 121 active+clean; 169 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 5.9 MiB/s rd, 2.2 MiB/s wr, 9 op/s 2024-09-21T09:43:59.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:43:59 smithi022 ceph-mon[98615]: osdmap e797: 8 total, 8 up, 8 in 2024-09-21T09:43:59.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:43:59 smithi022 ceph-mon[96736]: osdmap e796: 8 total, 8 up, 8 in 2024-09-21T09:43:59.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:43:59 smithi022 ceph-mon[96736]: pgmap v808: 121 pgs: 121 active+clean; 169 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 5.9 MiB/s rd, 2.2 MiB/s wr, 9 op/s 2024-09-21T09:43:59.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:43:59 smithi022 ceph-mon[96736]: osdmap e797: 8 total, 8 up, 8 in 2024-09-21T09:43:59.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:43:59 smithi064 ceph-mon[86709]: osdmap e796: 8 total, 8 up, 8 in 2024-09-21T09:43:59.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:43:59 smithi064 ceph-mon[86709]: pgmap v808: 121 pgs: 121 active+clean; 169 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 5.9 MiB/s rd, 2.2 MiB/s wr, 9 op/s 2024-09-21T09:43:59.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:43:59 smithi064 ceph-mon[86709]: osdmap e797: 8 total, 8 up, 8 in 2024-09-21T09:44:00.104 INFO:tasks.rados.rados.0.smithi022.stdout:3703: done (2 left) 2024-09-21T09:44:00.104 INFO:tasks.rados.rados.0.smithi022.stdout:3704: copy_from oid 36 from oid 28 current snap is 358 2024-09-21T09:44:00.104 INFO:tasks.rados.rados.0.smithi022.stdout:3698: expect (ObjNum 1156 snap 356 seq_num 1156) 2024-09-21T09:44:00.235 INFO:tasks.rados.rados.0.smithi022.stdout:3702: finishing write tid 1 to smithi02284960-29 2024-09-21T09:44:00.236 INFO:tasks.rados.rados.0.smithi022.stdout:3702: finishing write tid 2 to smithi02284960-29 2024-09-21T09:44:00.236 INFO:tasks.rados.rados.0.smithi022.stdout:3702: finishing write tid 3 to smithi02284960-29 2024-09-21T09:44:00.236 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 29 v 1845 (ObjNum 1165 snap 358 seq_num 1165) dirty exists 2024-09-21T09:44:00.236 INFO:tasks.rados.rados.0.smithi022.stdout:3702: left oid 29 (ObjNum 1165 snap 358 seq_num 1165) 2024-09-21T09:44:00.236 INFO:tasks.rados.rados.0.smithi022.stdout:3704: finishing copy_from racing read to smithi02284960-36 2024-09-21T09:44:00.236 INFO:tasks.rados.rados.0.smithi022.stdout:3704: finishing copy_from to smithi02284960-36 2024-09-21T09:44:00.236 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 36 v 1810 (ObjNum 1117 snap 341 seq_num 1117) dirty exists 2024-09-21T09:44:00.236 INFO:tasks.rados.rados.0.smithi022.stdout:3698: done (2 left) 2024-09-21T09:44:00.236 INFO:tasks.rados.rados.0.smithi022.stdout:3702: done (1 left) 2024-09-21T09:44:00.236 INFO:tasks.rados.rados.0.smithi022.stdout:3704: done (0 left) 2024-09-21T09:44:00.236 INFO:tasks.rados.rados.0.smithi022.stdout:3705: rollback oid 35 current snap is 358 2024-09-21T09:44:00.236 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 35 to 357 2024-09-21T09:44:00.237 INFO:tasks.rados.rados.0.smithi022.stdout:3706: snap_create 2024-09-21T09:44:00.239 INFO:tasks.rados.rados.0.smithi022.stdout:3705: finishing rollback tid 0 to smithi02284960-35 2024-09-21T09:44:00.239 INFO:tasks.rados.rados.0.smithi022.stdout:3705: finishing rollback tid 1 to smithi02284960-35 2024-09-21T09:44:00.239 INFO:tasks.rados.rados.0.smithi022.stdout:3705: finishing rollback tid 2 to smithi02284960-35 2024-09-21T09:44:00.239 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 35 v 1818 (ObjNum 1137 snap 349 seq_num 1137) dirty exists 2024-09-21T09:44:00.691 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:44:00 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:44:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:44:01.108 INFO:tasks.rados.rados.0.smithi022.stdout:3705: done (1 left) 2024-09-21T09:44:01.108 INFO:tasks.rados.rados.0.smithi022.stdout:3706: done (0 left) 2024-09-21T09:44:01.109 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 4 current snap is 359 2024-09-21T09:44:01.109 INFO:tasks.rados.rados.0.smithi022.stdout:3707: seq_num 1166 ranges {196608=335872} 2024-09-21T09:44:01.111 INFO:tasks.rados.rados.0.smithi022.stdout:3707: writing smithi02284960-4 from 196608 to 532480 tid 1 2024-09-21T09:44:01.111 INFO:tasks.rados.rados.0.smithi022.stdout:3708: snap_remove snap 346 2024-09-21T09:44:01.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:44:01 smithi064 ceph-mon[86709]: osdmap e798: 8 total, 8 up, 8 in 2024-09-21T09:44:01.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:44:01 smithi022 ceph-mon[98615]: osdmap e798: 8 total, 8 up, 8 in 2024-09-21T09:44:01.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:44:01 smithi022 ceph-mon[96736]: osdmap e798: 8 total, 8 up, 8 in 2024-09-21T09:44:02.116 INFO:tasks.rados.rados.0.smithi022.stdout:3708: done (1 left) 2024-09-21T09:44:02.116 INFO:tasks.rados.rados.0.smithi022.stdout:3709: snap_remove snap 358 2024-09-21T09:44:02.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:44:02 smithi064 ceph-mon[86709]: pgmap v811: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 166 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 133 KiB/s rd, 1.0 MiB/s wr, 3 op/s 2024-09-21T09:44:02.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:44:02 smithi064 ceph-mon[86709]: osdmap e799: 8 total, 8 up, 8 in 2024-09-21T09:44:02.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:44:02 smithi022 ceph-mon[98615]: pgmap v811: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 166 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 133 KiB/s rd, 1.0 MiB/s wr, 3 op/s 2024-09-21T09:44:02.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:44:02 smithi022 ceph-mon[98615]: osdmap e799: 8 total, 8 up, 8 in 2024-09-21T09:44:02.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:44:02 smithi022 ceph-mon[96736]: pgmap v811: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 166 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 133 KiB/s rd, 1.0 MiB/s wr, 3 op/s 2024-09-21T09:44:02.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:44:02 smithi022 ceph-mon[96736]: osdmap e799: 8 total, 8 up, 8 in 2024-09-21T09:44:03.119 INFO:tasks.rados.rados.0.smithi022.stdout:3709: done (1 left) 2024-09-21T09:44:03.119 INFO:tasks.rados.rados.0.smithi022.stdout:3710: delete oid 26 current snap is 359 2024-09-21T09:44:03.119 INFO:tasks.rados.rados.0.smithi022.stdout:3707: finishing write tid 1 to smithi02284960-4 2024-09-21T09:44:03.120 INFO:tasks.rados.rados.0.smithi022.stdout:3707: finishing write tid 2 to smithi02284960-4 2024-09-21T09:44:03.120 INFO:tasks.rados.rados.0.smithi022.stdout:3707: finishing write tid 3 to smithi02284960-4 2024-09-21T09:44:03.120 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 4 v 1994 (ObjNum 1166 snap 359 seq_num 1166) dirty exists 2024-09-21T09:44:03.120 INFO:tasks.rados.rados.0.smithi022.stdout:3707: left oid 4 (ObjNum 1166 snap 359 seq_num 1166) 2024-09-21T09:44:03.122 INFO:tasks.rados.rados.0.smithi022.stdout:3707: done (1 left) 2024-09-21T09:44:03.122 INFO:tasks.rados.rados.0.smithi022.stdout:3710: done (0 left) 2024-09-21T09:44:03.122 INFO:tasks.rados.rados.0.smithi022.stdout:3711: snap_remove snap 357 2024-09-21T09:44:03.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:44:03 smithi064 ceph-mon[86709]: osdmap e800: 8 total, 8 up, 8 in 2024-09-21T09:44:03.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:44:03 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:44:03.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:44:03 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:44:03.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:44:03 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:44:03.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:44:03 smithi022 ceph-mon[98615]: osdmap e800: 8 total, 8 up, 8 in 2024-09-21T09:44:03.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:44:03 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:44:03.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:44:03 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:44:03.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:44:03 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:44:03.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:44:03 smithi022 ceph-mon[96736]: osdmap e800: 8 total, 8 up, 8 in 2024-09-21T09:44:03.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:44:03 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:44:03.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:44:03 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:44:03.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:44:03 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:44:04.104 INFO:tasks.rados.rados.0.smithi022.stdout:3711: done (0 left) 2024-09-21T09:44:04.104 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 50 current snap is 359 2024-09-21T09:44:04.104 INFO:tasks.rados.rados.0.smithi022.stdout:3712: seq_num 1168 ranges {0=368640} 2024-09-21T09:44:04.107 INFO:tasks.rados.rados.0.smithi022.stdout:3712: writing smithi02284960-50 from 0 to 368640 tid 1 2024-09-21T09:44:04.107 INFO:tasks.rados.rados.0.smithi022.stdout:3713: delete oid 42 current snap is 359 2024-09-21T09:44:04.111 INFO:tasks.rados.rados.0.smithi022.stdout:3712: finishing write tid 1 to smithi02284960-50 2024-09-21T09:44:04.112 INFO:tasks.rados.rados.0.smithi022.stdout:3712: finishing write tid 2 to smithi02284960-50 2024-09-21T09:44:04.116 INFO:tasks.rados.rados.0.smithi022.stdout:3712: finishing write tid 3 to smithi02284960-50 2024-09-21T09:44:04.116 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 50 v 1892 (ObjNum 1168 snap 359 seq_num 1168) dirty exists 2024-09-21T09:44:04.116 INFO:tasks.rados.rados.0.smithi022.stdout:3712: left oid 50 (ObjNum 1168 snap 359 seq_num 1168) 2024-09-21T09:44:04.118 INFO:tasks.rados.rados.0.smithi022.stdout:3712: done (1 left) 2024-09-21T09:44:04.118 INFO:tasks.rados.rados.0.smithi022.stdout:3713: done (0 left) 2024-09-21T09:44:04.118 INFO:tasks.rados.rados.0.smithi022.stdout:3714: snap_create 2024-09-21T09:44:04.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:44:04 smithi064 ceph-mon[86709]: pgmap v814: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 156 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 1.4 MiB/s rd, 424 KiB/s wr, 15 op/s 2024-09-21T09:44:04.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:44:04 smithi064 ceph-mon[86709]: osdmap e801: 8 total, 8 up, 8 in 2024-09-21T09:44:04.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:44:04 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:44:04.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:44:04 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:44:04.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:44:04 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:44:04.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:44:04 smithi064 ceph-mon[86709]: osdmap e802: 8 total, 8 up, 8 in 2024-09-21T09:44:04.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:44:04 smithi022 ceph-mon[96736]: pgmap v814: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 156 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 1.4 MiB/s rd, 424 KiB/s wr, 15 op/s 2024-09-21T09:44:04.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:44:04 smithi022 ceph-mon[96736]: osdmap e801: 8 total, 8 up, 8 in 2024-09-21T09:44:04.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:44:04 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:44:04.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:44:04 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:44:04.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:44:04 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:44:04.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:44:04 smithi022 ceph-mon[96736]: osdmap e802: 8 total, 8 up, 8 in 2024-09-21T09:44:04.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:44:04 smithi022 ceph-mon[98615]: pgmap v814: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 156 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 1.4 MiB/s rd, 424 KiB/s wr, 15 op/s 2024-09-21T09:44:04.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:44:04 smithi022 ceph-mon[98615]: osdmap e801: 8 total, 8 up, 8 in 2024-09-21T09:44:04.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:44:04 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:44:04.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:44:04 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:44:04.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:44:04 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:44:04.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:44:04 smithi022 ceph-mon[98615]: osdmap e802: 8 total, 8 up, 8 in 2024-09-21T09:44:05.127 INFO:tasks.rados.rados.0.smithi022.stdout:3714: done (0 left) 2024-09-21T09:44:05.127 INFO:tasks.rados.rados.0.smithi022.stdout:3715: snap_create 2024-09-21T09:44:06.058 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:44:05 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:44:05] "GET /metrics HTTP/1.1" 200 36358 "" "Prometheus/2.51.0" 2024-09-21T09:44:06.128 INFO:tasks.rados.rados.0.smithi022.stdout:3715: done (0 left) 2024-09-21T09:44:06.129 INFO:tasks.rados.rados.0.smithi022.stdout:3716: rollback oid 11 current snap is 361 2024-09-21T09:44:06.129 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 11 to 360 2024-09-21T09:44:06.129 INFO:tasks.rados.rados.0.smithi022.stdout:3717: rollback oid 32 current snap is 361 2024-09-21T09:44:06.129 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 32 to 359 2024-09-21T09:44:06.129 INFO:tasks.rados.rados.0.smithi022.stdout:3718: delete oid 15 current snap is 361 2024-09-21T09:44:06.131 INFO:tasks.rados.rados.0.smithi022.stdout:3717: finishing rollback tid 1 to smithi02284960-32 2024-09-21T09:44:06.131 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 32 v 0 (ObjNum 129 snap 0 seq_num 3570349776) dirty dne 2024-09-21T09:44:06.131 INFO:tasks.rados.rados.0.smithi022.stdout:3716: finishing rollback tid 1 to smithi02284960-11 2024-09-21T09:44:06.131 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 11 v 0 (ObjNum 97 snap 0 seq_num 3569539968) dirty dne 2024-09-21T09:44:06.138 INFO:tasks.rados.rados.0.smithi022.stdout:3716: done (2 left) 2024-09-21T09:44:06.138 INFO:tasks.rados.rados.0.smithi022.stdout:3717: done (1 left) 2024-09-21T09:44:06.138 INFO:tasks.rados.rados.0.smithi022.stdout:3718: done (0 left) 2024-09-21T09:44:06.138 INFO:tasks.rados.rados.0.smithi022.stdout:3719: rollback oid 45 current snap is 361 2024-09-21T09:44:06.138 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 45 to 359 2024-09-21T09:44:06.138 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 40 current snap is 361 2024-09-21T09:44:06.138 INFO:tasks.rados.rados.0.smithi022.stdout:3720: seq_num 1171 ranges {0=417792} 2024-09-21T09:44:06.140 INFO:tasks.rados.rados.0.smithi022.stdout:3720: writing smithi02284960-40 from 0 to 417792 tid 1 2024-09-21T09:44:06.140 INFO:tasks.rados.rados.0.smithi022.stdout:3721: snap_create 2024-09-21T09:44:06.140 INFO:tasks.rados.rados.0.smithi022.stdout:3719: finishing rollback tid 1 to smithi02284960-45 2024-09-21T09:44:06.140 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 45 v 0 (ObjNum 129 snap 0 seq_num 2956276867) dirty dne 2024-09-21T09:44:06.145 INFO:tasks.rados.rados.0.smithi022.stdout:3720: finishing write tid 1 to smithi02284960-40 2024-09-21T09:44:06.145 INFO:tasks.rados.rados.0.smithi022.stdout:3720: finishing write tid 2 to smithi02284960-40 2024-09-21T09:44:06.147 INFO:tasks.rados.rados.0.smithi022.stdout:3720: finishing write tid 3 to smithi02284960-40 2024-09-21T09:44:06.147 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 40 v 1918 (ObjNum 1171 snap 361 seq_num 1171) dirty exists 2024-09-21T09:44:06.147 INFO:tasks.rados.rados.0.smithi022.stdout:3720: left oid 40 (ObjNum 1171 snap 361 seq_num 1171) 2024-09-21T09:44:06.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:44:06 smithi064 ceph-mon[86709]: pgmap v817: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 147 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 1.5 MiB/s wr, 15 op/s 2024-09-21T09:44:06.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:44:06 smithi064 ceph-mon[86709]: osdmap e803: 8 total, 8 up, 8 in 2024-09-21T09:44:06.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:44:06 smithi022 ceph-mon[98615]: pgmap v817: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 147 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 1.5 MiB/s wr, 15 op/s 2024-09-21T09:44:06.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:44:06 smithi022 ceph-mon[98615]: osdmap e803: 8 total, 8 up, 8 in 2024-09-21T09:44:06.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:44:06 smithi022 ceph-mon[96736]: pgmap v817: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 147 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 1.5 MiB/s wr, 15 op/s 2024-09-21T09:44:06.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:44:06 smithi022 ceph-mon[96736]: osdmap e803: 8 total, 8 up, 8 in 2024-09-21T09:44:07.141 INFO:tasks.rados.rados.0.smithi022.stdout:3719: done (2 left) 2024-09-21T09:44:07.142 INFO:tasks.rados.rados.0.smithi022.stdout:3720: done (1 left) 2024-09-21T09:44:07.142 INFO:tasks.rados.rados.0.smithi022.stdout:3721: done (0 left) 2024-09-21T09:44:07.142 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 13 current snap is 362 2024-09-21T09:44:07.142 INFO:tasks.rados.rados.0.smithi022.stdout:3722: seq_num 1172 ranges {0=434176} 2024-09-21T09:44:07.144 INFO:tasks.rados.rados.0.smithi022.stdout:3722: writing smithi02284960-13 from 0 to 434176 tid 1 2024-09-21T09:44:07.144 INFO:tasks.rados.rados.0.smithi022.stdout:3723: rollback oid 30 current snap is 362 2024-09-21T09:44:07.144 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 30 to 361 2024-09-21T09:44:07.144 INFO:tasks.rados.rados.0.smithi022.stdout:3724: snap_create 2024-09-21T09:44:07.148 INFO:tasks.rados.rados.0.smithi022.stdout:3723: finishing rollback tid 0 to smithi02284960-30 2024-09-21T09:44:07.148 INFO:tasks.rados.rados.0.smithi022.stdout:3723: finishing rollback tid 1 to smithi02284960-30 2024-09-21T09:44:07.148 INFO:tasks.rados.rados.0.smithi022.stdout:3723: finishing rollback tid 2 to smithi02284960-30 2024-09-21T09:44:07.149 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 30 v 1977 (ObjNum 1102 snap 337 seq_num 1102) dirty exists 2024-09-21T09:44:07.150 INFO:tasks.rados.rados.0.smithi022.stdout:3722: finishing write tid 1 to smithi02284960-13 2024-09-21T09:44:07.151 INFO:tasks.rados.rados.0.smithi022.stdout:3722: finishing write tid 2 to smithi02284960-13 2024-09-21T09:44:07.152 INFO:tasks.rados.rados.0.smithi022.stdout:3722: finishing write tid 3 to smithi02284960-13 2024-09-21T09:44:07.153 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 13 v 1986 (ObjNum 1172 snap 362 seq_num 1172) dirty exists 2024-09-21T09:44:07.153 INFO:tasks.rados.rados.0.smithi022.stdout:3722: left oid 13 (ObjNum 1172 snap 362 seq_num 1172) 2024-09-21T09:44:07.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:44:07 smithi064 ceph-mon[86709]: osdmap e804: 8 total, 8 up, 8 in 2024-09-21T09:44:07.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:44:07 smithi064 ceph-mon[86709]: pgmap v820: 121 pgs: 121 active+clean; 107 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 2.0 MiB/s rd, 1.5 MiB/s wr, 4 op/s 2024-09-21T09:44:07.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:44:07 smithi022 ceph-mon[96736]: osdmap e804: 8 total, 8 up, 8 in 2024-09-21T09:44:07.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:44:07 smithi022 ceph-mon[96736]: pgmap v820: 121 pgs: 121 active+clean; 107 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 2.0 MiB/s rd, 1.5 MiB/s wr, 4 op/s 2024-09-21T09:44:07.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:44:07 smithi022 ceph-mon[98615]: osdmap e804: 8 total, 8 up, 8 in 2024-09-21T09:44:07.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:44:07 smithi022 ceph-mon[98615]: pgmap v820: 121 pgs: 121 active+clean; 107 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 2.0 MiB/s rd, 1.5 MiB/s wr, 4 op/s 2024-09-21T09:44:08.143 INFO:tasks.rados.rados.0.smithi022.stdout:3722: done (2 left) 2024-09-21T09:44:08.144 INFO:tasks.rados.rados.0.smithi022.stdout:3723: done (1 left) 2024-09-21T09:44:08.144 INFO:tasks.rados.rados.0.smithi022.stdout:3724: done (0 left) 2024-09-21T09:44:08.144 INFO:tasks.rados.rados.0.smithi022.stdout:3725: read oid 41 snap -1 2024-09-21T09:44:08.144 INFO:tasks.rados.rados.0.smithi022.stdout:3725: expect (ObjNum 1120 snap 341 seq_num 1120) 2024-09-21T09:44:08.144 INFO:tasks.rados.rados.0.smithi022.stdout:3726: delete oid 28 current snap is 363 2024-09-21T09:44:08.147 INFO:tasks.rados.rados.0.smithi022.stdout:3726: done (1 left) 2024-09-21T09:44:08.147 INFO:tasks.rados.rados.0.smithi022.stdout:3727: snap_create 2024-09-21T09:44:08.157 INFO:tasks.rados.rados.0.smithi022.stdout:3725: expect (ObjNum 1120 snap 341 seq_num 1120) 2024-09-21T09:44:08.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:44:08 smithi064 ceph-mon[86709]: osdmap e805: 8 total, 8 up, 8 in 2024-09-21T09:44:08.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:44:08 smithi022 ceph-mon[98615]: osdmap e805: 8 total, 8 up, 8 in 2024-09-21T09:44:08.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:44:08 smithi022 ceph-mon[96736]: osdmap e805: 8 total, 8 up, 8 in 2024-09-21T09:44:09.106 INFO:tasks.rados.rados.0.smithi022.stdout:3725: done (1 left) 2024-09-21T09:44:09.106 INFO:tasks.rados.rados.0.smithi022.stdout:3727: done (0 left) 2024-09-21T09:44:09.106 INFO:tasks.rados.rados.0.smithi022.stdout:3728: snap_remove snap 361 2024-09-21T09:44:09.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:44:09 smithi064 ceph-mon[86709]: osdmap e806: 8 total, 8 up, 8 in 2024-09-21T09:44:09.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:44:09 smithi064 ceph-mon[86709]: pgmap v823: 121 pgs: 121 active+clean; 107 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 172 KiB/s wr, 2 op/s 2024-09-21T09:44:09.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:44:09 smithi064 ceph-mon[86709]: osdmap e807: 8 total, 8 up, 8 in 2024-09-21T09:44:09.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:44:09 smithi022 ceph-mon[98615]: osdmap e806: 8 total, 8 up, 8 in 2024-09-21T09:44:09.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:44:09 smithi022 ceph-mon[98615]: pgmap v823: 121 pgs: 121 active+clean; 107 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 172 KiB/s wr, 2 op/s 2024-09-21T09:44:09.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:44:09 smithi022 ceph-mon[98615]: osdmap e807: 8 total, 8 up, 8 in 2024-09-21T09:44:09.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:44:09 smithi022 ceph-mon[96736]: osdmap e806: 8 total, 8 up, 8 in 2024-09-21T09:44:09.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:44:09 smithi022 ceph-mon[96736]: pgmap v823: 121 pgs: 121 active+clean; 107 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 172 KiB/s wr, 2 op/s 2024-09-21T09:44:09.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:44:09 smithi022 ceph-mon[96736]: osdmap e807: 8 total, 8 up, 8 in 2024-09-21T09:44:10.114 INFO:tasks.rados.rados.0.smithi022.stdout:3728: done (0 left) 2024-09-21T09:44:10.114 INFO:tasks.rados.rados.0.smithi022.stdout:3729: snap_create 2024-09-21T09:44:10.691 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:44:10 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:44:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:44:11.118 INFO:tasks.rados.rados.0.smithi022.stdout:3729: done (0 left) 2024-09-21T09:44:11.118 INFO:tasks.rados.rados.0.smithi022.stdout:3730: copy_from oid 13 from oid 23 current snap is 365 2024-09-21T09:44:11.118 INFO:tasks.rados.rados.0.smithi022.stdout:3731: delete oid 7 current snap is 365 2024-09-21T09:44:11.121 INFO:tasks.rados.rados.0.smithi022.stdout:3731: done (1 left) 2024-09-21T09:44:11.121 INFO:tasks.rados.rados.0.smithi022.stdout:3732: snap_create 2024-09-21T09:44:11.138 INFO:tasks.rados.rados.0.smithi022.stdout:3730: finishing copy_from racing read to smithi02284960-13 2024-09-21T09:44:11.140 INFO:tasks.rados.rados.0.smithi022.stdout:3730: finishing copy_from to smithi02284960-13 2024-09-21T09:44:11.140 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 13 v 1843 (ObjNum 1156 snap 356 seq_num 1156) dirty exists 2024-09-21T09:44:11.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:44:11 smithi064 ceph-mon[86709]: osdmap e808: 8 total, 8 up, 8 in 2024-09-21T09:44:11.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:44:11 smithi022 ceph-mon[98615]: osdmap e808: 8 total, 8 up, 8 in 2024-09-21T09:44:11.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:44:11 smithi022 ceph-mon[96736]: osdmap e808: 8 total, 8 up, 8 in 2024-09-21T09:44:12.123 INFO:tasks.rados.rados.0.smithi022.stdout:3730: done (1 left) 2024-09-21T09:44:12.123 INFO:tasks.rados.rados.0.smithi022.stdout:3732: done (0 left) 2024-09-21T09:44:12.123 INFO:tasks.rados.rados.0.smithi022.stdout:3733: snap_create 2024-09-21T09:44:12.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:44:12 smithi064 ceph-mon[86709]: pgmap v826: 121 pgs: 121 active+clean; 108 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 1.4 MiB/s rd, 192 KiB/s wr, 2 op/s 2024-09-21T09:44:12.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:44:12 smithi064 ceph-mon[86709]: osdmap e809: 8 total, 8 up, 8 in 2024-09-21T09:44:12.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:44:12 smithi022 ceph-mon[98615]: pgmap v826: 121 pgs: 121 active+clean; 108 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 1.4 MiB/s rd, 192 KiB/s wr, 2 op/s 2024-09-21T09:44:12.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:44:12 smithi022 ceph-mon[98615]: osdmap e809: 8 total, 8 up, 8 in 2024-09-21T09:44:12.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:44:12 smithi022 ceph-mon[96736]: pgmap v826: 121 pgs: 121 active+clean; 108 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 1.4 MiB/s rd, 192 KiB/s wr, 2 op/s 2024-09-21T09:44:12.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:44:12 smithi022 ceph-mon[96736]: osdmap e809: 8 total, 8 up, 8 in 2024-09-21T09:44:13.129 INFO:tasks.rados.rados.0.smithi022.stdout:3733: done (0 left) 2024-09-21T09:44:13.129 INFO:tasks.rados.rados.0.smithi022.stdout:3734: snap_create 2024-09-21T09:44:13.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:44:13 smithi064 ceph-mon[86709]: osdmap e810: 8 total, 8 up, 8 in 2024-09-21T09:44:13.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:44:13 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:44:13.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:44:13 smithi022 ceph-mon[98615]: osdmap e810: 8 total, 8 up, 8 in 2024-09-21T09:44:13.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:44:13 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:44:13.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:44:13 smithi022 ceph-mon[96736]: osdmap e810: 8 total, 8 up, 8 in 2024-09-21T09:44:13.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:44:13 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:44:14.107 INFO:tasks.rados.rados.0.smithi022.stdout:3734: done (0 left) 2024-09-21T09:44:14.108 INFO:tasks.rados.rados.0.smithi022.stdout:3735: snap_create 2024-09-21T09:44:14.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:44:14 smithi064 ceph-mon[86709]: pgmap v829: 121 pgs: 121 active+clean; 111 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 1.4 MiB/s rd, 894 KiB/s wr, 3 op/s 2024-09-21T09:44:14.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:44:14 smithi064 ceph-mon[86709]: osdmap e811: 8 total, 8 up, 8 in 2024-09-21T09:44:14.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:44:14 smithi064 ceph-mon[86709]: osdmap e812: 8 total, 8 up, 8 in 2024-09-21T09:44:14.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:44:14 smithi022 ceph-mon[98615]: pgmap v829: 121 pgs: 121 active+clean; 111 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 1.4 MiB/s rd, 894 KiB/s wr, 3 op/s 2024-09-21T09:44:14.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:44:14 smithi022 ceph-mon[98615]: osdmap e811: 8 total, 8 up, 8 in 2024-09-21T09:44:14.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:44:14 smithi022 ceph-mon[98615]: osdmap e812: 8 total, 8 up, 8 in 2024-09-21T09:44:14.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:44:14 smithi022 ceph-mon[96736]: pgmap v829: 121 pgs: 121 active+clean; 111 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 1.4 MiB/s rd, 894 KiB/s wr, 3 op/s 2024-09-21T09:44:14.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:44:14 smithi022 ceph-mon[96736]: osdmap e811: 8 total, 8 up, 8 in 2024-09-21T09:44:14.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:44:14 smithi022 ceph-mon[96736]: osdmap e812: 8 total, 8 up, 8 in 2024-09-21T09:44:15.109 INFO:tasks.rados.rados.0.smithi022.stdout:3735: done (0 left) 2024-09-21T09:44:15.109 INFO:tasks.rados.rados.0.smithi022.stdout:3736: read oid 3 snap 360 2024-09-21T09:44:15.109 INFO:tasks.rados.rados.0.smithi022.stdout:3736: expect (ObjNum 1109 snap 338 seq_num 1109) 2024-09-21T09:44:15.109 INFO:tasks.rados.rados.0.smithi022.stdout:3737: delete oid 41 current snap is 369 2024-09-21T09:44:15.114 INFO:tasks.rados.rados.0.smithi022.stdout:3737: done (1 left) 2024-09-21T09:44:15.114 INFO:tasks.rados.rados.0.smithi022.stdout:3738: snap_remove snap 363 2024-09-21T09:44:16.058 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:44:15 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:44:15] "GET /metrics HTTP/1.1" 200 36356 "" "Prometheus/2.51.0" 2024-09-21T09:44:16.114 INFO:tasks.rados.rados.0.smithi022.stdout:3738: done (1 left) 2024-09-21T09:44:16.114 INFO:tasks.rados.rados.0.smithi022.stdout:3739: rollback oid 26 current snap is 369 2024-09-21T09:44:16.114 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 26 to 367 2024-09-21T09:44:16.114 INFO:tasks.rados.rados.0.smithi022.stdout:3736: expect (ObjNum 1109 snap 338 seq_num 1109) 2024-09-21T09:44:16.267 INFO:tasks.rados.rados.0.smithi022.stdout:3739: finishing rollback tid 1 to smithi02284960-26 2024-09-21T09:44:16.267 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 26 v 0 (ObjNum 49 snap 0 seq_num 2955784946) dirty dne 2024-09-21T09:44:16.268 INFO:tasks.rados.rados.0.smithi022.stdout:3736: done (1 left) 2024-09-21T09:44:16.268 INFO:tasks.rados.rados.0.smithi022.stdout:3739: done (0 left) 2024-09-21T09:44:16.268 INFO:tasks.rados.rados.0.smithi022.stdout:3740: copy_from oid 28 from oid 4 current snap is 369 2024-09-21T09:44:16.268 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 20 current snap is 369 2024-09-21T09:44:16.268 INFO:tasks.rados.rados.0.smithi022.stdout:3741: seq_num 1176 ranges {5947392=499712} 2024-09-21T09:44:16.303 INFO:tasks.rados.rados.0.smithi022.stdout:3741: writing smithi02284960-20 from 5947392 to 6447104 tid 1 2024-09-21T09:44:16.303 INFO:tasks.rados.rados.0.smithi022.stdout:3742: read oid 16 snap -1 2024-09-21T09:44:16.303 INFO:tasks.rados.rados.0.smithi022.stdout:3742: expect (ObjNum 1106 snap 338 seq_num 1106) 2024-09-21T09:44:16.303 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 44 current snap is 369 2024-09-21T09:44:16.303 INFO:tasks.rados.rados.0.smithi022.stdout:3740: finishing copy_from racing read to smithi02284960-28 2024-09-21T09:44:16.303 INFO:tasks.rados.rados.0.smithi022.stdout:3740: finishing copy_from to smithi02284960-28 2024-09-21T09:44:16.303 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 28 v 1994 (ObjNum 1166 snap 359 seq_num 1166) dirty exists 2024-09-21T09:44:16.304 INFO:tasks.rados.rados.0.smithi022.stdout:3743: seq_num 1177 ranges {7241728=516096} 2024-09-21T09:44:16.346 INFO:tasks.rados.rados.0.smithi022.stdout:3743: writing smithi02284960-44 from 7241728 to 7757824 tid 1 2024-09-21T09:44:16.346 INFO:tasks.rados.rados.0.smithi022.stdout:3740: done (3 left) 2024-09-21T09:44:16.346 INFO:tasks.rados.rados.0.smithi022.stdout:3744: snap_create 2024-09-21T09:44:16.346 INFO:tasks.rados.rados.0.smithi022.stdout:3741: finishing write tid 1 to smithi02284960-20 2024-09-21T09:44:16.347 INFO:tasks.rados.rados.0.smithi022.stdout:3741: finishing write tid 2 to smithi02284960-20 2024-09-21T09:44:16.347 INFO:tasks.rados.rados.0.smithi022.stdout:3741: finishing write tid 3 to smithi02284960-20 2024-09-21T09:44:16.347 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 20 v 2133 (ObjNum 1176 snap 369 seq_num 1176) dirty exists 2024-09-21T09:44:16.347 INFO:tasks.rados.rados.0.smithi022.stdout:3741: left oid 20 (ObjNum 1176 snap 369 seq_num 1176) 2024-09-21T09:44:16.347 INFO:tasks.rados.rados.0.smithi022.stdout:3742: expect (ObjNum 1106 snap 338 seq_num 1106) 2024-09-21T09:44:16.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:44:16 smithi064 ceph-mon[86709]: pgmap v832: 121 pgs: 121 active+clean; 111 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 792 KiB/s wr, 2 op/s 2024-09-21T09:44:16.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:44:16 smithi064 ceph-mon[86709]: osdmap e813: 8 total, 8 up, 8 in 2024-09-21T09:44:16.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:44:16 smithi022 ceph-mon[96736]: pgmap v832: 121 pgs: 121 active+clean; 111 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 792 KiB/s wr, 2 op/s 2024-09-21T09:44:16.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:44:16 smithi022 ceph-mon[96736]: osdmap e813: 8 total, 8 up, 8 in 2024-09-21T09:44:16.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:44:16 smithi022 ceph-mon[98615]: pgmap v832: 121 pgs: 121 active+clean; 111 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 792 KiB/s wr, 2 op/s 2024-09-21T09:44:16.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:44:16 smithi022 ceph-mon[98615]: osdmap e813: 8 total, 8 up, 8 in 2024-09-21T09:44:17.035 INFO:tasks.rados.rados.0.smithi022.stdout:3743: finishing write tid 1 to smithi02284960-44 2024-09-21T09:44:17.035 INFO:tasks.rados.rados.0.smithi022.stdout:3743: finishing write tid 2 to smithi02284960-44 2024-09-21T09:44:17.035 INFO:tasks.rados.rados.0.smithi022.stdout:3743: finishing write tid 3 to smithi02284960-44 2024-09-21T09:44:17.035 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 44 v 2089 (ObjNum 1177 snap 369 seq_num 1177) dirty exists 2024-09-21T09:44:17.035 INFO:tasks.rados.rados.0.smithi022.stdout:3743: left oid 44 (ObjNum 1177 snap 369 seq_num 1177) 2024-09-21T09:44:17.118 INFO:tasks.rados.rados.0.smithi022.stdout:3741: done (3 left) 2024-09-21T09:44:17.119 INFO:tasks.rados.rados.0.smithi022.stdout:3742: done (2 left) 2024-09-21T09:44:17.119 INFO:tasks.rados.rados.0.smithi022.stdout:3743: done (1 left) 2024-09-21T09:44:17.119 INFO:tasks.rados.rados.0.smithi022.stdout:3744: done (0 left) 2024-09-21T09:44:17.119 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 23 current snap is 370 2024-09-21T09:44:17.119 INFO:tasks.rados.rados.0.smithi022.stdout:3745: seq_num 1178 ranges {2809856=532480} 2024-09-21T09:44:17.137 INFO:tasks.rados.rados.0.smithi022.stdout:3745: writing smithi02284960-23 from 2809856 to 3342336 tid 1 2024-09-21T09:44:17.137 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 3 current snap is 370 2024-09-21T09:44:17.137 INFO:tasks.rados.rados.0.smithi022.stdout:3746: seq_num 1179 ranges {3891200=548864} 2024-09-21T09:44:17.162 INFO:tasks.rados.rados.0.smithi022.stdout:3746: writing smithi02284960-3 from 3891200 to 4440064 tid 1 2024-09-21T09:44:17.162 INFO:tasks.rados.rados.0.smithi022.stdout:3747: read oid 34 snap -1 2024-09-21T09:44:17.162 INFO:tasks.rados.rados.0.smithi022.stdout:3747: expect deleted 2024-09-21T09:44:17.162 INFO:tasks.rados.rados.0.smithi022.stdout:3748: rollback oid 19 current snap is 370 2024-09-21T09:44:17.162 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 19 to 364 2024-09-21T09:44:17.162 INFO:tasks.rados.rados.0.smithi022.stdout:3745: finishing write tid 1 to smithi02284960-23 2024-09-21T09:44:17.162 INFO:tasks.rados.rados.0.smithi022.stdout:3745: finishing write tid 2 to smithi02284960-23 2024-09-21T09:44:17.163 INFO:tasks.rados.rados.0.smithi022.stdout:3745: finishing write tid 3 to smithi02284960-23 2024-09-21T09:44:17.163 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 23 v 1847 (ObjNum 1178 snap 370 seq_num 1178) dirty exists 2024-09-21T09:44:17.163 INFO:tasks.rados.rados.0.smithi022.stdout:3745: left oid 23 (ObjNum 1178 snap 370 seq_num 1178) 2024-09-21T09:44:17.163 INFO:tasks.rados.rados.0.smithi022.stdout:3745: done (3 left) 2024-09-21T09:44:17.163 INFO:tasks.rados.rados.0.smithi022.stdout:3749: delete oid 5 current snap is 370 2024-09-21T09:44:17.167 INFO:tasks.rados.rados.0.smithi022.stdout:3747: done (3 left) 2024-09-21T09:44:17.167 INFO:tasks.rados.rados.0.smithi022.stdout:3749: done (2 left) 2024-09-21T09:44:17.167 INFO:tasks.rados.rados.0.smithi022.stdout:3750: rollback oid 10 current snap is 370 2024-09-21T09:44:17.168 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 10 to 368 2024-09-21T09:44:17.168 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 22 current snap is 370 2024-09-21T09:44:17.168 INFO:tasks.rados.rados.0.smithi022.stdout:3751: seq_num 1181 ranges {4202496=581632} 2024-09-21T09:44:17.194 INFO:tasks.rados.rados.0.smithi022.stdout:3751: writing smithi02284960-22 from 4202496 to 4784128 tid 1 2024-09-21T09:44:17.194 INFO:tasks.rados.rados.0.smithi022.stdout:3752: rollback oid 41 current snap is 370 2024-09-21T09:44:17.194 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 41 to 368 2024-09-21T09:44:17.194 INFO:tasks.rados.rados.0.smithi022.stdout:3748: finishing rollback tid 1 to smithi02284960-19 2024-09-21T09:44:17.194 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 19 v 0 (ObjNum 65 snap 0 seq_num 1886543455) dirty dne 2024-09-21T09:44:17.194 INFO:tasks.rados.rados.0.smithi022.stdout:3746: finishing write tid 1 to smithi02284960-3 2024-09-21T09:44:17.194 INFO:tasks.rados.rados.0.smithi022.stdout:3746: finishing write tid 2 to smithi02284960-3 2024-09-21T09:44:17.194 INFO:tasks.rados.rados.0.smithi022.stdout:3748: done (4 left) 2024-09-21T09:44:17.195 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 21 current snap is 370 2024-09-21T09:44:17.195 INFO:tasks.rados.rados.0.smithi022.stdout:3746: finishing write tid 3 to smithi02284960-3 2024-09-21T09:44:17.195 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 3 v 637 (ObjNum 1179 snap 370 seq_num 1179) dirty exists 2024-09-21T09:44:17.195 INFO:tasks.rados.rados.0.smithi022.stdout:3746: left oid 3 (ObjNum 1179 snap 370 seq_num 1179) 2024-09-21T09:44:17.195 INFO:tasks.rados.rados.0.smithi022.stdout:3750: finishing rollback tid 0 to smithi02284960-10 2024-09-21T09:44:17.195 INFO:tasks.rados.rados.0.smithi022.stdout:3750: finishing rollback tid 1 to smithi02284960-10 2024-09-21T09:44:17.195 INFO:tasks.rados.rados.0.smithi022.stdout:3750: finishing rollback tid 2 to smithi02284960-10 2024-09-21T09:44:17.195 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 10 v 1922 (ObjNum 1157 snap 356 seq_num 1157) dirty exists 2024-09-21T09:44:17.195 INFO:tasks.rados.rados.0.smithi022.stdout:3753: seq_num 1182 ranges {0=606208} 2024-09-21T09:44:17.197 INFO:tasks.rados.rados.0.smithi022.stdout:3753: writing smithi02284960-21 from 0 to 606208 tid 1 2024-09-21T09:44:17.197 INFO:tasks.rados.rados.0.smithi022.stdout:3746: done (4 left) 2024-09-21T09:44:17.197 INFO:tasks.rados.rados.0.smithi022.stdout:3750: done (3 left) 2024-09-21T09:44:17.197 INFO:tasks.rados.rados.0.smithi022.stdout:3754: snap_create 2024-09-21T09:44:17.197 INFO:tasks.rados.rados.0.smithi022.stdout:3752: finishing rollback tid 1 to smithi02284960-41 2024-09-21T09:44:17.197 INFO:tasks.rados.rados.0.smithi022.stdout:3752: finishing rollback tid 2 to smithi02284960-41 2024-09-21T09:44:17.197 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 41 v 1924 (ObjNum 1120 snap 341 seq_num 1120) dirty exists 2024-09-21T09:44:17.199 INFO:tasks.rados.rados.0.smithi022.stdout:3751: finishing write tid 1 to smithi02284960-22 2024-09-21T09:44:17.199 INFO:tasks.rados.rados.0.smithi022.stdout:3751: finishing write tid 2 to smithi02284960-22 2024-09-21T09:44:17.201 INFO:tasks.rados.rados.0.smithi022.stdout:3753: finishing write tid 1 to smithi02284960-21 2024-09-21T09:44:17.201 INFO:tasks.rados.rados.0.smithi022.stdout:3753: finishing write tid 2 to smithi02284960-21 2024-09-21T09:44:17.202 INFO:tasks.rados.rados.0.smithi022.stdout:3753: finishing write tid 3 to smithi02284960-21 2024-09-21T09:44:17.202 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 21 v 1849 (ObjNum 1182 snap 370 seq_num 1182) dirty exists 2024-09-21T09:44:17.202 INFO:tasks.rados.rados.0.smithi022.stdout:3753: left oid 21 (ObjNum 1182 snap 370 seq_num 1182) 2024-09-21T09:44:17.204 INFO:tasks.rados.rados.0.smithi022.stdout:3751: finishing write tid 3 to smithi02284960-22 2024-09-21T09:44:17.205 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 22 v 1988 (ObjNum 1181 snap 370 seq_num 1181) dirty exists 2024-09-21T09:44:17.205 INFO:tasks.rados.rados.0.smithi022.stdout:3751: left oid 22 (ObjNum 1181 snap 370 seq_num 1181) 2024-09-21T09:44:17.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:44:17 smithi064 ceph-mon[86709]: osdmap e814: 8 total, 8 up, 8 in 2024-09-21T09:44:17.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:44:17 smithi022 ceph-mon[98615]: osdmap e814: 8 total, 8 up, 8 in 2024-09-21T09:44:17.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:44:17 smithi022 ceph-mon[96736]: osdmap e814: 8 total, 8 up, 8 in 2024-09-21T09:44:18.125 INFO:tasks.rados.rados.0.smithi022.stdout:3751: done (3 left) 2024-09-21T09:44:18.125 INFO:tasks.rados.rados.0.smithi022.stdout:3752: done (2 left) 2024-09-21T09:44:18.125 INFO:tasks.rados.rados.0.smithi022.stdout:3753: done (1 left) 2024-09-21T09:44:18.125 INFO:tasks.rados.rados.0.smithi022.stdout:3754: done (0 left) 2024-09-21T09:44:18.125 INFO:tasks.rados.rados.0.smithi022.stdout:3755: delete oid 49 current snap is 371 2024-09-21T09:44:18.128 INFO:tasks.rados.rados.0.smithi022.stdout:3755: done (0 left) 2024-09-21T09:44:18.129 INFO:tasks.rados.rados.0.smithi022.stdout:3756: snap_remove snap 364 2024-09-21T09:44:18.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:44:18 smithi064 ceph-mon[86709]: pgmap v835: 121 pgs: 121 active+clean; 111 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-21T09:44:18.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:44:18 smithi064 ceph-mon[86709]: osdmap e815: 8 total, 8 up, 8 in 2024-09-21T09:44:18.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:44:18 smithi022 ceph-mon[98615]: pgmap v835: 121 pgs: 121 active+clean; 111 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-21T09:44:18.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:44:18 smithi022 ceph-mon[98615]: osdmap e815: 8 total, 8 up, 8 in 2024-09-21T09:44:18.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:44:18 smithi022 ceph-mon[96736]: pgmap v835: 121 pgs: 121 active+clean; 111 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-21T09:44:18.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:44:18 smithi022 ceph-mon[96736]: osdmap e815: 8 total, 8 up, 8 in 2024-09-21T09:44:19.109 INFO:tasks.rados.rados.0.smithi022.stdout:3756: done (0 left) 2024-09-21T09:44:19.109 INFO:tasks.rados.rados.0.smithi022.stdout:3757: rollback oid 20 current snap is 371 2024-09-21T09:44:19.109 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 20 to 359 2024-09-21T09:44:19.109 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 10 current snap is 371 2024-09-21T09:44:19.109 INFO:tasks.rados.rados.0.smithi022.stdout:3758: seq_num 1184 ranges {180224=638976} 2024-09-21T09:44:19.114 INFO:tasks.rados.rados.0.smithi022.stdout:3758: writing smithi02284960-10 from 180224 to 819200 tid 1 2024-09-21T09:44:19.114 INFO:tasks.rados.rados.0.smithi022.stdout:3759: copy_from oid 24 from oid 49 current snap is 371 2024-09-21T09:44:19.114 INFO:tasks.rados.rados.0.smithi022.stdout:3760: rollback oid 36 current snap is 371 2024-09-21T09:44:19.115 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 36 to 370 2024-09-21T09:44:19.115 INFO:tasks.rados.rados.0.smithi022.stdout:3761: snap_remove snap 362 2024-09-21T09:44:19.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:44:19 smithi064 ceph-mon[86709]: osdmap e816: 8 total, 8 up, 8 in 2024-09-21T09:44:19.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:44:19 smithi064 ceph-mon[86709]: osdmap e817: 8 total, 8 up, 8 in 2024-09-21T09:44:19.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:44:19 smithi022 ceph-mon[98615]: osdmap e816: 8 total, 8 up, 8 in 2024-09-21T09:44:19.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:44:19 smithi022 ceph-mon[98615]: osdmap e817: 8 total, 8 up, 8 in 2024-09-21T09:44:19.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:44:19 smithi022 ceph-mon[96736]: osdmap e816: 8 total, 8 up, 8 in 2024-09-21T09:44:19.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:44:19 smithi022 ceph-mon[96736]: osdmap e817: 8 total, 8 up, 8 in 2024-09-21T09:44:20.111 INFO:tasks.rados.rados.0.smithi022.stdout:3757: finishing rollback tid 0 to smithi02284960-20 2024-09-21T09:44:20.111 INFO:tasks.rados.rados.0.smithi022.stdout:3757: finishing rollback tid 1 to smithi02284960-20 2024-09-21T09:44:20.111 INFO:tasks.rados.rados.0.smithi022.stdout:3760: finishing rollback tid 0 to smithi02284960-36 2024-09-21T09:44:20.111 INFO:tasks.rados.rados.0.smithi022.stdout:3760: finishing rollback tid 1 to smithi02284960-36 2024-09-21T09:44:20.111 INFO:tasks.rados.rados.0.smithi022.stdout:3757: finishing rollback tid 2 to smithi02284960-20 2024-09-21T09:44:20.111 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 20 v 2137 (ObjNum 1122 snap 342 seq_num 1122) dirty exists 2024-09-21T09:44:20.111 INFO:tasks.rados.rados.0.smithi022.stdout:3758: finishing write tid 1 to smithi02284960-10 2024-09-21T09:44:20.111 INFO:tasks.rados.rados.0.smithi022.stdout:3760: finishing rollback tid 2 to smithi02284960-36 2024-09-21T09:44:20.111 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 36 v 1895 (ObjNum 1117 snap 341 seq_num 1117) dirty exists 2024-09-21T09:44:20.111 INFO:tasks.rados.rados.0.smithi022.stdout:3758: finishing write tid 2 to smithi02284960-10 2024-09-21T09:44:20.112 INFO:tasks.rados.rados.0.smithi022.stdout:3759: finishing copy_from to smithi02284960-24 2024-09-21T09:44:20.112 INFO:tasks.rados.rados.0.smithi022.stdout:3759: got expected ENOENT (src dne) 2024-09-21T09:44:20.112 INFO:tasks.rados.rados.0.smithi022.stdout:3759: finishing copy_from racing read to smithi02284960-24 2024-09-21T09:44:20.112 INFO:tasks.rados.rados.0.smithi022.stdout:3758: finishing write tid 3 to smithi02284960-10 2024-09-21T09:44:20.112 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 10 v 1926 (ObjNum 1184 snap 371 seq_num 1184) dirty exists 2024-09-21T09:44:20.112 INFO:tasks.rados.rados.0.smithi022.stdout:3758: left oid 10 (ObjNum 1184 snap 371 seq_num 1184) 2024-09-21T09:44:20.112 INFO:tasks.rados.rados.0.smithi022.stdout:3757: done (4 left) 2024-09-21T09:44:20.112 INFO:tasks.rados.rados.0.smithi022.stdout:3758: done (3 left) 2024-09-21T09:44:20.112 INFO:tasks.rados.rados.0.smithi022.stdout:3759: done (2 left) 2024-09-21T09:44:20.112 INFO:tasks.rados.rados.0.smithi022.stdout:3760: done (1 left) 2024-09-21T09:44:20.112 INFO:tasks.rados.rados.0.smithi022.stdout:3761: done (0 left) 2024-09-21T09:44:20.112 INFO:tasks.rados.rados.0.smithi022.stdout:3762: delete oid 8 current snap is 371 2024-09-21T09:44:20.118 INFO:tasks.rados.rados.0.smithi022.stdout:3762: done (0 left) 2024-09-21T09:44:20.118 INFO:tasks.rados.rados.0.smithi022.stdout:3763: read oid 27 snap -1 2024-09-21T09:44:20.118 INFO:tasks.rados.rados.0.smithi022.stdout:3763: expect (ObjNum 1008 snap 309 seq_num 1008) 2024-09-21T09:44:20.118 INFO:tasks.rados.rados.0.smithi022.stdout:3764: delete oid 16 current snap is 371 2024-09-21T09:44:20.123 INFO:tasks.rados.rados.0.smithi022.stdout:3764: done (1 left) 2024-09-21T09:44:20.123 INFO:tasks.rados.rados.0.smithi022.stdout:3765: snap_remove snap 360 2024-09-21T09:44:20.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:44:20 smithi064 ceph-mon[86709]: pgmap v838: 121 pgs: 121 active+clean; 113 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 542 KiB/s wr, 4 op/s 2024-09-21T09:44:20.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:44:20 smithi064 ceph-mon[86709]: osdmap e818: 8 total, 8 up, 8 in 2024-09-21T09:44:20.442 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:44:20 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:44:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:44:20.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:44:20 smithi022 ceph-mon[98615]: pgmap v838: 121 pgs: 121 active+clean; 113 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 542 KiB/s wr, 4 op/s 2024-09-21T09:44:20.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:44:20 smithi022 ceph-mon[98615]: osdmap e818: 8 total, 8 up, 8 in 2024-09-21T09:44:20.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:44:20 smithi022 ceph-mon[96736]: pgmap v838: 121 pgs: 121 active+clean; 113 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 542 KiB/s wr, 4 op/s 2024-09-21T09:44:20.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:44:20 smithi022 ceph-mon[96736]: osdmap e818: 8 total, 8 up, 8 in 2024-09-21T09:44:21.113 INFO:tasks.rados.rados.0.smithi022.stdout:3765: done (1 left) 2024-09-21T09:44:21.113 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 34 current snap is 371 2024-09-21T09:44:21.113 INFO:tasks.rados.rados.0.smithi022.stdout:3763: expect (ObjNum 1008 snap 309 seq_num 1008) 2024-09-21T09:44:21.121 INFO:tasks.rados.rados.0.smithi022.stdout:3766: seq_num 1187 ranges {0=688128} 2024-09-21T09:44:21.126 INFO:tasks.rados.rados.0.smithi022.stdout:3766: writing smithi02284960-34 from 0 to 688128 tid 1 2024-09-21T09:44:21.126 INFO:tasks.rados.rados.0.smithi022.stdout:3763: done (1 left) 2024-09-21T09:44:21.126 INFO:tasks.rados.rados.0.smithi022.stdout:3767: read oid 39 snap -1 2024-09-21T09:44:21.126 INFO:tasks.rados.rados.0.smithi022.stdout:3767: expect deleted 2024-09-21T09:44:21.126 INFO:tasks.rados.rados.0.smithi022.stdout:3768: rollback oid 16 current snap is 371 2024-09-21T09:44:21.126 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 16 to 369 2024-09-21T09:44:21.126 INFO:tasks.rados.rados.0.smithi022.stdout:3769: read oid 18 snap -1 2024-09-21T09:44:21.126 INFO:tasks.rados.rados.0.smithi022.stdout:3769: expect (ObjNum 1062 snap 319 seq_num 1062) 2024-09-21T09:44:21.126 INFO:tasks.rados.rados.0.smithi022.stdout:3770: rollback oid 23 current snap is 371 2024-09-21T09:44:21.126 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 23 to 367 2024-09-21T09:44:21.126 INFO:tasks.rados.rados.0.smithi022.stdout:3771: copy_from oid 10 from oid 31 current snap is 371 2024-09-21T09:44:21.126 INFO:tasks.rados.rados.0.smithi022.stdout:3772: copy_from oid 26 from oid 3 current snap is 371 2024-09-21T09:44:21.127 INFO:tasks.rados.rados.0.smithi022.stdout:3773: delete oid 37 current snap is 371 2024-09-21T09:44:21.130 INFO:tasks.rados.rados.0.smithi022.stdout:3770: finishing rollback tid 0 to smithi02284960-23 2024-09-21T09:44:21.131 INFO:tasks.rados.rados.0.smithi022.stdout:3770: finishing rollback tid 1 to smithi02284960-23 2024-09-21T09:44:21.133 INFO:tasks.rados.rados.0.smithi022.stdout:3770: finishing rollback tid 2 to smithi02284960-23 2024-09-21T09:44:21.133 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 23 v 1850 (ObjNum 1156 snap 356 seq_num 1156) dirty exists 2024-09-21T09:44:21.133 INFO:tasks.rados.rados.0.smithi022.stdout:3766: finishing write tid 1 to smithi02284960-34 2024-09-21T09:44:21.134 INFO:tasks.rados.rados.0.smithi022.stdout:3766: finishing write tid 2 to smithi02284960-34 2024-09-21T09:44:21.134 INFO:tasks.rados.rados.0.smithi022.stdout:3768: finishing rollback tid 1 to smithi02284960-16 2024-09-21T09:44:21.136 INFO:tasks.rados.rados.0.smithi022.stdout:3766: finishing write tid 3 to smithi02284960-34 2024-09-21T09:44:21.136 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 34 v 1851 (ObjNum 1187 snap 371 seq_num 1187) dirty exists 2024-09-21T09:44:21.136 INFO:tasks.rados.rados.0.smithi022.stdout:3766: left oid 34 (ObjNum 1187 snap 371 seq_num 1187) 2024-09-21T09:44:21.139 INFO:tasks.rados.rados.0.smithi022.stdout:3768: finishing rollback tid 2 to smithi02284960-16 2024-09-21T09:44:21.139 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 16 v 1755 (ObjNum 1106 snap 338 seq_num 1106) dirty exists 2024-09-21T09:44:21.139 INFO:tasks.rados.rados.0.smithi022.stdout:3766: done (7 left) 2024-09-21T09:44:21.139 INFO:tasks.rados.rados.0.smithi022.stdout:3767: done (6 left) 2024-09-21T09:44:21.139 INFO:tasks.rados.rados.0.smithi022.stdout:3768: done (5 left) 2024-09-21T09:44:21.139 INFO:tasks.rados.rados.0.smithi022.stdout:3770: done (4 left) 2024-09-21T09:44:21.140 INFO:tasks.rados.rados.0.smithi022.stdout:3773: done (3 left) 2024-09-21T09:44:21.140 INFO:tasks.rados.rados.0.smithi022.stdout:3774: read oid 35 snap -1 2024-09-21T09:44:21.140 INFO:tasks.rados.rados.0.smithi022.stdout:3774: expect (ObjNum 1137 snap 349 seq_num 1137) 2024-09-21T09:44:21.140 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 33 current snap is 371 2024-09-21T09:44:21.140 INFO:tasks.rados.rados.0.smithi022.stdout:3775: seq_num 1189 ranges {0=720896} 2024-09-21T09:44:21.144 INFO:tasks.rados.rados.0.smithi022.stdout:3775: writing smithi02284960-33 from 0 to 720896 tid 1 2024-09-21T09:44:21.144 INFO:tasks.rados.rados.0.smithi022.stdout:3776: rmattr oid 13 current snap is 371 2024-09-21T09:44:21.144 INFO:tasks.rados.rados.0.smithi022.stdout:3776: done (5 left) 2024-09-21T09:44:21.144 INFO:tasks.rados.rados.0.smithi022.stdout:3777: delete oid 21 current snap is 371 2024-09-21T09:44:21.145 INFO:tasks.rados.rados.0.smithi022.stdout:3769: expect (ObjNum 1062 snap 319 seq_num 1062) 2024-09-21T09:44:21.176 INFO:tasks.rados.rados.0.smithi022.stdout:3772: finishing copy_from racing read to smithi02284960-26 2024-09-21T09:44:21.176 INFO:tasks.rados.rados.0.smithi022.stdout:3771: finishing copy_from racing read to smithi02284960-10 2024-09-21T09:44:21.176 INFO:tasks.rados.rados.0.smithi022.stdout:3775: finishing write tid 1 to smithi02284960-33 2024-09-21T09:44:21.176 INFO:tasks.rados.rados.0.smithi022.stdout:3775: finishing write tid 2 to smithi02284960-33 2024-09-21T09:44:21.176 INFO:tasks.rados.rados.0.smithi022.stdout:3772: finishing copy_from to smithi02284960-26 2024-09-21T09:44:21.176 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 26 v 637 (ObjNum 1179 snap 370 seq_num 1179) dirty exists 2024-09-21T09:44:21.176 INFO:tasks.rados.rados.0.smithi022.stdout:3771: finishing copy_from to smithi02284960-10 2024-09-21T09:44:21.177 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 10 v 2105 (ObjNum 1160 snap 356 seq_num 1160) dirty exists 2024-09-21T09:44:21.177 INFO:tasks.rados.rados.0.smithi022.stdout:3775: finishing write tid 3 to smithi02284960-33 2024-09-21T09:44:21.177 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 33 v 2145 (ObjNum 1189 snap 371 seq_num 1189) dirty exists 2024-09-21T09:44:21.177 INFO:tasks.rados.rados.0.smithi022.stdout:3775: left oid 33 (ObjNum 1189 snap 371 seq_num 1189) 2024-09-21T09:44:21.177 INFO:tasks.rados.rados.0.smithi022.stdout:3774: expect (ObjNum 1137 snap 349 seq_num 1137) 2024-09-21T09:44:21.533 INFO:tasks.rados.rados.0.smithi022.stdout:3769: done (5 left) 2024-09-21T09:44:21.533 INFO:tasks.rados.rados.0.smithi022.stdout:3771: done (4 left) 2024-09-21T09:44:21.533 INFO:tasks.rados.rados.0.smithi022.stdout:3772: done (3 left) 2024-09-21T09:44:21.533 INFO:tasks.rados.rados.0.smithi022.stdout:3774: done (2 left) 2024-09-21T09:44:21.533 INFO:tasks.rados.rados.0.smithi022.stdout:3775: done (1 left) 2024-09-21T09:44:21.533 INFO:tasks.rados.rados.0.smithi022.stdout:3777: done (0 left) 2024-09-21T09:44:21.533 INFO:tasks.rados.rados.0.smithi022.stdout:3778: rmattr oid 6 current snap is 371 2024-09-21T09:44:21.533 INFO:tasks.rados.rados.0.smithi022.stdout:3778: done (0 left) 2024-09-21T09:44:21.533 INFO:tasks.rados.rados.0.smithi022.stdout:3779: copy_from oid 37 from oid 12 current snap is 371 2024-09-21T09:44:21.533 INFO:tasks.rados.rados.0.smithi022.stdout:3780: rollback oid 15 current snap is 371 2024-09-21T09:44:21.533 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 15 to 365 2024-09-21T09:44:21.533 INFO:tasks.rados.rados.0.smithi022.stdout:3781: delete oid 42 current snap is 371 2024-09-21T09:44:21.535 INFO:tasks.rados.rados.0.smithi022.stdout:3781: done (2 left) 2024-09-21T09:44:21.535 INFO:tasks.rados.rados.0.smithi022.stdout:3782: read oid 40 snap 359 2024-09-21T09:44:21.535 INFO:tasks.rados.rados.0.smithi022.stdout:3782: expect deleted 2024-09-21T09:44:21.535 INFO:tasks.rados.rados.0.smithi022.stdout:3783: setattr oid 43 current snap is 371 2024-09-21T09:44:21.538 INFO:tasks.rados.rados.0.smithi022.stdout:3782: done (3 left) 2024-09-21T09:44:21.538 INFO:tasks.rados.rados.0.smithi022.stdout:3784: snap_create 2024-09-21T09:44:21.539 INFO:tasks.rados.rados.0.smithi022.stdout:3780: finishing rollback tid 1 to smithi02284960-15 2024-09-21T09:44:21.539 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 15 v 2121 (ObjNum 65 snap 0 seq_num 2956983558) dirty dne 2024-09-21T09:44:21.541 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 43 v 2106 (ObjNum 33 snap 0 seq_num 2955212881) dirty exists 2024-09-21T09:44:21.551 INFO:tasks.rados.rados.0.smithi022.stdout:3779: finishing copy_from racing read to smithi02284960-37 2024-09-21T09:44:21.553 INFO:tasks.rados.rados.0.smithi022.stdout:3779: finishing copy_from to smithi02284960-37 2024-09-21T09:44:21.553 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 37 v 2107 (ObjNum 1103 snap 337 seq_num 1103) dirty exists 2024-09-21T09:44:22.136 INFO:tasks.rados.rados.0.smithi022.stdout:3779: done (3 left) 2024-09-21T09:44:22.136 INFO:tasks.rados.rados.0.smithi022.stdout:3780: done (2 left) 2024-09-21T09:44:22.136 INFO:tasks.rados.rados.0.smithi022.stdout:3783: done (1 left) 2024-09-21T09:44:22.137 INFO:tasks.rados.rados.0.smithi022.stdout:3784: done (0 left) 2024-09-21T09:44:22.137 INFO:tasks.rados.rados.0.smithi022.stdout:3785: rmattr oid 41 current snap is 372 2024-09-21T09:44:22.137 INFO:tasks.rados.rados.0.smithi022.stdout:3786: read oid 24 snap -1 2024-09-21T09:44:22.137 INFO:tasks.rados.rados.0.smithi022.stdout:3786: expect (ObjNum 1079 snap 326 seq_num 1079) 2024-09-21T09:44:22.137 INFO:tasks.rados.rados.0.smithi022.stdout:3787: delete oid 26 current snap is 372 2024-09-21T09:44:22.143 INFO:tasks.rados.rados.0.smithi022.stdout:3787: done (2 left) 2024-09-21T09:44:22.143 INFO:tasks.rados.rados.0.smithi022.stdout:3788: setattr oid 38 current snap is 372 2024-09-21T09:44:22.144 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 41 v 2107 (ObjNum 1120 snap 341 seq_num 1120) dirty exists 2024-09-21T09:44:22.144 INFO:tasks.rados.rados.0.smithi022.stdout:3785: done (2 left) 2024-09-21T09:44:22.144 INFO:tasks.rados.rados.0.smithi022.stdout:3789: snap_create 2024-09-21T09:44:22.150 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 38 v 1851 (ObjNum 1141 snap 350 seq_num 1141) dirty exists 2024-09-21T09:44:22.177 INFO:tasks.rados.rados.0.smithi022.stdout:3786: expect (ObjNum 1079 snap 326 seq_num 1079) 2024-09-21T09:44:22.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:44:22 smithi064 ceph-mon[86709]: pgmap v841: 121 pgs: 121 active+clean; 116 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 9.3 MiB/s rd, 946 KiB/s wr, 8 op/s 2024-09-21T09:44:22.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:44:22 smithi064 ceph-mon[86709]: osdmap e819: 8 total, 8 up, 8 in 2024-09-21T09:44:22.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:44:22 smithi022 ceph-mon[96736]: pgmap v841: 121 pgs: 121 active+clean; 116 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 9.3 MiB/s rd, 946 KiB/s wr, 8 op/s 2024-09-21T09:44:22.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:44:22 smithi022 ceph-mon[96736]: osdmap e819: 8 total, 8 up, 8 in 2024-09-21T09:44:22.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:44:22 smithi022 ceph-mon[98615]: pgmap v841: 121 pgs: 121 active+clean; 116 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 9.3 MiB/s rd, 946 KiB/s wr, 8 op/s 2024-09-21T09:44:22.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:44:22 smithi022 ceph-mon[98615]: osdmap e819: 8 total, 8 up, 8 in 2024-09-21T09:44:23.141 INFO:tasks.rados.rados.0.smithi022.stdout:3786: done (2 left) 2024-09-21T09:44:23.141 INFO:tasks.rados.rados.0.smithi022.stdout:3788: done (1 left) 2024-09-21T09:44:23.141 INFO:tasks.rados.rados.0.smithi022.stdout:3789: done (0 left) 2024-09-21T09:44:23.141 INFO:tasks.rados.rados.0.smithi022.stdout:3790: snap_remove snap 371 2024-09-21T09:44:23.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:44:23 smithi064 ceph-mon[86709]: osdmap e820: 8 total, 8 up, 8 in 2024-09-21T09:44:23.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:44:23 smithi064 ceph-mon[86709]: pgmap v844: 121 pgs: 121 active+clean; 121 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 6.6 MiB/s rd, 1.6 MiB/s wr, 8 op/s 2024-09-21T09:44:23.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:44:23 smithi022 ceph-mon[98615]: osdmap e820: 8 total, 8 up, 8 in 2024-09-21T09:44:23.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:44:23 smithi022 ceph-mon[98615]: pgmap v844: 121 pgs: 121 active+clean; 121 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 6.6 MiB/s rd, 1.6 MiB/s wr, 8 op/s 2024-09-21T09:44:23.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:44:23 smithi022 ceph-mon[96736]: osdmap e820: 8 total, 8 up, 8 in 2024-09-21T09:44:23.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:44:23 smithi022 ceph-mon[96736]: pgmap v844: 121 pgs: 121 active+clean; 121 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 6.6 MiB/s rd, 1.6 MiB/s wr, 8 op/s 2024-09-21T09:44:24.110 INFO:tasks.rados.rados.0.smithi022.stdout:3790: done (0 left) 2024-09-21T09:44:24.111 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 7 current snap is 373 2024-09-21T09:44:24.111 INFO:tasks.rados.rados.0.smithi022.stdout:3791: seq_num 1193 ranges {0=786432} 2024-09-21T09:44:24.115 INFO:tasks.rados.rados.0.smithi022.stdout:3791: writing smithi02284960-7 from 0 to 786432 tid 1 2024-09-21T09:44:24.115 INFO:tasks.rados.rados.0.smithi022.stdout:3792: snap_remove snap 365 2024-09-21T09:44:24.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:44:24 smithi064 ceph-mon[86709]: osdmap e821: 8 total, 8 up, 8 in 2024-09-21T09:44:24.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:44:24 smithi064 ceph-mon[86709]: osdmap e822: 8 total, 8 up, 8 in 2024-09-21T09:44:24.459 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:44:24 smithi022 ceph-mon[98615]: osdmap e821: 8 total, 8 up, 8 in 2024-09-21T09:44:24.459 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:44:24 smithi022 ceph-mon[98615]: osdmap e822: 8 total, 8 up, 8 in 2024-09-21T09:44:24.459 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:44:24 smithi022 ceph-mon[96736]: osdmap e821: 8 total, 8 up, 8 in 2024-09-21T09:44:24.459 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:44:24 smithi022 ceph-mon[96736]: osdmap e822: 8 total, 8 up, 8 in 2024-09-21T09:44:25.112 INFO:tasks.rados.rados.0.smithi022.stdout:3792: done (1 left) 2024-09-21T09:44:25.112 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 41 current snap is 373 2024-09-21T09:44:25.112 INFO:tasks.rados.rados.0.smithi022.stdout:3791: finishing write tid 1 to smithi02284960-7 2024-09-21T09:44:25.112 INFO:tasks.rados.rados.0.smithi022.stdout:3791: finishing write tid 2 to smithi02284960-7 2024-09-21T09:44:25.112 INFO:tasks.rados.rados.0.smithi022.stdout:3791: finishing write tid 3 to smithi02284960-7 2024-09-21T09:44:25.112 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 7 v 2147 (ObjNum 1193 snap 373 seq_num 1193) dirty exists 2024-09-21T09:44:25.112 INFO:tasks.rados.rados.0.smithi022.stdout:3791: left oid 7 (ObjNum 1193 snap 373 seq_num 1193) 2024-09-21T09:44:25.112 INFO:tasks.rados.rados.0.smithi022.stdout:3793: seq_num 1194 ranges {1966080=802816} 2024-09-21T09:44:25.127 INFO:tasks.rados.rados.0.smithi022.stdout:3793: writing smithi02284960-41 from 1966080 to 2768896 tid 1 2024-09-21T09:44:25.127 INFO:tasks.rados.rados.0.smithi022.stdout:3791: done (1 left) 2024-09-21T09:44:25.127 INFO:tasks.rados.rados.0.smithi022.stdout:3794: read oid 37 snap 369 2024-09-21T09:44:25.127 INFO:tasks.rados.rados.0.smithi022.stdout:3794: expect deleted 2024-09-21T09:44:25.127 INFO:tasks.rados.rados.0.smithi022.stdout:3795: snap_create 2024-09-21T09:44:25.133 INFO:tasks.rados.rados.0.smithi022.stdout:3793: finishing write tid 1 to smithi02284960-41 2024-09-21T09:44:25.134 INFO:tasks.rados.rados.0.smithi022.stdout:3793: finishing write tid 2 to smithi02284960-41 2024-09-21T09:44:25.135 INFO:tasks.rados.rados.0.smithi022.stdout:3793: finishing write tid 3 to smithi02284960-41 2024-09-21T09:44:25.135 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 41 v 2109 (ObjNum 1194 snap 373 seq_num 1194) dirty exists 2024-09-21T09:44:25.135 INFO:tasks.rados.rados.0.smithi022.stdout:3793: left oid 41 (ObjNum 1194 snap 373 seq_num 1194) 2024-09-21T09:44:26.058 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:44:25 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:44:25] "GET /metrics HTTP/1.1" 200 36356 "" "Prometheus/2.51.0" 2024-09-21T09:44:26.114 INFO:tasks.rados.rados.0.smithi022.stdout:3793: done (2 left) 2024-09-21T09:44:26.114 INFO:tasks.rados.rados.0.smithi022.stdout:3794: done (1 left) 2024-09-21T09:44:26.114 INFO:tasks.rados.rados.0.smithi022.stdout:3795: done (0 left) 2024-09-21T09:44:26.114 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 44 current snap is 374 2024-09-21T09:44:26.114 INFO:tasks.rados.rados.0.smithi022.stdout:3796: seq_num 1195 ranges {7757824=417792,8175616=401408} 2024-09-21T09:44:26.158 INFO:tasks.rados.rados.0.smithi022.stdout:3796: writing smithi02284960-44 from 7757824 to 8175616 tid 1 2024-09-21T09:44:26.161 INFO:tasks.rados.rados.0.smithi022.stdout:3796: writing smithi02284960-44 from 8175616 to 8577024 tid 2 2024-09-21T09:44:26.161 INFO:tasks.rados.rados.0.smithi022.stdout:3797: snap_remove snap 373 2024-09-21T09:44:26.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:44:26 smithi064 ceph-mon[86709]: pgmap v847: 121 pgs: 121 active+clean; 121 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 5.1 MiB/s rd, 1.4 MiB/s wr, 10 op/s 2024-09-21T09:44:26.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:44:26 smithi064 ceph-mon[86709]: osdmap e823: 8 total, 8 up, 8 in 2024-09-21T09:44:26.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:44:26 smithi022 ceph-mon[98615]: pgmap v847: 121 pgs: 121 active+clean; 121 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 5.1 MiB/s rd, 1.4 MiB/s wr, 10 op/s 2024-09-21T09:44:26.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:44:26 smithi022 ceph-mon[98615]: osdmap e823: 8 total, 8 up, 8 in 2024-09-21T09:44:26.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:44:26 smithi022 ceph-mon[96736]: pgmap v847: 121 pgs: 121 active+clean; 121 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 5.1 MiB/s rd, 1.4 MiB/s wr, 10 op/s 2024-09-21T09:44:26.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:44:26 smithi022 ceph-mon[96736]: osdmap e823: 8 total, 8 up, 8 in 2024-09-21T09:44:27.126 INFO:tasks.rados.rados.0.smithi022.stdout:3797: done (1 left) 2024-09-21T09:44:27.126 INFO:tasks.rados.rados.0.smithi022.stdout:3798: delete oid 42 current snap is 374 2024-09-21T09:44:27.126 INFO:tasks.rados.rados.0.smithi022.stdout:3796: finishing write tid 1 to smithi02284960-44 2024-09-21T09:44:27.127 INFO:tasks.rados.rados.0.smithi022.stdout:3796: finishing write tid 2 to smithi02284960-44 2024-09-21T09:44:27.127 INFO:tasks.rados.rados.0.smithi022.stdout:3796: finishing write tid 3 to smithi02284960-44 2024-09-21T09:44:27.127 INFO:tasks.rados.rados.0.smithi022.stdout:3796: finishing write tid 4 to smithi02284960-44 2024-09-21T09:44:27.127 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 44 v 2092 (ObjNum 1195 snap 374 seq_num 1195) dirty exists 2024-09-21T09:44:27.127 INFO:tasks.rados.rados.0.smithi022.stdout:3796: left oid 44 (ObjNum 1195 snap 374 seq_num 1195) 2024-09-21T09:44:27.137 INFO:tasks.rados.rados.0.smithi022.stdout:3796: done (1 left) 2024-09-21T09:44:27.137 INFO:tasks.rados.rados.0.smithi022.stdout:3798: done (0 left) 2024-09-21T09:44:27.137 INFO:tasks.rados.rados.0.smithi022.stdout:3799: setattr oid 19 current snap is 374 2024-09-21T09:44:27.138 INFO:tasks.rados.rados.0.smithi022.stdout:3800: snap_create 2024-09-21T09:44:27.141 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 19 v 2128 (ObjNum 33 snap 0 seq_num 3569755760) dirty exists 2024-09-21T09:44:27.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:44:27 smithi064 ceph-mon[86709]: osdmap e824: 8 total, 8 up, 8 in 2024-09-21T09:44:27.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:44:27 smithi022 ceph-mon[98615]: osdmap e824: 8 total, 8 up, 8 in 2024-09-21T09:44:27.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:44:27 smithi022 ceph-mon[96736]: osdmap e824: 8 total, 8 up, 8 in 2024-09-21T09:44:28.127 INFO:tasks.rados.rados.0.smithi022.stdout:3799: done (1 left) 2024-09-21T09:44:28.127 INFO:tasks.rados.rados.0.smithi022.stdout:3800: done (0 left) 2024-09-21T09:44:28.128 INFO:tasks.rados.rados.0.smithi022.stdout:3801: rollback oid 1 current snap is 375 2024-09-21T09:44:28.128 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 1 to 372 2024-09-21T09:44:28.128 INFO:tasks.rados.rados.0.smithi022.stdout:3802: rollback oid 18 current snap is 375 2024-09-21T09:44:28.128 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 18 to 372 2024-09-21T09:44:28.128 INFO:tasks.rados.rados.0.smithi022.stdout:3803: copy_from oid 6 from oid 4 current snap is 375 2024-09-21T09:44:28.128 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 15 current snap is 375 2024-09-21T09:44:28.128 INFO:tasks.rados.rados.0.smithi022.stdout:3804: seq_num 1197 ranges {0=450560,450560=401408} 2024-09-21T09:44:28.129 INFO:tasks.rados.rados.0.smithi022.stdout:3804: writing smithi02284960-15 from 0 to 450560 tid 1 2024-09-21T09:44:28.132 INFO:tasks.rados.rados.0.smithi022.stdout:3804: writing smithi02284960-15 from 450560 to 851968 tid 2 2024-09-21T09:44:28.132 INFO:tasks.rados.rados.0.smithi022.stdout:3805: read oid 39 snap 368 2024-09-21T09:44:28.132 INFO:tasks.rados.rados.0.smithi022.stdout:3805: expect deleted 2024-09-21T09:44:28.132 INFO:tasks.rados.rados.0.smithi022.stdout:3806: snap_create 2024-09-21T09:44:28.137 INFO:tasks.rados.rados.0.smithi022.stdout:3802: finishing rollback tid 0 to smithi02284960-18 2024-09-21T09:44:28.138 INFO:tasks.rados.rados.0.smithi022.stdout:3802: finishing rollback tid 1 to smithi02284960-18 2024-09-21T09:44:28.139 INFO:tasks.rados.rados.0.smithi022.stdout:3802: finishing rollback tid 2 to smithi02284960-18 2024-09-21T09:44:28.139 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 18 v 1758 (ObjNum 1062 snap 319 seq_num 1062) dirty exists 2024-09-21T09:44:28.140 INFO:tasks.rados.rados.0.smithi022.stdout:3801: finishing rollback tid 0 to smithi02284960-1 2024-09-21T09:44:28.140 INFO:tasks.rados.rados.0.smithi022.stdout:3804: finishing write tid 1 to smithi02284960-15 2024-09-21T09:44:28.140 INFO:tasks.rados.rados.0.smithi022.stdout:3804: finishing write tid 2 to smithi02284960-15 2024-09-21T09:44:28.140 INFO:tasks.rados.rados.0.smithi022.stdout:3804: finishing write tid 3 to smithi02284960-15 2024-09-21T09:44:28.141 INFO:tasks.rados.rados.0.smithi022.stdout:3801: finishing rollback tid 1 to smithi02284960-1 2024-09-21T09:44:28.141 INFO:tasks.rados.rados.0.smithi022.stdout:3803: finishing copy_from racing read to smithi02284960-6 2024-09-21T09:44:28.141 INFO:tasks.rados.rados.0.smithi022.stdout:3801: finishing rollback tid 2 to smithi02284960-1 2024-09-21T09:44:28.141 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 1 v 2095 (ObjNum 1120 snap 341 seq_num 1120) dirty exists 2024-09-21T09:44:28.141 INFO:tasks.rados.rados.0.smithi022.stdout:3803: finishing copy_from to smithi02284960-6 2024-09-21T09:44:28.141 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 6 v 1994 (ObjNum 1166 snap 359 seq_num 1166) dirty exists 2024-09-21T09:44:28.142 INFO:tasks.rados.rados.0.smithi022.stdout:3804: finishing write tid 4 to smithi02284960-15 2024-09-21T09:44:28.142 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 15 v 2131 (ObjNum 1197 snap 375 seq_num 1197) dirty exists 2024-09-21T09:44:28.142 INFO:tasks.rados.rados.0.smithi022.stdout:3804: left oid 15 (ObjNum 1197 snap 375 seq_num 1197) 2024-09-21T09:44:28.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:44:28 smithi064 ceph-mon[86709]: pgmap v850: 121 pgs: 121 active+clean; 135 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 2.6 MiB/s wr, 20 op/s 2024-09-21T09:44:28.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:44:28 smithi064 ceph-mon[86709]: osdmap e825: 8 total, 8 up, 8 in 2024-09-21T09:44:28.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:44:28 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:44:28.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:44:28 smithi022 ceph-mon[98615]: pgmap v850: 121 pgs: 121 active+clean; 135 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 2.6 MiB/s wr, 20 op/s 2024-09-21T09:44:28.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:44:28 smithi022 ceph-mon[98615]: osdmap e825: 8 total, 8 up, 8 in 2024-09-21T09:44:28.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:44:28 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:44:28.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:44:28 smithi022 ceph-mon[96736]: pgmap v850: 121 pgs: 121 active+clean; 135 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 2.6 MiB/s wr, 20 op/s 2024-09-21T09:44:28.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:44:28 smithi022 ceph-mon[96736]: osdmap e825: 8 total, 8 up, 8 in 2024-09-21T09:44:28.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:44:28 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:44:29.113 INFO:tasks.rados.rados.0.smithi022.stdout:3801: done (5 left) 2024-09-21T09:44:29.113 INFO:tasks.rados.rados.0.smithi022.stdout:3802: done (4 left) 2024-09-21T09:44:29.113 INFO:tasks.rados.rados.0.smithi022.stdout:3803: done (3 left) 2024-09-21T09:44:29.113 INFO:tasks.rados.rados.0.smithi022.stdout:3804: done (2 left) 2024-09-21T09:44:29.113 INFO:tasks.rados.rados.0.smithi022.stdout:3805: done (1 left) 2024-09-21T09:44:29.113 INFO:tasks.rados.rados.0.smithi022.stdout:3806: done (0 left) 2024-09-21T09:44:29.113 INFO:tasks.rados.rados.0.smithi022.stdout:3807: rmattr oid 27 current snap is 376 2024-09-21T09:44:29.113 INFO:tasks.rados.rados.0.smithi022.stdout:3808: copy_from oid 28 from oid 23 current snap is 376 2024-09-21T09:44:29.113 INFO:tasks.rados.rados.0.smithi022.stdout:3809: setattr oid 29 current snap is 376 2024-09-21T09:44:29.114 INFO:tasks.rados.rados.0.smithi022.stdout:3810: read oid 46 snap -1 2024-09-21T09:44:29.114 INFO:tasks.rados.rados.0.smithi022.stdout:3810: expect deleted 2024-09-21T09:44:29.114 INFO:tasks.rados.rados.0.smithi022.stdout:3811: rollback oid 4 current snap is 376 2024-09-21T09:44:29.114 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 4 to 359 2024-09-21T09:44:29.114 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 37 current snap is 376 2024-09-21T09:44:29.114 INFO:tasks.rados.rados.0.smithi022.stdout:3812: seq_num 1198 ranges {3989504=466944,4456448=401408} 2024-09-21T09:44:29.139 INFO:tasks.rados.rados.0.smithi022.stdout:3812: writing smithi02284960-37 from 3989504 to 4456448 tid 1 2024-09-21T09:44:29.142 INFO:tasks.rados.rados.0.smithi022.stdout:3812: writing smithi02284960-37 from 4456448 to 4857856 tid 2 2024-09-21T09:44:29.142 INFO:tasks.rados.rados.0.smithi022.stdout:3813: rollback oid 22 current snap is 376 2024-09-21T09:44:29.142 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 22 to 359 2024-09-21T09:44:29.142 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 27 v 1990 (ObjNum 1008 snap 309 seq_num 1008) dirty exists 2024-09-21T09:44:29.143 INFO:tasks.rados.rados.0.smithi022.stdout:3811: finishing rollback tid 0 to smithi02284960-4 2024-09-21T09:44:29.143 INFO:tasks.rados.rados.0.smithi022.stdout:3811: finishing rollback tid 1 to smithi02284960-4 2024-09-21T09:44:29.143 INFO:tasks.rados.rados.0.smithi022.stdout:3811: finishing rollback tid 2 to smithi02284960-4 2024-09-21T09:44:29.143 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 4 v 1999 (ObjNum 1166 snap 359 seq_num 1166) dirty exists 2024-09-21T09:44:29.143 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 29 v 1852 (ObjNum 1165 snap 358 seq_num 1165) dirty exists 2024-09-21T09:44:29.143 INFO:tasks.rados.rados.0.smithi022.stdout:3808: finishing copy_from racing read to smithi02284960-28 2024-09-21T09:44:29.143 INFO:tasks.rados.rados.0.smithi022.stdout:3808: finishing copy_from to smithi02284960-28 2024-09-21T09:44:29.143 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 28 v 1850 (ObjNum 1156 snap 356 seq_num 1156) dirty exists 2024-09-21T09:44:29.143 INFO:tasks.rados.rados.0.smithi022.stdout:3807: done (6 left) 2024-09-21T09:44:29.143 INFO:tasks.rados.rados.0.smithi022.stdout:3808: done (5 left) 2024-09-21T09:44:29.143 INFO:tasks.rados.rados.0.smithi022.stdout:3809: done (4 left) 2024-09-21T09:44:29.143 INFO:tasks.rados.rados.0.smithi022.stdout:3810: done (3 left) 2024-09-21T09:44:29.143 INFO:tasks.rados.rados.0.smithi022.stdout:3811: done (2 left) 2024-09-21T09:44:29.144 INFO:tasks.rados.rados.0.smithi022.stdout:3814: setattr oid 9 current snap is 376 2024-09-21T09:44:29.144 INFO:tasks.rados.rados.0.smithi022.stdout:3815: snap_create 2024-09-21T09:44:29.146 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 9 v 2000 (ObjNum 1062 snap 319 seq_num 1062) dirty exists 2024-09-21T09:44:29.147 INFO:tasks.rados.rados.0.smithi022.stdout:3813: finishing rollback tid 0 to smithi02284960-22 2024-09-21T09:44:29.147 INFO:tasks.rados.rados.0.smithi022.stdout:3813: finishing rollback tid 1 to smithi02284960-22 2024-09-21T09:44:29.148 INFO:tasks.rados.rados.0.smithi022.stdout:3812: finishing write tid 1 to smithi02284960-37 2024-09-21T09:44:29.148 INFO:tasks.rados.rados.0.smithi022.stdout:3813: finishing rollback tid 2 to smithi02284960-22 2024-09-21T09:44:29.148 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 22 v 1993 (ObjNum 1036 snap 315 seq_num 1036) dirty exists 2024-09-21T09:44:29.148 INFO:tasks.rados.rados.0.smithi022.stdout:3812: finishing write tid 2 to smithi02284960-37 2024-09-21T09:44:29.148 INFO:tasks.rados.rados.0.smithi022.stdout:3812: finishing write tid 3 to smithi02284960-37 2024-09-21T09:44:29.149 INFO:tasks.rados.rados.0.smithi022.stdout:3812: finishing write tid 4 to smithi02284960-37 2024-09-21T09:44:29.150 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 37 v 2135 (ObjNum 1198 snap 376 seq_num 1198) dirty exists 2024-09-21T09:44:29.150 INFO:tasks.rados.rados.0.smithi022.stdout:3812: left oid 37 (ObjNum 1198 snap 376 seq_num 1198) 2024-09-21T09:44:29.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:44:29 smithi064 ceph-mon[86709]: osdmap e826: 8 total, 8 up, 8 in 2024-09-21T09:44:29.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:44:29 smithi064 ceph-mon[86709]: pgmap v853: 121 pgs: 121 active+clean; 135 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 6.0 MiB/s rd, 2.5 MiB/s wr, 12 op/s 2024-09-21T09:44:29.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:44:29 smithi064 ceph-mon[86709]: osdmap e827: 8 total, 8 up, 8 in 2024-09-21T09:44:29.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:44:29 smithi022 ceph-mon[98615]: osdmap e826: 8 total, 8 up, 8 in 2024-09-21T09:44:29.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:44:29 smithi022 ceph-mon[98615]: pgmap v853: 121 pgs: 121 active+clean; 135 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 6.0 MiB/s rd, 2.5 MiB/s wr, 12 op/s 2024-09-21T09:44:29.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:44:29 smithi022 ceph-mon[98615]: osdmap e827: 8 total, 8 up, 8 in 2024-09-21T09:44:29.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:44:29 smithi022 ceph-mon[96736]: osdmap e826: 8 total, 8 up, 8 in 2024-09-21T09:44:29.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:44:29 smithi022 ceph-mon[96736]: pgmap v853: 121 pgs: 121 active+clean; 135 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 6.0 MiB/s rd, 2.5 MiB/s wr, 12 op/s 2024-09-21T09:44:29.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:44:29 smithi022 ceph-mon[96736]: osdmap e827: 8 total, 8 up, 8 in 2024-09-21T09:44:30.141 INFO:tasks.rados.rados.0.smithi022.stdout:3812: done (3 left) 2024-09-21T09:44:30.142 INFO:tasks.rados.rados.0.smithi022.stdout:3813: done (2 left) 2024-09-21T09:44:30.142 INFO:tasks.rados.rados.0.smithi022.stdout:3814: done (1 left) 2024-09-21T09:44:30.142 INFO:tasks.rados.rados.0.smithi022.stdout:3815: done (0 left) 2024-09-21T09:44:30.142 INFO:tasks.rados.rados.0.smithi022.stdout:3816: read oid 20 snap -1 2024-09-21T09:44:30.142 INFO:tasks.rados.rados.0.smithi022.stdout:3816: expect (ObjNum 1122 snap 342 seq_num 1122) 2024-09-21T09:44:30.142 INFO:tasks.rados.rados.0.smithi022.stdout:3817: snap_create 2024-09-21T09:44:30.162 INFO:tasks.rados.rados.0.smithi022.stdout:3816: expect (ObjNum 1122 snap 342 seq_num 1122) 2024-09-21T09:44:30.691 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:44:30 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:44:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:44:31.147 INFO:tasks.rados.rados.0.smithi022.stdout:3816: done (1 left) 2024-09-21T09:44:31.147 INFO:tasks.rados.rados.0.smithi022.stdout:3817: done (0 left) 2024-09-21T09:44:31.147 INFO:tasks.rados.rados.0.smithi022.stdout:3818: snap_create 2024-09-21T09:44:31.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:44:31 smithi064 ceph-mon[86709]: osdmap e828: 8 total, 8 up, 8 in 2024-09-21T09:44:31.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:44:31 smithi064 ceph-mon[86709]: pgmap v856: 121 pgs: 121 active+clean; 136 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 4.3 MiB/s rd, 330 KiB/s wr, 8 op/s 2024-09-21T09:44:31.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:44:31 smithi022 ceph-mon[98615]: osdmap e828: 8 total, 8 up, 8 in 2024-09-21T09:44:31.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:44:31 smithi022 ceph-mon[98615]: pgmap v856: 121 pgs: 121 active+clean; 136 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 4.3 MiB/s rd, 330 KiB/s wr, 8 op/s 2024-09-21T09:44:31.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:44:31 smithi022 ceph-mon[96736]: osdmap e828: 8 total, 8 up, 8 in 2024-09-21T09:44:31.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:44:31 smithi022 ceph-mon[96736]: pgmap v856: 121 pgs: 121 active+clean; 136 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 4.3 MiB/s rd, 330 KiB/s wr, 8 op/s 2024-09-21T09:44:32.149 INFO:tasks.rados.rados.0.smithi022.stdout:3818: done (0 left) 2024-09-21T09:44:32.149 INFO:tasks.rados.rados.0.smithi022.stdout:3819: rmattr oid 8 current snap is 379 2024-09-21T09:44:32.149 INFO:tasks.rados.rados.0.smithi022.stdout:3820: copy_from oid 30 from oid 7 current snap is 379 2024-09-21T09:44:32.149 INFO:tasks.rados.rados.0.smithi022.stdout:3821: setattr oid 47 current snap is 379 2024-09-21T09:44:32.149 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 8 v 0 (ObjNum 0 snap 0 seq_num 304) dirty dne 2024-09-21T09:44:32.150 INFO:tasks.rados.rados.0.smithi022.stdout:3819: done (2 left) 2024-09-21T09:44:32.150 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 25 current snap is 379 2024-09-21T09:44:32.150 INFO:tasks.rados.rados.0.smithi022.stdout:3822: seq_num 1199 ranges {270336=483328,753664=401408} 2024-09-21T09:44:32.154 INFO:tasks.rados.rados.0.smithi022.stdout:3822: writing smithi02284960-25 from 270336 to 753664 tid 1 2024-09-21T09:44:32.157 INFO:tasks.rados.rados.0.smithi022.stdout:3822: writing smithi02284960-25 from 753664 to 1155072 tid 2 2024-09-21T09:44:32.157 INFO:tasks.rados.rados.0.smithi022.stdout:3823: snap_remove snap 372 2024-09-21T09:44:32.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:44:32 smithi064 ceph-mon[86709]: osdmap e829: 8 total, 8 up, 8 in 2024-09-21T09:44:32.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:44:32 smithi022 ceph-mon[98615]: osdmap e829: 8 total, 8 up, 8 in 2024-09-21T09:44:32.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:44:32 smithi022 ceph-mon[96736]: osdmap e829: 8 total, 8 up, 8 in 2024-09-21T09:44:33.162 INFO:tasks.rados.rados.0.smithi022.stdout:3823: done (3 left) 2024-09-21T09:44:33.162 INFO:tasks.rados.rados.0.smithi022.stdout:3824: read oid 4 snap 366 2024-09-21T09:44:33.162 INFO:tasks.rados.rados.0.smithi022.stdout:3824: expect (ObjNum 1166 snap 359 seq_num 1166) 2024-09-21T09:44:33.162 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 17 current snap is 379 2024-09-21T09:44:33.162 INFO:tasks.rados.rados.0.smithi022.stdout:3825: seq_num 1200 ranges {0=499712,499712=401408} 2024-09-21T09:44:33.165 INFO:tasks.rados.rados.0.smithi022.stdout:3825: writing smithi02284960-17 from 0 to 499712 tid 1 2024-09-21T09:44:33.168 INFO:tasks.rados.rados.0.smithi022.stdout:3825: writing smithi02284960-17 from 499712 to 901120 tid 2 2024-09-21T09:44:33.168 INFO:tasks.rados.rados.0.smithi022.stdout:3826: snap_remove snap 378 2024-09-21T09:44:33.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:44:33 smithi064 ceph-mon[86709]: osdmap e830: 8 total, 8 up, 8 in 2024-09-21T09:44:33.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:44:33 smithi064 ceph-mon[86709]: pgmap v859: 121 pgs: 121 active+clean; 138 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 4.3 MiB/s rd, 750 KiB/s wr, 18 op/s 2024-09-21T09:44:33.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:44:33 smithi022 ceph-mon[98615]: osdmap e830: 8 total, 8 up, 8 in 2024-09-21T09:44:33.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:44:33 smithi022 ceph-mon[98615]: pgmap v859: 121 pgs: 121 active+clean; 138 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 4.3 MiB/s rd, 750 KiB/s wr, 18 op/s 2024-09-21T09:44:33.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:44:33 smithi022 ceph-mon[96736]: osdmap e830: 8 total, 8 up, 8 in 2024-09-21T09:44:33.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:44:33 smithi022 ceph-mon[96736]: pgmap v859: 121 pgs: 121 active+clean; 138 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 4.3 MiB/s rd, 750 KiB/s wr, 18 op/s 2024-09-21T09:44:34.114 INFO:tasks.rados.rados.0.smithi022.stdout:3826: done (5 left) 2024-09-21T09:44:34.114 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 12 current snap is 379 2024-09-21T09:44:34.114 INFO:tasks.rados.rados.0.smithi022.stdout:3827: seq_num 1201 ranges {3989504=524288,4513792=401408} 2024-09-21T09:44:34.139 INFO:tasks.rados.rados.0.smithi022.stdout:3827: writing smithi02284960-12 from 3989504 to 4513792 tid 1 2024-09-21T09:44:34.141 INFO:tasks.rados.rados.0.smithi022.stdout:3827: writing smithi02284960-12 from 4513792 to 4915200 tid 2 2024-09-21T09:44:34.141 INFO:tasks.rados.rados.0.smithi022.stdout:3828: snap_create 2024-09-21T09:44:34.141 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 47 v 2096 (ObjNum 33 snap 0 seq_num 3570165088) dirty exists 2024-09-21T09:44:34.141 INFO:tasks.rados.rados.0.smithi022.stdout:3820: finishing copy_from racing read to smithi02284960-30 2024-09-21T09:44:34.141 INFO:tasks.rados.rados.0.smithi022.stdout:3820: finishing copy_from to smithi02284960-30 2024-09-21T09:44:34.141 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 30 v 2147 (ObjNum 1193 snap 373 seq_num 1193) dirty exists 2024-09-21T09:44:34.141 INFO:tasks.rados.rados.0.smithi022.stdout:3822: finishing write tid 1 to smithi02284960-25 2024-09-21T09:44:34.142 INFO:tasks.rados.rados.0.smithi022.stdout:3822: finishing write tid 2 to smithi02284960-25 2024-09-21T09:44:34.142 INFO:tasks.rados.rados.0.smithi022.stdout:3822: finishing write tid 3 to smithi02284960-25 2024-09-21T09:44:34.142 INFO:tasks.rados.rados.0.smithi022.stdout:3822: finishing write tid 4 to smithi02284960-25 2024-09-21T09:44:34.142 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 25 v 2112 (ObjNum 1199 snap 379 seq_num 1199) dirty exists 2024-09-21T09:44:34.142 INFO:tasks.rados.rados.0.smithi022.stdout:3822: left oid 25 (ObjNum 1199 snap 379 seq_num 1199) 2024-09-21T09:44:34.142 INFO:tasks.rados.rados.0.smithi022.stdout:3825: finishing write tid 1 to smithi02284960-17 2024-09-21T09:44:34.142 INFO:tasks.rados.rados.0.smithi022.stdout:3825: finishing write tid 2 to smithi02284960-17 2024-09-21T09:44:34.142 INFO:tasks.rados.rados.0.smithi022.stdout:3825: finishing write tid 3 to smithi02284960-17 2024-09-21T09:44:34.142 INFO:tasks.rados.rados.0.smithi022.stdout:3824: expect (ObjNum 1166 snap 359 seq_num 1166) 2024-09-21T09:44:34.161 INFO:tasks.rados.rados.0.smithi022.stdout:3825: finishing write tid 4 to smithi02284960-17 2024-09-21T09:44:34.161 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 17 v 2150 (ObjNum 1200 snap 379 seq_num 1200) dirty exists 2024-09-21T09:44:34.161 INFO:tasks.rados.rados.0.smithi022.stdout:3825: left oid 17 (ObjNum 1200 snap 379 seq_num 1200) 2024-09-21T09:44:34.161 INFO:tasks.rados.rados.0.smithi022.stdout:3827: finishing write tid 1 to smithi02284960-12 2024-09-21T09:44:34.161 INFO:tasks.rados.rados.0.smithi022.stdout:3827: finishing write tid 2 to smithi02284960-12 2024-09-21T09:44:34.161 INFO:tasks.rados.rados.0.smithi022.stdout:3827: finishing write tid 3 to smithi02284960-12 2024-09-21T09:44:34.161 INFO:tasks.rados.rados.0.smithi022.stdout:3827: finishing write tid 4 to smithi02284960-12 2024-09-21T09:44:34.161 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 12 v 2141 (ObjNum 1201 snap 379 seq_num 1201) dirty exists 2024-09-21T09:44:34.161 INFO:tasks.rados.rados.0.smithi022.stdout:3827: left oid 12 (ObjNum 1201 snap 379 seq_num 1201) 2024-09-21T09:44:34.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:44:34 smithi064 ceph-mon[86709]: osdmap e831: 8 total, 8 up, 8 in 2024-09-21T09:44:34.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:44:34 smithi064 ceph-mon[86709]: osdmap e832: 8 total, 8 up, 8 in 2024-09-21T09:44:34.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:44:34 smithi022 ceph-mon[96736]: osdmap e831: 8 total, 8 up, 8 in 2024-09-21T09:44:34.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:44:34 smithi022 ceph-mon[96736]: osdmap e832: 8 total, 8 up, 8 in 2024-09-21T09:44:34.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:44:34 smithi022 ceph-mon[98615]: osdmap e831: 8 total, 8 up, 8 in 2024-09-21T09:44:34.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:44:34 smithi022 ceph-mon[98615]: osdmap e832: 8 total, 8 up, 8 in 2024-09-21T09:44:35.115 INFO:tasks.rados.rados.0.smithi022.stdout:3820: done (6 left) 2024-09-21T09:44:35.115 INFO:tasks.rados.rados.0.smithi022.stdout:3821: done (5 left) 2024-09-21T09:44:35.116 INFO:tasks.rados.rados.0.smithi022.stdout:3822: done (4 left) 2024-09-21T09:44:35.116 INFO:tasks.rados.rados.0.smithi022.stdout:3824: done (3 left) 2024-09-21T09:44:35.116 INFO:tasks.rados.rados.0.smithi022.stdout:3825: done (2 left) 2024-09-21T09:44:35.116 INFO:tasks.rados.rados.0.smithi022.stdout:3827: done (1 left) 2024-09-21T09:44:35.116 INFO:tasks.rados.rados.0.smithi022.stdout:3828: done (0 left) 2024-09-21T09:44:35.116 INFO:tasks.rados.rados.0.smithi022.stdout:3829: snap_create 2024-09-21T09:44:36.058 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:44:35 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:44:35] "GET /metrics HTTP/1.1" 200 36355 "" "Prometheus/2.51.0" 2024-09-21T09:44:36.117 INFO:tasks.rados.rados.0.smithi022.stdout:3829: done (0 left) 2024-09-21T09:44:36.117 INFO:tasks.rados.rados.0.smithi022.stdout:3830: rollback oid 29 current snap is 381 2024-09-21T09:44:36.117 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 29 to 375 2024-09-21T09:44:36.117 INFO:tasks.rados.rados.0.smithi022.stdout:3831: rmattr oid 15 current snap is 381 2024-09-21T09:44:36.118 INFO:tasks.rados.rados.0.smithi022.stdout:3831: done (1 left) 2024-09-21T09:44:36.118 INFO:tasks.rados.rados.0.smithi022.stdout:3832: rmattr oid 40 current snap is 381 2024-09-21T09:44:36.118 INFO:tasks.rados.rados.0.smithi022.stdout:3832: done (1 left) 2024-09-21T09:44:36.118 INFO:tasks.rados.rados.0.smithi022.stdout:3833: snap_remove snap 379 2024-09-21T09:44:36.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:44:36 smithi064 ceph-mon[86709]: pgmap v862: 121 pgs: 121 active+clean; 141 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 1.1 MiB/s wr, 14 op/s 2024-09-21T09:44:36.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:44:36 smithi064 ceph-mon[86709]: osdmap e833: 8 total, 8 up, 8 in 2024-09-21T09:44:36.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:44:36 smithi022 ceph-mon[98615]: pgmap v862: 121 pgs: 121 active+clean; 141 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 1.1 MiB/s wr, 14 op/s 2024-09-21T09:44:36.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:44:36 smithi022 ceph-mon[98615]: osdmap e833: 8 total, 8 up, 8 in 2024-09-21T09:44:36.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:44:36 smithi022 ceph-mon[96736]: pgmap v862: 121 pgs: 121 active+clean; 141 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 1.1 MiB/s wr, 14 op/s 2024-09-21T09:44:36.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:44:36 smithi022 ceph-mon[96736]: osdmap e833: 8 total, 8 up, 8 in 2024-09-21T09:44:37.129 INFO:tasks.rados.rados.0.smithi022.stdout:3833: done (1 left) 2024-09-21T09:44:37.129 INFO:tasks.rados.rados.0.smithi022.stdout:3834: delete oid 12 current snap is 381 2024-09-21T09:44:37.129 INFO:tasks.rados.rados.0.smithi022.stdout:3830: finishing rollback tid 0 to smithi02284960-29 2024-09-21T09:44:37.130 INFO:tasks.rados.rados.0.smithi022.stdout:3830: finishing rollback tid 1 to smithi02284960-29 2024-09-21T09:44:37.130 INFO:tasks.rados.rados.0.smithi022.stdout:3830: finishing rollback tid 2 to smithi02284960-29 2024-09-21T09:44:37.130 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 29 v 1855 (ObjNum 1165 snap 358 seq_num 1165) dirty exists 2024-09-21T09:44:37.142 INFO:tasks.rados.rados.0.smithi022.stdout:3830: done (1 left) 2024-09-21T09:44:37.142 INFO:tasks.rados.rados.0.smithi022.stdout:3834: done (0 left) 2024-09-21T09:44:37.143 INFO:tasks.rados.rados.0.smithi022.stdout:3835: snap_create 2024-09-21T09:44:37.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:44:37 smithi064 ceph-mon[86709]: osdmap e834: 8 total, 8 up, 8 in 2024-09-21T09:44:37.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:44:37 smithi022 ceph-mon[98615]: osdmap e834: 8 total, 8 up, 8 in 2024-09-21T09:44:37.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:44:37 smithi022 ceph-mon[96736]: osdmap e834: 8 total, 8 up, 8 in 2024-09-21T09:44:38.131 INFO:tasks.rados.rados.0.smithi022.stdout:3835: done (0 left) 2024-09-21T09:44:38.132 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 19 current snap is 382 2024-09-21T09:44:38.132 INFO:tasks.rados.rados.0.smithi022.stdout:3836: seq_num 1203 ranges {0=557056,557056=401408} 2024-09-21T09:44:38.135 INFO:tasks.rados.rados.0.smithi022.stdout:3836: writing smithi02284960-19 from 0 to 557056 tid 1 2024-09-21T09:44:38.137 INFO:tasks.rados.rados.0.smithi022.stdout:3836: writing smithi02284960-19 from 557056 to 958464 tid 2 2024-09-21T09:44:38.138 INFO:tasks.rados.rados.0.smithi022.stdout:3837: copy_from oid 20 from oid 49 current snap is 382 2024-09-21T09:44:38.138 INFO:tasks.rados.rados.0.smithi022.stdout:3838: snap_remove snap 359 2024-09-21T09:44:38.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:44:38 smithi064 ceph-mon[86709]: pgmap v865: 121 pgs: 121 active+clean; 144 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 391 KiB/s rd, 1.5 MiB/s wr, 14 op/s 2024-09-21T09:44:38.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:44:38 smithi064 ceph-mon[86709]: osdmap e835: 8 total, 8 up, 8 in 2024-09-21T09:44:38.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:44:38 smithi022 ceph-mon[96736]: pgmap v865: 121 pgs: 121 active+clean; 144 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 391 KiB/s rd, 1.5 MiB/s wr, 14 op/s 2024-09-21T09:44:38.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:44:38 smithi022 ceph-mon[96736]: osdmap e835: 8 total, 8 up, 8 in 2024-09-21T09:44:38.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:44:38 smithi022 ceph-mon[98615]: pgmap v865: 121 pgs: 121 active+clean; 144 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 391 KiB/s rd, 1.5 MiB/s wr, 14 op/s 2024-09-21T09:44:38.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:44:38 smithi022 ceph-mon[98615]: osdmap e835: 8 total, 8 up, 8 in 2024-09-21T09:44:39.145 INFO:tasks.rados.rados.0.smithi022.stdout:3837: finishing copy_from to smithi02284960-20 2024-09-21T09:44:39.145 INFO:tasks.rados.rados.0.smithi022.stdout:3837: got expected ENOENT (src dne) 2024-09-21T09:44:39.145 INFO:tasks.rados.rados.0.smithi022.stdout:3837: finishing copy_from racing read to smithi02284960-20 2024-09-21T09:44:39.145 INFO:tasks.rados.rados.0.smithi022.stdout:3836: finishing write tid 1 to smithi02284960-19 2024-09-21T09:44:39.146 INFO:tasks.rados.rados.0.smithi022.stdout:3836: finishing write tid 2 to smithi02284960-19 2024-09-21T09:44:39.146 INFO:tasks.rados.rados.0.smithi022.stdout:3836: finishing write tid 3 to smithi02284960-19 2024-09-21T09:44:39.146 INFO:tasks.rados.rados.0.smithi022.stdout:3836: finishing write tid 4 to smithi02284960-19 2024-09-21T09:44:39.146 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 19 v 2147 (ObjNum 1203 snap 382 seq_num 1203) dirty exists 2024-09-21T09:44:39.146 INFO:tasks.rados.rados.0.smithi022.stdout:3836: left oid 19 (ObjNum 1203 snap 382 seq_num 1203) 2024-09-21T09:44:39.146 INFO:tasks.rados.rados.0.smithi022.stdout:3836: done (2 left) 2024-09-21T09:44:39.146 INFO:tasks.rados.rados.0.smithi022.stdout:3837: done (1 left) 2024-09-21T09:44:39.146 INFO:tasks.rados.rados.0.smithi022.stdout:3838: done (0 left) 2024-09-21T09:44:39.146 INFO:tasks.rados.rados.0.smithi022.stdout:3839: rollback oid 48 current snap is 382 2024-09-21T09:44:39.146 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 48 to 370 2024-09-21T09:44:39.146 INFO:tasks.rados.rados.0.smithi022.stdout:3840: copy_from oid 27 from oid 45 current snap is 382 2024-09-21T09:44:39.146 INFO:tasks.rados.rados.0.smithi022.stdout:3841: delete oid 39 current snap is 382 2024-09-21T09:44:39.156 INFO:tasks.rados.rados.0.smithi022.stdout:3841: done (2 left) 2024-09-21T09:44:39.156 INFO:tasks.rados.rados.0.smithi022.stdout:3842: rollback oid 38 current snap is 382 2024-09-21T09:44:39.156 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 38 to 374 2024-09-21T09:44:39.156 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 23 current snap is 382 2024-09-21T09:44:39.156 INFO:tasks.rados.rados.0.smithi022.stdout:3843: seq_num 1205 ranges {2809856=589824,3399680=401408} 2024-09-21T09:44:39.171 INFO:tasks.rados.rados.0.smithi022.stdout:3843: writing smithi02284960-23 from 2809856 to 3399680 tid 1 2024-09-21T09:44:39.174 INFO:tasks.rados.rados.0.smithi022.stdout:3843: writing smithi02284960-23 from 3399680 to 3801088 tid 2 2024-09-21T09:44:39.174 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 42 current snap is 382 2024-09-21T09:44:39.174 INFO:tasks.rados.rados.0.smithi022.stdout:3840: finishing copy_from to smithi02284960-27 2024-09-21T09:44:39.174 INFO:tasks.rados.rados.0.smithi022.stdout:3840: got expected ENOENT (src dne) 2024-09-21T09:44:39.174 INFO:tasks.rados.rados.0.smithi022.stdout:3844: seq_num 1206 ranges {0=606208,606208=401408} 2024-09-21T09:44:39.177 INFO:tasks.rados.rados.0.smithi022.stdout:3844: writing smithi02284960-42 from 0 to 606208 tid 1 2024-09-21T09:44:39.180 INFO:tasks.rados.rados.0.smithi022.stdout:3844: writing smithi02284960-42 from 606208 to 1007616 tid 2 2024-09-21T09:44:39.180 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 22 current snap is 382 2024-09-21T09:44:39.180 INFO:tasks.rados.rados.0.smithi022.stdout:3840: finishing copy_from racing read to smithi02284960-27 2024-09-21T09:44:39.180 INFO:tasks.rados.rados.0.smithi022.stdout:3842: finishing rollback tid 0 to smithi02284960-38 2024-09-21T09:44:39.181 INFO:tasks.rados.rados.0.smithi022.stdout:3842: finishing rollback tid 1 to smithi02284960-38 2024-09-21T09:44:39.181 INFO:tasks.rados.rados.0.smithi022.stdout:3842: finishing rollback tid 2 to smithi02284960-38 2024-09-21T09:44:39.181 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 38 v 1858 (ObjNum 1141 snap 350 seq_num 1141) dirty exists 2024-09-21T09:44:39.181 INFO:tasks.rados.rados.0.smithi022.stdout:3839: finishing rollback tid 1 to smithi02284960-48 2024-09-21T09:44:39.181 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 48 v 0 (ObjNum 65 snap 0 seq_num 1885692788) dirty dne 2024-09-21T09:44:39.181 INFO:tasks.rados.rados.0.smithi022.stdout:3845: seq_num 1207 ranges {4202496=622592,4825088=401408} 2024-09-21T09:44:39.207 INFO:tasks.rados.rados.0.smithi022.stdout:3845: writing smithi02284960-22 from 4202496 to 4825088 tid 1 2024-09-21T09:44:39.209 INFO:tasks.rados.rados.0.smithi022.stdout:3845: writing smithi02284960-22 from 4825088 to 5226496 tid 2 2024-09-21T09:44:39.209 INFO:tasks.rados.rados.0.smithi022.stdout:3839: done (5 left) 2024-09-21T09:44:39.209 INFO:tasks.rados.rados.0.smithi022.stdout:3840: done (4 left) 2024-09-21T09:44:39.209 INFO:tasks.rados.rados.0.smithi022.stdout:3842: done (3 left) 2024-09-21T09:44:39.209 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 45 current snap is 382 2024-09-21T09:44:39.209 INFO:tasks.rados.rados.0.smithi022.stdout:3846: seq_num 1208 ranges {0=638976,638976=401408} 2024-09-21T09:44:39.212 INFO:tasks.rados.rados.0.smithi022.stdout:3846: writing smithi02284960-45 from 0 to 638976 tid 1 2024-09-21T09:44:39.216 INFO:tasks.rados.rados.0.smithi022.stdout:3846: writing smithi02284960-45 from 638976 to 1040384 tid 2 2024-09-21T09:44:39.216 INFO:tasks.rados.rados.0.smithi022.stdout:3847: delete oid 47 current snap is 382 2024-09-21T09:44:39.216 INFO:tasks.rados.rados.0.smithi022.stdout:3843: finishing write tid 1 to smithi02284960-23 2024-09-21T09:44:39.216 INFO:tasks.rados.rados.0.smithi022.stdout:3843: finishing write tid 2 to smithi02284960-23 2024-09-21T09:44:39.216 INFO:tasks.rados.rados.0.smithi022.stdout:3843: finishing write tid 3 to smithi02284960-23 2024-09-21T09:44:39.216 INFO:tasks.rados.rados.0.smithi022.stdout:3843: finishing write tid 4 to smithi02284960-23 2024-09-21T09:44:39.216 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 23 v 1861 (ObjNum 1205 snap 382 seq_num 1205) dirty exists 2024-09-21T09:44:39.216 INFO:tasks.rados.rados.0.smithi022.stdout:3843: left oid 23 (ObjNum 1205 snap 382 seq_num 1205) 2024-09-21T09:44:39.216 INFO:tasks.rados.rados.0.smithi022.stdout:3844: finishing write tid 1 to smithi02284960-42 2024-09-21T09:44:39.216 INFO:tasks.rados.rados.0.smithi022.stdout:3844: finishing write tid 2 to smithi02284960-42 2024-09-21T09:44:39.216 INFO:tasks.rados.rados.0.smithi022.stdout:3844: finishing write tid 3 to smithi02284960-42 2024-09-21T09:44:39.216 INFO:tasks.rados.rados.0.smithi022.stdout:3844: finishing write tid 4 to smithi02284960-42 2024-09-21T09:44:39.216 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 42 v 1996 (ObjNum 1206 snap 382 seq_num 1206) dirty exists 2024-09-21T09:44:39.216 INFO:tasks.rados.rados.0.smithi022.stdout:3844: left oid 42 (ObjNum 1206 snap 382 seq_num 1206) 2024-09-21T09:44:39.217 INFO:tasks.rados.rados.0.smithi022.stdout:3845: finishing write tid 1 to smithi02284960-22 2024-09-21T09:44:39.217 INFO:tasks.rados.rados.0.smithi022.stdout:3845: finishing write tid 2 to smithi02284960-22 2024-09-21T09:44:39.217 INFO:tasks.rados.rados.0.smithi022.stdout:3845: finishing write tid 3 to smithi02284960-22 2024-09-21T09:44:39.217 INFO:tasks.rados.rados.0.smithi022.stdout:3845: finishing write tid 4 to smithi02284960-22 2024-09-21T09:44:39.217 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 22 v 1999 (ObjNum 1207 snap 382 seq_num 1207) dirty exists 2024-09-21T09:44:39.217 INFO:tasks.rados.rados.0.smithi022.stdout:3845: left oid 22 (ObjNum 1207 snap 382 seq_num 1207) 2024-09-21T09:44:39.218 INFO:tasks.rados.rados.0.smithi022.stdout:3843: done (4 left) 2024-09-21T09:44:39.218 INFO:tasks.rados.rados.0.smithi022.stdout:3844: done (3 left) 2024-09-21T09:44:39.218 INFO:tasks.rados.rados.0.smithi022.stdout:3845: done (2 left) 2024-09-21T09:44:39.218 INFO:tasks.rados.rados.0.smithi022.stdout:3847: done (1 left) 2024-09-21T09:44:39.218 INFO:tasks.rados.rados.0.smithi022.stdout:3848: delete oid 1 current snap is 382 2024-09-21T09:44:39.218 INFO:tasks.rados.rados.0.smithi022.stdout:3846: finishing write tid 1 to smithi02284960-45 2024-09-21T09:44:39.219 INFO:tasks.rados.rados.0.smithi022.stdout:3846: finishing write tid 2 to smithi02284960-45 2024-09-21T09:44:39.219 INFO:tasks.rados.rados.0.smithi022.stdout:3846: finishing write tid 3 to smithi02284960-45 2024-09-21T09:44:39.220 INFO:tasks.rados.rados.0.smithi022.stdout:3846: finishing write tid 4 to smithi02284960-45 2024-09-21T09:44:39.220 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 45 v 1898 (ObjNum 1208 snap 382 seq_num 1208) dirty exists 2024-09-21T09:44:39.220 INFO:tasks.rados.rados.0.smithi022.stdout:3846: left oid 45 (ObjNum 1208 snap 382 seq_num 1208) 2024-09-21T09:44:39.221 INFO:tasks.rados.rados.0.smithi022.stdout:3846: done (1 left) 2024-09-21T09:44:39.221 INFO:tasks.rados.rados.0.smithi022.stdout:3848: done (0 left) 2024-09-21T09:44:39.221 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 37 current snap is 382 2024-09-21T09:44:39.221 INFO:tasks.rados.rados.0.smithi022.stdout:3849: seq_num 1211 ranges {4857856=688128,5545984=401408} 2024-09-21T09:44:39.251 INFO:tasks.rados.rados.0.smithi022.stdout:3849: writing smithi02284960-37 from 4857856 to 5545984 tid 1 2024-09-21T09:44:39.254 INFO:tasks.rados.rados.0.smithi022.stdout:3849: writing smithi02284960-37 from 5545984 to 5947392 tid 2 2024-09-21T09:44:39.254 INFO:tasks.rados.rados.0.smithi022.stdout:3850: snap_create 2024-09-21T09:44:39.258 INFO:tasks.rados.rados.0.smithi022.stdout:3849: finishing write tid 1 to smithi02284960-37 2024-09-21T09:44:39.258 INFO:tasks.rados.rados.0.smithi022.stdout:3849: finishing write tid 2 to smithi02284960-37 2024-09-21T09:44:39.258 INFO:tasks.rados.rados.0.smithi022.stdout:3849: finishing write tid 3 to smithi02284960-37 2024-09-21T09:44:39.260 INFO:tasks.rados.rados.0.smithi022.stdout:3849: finishing write tid 4 to smithi02284960-37 2024-09-21T09:44:39.260 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 37 v 2155 (ObjNum 1211 snap 382 seq_num 1211) dirty exists 2024-09-21T09:44:39.260 INFO:tasks.rados.rados.0.smithi022.stdout:3849: left oid 37 (ObjNum 1211 snap 382 seq_num 1211) 2024-09-21T09:44:39.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:44:39 smithi064 ceph-mon[86709]: osdmap e836: 8 total, 8 up, 8 in 2024-09-21T09:44:39.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:44:39 smithi064 ceph-mon[86709]: pgmap v868: 121 pgs: 121 active+clean; 144 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 391 KiB/s rd, 854 KiB/s wr, 9 op/s 2024-09-21T09:44:39.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:44:39 smithi022 ceph-mon[96736]: osdmap e836: 8 total, 8 up, 8 in 2024-09-21T09:44:39.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:44:39 smithi022 ceph-mon[96736]: pgmap v868: 121 pgs: 121 active+clean; 144 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 391 KiB/s rd, 854 KiB/s wr, 9 op/s 2024-09-21T09:44:39.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:44:39 smithi022 ceph-mon[98615]: osdmap e836: 8 total, 8 up, 8 in 2024-09-21T09:44:39.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:44:39 smithi022 ceph-mon[98615]: pgmap v868: 121 pgs: 121 active+clean; 144 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 391 KiB/s rd, 854 KiB/s wr, 9 op/s 2024-09-21T09:44:40.149 INFO:tasks.rados.rados.0.smithi022.stdout:3849: done (1 left) 2024-09-21T09:44:40.149 INFO:tasks.rados.rados.0.smithi022.stdout:3850: done (0 left) 2024-09-21T09:44:40.149 INFO:tasks.rados.rados.0.smithi022.stdout:3851: read oid 25 snap -1 2024-09-21T09:44:40.149 INFO:tasks.rados.rados.0.smithi022.stdout:3851: expect (ObjNum 1199 snap 379 seq_num 1199) 2024-09-21T09:44:40.149 INFO:tasks.rados.rados.0.smithi022.stdout:3852: rollback oid 21 current snap is 383 2024-09-21T09:44:40.149 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 21 to 376 2024-09-21T09:44:40.149 INFO:tasks.rados.rados.0.smithi022.stdout:3853: read oid 35 snap 369 2024-09-21T09:44:40.150 INFO:tasks.rados.rados.0.smithi022.stdout:3853: expect (ObjNum 1137 snap 349 seq_num 1137) 2024-09-21T09:44:40.150 INFO:tasks.rados.rados.0.smithi022.stdout:3854: snap_create 2024-09-21T09:44:40.154 INFO:tasks.rados.rados.0.smithi022.stdout:3852: finishing rollback tid 1 to smithi02284960-21 2024-09-21T09:44:40.154 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 21 v 1852 (ObjNum 49 snap 0 seq_num 1449218391) dirty dne 2024-09-21T09:44:40.163 INFO:tasks.rados.rados.0.smithi022.stdout:3851: expect (ObjNum 1199 snap 379 seq_num 1199) 2024-09-21T09:44:40.198 INFO:tasks.rados.rados.0.smithi022.stdout:3853: expect (ObjNum 1137 snap 349 seq_num 1137) 2024-09-21T09:44:40.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:44:40 smithi064 ceph-mon[86709]: osdmap e837: 8 total, 8 up, 8 in 2024-09-21T09:44:40.442 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:44:40 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:44:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:44:40.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:44:40 smithi022 ceph-mon[98615]: osdmap e837: 8 total, 8 up, 8 in 2024-09-21T09:44:40.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:44:40 smithi022 ceph-mon[96736]: osdmap e837: 8 total, 8 up, 8 in 2024-09-21T09:44:41.154 INFO:tasks.rados.rados.0.smithi022.stdout:3851: done (3 left) 2024-09-21T09:44:41.154 INFO:tasks.rados.rados.0.smithi022.stdout:3852: done (2 left) 2024-09-21T09:44:41.154 INFO:tasks.rados.rados.0.smithi022.stdout:3853: done (1 left) 2024-09-21T09:44:41.154 INFO:tasks.rados.rados.0.smithi022.stdout:3854: done (0 left) 2024-09-21T09:44:41.154 INFO:tasks.rados.rados.0.smithi022.stdout:3855: setattr oid 4 current snap is 384 2024-09-21T09:44:41.155 INFO:tasks.rados.rados.0.smithi022.stdout:3856: setattr oid 39 current snap is 384 2024-09-21T09:44:41.158 INFO:tasks.rados.rados.0.smithi022.stdout:3857: snap_create 2024-09-21T09:44:41.159 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 4 v 2001 (ObjNum 1166 snap 359 seq_num 1166) dirty exists 2024-09-21T09:44:41.162 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 39 v 2151 (ObjNum 161 snap 0 seq_num 0) dirty exists 2024-09-21T09:44:41.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:44:41 smithi064 ceph-mon[86709]: osdmap e838: 8 total, 8 up, 8 in 2024-09-21T09:44:41.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:44:41 smithi064 ceph-mon[86709]: pgmap v871: 121 pgs: 121 active+clean; 144 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 1 op/s 2024-09-21T09:44:41.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:44:41 smithi022 ceph-mon[98615]: osdmap e838: 8 total, 8 up, 8 in 2024-09-21T09:44:41.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:44:41 smithi022 ceph-mon[98615]: pgmap v871: 121 pgs: 121 active+clean; 144 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 1 op/s 2024-09-21T09:44:41.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:44:41 smithi022 ceph-mon[96736]: osdmap e838: 8 total, 8 up, 8 in 2024-09-21T09:44:41.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:44:41 smithi022 ceph-mon[96736]: pgmap v871: 121 pgs: 121 active+clean; 144 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 1 op/s 2024-09-21T09:44:42.159 INFO:tasks.rados.rados.0.smithi022.stdout:3855: done (2 left) 2024-09-21T09:44:42.160 INFO:tasks.rados.rados.0.smithi022.stdout:3856: done (1 left) 2024-09-21T09:44:42.160 INFO:tasks.rados.rados.0.smithi022.stdout:3857: done (0 left) 2024-09-21T09:44:42.160 INFO:tasks.rados.rados.0.smithi022.stdout:3858: rollback oid 29 current snap is 385 2024-09-21T09:44:42.160 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 29 to 367 2024-09-21T09:44:42.160 INFO:tasks.rados.rados.0.smithi022.stdout:3859: rollback oid 14 current snap is 385 2024-09-21T09:44:42.160 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 14 to 368 2024-09-21T09:44:42.160 INFO:tasks.rados.rados.0.smithi022.stdout:3860: snap_remove snap 377 2024-09-21T09:44:42.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:44:42 smithi064 ceph-mon[86709]: osdmap e839: 8 total, 8 up, 8 in 2024-09-21T09:44:42.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:44:42 smithi022 ceph-mon[98615]: osdmap e839: 8 total, 8 up, 8 in 2024-09-21T09:44:42.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:44:42 smithi022 ceph-mon[96736]: osdmap e839: 8 total, 8 up, 8 in 2024-09-21T09:44:43.166 INFO:tasks.rados.rados.0.smithi022.stdout:3860: done (2 left) 2024-09-21T09:44:43.167 INFO:tasks.rados.rados.0.smithi022.stdout:3861: rmattr oid 43 current snap is 385 2024-09-21T09:44:43.167 INFO:tasks.rados.rados.0.smithi022.stdout:3859: finishing rollback tid 1 to smithi02284960-14 2024-09-21T09:44:43.167 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 14 v 0 (ObjNum 161 snap 0 seq_num 1) dirty dne 2024-09-21T09:44:43.167 INFO:tasks.rados.rados.0.smithi022.stdout:3858: finishing rollback tid 0 to smithi02284960-29 2024-09-21T09:44:43.167 INFO:tasks.rados.rados.0.smithi022.stdout:3858: finishing rollback tid 1 to smithi02284960-29 2024-09-21T09:44:43.167 INFO:tasks.rados.rados.0.smithi022.stdout:3858: finishing rollback tid 2 to smithi02284960-29 2024-09-21T09:44:43.167 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 29 v 1864 (ObjNum 1165 snap 358 seq_num 1165) dirty exists 2024-09-21T09:44:43.167 INFO:tasks.rados.rados.0.smithi022.stdout:3858: done (2 left) 2024-09-21T09:44:43.167 INFO:tasks.rados.rados.0.smithi022.stdout:3859: done (1 left) 2024-09-21T09:44:43.167 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 4 current snap is 385 2024-09-21T09:44:43.167 INFO:tasks.rados.rados.0.smithi022.stdout:3862: seq_num 1212 ranges {532480=704512,1236992=401408} 2024-09-21T09:44:43.173 INFO:tasks.rados.rados.0.smithi022.stdout:3862: writing smithi02284960-4 from 532480 to 1236992 tid 1 2024-09-21T09:44:43.176 INFO:tasks.rados.rados.0.smithi022.stdout:3862: writing smithi02284960-4 from 1236992 to 1638400 tid 2 2024-09-21T09:44:43.176 INFO:tasks.rados.rados.0.smithi022.stdout:3863: read oid 23 snap -1 2024-09-21T09:44:43.176 INFO:tasks.rados.rados.0.smithi022.stdout:3863: expect (ObjNum 1205 snap 382 seq_num 1205) 2024-09-21T09:44:43.176 INFO:tasks.rados.rados.0.smithi022.stdout:3864: copy_from oid 11 from oid 36 current snap is 385 2024-09-21T09:44:43.177 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 43 v 2113 (ObjNum 273 snap 0 seq_num 0) dirty exists 2024-09-21T09:44:43.177 INFO:tasks.rados.rados.0.smithi022.stdout:3861: done (3 left) 2024-09-21T09:44:43.177 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 10 current snap is 385 2024-09-21T09:44:43.177 INFO:tasks.rados.rados.0.smithi022.stdout:3865: seq_num 1213 ranges {3981312=720896,4702208=401408} 2024-09-21T09:44:43.203 INFO:tasks.rados.rados.0.smithi022.stdout:3865: writing smithi02284960-10 from 3981312 to 4702208 tid 1 2024-09-21T09:44:43.207 INFO:tasks.rados.rados.0.smithi022.stdout:3865: writing smithi02284960-10 from 4702208 to 5103616 tid 2 2024-09-21T09:44:43.208 INFO:tasks.rados.rados.0.smithi022.stdout:3866: copy_from oid 35 from oid 38 current snap is 385 2024-09-21T09:44:43.208 INFO:tasks.rados.rados.0.smithi022.stdout:3862: finishing write tid 1 to smithi02284960-4 2024-09-21T09:44:43.208 INFO:tasks.rados.rados.0.smithi022.stdout:3862: finishing write tid 2 to smithi02284960-4 2024-09-21T09:44:43.208 INFO:tasks.rados.rados.0.smithi022.stdout:3862: finishing write tid 3 to smithi02284960-4 2024-09-21T09:44:43.208 INFO:tasks.rados.rados.0.smithi022.stdout:3862: finishing write tid 4 to smithi02284960-4 2024-09-21T09:44:43.208 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 4 v 2004 (ObjNum 1212 snap 385 seq_num 1212) dirty exists 2024-09-21T09:44:43.209 INFO:tasks.rados.rados.0.smithi022.stdout:3862: left oid 4 (ObjNum 1212 snap 385 seq_num 1212) 2024-09-21T09:44:43.209 INFO:tasks.rados.rados.0.smithi022.stdout:3863: expect (ObjNum 1205 snap 382 seq_num 1205) 2024-09-21T09:44:43.404 INFO:tasks.rados.rados.0.smithi022.stdout:3864: finishing copy_from racing read to smithi02284960-11 2024-09-21T09:44:43.404 INFO:tasks.rados.rados.0.smithi022.stdout:3865: finishing write tid 1 to smithi02284960-10 2024-09-21T09:44:43.404 INFO:tasks.rados.rados.0.smithi022.stdout:3865: finishing write tid 2 to smithi02284960-10 2024-09-21T09:44:43.404 INFO:tasks.rados.rados.0.smithi022.stdout:3865: finishing write tid 3 to smithi02284960-10 2024-09-21T09:44:43.404 INFO:tasks.rados.rados.0.smithi022.stdout:3864: finishing copy_from to smithi02284960-11 2024-09-21T09:44:43.405 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 11 v 1895 (ObjNum 1117 snap 341 seq_num 1117) dirty exists 2024-09-21T09:44:43.405 INFO:tasks.rados.rados.0.smithi022.stdout:3866: finishing copy_from racing read to smithi02284960-35 2024-09-21T09:44:43.405 INFO:tasks.rados.rados.0.smithi022.stdout:3865: finishing write tid 4 to smithi02284960-10 2024-09-21T09:44:43.405 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 10 v 2116 (ObjNum 1213 snap 385 seq_num 1213) dirty exists 2024-09-21T09:44:43.405 INFO:tasks.rados.rados.0.smithi022.stdout:3865: left oid 10 (ObjNum 1213 snap 385 seq_num 1213) 2024-09-21T09:44:43.405 INFO:tasks.rados.rados.0.smithi022.stdout:3866: finishing copy_from to smithi02284960-35 2024-09-21T09:44:43.405 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 35 v 1858 (ObjNum 1141 snap 350 seq_num 1141) dirty exists 2024-09-21T09:44:43.405 INFO:tasks.rados.rados.0.smithi022.stdout:3862: done (4 left) 2024-09-21T09:44:43.405 INFO:tasks.rados.rados.0.smithi022.stdout:3863: done (3 left) 2024-09-21T09:44:43.405 INFO:tasks.rados.rados.0.smithi022.stdout:3864: done (2 left) 2024-09-21T09:44:43.405 INFO:tasks.rados.rados.0.smithi022.stdout:3865: done (1 left) 2024-09-21T09:44:43.405 INFO:tasks.rados.rados.0.smithi022.stdout:3866: done (0 left) 2024-09-21T09:44:43.405 INFO:tasks.rados.rados.0.smithi022.stdout:3867: read oid 19 snap -1 2024-09-21T09:44:43.405 INFO:tasks.rados.rados.0.smithi022.stdout:3867: expect (ObjNum 1203 snap 382 seq_num 1203) 2024-09-21T09:44:43.406 INFO:tasks.rados.rados.0.smithi022.stdout:3868: read oid 35 snap 366 2024-09-21T09:44:43.406 INFO:tasks.rados.rados.0.smithi022.stdout:3868: expect (ObjNum 1137 snap 349 seq_num 1137) 2024-09-21T09:44:43.406 INFO:tasks.rados.rados.0.smithi022.stdout:3869: snap_create 2024-09-21T09:44:43.411 INFO:tasks.rados.rados.0.smithi022.stdout:3867: expect (ObjNum 1203 snap 382 seq_num 1203) 2024-09-21T09:44:43.438 INFO:tasks.rados.rados.0.smithi022.stdout:3868: expect (ObjNum 1137 snap 349 seq_num 1137) 2024-09-21T09:44:43.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:44:43 smithi064 ceph-mon[86709]: osdmap e840: 8 total, 8 up, 8 in 2024-09-21T09:44:43.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:44:43 smithi064 ceph-mon[86709]: pgmap v874: 121 pgs: 121 active+clean; 141 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 1023 B/s rd, 996 KiB/s wr, 5 op/s 2024-09-21T09:44:43.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:44:43 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:44:43.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:44:43 smithi022 ceph-mon[98615]: osdmap e840: 8 total, 8 up, 8 in 2024-09-21T09:44:43.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:44:43 smithi022 ceph-mon[98615]: pgmap v874: 121 pgs: 121 active+clean; 141 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 1023 B/s rd, 996 KiB/s wr, 5 op/s 2024-09-21T09:44:43.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:44:43 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:44:43.558 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 21 09:44:43 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-2[113367]: 2024-09-21T09:44:43.191+0000 7f02dd35f640 -1 snap_mapper.get_next_objects_to_trim::New Clone-Objects were added to Snap 1d2 after trimming was started 2024-09-21T09:44:43.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:44:43 smithi022 ceph-mon[96736]: osdmap e840: 8 total, 8 up, 8 in 2024-09-21T09:44:43.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:44:43 smithi022 ceph-mon[96736]: pgmap v874: 121 pgs: 121 active+clean; 141 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 1023 B/s rd, 996 KiB/s wr, 5 op/s 2024-09-21T09:44:43.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:44:43 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:44:44.116 INFO:tasks.rados.rados.0.smithi022.stdout:3867: done (2 left) 2024-09-21T09:44:44.116 INFO:tasks.rados.rados.0.smithi022.stdout:3868: done (1 left) 2024-09-21T09:44:44.116 INFO:tasks.rados.rados.0.smithi022.stdout:3869: done (0 left) 2024-09-21T09:44:44.116 INFO:tasks.rados.rados.0.smithi022.stdout:3870: read oid 14 snap 375 2024-09-21T09:44:44.116 INFO:tasks.rados.rados.0.smithi022.stdout:3870: expect deleted 2024-09-21T09:44:44.116 INFO:tasks.rados.rados.0.smithi022.stdout:3871: rollback oid 41 current snap is 386 2024-09-21T09:44:44.116 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 41 to 366 2024-09-21T09:44:44.117 INFO:tasks.rados.rados.0.smithi022.stdout:3872: delete oid 27 current snap is 386 2024-09-21T09:44:44.122 INFO:tasks.rados.rados.0.smithi022.stdout:3870: done (2 left) 2024-09-21T09:44:44.122 INFO:tasks.rados.rados.0.smithi022.stdout:3872: done (1 left) 2024-09-21T09:44:44.123 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 3 current snap is 386 2024-09-21T09:44:44.123 INFO:tasks.rados.rados.0.smithi022.stdout:3873: seq_num 1215 ranges {4440064=753664,5193728=401408} 2024-09-21T09:44:44.151 INFO:tasks.rados.rados.0.smithi022.stdout:3873: writing smithi02284960-3 from 4440064 to 5193728 tid 1 2024-09-21T09:44:44.153 INFO:tasks.rados.rados.0.smithi022.stdout:3873: writing smithi02284960-3 from 5193728 to 5595136 tid 2 2024-09-21T09:44:44.153 INFO:tasks.rados.rados.0.smithi022.stdout:3874: rmattr oid 42 current snap is 386 2024-09-21T09:44:44.153 INFO:tasks.rados.rados.0.smithi022.stdout:3871: finishing rollback tid 0 to smithi02284960-41 2024-09-21T09:44:44.153 INFO:tasks.rados.rados.0.smithi022.stdout:3871: finishing rollback tid 1 to smithi02284960-41 2024-09-21T09:44:44.153 INFO:tasks.rados.rados.0.smithi022.stdout:3871: finishing rollback tid 2 to smithi02284960-41 2024-09-21T09:44:44.153 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 41 v 2119 (ObjNum 1120 snap 341 seq_num 1120) dirty exists 2024-09-21T09:44:44.154 INFO:tasks.rados.rados.0.smithi022.stdout:3871: done (2 left) 2024-09-21T09:44:44.154 INFO:tasks.rados.rados.0.smithi022.stdout:3874: done (1 left) 2024-09-21T09:44:44.154 INFO:tasks.rados.rados.0.smithi022.stdout:3875: snap_remove snap 366 2024-09-21T09:44:44.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:44:44 smithi064 ceph-mon[86709]: osdmap e841: 8 total, 8 up, 8 in 2024-09-21T09:44:44.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:44:44 smithi064 ceph-mon[86709]: osdmap e842: 8 total, 8 up, 8 in 2024-09-21T09:44:44.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:44:44 smithi022 ceph-mon[98615]: osdmap e841: 8 total, 8 up, 8 in 2024-09-21T09:44:44.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:44:44 smithi022 ceph-mon[98615]: osdmap e842: 8 total, 8 up, 8 in 2024-09-21T09:44:44.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:44:44 smithi022 ceph-mon[96736]: osdmap e841: 8 total, 8 up, 8 in 2024-09-21T09:44:44.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:44:44 smithi022 ceph-mon[96736]: osdmap e842: 8 total, 8 up, 8 in 2024-09-21T09:44:45.117 INFO:tasks.rados.rados.0.smithi022.stdout:3875: done (1 left) 2024-09-21T09:44:45.117 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 21 current snap is 386 2024-09-21T09:44:45.118 INFO:tasks.rados.rados.0.smithi022.stdout:3876: seq_num 1216 ranges {0=770048,770048=401408} 2024-09-21T09:44:45.122 INFO:tasks.rados.rados.0.smithi022.stdout:3876: writing smithi02284960-21 from 0 to 770048 tid 1 2024-09-21T09:44:45.125 INFO:tasks.rados.rados.0.smithi022.stdout:3876: writing smithi02284960-21 from 770048 to 1171456 tid 2 2024-09-21T09:44:45.125 INFO:tasks.rados.rados.0.smithi022.stdout:3877: snap_remove snap 368 2024-09-21T09:44:46.058 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:44:45 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:44:45] "GET /metrics HTTP/1.1" 200 36352 "" "Prometheus/2.51.0" 2024-09-21T09:44:46.123 INFO:tasks.rados.rados.0.smithi022.stdout:3877: done (2 left) 2024-09-21T09:44:46.123 INFO:tasks.rados.rados.0.smithi022.stdout:3878: delete oid 46 current snap is 386 2024-09-21T09:44:46.123 INFO:tasks.rados.rados.0.smithi022.stdout:3873: finishing write tid 1 to smithi02284960-3 2024-09-21T09:44:46.123 INFO:tasks.rados.rados.0.smithi022.stdout:3873: finishing write tid 2 to smithi02284960-3 2024-09-21T09:44:46.123 INFO:tasks.rados.rados.0.smithi022.stdout:3873: finishing write tid 3 to smithi02284960-3 2024-09-21T09:44:46.123 INFO:tasks.rados.rados.0.smithi022.stdout:3873: finishing write tid 4 to smithi02284960-3 2024-09-21T09:44:46.123 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 3 v 640 (ObjNum 1215 snap 386 seq_num 1215) dirty exists 2024-09-21T09:44:46.123 INFO:tasks.rados.rados.0.smithi022.stdout:3873: left oid 3 (ObjNum 1215 snap 386 seq_num 1215) 2024-09-21T09:44:46.123 INFO:tasks.rados.rados.0.smithi022.stdout:3876: finishing write tid 1 to smithi02284960-21 2024-09-21T09:44:46.123 INFO:tasks.rados.rados.0.smithi022.stdout:3876: finishing write tid 2 to smithi02284960-21 2024-09-21T09:44:46.123 INFO:tasks.rados.rados.0.smithi022.stdout:3876: finishing write tid 3 to smithi02284960-21 2024-09-21T09:44:46.124 INFO:tasks.rados.rados.0.smithi022.stdout:3876: finishing write tid 4 to smithi02284960-21 2024-09-21T09:44:46.124 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 21 v 1855 (ObjNum 1216 snap 386 seq_num 1216) dirty exists 2024-09-21T09:44:46.124 INFO:tasks.rados.rados.0.smithi022.stdout:3876: left oid 21 (ObjNum 1216 snap 386 seq_num 1216) 2024-09-21T09:44:46.133 INFO:tasks.rados.rados.0.smithi022.stdout:3873: done (2 left) 2024-09-21T09:44:46.133 INFO:tasks.rados.rados.0.smithi022.stdout:3876: done (1 left) 2024-09-21T09:44:46.133 INFO:tasks.rados.rados.0.smithi022.stdout:3878: done (0 left) 2024-09-21T09:44:46.133 INFO:tasks.rados.rados.0.smithi022.stdout:3879: rollback oid 8 current snap is 386 2024-09-21T09:44:46.133 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 8 to 383 2024-09-21T09:44:46.133 INFO:tasks.rados.rados.0.smithi022.stdout:3880: read oid 28 snap -1 2024-09-21T09:44:46.133 INFO:tasks.rados.rados.0.smithi022.stdout:3880: expect (ObjNum 1156 snap 356 seq_num 1156) 2024-09-21T09:44:46.133 INFO:tasks.rados.rados.0.smithi022.stdout:3881: delete oid 43 current snap is 386 2024-09-21T09:44:46.140 INFO:tasks.rados.rados.0.smithi022.stdout:3881: done (2 left) 2024-09-21T09:44:46.140 INFO:tasks.rados.rados.0.smithi022.stdout:3882: snap_remove snap 370 2024-09-21T09:44:46.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:44:46 smithi064 ceph-mon[86709]: pgmap v877: 121 pgs: 121 active+clean; 138 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 5.0 MiB/s rd, 1.5 MiB/s wr, 8 op/s 2024-09-21T09:44:46.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:44:46 smithi064 ceph-mon[86709]: osdmap e843: 8 total, 8 up, 8 in 2024-09-21T09:44:46.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:44:46 smithi022 ceph-mon[96736]: pgmap v877: 121 pgs: 121 active+clean; 138 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 5.0 MiB/s rd, 1.5 MiB/s wr, 8 op/s 2024-09-21T09:44:46.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:44:46 smithi022 ceph-mon[96736]: osdmap e843: 8 total, 8 up, 8 in 2024-09-21T09:44:46.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:44:46 smithi022 ceph-mon[98615]: pgmap v877: 121 pgs: 121 active+clean; 138 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 5.0 MiB/s rd, 1.5 MiB/s wr, 8 op/s 2024-09-21T09:44:46.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:44:46 smithi022 ceph-mon[98615]: osdmap e843: 8 total, 8 up, 8 in 2024-09-21T09:44:47.132 INFO:tasks.rados.rados.0.smithi022.stdout:3882: done (2 left) 2024-09-21T09:44:47.132 INFO:tasks.rados.rados.0.smithi022.stdout:3883: rollback oid 48 current snap is 386 2024-09-21T09:44:47.132 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 48 to 375 2024-09-21T09:44:47.132 INFO:tasks.rados.rados.0.smithi022.stdout:3879: finishing rollback tid 1 to smithi02284960-8 2024-09-21T09:44:47.132 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 8 v 1996 (ObjNum 129 snap 0 seq_num 3569486840) dirty dne 2024-09-21T09:44:47.132 INFO:tasks.rados.rados.0.smithi022.stdout:3880: expect (ObjNum 1156 snap 356 seq_num 1156) 2024-09-21T09:44:47.263 INFO:tasks.rados.rados.0.smithi022.stdout:3883: finishing rollback tid 1 to smithi02284960-48 2024-09-21T09:44:47.263 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 48 v 0 (ObjNum 49 snap 0 seq_num 2955662579) dirty dne 2024-09-21T09:44:47.263 INFO:tasks.rados.rados.0.smithi022.stdout:3879: done (2 left) 2024-09-21T09:44:47.263 INFO:tasks.rados.rados.0.smithi022.stdout:3880: done (1 left) 2024-09-21T09:44:47.264 INFO:tasks.rados.rados.0.smithi022.stdout:3883: done (0 left) 2024-09-21T09:44:47.264 INFO:tasks.rados.rados.0.smithi022.stdout:3884: rollback oid 30 current snap is 386 2024-09-21T09:44:47.264 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 30 to 380 2024-09-21T09:44:47.264 INFO:tasks.rados.rados.0.smithi022.stdout:3885: rollback oid 27 current snap is 386 2024-09-21T09:44:47.264 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 27 to 384 2024-09-21T09:44:47.264 INFO:tasks.rados.rados.0.smithi022.stdout:3886: delete oid 24 current snap is 386 2024-09-21T09:44:47.267 INFO:tasks.rados.rados.0.smithi022.stdout:3884: finishing rollback tid 0 to smithi02284960-30 2024-09-21T09:44:47.267 INFO:tasks.rados.rados.0.smithi022.stdout:3885: finishing rollback tid 1 to smithi02284960-27 2024-09-21T09:44:47.267 INFO:tasks.rados.rados.0.smithi022.stdout:3884: finishing rollback tid 1 to smithi02284960-30 2024-09-21T09:44:47.267 INFO:tasks.rados.rados.0.smithi022.stdout:3885: finishing rollback tid 2 to smithi02284960-27 2024-09-21T09:44:47.267 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 27 v 2002 (ObjNum 1008 snap 309 seq_num 1008) dirty exists 2024-09-21T09:44:47.267 INFO:tasks.rados.rados.0.smithi022.stdout:3884: finishing rollback tid 2 to smithi02284960-30 2024-09-21T09:44:47.267 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 30 v 2154 (ObjNum 1193 snap 373 seq_num 1193) dirty exists 2024-09-21T09:44:47.268 INFO:tasks.rados.rados.0.smithi022.stdout:3884: done (2 left) 2024-09-21T09:44:47.268 INFO:tasks.rados.rados.0.smithi022.stdout:3885: done (1 left) 2024-09-21T09:44:47.268 INFO:tasks.rados.rados.0.smithi022.stdout:3886: done (0 left) 2024-09-21T09:44:47.268 INFO:tasks.rados.rados.0.smithi022.stdout:3887: delete oid 28 current snap is 386 2024-09-21T09:44:47.269 INFO:tasks.rados.rados.0.smithi022.stdout:3887: done (0 left) 2024-09-21T09:44:47.270 INFO:tasks.rados.rados.0.smithi022.stdout:3888: rmattr oid 44 current snap is 386 2024-09-21T09:44:47.270 INFO:tasks.rados.rados.0.smithi022.stdout:3888: done (0 left) 2024-09-21T09:44:47.270 INFO:tasks.rados.rados.0.smithi022.stdout:3889: read oid 7 snap -1 2024-09-21T09:44:47.270 INFO:tasks.rados.rados.0.smithi022.stdout:3889: expect (ObjNum 1193 snap 373 seq_num 1193) 2024-09-21T09:44:47.270 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 46 current snap is 386 2024-09-21T09:44:47.270 INFO:tasks.rados.rados.0.smithi022.stdout:3890: seq_num 1221 ranges {0=458752,458752=729088,1187840=73728} 2024-09-21T09:44:47.272 INFO:tasks.rados.rados.0.smithi022.stdout:3890: writing smithi02284960-46 from 0 to 458752 tid 1 2024-09-21T09:44:47.277 INFO:tasks.rados.rados.0.smithi022.stdout:3890: writing smithi02284960-46 from 458752 to 1187840 tid 2 2024-09-21T09:44:47.278 INFO:tasks.rados.rados.0.smithi022.stdout:3890: writing smithi02284960-46 from 1187840 to 1261568 tid 3 2024-09-21T09:44:47.278 INFO:tasks.rados.rados.0.smithi022.stdout:3891: copy_from oid 36 from oid 4 current snap is 386 2024-09-21T09:44:47.278 INFO:tasks.rados.rados.0.smithi022.stdout:3889: expect (ObjNum 1193 snap 373 seq_num 1193) 2024-09-21T09:44:47.302 INFO:tasks.rados.rados.0.smithi022.stdout:3890: finishing write tid 1 to smithi02284960-46 2024-09-21T09:44:47.302 INFO:tasks.rados.rados.0.smithi022.stdout:3890: finishing write tid 2 to smithi02284960-46 2024-09-21T09:44:47.302 INFO:tasks.rados.rados.0.smithi022.stdout:3890: finishing write tid 3 to smithi02284960-46 2024-09-21T09:44:47.302 INFO:tasks.rados.rados.0.smithi022.stdout:3890: finishing write tid 4 to smithi02284960-46 2024-09-21T09:44:47.302 INFO:tasks.rados.rados.0.smithi022.stdout:3890: finishing write tid 5 to smithi02284960-46 2024-09-21T09:44:47.302 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 46 v 2102 (ObjNum 1221 snap 386 seq_num 1221) dirty exists 2024-09-21T09:44:47.302 INFO:tasks.rados.rados.0.smithi022.stdout:3890: left oid 46 (ObjNum 1221 snap 386 seq_num 1221) 2024-09-21T09:44:47.302 INFO:tasks.rados.rados.0.smithi022.stdout:3891: finishing copy_from racing read to smithi02284960-36 2024-09-21T09:44:47.302 INFO:tasks.rados.rados.0.smithi022.stdout:3891: finishing copy_from to smithi02284960-36 2024-09-21T09:44:47.302 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 36 v 2004 (ObjNum 1212 snap 385 seq_num 1212) dirty exists 2024-09-21T09:44:47.302 INFO:tasks.rados.rados.0.smithi022.stdout:3889: done (2 left) 2024-09-21T09:44:47.302 INFO:tasks.rados.rados.0.smithi022.stdout:3890: done (1 left) 2024-09-21T09:44:47.303 INFO:tasks.rados.rados.0.smithi022.stdout:3891: done (0 left) 2024-09-21T09:44:47.303 INFO:tasks.rados.rados.0.smithi022.stdout:3892: read oid 25 snap -1 2024-09-21T09:44:47.303 INFO:tasks.rados.rados.0.smithi022.stdout:3892: expect (ObjNum 1199 snap 379 seq_num 1199) 2024-09-21T09:44:47.303 INFO:tasks.rados.rados.0.smithi022.stdout:3893: read oid 3 snap -1 2024-09-21T09:44:47.303 INFO:tasks.rados.rados.0.smithi022.stdout:3893: expect (ObjNum 1215 snap 386 seq_num 1215) 2024-09-21T09:44:47.303 INFO:tasks.rados.rados.0.smithi022.stdout:3894: copy_from oid 32 from oid 48 current snap is 386 2024-09-21T09:44:47.303 INFO:tasks.rados.rados.0.smithi022.stdout:3895: delete oid 46 current snap is 386 2024-09-21T09:44:47.303 INFO:tasks.rados.rados.0.smithi022.stdout:3894: finishing copy_from to smithi02284960-32 2024-09-21T09:44:47.304 INFO:tasks.rados.rados.0.smithi022.stdout:3894: got expected ENOENT (src dne) 2024-09-21T09:44:47.304 INFO:tasks.rados.rados.0.smithi022.stdout:3894: finishing copy_from racing read to smithi02284960-32 2024-09-21T09:44:47.304 INFO:tasks.rados.rados.0.smithi022.stdout:3894: done (3 left) 2024-09-21T09:44:47.304 INFO:tasks.rados.rados.0.smithi022.stdout:3895: done (2 left) 2024-09-21T09:44:47.304 INFO:tasks.rados.rados.0.smithi022.stdout:3896: snap_create 2024-09-21T09:44:47.310 INFO:tasks.rados.rados.0.smithi022.stdout:3892: expect (ObjNum 1199 snap 379 seq_num 1199) 2024-09-21T09:44:47.345 INFO:tasks.rados.rados.0.smithi022.stdout:3893: expect (ObjNum 1215 snap 386 seq_num 1215) 2024-09-21T09:44:47.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:44:47 smithi064 ceph-mon[86709]: osdmap e844: 8 total, 8 up, 8 in 2024-09-21T09:44:47.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:44:47 smithi022 ceph-mon[98615]: osdmap e844: 8 total, 8 up, 8 in 2024-09-21T09:44:47.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:44:47 smithi022 ceph-mon[96736]: osdmap e844: 8 total, 8 up, 8 in 2024-09-21T09:44:48.137 INFO:tasks.rados.rados.0.smithi022.stdout:3892: done (2 left) 2024-09-21T09:44:48.137 INFO:tasks.rados.rados.0.smithi022.stdout:3893: done (1 left) 2024-09-21T09:44:48.137 INFO:tasks.rados.rados.0.smithi022.stdout:3896: done (0 left) 2024-09-21T09:44:48.137 INFO:tasks.rados.rados.0.smithi022.stdout:3897: read oid 47 snap 381 2024-09-21T09:44:48.137 INFO:tasks.rados.rados.0.smithi022.stdout:3897: expect (ObjNum -724752096 snap 22094 seq_num 3569471696) 2024-09-21T09:44:48.137 INFO:tasks.rados.rados.0.smithi022.stdout:3898: copy_from oid 43 from oid 38 current snap is 387 2024-09-21T09:44:48.137 INFO:tasks.rados.rados.0.smithi022.stdout:3899: read oid 2 snap -1 2024-09-21T09:44:48.138 INFO:tasks.rados.rados.0.smithi022.stdout:3899: expect deleted 2024-09-21T09:44:48.138 INFO:tasks.rados.rados.0.smithi022.stdout:3900: read oid 10 snap 385 2024-09-21T09:44:48.138 INFO:tasks.rados.rados.0.smithi022.stdout:3900: expect (ObjNum 1213 snap 385 seq_num 1213) 2024-09-21T09:44:48.138 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 42 current snap is 387 2024-09-21T09:44:48.138 INFO:tasks.rados.rados.0.smithi022.stdout:3901: seq_num 1223 ranges {1007616=491520,1499136=491520,1990656=311296} 2024-09-21T09:44:48.145 INFO:tasks.rados.rados.0.smithi022.stdout:3901: writing smithi02284960-42 from 1007616 to 1499136 tid 1 2024-09-21T09:44:48.148 INFO:tasks.rados.rados.0.smithi022.stdout:3901: writing smithi02284960-42 from 1499136 to 1990656 tid 2 2024-09-21T09:44:48.150 INFO:tasks.rados.rados.0.smithi022.stdout:3901: writing smithi02284960-42 from 1990656 to 2301952 tid 3 2024-09-21T09:44:48.150 INFO:tasks.rados.rados.0.smithi022.stdout:3902: rollback oid 15 current snap is 387 2024-09-21T09:44:48.151 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 15 to 382 2024-09-21T09:44:48.151 INFO:tasks.rados.rados.0.smithi022.stdout:3897: expect (ObjNum -724752096 snap 22094 seq_num 3569471696) 2024-09-21T09:44:48.152 INFO:tasks.rados.rados.0.smithi022.stdout:3897: done (5 left) 2024-09-21T09:44:48.152 INFO:tasks.rados.rados.0.smithi022.stdout:3899: done (4 left) 2024-09-21T09:44:48.153 INFO:tasks.rados.rados.0.smithi022.stdout:3903: read oid 24 snap -1 2024-09-21T09:44:48.153 INFO:tasks.rados.rados.0.smithi022.stdout:3903: expect deleted 2024-09-21T09:44:48.153 INFO:tasks.rados.rados.0.smithi022.stdout:3904: snap_create 2024-09-21T09:44:48.156 INFO:tasks.rados.rados.0.smithi022.stdout:3902: finishing rollback tid 0 to smithi02284960-15 2024-09-21T09:44:48.157 INFO:tasks.rados.rados.0.smithi022.stdout:3901: finishing write tid 1 to smithi02284960-42 2024-09-21T09:44:48.158 INFO:tasks.rados.rados.0.smithi022.stdout:3901: finishing write tid 2 to smithi02284960-42 2024-09-21T09:44:48.158 INFO:tasks.rados.rados.0.smithi022.stdout:3902: finishing rollback tid 1 to smithi02284960-15 2024-09-21T09:44:48.159 INFO:tasks.rados.rados.0.smithi022.stdout:3902: finishing rollback tid 2 to smithi02284960-15 2024-09-21T09:44:48.159 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 15 v 2173 (ObjNum 1197 snap 375 seq_num 1197) dirty exists 2024-09-21T09:44:48.162 INFO:tasks.rados.rados.0.smithi022.stdout:3901: finishing write tid 3 to smithi02284960-42 2024-09-21T09:44:48.162 INFO:tasks.rados.rados.0.smithi022.stdout:3901: finishing write tid 4 to smithi02284960-42 2024-09-21T09:44:48.164 INFO:tasks.rados.rados.0.smithi022.stdout:3901: finishing write tid 5 to smithi02284960-42 2024-09-21T09:44:48.164 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 42 v 2006 (ObjNum 1223 snap 387 seq_num 1223) dirty exists 2024-09-21T09:44:48.164 INFO:tasks.rados.rados.0.smithi022.stdout:3901: left oid 42 (ObjNum 1223 snap 387 seq_num 1223) 2024-09-21T09:44:48.168 INFO:tasks.rados.rados.0.smithi022.stdout:3898: finishing copy_from racing read to smithi02284960-43 2024-09-21T09:44:48.173 INFO:tasks.rados.rados.0.smithi022.stdout:3900: expect (ObjNum 1213 snap 385 seq_num 1213) 2024-09-21T09:44:48.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:44:48 smithi064 ceph-mon[86709]: pgmap v880: 121 pgs: 121 active+clean; 147 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6.5 MiB/s rd, 2.3 MiB/s wr, 20 op/s 2024-09-21T09:44:48.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:44:48 smithi064 ceph-mon[86709]: osdmap e845: 8 total, 8 up, 8 in 2024-09-21T09:44:48.457 INFO:tasks.rados.rados.0.smithi022.stdout:3898: finishing copy_from to smithi02284960-43 2024-09-21T09:44:48.457 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 43 v 1858 (ObjNum 1141 snap 350 seq_num 1141) dirty exists 2024-09-21T09:44:48.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:44:48 smithi022 ceph-mon[98615]: pgmap v880: 121 pgs: 121 active+clean; 147 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6.5 MiB/s rd, 2.3 MiB/s wr, 20 op/s 2024-09-21T09:44:48.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:44:48 smithi022 ceph-mon[98615]: osdmap e845: 8 total, 8 up, 8 in 2024-09-21T09:44:48.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:44:48 smithi022 ceph-mon[96736]: pgmap v880: 121 pgs: 121 active+clean; 147 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6.5 MiB/s rd, 2.3 MiB/s wr, 20 op/s 2024-09-21T09:44:48.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:44:48 smithi022 ceph-mon[96736]: osdmap e845: 8 total, 8 up, 8 in 2024-09-21T09:44:49.117 INFO:tasks.rados.rados.0.smithi022.stdout:3898: done (5 left) 2024-09-21T09:44:49.117 INFO:tasks.rados.rados.0.smithi022.stdout:3900: done (4 left) 2024-09-21T09:44:49.118 INFO:tasks.rados.rados.0.smithi022.stdout:3901: done (3 left) 2024-09-21T09:44:49.118 INFO:tasks.rados.rados.0.smithi022.stdout:3902: done (2 left) 2024-09-21T09:44:49.118 INFO:tasks.rados.rados.0.smithi022.stdout:3903: done (1 left) 2024-09-21T09:44:49.118 INFO:tasks.rados.rados.0.smithi022.stdout:3904: done (0 left) 2024-09-21T09:44:49.118 INFO:tasks.rados.rados.0.smithi022.stdout:3905: read oid 49 snap 386 2024-09-21T09:44:49.118 INFO:tasks.rados.rados.0.smithi022.stdout:3905: expect deleted 2024-09-21T09:44:49.118 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 26 current snap is 388 2024-09-21T09:44:49.118 INFO:tasks.rados.rados.0.smithi022.stdout:3906: seq_num 1224 ranges {0=507904,507904=425984,933888=376832} 2024-09-21T09:44:49.120 INFO:tasks.rados.rados.0.smithi022.stdout:3906: writing smithi02284960-26 from 0 to 507904 tid 1 2024-09-21T09:44:49.122 INFO:tasks.rados.rados.0.smithi022.stdout:3906: writing smithi02284960-26 from 507904 to 933888 tid 2 2024-09-21T09:44:49.124 INFO:tasks.rados.rados.0.smithi022.stdout:3906: writing smithi02284960-26 from 933888 to 1310720 tid 3 2024-09-21T09:44:49.124 INFO:tasks.rados.rados.0.smithi022.stdout:3907: rollback oid 34 current snap is 388 2024-09-21T09:44:49.124 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 34 to 369 2024-09-21T09:44:49.125 INFO:tasks.rados.rados.0.smithi022.stdout:3905: done (2 left) 2024-09-21T09:44:49.125 INFO:tasks.rados.rados.0.smithi022.stdout:3908: delete oid 8 current snap is 388 2024-09-21T09:44:49.130 INFO:tasks.rados.rados.0.smithi022.stdout:3908: done (2 left) 2024-09-21T09:44:49.130 INFO:tasks.rados.rados.0.smithi022.stdout:3909: read oid 45 snap -1 2024-09-21T09:44:49.130 INFO:tasks.rados.rados.0.smithi022.stdout:3909: expect (ObjNum 1208 snap 382 seq_num 1208) 2024-09-21T09:44:49.131 INFO:tasks.rados.rados.0.smithi022.stdout:3910: copy_from oid 47 from oid 18 current snap is 388 2024-09-21T09:44:49.131 INFO:tasks.rados.rados.0.smithi022.stdout:3911: snap_remove snap 380 2024-09-21T09:44:49.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:44:49 smithi064 ceph-mon[86709]: osdmap e846: 8 total, 8 up, 8 in 2024-09-21T09:44:49.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:44:49 smithi064 ceph-mon[86709]: pgmap v883: 121 pgs: 121 active+clean; 152 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 3.3 MiB/s wr, 24 op/s 2024-09-21T09:44:49.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:44:49 smithi064 ceph-mon[86709]: osdmap e847: 8 total, 8 up, 8 in 2024-09-21T09:44:49.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:44:49 smithi022 ceph-mon[98615]: osdmap e846: 8 total, 8 up, 8 in 2024-09-21T09:44:49.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:44:49 smithi022 ceph-mon[98615]: pgmap v883: 121 pgs: 121 active+clean; 152 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 3.3 MiB/s wr, 24 op/s 2024-09-21T09:44:49.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:44:49 smithi022 ceph-mon[98615]: osdmap e847: 8 total, 8 up, 8 in 2024-09-21T09:44:49.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:44:49 smithi022 ceph-mon[96736]: osdmap e846: 8 total, 8 up, 8 in 2024-09-21T09:44:49.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:44:49 smithi022 ceph-mon[96736]: pgmap v883: 121 pgs: 121 active+clean; 152 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 3.3 MiB/s wr, 24 op/s 2024-09-21T09:44:49.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:44:49 smithi022 ceph-mon[96736]: osdmap e847: 8 total, 8 up, 8 in 2024-09-21T09:44:50.119 INFO:tasks.rados.rados.0.smithi022.stdout:3911: done (4 left) 2024-09-21T09:44:50.120 INFO:tasks.rados.rados.0.smithi022.stdout:3912: snap_remove snap 385 2024-09-21T09:44:50.691 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:44:50 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:44:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:44:51.126 INFO:tasks.rados.rados.0.smithi022.stdout:3912: done (4 left) 2024-09-21T09:44:51.126 INFO:tasks.rados.rados.0.smithi022.stdout:3913: rollback oid 24 current snap is 388 2024-09-21T09:44:51.126 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 24 to 369 2024-09-21T09:44:51.126 INFO:tasks.rados.rados.0.smithi022.stdout:3906: finishing write tid 1 to smithi02284960-26 2024-09-21T09:44:51.126 INFO:tasks.rados.rados.0.smithi022.stdout:3907: finishing rollback tid 0 to smithi02284960-34 2024-09-21T09:44:51.126 INFO:tasks.rados.rados.0.smithi022.stdout:3906: finishing write tid 2 to smithi02284960-26 2024-09-21T09:44:51.127 INFO:tasks.rados.rados.0.smithi022.stdout:3907: finishing rollback tid 1 to smithi02284960-34 2024-09-21T09:44:51.127 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 34 v 1857 (ObjNum -859622560 snap 32676 seq_num 304) dirty dne 2024-09-21T09:44:51.127 INFO:tasks.rados.rados.0.smithi022.stdout:3906: finishing write tid 3 to smithi02284960-26 2024-09-21T09:44:51.127 INFO:tasks.rados.rados.0.smithi022.stdout:3906: finishing write tid 4 to smithi02284960-26 2024-09-21T09:44:51.127 INFO:tasks.rados.rados.0.smithi022.stdout:3909: expect (ObjNum 1208 snap 382 seq_num 1208) 2024-09-21T09:44:51.160 INFO:tasks.rados.rados.0.smithi022.stdout:3906: finishing write tid 5 to smithi02284960-26 2024-09-21T09:44:51.160 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 26 v 2010 (ObjNum 1224 snap 388 seq_num 1224) dirty exists 2024-09-21T09:44:51.160 INFO:tasks.rados.rados.0.smithi022.stdout:3906: left oid 26 (ObjNum 1224 snap 388 seq_num 1224) 2024-09-21T09:44:51.160 INFO:tasks.rados.rados.0.smithi022.stdout:3910: finishing copy_from racing read to smithi02284960-47 2024-09-21T09:44:51.160 INFO:tasks.rados.rados.0.smithi022.stdout:3910: finishing copy_from to smithi02284960-47 2024-09-21T09:44:51.160 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 47 v 1758 (ObjNum 1062 snap 319 seq_num 1062) dirty exists 2024-09-21T09:44:51.160 INFO:tasks.rados.rados.0.smithi022.stdout:3913: finishing rollback tid 1 to smithi02284960-24 2024-09-21T09:44:51.160 INFO:tasks.rados.rados.0.smithi022.stdout:3913: finishing rollback tid 2 to smithi02284960-24 2024-09-21T09:44:51.160 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 24 v 2185 (ObjNum 1079 snap 326 seq_num 1079) dirty exists 2024-09-21T09:44:51.161 INFO:tasks.rados.rados.0.smithi022.stdout:3906: done (4 left) 2024-09-21T09:44:51.161 INFO:tasks.rados.rados.0.smithi022.stdout:3907: done (3 left) 2024-09-21T09:44:51.161 INFO:tasks.rados.rados.0.smithi022.stdout:3909: done (2 left) 2024-09-21T09:44:51.161 INFO:tasks.rados.rados.0.smithi022.stdout:3910: done (1 left) 2024-09-21T09:44:51.161 INFO:tasks.rados.rados.0.smithi022.stdout:3913: done (0 left) 2024-09-21T09:44:51.161 INFO:tasks.rados.rados.0.smithi022.stdout:3914: delete oid 32 current snap is 388 2024-09-21T09:44:51.162 INFO:tasks.rados.rados.0.smithi022.stdout:3914: done (0 left) 2024-09-21T09:44:51.162 INFO:tasks.rados.rados.0.smithi022.stdout:3915: snap_create 2024-09-21T09:44:51.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:44:51 smithi064 ceph-mon[86709]: osdmap e848: 8 total, 8 up, 8 in 2024-09-21T09:44:51.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:44:51 smithi022 ceph-mon[98615]: osdmap e848: 8 total, 8 up, 8 in 2024-09-21T09:44:51.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:44:51 smithi022 ceph-mon[96736]: osdmap e848: 8 total, 8 up, 8 in 2024-09-21T09:44:52.130 INFO:tasks.rados.rados.0.smithi022.stdout:3915: done (0 left) 2024-09-21T09:44:52.130 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 31 current snap is 389 2024-09-21T09:44:52.130 INFO:tasks.rados.rados.0.smithi022.stdout:3916: seq_num 1227 ranges {3981312=557056,4538368=483328,5021696=319488} 2024-09-21T09:44:52.154 INFO:tasks.rados.rados.0.smithi022.stdout:3916: writing smithi02284960-31 from 3981312 to 4538368 tid 1 2024-09-21T09:44:52.157 INFO:tasks.rados.rados.0.smithi022.stdout:3916: writing smithi02284960-31 from 4538368 to 5021696 tid 2 2024-09-21T09:44:52.159 INFO:tasks.rados.rados.0.smithi022.stdout:3916: writing smithi02284960-31 from 5021696 to 5341184 tid 3 2024-09-21T09:44:52.159 INFO:tasks.rados.rados.0.smithi022.stdout:3917: delete oid 30 current snap is 389 2024-09-21T09:44:52.162 INFO:tasks.rados.rados.0.smithi022.stdout:3917: done (1 left) 2024-09-21T09:44:52.162 INFO:tasks.rados.rados.0.smithi022.stdout:3918: delete oid 22 current snap is 389 2024-09-21T09:44:52.165 INFO:tasks.rados.rados.0.smithi022.stdout:3918: done (1 left) 2024-09-21T09:44:52.166 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 34 current snap is 389 2024-09-21T09:44:52.166 INFO:tasks.rados.rados.0.smithi022.stdout:3919: seq_num 1230 ranges {0=606208,606208=532480,1138688=270336} 2024-09-21T09:44:52.169 INFO:tasks.rados.rados.0.smithi022.stdout:3919: writing smithi02284960-34 from 0 to 606208 tid 1 2024-09-21T09:44:52.172 INFO:tasks.rados.rados.0.smithi022.stdout:3919: writing smithi02284960-34 from 606208 to 1138688 tid 2 2024-09-21T09:44:52.174 INFO:tasks.rados.rados.0.smithi022.stdout:3919: writing smithi02284960-34 from 1138688 to 1409024 tid 3 2024-09-21T09:44:52.174 INFO:tasks.rados.rados.0.smithi022.stdout:3920: snap_remove snap 375 2024-09-21T09:44:52.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:44:52 smithi064 ceph-mon[86709]: pgmap v886: 121 pgs: 121 active+clean; 155 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 2.6 MiB/s wr, 13 op/s 2024-09-21T09:44:52.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:44:52 smithi064 ceph-mon[86709]: osdmap e849: 8 total, 8 up, 8 in 2024-09-21T09:44:52.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:44:52 smithi022 ceph-mon[98615]: pgmap v886: 121 pgs: 121 active+clean; 155 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 2.6 MiB/s wr, 13 op/s 2024-09-21T09:44:52.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:44:52 smithi022 ceph-mon[98615]: osdmap e849: 8 total, 8 up, 8 in 2024-09-21T09:44:52.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:44:52 smithi022 ceph-mon[96736]: pgmap v886: 121 pgs: 121 active+clean; 155 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 2.6 MiB/s wr, 13 op/s 2024-09-21T09:44:52.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:44:52 smithi022 ceph-mon[96736]: osdmap e849: 8 total, 8 up, 8 in 2024-09-21T09:44:53.137 INFO:tasks.rados.rados.0.smithi022.stdout:3916: finishing write tid 1 to smithi02284960-31 2024-09-21T09:44:53.137 INFO:tasks.rados.rados.0.smithi022.stdout:3916: finishing write tid 2 to smithi02284960-31 2024-09-21T09:44:53.137 INFO:tasks.rados.rados.0.smithi022.stdout:3916: finishing write tid 3 to smithi02284960-31 2024-09-21T09:44:53.137 INFO:tasks.rados.rados.0.smithi022.stdout:3916: finishing write tid 4 to smithi02284960-31 2024-09-21T09:44:53.137 INFO:tasks.rados.rados.0.smithi022.stdout:3916: finishing write tid 5 to smithi02284960-31 2024-09-21T09:44:53.137 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 31 v 2194 (ObjNum 1227 snap 389 seq_num 1227) dirty exists 2024-09-21T09:44:53.138 INFO:tasks.rados.rados.0.smithi022.stdout:3916: left oid 31 (ObjNum 1227 snap 389 seq_num 1227) 2024-09-21T09:44:53.138 INFO:tasks.rados.rados.0.smithi022.stdout:3919: finishing write tid 1 to smithi02284960-34 2024-09-21T09:44:53.138 INFO:tasks.rados.rados.0.smithi022.stdout:3919: finishing write tid 2 to smithi02284960-34 2024-09-21T09:44:53.138 INFO:tasks.rados.rados.0.smithi022.stdout:3919: finishing write tid 3 to smithi02284960-34 2024-09-21T09:44:53.138 INFO:tasks.rados.rados.0.smithi022.stdout:3919: finishing write tid 4 to smithi02284960-34 2024-09-21T09:44:53.138 INFO:tasks.rados.rados.0.smithi022.stdout:3919: finishing write tid 5 to smithi02284960-34 2024-09-21T09:44:53.138 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 34 v 1861 (ObjNum 1230 snap 389 seq_num 1230) dirty exists 2024-09-21T09:44:53.138 INFO:tasks.rados.rados.0.smithi022.stdout:3919: left oid 34 (ObjNum 1230 snap 389 seq_num 1230) 2024-09-21T09:44:53.138 INFO:tasks.rados.rados.0.smithi022.stdout:3916: done (2 left) 2024-09-21T09:44:53.138 INFO:tasks.rados.rados.0.smithi022.stdout:3919: done (1 left) 2024-09-21T09:44:53.138 INFO:tasks.rados.rados.0.smithi022.stdout:3920: done (0 left) 2024-09-21T09:44:53.138 INFO:tasks.rados.rados.0.smithi022.stdout:3921: snap_create 2024-09-21T09:44:53.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:44:53 smithi064 ceph-mon[86709]: osdmap e850: 8 total, 8 up, 8 in 2024-09-21T09:44:53.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:44:53 smithi022 ceph-mon[98615]: osdmap e850: 8 total, 8 up, 8 in 2024-09-21T09:44:53.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:44:53 smithi022 ceph-mon[96736]: osdmap e850: 8 total, 8 up, 8 in 2024-09-21T09:44:54.120 INFO:tasks.rados.rados.0.smithi022.stdout:3921: done (0 left) 2024-09-21T09:44:54.120 INFO:tasks.rados.rados.0.smithi022.stdout:3922: rollback oid 21 current snap is 390 2024-09-21T09:44:54.120 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 21 to 386 2024-09-21T09:44:54.121 INFO:tasks.rados.rados.0.smithi022.stdout:3923: rollback oid 38 current snap is 390 2024-09-21T09:44:54.121 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 38 to 384 2024-09-21T09:44:54.121 INFO:tasks.rados.rados.0.smithi022.stdout:3924: snap_create 2024-09-21T09:44:54.127 INFO:tasks.rados.rados.0.smithi022.stdout:3922: finishing rollback tid 0 to smithi02284960-21 2024-09-21T09:44:54.127 INFO:tasks.rados.rados.0.smithi022.stdout:3922: finishing rollback tid 1 to smithi02284960-21 2024-09-21T09:44:54.127 INFO:tasks.rados.rados.0.smithi022.stdout:3922: finishing rollback tid 2 to smithi02284960-21 2024-09-21T09:44:54.127 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 21 v 1864 (ObjNum 1216 snap 386 seq_num 1216) dirty exists 2024-09-21T09:44:54.128 INFO:tasks.rados.rados.0.smithi022.stdout:3923: finishing rollback tid 0 to smithi02284960-38 2024-09-21T09:44:54.128 INFO:tasks.rados.rados.0.smithi022.stdout:3923: finishing rollback tid 1 to smithi02284960-38 2024-09-21T09:44:54.128 INFO:tasks.rados.rados.0.smithi022.stdout:3923: finishing rollback tid 2 to smithi02284960-38 2024-09-21T09:44:54.128 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 38 v 1867 (ObjNum 1141 snap 350 seq_num 1141) dirty exists 2024-09-21T09:44:54.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:44:54 smithi064 ceph-mon[86709]: pgmap v889: 121 pgs: 121 active+clean; 164 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 5.1 MiB/s rd, 1.7 MiB/s wr, 11 op/s 2024-09-21T09:44:54.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:44:54 smithi064 ceph-mon[86709]: osdmap e851: 8 total, 8 up, 8 in 2024-09-21T09:44:54.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:44:54 smithi064 ceph-mon[86709]: osdmap e852: 8 total, 8 up, 8 in 2024-09-21T09:44:54.459 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:44:54 smithi022 ceph-mon[98615]: pgmap v889: 121 pgs: 121 active+clean; 164 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 5.1 MiB/s rd, 1.7 MiB/s wr, 11 op/s 2024-09-21T09:44:54.459 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:44:54 smithi022 ceph-mon[98615]: osdmap e851: 8 total, 8 up, 8 in 2024-09-21T09:44:54.459 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:44:54 smithi022 ceph-mon[98615]: osdmap e852: 8 total, 8 up, 8 in 2024-09-21T09:44:54.460 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:44:54 smithi022 ceph-mon[96736]: pgmap v889: 121 pgs: 121 active+clean; 164 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 5.1 MiB/s rd, 1.7 MiB/s wr, 11 op/s 2024-09-21T09:44:54.460 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:44:54 smithi022 ceph-mon[96736]: osdmap e851: 8 total, 8 up, 8 in 2024-09-21T09:44:54.460 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:44:54 smithi022 ceph-mon[96736]: osdmap e852: 8 total, 8 up, 8 in 2024-09-21T09:44:55.121 INFO:tasks.rados.rados.0.smithi022.stdout:3922: done (2 left) 2024-09-21T09:44:55.121 INFO:tasks.rados.rados.0.smithi022.stdout:3923: done (1 left) 2024-09-21T09:44:55.121 INFO:tasks.rados.rados.0.smithi022.stdout:3924: done (0 left) 2024-09-21T09:44:55.121 INFO:tasks.rados.rados.0.smithi022.stdout:3925: read oid 9 snap 367 2024-09-21T09:44:55.121 INFO:tasks.rados.rados.0.smithi022.stdout:3925: expect (ObjNum 1062 snap 319 seq_num 1062) 2024-09-21T09:44:55.122 INFO:tasks.rados.rados.0.smithi022.stdout:3926: delete oid 37 current snap is 391 2024-09-21T09:44:55.125 INFO:tasks.rados.rados.0.smithi022.stdout:3926: done (1 left) 2024-09-21T09:44:55.125 INFO:tasks.rados.rados.0.smithi022.stdout:3927: rollback oid 8 current snap is 391 2024-09-21T09:44:55.125 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 8 to 367 2024-09-21T09:44:55.125 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 32 current snap is 391 2024-09-21T09:44:55.125 INFO:tasks.rados.rados.0.smithi022.stdout:3928: seq_num 1232 ranges {0=638976,638976=753664,1392640=49152} 2024-09-21T09:44:55.129 INFO:tasks.rados.rados.0.smithi022.stdout:3928: writing smithi02284960-32 from 0 to 638976 tid 1 2024-09-21T09:44:55.134 INFO:tasks.rados.rados.0.smithi022.stdout:3928: writing smithi02284960-32 from 638976 to 1392640 tid 2 2024-09-21T09:44:55.134 INFO:tasks.rados.rados.0.smithi022.stdout:3928: writing smithi02284960-32 from 1392640 to 1441792 tid 3 2024-09-21T09:44:55.134 INFO:tasks.rados.rados.0.smithi022.stdout:3929: rmattr oid 7 current snap is 391 2024-09-21T09:44:55.135 INFO:tasks.rados.rados.0.smithi022.stdout:3929: done (3 left) 2024-09-21T09:44:55.135 INFO:tasks.rados.rados.0.smithi022.stdout:3930: copy_from oid 6 from oid 34 current snap is 391 2024-09-21T09:44:55.135 INFO:tasks.rados.rados.0.smithi022.stdout:3927: finishing rollback tid 1 to smithi02284960-8 2024-09-21T09:44:55.135 INFO:tasks.rados.rados.0.smithi022.stdout:3927: finishing rollback tid 2 to smithi02284960-8 2024-09-21T09:44:55.135 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 8 v 2006 (ObjNum 1155 snap 356 seq_num 1155) dirty exists 2024-09-21T09:44:55.135 INFO:tasks.rados.rados.0.smithi022.stdout:3925: expect (ObjNum 1062 snap 319 seq_num 1062) 2024-09-21T09:44:55.164 INFO:tasks.rados.rados.0.smithi022.stdout:3928: finishing write tid 1 to smithi02284960-32 2024-09-21T09:44:55.164 INFO:tasks.rados.rados.0.smithi022.stdout:3928: finishing write tid 2 to smithi02284960-32 2024-09-21T09:44:55.164 INFO:tasks.rados.rados.0.smithi022.stdout:3928: finishing write tid 3 to smithi02284960-32 2024-09-21T09:44:55.165 INFO:tasks.rados.rados.0.smithi022.stdout:3928: finishing write tid 4 to smithi02284960-32 2024-09-21T09:44:55.165 INFO:tasks.rados.rados.0.smithi022.stdout:3928: finishing write tid 5 to smithi02284960-32 2024-09-21T09:44:55.165 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 32 v 2176 (ObjNum 1232 snap 391 seq_num 1232) dirty exists 2024-09-21T09:44:55.165 INFO:tasks.rados.rados.0.smithi022.stdout:3928: left oid 32 (ObjNum 1232 snap 391 seq_num 1232) 2024-09-21T09:44:55.165 INFO:tasks.rados.rados.0.smithi022.stdout:3930: finishing copy_from racing read to smithi02284960-6 2024-09-21T09:44:55.165 INFO:tasks.rados.rados.0.smithi022.stdout:3930: finishing copy_from to smithi02284960-6 2024-09-21T09:44:55.165 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 6 v 1861 (ObjNum 1230 snap 389 seq_num 1230) dirty exists 2024-09-21T09:44:55.165 INFO:tasks.rados.rados.0.smithi022.stdout:3925: done (3 left) 2024-09-21T09:44:55.165 INFO:tasks.rados.rados.0.smithi022.stdout:3927: done (2 left) 2024-09-21T09:44:55.165 INFO:tasks.rados.rados.0.smithi022.stdout:3928: done (1 left) 2024-09-21T09:44:55.165 INFO:tasks.rados.rados.0.smithi022.stdout:3930: done (0 left) 2024-09-21T09:44:55.165 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 2 current snap is 391 2024-09-21T09:44:55.165 INFO:tasks.rados.rados.0.smithi022.stdout:3931: seq_num 1233 ranges {0=655360,655360=638976,1294336=163840} 2024-09-21T09:44:55.167 INFO:tasks.rados.rados.0.smithi022.stdout:3931: writing smithi02284960-2 from 0 to 655360 tid 1 2024-09-21T09:44:55.171 INFO:tasks.rados.rados.0.smithi022.stdout:3931: writing smithi02284960-2 from 655360 to 1294336 tid 2 2024-09-21T09:44:55.172 INFO:tasks.rados.rados.0.smithi022.stdout:3931: writing smithi02284960-2 from 1294336 to 1458176 tid 3 2024-09-21T09:44:55.173 INFO:tasks.rados.rados.0.smithi022.stdout:3932: delete oid 9 current snap is 391 2024-09-21T09:44:55.173 INFO:tasks.rados.rados.0.smithi022.stdout:3931: finishing write tid 1 to smithi02284960-2 2024-09-21T09:44:55.176 INFO:tasks.rados.rados.0.smithi022.stdout:3932: done (1 left) 2024-09-21T09:44:55.176 INFO:tasks.rados.rados.0.smithi022.stdout:3933: rollback oid 10 current snap is 391 2024-09-21T09:44:55.176 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 10 to 384 2024-09-21T09:44:55.176 INFO:tasks.rados.rados.0.smithi022.stdout:3931: finishing write tid 2 to smithi02284960-2 2024-09-21T09:44:55.176 INFO:tasks.rados.rados.0.smithi022.stdout:3934: setattr oid 21 current snap is 391 2024-09-21T09:44:55.176 INFO:tasks.rados.rados.0.smithi022.stdout:3931: finishing write tid 3 to smithi02284960-2 2024-09-21T09:44:55.176 INFO:tasks.rados.rados.0.smithi022.stdout:3931: finishing write tid 4 to smithi02284960-2 2024-09-21T09:44:55.177 INFO:tasks.rados.rados.0.smithi022.stdout:3931: finishing write tid 5 to smithi02284960-2 2024-09-21T09:44:55.177 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 2 v 1868 (ObjNum 1233 snap 391 seq_num 1233) dirty exists 2024-09-21T09:44:55.178 INFO:tasks.rados.rados.0.smithi022.stdout:3931: left oid 2 (ObjNum 1233 snap 391 seq_num 1233) 2024-09-21T09:44:55.178 INFO:tasks.rados.rados.0.smithi022.stdout:3931: done (2 left) 2024-09-21T09:44:55.178 INFO:tasks.rados.rados.0.smithi022.stdout:3935: read oid 22 snap -1 2024-09-21T09:44:55.178 INFO:tasks.rados.rados.0.smithi022.stdout:3935: expect deleted 2024-09-21T09:44:55.178 INFO:tasks.rados.rados.0.smithi022.stdout:3936: read oid 20 snap -1 2024-09-21T09:44:55.178 INFO:tasks.rados.rados.0.smithi022.stdout:3936: expect (ObjNum 1122 snap 342 seq_num 1122) 2024-09-21T09:44:55.178 INFO:tasks.rados.rados.0.smithi022.stdout:3937: read oid 31 snap -1 2024-09-21T09:44:55.178 INFO:tasks.rados.rados.0.smithi022.stdout:3937: expect (ObjNum 1227 snap 389 seq_num 1227) 2024-09-21T09:44:55.178 INFO:tasks.rados.rados.0.smithi022.stdout:3938: snap_remove snap 382 2024-09-21T09:44:56.058 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:44:55 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:44:55] "GET /metrics HTTP/1.1" 200 36352 "" "Prometheus/2.51.0" 2024-09-21T09:44:56.140 INFO:tasks.rados.rados.0.smithi022.stdout:3938: done (5 left) 2024-09-21T09:44:56.141 INFO:tasks.rados.rados.0.smithi022.stdout:3939: copy_from oid 13 from oid 5 current snap is 391 2024-09-21T09:44:56.141 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 21 v 1869 (ObjNum 1216 snap 386 seq_num 1216) dirty exists 2024-09-21T09:44:56.141 INFO:tasks.rados.rados.0.smithi022.stdout:3933: finishing rollback tid 0 to smithi02284960-10 2024-09-21T09:44:56.141 INFO:tasks.rados.rados.0.smithi022.stdout:3933: finishing rollback tid 1 to smithi02284960-10 2024-09-21T09:44:56.141 INFO:tasks.rados.rados.0.smithi022.stdout:3933: finishing rollback tid 2 to smithi02284960-10 2024-09-21T09:44:56.141 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 10 v 2123 (ObjNum 1160 snap 356 seq_num 1160) dirty exists 2024-09-21T09:44:56.141 INFO:tasks.rados.rados.0.smithi022.stdout:3936: expect (ObjNum 1122 snap 342 seq_num 1122) 2024-09-21T09:44:56.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:44:56 smithi064 ceph-mon[86709]: pgmap v892: 121 pgs: 121 active+clean; 165 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 763 KiB/s rd, 980 KiB/s wr, 7 op/s 2024-09-21T09:44:56.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:44:56 smithi064 ceph-mon[86709]: osdmap e853: 8 total, 8 up, 8 in 2024-09-21T09:44:56.497 INFO:tasks.rados.rados.0.smithi022.stdout:3937: expect (ObjNum 1227 snap 389 seq_num 1227) 2024-09-21T09:44:56.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:44:56 smithi022 ceph-mon[98615]: pgmap v892: 121 pgs: 121 active+clean; 165 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 763 KiB/s rd, 980 KiB/s wr, 7 op/s 2024-09-21T09:44:56.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:44:56 smithi022 ceph-mon[98615]: osdmap e853: 8 total, 8 up, 8 in 2024-09-21T09:44:56.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:44:56 smithi022 ceph-mon[96736]: pgmap v892: 121 pgs: 121 active+clean; 165 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 763 KiB/s rd, 980 KiB/s wr, 7 op/s 2024-09-21T09:44:56.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:44:56 smithi022 ceph-mon[96736]: osdmap e853: 8 total, 8 up, 8 in 2024-09-21T09:44:56.787 INFO:tasks.rados.rados.0.smithi022.stdout:3939: finishing copy_from to smithi02284960-13 2024-09-21T09:44:56.787 INFO:tasks.rados.rados.0.smithi022.stdout:3939: got expected ENOENT (src dne) 2024-09-21T09:44:56.787 INFO:tasks.rados.rados.0.smithi022.stdout:3939: finishing copy_from racing read to smithi02284960-13 2024-09-21T09:44:56.787 INFO:tasks.rados.rados.0.smithi022.stdout:3933: done (5 left) 2024-09-21T09:44:56.787 INFO:tasks.rados.rados.0.smithi022.stdout:3934: done (4 left) 2024-09-21T09:44:56.787 INFO:tasks.rados.rados.0.smithi022.stdout:3935: done (3 left) 2024-09-21T09:44:56.787 INFO:tasks.rados.rados.0.smithi022.stdout:3936: done (2 left) 2024-09-21T09:44:56.788 INFO:tasks.rados.rados.0.smithi022.stdout:3937: done (1 left) 2024-09-21T09:44:56.788 INFO:tasks.rados.rados.0.smithi022.stdout:3939: done (0 left) 2024-09-21T09:44:56.788 INFO:tasks.rados.rados.0.smithi022.stdout:3940: delete oid 16 current snap is 391 2024-09-21T09:44:56.790 INFO:tasks.rados.rados.0.smithi022.stdout:3940: done (0 left) 2024-09-21T09:44:56.791 INFO:tasks.rados.rados.0.smithi022.stdout:3941: rollback oid 20 current snap is 391 2024-09-21T09:44:56.791 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 20 to 381 2024-09-21T09:44:56.791 INFO:tasks.rados.rados.0.smithi022.stdout:3942: snap_create 2024-09-21T09:44:56.795 INFO:tasks.rados.rados.0.smithi022.stdout:3941: finishing rollback tid 0 to smithi02284960-20 2024-09-21T09:44:56.795 INFO:tasks.rados.rados.0.smithi022.stdout:3941: finishing rollback tid 1 to smithi02284960-20 2024-09-21T09:44:56.795 INFO:tasks.rados.rados.0.smithi022.stdout:3941: finishing rollback tid 2 to smithi02284960-20 2024-09-21T09:44:56.795 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 20 v 2184 (ObjNum 1122 snap 342 seq_num 1122) dirty exists 2024-09-21T09:44:57.144 INFO:tasks.rados.rados.0.smithi022.stdout:3941: done (1 left) 2024-09-21T09:44:57.144 INFO:tasks.rados.rados.0.smithi022.stdout:3942: done (0 left) 2024-09-21T09:44:57.144 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 23 current snap is 392 2024-09-21T09:44:57.145 INFO:tasks.rados.rados.0.smithi022.stdout:3943: seq_num 1236 ranges {3801088=704512,4505600=688128,5193728=114688} 2024-09-21T09:44:57.169 INFO:tasks.rados.rados.0.smithi022.stdout:3943: writing smithi02284960-23 from 3801088 to 4505600 tid 1 2024-09-21T09:44:57.173 INFO:tasks.rados.rados.0.smithi022.stdout:3943: writing smithi02284960-23 from 4505600 to 5193728 tid 2 2024-09-21T09:44:57.174 INFO:tasks.rados.rados.0.smithi022.stdout:3943: writing smithi02284960-23 from 5193728 to 5308416 tid 3 2024-09-21T09:44:57.174 INFO:tasks.rados.rados.0.smithi022.stdout:3944: delete oid 13 current snap is 392 2024-09-21T09:44:57.177 INFO:tasks.rados.rados.0.smithi022.stdout:3943: finishing write tid 1 to smithi02284960-23 2024-09-21T09:44:57.178 INFO:tasks.rados.rados.0.smithi022.stdout:3943: finishing write tid 2 to smithi02284960-23 2024-09-21T09:44:57.178 INFO:tasks.rados.rados.0.smithi022.stdout:3943: finishing write tid 3 to smithi02284960-23 2024-09-21T09:44:57.179 INFO:tasks.rados.rados.0.smithi022.stdout:3943: finishing write tid 4 to smithi02284960-23 2024-09-21T09:44:57.180 INFO:tasks.rados.rados.0.smithi022.stdout:3943: finishing write tid 5 to smithi02284960-23 2024-09-21T09:44:57.180 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 23 v 1871 (ObjNum 1236 snap 392 seq_num 1236) dirty exists 2024-09-21T09:44:57.180 INFO:tasks.rados.rados.0.smithi022.stdout:3943: left oid 23 (ObjNum 1236 snap 392 seq_num 1236) 2024-09-21T09:44:57.180 INFO:tasks.rados.rados.0.smithi022.stdout:3943: done (1 left) 2024-09-21T09:44:57.180 INFO:tasks.rados.rados.0.smithi022.stdout:3944: done (0 left) 2024-09-21T09:44:57.180 INFO:tasks.rados.rados.0.smithi022.stdout:3945: copy_from oid 8 from oid 23 current snap is 392 2024-09-21T09:44:57.180 INFO:tasks.rados.rados.0.smithi022.stdout:3946: snap_remove snap 391 2024-09-21T09:44:57.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:44:57 smithi064 ceph-mon[86709]: osdmap e854: 8 total, 8 up, 8 in 2024-09-21T09:44:57.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:44:57 smithi064 ceph-mon[86709]: pgmap v895: 121 pgs: 121 active+clean; 175 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 6.7 MiB/s rd, 1.3 MiB/s wr, 10 op/s 2024-09-21T09:44:57.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:44:57 smithi022 ceph-mon[98615]: osdmap e854: 8 total, 8 up, 8 in 2024-09-21T09:44:57.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:44:57 smithi022 ceph-mon[98615]: pgmap v895: 121 pgs: 121 active+clean; 175 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 6.7 MiB/s rd, 1.3 MiB/s wr, 10 op/s 2024-09-21T09:44:57.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:44:57 smithi022 ceph-mon[96736]: osdmap e854: 8 total, 8 up, 8 in 2024-09-21T09:44:57.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:44:57 smithi022 ceph-mon[96736]: pgmap v895: 121 pgs: 121 active+clean; 175 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 6.7 MiB/s rd, 1.3 MiB/s wr, 10 op/s 2024-09-21T09:44:58.151 INFO:tasks.rados.rados.0.smithi022.stdout:3946: done (1 left) 2024-09-21T09:44:58.152 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 50 current snap is 392 2024-09-21T09:44:58.152 INFO:tasks.rados.rados.0.smithi022.stdout:3947: seq_num 1238 ranges {368640=737280,1105920=458752,1564672=344064} 2024-09-21T09:44:58.157 INFO:tasks.rados.rados.0.smithi022.stdout:3947: writing smithi02284960-50 from 368640 to 1105920 tid 1 2024-09-21T09:44:58.160 INFO:tasks.rados.rados.0.smithi022.stdout:3947: writing smithi02284960-50 from 1105920 to 1564672 tid 2 2024-09-21T09:44:58.162 INFO:tasks.rados.rados.0.smithi022.stdout:3947: writing smithi02284960-50 from 1564672 to 1908736 tid 3 2024-09-21T09:44:58.162 INFO:tasks.rados.rados.0.smithi022.stdout:3948: snap_create 2024-09-21T09:44:58.162 INFO:tasks.rados.rados.0.smithi022.stdout:3945: finishing copy_from racing read to smithi02284960-8 2024-09-21T09:44:58.162 INFO:tasks.rados.rados.0.smithi022.stdout:3945: finishing copy_from to smithi02284960-8 2024-09-21T09:44:58.163 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 8 v 1871 (ObjNum 1236 snap 392 seq_num 1236) dirty exists 2024-09-21T09:44:58.163 INFO:tasks.rados.rados.0.smithi022.stdout:3947: finishing write tid 1 to smithi02284960-50 2024-09-21T09:44:58.165 INFO:tasks.rados.rados.0.smithi022.stdout:3947: finishing write tid 2 to smithi02284960-50 2024-09-21T09:44:58.165 INFO:tasks.rados.rados.0.smithi022.stdout:3947: finishing write tid 3 to smithi02284960-50 2024-09-21T09:44:58.166 INFO:tasks.rados.rados.0.smithi022.stdout:3947: finishing write tid 4 to smithi02284960-50 2024-09-21T09:44:58.167 INFO:tasks.rados.rados.0.smithi022.stdout:3947: finishing write tid 5 to smithi02284960-50 2024-09-21T09:44:58.167 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 50 v 2008 (ObjNum 1238 snap 392 seq_num 1238) dirty exists 2024-09-21T09:44:58.167 INFO:tasks.rados.rados.0.smithi022.stdout:3947: left oid 50 (ObjNum 1238 snap 392 seq_num 1238) 2024-09-21T09:44:58.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:44:58 smithi064 ceph-mon[86709]: osdmap e855: 8 total, 8 up, 8 in 2024-09-21T09:44:58.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:44:58 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:44:58.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:44:58 smithi022 ceph-mon[98615]: osdmap e855: 8 total, 8 up, 8 in 2024-09-21T09:44:58.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:44:58 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:44:58.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:44:58 smithi022 ceph-mon[96736]: osdmap e855: 8 total, 8 up, 8 in 2024-09-21T09:44:58.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:44:58 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:44:59.121 INFO:tasks.rados.rados.0.smithi022.stdout:3945: done (2 left) 2024-09-21T09:44:59.122 INFO:tasks.rados.rados.0.smithi022.stdout:3947: done (1 left) 2024-09-21T09:44:59.122 INFO:tasks.rados.rados.0.smithi022.stdout:3948: done (0 left) 2024-09-21T09:44:59.122 INFO:tasks.rados.rados.0.smithi022.stdout:3949: copy_from oid 7 from oid 49 current snap is 393 2024-09-21T09:44:59.122 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 37 current snap is 393 2024-09-21T09:44:59.122 INFO:tasks.rados.rados.0.smithi022.stdout:3950: seq_num 1239 ranges {0=753664,753664=745472,1499136=57344} 2024-09-21T09:44:59.125 INFO:tasks.rados.rados.0.smithi022.stdout:3950: writing smithi02284960-37 from 0 to 753664 tid 1 2024-09-21T09:44:59.129 INFO:tasks.rados.rados.0.smithi022.stdout:3950: writing smithi02284960-37 from 753664 to 1499136 tid 2 2024-09-21T09:44:59.131 INFO:tasks.rados.rados.0.smithi022.stdout:3950: writing smithi02284960-37 from 1499136 to 1556480 tid 3 2024-09-21T09:44:59.131 INFO:tasks.rados.rados.0.smithi022.stdout:3951: copy_from oid 5 from oid 6 current snap is 393 2024-09-21T09:44:59.131 INFO:tasks.rados.rados.0.smithi022.stdout:3949: finishing copy_from to smithi02284960-7 2024-09-21T09:44:59.131 INFO:tasks.rados.rados.0.smithi022.stdout:3949: got expected ENOENT (src dne) 2024-09-21T09:44:59.131 INFO:tasks.rados.rados.0.smithi022.stdout:3949: finishing copy_from racing read to smithi02284960-7 2024-09-21T09:44:59.131 INFO:tasks.rados.rados.0.smithi022.stdout:3949: done (2 left) 2024-09-21T09:44:59.131 INFO:tasks.rados.rados.0.smithi022.stdout:3952: rollback oid 46 current snap is 393 2024-09-21T09:44:59.131 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 46 to 374 2024-09-21T09:44:59.131 INFO:tasks.rados.rados.0.smithi022.stdout:3953: rollback oid 33 current snap is 393 2024-09-21T09:44:59.131 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 33 to 386 2024-09-21T09:44:59.131 INFO:tasks.rados.rados.0.smithi022.stdout:3954: copy_from oid 34 from oid 48 current snap is 393 2024-09-21T09:44:59.131 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 26 current snap is 393 2024-09-21T09:44:59.131 INFO:tasks.rados.rados.0.smithi022.stdout:3955: seq_num 1240 ranges {1310720=778240,2088960=671744,2760704=131072} 2024-09-21T09:44:59.143 INFO:tasks.rados.rados.0.smithi022.stdout:3955: writing smithi02284960-26 from 1310720 to 2088960 tid 1 2024-09-21T09:44:59.147 INFO:tasks.rados.rados.0.smithi022.stdout:3955: writing smithi02284960-26 from 2088960 to 2760704 tid 2 2024-09-21T09:44:59.148 INFO:tasks.rados.rados.0.smithi022.stdout:3955: writing smithi02284960-26 from 2760704 to 2891776 tid 3 2024-09-21T09:44:59.149 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 43 current snap is 393 2024-09-21T09:44:59.149 INFO:tasks.rados.rados.0.smithi022.stdout:3952: finishing rollback tid 1 to smithi02284960-46 2024-09-21T09:44:59.149 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 46 v 0 (ObjNum 161 snap 0 seq_num 1) dirty dne 2024-09-21T09:44:59.149 INFO:tasks.rados.rados.0.smithi022.stdout:3950: finishing write tid 1 to smithi02284960-37 2024-09-21T09:44:59.149 INFO:tasks.rados.rados.0.smithi022.stdout:3950: finishing write tid 2 to smithi02284960-37 2024-09-21T09:44:59.149 INFO:tasks.rados.rados.0.smithi022.stdout:3954: finishing copy_from to smithi02284960-34 2024-09-21T09:44:59.149 INFO:tasks.rados.rados.0.smithi022.stdout:3954: got expected ENOENT (src dne) 2024-09-21T09:44:59.149 INFO:tasks.rados.rados.0.smithi022.stdout:3950: finishing write tid 3 to smithi02284960-37 2024-09-21T09:44:59.149 INFO:tasks.rados.rados.0.smithi022.stdout:3950: finishing write tid 4 to smithi02284960-37 2024-09-21T09:44:59.149 INFO:tasks.rados.rados.0.smithi022.stdout:3954: finishing copy_from racing read to smithi02284960-34 2024-09-21T09:44:59.149 INFO:tasks.rados.rados.0.smithi022.stdout:3953: finishing rollback tid 0 to smithi02284960-33 2024-09-21T09:44:59.149 INFO:tasks.rados.rados.0.smithi022.stdout:3953: finishing rollback tid 1 to smithi02284960-33 2024-09-21T09:44:59.149 INFO:tasks.rados.rados.0.smithi022.stdout:3953: finishing rollback tid 2 to smithi02284960-33 2024-09-21T09:44:59.149 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 33 v 2188 (ObjNum 1189 snap 371 seq_num 1189) dirty exists 2024-09-21T09:44:59.150 INFO:tasks.rados.rados.0.smithi022.stdout:3950: finishing write tid 5 to smithi02284960-37 2024-09-21T09:44:59.150 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 37 v 2220 (ObjNum 1239 snap 393 seq_num 1239) dirty exists 2024-09-21T09:44:59.150 INFO:tasks.rados.rados.0.smithi022.stdout:3950: left oid 37 (ObjNum 1239 snap 393 seq_num 1239) 2024-09-21T09:44:59.150 INFO:tasks.rados.rados.0.smithi022.stdout:3951: finishing copy_from racing read to smithi02284960-5 2024-09-21T09:44:59.150 INFO:tasks.rados.rados.0.smithi022.stdout:3951: finishing copy_from to smithi02284960-5 2024-09-21T09:44:59.150 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 5 v 1861 (ObjNum 1230 snap 389 seq_num 1230) dirty exists 2024-09-21T09:44:59.150 INFO:tasks.rados.rados.0.smithi022.stdout:3956: seq_num 1241 ranges {2318336=794624,3112960=557056,3670016=245760} 2024-09-21T09:44:59.165 INFO:tasks.rados.rados.0.smithi022.stdout:3956: writing smithi02284960-43 from 2318336 to 3112960 tid 1 2024-09-21T09:44:59.168 INFO:tasks.rados.rados.0.smithi022.stdout:3956: writing smithi02284960-43 from 3112960 to 3670016 tid 2 2024-09-21T09:44:59.170 INFO:tasks.rados.rados.0.smithi022.stdout:3956: writing smithi02284960-43 from 3670016 to 3915776 tid 3 2024-09-21T09:44:59.170 INFO:tasks.rados.rados.0.smithi022.stdout:3950: done (6 left) 2024-09-21T09:44:59.170 INFO:tasks.rados.rados.0.smithi022.stdout:3951: done (5 left) 2024-09-21T09:44:59.170 INFO:tasks.rados.rados.0.smithi022.stdout:3952: done (4 left) 2024-09-21T09:44:59.171 INFO:tasks.rados.rados.0.smithi022.stdout:3953: done (3 left) 2024-09-21T09:44:59.171 INFO:tasks.rados.rados.0.smithi022.stdout:3954: done (2 left) 2024-09-21T09:44:59.171 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 30 current snap is 393 2024-09-21T09:44:59.171 INFO:tasks.rados.rados.0.smithi022.stdout:3955: finishing write tid 1 to smithi02284960-26 2024-09-21T09:44:59.171 INFO:tasks.rados.rados.0.smithi022.stdout:3955: finishing write tid 2 to smithi02284960-26 2024-09-21T09:44:59.171 INFO:tasks.rados.rados.0.smithi022.stdout:3955: finishing write tid 3 to smithi02284960-26 2024-09-21T09:44:59.171 INFO:tasks.rados.rados.0.smithi022.stdout:3955: finishing write tid 4 to smithi02284960-26 2024-09-21T09:44:59.171 INFO:tasks.rados.rados.0.smithi022.stdout:3957: seq_num 1242 ranges {0=409600,409600=442368,851968=655360,1507328=106496} 2024-09-21T09:44:59.172 INFO:tasks.rados.rados.0.smithi022.stdout:3957: writing smithi02284960-30 from 0 to 409600 tid 1 2024-09-21T09:44:59.175 INFO:tasks.rados.rados.0.smithi022.stdout:3957: writing smithi02284960-30 from 409600 to 851968 tid 2 2024-09-21T09:44:59.178 INFO:tasks.rados.rados.0.smithi022.stdout:3957: writing smithi02284960-30 from 851968 to 1507328 tid 3 2024-09-21T09:44:59.179 INFO:tasks.rados.rados.0.smithi022.stdout:3957: writing smithi02284960-30 from 1507328 to 1613824 tid 4 2024-09-21T09:44:59.179 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 42 current snap is 393 2024-09-21T09:44:59.179 INFO:tasks.rados.rados.0.smithi022.stdout:3955: finishing write tid 5 to smithi02284960-26 2024-09-21T09:44:59.179 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 26 v 2016 (ObjNum 1240 snap 393 seq_num 1240) dirty exists 2024-09-21T09:44:59.179 INFO:tasks.rados.rados.0.smithi022.stdout:3955: left oid 26 (ObjNum 1240 snap 393 seq_num 1240) 2024-09-21T09:44:59.179 INFO:tasks.rados.rados.0.smithi022.stdout:3958: seq_num 1243 ranges {2301952=425984,2727936=729088,3457024=466944,3923968=8192} 2024-09-21T09:44:59.194 INFO:tasks.rados.rados.0.smithi022.stdout:3958: writing smithi02284960-42 from 2301952 to 2727936 tid 1 2024-09-21T09:44:59.198 INFO:tasks.rados.rados.0.smithi022.stdout:3958: writing smithi02284960-42 from 2727936 to 3457024 tid 2 2024-09-21T09:44:59.202 INFO:tasks.rados.rados.0.smithi022.stdout:3958: writing smithi02284960-42 from 3457024 to 3923968 tid 3 2024-09-21T09:44:59.202 INFO:tasks.rados.rados.0.smithi022.stdout:3958: writing smithi02284960-42 from 3923968 to 3932160 tid 4 2024-09-21T09:44:59.202 INFO:tasks.rados.rados.0.smithi022.stdout:3955: done (3 left) 2024-09-21T09:44:59.202 INFO:tasks.rados.rados.0.smithi022.stdout:3959: read oid 23 snap -1 2024-09-21T09:44:59.202 INFO:tasks.rados.rados.0.smithi022.stdout:3959: expect (ObjNum 1236 snap 392 seq_num 1236) 2024-09-21T09:44:59.202 INFO:tasks.rados.rados.0.smithi022.stdout:3960: copy_from oid 39 from oid 21 current snap is 393 2024-09-21T09:44:59.202 INFO:tasks.rados.rados.0.smithi022.stdout:3956: finishing write tid 1 to smithi02284960-43 2024-09-21T09:44:59.202 INFO:tasks.rados.rados.0.smithi022.stdout:3956: finishing write tid 2 to smithi02284960-43 2024-09-21T09:44:59.202 INFO:tasks.rados.rados.0.smithi022.stdout:3961: snap_remove snap 367 2024-09-21T09:44:59.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:44:59 smithi064 ceph-mon[86709]: osdmap e856: 8 total, 8 up, 8 in 2024-09-21T09:44:59.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:44:59 smithi064 ceph-mon[86709]: pgmap v898: 121 pgs: 121 active+clean; 178 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 5.9 MiB/s rd, 1.7 MiB/s wr, 15 op/s 2024-09-21T09:44:59.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:44:59 smithi064 ceph-mon[86709]: osdmap e857: 8 total, 8 up, 8 in 2024-09-21T09:44:59.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:44:59 smithi022 ceph-mon[98615]: osdmap e856: 8 total, 8 up, 8 in 2024-09-21T09:44:59.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:44:59 smithi022 ceph-mon[98615]: pgmap v898: 121 pgs: 121 active+clean; 178 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 5.9 MiB/s rd, 1.7 MiB/s wr, 15 op/s 2024-09-21T09:44:59.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:44:59 smithi022 ceph-mon[98615]: osdmap e857: 8 total, 8 up, 8 in 2024-09-21T09:44:59.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:44:59 smithi022 ceph-mon[96736]: osdmap e856: 8 total, 8 up, 8 in 2024-09-21T09:44:59.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:44:59 smithi022 ceph-mon[96736]: pgmap v898: 121 pgs: 121 active+clean; 178 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 5.9 MiB/s rd, 1.7 MiB/s wr, 15 op/s 2024-09-21T09:44:59.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:44:59 smithi022 ceph-mon[96736]: osdmap e857: 8 total, 8 up, 8 in 2024-09-21T09:45:00.159 INFO:tasks.rados.rados.0.smithi022.stdout:3956: finishing write tid 3 to smithi02284960-43 2024-09-21T09:45:00.159 INFO:tasks.rados.rados.0.smithi022.stdout:3956: finishing write tid 4 to smithi02284960-43 2024-09-21T09:45:00.159 INFO:tasks.rados.rados.0.smithi022.stdout:3956: finishing write tid 5 to smithi02284960-43 2024-09-21T09:45:00.159 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 43 v 2127 (ObjNum 1241 snap 393 seq_num 1241) dirty exists 2024-09-21T09:45:00.159 INFO:tasks.rados.rados.0.smithi022.stdout:3956: left oid 43 (ObjNum 1241 snap 393 seq_num 1241) 2024-09-21T09:45:00.159 INFO:tasks.rados.rados.0.smithi022.stdout:3957: finishing write tid 1 to smithi02284960-30 2024-09-21T09:45:00.160 INFO:tasks.rados.rados.0.smithi022.stdout:3957: finishing write tid 2 to smithi02284960-30 2024-09-21T09:45:00.160 INFO:tasks.rados.rados.0.smithi022.stdout:3957: finishing write tid 3 to smithi02284960-30 2024-09-21T09:45:00.160 INFO:tasks.rados.rados.0.smithi022.stdout:3957: finishing write tid 4 to smithi02284960-30 2024-09-21T09:45:00.160 INFO:tasks.rados.rados.0.smithi022.stdout:3957: finishing write tid 5 to smithi02284960-30 2024-09-21T09:45:00.160 INFO:tasks.rados.rados.0.smithi022.stdout:3957: finishing write tid 6 to smithi02284960-30 2024-09-21T09:45:00.160 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 30 v 2160 (ObjNum 1242 snap 393 seq_num 1242) dirty exists 2024-09-21T09:45:00.160 INFO:tasks.rados.rados.0.smithi022.stdout:3957: left oid 30 (ObjNum 1242 snap 393 seq_num 1242) 2024-09-21T09:45:00.160 INFO:tasks.rados.rados.0.smithi022.stdout:3958: finishing write tid 1 to smithi02284960-42 2024-09-21T09:45:00.160 INFO:tasks.rados.rados.0.smithi022.stdout:3958: finishing write tid 2 to smithi02284960-42 2024-09-21T09:45:00.160 INFO:tasks.rados.rados.0.smithi022.stdout:3958: finishing write tid 3 to smithi02284960-42 2024-09-21T09:45:00.160 INFO:tasks.rados.rados.0.smithi022.stdout:3958: finishing write tid 4 to smithi02284960-42 2024-09-21T09:45:00.160 INFO:tasks.rados.rados.0.smithi022.stdout:3958: finishing write tid 5 to smithi02284960-42 2024-09-21T09:45:00.160 INFO:tasks.rados.rados.0.smithi022.stdout:3958: finishing write tid 6 to smithi02284960-42 2024-09-21T09:45:00.160 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 42 v 2021 (ObjNum 1243 snap 393 seq_num 1243) dirty exists 2024-09-21T09:45:00.160 INFO:tasks.rados.rados.0.smithi022.stdout:3958: left oid 42 (ObjNum 1243 snap 393 seq_num 1243) 2024-09-21T09:45:00.160 INFO:tasks.rados.rados.0.smithi022.stdout:3960: finishing copy_from racing read to smithi02284960-39 2024-09-21T09:45:00.161 INFO:tasks.rados.rados.0.smithi022.stdout:3960: finishing copy_from to smithi02284960-39 2024-09-21T09:45:00.161 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 39 v 1869 (ObjNum 1216 snap 386 seq_num 1216) dirty exists 2024-09-21T09:45:00.161 INFO:tasks.rados.rados.0.smithi022.stdout:3959: expect (ObjNum 1236 snap 392 seq_num 1236) 2024-09-21T09:45:00.459 INFO:tasks.rados.rados.0.smithi022.stdout:3956: done (5 left) 2024-09-21T09:45:00.459 INFO:tasks.rados.rados.0.smithi022.stdout:3957: done (4 left) 2024-09-21T09:45:00.459 INFO:tasks.rados.rados.0.smithi022.stdout:3958: done (3 left) 2024-09-21T09:45:00.459 INFO:tasks.rados.rados.0.smithi022.stdout:3959: done (2 left) 2024-09-21T09:45:00.459 INFO:tasks.rados.rados.0.smithi022.stdout:3960: done (1 left) 2024-09-21T09:45:00.459 INFO:tasks.rados.rados.0.smithi022.stdout:3961: done (0 left) 2024-09-21T09:45:00.459 INFO:tasks.rados.rados.0.smithi022.stdout:3962: copy_from oid 45 from oid 30 current snap is 393 2024-09-21T09:45:00.460 INFO:tasks.rados.rados.0.smithi022.stdout:3963: rollback oid 35 current snap is 393 2024-09-21T09:45:00.460 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 35 to 384 2024-09-21T09:45:00.460 INFO:tasks.rados.rados.0.smithi022.stdout:3964: copy_from oid 11 from oid 41 current snap is 393 2024-09-21T09:45:00.460 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 14 current snap is 393 2024-09-21T09:45:00.460 INFO:tasks.rados.rados.0.smithi022.stdout:3965: seq_num 1244 ranges {0=442368,442368=614400,1056768=589824} 2024-09-21T09:45:00.461 INFO:tasks.rados.rados.0.smithi022.stdout:3965: writing smithi02284960-14 from 0 to 442368 tid 1 2024-09-21T09:45:00.466 INFO:tasks.rados.rados.0.smithi022.stdout:3965: writing smithi02284960-14 from 442368 to 1056768 tid 2 2024-09-21T09:45:00.470 INFO:tasks.rados.rados.0.smithi022.stdout:3965: writing smithi02284960-14 from 1056768 to 1646592 tid 3 2024-09-21T09:45:00.470 INFO:tasks.rados.rados.0.smithi022.stdout:3966: copy_from oid 32 from oid 5 current snap is 393 2024-09-21T09:45:00.470 INFO:tasks.rados.rados.0.smithi022.stdout:3963: finishing rollback tid 0 to smithi02284960-35 2024-09-21T09:45:00.470 INFO:tasks.rados.rados.0.smithi022.stdout:3963: finishing rollback tid 1 to smithi02284960-35 2024-09-21T09:45:00.470 INFO:tasks.rados.rados.0.smithi022.stdout:3963: finishing rollback tid 2 to smithi02284960-35 2024-09-21T09:45:00.470 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 35 v 1998 (ObjNum 1137 snap 349 seq_num 1137) dirty exists 2024-09-21T09:45:00.470 INFO:tasks.rados.rados.0.smithi022.stdout:3965: finishing write tid 1 to smithi02284960-14 2024-09-21T09:45:00.470 INFO:tasks.rados.rados.0.smithi022.stdout:3962: finishing copy_from racing read to smithi02284960-45 2024-09-21T09:45:00.470 INFO:tasks.rados.rados.0.smithi022.stdout:3963: done (4 left) 2024-09-21T09:45:00.470 INFO:tasks.rados.rados.0.smithi022.stdout:3967: snap_remove snap 381 2024-09-21T09:45:00.692 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:45:00 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:45:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:45:01.164 INFO:tasks.rados.rados.0.smithi022.stdout:3967: done (4 left) 2024-09-21T09:45:01.164 INFO:tasks.rados.rados.0.smithi022.stdout:3968: snap_create 2024-09-21T09:45:01.164 INFO:tasks.rados.rados.0.smithi022.stdout:3965: finishing write tid 2 to smithi02284960-14 2024-09-21T09:45:01.165 INFO:tasks.rados.rados.0.smithi022.stdout:3962: finishing copy_from to smithi02284960-45 2024-09-21T09:45:01.165 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 45 v 2160 (ObjNum 1242 snap 393 seq_num 1242) dirty exists 2024-09-21T09:45:01.165 INFO:tasks.rados.rados.0.smithi022.stdout:3964: finishing copy_from racing read to smithi02284960-11 2024-09-21T09:45:01.165 INFO:tasks.rados.rados.0.smithi022.stdout:3965: finishing write tid 3 to smithi02284960-14 2024-09-21T09:45:01.165 INFO:tasks.rados.rados.0.smithi022.stdout:3965: finishing write tid 4 to smithi02284960-14 2024-09-21T09:45:01.165 INFO:tasks.rados.rados.0.smithi022.stdout:3964: finishing copy_from to smithi02284960-11 2024-09-21T09:45:01.165 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 11 v 2119 (ObjNum 1120 snap 341 seq_num 1120) dirty exists 2024-09-21T09:45:01.165 INFO:tasks.rados.rados.0.smithi022.stdout:3966: finishing copy_from racing read to smithi02284960-32 2024-09-21T09:45:01.165 INFO:tasks.rados.rados.0.smithi022.stdout:3966: finishing copy_from to smithi02284960-32 2024-09-21T09:45:01.165 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 32 v 1861 (ObjNum 1230 snap 389 seq_num 1230) dirty exists 2024-09-21T09:45:01.165 INFO:tasks.rados.rados.0.smithi022.stdout:3965: finishing write tid 5 to smithi02284960-14 2024-09-21T09:45:01.165 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 14 v 1816 (ObjNum 1244 snap 393 seq_num 1244) dirty exists 2024-09-21T09:45:01.165 INFO:tasks.rados.rados.0.smithi022.stdout:3965: left oid 14 (ObjNum 1244 snap 393 seq_num 1244) 2024-09-21T09:45:01.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:01 smithi064 ceph-mon[86709]: osdmap e858: 8 total, 8 up, 8 in 2024-09-21T09:45:01.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:01 smithi064 ceph-mon[86709]: pgmap v901: 121 pgs: 121 active+clean; 180 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.7 MiB/s wr, 13 op/s 2024-09-21T09:45:01.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:01 smithi022 ceph-mon[96736]: osdmap e858: 8 total, 8 up, 8 in 2024-09-21T09:45:01.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:01 smithi022 ceph-mon[96736]: pgmap v901: 121 pgs: 121 active+clean; 180 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.7 MiB/s wr, 13 op/s 2024-09-21T09:45:01.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:01 smithi022 ceph-mon[98615]: osdmap e858: 8 total, 8 up, 8 in 2024-09-21T09:45:01.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:01 smithi022 ceph-mon[98615]: pgmap v901: 121 pgs: 121 active+clean; 180 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.7 MiB/s wr, 13 op/s 2024-09-21T09:45:02.165 INFO:tasks.rados.rados.0.smithi022.stdout:3962: done (4 left) 2024-09-21T09:45:02.165 INFO:tasks.rados.rados.0.smithi022.stdout:3964: done (3 left) 2024-09-21T09:45:02.165 INFO:tasks.rados.rados.0.smithi022.stdout:3965: done (2 left) 2024-09-21T09:45:02.165 INFO:tasks.rados.rados.0.smithi022.stdout:3966: done (1 left) 2024-09-21T09:45:02.166 INFO:tasks.rados.rados.0.smithi022.stdout:3968: done (0 left) 2024-09-21T09:45:02.166 INFO:tasks.rados.rados.0.smithi022.stdout:3969: rollback oid 42 current snap is 394 2024-09-21T09:45:02.166 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 42 to 384 2024-09-21T09:45:02.166 INFO:tasks.rados.rados.0.smithi022.stdout:3970: read oid 27 snap -1 2024-09-21T09:45:02.166 INFO:tasks.rados.rados.0.smithi022.stdout:3970: expect (ObjNum 1008 snap 309 seq_num 1008) 2024-09-21T09:45:02.166 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 32 current snap is 394 2024-09-21T09:45:02.166 INFO:tasks.rados.rados.0.smithi022.stdout:3971: seq_num 1245 ranges {1409024=458752,1867776=499712,2367488=425984,2793472=278528} 2024-09-21T09:45:02.175 INFO:tasks.rados.rados.0.smithi022.stdout:3971: writing smithi02284960-32 from 1409024 to 1867776 tid 1 2024-09-21T09:45:02.178 INFO:tasks.rados.rados.0.smithi022.stdout:3971: writing smithi02284960-32 from 1867776 to 2367488 tid 2 2024-09-21T09:45:02.182 INFO:tasks.rados.rados.0.smithi022.stdout:3971: writing smithi02284960-32 from 2367488 to 2793472 tid 3 2024-09-21T09:45:02.183 INFO:tasks.rados.rados.0.smithi022.stdout:3971: writing smithi02284960-32 from 2793472 to 3072000 tid 4 2024-09-21T09:45:02.183 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 1 current snap is 394 2024-09-21T09:45:02.184 INFO:tasks.rados.rados.0.smithi022.stdout:3969: finishing rollback tid 0 to smithi02284960-42 2024-09-21T09:45:02.184 INFO:tasks.rados.rados.0.smithi022.stdout:3969: finishing rollback tid 1 to smithi02284960-42 2024-09-21T09:45:02.184 INFO:tasks.rados.rados.0.smithi022.stdout:3969: finishing rollback tid 2 to smithi02284960-42 2024-09-21T09:45:02.184 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 42 v 2024 (ObjNum 1206 snap 382 seq_num 1206) dirty exists 2024-09-21T09:45:02.184 INFO:tasks.rados.rados.0.smithi022.stdout:3970: expect (ObjNum 1008 snap 309 seq_num 1008) 2024-09-21T09:45:02.191 INFO:tasks.rados.rados.0.smithi022.stdout:3971: finishing write tid 1 to smithi02284960-32 2024-09-21T09:45:02.191 INFO:tasks.rados.rados.0.smithi022.stdout:3971: finishing write tid 2 to smithi02284960-32 2024-09-21T09:45:02.191 INFO:tasks.rados.rados.0.smithi022.stdout:3971: finishing write tid 3 to smithi02284960-32 2024-09-21T09:45:02.191 INFO:tasks.rados.rados.0.smithi022.stdout:3971: finishing write tid 4 to smithi02284960-32 2024-09-21T09:45:02.191 INFO:tasks.rados.rados.0.smithi022.stdout:3971: finishing write tid 5 to smithi02284960-32 2024-09-21T09:45:02.191 INFO:tasks.rados.rados.0.smithi022.stdout:3972: seq_num 1246 ranges {0=475136,475136=786432,1261568=417792} 2024-09-21T09:45:02.194 INFO:tasks.rados.rados.0.smithi022.stdout:3972: writing smithi02284960-1 from 0 to 475136 tid 1 2024-09-21T09:45:02.198 INFO:tasks.rados.rados.0.smithi022.stdout:3972: writing smithi02284960-1 from 475136 to 1261568 tid 2 2024-09-21T09:45:02.201 INFO:tasks.rados.rados.0.smithi022.stdout:3972: writing smithi02284960-1 from 1261568 to 1679360 tid 3 2024-09-21T09:45:02.201 INFO:tasks.rados.rados.0.smithi022.stdout:3969: done (3 left) 2024-09-21T09:45:02.201 INFO:tasks.rados.rados.0.smithi022.stdout:3970: done (2 left) 2024-09-21T09:45:02.201 INFO:tasks.rados.rados.0.smithi022.stdout:3973: delete oid 19 current snap is 394 2024-09-21T09:45:02.201 INFO:tasks.rados.rados.0.smithi022.stdout:3971: finishing write tid 6 to smithi02284960-32 2024-09-21T09:45:02.201 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 32 v 2206 (ObjNum 1245 snap 394 seq_num 1245) dirty exists 2024-09-21T09:45:02.201 INFO:tasks.rados.rados.0.smithi022.stdout:3971: left oid 32 (ObjNum 1245 snap 394 seq_num 1245) 2024-09-21T09:45:02.201 INFO:tasks.rados.rados.0.smithi022.stdout:3972: finishing write tid 1 to smithi02284960-1 2024-09-21T09:45:02.203 INFO:tasks.rados.rados.0.smithi022.stdout:3972: finishing write tid 2 to smithi02284960-1 2024-09-21T09:45:02.204 INFO:tasks.rados.rados.0.smithi022.stdout:3972: finishing write tid 3 to smithi02284960-1 2024-09-21T09:45:02.204 INFO:tasks.rados.rados.0.smithi022.stdout:3971: done (2 left) 2024-09-21T09:45:02.204 INFO:tasks.rados.rados.0.smithi022.stdout:3973: done (1 left) 2024-09-21T09:45:02.204 INFO:tasks.rados.rados.0.smithi022.stdout:3974: snap_remove snap 383 2024-09-21T09:45:02.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:02 smithi064 ceph-mon[86709]: osdmap e859: 8 total, 8 up, 8 in 2024-09-21T09:45:02.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:02 smithi022 ceph-mon[98615]: osdmap e859: 8 total, 8 up, 8 in 2024-09-21T09:45:02.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:02 smithi022 ceph-mon[96736]: osdmap e859: 8 total, 8 up, 8 in 2024-09-21T09:45:03.176 INFO:tasks.rados.rados.0.smithi022.stdout:3974: done (1 left) 2024-09-21T09:45:03.177 INFO:tasks.rados.rados.0.smithi022.stdout:3975: read oid 19 snap -1 2024-09-21T09:45:03.177 INFO:tasks.rados.rados.0.smithi022.stdout:3975: expect deleted 2024-09-21T09:45:03.177 INFO:tasks.rados.rados.0.smithi022.stdout:3976: snap_create 2024-09-21T09:45:03.177 INFO:tasks.rados.rados.0.smithi022.stdout:3972: finishing write tid 4 to smithi02284960-1 2024-09-21T09:45:03.177 INFO:tasks.rados.rados.0.smithi022.stdout:3972: finishing write tid 5 to smithi02284960-1 2024-09-21T09:45:03.177 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 1 v 2123 (ObjNum 1246 snap 394 seq_num 1246) dirty exists 2024-09-21T09:45:03.177 INFO:tasks.rados.rados.0.smithi022.stdout:3972: left oid 1 (ObjNum 1246 snap 394 seq_num 1246) 2024-09-21T09:45:03.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:03 smithi064 ceph-mon[86709]: osdmap e860: 8 total, 8 up, 8 in 2024-09-21T09:45:03.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:03 smithi064 ceph-mon[86709]: pgmap v904: 121 pgs: 121 active+clean; 188 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 4.2 MiB/s wr, 12 op/s 2024-09-21T09:45:03.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:03 smithi022 ceph-mon[98615]: osdmap e860: 8 total, 8 up, 8 in 2024-09-21T09:45:03.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:03 smithi022 ceph-mon[98615]: pgmap v904: 121 pgs: 121 active+clean; 188 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 4.2 MiB/s wr, 12 op/s 2024-09-21T09:45:03.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:03 smithi022 ceph-mon[96736]: osdmap e860: 8 total, 8 up, 8 in 2024-09-21T09:45:03.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:03 smithi022 ceph-mon[96736]: pgmap v904: 121 pgs: 121 active+clean; 188 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 4.2 MiB/s wr, 12 op/s 2024-09-21T09:45:04.122 INFO:tasks.rados.rados.0.smithi022.stdout:3972: done (2 left) 2024-09-21T09:45:04.122 INFO:tasks.rados.rados.0.smithi022.stdout:3975: done (1 left) 2024-09-21T09:45:04.122 INFO:tasks.rados.rados.0.smithi022.stdout:3976: done (0 left) 2024-09-21T09:45:04.122 INFO:tasks.rados.rados.0.smithi022.stdout:3977: snap_create 2024-09-21T09:45:04.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:04 smithi064 ceph-mon[86709]: osdmap e861: 8 total, 8 up, 8 in 2024-09-21T09:45:04.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:04 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:45:04.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:04 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:45:04.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:04 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:45:04.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:04 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:45:04.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:04 smithi064 ceph-mon[86709]: osdmap e862: 8 total, 8 up, 8 in 2024-09-21T09:45:04.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:04 smithi022 ceph-mon[98615]: osdmap e861: 8 total, 8 up, 8 in 2024-09-21T09:45:04.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:04 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:45:04.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:04 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:45:04.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:04 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:45:04.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:04 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:45:04.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:04 smithi022 ceph-mon[98615]: osdmap e862: 8 total, 8 up, 8 in 2024-09-21T09:45:04.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:04 smithi022 ceph-mon[96736]: osdmap e861: 8 total, 8 up, 8 in 2024-09-21T09:45:04.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:04 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:45:04.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:04 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:45:04.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:04 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:45:04.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:04 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:45:04.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:04 smithi022 ceph-mon[96736]: osdmap e862: 8 total, 8 up, 8 in 2024-09-21T09:45:05.124 INFO:tasks.rados.rados.0.smithi022.stdout:3977: done (0 left) 2024-09-21T09:45:05.124 INFO:tasks.rados.rados.0.smithi022.stdout:3978: rmattr oid 7 current snap is 396 2024-09-21T09:45:05.124 INFO:tasks.rados.rados.0.smithi022.stdout:3978: done (0 left) 2024-09-21T09:45:05.124 INFO:tasks.rados.rados.0.smithi022.stdout:3979: read oid 17 snap 386 2024-09-21T09:45:05.124 INFO:tasks.rados.rados.0.smithi022.stdout:3979: expect (ObjNum 1200 snap 379 seq_num 1200) 2024-09-21T09:45:05.124 INFO:tasks.rados.rados.0.smithi022.stdout:3980: read oid 46 snap -1 2024-09-21T09:45:05.124 INFO:tasks.rados.rados.0.smithi022.stdout:3980: expect deleted 2024-09-21T09:45:05.124 INFO:tasks.rados.rados.0.smithi022.stdout:3981: snap_create 2024-09-21T09:45:05.128 INFO:tasks.rados.rados.0.smithi022.stdout:3979: expect (ObjNum 1200 snap 379 seq_num 1200) 2024-09-21T09:45:06.070 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:45:05 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:45:05] "GET /metrics HTTP/1.1" 200 36354 "" "Prometheus/2.51.0" 2024-09-21T09:45:06.186 INFO:tasks.rados.rados.0.smithi022.stdout:3979: done (2 left) 2024-09-21T09:45:06.187 INFO:tasks.rados.rados.0.smithi022.stdout:3980: done (1 left) 2024-09-21T09:45:06.187 INFO:tasks.rados.rados.0.smithi022.stdout:3981: done (0 left) 2024-09-21T09:45:06.187 INFO:tasks.rados.rados.0.smithi022.stdout:3982: snap_create 2024-09-21T09:45:06.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:06 smithi064 ceph-mon[86709]: pgmap v907: 121 pgs: 121 active+clean; 198 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 3.8 MiB/s rd, 2.9 MiB/s wr, 10 op/s 2024-09-21T09:45:06.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:06 smithi064 ceph-mon[86709]: osdmap e863: 8 total, 8 up, 8 in 2024-09-21T09:45:06.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:06 smithi022 ceph-mon[98615]: pgmap v907: 121 pgs: 121 active+clean; 198 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 3.8 MiB/s rd, 2.9 MiB/s wr, 10 op/s 2024-09-21T09:45:06.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:06 smithi022 ceph-mon[98615]: osdmap e863: 8 total, 8 up, 8 in 2024-09-21T09:45:06.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:06 smithi022 ceph-mon[96736]: pgmap v907: 121 pgs: 121 active+clean; 198 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 3.8 MiB/s rd, 2.9 MiB/s wr, 10 op/s 2024-09-21T09:45:06.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:06 smithi022 ceph-mon[96736]: osdmap e863: 8 total, 8 up, 8 in 2024-09-21T09:45:07.137 INFO:tasks.rados.rados.0.smithi022.stdout:3982: done (0 left) 2024-09-21T09:45:07.137 INFO:tasks.rados.rados.0.smithi022.stdout:3983: snap_remove snap 386 2024-09-21T09:45:07.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:07 smithi064 ceph-mon[86709]: osdmap e864: 8 total, 8 up, 8 in 2024-09-21T09:45:07.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:07 smithi022 ceph-mon[98615]: osdmap e864: 8 total, 8 up, 8 in 2024-09-21T09:45:07.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:07 smithi022 ceph-mon[96736]: osdmap e864: 8 total, 8 up, 8 in 2024-09-21T09:45:08.139 INFO:tasks.rados.rados.0.smithi022.stdout:3983: done (0 left) 2024-09-21T09:45:08.139 INFO:tasks.rados.rados.0.smithi022.stdout:3984: rmattr oid 50 current snap is 398 2024-09-21T09:45:08.139 INFO:tasks.rados.rados.0.smithi022.stdout:3984: done (0 left) 2024-09-21T09:45:08.139 INFO:tasks.rados.rados.0.smithi022.stdout:3985: read oid 21 snap 384 2024-09-21T09:45:08.139 INFO:tasks.rados.rados.0.smithi022.stdout:3985: expect deleted 2024-09-21T09:45:08.139 INFO:tasks.rados.rados.0.smithi022.stdout:3986: read oid 37 snap -1 2024-09-21T09:45:08.139 INFO:tasks.rados.rados.0.smithi022.stdout:3986: expect (ObjNum 1239 snap 393 seq_num 1239) 2024-09-21T09:45:08.139 INFO:tasks.rados.rados.0.smithi022.stdout:3987: rollback oid 26 current snap is 398 2024-09-21T09:45:08.139 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 26 to 369 2024-09-21T09:45:08.139 INFO:tasks.rados.rados.0.smithi022.stdout:3988: delete oid 47 current snap is 398 2024-09-21T09:45:08.152 INFO:tasks.rados.rados.0.smithi022.stdout:3985: done (3 left) 2024-09-21T09:45:08.152 INFO:tasks.rados.rados.0.smithi022.stdout:3988: done (2 left) 2024-09-21T09:45:08.152 INFO:tasks.rados.rados.0.smithi022.stdout:3989: setattr oid 18 current snap is 398 2024-09-21T09:45:08.152 INFO:tasks.rados.rados.0.smithi022.stdout:3990: rollback oid 36 current snap is 398 2024-09-21T09:45:08.152 INFO:tasks.rados.rados.0.smithi022.stdout:rollback oid 36 to 397 2024-09-21T09:45:08.152 INFO:tasks.rados.rados.0.smithi022.stdout:3991: snap_remove snap 395 2024-09-21T09:45:08.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:08 smithi064 ceph-mon[86709]: pgmap v910: 121 pgs: 121 active+clean; 203 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 4.5 MiB/s rd, 2.4 MiB/s wr, 9 op/s 2024-09-21T09:45:08.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:08 smithi064 ceph-mon[86709]: osdmap e865: 8 total, 8 up, 8 in 2024-09-21T09:45:08.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:08 smithi022 ceph-mon[96736]: pgmap v910: 121 pgs: 121 active+clean; 203 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 4.5 MiB/s rd, 2.4 MiB/s wr, 9 op/s 2024-09-21T09:45:08.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:08 smithi022 ceph-mon[96736]: osdmap e865: 8 total, 8 up, 8 in 2024-09-21T09:45:08.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:08 smithi022 ceph-mon[98615]: pgmap v910: 121 pgs: 121 active+clean; 203 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 4.5 MiB/s rd, 2.4 MiB/s wr, 9 op/s 2024-09-21T09:45:08.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:08 smithi022 ceph-mon[98615]: osdmap e865: 8 total, 8 up, 8 in 2024-09-21T09:45:09.125 INFO:tasks.rados.rados.0.smithi022.stdout:3991: done (4 left) 2024-09-21T09:45:09.125 INFO:tasks.rados.rados.0.smithi022.stdout:3992: read oid 19 snap -1 2024-09-21T09:45:09.125 INFO:tasks.rados.rados.0.smithi022.stdout:3992: expect deleted 2024-09-21T09:45:09.125 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 41 current snap is 398 2024-09-21T09:45:09.125 INFO:tasks.rados.rados.0.smithi022.stdout:3993: seq_num 1249 ranges {1966080=524288,2490368=483328,2973696=409600,3383296=311296} 2024-09-21T09:45:09.139 INFO:tasks.rados.rados.0.smithi022.stdout:3993: writing smithi02284960-41 from 1966080 to 2490368 tid 1 2024-09-21T09:45:09.141 INFO:tasks.rados.rados.0.smithi022.stdout:3993: writing smithi02284960-41 from 2490368 to 2973696 tid 2 2024-09-21T09:45:09.144 INFO:tasks.rados.rados.0.smithi022.stdout:3993: writing smithi02284960-41 from 2973696 to 3383296 tid 3 2024-09-21T09:45:09.146 INFO:tasks.rados.rados.0.smithi022.stdout:3993: writing smithi02284960-41 from 3383296 to 3694592 tid 4 2024-09-21T09:45:09.146 INFO:tasks.rados.rados.0.smithi022.stdout:3994: snap_remove snap 396 2024-09-21T09:45:09.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:09 smithi064 ceph-mon[86709]: osdmap e866: 8 total, 8 up, 8 in 2024-09-21T09:45:09.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:09 smithi064 ceph-mon[86709]: pgmap v913: 121 pgs: 121 active+clean; 203 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 747 KiB/s rd, 1.6 MiB/s wr, 6 op/s 2024-09-21T09:45:09.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:09 smithi064 ceph-mon[86709]: osdmap e867: 8 total, 8 up, 8 in 2024-09-21T09:45:09.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:09 smithi022 ceph-mon[98615]: osdmap e866: 8 total, 8 up, 8 in 2024-09-21T09:45:09.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:09 smithi022 ceph-mon[98615]: pgmap v913: 121 pgs: 121 active+clean; 203 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 747 KiB/s rd, 1.6 MiB/s wr, 6 op/s 2024-09-21T09:45:09.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:09 smithi022 ceph-mon[98615]: osdmap e867: 8 total, 8 up, 8 in 2024-09-21T09:45:09.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:09 smithi022 ceph-mon[96736]: osdmap e866: 8 total, 8 up, 8 in 2024-09-21T09:45:09.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:09 smithi022 ceph-mon[96736]: pgmap v913: 121 pgs: 121 active+clean; 203 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 747 KiB/s rd, 1.6 MiB/s wr, 6 op/s 2024-09-21T09:45:09.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:09 smithi022 ceph-mon[96736]: osdmap e867: 8 total, 8 up, 8 in 2024-09-21T09:45:10.152 INFO:tasks.rados.rados.0.smithi022.stdout:3987: finishing rollback tid 0 to smithi02284960-26 2024-09-21T09:45:10.152 INFO:tasks.rados.rados.0.smithi022.stdout:3987: finishing rollback tid 1 to smithi02284960-26 2024-09-21T09:45:10.152 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 26 v 2026 (ObjNum 161 snap 0 seq_num 1) dirty dne 2024-09-21T09:45:10.153 INFO:tasks.rados.rados.0.smithi022.stdout:3990: finishing rollback tid 0 to smithi02284960-36 2024-09-21T09:45:10.153 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 18 v 1760 (ObjNum 1062 snap 319 seq_num 1062) dirty exists 2024-09-21T09:45:10.153 INFO:tasks.rados.rados.0.smithi022.stdout:3990: finishing rollback tid 1 to smithi02284960-36 2024-09-21T09:45:10.153 INFO:tasks.rados.rados.0.smithi022.stdout:3990: finishing rollback tid 2 to smithi02284960-36 2024-09-21T09:45:10.153 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 36 v 2163 (ObjNum 1212 snap 385 seq_num 1212) dirty exists 2024-09-21T09:45:10.153 INFO:tasks.rados.rados.0.smithi022.stdout:3986: expect (ObjNum 1239 snap 393 seq_num 1239) 2024-09-21T09:45:10.197 INFO:tasks.rados.rados.0.smithi022.stdout:3993: finishing write tid 1 to smithi02284960-41 2024-09-21T09:45:10.198 INFO:tasks.rados.rados.0.smithi022.stdout:3993: finishing write tid 2 to smithi02284960-41 2024-09-21T09:45:10.198 INFO:tasks.rados.rados.0.smithi022.stdout:3993: finishing write tid 3 to smithi02284960-41 2024-09-21T09:45:10.198 INFO:tasks.rados.rados.0.smithi022.stdout:3993: finishing write tid 4 to smithi02284960-41 2024-09-21T09:45:10.198 INFO:tasks.rados.rados.0.smithi022.stdout:3993: finishing write tid 5 to smithi02284960-41 2024-09-21T09:45:10.198 INFO:tasks.rados.rados.0.smithi022.stdout:3993: finishing write tid 6 to smithi02284960-41 2024-09-21T09:45:10.198 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 41 v 2132 (ObjNum 1249 snap 398 seq_num 1249) dirty exists 2024-09-21T09:45:10.198 INFO:tasks.rados.rados.0.smithi022.stdout:3993: left oid 41 (ObjNum 1249 snap 398 seq_num 1249) 2024-09-21T09:45:10.198 INFO:tasks.rados.rados.0.smithi022.stdout:3986: done (6 left) 2024-09-21T09:45:10.198 INFO:tasks.rados.rados.0.smithi022.stdout:3987: done (5 left) 2024-09-21T09:45:10.198 INFO:tasks.rados.rados.0.smithi022.stdout:3989: done (4 left) 2024-09-21T09:45:10.198 INFO:tasks.rados.rados.0.smithi022.stdout:3990: done (3 left) 2024-09-21T09:45:10.198 INFO:tasks.rados.rados.0.smithi022.stdout:3992: done (2 left) 2024-09-21T09:45:10.198 INFO:tasks.rados.rados.0.smithi022.stdout:3993: done (1 left) 2024-09-21T09:45:10.199 INFO:tasks.rados.rados.0.smithi022.stdout:3994: done (0 left) 2024-09-21T09:45:10.199 INFO:tasks.rados.rados.0.smithi022.stdout:3995: read oid 24 snap -1 2024-09-21T09:45:10.199 INFO:tasks.rados.rados.0.smithi022.stdout:3995: expect (ObjNum 1079 snap 326 seq_num 1079) 2024-09-21T09:45:10.199 INFO:tasks.rados.rados.0.smithi022.stdout:3996: read oid 46 snap -1 2024-09-21T09:45:10.199 INFO:tasks.rados.rados.0.smithi022.stdout:3996: expect deleted 2024-09-21T09:45:10.199 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 23 current snap is 398 2024-09-21T09:45:10.199 INFO:tasks.rados.rados.0.smithi022.stdout:3997: seq_num 1250 ranges {5308416=540672,5849088=770048,6619136=434176} 2024-09-21T09:45:10.229 INFO:tasks.rados.rados.0.smithi022.stdout:3997: writing smithi02284960-23 from 5308416 to 5849088 tid 1 2024-09-21T09:45:10.234 INFO:tasks.rados.rados.0.smithi022.stdout:3997: writing smithi02284960-23 from 5849088 to 6619136 tid 2 2024-09-21T09:45:10.236 INFO:tasks.rados.rados.0.smithi022.stdout:3997: writing smithi02284960-23 from 6619136 to 7053312 tid 3 2024-09-21T09:45:10.236 INFO:tasks.rados.rados.0.smithi022.stdout:append oid 40 current snap is 398 2024-09-21T09:45:10.236 INFO:tasks.rados.rados.0.smithi022.stdout:3998: seq_num 1251 ranges {417792=557056,974848=655360,1630208=425984,2056192=122880} 2024-09-21T09:45:10.242 INFO:tasks.rados.rados.0.smithi022.stdout:3998: writing smithi02284960-40 from 417792 to 974848 tid 1 2024-09-21T09:45:10.245 INFO:tasks.rados.rados.0.smithi022.stdout:3998: writing smithi02284960-40 from 974848 to 1630208 tid 2 2024-09-21T09:45:10.248 INFO:tasks.rados.rados.0.smithi022.stdout:3998: writing smithi02284960-40 from 1630208 to 2056192 tid 3 2024-09-21T09:45:10.249 INFO:tasks.rados.rados.0.smithi022.stdout:3998: writing smithi02284960-40 from 2056192 to 2179072 tid 4 2024-09-21T09:45:10.249 INFO:tasks.rados.rados.0.smithi022.stdout:3999: read oid 21 snap -1 2024-09-21T09:45:10.249 INFO:tasks.rados.rados.0.smithi022.stdout:3999: expect (ObjNum 1216 snap 386 seq_num 1216) 2024-09-21T09:45:10.249 INFO:tasks.rados.rados.0.smithi022.stdout: waiting on 5 2024-09-21T09:45:10.250 INFO:tasks.rados.rados.0.smithi022.stdout:3995: expect (ObjNum 1079 snap 326 seq_num 1079) 2024-09-21T09:45:10.691 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:45:10 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:45:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:45:10.771 INFO:tasks.rados.rados.0.smithi022.stdout:3997: finishing write tid 1 to smithi02284960-23 2024-09-21T09:45:10.771 INFO:tasks.rados.rados.0.smithi022.stdout:3997: finishing write tid 2 to smithi02284960-23 2024-09-21T09:45:10.771 INFO:tasks.rados.rados.0.smithi022.stdout:3997: finishing write tid 3 to smithi02284960-23 2024-09-21T09:45:10.771 INFO:tasks.rados.rados.0.smithi022.stdout:3997: finishing write tid 4 to smithi02284960-23 2024-09-21T09:45:10.771 INFO:tasks.rados.rados.0.smithi022.stdout:3997: finishing write tid 5 to smithi02284960-23 2024-09-21T09:45:10.771 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 23 v 1875 (ObjNum 1250 snap 398 seq_num 1250) dirty exists 2024-09-21T09:45:10.771 INFO:tasks.rados.rados.0.smithi022.stdout:3997: left oid 23 (ObjNum 1250 snap 398 seq_num 1250) 2024-09-21T09:45:10.771 INFO:tasks.rados.rados.0.smithi022.stdout:3998: finishing write tid 1 to smithi02284960-40 2024-09-21T09:45:10.771 INFO:tasks.rados.rados.0.smithi022.stdout:3998: finishing write tid 2 to smithi02284960-40 2024-09-21T09:45:10.771 INFO:tasks.rados.rados.0.smithi022.stdout:3998: finishing write tid 3 to smithi02284960-40 2024-09-21T09:45:10.771 INFO:tasks.rados.rados.0.smithi022.stdout:3998: finishing write tid 4 to smithi02284960-40 2024-09-21T09:45:10.772 INFO:tasks.rados.rados.0.smithi022.stdout:3998: finishing write tid 5 to smithi02284960-40 2024-09-21T09:45:10.772 INFO:tasks.rados.rados.0.smithi022.stdout:3999: expect (ObjNum 1216 snap 386 seq_num 1216) 2024-09-21T09:45:10.803 INFO:tasks.rados.rados.0.smithi022.stdout:3998: finishing write tid 6 to smithi02284960-40 2024-09-21T09:45:10.803 INFO:tasks.rados.rados.0.smithi022.stdout:update_object_version oid 40 v 2137 (ObjNum 1251 snap 398 seq_num 1251) dirty exists 2024-09-21T09:45:10.804 INFO:tasks.rados.rados.0.smithi022.stdout:3998: left oid 40 (ObjNum 1251 snap 398 seq_num 1251) 2024-09-21T09:45:10.804 INFO:tasks.rados.rados.0.smithi022.stdout:3995: done (4 left) 2024-09-21T09:45:10.804 INFO:tasks.rados.rados.0.smithi022.stdout:3996: done (3 left) 2024-09-21T09:45:10.804 INFO:tasks.rados.rados.0.smithi022.stdout:3997: done (2 left) 2024-09-21T09:45:10.804 INFO:tasks.rados.rados.0.smithi022.stdout:3998: done (1 left) 2024-09-21T09:45:10.804 INFO:tasks.rados.rados.0.smithi022.stdout:3999: done (0 left) 2024-09-21T09:45:10.806 INFO:tasks.rados.rados.0.smithi022.stderr:0 errors. 2024-09-21T09:45:10.806 INFO:tasks.rados.rados.0.smithi022.stderr: 2024-09-21T09:45:10.808 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid d65e801e-77fa-11ef-baf6-efdc52797490 -- ceph osd dump --format=json 2024-09-21T09:45:11.102 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:45:11.152 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:11 smithi022 ceph-mon[96736]: osdmap e868: 8 total, 8 up, 8 in 2024-09-21T09:45:11.152 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:11 smithi022 ceph-mon[96736]: pgmap v916: 121 pgs: 121 active+clean; 204 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 422 KiB/s wr, 6 op/s 2024-09-21T09:45:11.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:11 smithi064 ceph-mon[86709]: osdmap e868: 8 total, 8 up, 8 in 2024-09-21T09:45:11.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:11 smithi064 ceph-mon[86709]: pgmap v916: 121 pgs: 121 active+clean; 204 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 422 KiB/s wr, 6 op/s 2024-09-21T09:45:11.507 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:11 smithi022 ceph-mon[98615]: osdmap e868: 8 total, 8 up, 8 in 2024-09-21T09:45:11.507 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:11 smithi022 ceph-mon[98615]: pgmap v916: 121 pgs: 121 active+clean; 204 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 422 KiB/s wr, 6 op/s 2024-09-21T09:45:13.364 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-09-21T09:45:13.364 INFO:teuthology.orchestra.run.smithi022.stdout:{"epoch":868,"fsid":"d65e801e-77fa-11ef-baf6-efdc52797490","created":"2024-09-21T09:22:04.474917+0000","modified":"2024-09-21T09:45:10.144203+0000","last_up_change":"2024-09-21T09:40:02.098011+0000","last_in_change":"2024-09-21T09:26:49.376072+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":27,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":5,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-21T09:25:07.405775+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-09-21T09:27:31.151960+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"57","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-09-21T09:27:34.558875+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"57","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"rbd","create_time":"2024-09-21T09:28:27.564651+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"66","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":66,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":3,"score_stable":3,"optimal_score":1,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":5,"pool_name":"unique_pool_0","create_time":"2024-09-21T09:31:18.331126+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":3,"size":3,"min_size":2,"crush_rule":1,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"868","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":486,"snap_epoch":868,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"teuthologyprofile","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":8192,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"a1fbb74d-ffae-4d5f-81ab-9677d258a95c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":329,"up_from":343,"up_thru":578,"down_at":330,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6802","nonce":1264061432},{"type":"v1","addr":"172.21.15.22:6803","nonce":1264061432}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6804","nonce":1264061432},{"type":"v1","addr":"172.21.15.22:6805","nonce":1264061432}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6808","nonce":1264061432},{"type":"v1","addr":"172.21.15.22:6809","nonce":1264061432}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6806","nonce":1264061432},{"type":"v1","addr":"172.21.15.22:6807","nonce":1264061432}]},"public_addr":"172.21.15.22:6803/1264061432","cluster_addr":"172.21.15.22:6805/1264061432","heartbeat_back_addr":"172.21.15.22:6809/1264061432","heartbeat_front_addr":"172.21.15.22:6807/1264061432","state":["exists","up"]},{"osd":1,"uuid":"fa4c9385-ac93-4983-92b4-f23f0a3731db","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":363,"up_from":376,"up_thru":578,"down_at":364,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6810","nonce":2947123190},{"type":"v1","addr":"172.21.15.22:6811","nonce":2947123190}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6812","nonce":2947123190},{"type":"v1","addr":"172.21.15.22:6813","nonce":2947123190}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6816","nonce":2947123190},{"type":"v1","addr":"172.21.15.22:6817","nonce":2947123190}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6814","nonce":2947123190},{"type":"v1","addr":"172.21.15.22:6815","nonce":2947123190}]},"public_addr":"172.21.15.22:6811/2947123190","cluster_addr":"172.21.15.22:6813/2947123190","heartbeat_back_addr":"172.21.15.22:6817/2947123190","heartbeat_front_addr":"172.21.15.22:6815/2947123190","state":["exists","up"]},{"osd":2,"uuid":"1c27d056-2218-47dd-8f13-bc1c13888483","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":397,"up_from":410,"up_thru":578,"down_at":398,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6818","nonce":4029603729},{"type":"v1","addr":"172.21.15.22:6819","nonce":4029603729}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6820","nonce":4029603729},{"type":"v1","addr":"172.21.15.22:6821","nonce":4029603729}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6824","nonce":4029603729},{"type":"v1","addr":"172.21.15.22:6825","nonce":4029603729}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6822","nonce":4029603729},{"type":"v1","addr":"172.21.15.22:6823","nonce":4029603729}]},"public_addr":"172.21.15.22:6819/4029603729","cluster_addr":"172.21.15.22:6821/4029603729","heartbeat_back_addr":"172.21.15.22:6825/4029603729","heartbeat_front_addr":"172.21.15.22:6823/4029603729","state":["exists","up"]},{"osd":3,"uuid":"8668a232-df4a-46ba-b97d-3ed2f6ddeab2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":431,"up_from":443,"up_thru":578,"down_at":432,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6826","nonce":4226372252},{"type":"v1","addr":"172.21.15.22:6827","nonce":4226372252}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6828","nonce":4226372252},{"type":"v1","addr":"172.21.15.22:6829","nonce":4226372252}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6832","nonce":4226372252},{"type":"v1","addr":"172.21.15.22:6833","nonce":4226372252}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6830","nonce":4226372252},{"type":"v1","addr":"172.21.15.22:6831","nonce":4226372252}]},"public_addr":"172.21.15.22:6827/4226372252","cluster_addr":"172.21.15.22:6829/4226372252","heartbeat_back_addr":"172.21.15.22:6833/4226372252","heartbeat_front_addr":"172.21.15.22:6831/4226372252","state":["exists","up"]},{"osd":4,"uuid":"44dd8dcc-dbf7-41f7-9ec2-e66922e6c521","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":30,"last_clean_end":465,"up_from":477,"up_thru":578,"down_at":466,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2985474018},{"type":"v1","addr":"172.21.15.64:6801","nonce":2985474018}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6802","nonce":2985474018},{"type":"v1","addr":"172.21.15.64:6803","nonce":2985474018}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6806","nonce":2985474018},{"type":"v1","addr":"172.21.15.64:6807","nonce":2985474018}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":2985474018},{"type":"v1","addr":"172.21.15.64:6805","nonce":2985474018}]},"public_addr":"172.21.15.64:6801/2985474018","cluster_addr":"172.21.15.64:6803/2985474018","heartbeat_back_addr":"172.21.15.64:6807/2985474018","heartbeat_front_addr":"172.21.15.64:6805/2985474018","state":["exists","up"]},{"osd":5,"uuid":"a920292e-6301-4b67-b2a1-30f4788f8fd3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":36,"last_clean_end":498,"up_from":512,"up_thru":578,"down_at":499,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":2760953518},{"type":"v1","addr":"172.21.15.64:6809","nonce":2760953518}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2760953518},{"type":"v1","addr":"172.21.15.64:6811","nonce":2760953518}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2760953518},{"type":"v1","addr":"172.21.15.64:6815","nonce":2760953518}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6812","nonce":2760953518},{"type":"v1","addr":"172.21.15.64:6813","nonce":2760953518}]},"public_addr":"172.21.15.64:6809/2760953518","cluster_addr":"172.21.15.64:6811/2760953518","heartbeat_back_addr":"172.21.15.64:6815/2760953518","heartbeat_front_addr":"172.21.15.64:6813/2760953518","state":["exists","up"]},{"osd":6,"uuid":"ed01fe25-9b7d-48b9-b054-e054280e5826","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":42,"last_clean_end":532,"up_from":543,"up_thru":578,"down_at":533,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1881689898},{"type":"v1","addr":"172.21.15.64:6817","nonce":1881689898}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1881689898},{"type":"v1","addr":"172.21.15.64:6819","nonce":1881689898}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1881689898},{"type":"v1","addr":"172.21.15.64:6823","nonce":1881689898}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1881689898},{"type":"v1","addr":"172.21.15.64:6821","nonce":1881689898}]},"public_addr":"172.21.15.64:6817/1881689898","cluster_addr":"172.21.15.64:6819/1881689898","heartbeat_back_addr":"172.21.15.64:6823/1881689898","heartbeat_front_addr":"172.21.15.64:6821/1881689898","state":["exists","up"]},{"osd":7,"uuid":"a9433be2-fcc7-46f5-91ee-13fbe71bf2f7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":48,"last_clean_end":565,"up_from":578,"up_thru":578,"down_at":566,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":927270672},{"type":"v1","addr":"172.21.15.64:6825","nonce":927270672}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":927270672},{"type":"v1","addr":"172.21.15.64:6827","nonce":927270672}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":927270672},{"type":"v1","addr":"172.21.15.64:6831","nonce":927270672}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":927270672},{"type":"v1","addr":"172.21.15.64:6829","nonce":927270672}]},"public_addr":"172.21.15.64:6825/927270672","cluster_addr":"172.21.15.64:6827/927270672","heartbeat_back_addr":"172.21.15.64:6831/927270672","heartbeat_front_addr":"172.21.15.64:6829/927270672","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:24:15.825542+0000","dead_epoch":329},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:24:40.497779+0000","dead_epoch":363},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:25:04.918091+0000","dead_epoch":397},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:25:30.663725+0000","dead_epoch":431},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:25:53.521949+0000","dead_epoch":465},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:26:17.565302+0000","dead_epoch":498},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:26:40.876016+0000","dead_epoch":532},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:27:04.436407+0000","dead_epoch":565}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.3","mappings":[{"from":7,"to":4}]},{"pgid":"3.a","mappings":[{"from":4,"to":3}]},{"pgid":"3.b","mappings":[{"from":0,"to":2}]},{"pgid":"3.17","mappings":[{"from":0,"to":2}]},{"pgid":"3.22","mappings":[{"from":0,"to":1}]},{"pgid":"3.24","mappings":[{"from":4,"to":2}]},{"pgid":"3.37","mappings":[{"from":5,"to":1}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.22:0/1728251114":"2024-09-22T09:22:26.473640+0000","172.21.15.64:0/2593942244":"2024-09-22T09:34:00.103818+0000","172.21.15.22:6801/2706373818":"2024-09-22T09:28:26.777111+0000","172.21.15.22:0/1022854075":"2024-09-22T09:22:26.473640+0000","172.21.15.22:0/3644531628":"2024-09-22T09:32:55.001383+0000","172.21.15.64:0/2254922149":"2024-09-22T09:34:00.103818+0000","172.21.15.64:6835/3987771184":"2024-09-22T09:34:00.103818+0000","172.21.15.22:0/1467628521":"2024-09-22T09:22:26.473640+0000","172.21.15.22:0/4254041618":"2024-09-22T09:28:26.777111+0000","172.21.15.22:6800/3088744553":"2024-09-22T09:32:55.001383+0000","172.21.15.22:6801/3012222320":"2024-09-22T09:34:42.310852+0000","172.21.15.22:0/1238499882":"2024-09-22T09:34:42.310852+0000","172.21.15.64:6834/3987771184":"2024-09-22T09:34:00.103818+0000","172.21.15.22:0/2059096138":"2024-09-22T09:34:42.310852+0000","172.21.15.22:0/4075962746":"2024-09-22T09:32:55.001383+0000","172.21.15.64:6832/3589447551":"2024-09-22T09:40:22.989105+0000","172.21.15.22:0/1998705954":"2024-09-22T09:28:26.777111+0000","172.21.15.22:0/2669480478":"2024-09-22T09:22:50.910682+0000","172.21.15.22:0/1065832205":"2024-09-22T09:34:42.310852+0000","172.21.15.64:0/3221881884":"2024-09-22T09:34:00.103818+0000","172.21.15.64:0/2724345611":"2024-09-22T09:34:00.103818+0000","172.21.15.22:0/4200462415":"2024-09-22T09:22:50.910682+0000","172.21.15.22:0/1937461032":"2024-09-22T09:28:26.777111+0000","172.21.15.64:0/40177038":"2024-09-22T09:34:00.103818+0000","172.21.15.22:6801/3088744553":"2024-09-22T09:32:55.001383+0000","172.21.15.22:0/2451188142":"2024-09-22T09:34:42.310852+0000","172.21.15.22:6800/3012222320":"2024-09-22T09:34:42.310852+0000","172.21.15.22:0/4095519345":"2024-09-22T09:34:42.310852+0000","172.21.15.22:0/932593602":"2024-09-22T09:32:55.001383+0000","172.21.15.22:0/438209926":"2024-09-22T09:32:55.001383+0000","172.21.15.22:6800/2983422368":"2024-09-22T09:22:26.473640+0000","172.21.15.22:0/3249188488":"2024-09-22T09:28:26.777111+0000","172.21.15.64:6833/3589447551":"2024-09-22T09:40:22.989105+0000","172.21.15.22:6800/2706373818":"2024-09-22T09:28:26.777111+0000","172.21.15.64:0/1545650826":"2024-09-22T09:34:00.103818+0000","172.21.15.22:6800/2821000087":"2024-09-22T09:22:50.910682+0000","172.21.15.22:0/2111825743":"2024-09-22T09:32:55.001383+0000","172.21.15.22:0/3766882712":"2024-09-22T09:22:50.910682+0000","172.21.15.22:6801/2821000087":"2024-09-22T09:22:50.910682+0000","172.21.15.22:0/2844144671":"2024-09-22T09:34:42.310852+0000","172.21.15.22:6801/2983422368":"2024-09-22T09:22:26.473640+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"},"teuthologyprofile":{"crush-device-class":"","crush-failure-domain":"osd","crush-root":"default","jerasure-per-chunk-alignment":"false","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van","w":"8"}},"removed_snaps_queue":[{"pool":5,"snaps":[{"begin":472,"length":1},{"begin":475,"length":1},{"begin":484,"length":2}]}],"new_removed_snaps":[{"pool":5,"snaps":[{"begin":485,"length":1}]}],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-21T09:45:13.680 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:13 smithi022 ceph-mon[98615]: pgmap v917: 121 pgs: 121 active+clean; 204 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 4.8 MiB/s rd, 316 KiB/s wr, 6 op/s 2024-09-21T09:45:13.680 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:13 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:45:13.680 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:13 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/4157722143' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-21T09:45:13.680 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:13 smithi022 ceph-mon[96736]: pgmap v917: 121 pgs: 121 active+clean; 204 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 4.8 MiB/s rd, 316 KiB/s wr, 6 op/s 2024-09-21T09:45:13.681 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:13 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:45:13.681 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:13 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/4157722143' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-21T09:45:13.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:13 smithi064 ceph-mon[86709]: pgmap v917: 121 pgs: 121 active+clean; 204 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 4.8 MiB/s rd, 316 KiB/s wr, 6 op/s 2024-09-21T09:45:13.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:13 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:45:13.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:13 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/4157722143' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-21T09:45:13.990 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid d65e801e-77fa-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T09:45:14.248 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:45:15.411 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:15 smithi022 ceph-mon[98615]: osdmap e869: 8 total, 8 up, 8 in 2024-09-21T09:45:15.411 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:15 smithi022 ceph-mon[96736]: osdmap e869: 8 total, 8 up, 8 in 2024-09-21T09:45:15.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:15 smithi064 ceph-mon[86709]: osdmap e869: 8 total, 8 up, 8 in 2024-09-21T09:45:15.957 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:45:15 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:45:15] "GET /metrics HTTP/1.1" 200 36356 "" "Prometheus/2.51.0" 2024-09-21T09:45:16.131 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-09-21T09:45:16.132 INFO:teuthology.orchestra.run.smithi022.stderr:dumped all 2024-09-21T09:45:16.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:16 smithi022 ceph-mon[98615]: pgmap v919: 121 pgs: 121 active+clean; 206 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 4.9 MiB/s rd, 565 KiB/s wr, 7 op/s 2024-09-21T09:45:16.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:16 smithi022 ceph-mon[96736]: pgmap v919: 121 pgs: 121 active+clean; 206 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 4.9 MiB/s rd, 565 KiB/s wr, 7 op/s 2024-09-21T09:45:16.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:16 smithi064 ceph-mon[86709]: pgmap v919: 121 pgs: 121 active+clean; 206 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 4.9 MiB/s rd, 565 KiB/s wr, 7 op/s 2024-09-21T09:45:16.895 INFO:teuthology.orchestra.run.smithi022.stdout:{"pg_ready":true,"pg_map":{"version":919,"stamp":"2024-09-21T09:45:14.477238+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":215989245,"num_objects":155,"num_object_clones":81,"num_object_copies":465,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":155,"num_whiteouts":10,"num_read":4467,"num_read_kb":4994556,"num_write":8373,"num_write_kb":1766825,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":141,"num_bytes_recovered":473293344,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":18644,"ondisk_log_size":18644,"up":363,"acting":363,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":363,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2299784,"kb_used_data":357580,"kb_used_omap":286,"kb_used_meta":1941793,"kb_avail":747661432,"statfs":{"total":767960285184,"available":765605306368,"internally_reserved":0,"allocated":366161920,"data_stored":555857663,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":293714,"internal_metadata":1988396206},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":3473408,"num_objects":6,"num_object_clones":6,"num_object_copies":18,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6,"num_whiteouts":2,"num_read":14,"num_read_kb":29834,"num_write":34,"num_write_kb":3392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.001847"},"pg_stats":[{"pgid":"3.3f","version":"0'0","reported_seq":1109,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.155126+0000","last_change":"2024-09-21T09:40:03.133624+0000","last_active":"2024-09-21T09:45:10.155126+0000","last_peered":"2024-09-21T09:45:10.155126+0000","last_clean":"2024-09-21T09:45:10.155126+0000","last_became_active":"2024-09-21T09:40:03.133206+0000","last_became_peered":"2024-09-21T09:40:03.133206+0000","last_unstale":"2024-09-21T09:45:10.155126+0000","last_undegraded":"2024-09-21T09:45:10.155126+0000","last_fullsized":"2024-09-21T09:45:10.155126+0000","mapping_epoch":578,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":579,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:39:17.983933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3e","version":"0'0","reported_seq":1319,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.156378+0000","last_change":"2024-09-21T09:39:27.026349+0000","last_active":"2024-09-21T09:45:10.156378+0000","last_peered":"2024-09-21T09:45:10.156378+0000","last_clean":"2024-09-21T09:45:10.156378+0000","last_became_active":"2024-09-21T09:39:27.025567+0000","last_became_peered":"2024-09-21T09:39:27.025567+0000","last_unstale":"2024-09-21T09:45:10.156378+0000","last_undegraded":"2024-09-21T09:45:10.156378+0000","last_fullsized":"2024-09-21T09:45:10.156378+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:46:41.891275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3d","version":"0'0","reported_seq":1335,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.150350+0000","last_change":"2024-09-21T09:37:12.120503+0000","last_active":"2024-09-21T09:45:10.150350+0000","last_peered":"2024-09-21T09:45:10.150350+0000","last_clean":"2024-09-21T09:45:10.150350+0000","last_became_active":"2024-09-21T09:37:12.120208+0000","last_became_peered":"2024-09-21T09:37:12.120208+0000","last_unstale":"2024-09-21T09:45:10.150350+0000","last_undegraded":"2024-09-21T09:45:10.150350+0000","last_fullsized":"2024-09-21T09:45:10.150350+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:44:54.459586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3c","version":"0'0","reported_seq":1182,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.152358+0000","last_change":"2024-09-21T09:40:03.129991+0000","last_active":"2024-09-21T09:45:10.152358+0000","last_peered":"2024-09-21T09:45:10.152358+0000","last_clean":"2024-09-21T09:45:10.152358+0000","last_became_active":"2024-09-21T09:40:03.128714+0000","last_became_peered":"2024-09-21T09:40:03.128714+0000","last_unstale":"2024-09-21T09:45:10.152358+0000","last_undegraded":"2024-09-21T09:45:10.152358+0000","last_fullsized":"2024-09-21T09:45:10.152358+0000","mapping_epoch":578,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":579,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:23:20.830868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3b","version":"0'0","reported_seq":1126,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.154124+0000","last_change":"2024-09-21T09:40:03.128974+0000","last_active":"2024-09-21T09:45:10.154124+0000","last_peered":"2024-09-21T09:45:10.154124+0000","last_clean":"2024-09-21T09:45:10.154124+0000","last_became_active":"2024-09-21T09:40:03.127406+0000","last_became_peered":"2024-09-21T09:40:03.127406+0000","last_unstale":"2024-09-21T09:45:10.154124+0000","last_undegraded":"2024-09-21T09:45:10.154124+0000","last_fullsized":"2024-09-21T09:45:10.154124+0000","mapping_epoch":578,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":579,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:53:03.806539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3a","version":"0'0","reported_seq":1162,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.152113+0000","last_change":"2024-09-21T09:38:55.252898+0000","last_active":"2024-09-21T09:45:10.152113+0000","last_peered":"2024-09-21T09:45:10.152113+0000","last_clean":"2024-09-21T09:45:10.152113+0000","last_became_active":"2024-09-21T09:38:55.251306+0000","last_became_peered":"2024-09-21T09:38:55.251306+0000","last_unstale":"2024-09-21T09:45:10.152113+0000","last_undegraded":"2024-09-21T09:45:10.152113+0000","last_fullsized":"2024-09-21T09:45:10.152113+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:24:33.224697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.39","version":"0'0","reported_seq":1126,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.153969+0000","last_change":"2024-09-21T09:40:03.134355+0000","last_active":"2024-09-21T09:45:10.153969+0000","last_peered":"2024-09-21T09:45:10.153969+0000","last_clean":"2024-09-21T09:45:10.153969+0000","last_became_active":"2024-09-21T09:40:03.134009+0000","last_became_peered":"2024-09-21T09:40:03.134009+0000","last_unstale":"2024-09-21T09:45:10.153969+0000","last_undegraded":"2024-09-21T09:45:10.153969+0000","last_fullsized":"2024-09-21T09:45:10.153969+0000","mapping_epoch":578,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":579,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:55:41.231906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.38","version":"0'0","reported_seq":1243,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.153714+0000","last_change":"2024-09-21T09:39:27.025553+0000","last_active":"2024-09-21T09:45:10.153714+0000","last_peered":"2024-09-21T09:45:10.153714+0000","last_clean":"2024-09-21T09:45:10.153714+0000","last_became_active":"2024-09-21T09:39:27.025206+0000","last_became_peered":"2024-09-21T09:39:27.025206+0000","last_unstale":"2024-09-21T09:45:10.153714+0000","last_undegraded":"2024-09-21T09:45:10.153714+0000","last_fullsized":"2024-09-21T09:45:10.153714+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:56:58.975981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.37","version":"0'0","reported_seq":1117,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.155097+0000","last_change":"2024-09-21T09:40:03.133462+0000","last_active":"2024-09-21T09:45:10.155097+0000","last_peered":"2024-09-21T09:45:10.155097+0000","last_clean":"2024-09-21T09:45:10.155097+0000","last_became_active":"2024-09-21T09:40:03.132847+0000","last_became_peered":"2024-09-21T09:40:03.132847+0000","last_unstale":"2024-09-21T09:45:10.155097+0000","last_undegraded":"2024-09-21T09:45:10.155097+0000","last_fullsized":"2024-09-21T09:45:10.155097+0000","mapping_epoch":578,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":579,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:38:30.733751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.36","version":"0'0","reported_seq":1140,"reported_epoch":867,"state":"active+clean","last_fresh":"2024-09-21T09:45:09.130198+0000","last_change":"2024-09-21T09:39:27.028659+0000","last_active":"2024-09-21T09:45:09.130198+0000","last_peered":"2024-09-21T09:45:09.130198+0000","last_clean":"2024-09-21T09:45:09.130198+0000","last_became_active":"2024-09-21T09:39:27.027714+0000","last_became_peered":"2024-09-21T09:39:27.027714+0000","last_unstale":"2024-09-21T09:45:09.130198+0000","last_undegraded":"2024-09-21T09:45:09.130198+0000","last_fullsized":"2024-09-21T09:45:09.130198+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:32:45.582601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.35","version":"0'0","reported_seq":1140,"reported_epoch":867,"state":"active+clean","last_fresh":"2024-09-21T09:45:09.129563+0000","last_change":"2024-09-21T09:39:27.032384+0000","last_active":"2024-09-21T09:45:09.129563+0000","last_peered":"2024-09-21T09:45:09.129563+0000","last_clean":"2024-09-21T09:45:09.129563+0000","last_became_active":"2024-09-21T09:39:27.032089+0000","last_became_peered":"2024-09-21T09:39:27.032089+0000","last_unstale":"2024-09-21T09:45:09.129563+0000","last_undegraded":"2024-09-21T09:45:09.129563+0000","last_fullsized":"2024-09-21T09:45:09.129563+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:57:51.777183+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.34","version":"0'0","reported_seq":1199,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.148492+0000","last_change":"2024-09-21T09:39:27.038040+0000","last_active":"2024-09-21T09:45:10.148492+0000","last_peered":"2024-09-21T09:45:10.148492+0000","last_clean":"2024-09-21T09:45:10.148492+0000","last_became_active":"2024-09-21T09:39:27.037573+0000","last_became_peered":"2024-09-21T09:39:27.037573+0000","last_unstale":"2024-09-21T09:45:10.148492+0000","last_undegraded":"2024-09-21T09:45:10.148492+0000","last_fullsized":"2024-09-21T09:45:10.148492+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:10:25.444335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.33","version":"0'0","reported_seq":1243,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.154096+0000","last_change":"2024-09-21T09:39:27.025275+0000","last_active":"2024-09-21T09:45:10.154096+0000","last_peered":"2024-09-21T09:45:10.154096+0000","last_clean":"2024-09-21T09:45:10.154096+0000","last_became_active":"2024-09-21T09:39:27.024926+0000","last_became_peered":"2024-09-21T09:39:27.024926+0000","last_unstale":"2024-09-21T09:45:10.154096+0000","last_undegraded":"2024-09-21T09:45:10.154096+0000","last_fullsized":"2024-09-21T09:45:10.154096+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:25:02.323230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.32","version":"0'0","reported_seq":1334,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.148611+0000","last_change":"2024-09-21T09:38:55.253737+0000","last_active":"2024-09-21T09:45:10.148611+0000","last_peered":"2024-09-21T09:45:10.148611+0000","last_clean":"2024-09-21T09:45:10.148611+0000","last_became_active":"2024-09-21T09:38:55.253166+0000","last_became_peered":"2024-09-21T09:38:55.253166+0000","last_unstale":"2024-09-21T09:45:10.148611+0000","last_undegraded":"2024-09-21T09:45:10.148611+0000","last_fullsized":"2024-09-21T09:45:10.148611+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:24:31.754234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.31","version":"0'0","reported_seq":1311,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.157856+0000","last_change":"2024-09-21T09:38:55.245920+0000","last_active":"2024-09-21T09:45:10.157856+0000","last_peered":"2024-09-21T09:45:10.157856+0000","last_clean":"2024-09-21T09:45:10.157856+0000","last_became_active":"2024-09-21T09:38:55.245587+0000","last_became_peered":"2024-09-21T09:38:55.245587+0000","last_unstale":"2024-09-21T09:45:10.157856+0000","last_undegraded":"2024-09-21T09:45:10.157856+0000","last_fullsized":"2024-09-21T09:45:10.157856+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:12:11.583592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.30","version":"0'0","reported_seq":1164,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.151464+0000","last_change":"2024-09-21T09:40:03.128190+0000","last_active":"2024-09-21T09:45:10.151464+0000","last_peered":"2024-09-21T09:45:10.151464+0000","last_clean":"2024-09-21T09:45:10.151464+0000","last_became_active":"2024-09-21T09:40:03.127879+0000","last_became_peered":"2024-09-21T09:40:03.127879+0000","last_unstale":"2024-09-21T09:45:10.151464+0000","last_undegraded":"2024-09-21T09:45:10.151464+0000","last_fullsized":"2024-09-21T09:45:10.151464+0000","mapping_epoch":578,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":579,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:02:47.140409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2f","version":"0'0","reported_seq":1173,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.152501+0000","last_change":"2024-09-21T09:40:03.128884+0000","last_active":"2024-09-21T09:45:10.152501+0000","last_peered":"2024-09-21T09:45:10.152501+0000","last_clean":"2024-09-21T09:45:10.152501+0000","last_became_active":"2024-09-21T09:40:03.127675+0000","last_became_peered":"2024-09-21T09:40:03.127675+0000","last_unstale":"2024-09-21T09:45:10.152501+0000","last_undegraded":"2024-09-21T09:45:10.152501+0000","last_fullsized":"2024-09-21T09:45:10.152501+0000","mapping_epoch":578,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":579,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:22:13.412134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2e","version":"0'0","reported_seq":1274,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.150335+0000","last_change":"2024-09-21T09:40:03.123090+0000","last_active":"2024-09-21T09:45:10.150335+0000","last_peered":"2024-09-21T09:45:10.150335+0000","last_clean":"2024-09-21T09:45:10.150335+0000","last_became_active":"2024-09-21T09:40:03.122775+0000","last_became_peered":"2024-09-21T09:40:03.122775+0000","last_unstale":"2024-09-21T09:45:10.150335+0000","last_undegraded":"2024-09-21T09:45:10.150335+0000","last_fullsized":"2024-09-21T09:45:10.150335+0000","mapping_epoch":578,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":579,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:15:03.996196+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,7],"acting":[2,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2d","version":"0'0","reported_seq":1109,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.153647+0000","last_change":"2024-09-21T09:40:03.127719+0000","last_active":"2024-09-21T09:45:10.153647+0000","last_peered":"2024-09-21T09:45:10.153647+0000","last_clean":"2024-09-21T09:45:10.153647+0000","last_became_active":"2024-09-21T09:40:03.127435+0000","last_became_peered":"2024-09-21T09:40:03.127435+0000","last_unstale":"2024-09-21T09:45:10.153647+0000","last_undegraded":"2024-09-21T09:45:10.153647+0000","last_fullsized":"2024-09-21T09:45:10.153647+0000","mapping_epoch":578,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":579,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:26:14.723006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2c","version":"0'0","reported_seq":1147,"reported_epoch":867,"state":"active+clean","last_fresh":"2024-09-21T09:45:09.129623+0000","last_change":"2024-09-21T09:39:27.037673+0000","last_active":"2024-09-21T09:45:09.129623+0000","last_peered":"2024-09-21T09:45:09.129623+0000","last_clean":"2024-09-21T09:45:09.129623+0000","last_became_active":"2024-09-21T09:39:27.037278+0000","last_became_peered":"2024-09-21T09:39:27.037278+0000","last_unstale":"2024-09-21T09:45:09.129623+0000","last_undegraded":"2024-09-21T09:45:09.129623+0000","last_fullsized":"2024-09-21T09:45:09.129623+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:38:02.454042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2b","version":"0'0","reported_seq":1335,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.148832+0000","last_change":"2024-09-21T09:40:03.128335+0000","last_active":"2024-09-21T09:45:10.148832+0000","last_peered":"2024-09-21T09:45:10.148832+0000","last_clean":"2024-09-21T09:45:10.148832+0000","last_became_active":"2024-09-21T09:40:03.126632+0000","last_became_peered":"2024-09-21T09:40:03.126632+0000","last_unstale":"2024-09-21T09:45:10.148832+0000","last_undegraded":"2024-09-21T09:45:10.148832+0000","last_fullsized":"2024-09-21T09:45:10.148832+0000","mapping_epoch":578,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":579,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:56:53.593583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2a","version":"0'0","reported_seq":1156,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.151723+0000","last_change":"2024-09-21T09:40:03.128906+0000","last_active":"2024-09-21T09:45:10.151723+0000","last_peered":"2024-09-21T09:45:10.151723+0000","last_clean":"2024-09-21T09:45:10.151723+0000","last_became_active":"2024-09-21T09:40:03.128482+0000","last_became_peered":"2024-09-21T09:40:03.128482+0000","last_unstale":"2024-09-21T09:45:10.151723+0000","last_undegraded":"2024-09-21T09:45:10.151723+0000","last_fullsized":"2024-09-21T09:45:10.151723+0000","mapping_epoch":578,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":579,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:04:28.762024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.29","version":"0'0","reported_seq":1243,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.153888+0000","last_change":"2024-09-21T09:38:19.265335+0000","last_active":"2024-09-21T09:45:10.153888+0000","last_peered":"2024-09-21T09:45:10.153888+0000","last_clean":"2024-09-21T09:45:10.153888+0000","last_became_active":"2024-09-21T09:38:19.264992+0000","last_became_peered":"2024-09-21T09:38:19.264992+0000","last_unstale":"2024-09-21T09:45:10.153888+0000","last_undegraded":"2024-09-21T09:45:10.153888+0000","last_fullsized":"2024-09-21T09:45:10.153888+0000","mapping_epoch":477,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:06:46.562732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.28","version":"0'0","reported_seq":1187,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.151612+0000","last_change":"2024-09-21T09:38:55.241432+0000","last_active":"2024-09-21T09:45:10.151612+0000","last_peered":"2024-09-21T09:45:10.151612+0000","last_clean":"2024-09-21T09:45:10.151612+0000","last_became_active":"2024-09-21T09:38:55.237695+0000","last_became_peered":"2024-09-21T09:38:55.237695+0000","last_unstale":"2024-09-21T09:45:10.151612+0000","last_undegraded":"2024-09-21T09:45:10.151612+0000","last_fullsized":"2024-09-21T09:45:10.151612+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:59:17.781598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.27","version":"0'0","reported_seq":1235,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.154427+0000","last_change":"2024-09-21T09:38:19.265421+0000","last_active":"2024-09-21T09:45:10.154427+0000","last_peered":"2024-09-21T09:45:10.154427+0000","last_clean":"2024-09-21T09:45:10.154427+0000","last_became_active":"2024-09-21T09:38:19.265067+0000","last_became_peered":"2024-09-21T09:38:19.265067+0000","last_unstale":"2024-09-21T09:45:10.154427+0000","last_undegraded":"2024-09-21T09:45:10.154427+0000","last_fullsized":"2024-09-21T09:45:10.154427+0000","mapping_epoch":477,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:29:04.200413+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.26","version":"0'0","reported_seq":1141,"reported_epoch":867,"state":"active+clean","last_fresh":"2024-09-21T09:45:09.129774+0000","last_change":"2024-09-21T09:40:03.122979+0000","last_active":"2024-09-21T09:45:09.129774+0000","last_peered":"2024-09-21T09:45:09.129774+0000","last_clean":"2024-09-21T09:45:09.129774+0000","last_became_active":"2024-09-21T09:40:03.122631+0000","last_became_peered":"2024-09-21T09:40:03.122631+0000","last_unstale":"2024-09-21T09:45:09.129774+0000","last_undegraded":"2024-09-21T09:45:09.129774+0000","last_fullsized":"2024-09-21T09:45:09.129774+0000","mapping_epoch":578,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":579,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:16:32.433324+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,4],"acting":[6,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.25","version":"0'0","reported_seq":1311,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.155532+0000","last_change":"2024-09-21T09:37:45.062756+0000","last_active":"2024-09-21T09:45:10.155532+0000","last_peered":"2024-09-21T09:45:10.155532+0000","last_clean":"2024-09-21T09:45:10.155532+0000","last_became_active":"2024-09-21T09:37:45.062133+0000","last_became_peered":"2024-09-21T09:37:45.062133+0000","last_unstale":"2024-09-21T09:45:10.155532+0000","last_undegraded":"2024-09-21T09:45:10.155532+0000","last_fullsized":"2024-09-21T09:45:10.155532+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:29:26.597723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.24","version":"0'0","reported_seq":1251,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.154228+0000","last_change":"2024-09-21T09:37:45.072422+0000","last_active":"2024-09-21T09:45:10.154228+0000","last_peered":"2024-09-21T09:45:10.154228+0000","last_clean":"2024-09-21T09:45:10.154228+0000","last_became_active":"2024-09-21T09:37:45.071949+0000","last_became_peered":"2024-09-21T09:37:45.071949+0000","last_unstale":"2024-09-21T09:45:10.154228+0000","last_undegraded":"2024-09-21T09:45:10.154228+0000","last_fullsized":"2024-09-21T09:45:10.154228+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:18:52.128932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.23","version":"0'0","reported_seq":1335,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.149366+0000","last_change":"2024-09-21T09:38:19.267789+0000","last_active":"2024-09-21T09:45:10.149366+0000","last_peered":"2024-09-21T09:45:10.149366+0000","last_clean":"2024-09-21T09:45:10.149366+0000","last_became_active":"2024-09-21T09:38:19.266709+0000","last_became_peered":"2024-09-21T09:38:19.266709+0000","last_unstale":"2024-09-21T09:45:10.149366+0000","last_undegraded":"2024-09-21T09:45:10.149366+0000","last_fullsized":"2024-09-21T09:45:10.149366+0000","mapping_epoch":477,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:13:14.596046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.22","version":"0'0","reported_seq":1198,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.151782+0000","last_change":"2024-09-21T09:38:55.248636+0000","last_active":"2024-09-21T09:45:10.151782+0000","last_peered":"2024-09-21T09:45:10.151782+0000","last_clean":"2024-09-21T09:45:10.151782+0000","last_became_active":"2024-09-21T09:38:55.246774+0000","last_became_peered":"2024-09-21T09:38:55.246774+0000","last_unstale":"2024-09-21T09:45:10.151782+0000","last_undegraded":"2024-09-21T09:45:10.151782+0000","last_fullsized":"2024-09-21T09:45:10.151782+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:15:19.102093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.21","version":"0'0","reported_seq":1274,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.148003+0000","last_change":"2024-09-21T09:39:27.033409+0000","last_active":"2024-09-21T09:45:10.148003+0000","last_peered":"2024-09-21T09:45:10.148003+0000","last_clean":"2024-09-21T09:45:10.148003+0000","last_became_active":"2024-09-21T09:39:27.033042+0000","last_became_peered":"2024-09-21T09:39:27.033042+0000","last_unstale":"2024-09-21T09:45:10.148003+0000","last_undegraded":"2024-09-21T09:45:10.148003+0000","last_fullsized":"2024-09-21T09:45:10.148003+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:11:14.771344+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.20","version":"0'0","reported_seq":1140,"reported_epoch":867,"state":"active+clean","last_fresh":"2024-09-21T09:45:09.128960+0000","last_change":"2024-09-21T09:39:27.032576+0000","last_active":"2024-09-21T09:45:09.128960+0000","last_peered":"2024-09-21T09:45:09.128960+0000","last_clean":"2024-09-21T09:45:09.128960+0000","last_became_active":"2024-09-21T09:39:27.032072+0000","last_became_peered":"2024-09-21T09:39:27.032072+0000","last_unstale":"2024-09-21T09:45:09.128960+0000","last_undegraded":"2024-09-21T09:45:09.128960+0000","last_fullsized":"2024-09-21T09:45:09.128960+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:20:51.553343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1a","version":"0'0","reported_seq":1231,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.148152+0000","last_change":"2024-09-21T09:38:19.269857+0000","last_active":"2024-09-21T09:45:10.148152+0000","last_peered":"2024-09-21T09:45:10.148152+0000","last_clean":"2024-09-21T09:45:10.148152+0000","last_became_active":"2024-09-21T09:38:19.269370+0000","last_became_peered":"2024-09-21T09:38:19.269370+0000","last_unstale":"2024-09-21T09:45:10.148152+0000","last_undegraded":"2024-09-21T09:45:10.148152+0000","last_fullsized":"2024-09-21T09:45:10.148152+0000","mapping_epoch":477,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:10:45.731493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1b","version":"599'2","reported_seq":1250,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.154828+0000","last_change":"2024-09-21T09:40:03.122663+0000","last_active":"2024-09-21T09:45:10.154828+0000","last_peered":"2024-09-21T09:45:10.154828+0000","last_clean":"2024-09-21T09:45:10.154828+0000","last_became_active":"2024-09-21T09:40:03.122284+0000","last_became_peered":"2024-09-21T09:40:03.122284+0000","last_unstale":"2024-09-21T09:45:10.154828+0000","last_undegraded":"2024-09-21T09:45:10.154828+0000","last_fullsized":"2024-09-21T09:45:10.154828+0000","mapping_epoch":578,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":579,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:59:27.379249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"0'0","reported_seq":1334,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.148943+0000","last_change":"2024-09-21T09:40:03.128597+0000","last_active":"2024-09-21T09:45:10.148943+0000","last_peered":"2024-09-21T09:45:10.148943+0000","last_clean":"2024-09-21T09:45:10.148943+0000","last_became_active":"2024-09-21T09:40:03.127020+0000","last_became_peered":"2024-09-21T09:40:03.127020+0000","last_unstale":"2024-09-21T09:45:10.148943+0000","last_undegraded":"2024-09-21T09:45:10.148943+0000","last_fullsized":"2024-09-21T09:45:10.148943+0000","mapping_epoch":578,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":579,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:16:28.183642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1a","version":"599'2","reported_seq":1146,"reported_epoch":867,"state":"active+clean","last_fresh":"2024-09-21T09:45:09.129279+0000","last_change":"2024-09-21T09:40:03.124251+0000","last_active":"2024-09-21T09:45:09.129279+0000","last_peered":"2024-09-21T09:45:09.129279+0000","last_clean":"2024-09-21T09:45:09.129279+0000","last_became_active":"2024-09-21T09:40:03.123793+0000","last_became_peered":"2024-09-21T09:40:03.123793+0000","last_unstale":"2024-09-21T09:45:09.129279+0000","last_undegraded":"2024-09-21T09:45:09.129279+0000","last_fullsized":"2024-09-21T09:45:09.129279+0000","mapping_epoch":578,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":579,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:45:22.942095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.19","version":"0'0","reported_seq":1254,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.153558+0000","last_change":"2024-09-21T09:39:27.025369+0000","last_active":"2024-09-21T09:45:10.153558+0000","last_peered":"2024-09-21T09:45:10.153558+0000","last_clean":"2024-09-21T09:45:10.153558+0000","last_became_active":"2024-09-21T09:39:27.025047+0000","last_became_peered":"2024-09-21T09:39:27.025047+0000","last_unstale":"2024-09-21T09:45:10.153558+0000","last_undegraded":"2024-09-21T09:45:10.153558+0000","last_fullsized":"2024-09-21T09:45:10.153558+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:03:11.966841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1249,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.153775+0000","last_change":"2024-09-21T09:37:45.072348+0000","last_active":"2024-09-21T09:45:10.153775+0000","last_peered":"2024-09-21T09:45:10.153775+0000","last_clean":"2024-09-21T09:45:10.153775+0000","last_became_active":"2024-09-21T09:37:45.071608+0000","last_became_peered":"2024-09-21T09:37:45.071608+0000","last_unstale":"2024-09-21T09:45:10.153775+0000","last_undegraded":"2024-09-21T09:45:10.153775+0000","last_fullsized":"2024-09-21T09:45:10.153775+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:38:34.389636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.18","version":"0'0","reported_seq":1176,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.151490+0000","last_change":"2024-09-21T09:40:03.128725+0000","last_active":"2024-09-21T09:45:10.151490+0000","last_peered":"2024-09-21T09:45:10.151490+0000","last_clean":"2024-09-21T09:45:10.151490+0000","last_became_active":"2024-09-21T09:40:03.128397+0000","last_became_peered":"2024-09-21T09:40:03.128397+0000","last_unstale":"2024-09-21T09:45:10.151490+0000","last_undegraded":"2024-09-21T09:45:10.151490+0000","last_fullsized":"2024-09-21T09:45:10.151490+0000","mapping_epoch":578,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":579,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:28:53.796129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,7],"acting":[5,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1311,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.157785+0000","last_change":"2024-09-21T09:38:19.262979+0000","last_active":"2024-09-21T09:45:10.157785+0000","last_peered":"2024-09-21T09:45:10.157785+0000","last_clean":"2024-09-21T09:45:10.157785+0000","last_became_active":"2024-09-21T09:38:19.262588+0000","last_became_peered":"2024-09-21T09:38:19.262588+0000","last_unstale":"2024-09-21T09:45:10.157785+0000","last_undegraded":"2024-09-21T09:45:10.157785+0000","last_fullsized":"2024-09-21T09:45:10.157785+0000","mapping_epoch":477,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:51:37.655965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.17","version":"0'0","reported_seq":1143,"reported_epoch":867,"state":"active+clean","last_fresh":"2024-09-21T09:45:09.129831+0000","last_change":"2024-09-21T09:39:27.028871+0000","last_active":"2024-09-21T09:45:09.129831+0000","last_peered":"2024-09-21T09:45:09.129831+0000","last_clean":"2024-09-21T09:45:09.129831+0000","last_became_active":"2024-09-21T09:39:27.028275+0000","last_became_peered":"2024-09-21T09:39:27.028275+0000","last_unstale":"2024-09-21T09:45:09.129831+0000","last_undegraded":"2024-09-21T09:45:09.129831+0000","last_fullsized":"2024-09-21T09:45:09.129831+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:06:44.746952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1156,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.152054+0000","last_change":"2024-09-21T09:40:03.130382+0000","last_active":"2024-09-21T09:45:10.152054+0000","last_peered":"2024-09-21T09:45:10.152054+0000","last_clean":"2024-09-21T09:45:10.152054+0000","last_became_active":"2024-09-21T09:40:03.130066+0000","last_became_peered":"2024-09-21T09:40:03.130066+0000","last_unstale":"2024-09-21T09:45:10.152054+0000","last_undegraded":"2024-09-21T09:45:10.152054+0000","last_fullsized":"2024-09-21T09:45:10.152054+0000","mapping_epoch":578,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":579,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:33:52.792124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.16","version":"0'0","reported_seq":1192,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.152366+0000","last_change":"2024-09-21T09:39:27.025863+0000","last_active":"2024-09-21T09:45:10.152366+0000","last_peered":"2024-09-21T09:45:10.152366+0000","last_clean":"2024-09-21T09:45:10.152366+0000","last_became_active":"2024-09-21T09:39:27.025527+0000","last_became_peered":"2024-09-21T09:39:27.025527+0000","last_unstale":"2024-09-21T09:45:10.152366+0000","last_undegraded":"2024-09-21T09:45:10.152366+0000","last_fullsized":"2024-09-21T09:45:10.152366+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:38:58.317196+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1275,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.151448+0000","last_change":"2024-09-21T09:38:55.243299+0000","last_active":"2024-09-21T09:45:10.151448+0000","last_peered":"2024-09-21T09:45:10.151448+0000","last_clean":"2024-09-21T09:45:10.151448+0000","last_became_active":"2024-09-21T09:38:55.242965+0000","last_became_peered":"2024-09-21T09:38:55.242965+0000","last_unstale":"2024-09-21T09:45:10.151448+0000","last_undegraded":"2024-09-21T09:45:10.151448+0000","last_fullsized":"2024-09-21T09:45:10.151448+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:40:46.942072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.15","version":"57'3","reported_seq":1327,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.155711+0000","last_change":"2024-09-21T09:38:55.246744+0000","last_active":"2024-09-21T09:45:10.155711+0000","last_peered":"2024-09-21T09:45:10.155711+0000","last_clean":"2024-09-21T09:45:10.155711+0000","last_became_active":"2024-09-21T09:38:55.246363+0000","last_became_peered":"2024-09-21T09:38:55.246363+0000","last_unstale":"2024-09-21T09:45:10.155711+0000","last_undegraded":"2024-09-21T09:45:10.155711+0000","last_fullsized":"2024-09-21T09:45:10.155711+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:17:48.782227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":7,"num_read_kb":7,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"0'0","reported_seq":1231,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.148718+0000","last_change":"2024-09-21T09:40:03.122217+0000","last_active":"2024-09-21T09:45:10.148718+0000","last_peered":"2024-09-21T09:45:10.148718+0000","last_clean":"2024-09-21T09:45:10.148718+0000","last_became_active":"2024-09-21T09:40:03.121913+0000","last_became_peered":"2024-09-21T09:40:03.121913+0000","last_unstale":"2024-09-21T09:45:10.148718+0000","last_undegraded":"2024-09-21T09:45:10.148718+0000","last_fullsized":"2024-09-21T09:45:10.148718+0000","mapping_epoch":578,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":579,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:48:20.988208+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.14","version":"599'29","reported_seq":1178,"reported_epoch":867,"state":"active+clean","last_fresh":"2024-09-21T09:45:09.129565+0000","last_change":"2024-09-21T09:39:27.025864+0000","last_active":"2024-09-21T09:45:09.129565+0000","last_peered":"2024-09-21T09:45:09.129565+0000","last_clean":"2024-09-21T09:45:09.129565+0000","last_became_active":"2024-09-21T09:39:27.025031+0000","last_became_peered":"2024-09-21T09:39:27.025031+0000","last_unstale":"2024-09-21T09:45:09.129565+0000","last_undegraded":"2024-09-21T09:45:09.129565+0000","last_fullsized":"2024-09-21T09:45:09.129565+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":29,"log_dups_size":0,"ondisk_log_size":29,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:41:33.069674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":5816,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":10,"num_read_kb":14,"num_write":28,"num_write_kb":25,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"0'0","reported_seq":1126,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.153786+0000","last_change":"2024-09-21T09:40:03.118862+0000","last_active":"2024-09-21T09:45:10.153786+0000","last_peered":"2024-09-21T09:45:10.153786+0000","last_clean":"2024-09-21T09:45:10.153786+0000","last_became_active":"2024-09-21T09:40:03.118575+0000","last_became_peered":"2024-09-21T09:40:03.118575+0000","last_unstale":"2024-09-21T09:45:10.153786+0000","last_undegraded":"2024-09-21T09:45:10.153786+0000","last_fullsized":"2024-09-21T09:45:10.153786+0000","mapping_epoch":578,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":579,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:49:33.511745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.13","version":"57'2","reported_seq":1343,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.149055+0000","last_change":"2024-09-21T09:38:19.267556+0000","last_active":"2024-09-21T09:45:10.149055+0000","last_peered":"2024-09-21T09:45:10.149055+0000","last_clean":"2024-09-21T09:45:10.149055+0000","last_became_active":"2024-09-21T09:38:19.266256+0000","last_became_peered":"2024-09-21T09:38:19.266256+0000","last_unstale":"2024-09-21T09:45:10.149055+0000","last_undegraded":"2024-09-21T09:45:10.149055+0000","last_fullsized":"2024-09-21T09:45:10.149055+0000","mapping_epoch":477,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:56:00.449930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":7,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1334,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.148483+0000","last_change":"2024-09-21T09:40:03.126011+0000","last_active":"2024-09-21T09:45:10.148483+0000","last_peered":"2024-09-21T09:45:10.148483+0000","last_clean":"2024-09-21T09:45:10.148483+0000","last_became_active":"2024-09-21T09:40:03.125683+0000","last_became_peered":"2024-09-21T09:40:03.125683+0000","last_unstale":"2024-09-21T09:45:10.148483+0000","last_undegraded":"2024-09-21T09:45:10.148483+0000","last_fullsized":"2024-09-21T09:45:10.148483+0000","mapping_epoch":578,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":579,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:42:54.971931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.12","version":"0'0","reported_seq":1184,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.152226+0000","last_change":"2024-09-21T09:40:03.129575+0000","last_active":"2024-09-21T09:45:10.152226+0000","last_peered":"2024-09-21T09:45:10.152226+0000","last_clean":"2024-09-21T09:45:10.152226+0000","last_became_active":"2024-09-21T09:40:03.129280+0000","last_became_peered":"2024-09-21T09:40:03.129280+0000","last_unstale":"2024-09-21T09:45:10.152226+0000","last_undegraded":"2024-09-21T09:45:10.152226+0000","last_fullsized":"2024-09-21T09:45:10.152226+0000","mapping_epoch":578,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":579,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:18:19.281113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"0'0","reported_seq":1126,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.153581+0000","last_change":"2024-09-21T09:40:03.129471+0000","last_active":"2024-09-21T09:45:10.153581+0000","last_peered":"2024-09-21T09:45:10.153581+0000","last_clean":"2024-09-21T09:45:10.153581+0000","last_became_active":"2024-09-21T09:40:03.129158+0000","last_became_peered":"2024-09-21T09:40:03.129158+0000","last_unstale":"2024-09-21T09:45:10.153581+0000","last_undegraded":"2024-09-21T09:45:10.153581+0000","last_fullsized":"2024-09-21T09:45:10.153581+0000","mapping_epoch":578,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":579,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:55:18.340725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.11","version":"0'0","reported_seq":1150,"reported_epoch":867,"state":"active+clean","last_fresh":"2024-09-21T09:45:09.129224+0000","last_change":"2024-09-21T09:39:27.029860+0000","last_active":"2024-09-21T09:45:09.129224+0000","last_peered":"2024-09-21T09:45:09.129224+0000","last_clean":"2024-09-21T09:45:09.129224+0000","last_became_active":"2024-09-21T09:39:27.029505+0000","last_became_peered":"2024-09-21T09:39:27.029505+0000","last_unstale":"2024-09-21T09:45:09.129224+0000","last_undegraded":"2024-09-21T09:45:09.129224+0000","last_fullsized":"2024-09-21T09:45:09.129224+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:40:23.423157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.10","version":"0'0","reported_seq":1140,"reported_epoch":867,"state":"active+clean","last_fresh":"2024-09-21T09:45:09.128976+0000","last_change":"2024-09-21T09:39:27.032334+0000","last_active":"2024-09-21T09:45:09.128976+0000","last_peered":"2024-09-21T09:45:09.128976+0000","last_clean":"2024-09-21T09:45:09.128976+0000","last_became_active":"2024-09-21T09:39:27.031663+0000","last_became_peered":"2024-09-21T09:39:27.031663+0000","last_unstale":"2024-09-21T09:45:09.128976+0000","last_undegraded":"2024-09-21T09:45:09.128976+0000","last_fullsized":"2024-09-21T09:45:09.128976+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:29:32.078281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.10","version":"57'8","reported_seq":1299,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.149149+0000","last_change":"2024-09-21T09:37:12.119634+0000","last_active":"2024-09-21T09:45:10.149149+0000","last_peered":"2024-09-21T09:45:10.149149+0000","last_clean":"2024-09-21T09:45:10.149149+0000","last_became_active":"2024-09-21T09:37:12.118806+0000","last_became_peered":"2024-09-21T09:37:12.118806+0000","last_unstale":"2024-09-21T09:45:10.149149+0000","last_undegraded":"2024-09-21T09:45:10.149149+0000","last_fullsized":"2024-09-21T09:45:10.149149+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:05:09.858207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":24,"num_read_kb":24,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1109,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.153887+0000","last_change":"2024-09-21T09:40:03.123363+0000","last_active":"2024-09-21T09:45:10.153887+0000","last_peered":"2024-09-21T09:45:10.153887+0000","last_clean":"2024-09-21T09:45:10.153887+0000","last_became_active":"2024-09-21T09:40:03.122860+0000","last_became_peered":"2024-09-21T09:40:03.122860+0000","last_unstale":"2024-09-21T09:45:10.153887+0000","last_undegraded":"2024-09-21T09:45:10.153887+0000","last_fullsized":"2024-09-21T09:45:10.153887+0000","mapping_epoch":578,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":579,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:42:38.650555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.8","version":"866'2218","reported_seq":5486,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.157598+0000","last_change":"2024-09-21T09:45:10.157897+0000","last_active":"2024-09-21T09:45:10.157598+0000","last_peered":"2024-09-21T09:45:10.157598+0000","last_clean":"2024-09-21T09:45:10.157598+0000","last_became_active":"2024-09-21T09:37:12.122899+0000","last_became_peered":"2024-09-21T09:37:12.122899+0000","last_unstale":"2024-09-21T09:45:10.157598+0000","last_undegraded":"2024-09-21T09:45:10.157598+0000","last_fullsized":"2024-09-21T09:45:10.157598+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":70,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:31:18.894586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:31:18.894586+0000","last_clean_scrub_stamp":"2024-09-21T09:31:18.894586+0000","objects_scrubbed":0,"log_size":2218,"log_dups_size":0,"ondisk_log_size":2218,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:47:55.620219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0083080680000000001,"stat_sum":{"num_bytes":15400960,"num_objects":12,"num_object_clones":7,"num_object_copies":36,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":12,"num_whiteouts":0,"num_read":437,"num_read_kb":523177,"num_write":993,"num_write_kb":177176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":19,"num_bytes_recovered":34504704,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"1d8","length":"1"},{"start":"1db","length":"1"},{"start":"1e4","length":"2"}]},{"pgid":"2.f","version":"57'1","reported_seq":1205,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.149457+0000","last_change":"2024-09-21T09:40:03.124550+0000","last_active":"2024-09-21T09:45:10.149457+0000","last_peered":"2024-09-21T09:45:10.149457+0000","last_clean":"2024-09-21T09:45:10.149457+0000","last_became_active":"2024-09-21T09:40:03.124167+0000","last_became_peered":"2024-09-21T09:40:03.124167+0000","last_unstale":"2024-09-21T09:45:10.149457+0000","last_undegraded":"2024-09-21T09:45:10.149457+0000","last_fullsized":"2024-09-21T09:45:10.149457+0000","mapping_epoch":578,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":579,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:11:07.825319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"0'0","reported_seq":1126,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.154913+0000","last_change":"2024-09-21T09:40:03.131963+0000","last_active":"2024-09-21T09:45:10.154913+0000","last_peered":"2024-09-21T09:45:10.154913+0000","last_clean":"2024-09-21T09:45:10.154913+0000","last_became_active":"2024-09-21T09:40:03.131638+0000","last_became_peered":"2024-09-21T09:40:03.131638+0000","last_unstale":"2024-09-21T09:45:10.154913+0000","last_undegraded":"2024-09-21T09:45:10.154913+0000","last_fullsized":"2024-09-21T09:45:10.154913+0000","mapping_epoch":578,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":579,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:14:49.344307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.9","version":"868'1590","reported_seq":4568,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.159720+0000","last_change":"2024-09-21T09:45:10.160743+0000","last_active":"2024-09-21T09:45:10.159720+0000","last_peered":"2024-09-21T09:45:10.159720+0000","last_clean":"2024-09-21T09:45:10.159720+0000","last_became_active":"2024-09-21T09:40:03.131443+0000","last_became_peered":"2024-09-21T09:40:03.131443+0000","last_unstale":"2024-09-21T09:45:10.159720+0000","last_undegraded":"2024-09-21T09:45:10.159720+0000","last_fullsized":"2024-09-21T09:45:10.159720+0000","mapping_epoch":578,"log_start":"0'0","ondisk_log_start":"0'0","created":70,"last_epoch_clean":579,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:31:18.894586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:31:18.894586+0000","last_clean_scrub_stamp":"2024-09-21T09:31:18.894586+0000","objects_scrubbed":0,"log_size":1590,"log_dups_size":0,"ondisk_log_size":1590,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:26:32.761199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0072782630000000001,"stat_sum":{"num_bytes":15622144,"num_objects":17,"num_object_clones":12,"num_object_copies":51,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":17,"num_whiteouts":3,"num_read":314,"num_read_kb":197507,"num_write":626,"num_write_kb":182112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":13287424,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[{"start":"1d8","length":"1"},{"start":"1db","length":"1"},{"start":"1e4","length":"2"}]},{"pgid":"2.e","version":"57'2","reported_seq":1283,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.150493+0000","last_change":"2024-09-21T09:40:03.123429+0000","last_active":"2024-09-21T09:45:10.150493+0000","last_peered":"2024-09-21T09:45:10.150493+0000","last_clean":"2024-09-21T09:45:10.150493+0000","last_became_active":"2024-09-21T09:40:03.123065+0000","last_became_peered":"2024-09-21T09:40:03.123065+0000","last_unstale":"2024-09-21T09:45:10.150493+0000","last_undegraded":"2024-09-21T09:45:10.150493+0000","last_fullsized":"2024-09-21T09:45:10.150493+0000","mapping_epoch":578,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":579,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:00:08.675515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":6,"num_read_kb":6,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"0'0","reported_seq":1126,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.155229+0000","last_change":"2024-09-21T09:40:03.133147+0000","last_active":"2024-09-21T09:45:10.155229+0000","last_peered":"2024-09-21T09:45:10.155229+0000","last_clean":"2024-09-21T09:45:10.155229+0000","last_became_active":"2024-09-21T09:40:03.132798+0000","last_became_peered":"2024-09-21T09:40:03.132798+0000","last_unstale":"2024-09-21T09:45:10.155229+0000","last_undegraded":"2024-09-21T09:45:10.155229+0000","last_fullsized":"2024-09-21T09:45:10.155229+0000","mapping_epoch":578,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":579,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:49:38.148383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.a","version":"868'827","reported_seq":4120,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.154621+0000","last_change":"2024-09-21T09:45:10.155595+0000","last_active":"2024-09-21T09:45:10.154621+0000","last_peered":"2024-09-21T09:45:10.154621+0000","last_clean":"2024-09-21T09:45:10.154621+0000","last_became_active":"2024-09-21T09:38:19.267344+0000","last_became_peered":"2024-09-21T09:38:19.267344+0000","last_unstale":"2024-09-21T09:45:10.154621+0000","last_undegraded":"2024-09-21T09:45:10.154621+0000","last_fullsized":"2024-09-21T09:45:10.154621+0000","mapping_epoch":477,"log_start":"0'0","ondisk_log_start":"0'0","created":70,"last_epoch_clean":478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:31:18.894586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:31:18.894586+0000","last_clean_scrub_stamp":"2024-09-21T09:31:18.894586+0000","objects_scrubbed":0,"log_size":827,"log_dups_size":0,"ondisk_log_size":827,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:25:04.569868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0066714970000000002,"stat_sum":{"num_bytes":19259392,"num_objects":6,"num_object_clones":4,"num_object_copies":18,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6,"num_whiteouts":1,"num_read":154,"num_read_kb":107106,"num_write":397,"num_write_kb":67096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":22446080,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"1d8","length":"1"},{"start":"1db","length":"1"},{"start":"1e4","length":"2"}]},{"pgid":"2.d","version":"57'1","reported_seq":1318,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.155811+0000","last_change":"2024-09-21T09:38:19.263081+0000","last_active":"2024-09-21T09:45:10.155811+0000","last_peered":"2024-09-21T09:45:10.155811+0000","last_clean":"2024-09-21T09:45:10.155811+0000","last_became_active":"2024-09-21T09:38:19.262718+0000","last_became_peered":"2024-09-21T09:38:19.262718+0000","last_unstale":"2024-09-21T09:45:10.155811+0000","last_undegraded":"2024-09-21T09:45:10.155811+0000","last_fullsized":"2024-09-21T09:45:10.155811+0000","mapping_epoch":477,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:12:28.211051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1181,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.152490+0000","last_change":"2024-09-21T09:39:27.026061+0000","last_active":"2024-09-21T09:45:10.152490+0000","last_peered":"2024-09-21T09:45:10.152490+0000","last_clean":"2024-09-21T09:45:10.152490+0000","last_became_active":"2024-09-21T09:39:27.025616+0000","last_became_peered":"2024-09-21T09:39:27.025616+0000","last_unstale":"2024-09-21T09:45:10.152490+0000","last_undegraded":"2024-09-21T09:45:10.152490+0000","last_fullsized":"2024-09-21T09:45:10.152490+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:03:30.870489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.b","version":"0'0","reported_seq":3306,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.148653+0000","last_change":"2024-09-21T09:45:10.149157+0000","last_active":"2024-09-21T09:45:10.148653+0000","last_peered":"2024-09-21T09:45:10.148653+0000","last_clean":"2024-09-21T09:45:10.148653+0000","last_became_active":"2024-09-21T09:38:55.232696+0000","last_became_peered":"2024-09-21T09:38:55.232696+0000","last_unstale":"2024-09-21T09:45:10.148653+0000","last_undegraded":"2024-09-21T09:45:10.148653+0000","last_fullsized":"2024-09-21T09:45:10.148653+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":70,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:31:18.894586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:31:18.894586+0000","last_clean_scrub_stamp":"2024-09-21T09:31:18.894586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:49:34.596335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00079894199999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"1d8","length":"1"},{"start":"1db","length":"1"},{"start":"1e4","length":"2"}]},{"pgid":"2.c","version":"0'0","reported_seq":1285,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.150887+0000","last_change":"2024-09-21T09:38:55.243558+0000","last_active":"2024-09-21T09:45:10.150887+0000","last_peered":"2024-09-21T09:45:10.150887+0000","last_clean":"2024-09-21T09:45:10.150887+0000","last_became_active":"2024-09-21T09:38:55.243224+0000","last_became_peered":"2024-09-21T09:38:55.243224+0000","last_unstale":"2024-09-21T09:45:10.150887+0000","last_undegraded":"2024-09-21T09:45:10.150887+0000","last_fullsized":"2024-09-21T09:45:10.150887+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:24:48.860630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"0'0","reported_seq":1109,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.154462+0000","last_change":"2024-09-21T09:40:03.126104+0000","last_active":"2024-09-21T09:45:10.154462+0000","last_peered":"2024-09-21T09:45:10.154462+0000","last_clean":"2024-09-21T09:45:10.154462+0000","last_became_active":"2024-09-21T09:40:03.125807+0000","last_became_peered":"2024-09-21T09:40:03.125807+0000","last_unstale":"2024-09-21T09:45:10.154462+0000","last_undegraded":"2024-09-21T09:45:10.154462+0000","last_fullsized":"2024-09-21T09:45:10.154462+0000","mapping_epoch":578,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":579,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:26:17.938817+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.4","version":"866'2256","reported_seq":5235,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.214775+0000","last_change":"2024-09-21T09:45:10.155226+0000","last_active":"2024-09-21T09:45:10.214775+0000","last_peered":"2024-09-21T09:45:10.214775+0000","last_clean":"2024-09-21T09:45:10.214775+0000","last_became_active":"2024-09-21T09:40:03.130119+0000","last_became_peered":"2024-09-21T09:40:03.130119+0000","last_unstale":"2024-09-21T09:45:10.214775+0000","last_undegraded":"2024-09-21T09:45:10.214775+0000","last_fullsized":"2024-09-21T09:45:10.214775+0000","mapping_epoch":578,"log_start":"0'0","ondisk_log_start":"0'0","created":70,"last_epoch_clean":579,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:31:18.894586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:31:18.894586+0000","last_clean_scrub_stamp":"2024-09-21T09:31:18.894586+0000","objects_scrubbed":0,"log_size":2256,"log_dups_size":0,"ondisk_log_size":2256,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:44:07.125758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011706500000000001,"stat_sum":{"num_bytes":33439744,"num_objects":15,"num_object_clones":9,"num_object_copies":45,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":15,"num_whiteouts":2,"num_read":450,"num_read_kb":566268,"num_write":994,"num_write_kb":222552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":52707328,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[{"start":"1d8","length":"1"},{"start":"1db","length":"1"},{"start":"1e4","length":"2"}]},{"pgid":"4.5","version":"0'0","reported_seq":1139,"reported_epoch":867,"state":"active+clean","last_fresh":"2024-09-21T09:45:09.130022+0000","last_change":"2024-09-21T09:40:03.127593+0000","last_active":"2024-09-21T09:45:09.130022+0000","last_peered":"2024-09-21T09:45:09.130022+0000","last_clean":"2024-09-21T09:45:09.130022+0000","last_became_active":"2024-09-21T09:40:03.127301+0000","last_became_peered":"2024-09-21T09:40:03.127301+0000","last_unstale":"2024-09-21T09:45:09.130022+0000","last_undegraded":"2024-09-21T09:45:09.130022+0000","last_fullsized":"2024-09-21T09:45:09.130022+0000","mapping_epoch":578,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":579,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:28:27.830318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:28:27.830318+0000","last_clean_scrub_stamp":"2024-09-21T09:28:27.830318+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:19:42.156705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030427299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"271'136","reported_seq":1349,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.154055+0000","last_change":"2024-09-21T09:40:03.134201+0000","last_active":"2024-09-21T09:45:10.154055+0000","last_peered":"2024-09-21T09:45:10.154055+0000","last_clean":"2024-09-21T09:45:10.154055+0000","last_became_active":"2024-09-21T09:40:03.133635+0000","last_became_peered":"2024-09-21T09:40:03.133635+0000","last_unstale":"2024-09-21T09:45:10.154055+0000","last_undegraded":"2024-09-21T09:45:10.154055+0000","last_fullsized":"2024-09-21T09:45:10.154055+0000","mapping_epoch":578,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":579,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:25:07.646659+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:25:07.646659+0000","last_clean_scrub_stamp":"2024-09-21T09:25:07.646659+0000","objects_scrubbed":0,"log_size":136,"log_dups_size":0,"ondisk_log_size":136,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:17:56.567476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":618,"num_read_kb":1145,"num_write":221,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":1193,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.151721+0000","last_change":"2024-09-21T09:38:55.237761+0000","last_active":"2024-09-21T09:45:10.151721+0000","last_peered":"2024-09-21T09:45:10.151721+0000","last_clean":"2024-09-21T09:45:10.151721+0000","last_became_active":"2024-09-21T09:38:55.237288+0000","last_became_peered":"2024-09-21T09:38:55.237288+0000","last_unstale":"2024-09-21T09:45:10.151721+0000","last_undegraded":"2024-09-21T09:45:10.151721+0000","last_fullsized":"2024-09-21T09:45:10.151721+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:20:27.638738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1243,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.154060+0000","last_change":"2024-09-21T09:39:27.026324+0000","last_active":"2024-09-21T09:45:10.154060+0000","last_peered":"2024-09-21T09:45:10.154060+0000","last_clean":"2024-09-21T09:45:10.154060+0000","last_became_active":"2024-09-21T09:39:27.026024+0000","last_became_peered":"2024-09-21T09:39:27.026024+0000","last_unstale":"2024-09-21T09:45:10.154060+0000","last_undegraded":"2024-09-21T09:45:10.154060+0000","last_fullsized":"2024-09-21T09:45:10.154060+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:10:50.254361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.7","version":"866'908","reported_seq":4085,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.251926+0000","last_change":"2024-09-21T09:45:10.154436+0000","last_active":"2024-09-21T09:45:10.251926+0000","last_peered":"2024-09-21T09:45:10.251926+0000","last_clean":"2024-09-21T09:45:10.251926+0000","last_became_active":"2024-09-21T09:38:55.243966+0000","last_became_peered":"2024-09-21T09:38:55.243966+0000","last_unstale":"2024-09-21T09:45:10.251926+0000","last_undegraded":"2024-09-21T09:45:10.251926+0000","last_fullsized":"2024-09-21T09:45:10.251926+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":70,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:31:18.894586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:31:18.894586+0000","last_clean_scrub_stamp":"2024-09-21T09:31:18.894586+0000","objects_scrubbed":0,"log_size":908,"log_dups_size":0,"ondisk_log_size":908,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:46:24.495090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00151133,"stat_sum":{"num_bytes":4726784,"num_objects":6,"num_object_clones":3,"num_object_copies":18,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6,"num_whiteouts":0,"num_read":206,"num_read_kb":189716,"num_write":420,"num_write_kb":98152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":8093696,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[{"start":"1d8","length":"1"},{"start":"1db","length":"1"},{"start":"1e4","length":"2"}]},{"pgid":"4.6","version":"0'0","reported_seq":1344,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.148869+0000","last_change":"2024-09-21T09:37:12.120873+0000","last_active":"2024-09-21T09:45:10.148869+0000","last_peered":"2024-09-21T09:45:10.148869+0000","last_clean":"2024-09-21T09:45:10.148869+0000","last_became_active":"2024-09-21T09:37:12.120338+0000","last_became_peered":"2024-09-21T09:37:12.120338+0000","last_unstale":"2024-09-21T09:45:10.148869+0000","last_undegraded":"2024-09-21T09:45:10.148869+0000","last_fullsized":"2024-09-21T09:45:10.148869+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:28:27.830318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:28:27.830318+0000","last_clean_scrub_stamp":"2024-09-21T09:28:27.830318+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:05:57.622287+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1110,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.153907+0000","last_change":"2024-09-21T09:40:03.133272+0000","last_active":"2024-09-21T09:45:10.153907+0000","last_peered":"2024-09-21T09:45:10.153907+0000","last_clean":"2024-09-21T09:45:10.153907+0000","last_became_active":"2024-09-21T09:40:03.132972+0000","last_became_peered":"2024-09-21T09:40:03.132972+0000","last_unstale":"2024-09-21T09:45:10.153907+0000","last_undegraded":"2024-09-21T09:45:10.153907+0000","last_fullsized":"2024-09-21T09:45:10.153907+0000","mapping_epoch":578,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":579,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:27:09.946866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1335,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.147981+0000","last_change":"2024-09-21T09:38:19.265257+0000","last_active":"2024-09-21T09:45:10.147981+0000","last_peered":"2024-09-21T09:45:10.147981+0000","last_clean":"2024-09-21T09:45:10.147981+0000","last_became_active":"2024-09-21T09:38:19.264885+0000","last_became_peered":"2024-09-21T09:38:19.264885+0000","last_unstale":"2024-09-21T09:45:10.147981+0000","last_undegraded":"2024-09-21T09:45:10.147981+0000","last_fullsized":"2024-09-21T09:45:10.147981+0000","mapping_epoch":477,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:04:06.019970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.6","version":"866'1541","reported_seq":4851,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.157673+0000","last_change":"2024-09-21T09:45:10.157952+0000","last_active":"2024-09-21T09:45:10.157673+0000","last_peered":"2024-09-21T09:45:10.157673+0000","last_clean":"2024-09-21T09:45:10.157673+0000","last_became_active":"2024-09-21T09:38:55.243902+0000","last_became_peered":"2024-09-21T09:38:55.243902+0000","last_unstale":"2024-09-21T09:45:10.157673+0000","last_undegraded":"2024-09-21T09:45:10.157673+0000","last_fullsized":"2024-09-21T09:45:10.157673+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":70,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:31:18.894586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:31:18.894586+0000","last_clean_scrub_stamp":"2024-09-21T09:31:18.894586+0000","objects_scrubbed":0,"log_size":1541,"log_dups_size":0,"ondisk_log_size":1541,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:52:01.218411+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0069214480000000002,"stat_sum":{"num_bytes":26296320,"num_objects":11,"num_object_clones":7,"num_object_copies":33,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":11,"num_whiteouts":1,"num_read":300,"num_read_kb":698483,"num_write":723,"num_write_kb":171328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":73089024,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"1d8","length":"1"},{"start":"1db","length":"1"},{"start":"1e4","length":"2"}]},{"pgid":"4.7","version":"0'0","reported_seq":1316,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.156191+0000","last_change":"2024-09-21T09:38:55.246205+0000","last_active":"2024-09-21T09:45:10.156191+0000","last_peered":"2024-09-21T09:45:10.156191+0000","last_clean":"2024-09-21T09:45:10.156191+0000","last_became_active":"2024-09-21T09:38:55.245792+0000","last_became_peered":"2024-09-21T09:38:55.245792+0000","last_unstale":"2024-09-21T09:45:10.156191+0000","last_undegraded":"2024-09-21T09:45:10.156191+0000","last_fullsized":"2024-09-21T09:45:10.156191+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:28:27.830318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:28:27.830318+0000","last_clean_scrub_stamp":"2024-09-21T09:28:27.830318+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:23:11.829685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00058044900000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1285,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.148120+0000","last_change":"2024-09-21T09:37:45.056740+0000","last_active":"2024-09-21T09:45:10.148120+0000","last_peered":"2024-09-21T09:45:10.148120+0000","last_clean":"2024-09-21T09:45:10.148120+0000","last_became_active":"2024-09-21T09:37:45.056385+0000","last_became_peered":"2024-09-21T09:37:45.056385+0000","last_unstale":"2024-09-21T09:45:10.148120+0000","last_undegraded":"2024-09-21T09:45:10.148120+0000","last_fullsized":"2024-09-21T09:45:10.148120+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:05:53.074827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1312,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.156528+0000","last_change":"2024-09-21T09:39:27.024062+0000","last_active":"2024-09-21T09:45:10.156528+0000","last_peered":"2024-09-21T09:45:10.156528+0000","last_clean":"2024-09-21T09:45:10.156528+0000","last_became_active":"2024-09-21T09:39:27.023711+0000","last_became_peered":"2024-09-21T09:39:27.023711+0000","last_unstale":"2024-09-21T09:45:10.156528+0000","last_undegraded":"2024-09-21T09:45:10.156528+0000","last_fullsized":"2024-09-21T09:45:10.156528+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:27:39.966745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.5","version":"868'1112","reported_seq":4556,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.157307+0000","last_change":"2024-09-21T09:45:10.158521+0000","last_active":"2024-09-21T09:45:10.157307+0000","last_peered":"2024-09-21T09:45:10.157307+0000","last_clean":"2024-09-21T09:45:10.157307+0000","last_became_active":"2024-09-21T09:38:19.266681+0000","last_became_peered":"2024-09-21T09:38:19.266681+0000","last_unstale":"2024-09-21T09:45:10.157307+0000","last_undegraded":"2024-09-21T09:45:10.157307+0000","last_fullsized":"2024-09-21T09:45:10.157307+0000","mapping_epoch":477,"log_start":"0'0","ondisk_log_start":"0'0","created":70,"last_epoch_clean":478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:31:18.894586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:31:18.894586+0000","last_clean_scrub_stamp":"2024-09-21T09:31:18.894586+0000","objects_scrubbed":0,"log_size":1112,"log_dups_size":0,"ondisk_log_size":1112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:27:41.671086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0075282409999999998,"stat_sum":{"num_bytes":11681792,"num_objects":8,"num_object_clones":5,"num_object_copies":24,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":8,"num_whiteouts":0,"num_read":233,"num_read_kb":339620,"num_write":437,"num_write_kb":89760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":13959168,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"1d8","length":"1"},{"start":"1db","length":"1"},{"start":"1e4","length":"2"}]},{"pgid":"4.4","version":"0'0","reported_seq":1322,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.157081+0000","last_change":"2024-09-21T09:37:45.064173+0000","last_active":"2024-09-21T09:45:10.157081+0000","last_peered":"2024-09-21T09:45:10.157081+0000","last_clean":"2024-09-21T09:45:10.157081+0000","last_became_active":"2024-09-21T09:37:45.063539+0000","last_became_peered":"2024-09-21T09:37:45.063539+0000","last_unstale":"2024-09-21T09:45:10.157081+0000","last_undegraded":"2024-09-21T09:45:10.157081+0000","last_fullsized":"2024-09-21T09:45:10.157081+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:28:27.830318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:28:27.830318+0000","last_clean_scrub_stamp":"2024-09-21T09:28:27.830318+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:26:51.829388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":1160,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.151999+0000","last_change":"2024-09-21T09:39:27.031886+0000","last_active":"2024-09-21T09:45:10.151999+0000","last_peered":"2024-09-21T09:45:10.151999+0000","last_clean":"2024-09-21T09:45:10.151999+0000","last_became_active":"2024-09-21T09:39:27.031542+0000","last_became_peered":"2024-09-21T09:39:27.031542+0000","last_unstale":"2024-09-21T09:45:10.151999+0000","last_undegraded":"2024-09-21T09:45:10.151999+0000","last_fullsized":"2024-09-21T09:45:10.151999+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:21:48.103445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1200,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.148676+0000","last_change":"2024-09-21T09:39:27.033788+0000","last_active":"2024-09-21T09:45:10.148676+0000","last_peered":"2024-09-21T09:45:10.148676+0000","last_clean":"2024-09-21T09:45:10.148676+0000","last_became_active":"2024-09-21T09:39:27.033464+0000","last_became_peered":"2024-09-21T09:39:27.033464+0000","last_unstale":"2024-09-21T09:45:10.148676+0000","last_undegraded":"2024-09-21T09:45:10.148676+0000","last_fullsized":"2024-09-21T09:45:10.148676+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:55:20.057561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.3","version":"866'711","reported_seq":4225,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.150475+0000","last_change":"2024-09-21T09:45:10.150855+0000","last_active":"2024-09-21T09:45:10.150475+0000","last_peered":"2024-09-21T09:45:10.150475+0000","last_clean":"2024-09-21T09:45:10.150475+0000","last_became_active":"2024-09-21T09:39:27.038422+0000","last_became_peered":"2024-09-21T09:39:27.038422+0000","last_unstale":"2024-09-21T09:45:10.150475+0000","last_undegraded":"2024-09-21T09:45:10.150475+0000","last_fullsized":"2024-09-21T09:45:10.150475+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":70,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:31:18.894586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:31:18.894586+0000","last_clean_scrub_stamp":"2024-09-21T09:31:18.894586+0000","objects_scrubbed":0,"log_size":711,"log_dups_size":0,"ondisk_log_size":711,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:08:52.980522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001141623,"stat_sum":{"num_bytes":19587072,"num_objects":6,"num_object_clones":4,"num_object_copies":18,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6,"num_whiteouts":1,"num_read":213,"num_read_kb":355011,"num_write":253,"num_write_kb":55536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":5939200,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"1d8","length":"1"},{"start":"1db","length":"1"},{"start":"1e4","length":"2"}]},{"pgid":"4.2","version":"66'2","reported_seq":1324,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.156098+0000","last_change":"2024-09-21T09:38:55.253738+0000","last_active":"2024-09-21T09:45:10.156098+0000","last_peered":"2024-09-21T09:45:10.156098+0000","last_clean":"2024-09-21T09:45:10.156098+0000","last_became_active":"2024-09-21T09:38:55.253306+0000","last_became_peered":"2024-09-21T09:38:55.253306+0000","last_unstale":"2024-09-21T09:45:10.156098+0000","last_undegraded":"2024-09-21T09:45:10.156098+0000","last_fullsized":"2024-09-21T09:45:10.156098+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:28:27.830318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:28:27.830318+0000","last_clean_scrub_stamp":"2024-09-21T09:28:27.830318+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:25:57.120032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00080024800000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1322,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.157240+0000","last_change":"2024-09-21T09:40:03.125865+0000","last_active":"2024-09-21T09:45:10.157240+0000","last_peered":"2024-09-21T09:45:10.157240+0000","last_clean":"2024-09-21T09:45:10.157240+0000","last_became_active":"2024-09-21T09:40:03.125522+0000","last_became_peered":"2024-09-21T09:40:03.125522+0000","last_unstale":"2024-09-21T09:45:10.157240+0000","last_undegraded":"2024-09-21T09:45:10.157240+0000","last_fullsized":"2024-09-21T09:45:10.157240+0000","mapping_epoch":578,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":579,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:17:02.672862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1188,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.152192+0000","last_change":"2024-09-21T09:38:55.243517+0000","last_active":"2024-09-21T09:45:10.152192+0000","last_peered":"2024-09-21T09:45:10.152192+0000","last_clean":"2024-09-21T09:45:10.152192+0000","last_became_active":"2024-09-21T09:38:55.242211+0000","last_became_peered":"2024-09-21T09:38:55.242211+0000","last_unstale":"2024-09-21T09:45:10.152192+0000","last_undegraded":"2024-09-21T09:45:10.152192+0000","last_fullsized":"2024-09-21T09:45:10.152192+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:06:43.469077+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.2","version":"867'1824","reported_seq":4854,"reported_epoch":867,"state":"active+clean","last_fresh":"2024-09-21T09:45:09.150102+0000","last_change":"2024-09-21T09:45:09.130608+0000","last_active":"2024-09-21T09:45:09.150102+0000","last_peered":"2024-09-21T09:45:09.150102+0000","last_clean":"2024-09-21T09:45:09.150102+0000","last_became_active":"2024-09-21T09:39:27.032640+0000","last_became_peered":"2024-09-21T09:39:27.032640+0000","last_unstale":"2024-09-21T09:45:09.150102+0000","last_undegraded":"2024-09-21T09:45:09.150102+0000","last_fullsized":"2024-09-21T09:45:09.150102+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":70,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:31:18.894586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:31:18.894586+0000","last_clean_scrub_stamp":"2024-09-21T09:31:18.894586+0000","objects_scrubbed":0,"log_size":1824,"log_dups_size":0,"ondisk_log_size":1824,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:55:52.343938+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00060815200000000004,"stat_sum":{"num_bytes":17235968,"num_objects":13,"num_object_clones":8,"num_object_copies":39,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":13,"num_whiteouts":0,"num_read":317,"num_read_kb":419001,"num_write":883,"num_write_kb":194848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":91979776,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[{"start":"1d8","length":"1"},{"start":"1db","length":"1"},{"start":"1e4","length":"1"}]},{"pgid":"4.3","version":"64'1","reported_seq":1345,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.149252+0000","last_change":"2024-09-21T09:40:03.128688+0000","last_active":"2024-09-21T09:45:10.149252+0000","last_peered":"2024-09-21T09:45:10.149252+0000","last_clean":"2024-09-21T09:45:10.149252+0000","last_became_active":"2024-09-21T09:40:03.127758+0000","last_became_peered":"2024-09-21T09:40:03.127758+0000","last_unstale":"2024-09-21T09:45:10.149252+0000","last_undegraded":"2024-09-21T09:45:10.149252+0000","last_fullsized":"2024-09-21T09:45:10.149252+0000","mapping_epoch":578,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":579,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:28:27.830318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:28:27.830318+0000","last_clean_scrub_stamp":"2024-09-21T09:28:27.830318+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:51:20.031888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1129,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.153414+0000","last_change":"2024-09-21T09:40:03.134083+0000","last_active":"2024-09-21T09:45:10.153414+0000","last_peered":"2024-09-21T09:45:10.153414+0000","last_clean":"2024-09-21T09:45:10.153414+0000","last_became_active":"2024-09-21T09:40:03.133598+0000","last_became_peered":"2024-09-21T09:40:03.133598+0000","last_unstale":"2024-09-21T09:45:10.153414+0000","last_undegraded":"2024-09-21T09:45:10.153414+0000","last_fullsized":"2024-09-21T09:45:10.153414+0000","mapping_epoch":578,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":579,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:20:24.891085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1312,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.157426+0000","last_change":"2024-09-21T09:38:55.247226+0000","last_active":"2024-09-21T09:45:10.157426+0000","last_peered":"2024-09-21T09:45:10.157426+0000","last_clean":"2024-09-21T09:45:10.157426+0000","last_became_active":"2024-09-21T09:38:55.246742+0000","last_became_peered":"2024-09-21T09:38:55.246742+0000","last_unstale":"2024-09-21T09:45:10.157426+0000","last_undegraded":"2024-09-21T09:45:10.157426+0000","last_fullsized":"2024-09-21T09:45:10.157426+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:46:58.785698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.1","version":"0'0","reported_seq":3201,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.148868+0000","last_change":"2024-09-21T09:45:10.149279+0000","last_active":"2024-09-21T09:45:10.148868+0000","last_peered":"2024-09-21T09:45:10.148868+0000","last_clean":"2024-09-21T09:45:10.148868+0000","last_became_active":"2024-09-21T09:40:03.122678+0000","last_became_peered":"2024-09-21T09:40:03.122678+0000","last_unstale":"2024-09-21T09:45:10.148868+0000","last_undegraded":"2024-09-21T09:45:10.148868+0000","last_fullsized":"2024-09-21T09:45:10.148868+0000","mapping_epoch":578,"log_start":"0'0","ondisk_log_start":"0'0","created":70,"last_epoch_clean":579,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:31:18.894586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:31:18.894586+0000","last_clean_scrub_stamp":"2024-09-21T09:31:18.894586+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:01:59.954455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00093911400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[{"start":"1d8","length":"1"},{"start":"1db","length":"1"},{"start":"1e4","length":"2"}]},{"pgid":"4.0","version":"0'0","reported_seq":1249,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.153665+0000","last_change":"2024-09-21T09:40:03.126964+0000","last_active":"2024-09-21T09:45:10.153665+0000","last_peered":"2024-09-21T09:45:10.153665+0000","last_clean":"2024-09-21T09:45:10.153665+0000","last_became_active":"2024-09-21T09:40:03.126386+0000","last_became_peered":"2024-09-21T09:40:03.126386+0000","last_unstale":"2024-09-21T09:45:10.153665+0000","last_undegraded":"2024-09-21T09:45:10.153665+0000","last_fullsized":"2024-09-21T09:45:10.153665+0000","mapping_epoch":578,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":579,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:28:27.830318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:28:27.830318+0000","last_clean_scrub_stamp":"2024-09-21T09:28:27.830318+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:02:04.535091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034591199999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"603'2","reported_seq":1319,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.156612+0000","last_change":"2024-09-21T09:39:27.026421+0000","last_active":"2024-09-21T09:45:10.156612+0000","last_peered":"2024-09-21T09:45:10.156612+0000","last_clean":"2024-09-21T09:45:10.156612+0000","last_became_active":"2024-09-21T09:39:27.026045+0000","last_became_peered":"2024-09-21T09:39:27.026045+0000","last_unstale":"2024-09-21T09:45:10.156612+0000","last_undegraded":"2024-09-21T09:45:10.156612+0000","last_fullsized":"2024-09-21T09:45:10.156612+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:23:30.404250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":2,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"0'0","reported_seq":1251,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.154602+0000","last_change":"2024-09-21T09:40:03.124602+0000","last_active":"2024-09-21T09:45:10.154602+0000","last_peered":"2024-09-21T09:45:10.154602+0000","last_clean":"2024-09-21T09:45:10.154602+0000","last_became_active":"2024-09-21T09:40:03.124319+0000","last_became_peered":"2024-09-21T09:40:03.124319+0000","last_unstale":"2024-09-21T09:45:10.154602+0000","last_undegraded":"2024-09-21T09:45:10.154602+0000","last_fullsized":"2024-09-21T09:45:10.154602+0000","mapping_epoch":578,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":579,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:38:23.059604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.0","version":"868'1158","reported_seq":4400,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.238547+0000","last_change":"2024-09-21T09:45:10.155061+0000","last_active":"2024-09-21T09:45:10.238547+0000","last_peered":"2024-09-21T09:45:10.238547+0000","last_clean":"2024-09-21T09:45:10.238547+0000","last_became_active":"2024-09-21T09:38:19.265542+0000","last_became_peered":"2024-09-21T09:38:19.265542+0000","last_unstale":"2024-09-21T09:45:10.238547+0000","last_undegraded":"2024-09-21T09:45:10.238547+0000","last_fullsized":"2024-09-21T09:45:10.238547+0000","mapping_epoch":477,"log_start":"0'0","ondisk_log_start":"0'0","created":70,"last_epoch_clean":478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:31:18.894586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:31:18.894586+0000","last_clean_scrub_stamp":"2024-09-21T09:31:18.894586+0000","objects_scrubbed":0,"log_size":1158,"log_dups_size":0,"ondisk_log_size":1158,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:23:10.238136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00068714799999999995,"stat_sum":{"num_bytes":12500992,"num_objects":13,"num_object_clones":10,"num_object_copies":39,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":13,"num_whiteouts":0,"num_read":227,"num_read_kb":296041,"num_write":582,"num_write_kb":113504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":25616384,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[{"start":"1d8","length":"1"},{"start":"1db","length":"1"},{"start":"1e4","length":"2"}]},{"pgid":"4.1","version":"0'0","reported_seq":1202,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.148212+0000","last_change":"2024-09-21T09:39:27.038382+0000","last_active":"2024-09-21T09:45:10.148212+0000","last_peered":"2024-09-21T09:45:10.148212+0000","last_clean":"2024-09-21T09:45:10.148212+0000","last_became_active":"2024-09-21T09:39:27.037568+0000","last_became_peered":"2024-09-21T09:39:27.037568+0000","last_unstale":"2024-09-21T09:45:10.148212+0000","last_undegraded":"2024-09-21T09:45:10.148212+0000","last_fullsized":"2024-09-21T09:45:10.148212+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:28:27.830318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:28:27.830318+0000","last_clean_scrub_stamp":"2024-09-21T09:28:27.830318+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:42:01.470953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"57'2","reported_seq":1122,"reported_epoch":867,"state":"active+clean","last_fresh":"2024-09-21T09:45:09.129681+0000","last_change":"2024-09-21T09:40:03.123911+0000","last_active":"2024-09-21T09:45:09.129681+0000","last_peered":"2024-09-21T09:45:09.129681+0000","last_clean":"2024-09-21T09:45:09.129681+0000","last_became_active":"2024-09-21T09:40:03.123569+0000","last_became_peered":"2024-09-21T09:40:03.123569+0000","last_unstale":"2024-09-21T09:45:09.129681+0000","last_undegraded":"2024-09-21T09:45:09.129681+0000","last_fullsized":"2024-09-21T09:45:09.129681+0000","mapping_epoch":578,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":579,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:09:56.105115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":6,"num_read_kb":6,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"0'0","reported_seq":1335,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.148745+0000","last_change":"2024-09-21T09:38:19.265298+0000","last_active":"2024-09-21T09:45:10.148745+0000","last_peered":"2024-09-21T09:45:10.148745+0000","last_clean":"2024-09-21T09:45:10.148745+0000","last_became_active":"2024-09-21T09:38:19.264894+0000","last_became_peered":"2024-09-21T09:38:19.264894+0000","last_unstale":"2024-09-21T09:45:10.148745+0000","last_undegraded":"2024-09-21T09:45:10.148745+0000","last_fullsized":"2024-09-21T09:45:10.148745+0000","mapping_epoch":477,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:47:36.892264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.f","version":"861'368","reported_seq":3529,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.153040+0000","last_change":"2024-09-21T09:45:10.153463+0000","last_active":"2024-09-21T09:45:10.153040+0000","last_peered":"2024-09-21T09:45:10.153040+0000","last_clean":"2024-09-21T09:45:10.153040+0000","last_became_active":"2024-09-21T09:39:27.025643+0000","last_became_peered":"2024-09-21T09:39:27.025643+0000","last_unstale":"2024-09-21T09:45:10.153040+0000","last_undegraded":"2024-09-21T09:45:10.153040+0000","last_fullsized":"2024-09-21T09:45:10.153040+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":70,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:31:18.894586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:31:18.894586+0000","last_clean_scrub_stamp":"2024-09-21T09:31:18.894586+0000","objects_scrubbed":0,"log_size":368,"log_dups_size":0,"ondisk_log_size":368,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:23:05.327252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00069220900000000005,"stat_sum":{"num_bytes":5595136,"num_objects":3,"num_object_clones":2,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":71,"num_read_kb":101574,"num_write":98,"num_write_kb":24656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":1712128,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[{"start":"1d8","length":"1"},{"start":"1db","length":"1"},{"start":"1e4","length":"2"}]},{"pgid":"2.8","version":"0'0","reported_seq":1110,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.154240+0000","last_change":"2024-09-21T09:40:03.134999+0000","last_active":"2024-09-21T09:45:10.154240+0000","last_peered":"2024-09-21T09:45:10.154240+0000","last_clean":"2024-09-21T09:45:10.154240+0000","last_became_active":"2024-09-21T09:40:03.134316+0000","last_became_peered":"2024-09-21T09:40:03.134316+0000","last_unstale":"2024-09-21T09:45:10.154240+0000","last_undegraded":"2024-09-21T09:45:10.154240+0000","last_fullsized":"2024-09-21T09:45:10.154240+0000","mapping_epoch":578,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":579,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:38:00.898982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1226,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.148604+0000","last_change":"2024-09-21T09:40:03.123591+0000","last_active":"2024-09-21T09:45:10.148604+0000","last_peered":"2024-09-21T09:45:10.148604+0000","last_clean":"2024-09-21T09:45:10.148604+0000","last_became_active":"2024-09-21T09:40:03.122310+0000","last_became_peered":"2024-09-21T09:40:03.122310+0000","last_unstale":"2024-09-21T09:45:10.148604+0000","last_undegraded":"2024-09-21T09:45:10.148604+0000","last_fullsized":"2024-09-21T09:45:10.148604+0000","mapping_epoch":578,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":579,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:50:53.123253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.e","version":"858'788","reported_seq":4035,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.149753+0000","last_change":"2024-09-21T09:45:10.150032+0000","last_active":"2024-09-21T09:45:10.149753+0000","last_peered":"2024-09-21T09:45:10.149753+0000","last_clean":"2024-09-21T09:45:10.149753+0000","last_became_active":"2024-09-21T09:38:55.242388+0000","last_became_peered":"2024-09-21T09:38:55.242388+0000","last_unstale":"2024-09-21T09:45:10.149753+0000","last_undegraded":"2024-09-21T09:45:10.149753+0000","last_fullsized":"2024-09-21T09:45:10.149753+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":70,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:31:18.894586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:31:18.894586+0000","last_clean_scrub_stamp":"2024-09-21T09:31:18.894586+0000","objects_scrubbed":0,"log_size":788,"log_dups_size":0,"ondisk_log_size":788,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:50:37.983591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00057494700000000002,"stat_sum":{"num_bytes":7249920,"num_objects":3,"num_object_clones":1,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":1,"num_read":167,"num_read_kb":226408,"num_write":342,"num_write_kb":78304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":54640640,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[{"start":"1d8","length":"1"},{"start":"1db","length":"1"},{"start":"1e4","length":"2"}]},{"pgid":"2.9","version":"0'0","reported_seq":1311,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.157938+0000","last_change":"2024-09-21T09:40:03.123997+0000","last_active":"2024-09-21T09:45:10.157938+0000","last_peered":"2024-09-21T09:45:10.157938+0000","last_clean":"2024-09-21T09:45:10.157938+0000","last_became_active":"2024-09-21T09:40:03.123702+0000","last_became_peered":"2024-09-21T09:40:03.123702+0000","last_unstale":"2024-09-21T09:45:10.157938+0000","last_undegraded":"2024-09-21T09:45:10.157938+0000","last_fullsized":"2024-09-21T09:45:10.157938+0000","mapping_epoch":578,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":579,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:38:56.257731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1243,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.153779+0000","last_change":"2024-09-21T09:40:03.126679+0000","last_active":"2024-09-21T09:45:10.153779+0000","last_peered":"2024-09-21T09:45:10.153779+0000","last_clean":"2024-09-21T09:45:10.153779+0000","last_became_active":"2024-09-21T09:40:03.125915+0000","last_became_peered":"2024-09-21T09:40:03.125915+0000","last_unstale":"2024-09-21T09:45:10.153779+0000","last_undegraded":"2024-09-21T09:45:10.153779+0000","last_fullsized":"2024-09-21T09:45:10.153779+0000","mapping_epoch":578,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":579,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:01:17.722644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.d","version":"868'2010","reported_seq":5286,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.155836+0000","last_change":"2024-09-21T09:45:10.157154+0000","last_active":"2024-09-21T09:45:10.155836+0000","last_peered":"2024-09-21T09:45:10.155836+0000","last_clean":"2024-09-21T09:45:10.155836+0000","last_became_active":"2024-09-21T09:40:03.129327+0000","last_became_peered":"2024-09-21T09:40:03.129327+0000","last_unstale":"2024-09-21T09:45:10.155836+0000","last_undegraded":"2024-09-21T09:45:10.155836+0000","last_fullsized":"2024-09-21T09:45:10.155836+0000","mapping_epoch":578,"log_start":"0'0","ondisk_log_start":"0'0","created":70,"last_epoch_clean":579,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:31:18.894586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:31:18.894586+0000","last_clean_scrub_stamp":"2024-09-21T09:31:18.894586+0000","objects_scrubbed":0,"log_size":2010,"log_dups_size":0,"ondisk_log_size":2010,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:50:23.393864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0087446020000000006,"stat_sum":{"num_bytes":20660224,"num_objects":9,"num_object_clones":5,"num_object_copies":27,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":9,"num_whiteouts":1,"num_read":409,"num_read_kb":459384,"num_write":788,"num_write_kb":176328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15,"num_bytes_recovered":58548224,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"1d8","length":"1"},{"start":"1db","length":"1"},{"start":"1e4","length":"2"}]},{"pgid":"2.a","version":"0'0","reported_seq":1311,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.155866+0000","last_change":"2024-09-21T09:40:03.124728+0000","last_active":"2024-09-21T09:45:10.155866+0000","last_peered":"2024-09-21T09:45:10.155866+0000","last_clean":"2024-09-21T09:45:10.155866+0000","last_became_active":"2024-09-21T09:40:03.124416+0000","last_became_peered":"2024-09-21T09:40:03.124416+0000","last_unstale":"2024-09-21T09:45:10.155866+0000","last_undegraded":"2024-09-21T09:45:10.155866+0000","last_fullsized":"2024-09-21T09:45:10.155866+0000","mapping_epoch":578,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":579,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:40:23.381710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"0'0","reported_seq":1245,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.154868+0000","last_change":"2024-09-21T09:38:19.265803+0000","last_active":"2024-09-21T09:45:10.154868+0000","last_peered":"2024-09-21T09:45:10.154868+0000","last_clean":"2024-09-21T09:45:10.154868+0000","last_became_active":"2024-09-21T09:38:19.265156+0000","last_became_peered":"2024-09-21T09:38:19.265156+0000","last_unstale":"2024-09-21T09:45:10.154868+0000","last_undegraded":"2024-09-21T09:45:10.154868+0000","last_fullsized":"2024-09-21T09:45:10.154868+0000","mapping_epoch":477,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:25:39.742890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.c","version":"866'1126","reported_seq":4565,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.157340+0000","last_change":"2024-09-21T09:45:10.158293+0000","last_active":"2024-09-21T09:45:10.157340+0000","last_peered":"2024-09-21T09:45:10.157340+0000","last_clean":"2024-09-21T09:45:10.157340+0000","last_became_active":"2024-09-21T09:38:19.269789+0000","last_became_peered":"2024-09-21T09:38:19.269789+0000","last_unstale":"2024-09-21T09:45:10.157340+0000","last_undegraded":"2024-09-21T09:45:10.157340+0000","last_fullsized":"2024-09-21T09:45:10.157340+0000","mapping_epoch":477,"log_start":"0'0","ondisk_log_start":"0'0","created":70,"last_epoch_clean":478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:31:18.894586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:31:18.894586+0000","last_clean_scrub_stamp":"2024-09-21T09:31:18.894586+0000","objects_scrubbed":0,"log_size":1126,"log_dups_size":0,"ondisk_log_size":1126,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:24:46.533905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0012035360000000001,"stat_sum":{"num_bytes":6135808,"num_objects":7,"num_object_clones":4,"num_object_copies":21,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7,"num_whiteouts":0,"num_read":272,"num_read_kb":514029,"num_write":546,"num_write_kb":113808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":16179200,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"1d8","length":"1"},{"start":"1db","length":"1"},{"start":"1e4","length":"2"}]},{"pgid":"2.b","version":"599'3","reported_seq":1127,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.153433+0000","last_change":"2024-09-21T09:40:03.126410+0000","last_active":"2024-09-21T09:45:10.153433+0000","last_peered":"2024-09-21T09:45:10.153433+0000","last_clean":"2024-09-21T09:45:10.153433+0000","last_became_active":"2024-09-21T09:40:03.126045+0000","last_became_peered":"2024-09-21T09:40:03.126045+0000","last_unstale":"2024-09-21T09:45:10.153433+0000","last_undegraded":"2024-09-21T09:45:10.153433+0000","last_fullsized":"2024-09-21T09:45:10.153433+0000","mapping_epoch":578,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":579,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:29:26.756716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1164,"reported_epoch":867,"state":"active+clean","last_fresh":"2024-09-21T09:45:09.129617+0000","last_change":"2024-09-21T09:39:27.027861+0000","last_active":"2024-09-21T09:45:09.129617+0000","last_peered":"2024-09-21T09:45:09.129617+0000","last_clean":"2024-09-21T09:45:09.129617+0000","last_became_active":"2024-09-21T09:39:27.027526+0000","last_became_peered":"2024-09-21T09:39:27.027526+0000","last_unstale":"2024-09-21T09:45:09.129617+0000","last_undegraded":"2024-09-21T09:45:09.129617+0000","last_fullsized":"2024-09-21T09:45:09.129617+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:47:15.319396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1d","version":"0'0","reported_seq":1173,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.151989+0000","last_change":"2024-09-21T09:39:27.031259+0000","last_active":"2024-09-21T09:45:10.151989+0000","last_peered":"2024-09-21T09:45:10.151989+0000","last_clean":"2024-09-21T09:45:10.151989+0000","last_became_active":"2024-09-21T09:39:27.030945+0000","last_became_peered":"2024-09-21T09:39:27.030945+0000","last_unstale":"2024-09-21T09:45:10.151989+0000","last_undegraded":"2024-09-21T09:45:10.151989+0000","last_fullsized":"2024-09-21T09:45:10.151989+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:57:58.055425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1c","version":"57'2","reported_seq":1127,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.153817+0000","last_change":"2024-09-21T09:40:03.129124+0000","last_active":"2024-09-21T09:45:10.153817+0000","last_peered":"2024-09-21T09:45:10.153817+0000","last_clean":"2024-09-21T09:45:10.153817+0000","last_became_active":"2024-09-21T09:40:03.128796+0000","last_became_peered":"2024-09-21T09:40:03.128796+0000","last_unstale":"2024-09-21T09:45:10.153817+0000","last_undegraded":"2024-09-21T09:45:10.153817+0000","last_fullsized":"2024-09-21T09:45:10.153817+0000","mapping_epoch":578,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":579,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:28:14.324206+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":6,"num_read_kb":6,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1c","version":"0'0","reported_seq":1179,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.152616+0000","last_change":"2024-09-21T09:38:55.241890+0000","last_active":"2024-09-21T09:45:10.152616+0000","last_peered":"2024-09-21T09:45:10.152616+0000","last_clean":"2024-09-21T09:45:10.152616+0000","last_became_active":"2024-09-21T09:38:55.241534+0000","last_became_peered":"2024-09-21T09:38:55.241534+0000","last_unstale":"2024-09-21T09:45:10.152616+0000","last_undegraded":"2024-09-21T09:45:10.152616+0000","last_fullsized":"2024-09-21T09:45:10.152616+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:52:56.999064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1d","version":"57'2","reported_seq":1116,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.154341+0000","last_change":"2024-09-21T09:40:03.135427+0000","last_active":"2024-09-21T09:45:10.154341+0000","last_peered":"2024-09-21T09:45:10.154341+0000","last_clean":"2024-09-21T09:45:10.154341+0000","last_became_active":"2024-09-21T09:40:03.134395+0000","last_became_peered":"2024-09-21T09:40:03.134395+0000","last_unstale":"2024-09-21T09:45:10.154341+0000","last_undegraded":"2024-09-21T09:45:10.154341+0000","last_fullsized":"2024-09-21T09:45:10.154341+0000","mapping_epoch":578,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":579,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:53:47.621184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":6,"num_read_kb":6,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1f","version":"0'0","reported_seq":1335,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.149244+0000","last_change":"2024-09-21T09:38:55.243482+0000","last_active":"2024-09-21T09:45:10.149244+0000","last_peered":"2024-09-21T09:45:10.149244+0000","last_clean":"2024-09-21T09:45:10.149244+0000","last_became_active":"2024-09-21T09:38:55.243137+0000","last_became_peered":"2024-09-21T09:38:55.243137+0000","last_unstale":"2024-09-21T09:45:10.149244+0000","last_undegraded":"2024-09-21T09:45:10.149244+0000","last_fullsized":"2024-09-21T09:45:10.149244+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:02:51.353824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1e","version":"599'2","reported_seq":1256,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.154653+0000","last_change":"2024-09-21T09:38:55.242343+0000","last_active":"2024-09-21T09:45:10.154653+0000","last_peered":"2024-09-21T09:45:10.154653+0000","last_clean":"2024-09-21T09:45:10.154653+0000","last_became_active":"2024-09-21T09:38:55.241975+0000","last_became_peered":"2024-09-21T09:38:55.241975+0000","last_unstale":"2024-09-21T09:45:10.154653+0000","last_undegraded":"2024-09-21T09:45:10.154653+0000","last_fullsized":"2024-09-21T09:45:10.154653+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:44:27.736713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"0'0","reported_seq":1235,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.154603+0000","last_change":"2024-09-21T09:39:27.027385+0000","last_active":"2024-09-21T09:45:10.154603+0000","last_peered":"2024-09-21T09:45:10.154603+0000","last_clean":"2024-09-21T09:45:10.154603+0000","last_became_active":"2024-09-21T09:39:27.027038+0000","last_became_peered":"2024-09-21T09:39:27.027038+0000","last_unstale":"2024-09-21T09:45:10.154603+0000","last_undegraded":"2024-09-21T09:45:10.154603+0000","last_fullsized":"2024-09-21T09:45:10.154603+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:34.609070+0000","last_clean_scrub_stamp":"2024-09-21T09:27:34.609070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:35:37.403698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1f","version":"599'5","reported_seq":1347,"reported_epoch":868,"state":"active+clean","last_fresh":"2024-09-21T09:45:10.149621+0000","last_change":"2024-09-21T09:40:03.126119+0000","last_active":"2024-09-21T09:45:10.149621+0000","last_peered":"2024-09-21T09:45:10.149621+0000","last_clean":"2024-09-21T09:45:10.149621+0000","last_became_active":"2024-09-21T09:40:03.125721+0000","last_became_peered":"2024-09-21T09:40:03.125721+0000","last_unstale":"2024-09-21T09:45:10.149621+0000","last_undegraded":"2024-09-21T09:45:10.149621+0000","last_fullsized":"2024-09-21T09:45:10.149621+0000","mapping_epoch":578,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":579,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:27:31.247150+0000","last_clean_scrub_stamp":"2024-09-21T09:27:31.247150+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:27:39.147744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":4,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":5,"num_pg":16,"stat_sum":{"num_bytes":215392256,"num_objects":129,"num_object_clones":81,"num_object_copies":387,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":129,"num_whiteouts":10,"num_read":3770,"num_read_kb":4993325,"num_write":8082,"num_write_kb":1765160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":139,"num_bytes_recovered":472702976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":286523392,"data_stored":509562880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":18437,"ondisk_log_size":18437,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":192,"acting":192,"num_store_stats":8},{"poolid":2,"num_pg":32,"stat_sum":{"num_bytes":6602,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":79,"num_read_kb":86,"num_write":68,"num_write_kb":35,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":110592,"data_stored":19806,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":48082,"internal_metadata":0},"log_size":68,"ondisk_log_size":68,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":618,"num_read_kb":1145,"num_write":221,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":136,"ondisk_log_size":136,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":578,"seq":2482491097151,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":211684,"kb_used_data":40472,"kb_used_omap":34,"kb_used_meta":171165,"kb_avail":93533468,"statfs":{"total":95995035648,"available":95778271232,"internally_reserved":0,"allocated":41443328,"data_stored":61884041,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":35818,"internal_metadata":175272982},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":6,"up_from":543,"seq":2332167241798,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":186348,"kb_used_data":34008,"kb_used_omap":28,"kb_used_meta":152291,"kb_avail":93558804,"statfs":{"total":95995035648,"available":95804215296,"internally_reserved":0,"allocated":34824192,"data_stored":54759138,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29247,"internal_metadata":155946433},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":512,"seq":2199023255629,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":345212,"kb_used_data":62060,"kb_used_omap":55,"kb_used_meta":283080,"kb_avail":93399940,"statfs":{"total":95995035648,"available":95641538560,"internally_reserved":0,"allocated":63549440,"data_stored":88856894,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":56524,"internal_metadata":289874740},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":477,"seq":2048699400276,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":197212,"kb_used_data":35536,"kb_used_omap":30,"kb_used_meta":161633,"kb_avail":93547940,"statfs":{"total":95995035648,"available":95793090560,"internally_reserved":0,"allocated":36388864,"data_stored":61609212,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31646,"internal_metadata":165512290},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":443,"seq":1902670512219,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":180480,"kb_used_data":26732,"kb_used_omap":17,"kb_used_meta":153710,"kb_avail":93564672,"statfs":{"total":95995035648,"available":95810224128,"internally_reserved":0,"allocated":27373568,"data_stored":47774001,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":17994,"internal_metadata":157399478},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":410,"seq":1760936591457,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":370936,"kb_used_data":57572,"kb_used_omap":39,"kb_used_meta":313304,"kb_avail":93374216,"statfs":{"total":95995035648,"available":95615197184,"internally_reserved":0,"allocated":58953728,"data_stored":84076615,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":40249,"internal_metadata":320824007},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":376,"seq":1614907703400,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":450288,"kb_used_data":54564,"kb_used_omap":39,"kb_used_meta":395672,"kb_avail":93294864,"statfs":{"total":95995035648,"available":95533940736,"internally_reserved":0,"allocated":55873536,"data_stored":90063588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":40350,"internal_metadata":405168738},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":343,"seq":1473173782639,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":357624,"kb_used_data":46636,"kb_used_omap":40,"kb_used_meta":310935,"kb_avail":93387528,"statfs":{"total":95995035648,"available":95628828672,"internally_reserved":0,"allocated":47755264,"data_stored":66834174,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":41886,"internal_metadata":318397538},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":716,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5940,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4702,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6264,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6648,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6724,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5906,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5569,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":5816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6153,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6082,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":37425152,"data_stored":60678144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":46137344,"data_stored":84496384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":49262592,"data_stored":78528512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":17625088,"data_stored":42201088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":26644480,"data_stored":56041472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":53800960,"data_stored":83283968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24510464,"data_stored":48607232,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":31117312,"data_stored":55726080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T09:45:16.900 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_0 2024-09-21T09:45:16.900 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid d65e801e-77fa-11ef-baf6-efdc52797490 -- ceph osd pool rm unique_pool_0 unique_pool_0 --yes-i-really-really-mean-it 2024-09-21T09:45:17.158 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:45:17.411 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:17 smithi022 ceph-mon[98615]: from='client.34590 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:45:17.411 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:17 smithi022 ceph-mon[96736]: from='client.34590 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:45:17.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:17 smithi064 ceph-mon[86709]: from='client.34590 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:45:18.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:18 smithi022 ceph-mon[98615]: pgmap v920: 121 pgs: 121 active+clean; 208 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 4.0 MiB/s rd, 695 KiB/s wr, 7 op/s 2024-09-21T09:45:18.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:18 smithi022 ceph-mon[96736]: pgmap v920: 121 pgs: 121 active+clean; 208 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 4.0 MiB/s rd, 695 KiB/s wr, 7 op/s 2024-09-21T09:45:18.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:18 smithi064 ceph-mon[86709]: pgmap v920: 121 pgs: 121 active+clean; 208 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 4.0 MiB/s rd, 695 KiB/s wr, 7 op/s 2024-09-21T09:45:19.122 INFO:teuthology.orchestra.run.smithi022.stderr:pool 'unique_pool_0' removed 2024-09-21T09:45:19.441 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:19 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/3394006324' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true} : dispatch 2024-09-21T09:45:19.441 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:19 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/3394006324' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2024-09-21T09:45:19.441 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:19 smithi022 ceph-mon[98615]: osdmap e870: 8 total, 8 up, 8 in 2024-09-21T09:45:19.441 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:19 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/3394006324' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true} : dispatch 2024-09-21T09:45:19.441 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:19 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/3394006324' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2024-09-21T09:45:19.441 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:19 smithi022 ceph-mon[96736]: osdmap e870: 8 total, 8 up, 8 in 2024-09-21T09:45:19.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:19 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/3394006324' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true} : dispatch 2024-09-21T09:45:19.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:19 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/3394006324' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2024-09-21T09:45:19.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:19 smithi064 ceph-mon[86709]: osdmap e870: 8 total, 8 up, 8 in 2024-09-21T09:45:19.783 INFO:teuthology.task.full_sequential:In full_sequential, running task print... 2024-09-21T09:45:19.784 INFO:teuthology.task.print:**** done end ec-rados-default.yaml 2024-09-21T09:45:19.784 INFO:teuthology.task.full_sequential:In full_sequential, running task print... 2024-09-21T09:45:19.784 INFO:teuthology.task.print:**** done start rados_api.yaml 2024-09-21T09:45:19.784 INFO:teuthology.task.full_sequential:In full_sequential, running task workunit... 2024-09-21T09:45:19.784 INFO:tasks.workunit:Pulling workunits from ref quincy 2024-09-21T09:45:19.785 INFO:tasks.workunit:Making a separate scratch dir for every client... 2024-09-21T09:45:19.785 DEBUG:teuthology.orchestra.run.smithi022:> stat -- /home/ubuntu/cephtest/mnt.0 2024-09-21T09:45:19.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:45:19.802 INFO:teuthology.orchestra.run.smithi022.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2024-09-21T09:45:19.802 DEBUG:teuthology.orchestra.run.smithi022:> mkdir -- /home/ubuntu/cephtest/mnt.0 2024-09-21T09:45:19.859 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2024-09-21T09:45:19.859 DEBUG:teuthology.orchestra.run.smithi022:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2024-09-21T09:45:19.915 INFO:tasks.workunit:timeout=3h 2024-09-21T09:45:19.915 INFO:tasks.workunit:cleanup=True 2024-09-21T09:45:19.915 DEBUG:teuthology.orchestra.run.smithi022:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch quincy https://git.ceph.com/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2024-09-21T09:45:19.971 INFO:tasks.workunit.client.0.smithi022.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2024-09-21T09:45:20.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:20 smithi064 ceph-mon[86709]: pgmap v921: 121 pgs: 121 active+clean; 208 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 3.5 MiB/s rd, 614 KiB/s wr, 6 op/s 2024-09-21T09:45:20.442 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:45:20 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:45:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:45:20.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:20 smithi022 ceph-mon[96736]: pgmap v921: 121 pgs: 121 active+clean; 208 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 3.5 MiB/s rd, 614 KiB/s wr, 6 op/s 2024-09-21T09:45:20.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:20 smithi022 ceph-mon[98615]: pgmap v921: 121 pgs: 121 active+clean; 208 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 3.5 MiB/s rd, 614 KiB/s wr, 6 op/s 2024-09-21T09:45:21.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:21 smithi064 ceph-mon[86709]: pgmap v923: 105 pgs: 105 active+clean; 583 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:45:21.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:21 smithi022 ceph-mon[96736]: pgmap v923: 105 pgs: 105 active+clean; 583 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:45:21.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:21 smithi022 ceph-mon[98615]: pgmap v923: 105 pgs: 105 active+clean; 583 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:45:23.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:23 smithi022 ceph-mon[96736]: pgmap v924: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:45:23.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:23 smithi022 ceph-mon[98615]: pgmap v924: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:45:23.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:23 smithi064 ceph-mon[86709]: pgmap v924: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:45:25.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:25 smithi022 ceph-mon[98615]: pgmap v925: 105 pgs: 105 active+clean; 583 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:45:25.808 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:45:25 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:45:25] "GET /metrics HTTP/1.1" 200 36356 "" "Prometheus/2.51.0" 2024-09-21T09:45:25.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:25 smithi022 ceph-mon[96736]: pgmap v925: 105 pgs: 105 active+clean; 583 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:45:25.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:25 smithi064 ceph-mon[86709]: pgmap v925: 105 pgs: 105 active+clean; 583 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:45:27.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:27 smithi022 ceph-mon[98615]: pgmap v926: 105 pgs: 105 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:45:27.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:27 smithi022 ceph-mon[96736]: pgmap v926: 105 pgs: 105 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:45:27.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:27 smithi064 ceph-mon[86709]: pgmap v926: 105 pgs: 105 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:45:28.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:28 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:45:28.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:28 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:45:28.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:28 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:45:29.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:29 smithi022 ceph-mon[98615]: pgmap v927: 105 pgs: 105 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:45:29.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:29 smithi022 ceph-mon[96736]: pgmap v927: 105 pgs: 105 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:45:29.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:29 smithi064 ceph-mon[86709]: pgmap v927: 105 pgs: 105 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:45:30.691 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:45:30 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:45:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:45:31.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:31 smithi022 ceph-mon[98615]: pgmap v928: 105 pgs: 105 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:45:31.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:31 smithi022 ceph-mon[96736]: pgmap v928: 105 pgs: 105 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:45:31.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:31 smithi064 ceph-mon[86709]: pgmap v928: 105 pgs: 105 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:45:32.805 DEBUG:teuthology.orchestra.run.smithi022:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2024-09-21T09:45:32.862 INFO:tasks.workunit.client.0.smithi022.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2024-09-21T09:45:32.864 INFO:tasks.workunit.client.0.smithi022.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-09-21T09:45:32.864 INFO:tasks.workunit.client.0.smithi022.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2024-09-21T09:45:32.914 INFO:tasks.workunit.client.0.smithi022.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2024-09-21T09:45:32.964 INFO:tasks.workunit.client.0.smithi022.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2024-09-21T09:45:33.004 INFO:tasks.workunit.client.0.smithi022.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-09-21T09:45:33.006 INFO:tasks.workunit.client.0.smithi022.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-09-21T09:45:33.006 INFO:tasks.workunit.client.0.smithi022.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2024-09-21T09:45:33.046 INFO:tasks.workunit.client.0.smithi022.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-09-21T09:45:33.050 DEBUG:teuthology.orchestra.run.smithi022:> set -ex 2024-09-21T09:45:33.050 DEBUG:teuthology.orchestra.run.smithi022:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2024-09-21T09:45:33.106 INFO:tasks.workunit:Running workunits matching cls on client.0... 2024-09-21T09:45:33.107 INFO:tasks.workunit:Running workunit cls/test_cls_2pc_queue.sh... 2024-09-21T09:45:33.107 DEBUG:teuthology.orchestra.run.smithi022:workunit test cls/test_cls_2pc_queue.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_2pc_queue.sh 2024-09-21T09:45:33.173 INFO:tasks.workunit.client.0.smithi022.stdout:Running main() from gmock_main.cc 2024-09-21T09:45:33.173 INFO:tasks.workunit.client.0.smithi022.stdout:[==========] Running 22 tests from 1 test suite. 2024-09-21T09:45:33.173 INFO:tasks.workunit.client.0.smithi022.stdout:[----------] Global test environment set-up. 2024-09-21T09:45:33.173 INFO:tasks.workunit.client.0.smithi022.stdout:[----------] 22 tests from TestCls2PCQueue 2024-09-21T09:45:33.173 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestCls2PCQueue.GetCapacity 2024-09-21T09:45:33.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:33 smithi022 ceph-mon[98615]: pgmap v929: 105 pgs: 105 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:45:33.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:33 smithi022 ceph-mon[96736]: pgmap v929: 105 pgs: 105 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:45:33.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:33 smithi064 ceph-mon[86709]: pgmap v929: 105 pgs: 105 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:45:34.842 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:34 smithi064 ceph-mon[86709]: osdmap e871: 8 total, 8 up, 8 in 2024-09-21T09:45:34.843 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:34 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/2302286091' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:45:35.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:34 smithi022 ceph-mon[98615]: osdmap e871: 8 total, 8 up, 8 in 2024-09-21T09:45:35.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:34 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/2302286091' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:45:35.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:34 smithi022 ceph-mon[96736]: osdmap e871: 8 total, 8 up, 8 in 2024-09-21T09:45:35.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:34 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/2302286091' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:45:35.574 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestCls2PCQueue.GetCapacity (2401 ms) 2024-09-21T09:45:35.574 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestCls2PCQueue.AsyncGetCapacity 2024-09-21T09:45:35.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:35 smithi064 ceph-mon[86709]: pgmap v931: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:45:35.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:35 smithi064 ceph-mon[86709]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:45:35.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:35 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/2302286091' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:45:35.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:35 smithi064 ceph-mon[86709]: osdmap e872: 8 total, 8 up, 8 in 2024-09-21T09:45:36.058 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:45:35 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:45:35] "GET /metrics HTTP/1.1" 200 34004 "" "Prometheus/2.51.0" 2024-09-21T09:45:36.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:35 smithi022 ceph-mon[98615]: pgmap v931: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:45:36.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:35 smithi022 ceph-mon[98615]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:45:36.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:35 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/2302286091' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:45:36.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:35 smithi022 ceph-mon[98615]: osdmap e872: 8 total, 8 up, 8 in 2024-09-21T09:45:36.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:35 smithi022 ceph-mon[96736]: pgmap v931: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:45:36.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:35 smithi022 ceph-mon[96736]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:45:36.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:35 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/2302286091' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:45:36.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:35 smithi022 ceph-mon[96736]: osdmap e872: 8 total, 8 up, 8 in 2024-09-21T09:45:36.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:36 smithi022 ceph-mon[98615]: osdmap e873: 8 total, 8 up, 8 in 2024-09-21T09:45:36.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:36 smithi022 ceph-mon[96736]: osdmap e873: 8 total, 8 up, 8 in 2024-09-21T09:45:36.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:36 smithi064 ceph-mon[86709]: osdmap e873: 8 total, 8 up, 8 in 2024-09-21T09:45:37.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:37 smithi064 ceph-mon[86709]: pgmap v934: 105 pgs: 105 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:45:37.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:37 smithi064 ceph-mon[86709]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:45:37.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:37 smithi064 ceph-mon[86709]: Cluster is now healthy 2024-09-21T09:45:37.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:37 smithi064 ceph-mon[86709]: osdmap e874: 8 total, 8 up, 8 in 2024-09-21T09:45:37.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:37 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/2126541197' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:45:38.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:37 smithi022 ceph-mon[96736]: pgmap v934: 105 pgs: 105 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:45:38.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:37 smithi022 ceph-mon[96736]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:45:38.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:37 smithi022 ceph-mon[96736]: Cluster is now healthy 2024-09-21T09:45:38.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:37 smithi022 ceph-mon[96736]: osdmap e874: 8 total, 8 up, 8 in 2024-09-21T09:45:38.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:37 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/2126541197' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:45:38.059 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:37 smithi022 ceph-mon[98615]: pgmap v934: 105 pgs: 105 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:45:38.059 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:37 smithi022 ceph-mon[98615]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:45:38.059 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:37 smithi022 ceph-mon[98615]: Cluster is now healthy 2024-09-21T09:45:38.059 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:37 smithi022 ceph-mon[98615]: osdmap e874: 8 total, 8 up, 8 in 2024-09-21T09:45:38.059 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:37 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/2126541197' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:45:38.593 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestCls2PCQueue.AsyncGetCapacity (3019 ms) 2024-09-21T09:45:38.593 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestCls2PCQueue.Reserve 2024-09-21T09:45:38.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:38 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/2126541197' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:45:38.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:38 smithi064 ceph-mon[86709]: osdmap e875: 8 total, 8 up, 8 in 2024-09-21T09:45:39.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:38 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/2126541197' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:45:39.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:38 smithi022 ceph-mon[98615]: osdmap e875: 8 total, 8 up, 8 in 2024-09-21T09:45:39.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:38 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/2126541197' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:45:39.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:38 smithi022 ceph-mon[96736]: osdmap e875: 8 total, 8 up, 8 in 2024-09-21T09:45:39.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:39 smithi064 ceph-mon[86709]: pgmap v937: 137 pgs: 3 creating+peering, 15 unknown, 119 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 255 B/s wr, 0 op/s 2024-09-21T09:45:39.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:39 smithi064 ceph-mon[86709]: osdmap e876: 8 total, 8 up, 8 in 2024-09-21T09:45:40.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:39 smithi022 ceph-mon[96736]: pgmap v937: 137 pgs: 3 creating+peering, 15 unknown, 119 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 255 B/s wr, 0 op/s 2024-09-21T09:45:40.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:39 smithi022 ceph-mon[96736]: osdmap e876: 8 total, 8 up, 8 in 2024-09-21T09:45:40.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:39 smithi022 ceph-mon[98615]: pgmap v937: 137 pgs: 3 creating+peering, 15 unknown, 119 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 255 B/s wr, 0 op/s 2024-09-21T09:45:40.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:39 smithi022 ceph-mon[98615]: osdmap e876: 8 total, 8 up, 8 in 2024-09-21T09:45:40.599 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:45:40 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:45:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:45:40.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:40 smithi064 ceph-mon[86709]: osdmap e877: 8 total, 8 up, 8 in 2024-09-21T09:45:40.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:40 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/1407823674' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:45:40.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:40 smithi064 ceph-mon[86709]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:45:41.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:40 smithi022 ceph-mon[98615]: osdmap e877: 8 total, 8 up, 8 in 2024-09-21T09:45:41.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:40 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/1407823674' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:45:41.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:40 smithi022 ceph-mon[98615]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:45:41.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:40 smithi022 ceph-mon[96736]: osdmap e877: 8 total, 8 up, 8 in 2024-09-21T09:45:41.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:40 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/1407823674' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:45:41.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:40 smithi022 ceph-mon[96736]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:45:41.612 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestCls2PCQueue.Reserve (3019 ms) 2024-09-21T09:45:41.612 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestCls2PCQueue.AsyncReserve 2024-09-21T09:45:41.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:41 smithi064 ceph-mon[86709]: pgmap v940: 137 pgs: 3 creating+peering, 29 unknown, 105 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:45:41.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:41 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/1407823674' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:45:41.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:41 smithi064 ceph-mon[86709]: osdmap e878: 8 total, 8 up, 8 in 2024-09-21T09:45:42.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:41 smithi022 ceph-mon[98615]: pgmap v940: 137 pgs: 3 creating+peering, 29 unknown, 105 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:45:42.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:41 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/1407823674' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:45:42.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:41 smithi022 ceph-mon[98615]: osdmap e878: 8 total, 8 up, 8 in 2024-09-21T09:45:42.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:41 smithi022 ceph-mon[96736]: pgmap v940: 137 pgs: 3 creating+peering, 29 unknown, 105 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:45:42.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:41 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/1407823674' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:45:42.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:41 smithi022 ceph-mon[96736]: osdmap e878: 8 total, 8 up, 8 in 2024-09-21T09:45:42.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:42 smithi064 ceph-mon[86709]: osdmap e879: 8 total, 8 up, 8 in 2024-09-21T09:45:43.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:42 smithi022 ceph-mon[98615]: osdmap e879: 8 total, 8 up, 8 in 2024-09-21T09:45:43.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:42 smithi022 ceph-mon[96736]: osdmap e879: 8 total, 8 up, 8 in 2024-09-21T09:45:43.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:43 smithi064 ceph-mon[86709]: pgmap v943: 105 pgs: 105 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:45:43.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:43 smithi064 ceph-mon[86709]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:45:43.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:43 smithi064 ceph-mon[86709]: Cluster is now healthy 2024-09-21T09:45:43.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:43 smithi064 ceph-mon[86709]: osdmap e880: 8 total, 8 up, 8 in 2024-09-21T09:45:43.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:43 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/4215233085' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:45:43.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:43 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:45:44.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:43 smithi022 ceph-mon[98615]: pgmap v943: 105 pgs: 105 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:45:44.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:43 smithi022 ceph-mon[98615]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:45:44.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:43 smithi022 ceph-mon[98615]: Cluster is now healthy 2024-09-21T09:45:44.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:43 smithi022 ceph-mon[98615]: osdmap e880: 8 total, 8 up, 8 in 2024-09-21T09:45:44.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:43 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/4215233085' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:45:44.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:43 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:45:44.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:43 smithi022 ceph-mon[96736]: pgmap v943: 105 pgs: 105 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:45:44.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:43 smithi022 ceph-mon[96736]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:45:44.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:43 smithi022 ceph-mon[96736]: Cluster is now healthy 2024-09-21T09:45:44.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:43 smithi022 ceph-mon[96736]: osdmap e880: 8 total, 8 up, 8 in 2024-09-21T09:45:44.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:43 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/4215233085' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:45:44.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:43 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:45:44.633 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestCls2PCQueue.AsyncReserve (3021 ms) 2024-09-21T09:45:44.634 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestCls2PCQueue.Commit 2024-09-21T09:45:44.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:44 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/4215233085' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:45:44.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:44 smithi064 ceph-mon[86709]: osdmap e881: 8 total, 8 up, 8 in 2024-09-21T09:45:45.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:44 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/4215233085' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:45:45.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:44 smithi022 ceph-mon[98615]: osdmap e881: 8 total, 8 up, 8 in 2024-09-21T09:45:45.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:44 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/4215233085' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:45:45.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:44 smithi022 ceph-mon[96736]: osdmap e881: 8 total, 8 up, 8 in 2024-09-21T09:45:45.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:45 smithi064 ceph-mon[86709]: pgmap v946: 137 pgs: 18 creating+peering, 14 unknown, 105 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:45:45.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:45 smithi064 ceph-mon[86709]: osdmap e882: 8 total, 8 up, 8 in 2024-09-21T09:45:46.058 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:45:45 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:45:45] "GET /metrics HTTP/1.1" 200 36312 "" "Prometheus/2.51.0" 2024-09-21T09:45:46.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:45 smithi022 ceph-mon[98615]: pgmap v946: 137 pgs: 18 creating+peering, 14 unknown, 105 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:45:46.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:45 smithi022 ceph-mon[98615]: osdmap e882: 8 total, 8 up, 8 in 2024-09-21T09:45:46.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:45 smithi022 ceph-mon[96736]: pgmap v946: 137 pgs: 18 creating+peering, 14 unknown, 105 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:45:46.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:45 smithi022 ceph-mon[96736]: osdmap e882: 8 total, 8 up, 8 in 2024-09-21T09:45:46.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:46 smithi064 ceph-mon[86709]: osdmap e883: 8 total, 8 up, 8 in 2024-09-21T09:45:46.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:46 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/142367546' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-5","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:45:47.057 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:46 smithi022 ceph-mon[98615]: osdmap e883: 8 total, 8 up, 8 in 2024-09-21T09:45:47.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:46 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/142367546' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-5","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:45:47.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:46 smithi022 ceph-mon[96736]: osdmap e883: 8 total, 8 up, 8 in 2024-09-21T09:45:47.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:46 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/142367546' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-5","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:45:47.651 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestCls2PCQueue.Commit (3017 ms) 2024-09-21T09:45:47.651 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestCls2PCQueue.Abort 2024-09-21T09:45:47.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:47 smithi064 ceph-mon[86709]: pgmap v949: 137 pgs: 6 creating+peering, 26 unknown, 105 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:45:47.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:47 smithi064 ceph-mon[86709]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:45:47.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:47 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/142367546' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-5","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:45:47.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:47 smithi064 ceph-mon[86709]: osdmap e884: 8 total, 8 up, 8 in 2024-09-21T09:45:48.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:47 smithi022 ceph-mon[98615]: pgmap v949: 137 pgs: 6 creating+peering, 26 unknown, 105 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:45:48.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:47 smithi022 ceph-mon[98615]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:45:48.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:47 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/142367546' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-5","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:45:48.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:47 smithi022 ceph-mon[98615]: osdmap e884: 8 total, 8 up, 8 in 2024-09-21T09:45:48.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:47 smithi022 ceph-mon[96736]: pgmap v949: 137 pgs: 6 creating+peering, 26 unknown, 105 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:45:48.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:47 smithi022 ceph-mon[96736]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:45:48.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:47 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/142367546' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-5","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:45:48.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:47 smithi022 ceph-mon[96736]: osdmap e884: 8 total, 8 up, 8 in 2024-09-21T09:45:48.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:48 smithi064 ceph-mon[86709]: osdmap e885: 8 total, 8 up, 8 in 2024-09-21T09:45:49.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:48 smithi022 ceph-mon[98615]: osdmap e885: 8 total, 8 up, 8 in 2024-09-21T09:45:49.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:48 smithi022 ceph-mon[96736]: osdmap e885: 8 total, 8 up, 8 in 2024-09-21T09:45:49.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:49 smithi064 ceph-mon[86709]: pgmap v952: 105 pgs: 105 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:45:49.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:49 smithi064 ceph-mon[86709]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:45:49.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:49 smithi064 ceph-mon[86709]: Cluster is now healthy 2024-09-21T09:45:49.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:49 smithi064 ceph-mon[86709]: osdmap e886: 8 total, 8 up, 8 in 2024-09-21T09:45:49.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:49 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/3689230574' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-6","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:45:49.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:49 smithi064 ceph-mon[86709]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-6","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:45:50.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:49 smithi022 ceph-mon[98615]: pgmap v952: 105 pgs: 105 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:45:50.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:49 smithi022 ceph-mon[98615]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:45:50.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:49 smithi022 ceph-mon[98615]: Cluster is now healthy 2024-09-21T09:45:50.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:49 smithi022 ceph-mon[98615]: osdmap e886: 8 total, 8 up, 8 in 2024-09-21T09:45:50.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:49 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/3689230574' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-6","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:45:50.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:49 smithi022 ceph-mon[98615]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-6","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:45:50.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:49 smithi022 ceph-mon[96736]: pgmap v952: 105 pgs: 105 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:45:50.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:49 smithi022 ceph-mon[96736]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:45:50.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:49 smithi022 ceph-mon[96736]: Cluster is now healthy 2024-09-21T09:45:50.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:49 smithi022 ceph-mon[96736]: osdmap e886: 8 total, 8 up, 8 in 2024-09-21T09:45:50.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:49 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/3689230574' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-6","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:45:50.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:49 smithi022 ceph-mon[96736]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-6","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:45:50.671 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:45:50 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:45:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:45:50.674 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestCls2PCQueue.Abort (3023 ms) 2024-09-21T09:45:50.674 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestCls2PCQueue.ReserveError 2024-09-21T09:45:50.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:50 smithi064 ceph-mon[86709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-6","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:45:50.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:50 smithi064 ceph-mon[86709]: osdmap e887: 8 total, 8 up, 8 in 2024-09-21T09:45:51.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:50 smithi022 ceph-mon[98615]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-6","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:45:51.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:50 smithi022 ceph-mon[98615]: osdmap e887: 8 total, 8 up, 8 in 2024-09-21T09:45:51.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:50 smithi022 ceph-mon[96736]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-6","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:45:51.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:50 smithi022 ceph-mon[96736]: osdmap e887: 8 total, 8 up, 8 in 2024-09-21T09:45:51.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:51 smithi064 ceph-mon[86709]: pgmap v955: 137 pgs: 22 unknown, 115 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:45:51.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:51 smithi064 ceph-mon[86709]: osdmap e888: 8 total, 8 up, 8 in 2024-09-21T09:45:52.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:51 smithi022 ceph-mon[98615]: pgmap v955: 137 pgs: 22 unknown, 115 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:45:52.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:51 smithi022 ceph-mon[98615]: osdmap e888: 8 total, 8 up, 8 in 2024-09-21T09:45:52.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:51 smithi022 ceph-mon[96736]: pgmap v955: 137 pgs: 22 unknown, 115 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:45:52.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:51 smithi022 ceph-mon[96736]: osdmap e888: 8 total, 8 up, 8 in 2024-09-21T09:45:52.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:52 smithi064 ceph-mon[86709]: osdmap e889: 8 total, 8 up, 8 in 2024-09-21T09:45:52.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:52 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/2420298693' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-7","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:45:53.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:52 smithi022 ceph-mon[96736]: osdmap e889: 8 total, 8 up, 8 in 2024-09-21T09:45:53.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:52 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/2420298693' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-7","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:45:53.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:52 smithi022 ceph-mon[98615]: osdmap e889: 8 total, 8 up, 8 in 2024-09-21T09:45:53.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:52 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/2420298693' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-7","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:45:53.692 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestCls2PCQueue.ReserveError (3018 ms) 2024-09-21T09:45:53.692 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestCls2PCQueue.CommitError 2024-09-21T09:45:53.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:53 smithi064 ceph-mon[86709]: pgmap v958: 137 pgs: 5 creating+peering, 27 unknown, 105 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:45:53.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:53 smithi064 ceph-mon[86709]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:45:53.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:53 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/2420298693' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-7","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:45:53.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:53 smithi064 ceph-mon[86709]: osdmap e890: 8 total, 8 up, 8 in 2024-09-21T09:45:54.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:53 smithi022 ceph-mon[98615]: pgmap v958: 137 pgs: 5 creating+peering, 27 unknown, 105 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:45:54.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:53 smithi022 ceph-mon[98615]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:45:54.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:53 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/2420298693' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-7","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:45:54.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:53 smithi022 ceph-mon[98615]: osdmap e890: 8 total, 8 up, 8 in 2024-09-21T09:45:54.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:53 smithi022 ceph-mon[96736]: pgmap v958: 137 pgs: 5 creating+peering, 27 unknown, 105 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:45:54.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:53 smithi022 ceph-mon[96736]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:45:54.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:53 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/2420298693' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-7","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:45:54.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:53 smithi022 ceph-mon[96736]: osdmap e890: 8 total, 8 up, 8 in 2024-09-21T09:45:55.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:54 smithi022 ceph-mon[98615]: osdmap e891: 8 total, 8 up, 8 in 2024-09-21T09:45:55.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:54 smithi022 ceph-mon[96736]: osdmap e891: 8 total, 8 up, 8 in 2024-09-21T09:45:55.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:54 smithi064 ceph-mon[86709]: osdmap e891: 8 total, 8 up, 8 in 2024-09-21T09:45:55.959 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:45:55 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:45:55] "GET /metrics HTTP/1.1" 200 36312 "" "Prometheus/2.51.0" 2024-09-21T09:45:55.960 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:55 smithi022 ceph-mon[98615]: pgmap v961: 105 pgs: 105 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:45:55.960 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:55 smithi022 ceph-mon[98615]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:45:55.960 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:55 smithi022 ceph-mon[98615]: Cluster is now healthy 2024-09-21T09:45:55.960 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:55 smithi022 ceph-mon[98615]: osdmap e892: 8 total, 8 up, 8 in 2024-09-21T09:45:55.960 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:55 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/1789590434' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-8","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:45:55.960 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:55 smithi022 ceph-mon[98615]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-8","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:45:55.960 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:55 smithi022 ceph-mon[96736]: pgmap v961: 105 pgs: 105 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:45:55.960 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:55 smithi022 ceph-mon[96736]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:45:55.961 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:55 smithi022 ceph-mon[96736]: Cluster is now healthy 2024-09-21T09:45:55.961 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:55 smithi022 ceph-mon[96736]: osdmap e892: 8 total, 8 up, 8 in 2024-09-21T09:45:55.961 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:55 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/1789590434' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-8","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:45:55.961 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:55 smithi022 ceph-mon[96736]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-8","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:45:56.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:55 smithi064 ceph-mon[86709]: pgmap v961: 105 pgs: 105 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:45:56.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:55 smithi064 ceph-mon[86709]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:45:56.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:55 smithi064 ceph-mon[86709]: Cluster is now healthy 2024-09-21T09:45:56.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:55 smithi064 ceph-mon[86709]: osdmap e892: 8 total, 8 up, 8 in 2024-09-21T09:45:56.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:55 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/1789590434' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-8","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:45:56.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:55 smithi064 ceph-mon[86709]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-8","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:45:56.714 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestCls2PCQueue.CommitError (3022 ms) 2024-09-21T09:45:56.714 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestCls2PCQueue.AbortError 2024-09-21T09:45:56.975 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:56 smithi022 ceph-mon[98615]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-8","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:45:56.975 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:56 smithi022 ceph-mon[98615]: osdmap e893: 8 total, 8 up, 8 in 2024-09-21T09:45:56.975 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:56 smithi022 ceph-mon[96736]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-8","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:45:56.975 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:56 smithi022 ceph-mon[96736]: osdmap e893: 8 total, 8 up, 8 in 2024-09-21T09:45:57.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:56 smithi064 ceph-mon[86709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-8","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:45:57.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:56 smithi064 ceph-mon[86709]: osdmap e893: 8 total, 8 up, 8 in 2024-09-21T09:45:58.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:57 smithi022 ceph-mon[98615]: pgmap v964: 137 pgs: 7 creating+peering, 15 unknown, 115 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:45:58.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:57 smithi022 ceph-mon[98615]: osdmap e894: 8 total, 8 up, 8 in 2024-09-21T09:45:58.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:57 smithi022 ceph-mon[96736]: pgmap v964: 137 pgs: 7 creating+peering, 15 unknown, 115 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:45:58.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:57 smithi022 ceph-mon[96736]: osdmap e894: 8 total, 8 up, 8 in 2024-09-21T09:45:58.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:57 smithi064 ceph-mon[86709]: pgmap v964: 137 pgs: 7 creating+peering, 15 unknown, 115 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:45:58.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:57 smithi064 ceph-mon[86709]: osdmap e894: 8 total, 8 up, 8 in 2024-09-21T09:45:59.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:58 smithi022 ceph-mon[96736]: osdmap e895: 8 total, 8 up, 8 in 2024-09-21T09:45:59.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:58 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/1891123548' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-9","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:45:59.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:58 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:45:59.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:58 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/1891123548' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-9","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:45:59.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:58 smithi022 ceph-mon[96736]: osdmap e896: 8 total, 8 up, 8 in 2024-09-21T09:45:59.059 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:58 smithi022 ceph-mon[98615]: osdmap e895: 8 total, 8 up, 8 in 2024-09-21T09:45:59.059 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:58 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/1891123548' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-9","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:45:59.059 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:58 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:45:59.059 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:58 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/1891123548' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-9","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:45:59.059 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:58 smithi022 ceph-mon[98615]: osdmap e896: 8 total, 8 up, 8 in 2024-09-21T09:45:59.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:58 smithi064 ceph-mon[86709]: osdmap e895: 8 total, 8 up, 8 in 2024-09-21T09:45:59.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:58 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/1891123548' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-9","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:45:59.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:58 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:45:59.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:58 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/1891123548' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-9","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:45:59.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:58 smithi064 ceph-mon[86709]: osdmap e896: 8 total, 8 up, 8 in 2024-09-21T09:45:59.732 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestCls2PCQueue.AbortError (3018 ms) 2024-09-21T09:45:59.732 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestCls2PCQueue.MultiReserve 2024-09-21T09:46:00.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:59 smithi022 ceph-mon[98615]: pgmap v967: 137 pgs: 15 creating+peering, 17 unknown, 105 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:46:00.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:59 smithi022 ceph-mon[98615]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:46:00.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:45:59 smithi022 ceph-mon[98615]: osdmap e897: 8 total, 8 up, 8 in 2024-09-21T09:46:00.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:59 smithi022 ceph-mon[96736]: pgmap v967: 137 pgs: 15 creating+peering, 17 unknown, 105 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:46:00.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:59 smithi022 ceph-mon[96736]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:46:00.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:45:59 smithi022 ceph-mon[96736]: osdmap e897: 8 total, 8 up, 8 in 2024-09-21T09:46:00.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:59 smithi064 ceph-mon[86709]: pgmap v967: 137 pgs: 15 creating+peering, 17 unknown, 105 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:46:00.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:59 smithi064 ceph-mon[86709]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:46:00.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:45:59 smithi064 ceph-mon[86709]: osdmap e897: 8 total, 8 up, 8 in 2024-09-21T09:46:00.691 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:46:00 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:46:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:46:01.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:00 smithi022 ceph-mon[98615]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:46:01.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:00 smithi022 ceph-mon[98615]: Cluster is now healthy 2024-09-21T09:46:01.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:00 smithi022 ceph-mon[98615]: osdmap e898: 8 total, 8 up, 8 in 2024-09-21T09:46:01.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:00 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/1938839481' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-10","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:46:01.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:00 smithi022 ceph-mon[96736]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:46:01.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:00 smithi022 ceph-mon[96736]: Cluster is now healthy 2024-09-21T09:46:01.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:00 smithi022 ceph-mon[96736]: osdmap e898: 8 total, 8 up, 8 in 2024-09-21T09:46:01.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:00 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/1938839481' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-10","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:46:01.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:00 smithi064 ceph-mon[86709]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:46:01.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:00 smithi064 ceph-mon[86709]: Cluster is now healthy 2024-09-21T09:46:01.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:00 smithi064 ceph-mon[86709]: osdmap e898: 8 total, 8 up, 8 in 2024-09-21T09:46:01.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:00 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/1938839481' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-10","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:46:02.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:01 smithi022 ceph-mon[98615]: pgmap v970: 105 pgs: 105 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:46:02.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:01 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/1938839481' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-10","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:46:02.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:01 smithi022 ceph-mon[98615]: osdmap e899: 8 total, 8 up, 8 in 2024-09-21T09:46:02.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:01 smithi022 ceph-mon[96736]: pgmap v970: 105 pgs: 105 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:46:02.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:01 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/1938839481' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-10","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:46:02.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:01 smithi022 ceph-mon[96736]: osdmap e899: 8 total, 8 up, 8 in 2024-09-21T09:46:02.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:01 smithi064 ceph-mon[86709]: pgmap v970: 105 pgs: 105 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:46:02.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:01 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/1938839481' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-10","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:46:02.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:01 smithi064 ceph-mon[86709]: osdmap e899: 8 total, 8 up, 8 in 2024-09-21T09:46:02.750 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestCls2PCQueue.MultiReserve (3018 ms) 2024-09-21T09:46:02.750 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestCls2PCQueue.MultiCommit 2024-09-21T09:46:04.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:03 smithi022 ceph-mon[96736]: pgmap v973: 137 pgs: 7 creating+peering, 20 unknown, 110 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:46:04.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:03 smithi022 ceph-mon[96736]: osdmap e900: 8 total, 8 up, 8 in 2024-09-21T09:46:04.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:03 smithi022 ceph-mon[98615]: pgmap v973: 137 pgs: 7 creating+peering, 20 unknown, 110 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:46:04.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:03 smithi022 ceph-mon[98615]: osdmap e900: 8 total, 8 up, 8 in 2024-09-21T09:46:04.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:03 smithi064 ceph-mon[86709]: pgmap v973: 137 pgs: 7 creating+peering, 20 unknown, 110 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:46:04.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:03 smithi064 ceph-mon[86709]: osdmap e900: 8 total, 8 up, 8 in 2024-09-21T09:46:05.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:04 smithi022 ceph-mon[98615]: osdmap e901: 8 total, 8 up, 8 in 2024-09-21T09:46:05.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:04 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/2394677018' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-11","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:46:05.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:04 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:46:05.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:04 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:46:05.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:04 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:46:05.059 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:04 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:46:05.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:04 smithi022 ceph-mon[96736]: osdmap e901: 8 total, 8 up, 8 in 2024-09-21T09:46:05.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:04 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/2394677018' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-11","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:46:05.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:04 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:46:05.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:04 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:46:05.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:04 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:46:05.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:04 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:46:05.093 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:04 smithi064 ceph-mon[86709]: osdmap e901: 8 total, 8 up, 8 in 2024-09-21T09:46:05.093 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:04 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/2394677018' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-11","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:46:05.093 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:04 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:46:05.093 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:04 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:46:05.093 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:04 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:46:05.093 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:04 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:46:05.771 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestCls2PCQueue.MultiCommit (3021 ms) 2024-09-21T09:46:05.771 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestCls2PCQueue.MultiAbort 2024-09-21T09:46:06.058 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:46:05 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:46:05] "GET /metrics HTTP/1.1" 200 36371 "" "Prometheus/2.51.0" 2024-09-21T09:46:06.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:05 smithi022 ceph-mon[98615]: pgmap v976: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:46:06.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:05 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/2394677018' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-11","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:46:06.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:05 smithi022 ceph-mon[98615]: osdmap e902: 8 total, 8 up, 8 in 2024-09-21T09:46:06.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:05 smithi022 ceph-mon[98615]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:46:06.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:05 smithi022 ceph-mon[96736]: pgmap v976: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:46:06.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:05 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/2394677018' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-11","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:46:06.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:05 smithi022 ceph-mon[96736]: osdmap e902: 8 total, 8 up, 8 in 2024-09-21T09:46:06.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:05 smithi022 ceph-mon[96736]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:46:06.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:05 smithi064 ceph-mon[86709]: pgmap v976: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:46:06.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:05 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/2394677018' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-11","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:46:06.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:05 smithi064 ceph-mon[86709]: osdmap e902: 8 total, 8 up, 8 in 2024-09-21T09:46:06.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:05 smithi064 ceph-mon[86709]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:46:07.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:06 smithi022 ceph-mon[98615]: osdmap e903: 8 total, 8 up, 8 in 2024-09-21T09:46:07.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:06 smithi022 ceph-mon[96736]: osdmap e903: 8 total, 8 up, 8 in 2024-09-21T09:46:07.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:06 smithi064 ceph-mon[86709]: osdmap e903: 8 total, 8 up, 8 in 2024-09-21T09:46:08.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:07 smithi022 ceph-mon[96736]: pgmap v979: 105 pgs: 105 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:46:08.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:07 smithi022 ceph-mon[96736]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:46:08.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:07 smithi022 ceph-mon[96736]: Cluster is now healthy 2024-09-21T09:46:08.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:07 smithi022 ceph-mon[96736]: osdmap e904: 8 total, 8 up, 8 in 2024-09-21T09:46:08.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:07 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/4233348951' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-12","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:46:08.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:07 smithi022 ceph-mon[96736]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-12","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:46:08.059 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:07 smithi022 ceph-mon[98615]: pgmap v979: 105 pgs: 105 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:46:08.059 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:07 smithi022 ceph-mon[98615]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:46:08.059 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:07 smithi022 ceph-mon[98615]: Cluster is now healthy 2024-09-21T09:46:08.059 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:07 smithi022 ceph-mon[98615]: osdmap e904: 8 total, 8 up, 8 in 2024-09-21T09:46:08.059 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:07 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/4233348951' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-12","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:46:08.059 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:07 smithi022 ceph-mon[98615]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-12","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:46:08.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:07 smithi064 ceph-mon[86709]: pgmap v979: 105 pgs: 105 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:46:08.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:07 smithi064 ceph-mon[86709]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:46:08.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:07 smithi064 ceph-mon[86709]: Cluster is now healthy 2024-09-21T09:46:08.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:07 smithi064 ceph-mon[86709]: osdmap e904: 8 total, 8 up, 8 in 2024-09-21T09:46:08.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:07 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/4233348951' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-12","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:46:08.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:07 smithi064 ceph-mon[86709]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-12","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:46:08.789 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestCls2PCQueue.MultiAbort (3018 ms) 2024-09-21T09:46:08.790 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestCls2PCQueue.ReserveCommit 2024-09-21T09:46:09.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:08 smithi022 ceph-mon[98615]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-12","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:46:09.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:08 smithi022 ceph-mon[98615]: osdmap e905: 8 total, 8 up, 8 in 2024-09-21T09:46:09.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:08 smithi022 ceph-mon[96736]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-12","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:46:09.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:08 smithi022 ceph-mon[96736]: osdmap e905: 8 total, 8 up, 8 in 2024-09-21T09:46:09.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:08 smithi064 ceph-mon[86709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-12","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:46:09.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:08 smithi064 ceph-mon[86709]: osdmap e905: 8 total, 8 up, 8 in 2024-09-21T09:46:10.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:09 smithi022 ceph-mon[98615]: pgmap v982: 137 pgs: 3 creating+peering, 14 unknown, 120 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:46:10.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:09 smithi022 ceph-mon[98615]: osdmap e906: 8 total, 8 up, 8 in 2024-09-21T09:46:10.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:09 smithi022 ceph-mon[96736]: pgmap v982: 137 pgs: 3 creating+peering, 14 unknown, 120 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:46:10.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:09 smithi022 ceph-mon[96736]: osdmap e906: 8 total, 8 up, 8 in 2024-09-21T09:46:10.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:09 smithi064 ceph-mon[86709]: pgmap v982: 137 pgs: 3 creating+peering, 14 unknown, 120 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:46:10.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:09 smithi064 ceph-mon[86709]: osdmap e906: 8 total, 8 up, 8 in 2024-09-21T09:46:10.691 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:46:10 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:46:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:46:11.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:10 smithi022 ceph-mon[98615]: osdmap e907: 8 total, 8 up, 8 in 2024-09-21T09:46:11.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:10 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/2685402188' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-13","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:46:11.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:10 smithi022 ceph-mon[96736]: osdmap e907: 8 total, 8 up, 8 in 2024-09-21T09:46:11.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:10 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/2685402188' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-13","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:46:11.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:10 smithi064 ceph-mon[86709]: osdmap e907: 8 total, 8 up, 8 in 2024-09-21T09:46:11.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:10 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/2685402188' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-13","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:46:11.807 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestCls2PCQueue.ReserveCommit (3017 ms) 2024-09-21T09:46:11.807 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestCls2PCQueue.ReserveAbort 2024-09-21T09:46:12.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:11 smithi022 ceph-mon[98615]: pgmap v985: 137 pgs: 4 creating+peering, 28 unknown, 105 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:46:12.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:11 smithi022 ceph-mon[98615]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:46:12.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:11 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/2685402188' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-13","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:46:12.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:11 smithi022 ceph-mon[98615]: osdmap e908: 8 total, 8 up, 8 in 2024-09-21T09:46:12.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:11 smithi022 ceph-mon[96736]: pgmap v985: 137 pgs: 4 creating+peering, 28 unknown, 105 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:46:12.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:11 smithi022 ceph-mon[96736]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:46:12.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:11 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/2685402188' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-13","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:46:12.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:11 smithi022 ceph-mon[96736]: osdmap e908: 8 total, 8 up, 8 in 2024-09-21T09:46:12.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:11 smithi064 ceph-mon[86709]: pgmap v985: 137 pgs: 4 creating+peering, 28 unknown, 105 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:46:12.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:11 smithi064 ceph-mon[86709]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:46:12.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:11 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/2685402188' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-13","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:46:12.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:11 smithi064 ceph-mon[86709]: osdmap e908: 8 total, 8 up, 8 in 2024-09-21T09:46:13.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:12 smithi064 ceph-mon[86709]: osdmap e909: 8 total, 8 up, 8 in 2024-09-21T09:46:13.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:12 smithi022 ceph-mon[98615]: osdmap e909: 8 total, 8 up, 8 in 2024-09-21T09:46:13.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:12 smithi022 ceph-mon[96736]: osdmap e909: 8 total, 8 up, 8 in 2024-09-21T09:46:14.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:13 smithi064 ceph-mon[86709]: pgmap v988: 105 pgs: 105 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:46:14.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:13 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:46:14.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:13 smithi064 ceph-mon[86709]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:46:14.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:13 smithi064 ceph-mon[86709]: Cluster is now healthy 2024-09-21T09:46:14.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:13 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:46:14.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:13 smithi064 ceph-mon[86709]: osdmap e910: 8 total, 8 up, 8 in 2024-09-21T09:46:14.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:13 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/697297707' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-14","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:46:14.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:13 smithi022 ceph-mon[96736]: pgmap v988: 105 pgs: 105 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:46:14.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:13 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:46:14.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:13 smithi022 ceph-mon[96736]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:46:14.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:13 smithi022 ceph-mon[96736]: Cluster is now healthy 2024-09-21T09:46:14.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:13 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:46:14.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:13 smithi022 ceph-mon[96736]: osdmap e910: 8 total, 8 up, 8 in 2024-09-21T09:46:14.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:13 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/697297707' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-14","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:46:14.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:13 smithi022 ceph-mon[98615]: pgmap v988: 105 pgs: 105 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:46:14.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:13 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:46:14.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:13 smithi022 ceph-mon[98615]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:46:14.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:13 smithi022 ceph-mon[98615]: Cluster is now healthy 2024-09-21T09:46:14.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:13 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:46:14.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:13 smithi022 ceph-mon[98615]: osdmap e910: 8 total, 8 up, 8 in 2024-09-21T09:46:14.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:13 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/697297707' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-14","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:46:14.832 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestCls2PCQueue.ReserveAbort (3025 ms) 2024-09-21T09:46:14.832 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestCls2PCQueue.ManualCleanup 2024-09-21T09:46:15.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:14 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/697297707' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-14","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:46:15.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:14 smithi064 ceph-mon[86709]: osdmap e911: 8 total, 8 up, 8 in 2024-09-21T09:46:15.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:14 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/697297707' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-14","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:46:15.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:14 smithi022 ceph-mon[98615]: osdmap e911: 8 total, 8 up, 8 in 2024-09-21T09:46:15.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:14 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/697297707' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-14","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:46:15.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:14 smithi022 ceph-mon[96736]: osdmap e911: 8 total, 8 up, 8 in 2024-09-21T09:46:16.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:15 smithi022 ceph-mon[98615]: pgmap v991: 137 pgs: 7 creating+peering, 25 unknown, 105 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:46:16.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:15 smithi022 ceph-mon[98615]: osdmap e912: 8 total, 8 up, 8 in 2024-09-21T09:46:16.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:15 smithi022 ceph-mon[96736]: pgmap v991: 137 pgs: 7 creating+peering, 25 unknown, 105 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:46:16.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:15 smithi022 ceph-mon[96736]: osdmap e912: 8 total, 8 up, 8 in 2024-09-21T09:46:16.059 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:46:15 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:46:15] "GET /metrics HTTP/1.1" 200 35552 "" "Prometheus/2.51.0" 2024-09-21T09:46:16.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:15 smithi064 ceph-mon[86709]: pgmap v991: 137 pgs: 7 creating+peering, 25 unknown, 105 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:46:16.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:15 smithi064 ceph-mon[86709]: osdmap e912: 8 total, 8 up, 8 in 2024-09-21T09:46:17.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:16 smithi064 ceph-mon[86709]: osdmap e913: 8 total, 8 up, 8 in 2024-09-21T09:46:17.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:16 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/3131819767' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-15","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:46:17.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:16 smithi064 ceph-mon[86709]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-15","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:46:17.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:16 smithi022 ceph-mon[98615]: osdmap e913: 8 total, 8 up, 8 in 2024-09-21T09:46:17.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:16 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/3131819767' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-15","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:46:17.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:16 smithi022 ceph-mon[98615]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-15","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:46:17.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:16 smithi022 ceph-mon[96736]: osdmap e913: 8 total, 8 up, 8 in 2024-09-21T09:46:17.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:16 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/3131819767' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-15","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:46:17.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:16 smithi022 ceph-mon[96736]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-15","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:46:17.848 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestCls2PCQueue.ManualCleanup (3016 ms) 2024-09-21T09:46:17.848 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestCls2PCQueue.Cleanup 2024-09-21T09:46:18.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:17 smithi064 ceph-mon[86709]: pgmap v994: 137 pgs: 10 creating+peering, 22 unknown, 105 active+clean; 583 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:46:18.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:17 smithi064 ceph-mon[86709]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:46:18.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:17 smithi064 ceph-mon[86709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-15","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:46:18.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:17 smithi064 ceph-mon[86709]: osdmap e914: 8 total, 8 up, 8 in 2024-09-21T09:46:18.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:17 smithi022 ceph-mon[98615]: pgmap v994: 137 pgs: 10 creating+peering, 22 unknown, 105 active+clean; 583 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:46:18.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:17 smithi022 ceph-mon[98615]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:46:18.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:17 smithi022 ceph-mon[98615]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-15","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:46:18.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:17 smithi022 ceph-mon[98615]: osdmap e914: 8 total, 8 up, 8 in 2024-09-21T09:46:18.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:17 smithi022 ceph-mon[96736]: pgmap v994: 137 pgs: 10 creating+peering, 22 unknown, 105 active+clean; 583 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:46:18.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:17 smithi022 ceph-mon[96736]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:46:18.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:17 smithi022 ceph-mon[96736]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-15","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:46:18.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:17 smithi022 ceph-mon[96736]: osdmap e914: 8 total, 8 up, 8 in 2024-09-21T09:46:19.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:18 smithi064 ceph-mon[86709]: osdmap e915: 8 total, 8 up, 8 in 2024-09-21T09:46:19.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:18 smithi022 ceph-mon[98615]: osdmap e915: 8 total, 8 up, 8 in 2024-09-21T09:46:19.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:18 smithi022 ceph-mon[96736]: osdmap e915: 8 total, 8 up, 8 in 2024-09-21T09:46:20.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:19 smithi064 ceph-mon[86709]: pgmap v997: 105 pgs: 105 active+clean; 583 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:46:20.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:19 smithi064 ceph-mon[86709]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:46:20.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:19 smithi064 ceph-mon[86709]: Cluster is now healthy 2024-09-21T09:46:20.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:19 smithi064 ceph-mon[86709]: osdmap e916: 8 total, 8 up, 8 in 2024-09-21T09:46:20.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:19 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/481227195' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-16","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:46:20.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:19 smithi064 ceph-mon[86709]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-16","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:46:20.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:19 smithi022 ceph-mon[96736]: pgmap v997: 105 pgs: 105 active+clean; 583 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:46:20.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:19 smithi022 ceph-mon[96736]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:46:20.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:19 smithi022 ceph-mon[96736]: Cluster is now healthy 2024-09-21T09:46:20.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:19 smithi022 ceph-mon[96736]: osdmap e916: 8 total, 8 up, 8 in 2024-09-21T09:46:20.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:19 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/481227195' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-16","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:46:20.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:19 smithi022 ceph-mon[96736]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-16","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:46:20.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:19 smithi022 ceph-mon[98615]: pgmap v997: 105 pgs: 105 active+clean; 583 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:46:20.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:19 smithi022 ceph-mon[98615]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:46:20.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:19 smithi022 ceph-mon[98615]: Cluster is now healthy 2024-09-21T09:46:20.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:19 smithi022 ceph-mon[98615]: osdmap e916: 8 total, 8 up, 8 in 2024-09-21T09:46:20.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:19 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/481227195' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-16","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:46:20.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:19 smithi022 ceph-mon[98615]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-16","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:46:20.691 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:46:20 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:46:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:46:20.878 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestCls2PCQueue.Cleanup (3029 ms) 2024-09-21T09:46:20.878 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestCls2PCQueue.MultiProducer 2024-09-21T09:46:21.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:20 smithi064 ceph-mon[86709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-16","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:46:21.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:20 smithi064 ceph-mon[86709]: osdmap e917: 8 total, 8 up, 8 in 2024-09-21T09:46:21.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:20 smithi022 ceph-mon[98615]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-16","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:46:21.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:20 smithi022 ceph-mon[98615]: osdmap e917: 8 total, 8 up, 8 in 2024-09-21T09:46:21.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:20 smithi022 ceph-mon[96736]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-16","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:46:21.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:20 smithi022 ceph-mon[96736]: osdmap e917: 8 total, 8 up, 8 in 2024-09-21T09:46:22.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:21 smithi064 ceph-mon[86709]: pgmap v1000: 137 pgs: 25 unknown, 112 active+clean; 583 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:46:22.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:21 smithi064 ceph-mon[86709]: osdmap e918: 8 total, 8 up, 8 in 2024-09-21T09:46:22.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:21 smithi022 ceph-mon[98615]: pgmap v1000: 137 pgs: 25 unknown, 112 active+clean; 583 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:46:22.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:21 smithi022 ceph-mon[98615]: osdmap e918: 8 total, 8 up, 8 in 2024-09-21T09:46:22.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:21 smithi022 ceph-mon[96736]: pgmap v1000: 137 pgs: 25 unknown, 112 active+clean; 583 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:46:22.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:21 smithi022 ceph-mon[96736]: osdmap e918: 8 total, 8 up, 8 in 2024-09-21T09:46:23.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:22 smithi064 ceph-mon[86709]: osdmap e919: 8 total, 8 up, 8 in 2024-09-21T09:46:23.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:22 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/3475124396' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-17","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:46:23.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:22 smithi022 ceph-mon[98615]: osdmap e919: 8 total, 8 up, 8 in 2024-09-21T09:46:23.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:22 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/3475124396' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-17","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:46:23.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:22 smithi022 ceph-mon[96736]: osdmap e919: 8 total, 8 up, 8 in 2024-09-21T09:46:23.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:22 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/3475124396' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-17","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:46:24.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:23 smithi064 ceph-mon[86709]: pgmap v1003: 137 pgs: 6 creating+peering, 26 unknown, 105 active+clean; 583 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:46:24.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:23 smithi064 ceph-mon[86709]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:46:24.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:23 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/3475124396' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-17","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:46:24.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:23 smithi064 ceph-mon[86709]: osdmap e920: 8 total, 8 up, 8 in 2024-09-21T09:46:24.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:23 smithi022 ceph-mon[98615]: pgmap v1003: 137 pgs: 6 creating+peering, 26 unknown, 105 active+clean; 583 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:46:24.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:23 smithi022 ceph-mon[98615]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:46:24.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:23 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/3475124396' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-17","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:46:24.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:23 smithi022 ceph-mon[98615]: osdmap e920: 8 total, 8 up, 8 in 2024-09-21T09:46:24.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:23 smithi022 ceph-mon[96736]: pgmap v1003: 137 pgs: 6 creating+peering, 26 unknown, 105 active+clean; 583 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:46:24.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:23 smithi022 ceph-mon[96736]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:46:24.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:23 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/3475124396' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-17","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:46:24.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:23 smithi022 ceph-mon[96736]: osdmap e920: 8 total, 8 up, 8 in 2024-09-21T09:46:25.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:24 smithi064 ceph-mon[86709]: osdmap e921: 8 total, 8 up, 8 in 2024-09-21T09:46:25.209 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:24 smithi022 ceph-mon[98615]: osdmap e921: 8 total, 8 up, 8 in 2024-09-21T09:46:25.209 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:24 smithi022 ceph-mon[96736]: osdmap e921: 8 total, 8 up, 8 in 2024-09-21T09:46:26.058 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:46:25 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:46:25] "GET /metrics HTTP/1.1" 200 35552 "" "Prometheus/2.51.0" 2024-09-21T09:46:26.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:25 smithi022 ceph-mon[98615]: pgmap v1006: 137 pgs: 6 creating+peering, 18 unknown, 113 active+clean; 583 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:46:26.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:25 smithi022 ceph-mon[98615]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:46:26.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:25 smithi022 ceph-mon[98615]: Cluster is now healthy 2024-09-21T09:46:26.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:25 smithi022 ceph-mon[96736]: pgmap v1006: 137 pgs: 6 creating+peering, 18 unknown, 113 active+clean; 583 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:46:26.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:25 smithi022 ceph-mon[96736]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:46:26.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:25 smithi022 ceph-mon[96736]: Cluster is now healthy 2024-09-21T09:46:26.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:25 smithi064 ceph-mon[86709]: pgmap v1006: 137 pgs: 6 creating+peering, 18 unknown, 113 active+clean; 583 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:46:26.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:25 smithi064 ceph-mon[86709]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:46:26.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:25 smithi064 ceph-mon[86709]: Cluster is now healthy 2024-09-21T09:46:28.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:27 smithi064 ceph-mon[86709]: pgmap v1007: 137 pgs: 6 creating+peering, 131 active+clean; 1.4 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s rd, 6.4 MiB/s wr, 7.21k op/s 2024-09-21T09:46:28.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:27 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:46:28.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:27 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:46:28.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:27 smithi022 ceph-mon[98615]: pgmap v1007: 137 pgs: 6 creating+peering, 131 active+clean; 1.4 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s rd, 6.4 MiB/s wr, 7.21k op/s 2024-09-21T09:46:28.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:27 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:46:28.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:27 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:46:28.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:27 smithi022 ceph-mon[96736]: pgmap v1007: 137 pgs: 6 creating+peering, 131 active+clean; 1.4 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s rd, 6.4 MiB/s wr, 7.21k op/s 2024-09-21T09:46:28.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:27 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:46:28.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:27 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:46:30.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:29 smithi064 ceph-mon[86709]: pgmap v1008: 137 pgs: 137 active+clean; 1.4 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 5.4 MiB/s wr, 6.12k op/s 2024-09-21T09:46:30.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:29 smithi022 ceph-mon[98615]: pgmap v1008: 137 pgs: 137 active+clean; 1.4 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 5.4 MiB/s wr, 6.12k op/s 2024-09-21T09:46:30.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:29 smithi022 ceph-mon[96736]: pgmap v1008: 137 pgs: 137 active+clean; 1.4 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 5.4 MiB/s wr, 6.12k op/s 2024-09-21T09:46:30.691 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:46:30 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:46:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:46:32.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:31 smithi064 ceph-mon[86709]: pgmap v1009: 137 pgs: 137 active+clean; 1.4 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 4.5 MiB/s wr, 5.07k op/s 2024-09-21T09:46:32.307 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:31 smithi022 ceph-mon[98615]: pgmap v1009: 137 pgs: 137 active+clean; 1.4 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 4.5 MiB/s wr, 5.07k op/s 2024-09-21T09:46:32.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:31 smithi022 ceph-mon[96736]: pgmap v1009: 137 pgs: 137 active+clean; 1.4 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 4.5 MiB/s wr, 5.07k op/s 2024-09-21T09:46:34.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:33 smithi064 ceph-mon[86709]: pgmap v1010: 137 pgs: 137 active+clean; 2.3 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 58 MiB/s rd, 7.7 MiB/s wr, 8.64k op/s 2024-09-21T09:46:34.307 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:33 smithi022 ceph-mon[98615]: pgmap v1010: 137 pgs: 137 active+clean; 2.3 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 58 MiB/s rd, 7.7 MiB/s wr, 8.64k op/s 2024-09-21T09:46:34.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:33 smithi022 ceph-mon[96736]: pgmap v1010: 137 pgs: 137 active+clean; 2.3 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 58 MiB/s rd, 7.7 MiB/s wr, 8.64k op/s 2024-09-21T09:46:35.912 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestCls2PCQueue.MultiProducer (15035 ms) 2024-09-21T09:46:35.912 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestCls2PCQueue.AsyncConsumer 2024-09-21T09:46:36.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:35 smithi022 ceph-mon[98615]: pgmap v1011: 137 pgs: 137 active+clean; 2.3 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 53 MiB/s rd, 6.9 MiB/s wr, 7.83k op/s 2024-09-21T09:46:36.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:35 smithi022 ceph-mon[96736]: pgmap v1011: 137 pgs: 137 active+clean; 2.3 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 53 MiB/s rd, 6.9 MiB/s wr, 7.83k op/s 2024-09-21T09:46:36.058 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:46:35 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:46:35] "GET /metrics HTTP/1.1" 200 36504 "" "Prometheus/2.51.0" 2024-09-21T09:46:36.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:35 smithi064 ceph-mon[86709]: pgmap v1011: 137 pgs: 137 active+clean; 2.3 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 53 MiB/s rd, 6.9 MiB/s wr, 7.83k op/s 2024-09-21T09:46:37.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:36 smithi064 ceph-mon[86709]: osdmap e922: 8 total, 8 up, 8 in 2024-09-21T09:46:37.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:36 smithi022 ceph-mon[96736]: osdmap e922: 8 total, 8 up, 8 in 2024-09-21T09:46:37.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:36 smithi022 ceph-mon[98615]: osdmap e922: 8 total, 8 up, 8 in 2024-09-21T09:46:38.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:37 smithi064 ceph-mon[86709]: pgmap v1013: 105 pgs: 105 active+clean; 583 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:46:38.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:37 smithi064 ceph-mon[86709]: osdmap e923: 8 total, 8 up, 8 in 2024-09-21T09:46:38.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:37 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/1514439874' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-18","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:46:38.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:37 smithi022 ceph-mon[98615]: pgmap v1013: 105 pgs: 105 active+clean; 583 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:46:38.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:37 smithi022 ceph-mon[98615]: osdmap e923: 8 total, 8 up, 8 in 2024-09-21T09:46:38.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:37 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/1514439874' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-18","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:46:38.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:37 smithi022 ceph-mon[96736]: pgmap v1013: 105 pgs: 105 active+clean; 583 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:46:38.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:37 smithi022 ceph-mon[96736]: osdmap e923: 8 total, 8 up, 8 in 2024-09-21T09:46:38.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:37 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/1514439874' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-18","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:46:39.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:38 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/1514439874' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-18","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:46:39.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:38 smithi064 ceph-mon[86709]: osdmap e924: 8 total, 8 up, 8 in 2024-09-21T09:46:39.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:38 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/1514439874' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-18","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:46:39.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:38 smithi022 ceph-mon[98615]: osdmap e924: 8 total, 8 up, 8 in 2024-09-21T09:46:39.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:38 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/1514439874' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-18","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:46:39.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:38 smithi022 ceph-mon[96736]: osdmap e924: 8 total, 8 up, 8 in 2024-09-21T09:46:39.941 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestCls2PCQueue.AsyncConsumer (4029 ms) 2024-09-21T09:46:39.942 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestCls2PCQueue.MultiProducerConsumer 2024-09-21T09:46:40.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:39 smithi064 ceph-mon[86709]: pgmap v1016: 137 pgs: 5 creating+activating, 3 creating+peering, 10 unknown, 119 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:46:40.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:39 smithi064 ceph-mon[86709]: osdmap e925: 8 total, 8 up, 8 in 2024-09-21T09:46:40.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:39 smithi022 ceph-mon[98615]: pgmap v1016: 137 pgs: 5 creating+activating, 3 creating+peering, 10 unknown, 119 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:46:40.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:39 smithi022 ceph-mon[98615]: osdmap e925: 8 total, 8 up, 8 in 2024-09-21T09:46:40.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:39 smithi022 ceph-mon[96736]: pgmap v1016: 137 pgs: 5 creating+activating, 3 creating+peering, 10 unknown, 119 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:46:40.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:39 smithi022 ceph-mon[96736]: osdmap e925: 8 total, 8 up, 8 in 2024-09-21T09:46:40.692 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:46:40 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:46:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:46:41.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:40 smithi022 ceph-mon[98615]: osdmap e926: 8 total, 8 up, 8 in 2024-09-21T09:46:41.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:40 smithi022 ceph-mon[96736]: osdmap e926: 8 total, 8 up, 8 in 2024-09-21T09:46:41.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:40 smithi064 ceph-mon[86709]: osdmap e926: 8 total, 8 up, 8 in 2024-09-21T09:46:42.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:41 smithi022 ceph-mon[98615]: pgmap v1019: 105 pgs: 105 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:46:42.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:41 smithi022 ceph-mon[98615]: osdmap e927: 8 total, 8 up, 8 in 2024-09-21T09:46:42.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:41 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/2866668372' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-19","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:46:42.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:41 smithi022 ceph-mon[98615]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-19","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:46:42.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:41 smithi022 ceph-mon[96736]: pgmap v1019: 105 pgs: 105 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:46:42.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:41 smithi022 ceph-mon[96736]: osdmap e927: 8 total, 8 up, 8 in 2024-09-21T09:46:42.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:41 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/2866668372' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-19","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:46:42.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:41 smithi022 ceph-mon[96736]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-19","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:46:42.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:41 smithi064 ceph-mon[86709]: pgmap v1019: 105 pgs: 105 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:46:42.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:41 smithi064 ceph-mon[86709]: osdmap e927: 8 total, 8 up, 8 in 2024-09-21T09:46:42.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:41 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/2866668372' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-19","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:46:42.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:41 smithi064 ceph-mon[86709]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-19","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:46:43.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:42 smithi022 ceph-mon[98615]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-19","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:46:43.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:42 smithi022 ceph-mon[98615]: osdmap e928: 8 total, 8 up, 8 in 2024-09-21T09:46:43.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:42 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:46:43.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:42 smithi022 ceph-mon[96736]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-19","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:46:43.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:42 smithi022 ceph-mon[96736]: osdmap e928: 8 total, 8 up, 8 in 2024-09-21T09:46:43.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:42 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:46:43.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:42 smithi064 ceph-mon[86709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-19","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:46:43.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:42 smithi064 ceph-mon[86709]: osdmap e928: 8 total, 8 up, 8 in 2024-09-21T09:46:43.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:42 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:46:44.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:43 smithi022 ceph-mon[96736]: pgmap v1022: 137 pgs: 11 creating+peering, 19 unknown, 107 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:46:44.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:43 smithi022 ceph-mon[96736]: osdmap e929: 8 total, 8 up, 8 in 2024-09-21T09:46:44.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:43 smithi022 ceph-mon[98615]: pgmap v1022: 137 pgs: 11 creating+peering, 19 unknown, 107 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:46:44.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:43 smithi022 ceph-mon[98615]: osdmap e929: 8 total, 8 up, 8 in 2024-09-21T09:46:44.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:43 smithi064 ceph-mon[86709]: pgmap v1022: 137 pgs: 11 creating+peering, 19 unknown, 107 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:46:44.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:43 smithi064 ceph-mon[86709]: osdmap e929: 8 total, 8 up, 8 in 2024-09-21T09:46:46.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:45 smithi022 ceph-mon[96736]: pgmap v1024: 137 pgs: 11 creating+peering, 8 unknown, 118 active+clean; 892 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.8 MiB/s rd, 2.7 MiB/s wr, 3.20k op/s 2024-09-21T09:46:46.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:45 smithi022 ceph-mon[98615]: pgmap v1024: 137 pgs: 11 creating+peering, 8 unknown, 118 active+clean; 892 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.8 MiB/s rd, 2.7 MiB/s wr, 3.20k op/s 2024-09-21T09:46:46.059 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:46:45 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:46:45] "GET /metrics HTTP/1.1" 200 36447 "" "Prometheus/2.51.0" 2024-09-21T09:46:46.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:45 smithi064 ceph-mon[86709]: pgmap v1024: 137 pgs: 11 creating+peering, 8 unknown, 118 active+clean; 892 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.8 MiB/s rd, 2.7 MiB/s wr, 3.20k op/s 2024-09-21T09:46:48.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:47 smithi022 ceph-mon[98615]: pgmap v1025: 137 pgs: 137 active+clean; 892 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 7.5 MiB/s rd, 2.1 MiB/s wr, 2.44k op/s 2024-09-21T09:46:48.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:47 smithi022 ceph-mon[96736]: pgmap v1025: 137 pgs: 137 active+clean; 892 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 7.5 MiB/s rd, 2.1 MiB/s wr, 2.44k op/s 2024-09-21T09:46:48.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:47 smithi064 ceph-mon[86709]: pgmap v1025: 137 pgs: 137 active+clean; 892 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 7.5 MiB/s rd, 2.1 MiB/s wr, 2.44k op/s 2024-09-21T09:46:50.214 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:49 smithi064 ceph-mon[86709]: pgmap v1026: 137 pgs: 137 active+clean; 1.6 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 70 MiB/s rd, 6.4 MiB/s wr, 7.56k op/s 2024-09-21T09:46:50.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:49 smithi022 ceph-mon[98615]: pgmap v1026: 137 pgs: 137 active+clean; 1.6 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 70 MiB/s rd, 6.4 MiB/s wr, 7.56k op/s 2024-09-21T09:46:50.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:49 smithi022 ceph-mon[96736]: pgmap v1026: 137 pgs: 137 active+clean; 1.6 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 70 MiB/s rd, 6.4 MiB/s wr, 7.56k op/s 2024-09-21T09:46:50.691 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:46:50 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:46:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:46:52.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:51 smithi022 ceph-mon[98615]: pgmap v1027: 137 pgs: 137 active+clean; 1.6 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 62 MiB/s rd, 5.6 MiB/s wr, 6.68k op/s 2024-09-21T09:46:52.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:51 smithi022 ceph-mon[96736]: pgmap v1027: 137 pgs: 137 active+clean; 1.6 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 62 MiB/s rd, 5.6 MiB/s wr, 6.68k op/s 2024-09-21T09:46:52.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:51 smithi064 ceph-mon[86709]: pgmap v1027: 137 pgs: 137 active+clean; 1.6 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 62 MiB/s rd, 5.6 MiB/s wr, 6.68k op/s 2024-09-21T09:46:54.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:53 smithi022 ceph-mon[98615]: pgmap v1028: 137 pgs: 137 active+clean; 1.6 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 53 MiB/s rd, 4.8 MiB/s wr, 5.71k op/s 2024-09-21T09:46:54.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:53 smithi022 ceph-mon[96736]: pgmap v1028: 137 pgs: 137 active+clean; 1.6 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 53 MiB/s rd, 4.8 MiB/s wr, 5.71k op/s 2024-09-21T09:46:54.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:53 smithi064 ceph-mon[86709]: pgmap v1028: 137 pgs: 137 active+clean; 1.6 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 53 MiB/s rd, 4.8 MiB/s wr, 5.71k op/s 2024-09-21T09:46:55.967 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:46:55 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:46:55] "GET /metrics HTTP/1.1" 200 36447 "" "Prometheus/2.51.0" 2024-09-21T09:46:56.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:55 smithi022 ceph-mon[98615]: pgmap v1029: 137 pgs: 137 active+clean; 1.6 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 71 MiB/s rd, 5.5 MiB/s wr, 6.63k op/s 2024-09-21T09:46:56.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:55 smithi022 ceph-mon[96736]: pgmap v1029: 137 pgs: 137 active+clean; 1.6 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 71 MiB/s rd, 5.5 MiB/s wr, 6.63k op/s 2024-09-21T09:46:56.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:55 smithi064 ceph-mon[86709]: pgmap v1029: 137 pgs: 137 active+clean; 1.6 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 71 MiB/s rd, 5.5 MiB/s wr, 6.63k op/s 2024-09-21T09:46:58.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:57 smithi022 ceph-mon[98615]: pgmap v1030: 137 pgs: 137 active+clean; 1.6 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 65 MiB/s rd, 4.2 MiB/s wr, 5.16k op/s 2024-09-21T09:46:58.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:57 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:46:58.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:57 smithi022 ceph-mon[96736]: pgmap v1030: 137 pgs: 137 active+clean; 1.6 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 65 MiB/s rd, 4.2 MiB/s wr, 5.16k op/s 2024-09-21T09:46:58.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:57 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:46:58.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:57 smithi064 ceph-mon[86709]: pgmap v1030: 137 pgs: 137 active+clean; 1.6 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 65 MiB/s rd, 4.2 MiB/s wr, 5.16k op/s 2024-09-21T09:46:58.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:57 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:46:58.985 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestCls2PCQueue.MultiProducerConsumer (19043 ms) 2024-09-21T09:46:58.985 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestCls2PCQueue.ReserveSpillover 2024-09-21T09:47:00.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:59 smithi022 ceph-mon[96736]: pgmap v1031: 137 pgs: 137 active+clean; 1.6 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 115 MiB/s rd, 5.3 MiB/s wr, 6.59k op/s 2024-09-21T09:47:00.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:46:59 smithi022 ceph-mon[96736]: osdmap e930: 8 total, 8 up, 8 in 2024-09-21T09:47:00.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:59 smithi022 ceph-mon[98615]: pgmap v1031: 137 pgs: 137 active+clean; 1.6 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 115 MiB/s rd, 5.3 MiB/s wr, 6.59k op/s 2024-09-21T09:47:00.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:46:59 smithi022 ceph-mon[98615]: osdmap e930: 8 total, 8 up, 8 in 2024-09-21T09:47:00.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:59 smithi064 ceph-mon[86709]: pgmap v1031: 137 pgs: 137 active+clean; 1.6 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 115 MiB/s rd, 5.3 MiB/s wr, 6.59k op/s 2024-09-21T09:47:00.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:46:59 smithi064 ceph-mon[86709]: osdmap e930: 8 total, 8 up, 8 in 2024-09-21T09:47:00.442 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:47:00 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:47:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:47:01.310 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:00 smithi022 ceph-mon[96736]: osdmap e931: 8 total, 8 up, 8 in 2024-09-21T09:47:01.311 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:00 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/1178354047' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-20","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:47:01.311 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:00 smithi022 ceph-mon[96736]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-20","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:47:01.311 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:00 smithi022 ceph-mon[98615]: osdmap e931: 8 total, 8 up, 8 in 2024-09-21T09:47:01.311 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:00 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/1178354047' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-20","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:47:01.311 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:00 smithi022 ceph-mon[98615]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-20","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:47:01.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:00 smithi064 ceph-mon[86709]: osdmap e931: 8 total, 8 up, 8 in 2024-09-21T09:47:01.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:00 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/1178354047' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-20","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:47:01.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:00 smithi064 ceph-mon[86709]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-20","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:47:02.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:01 smithi022 ceph-mon[98615]: pgmap v1034: 137 pgs: 6 creating+peering, 26 unknown, 105 active+clean; 583 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:47:02.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:01 smithi022 ceph-mon[98615]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:47:02.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:01 smithi022 ceph-mon[98615]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-20","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:47:02.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:01 smithi022 ceph-mon[98615]: osdmap e932: 8 total, 8 up, 8 in 2024-09-21T09:47:02.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:01 smithi022 ceph-mon[96736]: pgmap v1034: 137 pgs: 6 creating+peering, 26 unknown, 105 active+clean; 583 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:47:02.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:01 smithi022 ceph-mon[96736]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:47:02.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:01 smithi022 ceph-mon[96736]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-20","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:47:02.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:01 smithi022 ceph-mon[96736]: osdmap e932: 8 total, 8 up, 8 in 2024-09-21T09:47:02.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:01 smithi064 ceph-mon[86709]: pgmap v1034: 137 pgs: 6 creating+peering, 26 unknown, 105 active+clean; 583 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:47:02.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:01 smithi064 ceph-mon[86709]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:47:02.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:01 smithi064 ceph-mon[86709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-20","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:47:02.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:01 smithi064 ceph-mon[86709]: osdmap e932: 8 total, 8 up, 8 in 2024-09-21T09:47:03.010 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestCls2PCQueue.ReserveSpillover (4025 ms) 2024-09-21T09:47:03.010 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestCls2PCQueue.CommitSpillover 2024-09-21T09:47:03.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:03 smithi022 ceph-mon[98615]: osdmap e933: 8 total, 8 up, 8 in 2024-09-21T09:47:03.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:03 smithi022 ceph-mon[96736]: osdmap e933: 8 total, 8 up, 8 in 2024-09-21T09:47:03.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:03 smithi064 ceph-mon[86709]: osdmap e933: 8 total, 8 up, 8 in 2024-09-21T09:47:04.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:04 smithi022 ceph-mon[98615]: pgmap v1037: 137 pgs: 6 creating+peering, 17 unknown, 114 active+clean; 606 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:47:04.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:04 smithi022 ceph-mon[98615]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:47:04.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:04 smithi022 ceph-mon[98615]: Cluster is now healthy 2024-09-21T09:47:04.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:04 smithi022 ceph-mon[98615]: osdmap e934: 8 total, 8 up, 8 in 2024-09-21T09:47:04.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:04 smithi022 ceph-mon[96736]: pgmap v1037: 137 pgs: 6 creating+peering, 17 unknown, 114 active+clean; 606 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:47:04.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:04 smithi022 ceph-mon[96736]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:47:04.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:04 smithi022 ceph-mon[96736]: Cluster is now healthy 2024-09-21T09:47:04.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:04 smithi022 ceph-mon[96736]: osdmap e934: 8 total, 8 up, 8 in 2024-09-21T09:47:04.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:04 smithi064 ceph-mon[86709]: pgmap v1037: 137 pgs: 6 creating+peering, 17 unknown, 114 active+clean; 606 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:47:04.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:04 smithi064 ceph-mon[86709]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:47:04.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:04 smithi064 ceph-mon[86709]: Cluster is now healthy 2024-09-21T09:47:04.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:04 smithi064 ceph-mon[86709]: osdmap e934: 8 total, 8 up, 8 in 2024-09-21T09:47:05.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:05 smithi022 ceph-mon[98615]: osdmap e935: 8 total, 8 up, 8 in 2024-09-21T09:47:05.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:05 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/2764015778' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-21","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:47:05.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:05 smithi022 ceph-mon[98615]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-21","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:47:05.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:05 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:47:05.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:05 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:47:05.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:05 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:47:05.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:05 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:47:05.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:05 smithi022 ceph-mon[96736]: osdmap e935: 8 total, 8 up, 8 in 2024-09-21T09:47:05.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:05 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/2764015778' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-21","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:47:05.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:05 smithi022 ceph-mon[96736]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-21","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:47:05.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:05 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:47:05.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:05 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:47:05.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:05 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:47:05.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:05 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:47:05.342 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:05 smithi064 ceph-mon[86709]: osdmap e935: 8 total, 8 up, 8 in 2024-09-21T09:47:05.343 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:05 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/2764015778' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-21","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:47:05.343 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:05 smithi064 ceph-mon[86709]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-21","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:47:05.343 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:05 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:47:05.343 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:05 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:47:05.343 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:05 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:47:05.343 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:05 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:47:06.015 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:47:05 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:47:05] "GET /metrics HTTP/1.1" 200 36504 "" "Prometheus/2.51.0" 2024-09-21T09:47:06.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:06 smithi022 ceph-mon[96736]: pgmap v1040: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:47:06.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:06 smithi022 ceph-mon[96736]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:47:06.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:06 smithi022 ceph-mon[96736]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-21","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:47:06.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:06 smithi022 ceph-mon[96736]: osdmap e936: 8 total, 8 up, 8 in 2024-09-21T09:47:06.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:06 smithi022 ceph-mon[98615]: pgmap v1040: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:47:06.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:06 smithi022 ceph-mon[98615]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:47:06.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:06 smithi022 ceph-mon[98615]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-21","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:47:06.309 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:06 smithi022 ceph-mon[98615]: osdmap e936: 8 total, 8 up, 8 in 2024-09-21T09:47:06.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:06 smithi064 ceph-mon[86709]: pgmap v1040: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:47:06.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:06 smithi064 ceph-mon[86709]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:47:06.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:06 smithi064 ceph-mon[86709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-21","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:47:06.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:06 smithi064 ceph-mon[86709]: osdmap e936: 8 total, 8 up, 8 in 2024-09-21T09:47:07.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:07 smithi022 ceph-mon[98615]: osdmap e937: 8 total, 8 up, 8 in 2024-09-21T09:47:07.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:07 smithi022 ceph-mon[96736]: osdmap e937: 8 total, 8 up, 8 in 2024-09-21T09:47:07.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:07 smithi064 ceph-mon[86709]: osdmap e937: 8 total, 8 up, 8 in 2024-09-21T09:47:08.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:08 smithi022 ceph-mon[98615]: pgmap v1043: 137 pgs: 13 unknown, 124 active+clean; 583 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:47:08.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:08 smithi022 ceph-mon[98615]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:47:08.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:08 smithi022 ceph-mon[98615]: Cluster is now healthy 2024-09-21T09:47:08.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:08 smithi022 ceph-mon[96736]: pgmap v1043: 137 pgs: 13 unknown, 124 active+clean; 583 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:47:08.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:08 smithi022 ceph-mon[96736]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:47:08.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:08 smithi022 ceph-mon[96736]: Cluster is now healthy 2024-09-21T09:47:08.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:08 smithi064 ceph-mon[86709]: pgmap v1043: 137 pgs: 13 unknown, 124 active+clean; 583 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:47:08.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:08 smithi064 ceph-mon[86709]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:47:08.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:08 smithi064 ceph-mon[86709]: Cluster is now healthy 2024-09-21T09:47:09.041 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestCls2PCQueue.CommitSpillover (6031 ms) 2024-09-21T09:47:09.041 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestCls2PCQueue.AbortSpillover 2024-09-21T09:47:10.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:10 smithi022 ceph-mon[98615]: pgmap v1044: 137 pgs: 137 active+clean; 663 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.1 MiB/s rd, 5.4 MiB/s wr, 1.99k op/s 2024-09-21T09:47:10.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:10 smithi022 ceph-mon[98615]: osdmap e938: 8 total, 8 up, 8 in 2024-09-21T09:47:10.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:10 smithi022 ceph-mon[96736]: pgmap v1044: 137 pgs: 137 active+clean; 663 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.1 MiB/s rd, 5.4 MiB/s wr, 1.99k op/s 2024-09-21T09:47:10.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:10 smithi022 ceph-mon[96736]: osdmap e938: 8 total, 8 up, 8 in 2024-09-21T09:47:10.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:10 smithi064 ceph-mon[86709]: pgmap v1044: 137 pgs: 137 active+clean; 663 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.1 MiB/s rd, 5.4 MiB/s wr, 1.99k op/s 2024-09-21T09:47:10.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:10 smithi064 ceph-mon[86709]: osdmap e938: 8 total, 8 up, 8 in 2024-09-21T09:47:10.442 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:47:10 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:47:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:47:11.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:11 smithi022 ceph-mon[98615]: osdmap e939: 8 total, 8 up, 8 in 2024-09-21T09:47:11.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:11 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/1716877351' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-22","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:47:11.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:11 smithi022 ceph-mon[98615]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-22","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:47:11.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:11 smithi022 ceph-mon[96736]: osdmap e939: 8 total, 8 up, 8 in 2024-09-21T09:47:11.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:11 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/1716877351' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-22","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:47:11.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:11 smithi022 ceph-mon[96736]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-22","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:47:11.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:11 smithi064 ceph-mon[86709]: osdmap e939: 8 total, 8 up, 8 in 2024-09-21T09:47:11.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:11 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/1716877351' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-22","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:47:11.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:11 smithi064 ceph-mon[86709]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-22","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:47:12.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:12 smithi022 ceph-mon[96736]: pgmap v1047: 137 pgs: 7 creating+peering, 25 unknown, 105 active+clean; 583 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:47:12.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:12 smithi022 ceph-mon[96736]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:47:12.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:12 smithi022 ceph-mon[96736]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-22","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:47:12.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:12 smithi022 ceph-mon[96736]: osdmap e940: 8 total, 8 up, 8 in 2024-09-21T09:47:12.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:12 smithi022 ceph-mon[98615]: pgmap v1047: 137 pgs: 7 creating+peering, 25 unknown, 105 active+clean; 583 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:47:12.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:12 smithi022 ceph-mon[98615]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:47:12.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:12 smithi022 ceph-mon[98615]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-22","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:47:12.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:12 smithi022 ceph-mon[98615]: osdmap e940: 8 total, 8 up, 8 in 2024-09-21T09:47:12.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:12 smithi064 ceph-mon[86709]: pgmap v1047: 137 pgs: 7 creating+peering, 25 unknown, 105 active+clean; 583 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:47:12.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:12 smithi064 ceph-mon[86709]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:47:12.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:12 smithi064 ceph-mon[86709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-128626-22","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:47:12.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:12 smithi064 ceph-mon[86709]: osdmap e940: 8 total, 8 up, 8 in 2024-09-21T09:47:13.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:13 smithi022 ceph-mon[96736]: osdmap e941: 8 total, 8 up, 8 in 2024-09-21T09:47:13.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:13 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:47:13.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:13 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:47:13.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:13 smithi022 ceph-mon[98615]: osdmap e941: 8 total, 8 up, 8 in 2024-09-21T09:47:13.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:13 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:47:13.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:13 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:47:13.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:13 smithi064 ceph-mon[86709]: osdmap e941: 8 total, 8 up, 8 in 2024-09-21T09:47:13.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:13 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:47:13.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:13 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:47:14.066 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestCls2PCQueue.AbortSpillover (5025 ms) 2024-09-21T09:47:14.067 INFO:tasks.workunit.client.0.smithi022.stdout:[----------] 22 tests from TestCls2PCQueue (100893 ms total) 2024-09-21T09:47:14.067 INFO:tasks.workunit.client.0.smithi022.stdout: 2024-09-21T09:47:14.067 INFO:tasks.workunit.client.0.smithi022.stdout:[----------] Global test environment tear-down 2024-09-21T09:47:14.067 INFO:tasks.workunit.client.0.smithi022.stdout:[==========] 22 tests from 1 test suite ran. (100893 ms total) 2024-09-21T09:47:14.067 INFO:tasks.workunit.client.0.smithi022.stdout:[ PASSED ] 22 tests. 2024-09-21T09:47:14.070 INFO:teuthology.orchestra.run:Running command with timeout 3600 2024-09-21T09:47:14.070 DEBUG:teuthology.orchestra.run.smithi022:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2024-09-21T09:47:14.102 INFO:tasks.workunit:Running workunit cls/test_cls_cas.sh... 2024-09-21T09:47:14.102 DEBUG:teuthology.orchestra.run.smithi022:workunit test cls/test_cls_cas.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_cas.sh 2024-09-21T09:47:14.173 INFO:tasks.workunit.client.0.smithi022.stdout:Running main() from gmock_main.cc 2024-09-21T09:47:14.173 INFO:tasks.workunit.client.0.smithi022.stdout:[==========] Running 6 tests from 2 test suites. 2024-09-21T09:47:14.173 INFO:tasks.workunit.client.0.smithi022.stdout:[----------] Global test environment set-up. 2024-09-21T09:47:14.173 INFO:tasks.workunit.client.0.smithi022.stdout:[----------] 5 tests from cls_cas 2024-09-21T09:47:14.173 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] cls_cas.get_put 2024-09-21T09:47:14.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:14 smithi022 ceph-mon[98615]: pgmap v1050: 137 pgs: 7 creating+peering, 16 unknown, 114 active+clean; 583 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:47:14.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:14 smithi022 ceph-mon[98615]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:47:14.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:14 smithi022 ceph-mon[98615]: Cluster is now healthy 2024-09-21T09:47:14.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:14 smithi022 ceph-mon[96736]: pgmap v1050: 137 pgs: 7 creating+peering, 16 unknown, 114 active+clean; 583 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:47:14.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:14 smithi022 ceph-mon[96736]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:47:14.309 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:14 smithi022 ceph-mon[96736]: Cluster is now healthy 2024-09-21T09:47:14.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:14 smithi064 ceph-mon[86709]: pgmap v1050: 137 pgs: 7 creating+peering, 16 unknown, 114 active+clean; 583 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:47:14.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:14 smithi064 ceph-mon[86709]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:47:14.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:14 smithi064 ceph-mon[86709]: Cluster is now healthy 2024-09-21T09:47:15.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:15 smithi064 ceph-mon[86709]: osdmap e942: 8 total, 8 up, 8 in 2024-09-21T09:47:15.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:15 smithi022 ceph-mon[98615]: osdmap e942: 8 total, 8 up, 8 in 2024-09-21T09:47:15.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:15 smithi022 ceph-mon[96736]: osdmap e942: 8 total, 8 up, 8 in 2024-09-21T09:47:16.058 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:47:15 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:47:15] "GET /metrics HTTP/1.1" 200 36473 "" "Prometheus/2.51.0" 2024-09-21T09:47:16.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:16 smithi064 ceph-mon[86709]: pgmap v1052: 105 pgs: 105 active+clean; 583 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:47:16.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:16 smithi064 ceph-mon[86709]: osdmap e943: 8 total, 8 up, 8 in 2024-09-21T09:47:16.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:16 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/3779791965' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129314-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:47:16.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:16 smithi022 ceph-mon[98615]: pgmap v1052: 105 pgs: 105 active+clean; 583 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:47:16.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:16 smithi022 ceph-mon[98615]: osdmap e943: 8 total, 8 up, 8 in 2024-09-21T09:47:16.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:16 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/3779791965' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129314-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:47:16.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:16 smithi022 ceph-mon[96736]: pgmap v1052: 105 pgs: 105 active+clean; 583 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:47:16.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:16 smithi022 ceph-mon[96736]: osdmap e943: 8 total, 8 up, 8 in 2024-09-21T09:47:16.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:16 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/3779791965' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129314-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:47:17.089 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] cls_cas.get_put (2916 ms) 2024-09-21T09:47:17.089 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] cls_cas.wrong_put 2024-09-21T09:47:17.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:17 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/3779791965' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129314-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:47:17.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:17 smithi064 ceph-mon[86709]: osdmap e944: 8 total, 8 up, 8 in 2024-09-21T09:47:17.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:17 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/3779791965' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129314-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:47:17.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:17 smithi022 ceph-mon[98615]: osdmap e944: 8 total, 8 up, 8 in 2024-09-21T09:47:17.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:17 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/3779791965' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129314-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:47:17.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:17 smithi022 ceph-mon[96736]: osdmap e944: 8 total, 8 up, 8 in 2024-09-21T09:47:18.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:18 smithi064 ceph-mon[86709]: pgmap v1055: 137 pgs: 4 creating+activating, 8 creating+peering, 16 unknown, 109 active+clean; 583 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:47:18.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:18 smithi064 ceph-mon[86709]: osdmap e945: 8 total, 8 up, 8 in 2024-09-21T09:47:18.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:18 smithi022 ceph-mon[98615]: pgmap v1055: 137 pgs: 4 creating+activating, 8 creating+peering, 16 unknown, 109 active+clean; 583 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:47:18.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:18 smithi022 ceph-mon[98615]: osdmap e945: 8 total, 8 up, 8 in 2024-09-21T09:47:18.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:18 smithi022 ceph-mon[96736]: pgmap v1055: 137 pgs: 4 creating+activating, 8 creating+peering, 16 unknown, 109 active+clean; 583 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:47:18.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:18 smithi022 ceph-mon[96736]: osdmap e945: 8 total, 8 up, 8 in 2024-09-21T09:47:19.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:19 smithi064 ceph-mon[86709]: osdmap e946: 8 total, 8 up, 8 in 2024-09-21T09:47:19.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:19 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/4283146125' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129314-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:47:19.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:19 smithi064 ceph-mon[86709]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129314-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:47:19.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:19 smithi022 ceph-mon[98615]: osdmap e946: 8 total, 8 up, 8 in 2024-09-21T09:47:19.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:19 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/4283146125' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129314-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:47:19.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:19 smithi022 ceph-mon[98615]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129314-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:47:19.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:19 smithi022 ceph-mon[96736]: osdmap e946: 8 total, 8 up, 8 in 2024-09-21T09:47:19.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:19 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/4283146125' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129314-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:47:19.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:19 smithi022 ceph-mon[96736]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129314-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:47:20.110 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] cls_cas.wrong_put (3021 ms) 2024-09-21T09:47:20.110 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] cls_cas.dup_get 2024-09-21T09:47:20.441 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:47:20 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:47:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:47:20.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:20 smithi064 ceph-mon[86709]: pgmap v1058: 137 pgs: 5 creating+peering, 27 unknown, 105 active+clean; 583 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:47:20.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:20 smithi064 ceph-mon[86709]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:47:20.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:20 smithi064 ceph-mon[86709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129314-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:47:20.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:20 smithi064 ceph-mon[86709]: osdmap e947: 8 total, 8 up, 8 in 2024-09-21T09:47:20.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:20 smithi022 ceph-mon[98615]: pgmap v1058: 137 pgs: 5 creating+peering, 27 unknown, 105 active+clean; 583 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:47:20.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:20 smithi022 ceph-mon[98615]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:47:20.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:20 smithi022 ceph-mon[98615]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129314-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:47:20.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:20 smithi022 ceph-mon[98615]: osdmap e947: 8 total, 8 up, 8 in 2024-09-21T09:47:20.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:20 smithi022 ceph-mon[96736]: pgmap v1058: 137 pgs: 5 creating+peering, 27 unknown, 105 active+clean; 583 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:47:20.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:20 smithi022 ceph-mon[96736]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:47:20.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:20 smithi022 ceph-mon[96736]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129314-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:47:20.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:20 smithi022 ceph-mon[96736]: osdmap e947: 8 total, 8 up, 8 in 2024-09-21T09:47:21.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:21 smithi064 ceph-mon[86709]: osdmap e948: 8 total, 8 up, 8 in 2024-09-21T09:47:21.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:21 smithi022 ceph-mon[96736]: osdmap e948: 8 total, 8 up, 8 in 2024-09-21T09:47:21.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:21 smithi022 ceph-mon[98615]: osdmap e948: 8 total, 8 up, 8 in 2024-09-21T09:47:22.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:22 smithi064 ceph-mon[86709]: pgmap v1061: 105 pgs: 105 active+clean; 583 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:47:22.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:22 smithi064 ceph-mon[86709]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:47:22.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:22 smithi064 ceph-mon[86709]: Cluster is now healthy 2024-09-21T09:47:22.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:22 smithi064 ceph-mon[86709]: osdmap e949: 8 total, 8 up, 8 in 2024-09-21T09:47:22.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:22 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/2729004397' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129314-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:47:22.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:22 smithi022 ceph-mon[98615]: pgmap v1061: 105 pgs: 105 active+clean; 583 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:47:22.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:22 smithi022 ceph-mon[98615]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:47:22.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:22 smithi022 ceph-mon[98615]: Cluster is now healthy 2024-09-21T09:47:22.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:22 smithi022 ceph-mon[98615]: osdmap e949: 8 total, 8 up, 8 in 2024-09-21T09:47:22.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:22 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/2729004397' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129314-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:47:22.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:22 smithi022 ceph-mon[96736]: pgmap v1061: 105 pgs: 105 active+clean; 583 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:47:22.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:22 smithi022 ceph-mon[96736]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:47:22.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:22 smithi022 ceph-mon[96736]: Cluster is now healthy 2024-09-21T09:47:22.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:22 smithi022 ceph-mon[96736]: osdmap e949: 8 total, 8 up, 8 in 2024-09-21T09:47:22.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:22 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/2729004397' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129314-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:47:23.136 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] cls_cas.dup_get (3025 ms) 2024-09-21T09:47:23.136 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] cls_cas.dup_put 2024-09-21T09:47:23.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:23 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/2729004397' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129314-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:47:23.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:23 smithi064 ceph-mon[86709]: osdmap e950: 8 total, 8 up, 8 in 2024-09-21T09:47:23.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:23 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/2729004397' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129314-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:47:23.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:23 smithi022 ceph-mon[98615]: osdmap e950: 8 total, 8 up, 8 in 2024-09-21T09:47:23.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:23 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/2729004397' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129314-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:47:23.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:23 smithi022 ceph-mon[96736]: osdmap e950: 8 total, 8 up, 8 in 2024-09-21T09:47:24.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:24 smithi064 ceph-mon[86709]: pgmap v1064: 137 pgs: 3 creating+peering, 28 unknown, 106 active+clean; 583 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:47:24.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:24 smithi064 ceph-mon[86709]: osdmap e951: 8 total, 8 up, 8 in 2024-09-21T09:47:24.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:24 smithi022 ceph-mon[98615]: pgmap v1064: 137 pgs: 3 creating+peering, 28 unknown, 106 active+clean; 583 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:47:24.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:24 smithi022 ceph-mon[98615]: osdmap e951: 8 total, 8 up, 8 in 2024-09-21T09:47:24.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:24 smithi022 ceph-mon[96736]: pgmap v1064: 137 pgs: 3 creating+peering, 28 unknown, 106 active+clean; 583 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:47:24.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:24 smithi022 ceph-mon[96736]: osdmap e951: 8 total, 8 up, 8 in 2024-09-21T09:47:25.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:25 smithi064 ceph-mon[86709]: osdmap e952: 8 total, 8 up, 8 in 2024-09-21T09:47:25.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:25 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/3694072483' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129314-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:47:25.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:25 smithi064 ceph-mon[86709]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129314-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:47:25.459 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:25 smithi022 ceph-mon[98615]: osdmap e952: 8 total, 8 up, 8 in 2024-09-21T09:47:25.459 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:25 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/3694072483' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129314-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:47:25.459 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:25 smithi022 ceph-mon[98615]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129314-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:47:25.460 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:25 smithi022 ceph-mon[96736]: osdmap e952: 8 total, 8 up, 8 in 2024-09-21T09:47:25.460 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:25 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/3694072483' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129314-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:47:25.460 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:25 smithi022 ceph-mon[96736]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129314-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:47:26.058 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:47:25 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:47:25] "GET /metrics HTTP/1.1" 200 36473 "" "Prometheus/2.51.0" 2024-09-21T09:47:26.154 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] cls_cas.dup_put (3018 ms) 2024-09-21T09:47:26.154 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] cls_cas.get_wrong_data 2024-09-21T09:47:26.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:26 smithi064 ceph-mon[86709]: pgmap v1067: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:47:26.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:26 smithi064 ceph-mon[86709]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:47:26.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:26 smithi064 ceph-mon[86709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129314-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:47:26.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:26 smithi064 ceph-mon[86709]: osdmap e953: 8 total, 8 up, 8 in 2024-09-21T09:47:26.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:26 smithi022 ceph-mon[96736]: pgmap v1067: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:47:26.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:26 smithi022 ceph-mon[96736]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:47:26.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:26 smithi022 ceph-mon[96736]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129314-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:47:26.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:26 smithi022 ceph-mon[96736]: osdmap e953: 8 total, 8 up, 8 in 2024-09-21T09:47:26.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:26 smithi022 ceph-mon[98615]: pgmap v1067: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:47:26.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:26 smithi022 ceph-mon[98615]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:47:26.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:26 smithi022 ceph-mon[98615]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129314-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:47:26.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:26 smithi022 ceph-mon[98615]: osdmap e953: 8 total, 8 up, 8 in 2024-09-21T09:47:27.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:27 smithi064 ceph-mon[86709]: osdmap e954: 8 total, 8 up, 8 in 2024-09-21T09:47:27.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:27 smithi022 ceph-mon[98615]: osdmap e954: 8 total, 8 up, 8 in 2024-09-21T09:47:27.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:27 smithi022 ceph-mon[96736]: osdmap e954: 8 total, 8 up, 8 in 2024-09-21T09:47:28.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:28 smithi064 ceph-mon[86709]: pgmap v1070: 105 pgs: 105 active+clean; 583 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:47:28.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:28 smithi064 ceph-mon[86709]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:47:28.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:28 smithi064 ceph-mon[86709]: Cluster is now healthy 2024-09-21T09:47:28.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:28 smithi064 ceph-mon[86709]: osdmap e955: 8 total, 8 up, 8 in 2024-09-21T09:47:28.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:28 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/3460286327' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129314-5","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:47:28.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:28 smithi064 ceph-mon[86709]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129314-5","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:47:28.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:28 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:47:28.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:28 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:47:28.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:28 smithi022 ceph-mon[98615]: pgmap v1070: 105 pgs: 105 active+clean; 583 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:47:28.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:28 smithi022 ceph-mon[98615]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:47:28.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:28 smithi022 ceph-mon[98615]: Cluster is now healthy 2024-09-21T09:47:28.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:28 smithi022 ceph-mon[98615]: osdmap e955: 8 total, 8 up, 8 in 2024-09-21T09:47:28.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:28 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/3460286327' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129314-5","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:47:28.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:28 smithi022 ceph-mon[98615]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129314-5","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:47:28.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:28 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:47:28.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:28 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:47:28.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:28 smithi022 ceph-mon[96736]: pgmap v1070: 105 pgs: 105 active+clean; 583 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:47:28.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:28 smithi022 ceph-mon[96736]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:47:28.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:28 smithi022 ceph-mon[96736]: Cluster is now healthy 2024-09-21T09:47:28.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:28 smithi022 ceph-mon[96736]: osdmap e955: 8 total, 8 up, 8 in 2024-09-21T09:47:28.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:28 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/3460286327' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129314-5","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:47:28.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:28 smithi022 ceph-mon[96736]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129314-5","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:47:28.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:28 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:47:28.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:28 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:47:29.176 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] cls_cas.get_wrong_data (3022 ms) 2024-09-21T09:47:29.177 INFO:tasks.workunit.client.0.smithi022.stdout:[----------] 5 tests from cls_cas (15003 ms total) 2024-09-21T09:47:29.177 INFO:tasks.workunit.client.0.smithi022.stdout: 2024-09-21T09:47:29.177 INFO:tasks.workunit.client.0.smithi022.stdout:[----------] 1 test from chunk_refs_t 2024-09-21T09:47:29.177 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] chunk_refs_t.size 2024-09-21T09:47:29.177 INFO:tasks.workunit.client.0.smithi022.stdout:1 60 by_object 2024-09-21T09:47:29.177 INFO:tasks.workunit.client.0.smithi022.stdout:2 103 by_object 2024-09-21T09:47:29.177 INFO:tasks.workunit.client.0.smithi022.stdout:4 189 by_object 2024-09-21T09:47:29.177 INFO:tasks.workunit.client.0.smithi022.stdout:8 361 by_object 2024-09-21T09:47:29.177 INFO:tasks.workunit.client.0.smithi022.stdout:16 112 by_hash(32 bits) 2024-09-21T09:47:29.177 INFO:tasks.workunit.client.0.smithi022.stdout:32 208 by_hash(32 bits) 2024-09-21T09:47:29.177 INFO:tasks.workunit.client.0.smithi022.stdout:64 400 by_hash(32 bits) 2024-09-21T09:47:29.178 INFO:tasks.workunit.client.0.smithi022.stdout:128 509 by_hash(10 bits) 2024-09-21T09:47:29.178 INFO:tasks.workunit.client.0.smithi022.stdout:256 499 by_hash(7 bits) 2024-09-21T09:47:29.179 INFO:tasks.workunit.client.0.smithi022.stdout:512 462 by_hash(5 bits) 2024-09-21T09:47:29.179 INFO:tasks.workunit.client.0.smithi022.stdout:1024 466 by_hash(5 bits) 2024-09-21T09:47:29.179 INFO:tasks.workunit.client.0.smithi022.stdout:2048 466 by_hash(5 bits) 2024-09-21T09:47:29.179 INFO:tasks.workunit.client.0.smithi022.stdout:4096 470 by_hash(5 bits) 2024-09-21T09:47:29.179 INFO:tasks.workunit.client.0.smithi022.stdout:8192 498 by_hash(4 bits) 2024-09-21T09:47:29.180 INFO:tasks.workunit.client.0.smithi022.stdout:16384 278 by_hash(3 bits) 2024-09-21T09:47:29.182 INFO:tasks.workunit.client.0.smithi022.stdout:32768 337 by_hash(2 bits) 2024-09-21T09:47:29.186 INFO:tasks.workunit.client.0.smithi022.stdout:65536 367 by_hash(2 bits) 2024-09-21T09:47:29.194 INFO:tasks.workunit.client.0.smithi022.stdout:131072 387 by_hash(1 bits) 2024-09-21T09:47:29.209 INFO:tasks.workunit.client.0.smithi022.stdout:262144 331 by_pool 2024-09-21T09:47:29.239 INFO:tasks.workunit.client.0.smithi022.stdout:524288 21 count 2024-09-21T09:47:29.290 INFO:tasks.workunit.client.0.smithi022.stdout:1048576 21 count 2024-09-21T09:47:29.402 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] chunk_refs_t.size (226 ms) 2024-09-21T09:47:29.402 INFO:tasks.workunit.client.0.smithi022.stdout:[----------] 1 test from chunk_refs_t (226 ms total) 2024-09-21T09:47:29.402 INFO:tasks.workunit.client.0.smithi022.stdout: 2024-09-21T09:47:29.402 INFO:tasks.workunit.client.0.smithi022.stdout:[----------] Global test environment tear-down 2024-09-21T09:47:29.402 INFO:tasks.workunit.client.0.smithi022.stdout:[==========] 6 tests from 2 test suites ran. (15229 ms total) 2024-09-21T09:47:29.402 INFO:tasks.workunit.client.0.smithi022.stdout:[ PASSED ] 6 tests. 2024-09-21T09:47:29.404 INFO:teuthology.orchestra.run:Running command with timeout 3600 2024-09-21T09:47:29.404 DEBUG:teuthology.orchestra.run.smithi022:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2024-09-21T09:47:29.425 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:29 smithi022 ceph-mon[98615]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129314-5","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:47:29.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:29 smithi022 ceph-mon[98615]: osdmap e956: 8 total, 8 up, 8 in 2024-09-21T09:47:29.426 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:29 smithi022 ceph-mon[98615]: pgmap v1073: 137 pgs: 7 creating+peering, 16 unknown, 114 active+clean; 583 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:47:29.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:29 smithi022 ceph-mon[96736]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129314-5","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:47:29.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:29 smithi022 ceph-mon[96736]: osdmap e956: 8 total, 8 up, 8 in 2024-09-21T09:47:29.426 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:29 smithi022 ceph-mon[96736]: pgmap v1073: 137 pgs: 7 creating+peering, 16 unknown, 114 active+clean; 583 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:47:29.429 INFO:tasks.workunit:Running workunit cls/test_cls_cmpomap.sh... 2024-09-21T09:47:29.429 DEBUG:teuthology.orchestra.run.smithi022:workunit test cls/test_cls_cmpomap.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_cmpomap.sh 2024-09-21T09:47:29.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:29 smithi064 ceph-mon[86709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129314-5","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:47:29.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:29 smithi064 ceph-mon[86709]: osdmap e956: 8 total, 8 up, 8 in 2024-09-21T09:47:29.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:29 smithi064 ceph-mon[86709]: pgmap v1073: 137 pgs: 7 creating+peering, 16 unknown, 114 active+clean; 583 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:47:29.497 INFO:tasks.workunit.client.0.smithi022.stdout:Running main() from gmock_main.cc 2024-09-21T09:47:29.497 INFO:tasks.workunit.client.0.smithi022.stdout:[==========] Running 31 tests from 1 test suite. 2024-09-21T09:47:29.497 INFO:tasks.workunit.client.0.smithi022.stdout:[----------] Global test environment set-up. 2024-09-21T09:47:30.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:30 smithi064 ceph-mon[86709]: osdmap e957: 8 total, 8 up, 8 in 2024-09-21T09:47:30.442 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:47:30 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:47:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:47:30.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:30 smithi022 ceph-mon[98615]: osdmap e957: 8 total, 8 up, 8 in 2024-09-21T09:47:30.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:30 smithi022 ceph-mon[96736]: osdmap e957: 8 total, 8 up, 8 in 2024-09-21T09:47:31.186 INFO:tasks.workunit.client.0.smithi022.stdout:[----------] 31 tests from CmpOmap 2024-09-21T09:47:31.186 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] CmpOmap.cmp_vals_noexist_str 2024-09-21T09:47:31.202 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] CmpOmap.cmp_vals_noexist_str (16 ms) 2024-09-21T09:47:31.202 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] CmpOmap.cmp_vals_noexist_str_default 2024-09-21T09:47:31.208 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] CmpOmap.cmp_vals_noexist_str_default (6 ms) 2024-09-21T09:47:31.208 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] CmpOmap.cmp_vals_noexist_u64 2024-09-21T09:47:31.211 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] CmpOmap.cmp_vals_noexist_u64 (3 ms) 2024-09-21T09:47:31.211 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] CmpOmap.cmp_vals_noexist_u64_default 2024-09-21T09:47:31.215 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] CmpOmap.cmp_vals_noexist_u64_default (3 ms) 2024-09-21T09:47:31.215 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] CmpOmap.cmp_vals_str 2024-09-21T09:47:31.220 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] CmpOmap.cmp_vals_str (6 ms) 2024-09-21T09:47:31.220 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] CmpOmap.cmp_vals_u64 2024-09-21T09:47:31.227 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] CmpOmap.cmp_vals_u64 (7 ms) 2024-09-21T09:47:31.227 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] CmpOmap.cmp_vals_u64_invalid_input 2024-09-21T09:47:31.231 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] CmpOmap.cmp_vals_u64_invalid_input (4 ms) 2024-09-21T09:47:31.231 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] CmpOmap.cmp_vals_u64_invalid_default 2024-09-21T09:47:31.234 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] CmpOmap.cmp_vals_u64_invalid_default (3 ms) 2024-09-21T09:47:31.234 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] CmpOmap.cmp_vals_u64_empty_default 2024-09-21T09:47:31.236 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] CmpOmap.cmp_vals_u64_empty_default (2 ms) 2024-09-21T09:47:31.236 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] CmpOmap.cmp_vals_u64_invalid_value 2024-09-21T09:47:31.240 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] CmpOmap.cmp_vals_u64_invalid_value (4 ms) 2024-09-21T09:47:31.240 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] CmpOmap.cmp_vals_at_max_keys 2024-09-21T09:47:31.241 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] CmpOmap.cmp_vals_at_max_keys (1 ms) 2024-09-21T09:47:31.241 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] CmpOmap.cmp_vals_over_max_keys 2024-09-21T09:47:31.241 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] CmpOmap.cmp_vals_over_max_keys (1 ms) 2024-09-21T09:47:31.242 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] CmpOmap.cmp_set_vals_noexist_str 2024-09-21T09:47:31.247 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] CmpOmap.cmp_set_vals_noexist_str (6 ms) 2024-09-21T09:47:31.248 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] CmpOmap.cmp_set_vals_noexist_str_default 2024-09-21T09:47:31.254 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] CmpOmap.cmp_set_vals_noexist_str_default (6 ms) 2024-09-21T09:47:31.254 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] CmpOmap.cmp_set_vals_noexist_u64 2024-09-21T09:47:31.259 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] CmpOmap.cmp_set_vals_noexist_u64 (6 ms) 2024-09-21T09:47:31.260 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] CmpOmap.cmp_set_vals_noexist_u64_default 2024-09-21T09:47:31.267 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] CmpOmap.cmp_set_vals_noexist_u64_default (7 ms) 2024-09-21T09:47:31.267 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] CmpOmap.cmp_set_vals_str 2024-09-21T09:47:31.274 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] CmpOmap.cmp_set_vals_str (8 ms) 2024-09-21T09:47:31.275 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] CmpOmap.cmp_set_vals_u64 2024-09-21T09:47:31.282 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] CmpOmap.cmp_set_vals_u64 (7 ms) 2024-09-21T09:47:31.282 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] CmpOmap.cmp_set_vals_u64_einval 2024-09-21T09:47:31.284 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] CmpOmap.cmp_set_vals_u64_einval (2 ms) 2024-09-21T09:47:31.284 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] CmpOmap.cmp_set_vals_u64_eio 2024-09-21T09:47:31.286 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] CmpOmap.cmp_set_vals_u64_eio (2 ms) 2024-09-21T09:47:31.286 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] CmpOmap.cmp_set_vals_at_max_keys 2024-09-21T09:47:31.287 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] CmpOmap.cmp_set_vals_at_max_keys (1 ms) 2024-09-21T09:47:31.287 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] CmpOmap.cmp_set_vals_over_max_keys 2024-09-21T09:47:31.288 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] CmpOmap.cmp_set_vals_over_max_keys (1 ms) 2024-09-21T09:47:31.288 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] CmpOmap.cmp_rm_keys_noexist_str 2024-09-21T09:47:31.294 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] CmpOmap.cmp_rm_keys_noexist_str (7 ms) 2024-09-21T09:47:31.295 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] CmpOmap.cmp_rm_keys_noexist_u64 2024-09-21T09:47:31.300 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] CmpOmap.cmp_rm_keys_noexist_u64 (5 ms) 2024-09-21T09:47:31.300 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] CmpOmap.cmp_rm_keys_str 2024-09-21T09:47:31.307 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] CmpOmap.cmp_rm_keys_str (7 ms) 2024-09-21T09:47:31.307 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] CmpOmap.cmp_rm_keys_u64 2024-09-21T09:47:31.315 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] CmpOmap.cmp_rm_keys_u64 (8 ms) 2024-09-21T09:47:31.315 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] CmpOmap.cmp_rm_keys_u64_einval 2024-09-21T09:47:31.317 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] CmpOmap.cmp_rm_keys_u64_einval (2 ms) 2024-09-21T09:47:31.317 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] CmpOmap.cmp_rm_keys_u64_eio 2024-09-21T09:47:31.319 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] CmpOmap.cmp_rm_keys_u64_eio (2 ms) 2024-09-21T09:47:31.319 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] CmpOmap.cmp_rm_keys_at_max_keys 2024-09-21T09:47:31.320 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] CmpOmap.cmp_rm_keys_at_max_keys (1 ms) 2024-09-21T09:47:31.320 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] CmpOmap.cmp_rm_keys_over_max_keys 2024-09-21T09:47:31.320 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] CmpOmap.cmp_rm_keys_over_max_keys (1 ms) 2024-09-21T09:47:31.321 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] CmpOmap.cmp_rm_keys_u64_empty 2024-09-21T09:47:31.329 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] CmpOmap.cmp_rm_keys_u64_empty (8 ms) 2024-09-21T09:47:31.329 INFO:tasks.workunit.client.0.smithi022.stdout:[----------] 31 tests from CmpOmap (143 ms total) 2024-09-21T09:47:31.329 INFO:tasks.workunit.client.0.smithi022.stdout: 2024-09-21T09:47:31.329 INFO:tasks.workunit.client.0.smithi022.stdout:[----------] Global test environment tear-down 2024-09-21T09:47:31.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:31 smithi064 ceph-mon[86709]: osdmap e958: 8 total, 8 up, 8 in 2024-09-21T09:47:31.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:31 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/1406670895' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129437-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:47:31.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:31 smithi064 ceph-mon[86709]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129437-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:47:31.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:31 smithi064 ceph-mon[86709]: pgmap v1076: 137 pgs: 8 creating+peering, 24 unknown, 105 active+clean; 583 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:47:31.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:31 smithi022 ceph-mon[98615]: osdmap e958: 8 total, 8 up, 8 in 2024-09-21T09:47:31.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:31 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/1406670895' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129437-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:47:31.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:31 smithi022 ceph-mon[98615]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129437-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:47:31.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:31 smithi022 ceph-mon[98615]: pgmap v1076: 137 pgs: 8 creating+peering, 24 unknown, 105 active+clean; 583 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:47:31.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:31 smithi022 ceph-mon[96736]: osdmap e958: 8 total, 8 up, 8 in 2024-09-21T09:47:31.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:31 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/1406670895' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129437-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:47:31.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:31 smithi022 ceph-mon[96736]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129437-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:47:31.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:31 smithi022 ceph-mon[96736]: pgmap v1076: 137 pgs: 8 creating+peering, 24 unknown, 105 active+clean; 583 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:47:32.192 INFO:tasks.workunit.client.0.smithi022.stdout:[==========] 31 tests from 1 test suite ran. (2695 ms total) 2024-09-21T09:47:32.192 INFO:tasks.workunit.client.0.smithi022.stdout:[ PASSED ] 31 tests. 2024-09-21T09:47:32.195 INFO:teuthology.orchestra.run:Running command with timeout 3600 2024-09-21T09:47:32.196 DEBUG:teuthology.orchestra.run.smithi022:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2024-09-21T09:47:32.265 INFO:tasks.workunit:Running workunit cls/test_cls_hello.sh... 2024-09-21T09:47:32.265 DEBUG:teuthology.orchestra.run.smithi022:workunit test cls/test_cls_hello.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_hello.sh 2024-09-21T09:47:32.336 INFO:tasks.workunit.client.0.smithi022.stdout:Running main() from gmock_main.cc 2024-09-21T09:47:32.337 INFO:tasks.workunit.client.0.smithi022.stdout:[==========] Running 6 tests from 1 test suite. 2024-09-21T09:47:32.337 INFO:tasks.workunit.client.0.smithi022.stdout:[----------] Global test environment set-up. 2024-09-21T09:47:32.337 INFO:tasks.workunit.client.0.smithi022.stdout:[----------] 6 tests from ClsHello 2024-09-21T09:47:32.337 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] ClsHello.SayHello 2024-09-21T09:47:32.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:32 smithi064 ceph-mon[86709]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:47:32.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:32 smithi064 ceph-mon[86709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129437-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:47:32.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:32 smithi064 ceph-mon[86709]: osdmap e959: 8 total, 8 up, 8 in 2024-09-21T09:47:32.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:32 smithi022 ceph-mon[98615]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:47:32.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:32 smithi022 ceph-mon[98615]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129437-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:47:32.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:32 smithi022 ceph-mon[98615]: osdmap e959: 8 total, 8 up, 8 in 2024-09-21T09:47:32.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:32 smithi022 ceph-mon[96736]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:47:32.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:32 smithi022 ceph-mon[96736]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129437-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:47:32.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:32 smithi022 ceph-mon[96736]: osdmap e959: 8 total, 8 up, 8 in 2024-09-21T09:47:33.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:33 smithi022 ceph-mon[96736]: osdmap e960: 8 total, 8 up, 8 in 2024-09-21T09:47:33.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:33 smithi022 ceph-mon[96736]: pgmap v1079: 105 pgs: 105 active+clean; 583 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:47:33.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:33 smithi022 ceph-mon[98615]: osdmap e960: 8 total, 8 up, 8 in 2024-09-21T09:47:33.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:33 smithi022 ceph-mon[98615]: pgmap v1079: 105 pgs: 105 active+clean; 583 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:47:33.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:33 smithi064 ceph-mon[86709]: osdmap e960: 8 total, 8 up, 8 in 2024-09-21T09:47:33.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:33 smithi064 ceph-mon[86709]: pgmap v1079: 105 pgs: 105 active+clean; 583 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:47:34.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:34 smithi022 ceph-mon[96736]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:47:34.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:34 smithi022 ceph-mon[96736]: Cluster is now healthy 2024-09-21T09:47:34.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:34 smithi022 ceph-mon[96736]: osdmap e961: 8 total, 8 up, 8 in 2024-09-21T09:47:34.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:34 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/1363444566' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129492-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:47:34.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:34 smithi022 ceph-mon[96736]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129492-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:47:34.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:34 smithi022 ceph-mon[98615]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:47:34.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:34 smithi022 ceph-mon[98615]: Cluster is now healthy 2024-09-21T09:47:34.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:34 smithi022 ceph-mon[98615]: osdmap e961: 8 total, 8 up, 8 in 2024-09-21T09:47:34.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:34 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/1363444566' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129492-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:47:34.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:34 smithi022 ceph-mon[98615]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129492-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:47:34.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:34 smithi064 ceph-mon[86709]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:47:34.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:34 smithi064 ceph-mon[86709]: Cluster is now healthy 2024-09-21T09:47:34.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:34 smithi064 ceph-mon[86709]: osdmap e961: 8 total, 8 up, 8 in 2024-09-21T09:47:34.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:34 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/1363444566' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129492-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:47:34.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:34 smithi064 ceph-mon[86709]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129492-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:47:35.211 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] ClsHello.SayHello (2874 ms) 2024-09-21T09:47:35.211 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] ClsHello.RecordHello 2024-09-21T09:47:35.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:35 smithi022 ceph-mon[96736]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129492-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:47:35.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:35 smithi022 ceph-mon[96736]: osdmap e962: 8 total, 8 up, 8 in 2024-09-21T09:47:35.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:35 smithi022 ceph-mon[96736]: pgmap v1082: 137 pgs: 2 creating+peering, 30 unknown, 105 active+clean; 583 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:47:35.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:35 smithi022 ceph-mon[98615]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129492-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:47:35.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:35 smithi022 ceph-mon[98615]: osdmap e962: 8 total, 8 up, 8 in 2024-09-21T09:47:35.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:35 smithi022 ceph-mon[98615]: pgmap v1082: 137 pgs: 2 creating+peering, 30 unknown, 105 active+clean; 583 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:47:35.593 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:35 smithi064 ceph-mon[86709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129492-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:47:35.593 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:35 smithi064 ceph-mon[86709]: osdmap e962: 8 total, 8 up, 8 in 2024-09-21T09:47:35.593 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:35 smithi064 ceph-mon[86709]: pgmap v1082: 137 pgs: 2 creating+peering, 30 unknown, 105 active+clean; 583 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:47:36.058 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:47:35 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:47:35] "GET /metrics HTTP/1.1" 200 36441 "" "Prometheus/2.51.0" 2024-09-21T09:47:36.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:36 smithi022 ceph-mon[98615]: osdmap e963: 8 total, 8 up, 8 in 2024-09-21T09:47:36.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:36 smithi022 ceph-mon[96736]: osdmap e963: 8 total, 8 up, 8 in 2024-09-21T09:47:36.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:36 smithi064 ceph-mon[86709]: osdmap e963: 8 total, 8 up, 8 in 2024-09-21T09:47:37.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:37 smithi022 ceph-mon[98615]: osdmap e964: 8 total, 8 up, 8 in 2024-09-21T09:47:37.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:37 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/1256759865' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129492-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:47:37.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:37 smithi022 ceph-mon[98615]: pgmap v1085: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:47:37.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:37 smithi022 ceph-mon[96736]: osdmap e964: 8 total, 8 up, 8 in 2024-09-21T09:47:37.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:37 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/1256759865' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129492-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:47:37.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:37 smithi022 ceph-mon[96736]: pgmap v1085: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:47:37.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:37 smithi064 ceph-mon[86709]: osdmap e964: 8 total, 8 up, 8 in 2024-09-21T09:47:37.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:37 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/1256759865' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129492-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:47:37.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:37 smithi064 ceph-mon[86709]: pgmap v1085: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:47:38.226 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] ClsHello.RecordHello (3015 ms) 2024-09-21T09:47:38.226 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] ClsHello.WriteReturnData 2024-09-21T09:47:38.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:38 smithi022 ceph-mon[98615]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:47:38.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:38 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/1256759865' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129492-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:47:38.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:38 smithi022 ceph-mon[98615]: osdmap e965: 8 total, 8 up, 8 in 2024-09-21T09:47:38.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:38 smithi022 ceph-mon[96736]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:47:38.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:38 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/1256759865' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129492-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:47:38.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:38 smithi022 ceph-mon[96736]: osdmap e965: 8 total, 8 up, 8 in 2024-09-21T09:47:38.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:38 smithi064 ceph-mon[86709]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:47:38.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:38 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/1256759865' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129492-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:47:38.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:38 smithi064 ceph-mon[86709]: osdmap e965: 8 total, 8 up, 8 in 2024-09-21T09:47:39.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:39 smithi022 ceph-mon[98615]: osdmap e966: 8 total, 8 up, 8 in 2024-09-21T09:47:39.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:39 smithi022 ceph-mon[98615]: pgmap v1088: 105 pgs: 105 active+clean; 583 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:47:39.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:39 smithi022 ceph-mon[96736]: osdmap e966: 8 total, 8 up, 8 in 2024-09-21T09:47:39.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:39 smithi022 ceph-mon[96736]: pgmap v1088: 105 pgs: 105 active+clean; 583 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:47:39.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:39 smithi064 ceph-mon[86709]: osdmap e966: 8 total, 8 up, 8 in 2024-09-21T09:47:39.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:39 smithi064 ceph-mon[86709]: pgmap v1088: 105 pgs: 105 active+clean; 583 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:47:40.243 INFO:tasks.workunit.client.0.smithi022.stdout:require_osd_release = squid 2024-09-21T09:47:40.259 INFO:tasks.workunit.client.0.smithi022.stdout:0 2024-09-21T09:47:40.261 INFO:tasks.workunit.client.0.smithi022.stdout:1 2024-09-21T09:47:40.262 INFO:tasks.workunit.client.0.smithi022.stdout:2 2024-09-21T09:47:40.263 INFO:tasks.workunit.client.0.smithi022.stdout:3 2024-09-21T09:47:40.264 INFO:tasks.workunit.client.0.smithi022.stdout:4 2024-09-21T09:47:40.265 INFO:tasks.workunit.client.0.smithi022.stdout:5 2024-09-21T09:47:40.266 INFO:tasks.workunit.client.0.smithi022.stdout:6 2024-09-21T09:47:40.267 INFO:tasks.workunit.client.0.smithi022.stdout:7 2024-09-21T09:47:40.268 INFO:tasks.workunit.client.0.smithi022.stdout:8 2024-09-21T09:47:40.269 INFO:tasks.workunit.client.0.smithi022.stdout:9 2024-09-21T09:47:40.270 INFO:tasks.workunit.client.0.smithi022.stdout:10 2024-09-21T09:47:40.271 INFO:tasks.workunit.client.0.smithi022.stdout:11 2024-09-21T09:47:40.272 INFO:tasks.workunit.client.0.smithi022.stdout:12 2024-09-21T09:47:40.273 INFO:tasks.workunit.client.0.smithi022.stdout:13 2024-09-21T09:47:40.274 INFO:tasks.workunit.client.0.smithi022.stdout:14 2024-09-21T09:47:40.275 INFO:tasks.workunit.client.0.smithi022.stdout:15 2024-09-21T09:47:40.276 INFO:tasks.workunit.client.0.smithi022.stdout:16 2024-09-21T09:47:40.277 INFO:tasks.workunit.client.0.smithi022.stdout:17 2024-09-21T09:47:40.278 INFO:tasks.workunit.client.0.smithi022.stdout:18 2024-09-21T09:47:40.279 INFO:tasks.workunit.client.0.smithi022.stdout:19 2024-09-21T09:47:40.280 INFO:tasks.workunit.client.0.smithi022.stdout:20 2024-09-21T09:47:40.281 INFO:tasks.workunit.client.0.smithi022.stdout:21 2024-09-21T09:47:40.282 INFO:tasks.workunit.client.0.smithi022.stdout:22 2024-09-21T09:47:40.283 INFO:tasks.workunit.client.0.smithi022.stdout:23 2024-09-21T09:47:40.284 INFO:tasks.workunit.client.0.smithi022.stdout:24 2024-09-21T09:47:40.285 INFO:tasks.workunit.client.0.smithi022.stdout:25 2024-09-21T09:47:40.286 INFO:tasks.workunit.client.0.smithi022.stdout:26 2024-09-21T09:47:40.287 INFO:tasks.workunit.client.0.smithi022.stdout:27 2024-09-21T09:47:40.288 INFO:tasks.workunit.client.0.smithi022.stdout:28 2024-09-21T09:47:40.289 INFO:tasks.workunit.client.0.smithi022.stdout:29 2024-09-21T09:47:40.290 INFO:tasks.workunit.client.0.smithi022.stdout:30 2024-09-21T09:47:40.291 INFO:tasks.workunit.client.0.smithi022.stdout:31 2024-09-21T09:47:40.292 INFO:tasks.workunit.client.0.smithi022.stdout:32 2024-09-21T09:47:40.293 INFO:tasks.workunit.client.0.smithi022.stdout:33 2024-09-21T09:47:40.294 INFO:tasks.workunit.client.0.smithi022.stdout:34 2024-09-21T09:47:40.295 INFO:tasks.workunit.client.0.smithi022.stdout:35 2024-09-21T09:47:40.296 INFO:tasks.workunit.client.0.smithi022.stdout:36 2024-09-21T09:47:40.297 INFO:tasks.workunit.client.0.smithi022.stdout:37 2024-09-21T09:47:40.298 INFO:tasks.workunit.client.0.smithi022.stdout:38 2024-09-21T09:47:40.299 INFO:tasks.workunit.client.0.smithi022.stdout:39 2024-09-21T09:47:40.300 INFO:tasks.workunit.client.0.smithi022.stdout:40 2024-09-21T09:47:40.301 INFO:tasks.workunit.client.0.smithi022.stdout:41 2024-09-21T09:47:40.302 INFO:tasks.workunit.client.0.smithi022.stdout:42 2024-09-21T09:47:40.302 INFO:tasks.workunit.client.0.smithi022.stdout:43 2024-09-21T09:47:40.303 INFO:tasks.workunit.client.0.smithi022.stdout:44 2024-09-21T09:47:40.304 INFO:tasks.workunit.client.0.smithi022.stdout:45 2024-09-21T09:47:40.306 INFO:tasks.workunit.client.0.smithi022.stdout:46 2024-09-21T09:47:40.306 INFO:tasks.workunit.client.0.smithi022.stdout:47 2024-09-21T09:47:40.307 INFO:tasks.workunit.client.0.smithi022.stdout:48 2024-09-21T09:47:40.308 INFO:tasks.workunit.client.0.smithi022.stdout:49 2024-09-21T09:47:40.309 INFO:tasks.workunit.client.0.smithi022.stdout:50 2024-09-21T09:47:40.310 INFO:tasks.workunit.client.0.smithi022.stdout:51 2024-09-21T09:47:40.311 INFO:tasks.workunit.client.0.smithi022.stdout:52 2024-09-21T09:47:40.312 INFO:tasks.workunit.client.0.smithi022.stdout:53 2024-09-21T09:47:40.313 INFO:tasks.workunit.client.0.smithi022.stdout:54 2024-09-21T09:47:40.314 INFO:tasks.workunit.client.0.smithi022.stdout:55 2024-09-21T09:47:40.315 INFO:tasks.workunit.client.0.smithi022.stdout:56 2024-09-21T09:47:40.316 INFO:tasks.workunit.client.0.smithi022.stdout:57 2024-09-21T09:47:40.317 INFO:tasks.workunit.client.0.smithi022.stdout:58 2024-09-21T09:47:40.318 INFO:tasks.workunit.client.0.smithi022.stdout:59 2024-09-21T09:47:40.319 INFO:tasks.workunit.client.0.smithi022.stdout:60 2024-09-21T09:47:40.320 INFO:tasks.workunit.client.0.smithi022.stdout:61 2024-09-21T09:47:40.321 INFO:tasks.workunit.client.0.smithi022.stdout:62 2024-09-21T09:47:40.322 INFO:tasks.workunit.client.0.smithi022.stdout:63 2024-09-21T09:47:40.322 INFO:tasks.workunit.client.0.smithi022.stdout:64 2024-09-21T09:47:40.323 INFO:tasks.workunit.client.0.smithi022.stdout:65 2024-09-21T09:47:40.324 INFO:tasks.workunit.client.0.smithi022.stdout:66 2024-09-21T09:47:40.325 INFO:tasks.workunit.client.0.smithi022.stdout:67 2024-09-21T09:47:40.326 INFO:tasks.workunit.client.0.smithi022.stdout:68 2024-09-21T09:47:40.327 INFO:tasks.workunit.client.0.smithi022.stdout:69 2024-09-21T09:47:40.328 INFO:tasks.workunit.client.0.smithi022.stdout:70 2024-09-21T09:47:40.329 INFO:tasks.workunit.client.0.smithi022.stdout:71 2024-09-21T09:47:40.330 INFO:tasks.workunit.client.0.smithi022.stdout:72 2024-09-21T09:47:40.331 INFO:tasks.workunit.client.0.smithi022.stdout:73 2024-09-21T09:47:40.332 INFO:tasks.workunit.client.0.smithi022.stdout:74 2024-09-21T09:47:40.333 INFO:tasks.workunit.client.0.smithi022.stdout:75 2024-09-21T09:47:40.334 INFO:tasks.workunit.client.0.smithi022.stdout:76 2024-09-21T09:47:40.335 INFO:tasks.workunit.client.0.smithi022.stdout:77 2024-09-21T09:47:40.336 INFO:tasks.workunit.client.0.smithi022.stdout:78 2024-09-21T09:47:40.337 INFO:tasks.workunit.client.0.smithi022.stdout:79 2024-09-21T09:47:40.338 INFO:tasks.workunit.client.0.smithi022.stdout:80 2024-09-21T09:47:40.338 INFO:tasks.workunit.client.0.smithi022.stdout:81 2024-09-21T09:47:40.339 INFO:tasks.workunit.client.0.smithi022.stdout:82 2024-09-21T09:47:40.340 INFO:tasks.workunit.client.0.smithi022.stdout:83 2024-09-21T09:47:40.341 INFO:tasks.workunit.client.0.smithi022.stdout:84 2024-09-21T09:47:40.342 INFO:tasks.workunit.client.0.smithi022.stdout:85 2024-09-21T09:47:40.343 INFO:tasks.workunit.client.0.smithi022.stdout:86 2024-09-21T09:47:40.344 INFO:tasks.workunit.client.0.smithi022.stdout:87 2024-09-21T09:47:40.345 INFO:tasks.workunit.client.0.smithi022.stdout:88 2024-09-21T09:47:40.346 INFO:tasks.workunit.client.0.smithi022.stdout:89 2024-09-21T09:47:40.347 INFO:tasks.workunit.client.0.smithi022.stdout:90 2024-09-21T09:47:40.348 INFO:tasks.workunit.client.0.smithi022.stdout:91 2024-09-21T09:47:40.349 INFO:tasks.workunit.client.0.smithi022.stdout:92 2024-09-21T09:47:40.350 INFO:tasks.workunit.client.0.smithi022.stdout:93 2024-09-21T09:47:40.351 INFO:tasks.workunit.client.0.smithi022.stdout:94 2024-09-21T09:47:40.352 INFO:tasks.workunit.client.0.smithi022.stdout:95 2024-09-21T09:47:40.352 INFO:tasks.workunit.client.0.smithi022.stdout:96 2024-09-21T09:47:40.353 INFO:tasks.workunit.client.0.smithi022.stdout:97 2024-09-21T09:47:40.354 INFO:tasks.workunit.client.0.smithi022.stdout:98 2024-09-21T09:47:40.355 INFO:tasks.workunit.client.0.smithi022.stdout:99 2024-09-21T09:47:40.356 INFO:tasks.workunit.client.0.smithi022.stdout:100 2024-09-21T09:47:40.357 INFO:tasks.workunit.client.0.smithi022.stdout:101 2024-09-21T09:47:40.358 INFO:tasks.workunit.client.0.smithi022.stdout:102 2024-09-21T09:47:40.359 INFO:tasks.workunit.client.0.smithi022.stdout:103 2024-09-21T09:47:40.360 INFO:tasks.workunit.client.0.smithi022.stdout:104 2024-09-21T09:47:40.362 INFO:tasks.workunit.client.0.smithi022.stdout:105 2024-09-21T09:47:40.362 INFO:tasks.workunit.client.0.smithi022.stdout:106 2024-09-21T09:47:40.363 INFO:tasks.workunit.client.0.smithi022.stdout:107 2024-09-21T09:47:40.364 INFO:tasks.workunit.client.0.smithi022.stdout:108 2024-09-21T09:47:40.365 INFO:tasks.workunit.client.0.smithi022.stdout:109 2024-09-21T09:47:40.366 INFO:tasks.workunit.client.0.smithi022.stdout:110 2024-09-21T09:47:40.370 INFO:tasks.workunit.client.0.smithi022.stdout:111 2024-09-21T09:47:40.370 INFO:tasks.workunit.client.0.smithi022.stdout:112 2024-09-21T09:47:40.370 INFO:tasks.workunit.client.0.smithi022.stdout:113 2024-09-21T09:47:40.370 INFO:tasks.workunit.client.0.smithi022.stdout:114 2024-09-21T09:47:40.370 INFO:tasks.workunit.client.0.smithi022.stdout:115 2024-09-21T09:47:40.371 INFO:tasks.workunit.client.0.smithi022.stdout:116 2024-09-21T09:47:40.372 INFO:tasks.workunit.client.0.smithi022.stdout:117 2024-09-21T09:47:40.373 INFO:tasks.workunit.client.0.smithi022.stdout:118 2024-09-21T09:47:40.374 INFO:tasks.workunit.client.0.smithi022.stdout:119 2024-09-21T09:47:40.375 INFO:tasks.workunit.client.0.smithi022.stdout:120 2024-09-21T09:47:40.376 INFO:tasks.workunit.client.0.smithi022.stdout:121 2024-09-21T09:47:40.377 INFO:tasks.workunit.client.0.smithi022.stdout:122 2024-09-21T09:47:40.378 INFO:tasks.workunit.client.0.smithi022.stdout:123 2024-09-21T09:47:40.379 INFO:tasks.workunit.client.0.smithi022.stdout:124 2024-09-21T09:47:40.380 INFO:tasks.workunit.client.0.smithi022.stdout:125 2024-09-21T09:47:40.381 INFO:tasks.workunit.client.0.smithi022.stdout:126 2024-09-21T09:47:40.382 INFO:tasks.workunit.client.0.smithi022.stdout:127 2024-09-21T09:47:40.383 INFO:tasks.workunit.client.0.smithi022.stdout:128 2024-09-21T09:47:40.384 INFO:tasks.workunit.client.0.smithi022.stdout:129 2024-09-21T09:47:40.385 INFO:tasks.workunit.client.0.smithi022.stdout:130 2024-09-21T09:47:40.386 INFO:tasks.workunit.client.0.smithi022.stdout:131 2024-09-21T09:47:40.387 INFO:tasks.workunit.client.0.smithi022.stdout:132 2024-09-21T09:47:40.388 INFO:tasks.workunit.client.0.smithi022.stdout:133 2024-09-21T09:47:40.389 INFO:tasks.workunit.client.0.smithi022.stdout:134 2024-09-21T09:47:40.389 INFO:tasks.workunit.client.0.smithi022.stdout:135 2024-09-21T09:47:40.390 INFO:tasks.workunit.client.0.smithi022.stdout:136 2024-09-21T09:47:40.391 INFO:tasks.workunit.client.0.smithi022.stdout:137 2024-09-21T09:47:40.392 INFO:tasks.workunit.client.0.smithi022.stdout:138 2024-09-21T09:47:40.393 INFO:tasks.workunit.client.0.smithi022.stdout:139 2024-09-21T09:47:40.394 INFO:tasks.workunit.client.0.smithi022.stdout:140 2024-09-21T09:47:40.395 INFO:tasks.workunit.client.0.smithi022.stdout:141 2024-09-21T09:47:40.396 INFO:tasks.workunit.client.0.smithi022.stdout:142 2024-09-21T09:47:40.397 INFO:tasks.workunit.client.0.smithi022.stdout:143 2024-09-21T09:47:40.398 INFO:tasks.workunit.client.0.smithi022.stdout:144 2024-09-21T09:47:40.399 INFO:tasks.workunit.client.0.smithi022.stdout:145 2024-09-21T09:47:40.399 INFO:tasks.workunit.client.0.smithi022.stdout:146 2024-09-21T09:47:40.400 INFO:tasks.workunit.client.0.smithi022.stdout:147 2024-09-21T09:47:40.401 INFO:tasks.workunit.client.0.smithi022.stdout:148 2024-09-21T09:47:40.402 INFO:tasks.workunit.client.0.smithi022.stdout:149 2024-09-21T09:47:40.403 INFO:tasks.workunit.client.0.smithi022.stdout:150 2024-09-21T09:47:40.404 INFO:tasks.workunit.client.0.smithi022.stdout:151 2024-09-21T09:47:40.404 INFO:tasks.workunit.client.0.smithi022.stdout:152 2024-09-21T09:47:40.405 INFO:tasks.workunit.client.0.smithi022.stdout:153 2024-09-21T09:47:40.406 INFO:tasks.workunit.client.0.smithi022.stdout:154 2024-09-21T09:47:40.407 INFO:tasks.workunit.client.0.smithi022.stdout:155 2024-09-21T09:47:40.408 INFO:tasks.workunit.client.0.smithi022.stdout:156 2024-09-21T09:47:40.409 INFO:tasks.workunit.client.0.smithi022.stdout:157 2024-09-21T09:47:40.409 INFO:tasks.workunit.client.0.smithi022.stdout:158 2024-09-21T09:47:40.410 INFO:tasks.workunit.client.0.smithi022.stdout:159 2024-09-21T09:47:40.411 INFO:tasks.workunit.client.0.smithi022.stdout:160 2024-09-21T09:47:40.412 INFO:tasks.workunit.client.0.smithi022.stdout:161 2024-09-21T09:47:40.413 INFO:tasks.workunit.client.0.smithi022.stdout:162 2024-09-21T09:47:40.414 INFO:tasks.workunit.client.0.smithi022.stdout:163 2024-09-21T09:47:40.415 INFO:tasks.workunit.client.0.smithi022.stdout:164 2024-09-21T09:47:40.416 INFO:tasks.workunit.client.0.smithi022.stdout:165 2024-09-21T09:47:40.416 INFO:tasks.workunit.client.0.smithi022.stdout:166 2024-09-21T09:47:40.417 INFO:tasks.workunit.client.0.smithi022.stdout:167 2024-09-21T09:47:40.418 INFO:tasks.workunit.client.0.smithi022.stdout:168 2024-09-21T09:47:40.419 INFO:tasks.workunit.client.0.smithi022.stdout:169 2024-09-21T09:47:40.420 INFO:tasks.workunit.client.0.smithi022.stdout:170 2024-09-21T09:47:40.421 INFO:tasks.workunit.client.0.smithi022.stdout:171 2024-09-21T09:47:40.422 INFO:tasks.workunit.client.0.smithi022.stdout:172 2024-09-21T09:47:40.423 INFO:tasks.workunit.client.0.smithi022.stdout:173 2024-09-21T09:47:40.424 INFO:tasks.workunit.client.0.smithi022.stdout:174 2024-09-21T09:47:40.424 INFO:tasks.workunit.client.0.smithi022.stdout:175 2024-09-21T09:47:40.425 INFO:tasks.workunit.client.0.smithi022.stdout:176 2024-09-21T09:47:40.426 INFO:tasks.workunit.client.0.smithi022.stdout:177 2024-09-21T09:47:40.427 INFO:tasks.workunit.client.0.smithi022.stdout:178 2024-09-21T09:47:40.428 INFO:tasks.workunit.client.0.smithi022.stdout:179 2024-09-21T09:47:40.429 INFO:tasks.workunit.client.0.smithi022.stdout:180 2024-09-21T09:47:40.430 INFO:tasks.workunit.client.0.smithi022.stdout:181 2024-09-21T09:47:40.431 INFO:tasks.workunit.client.0.smithi022.stdout:182 2024-09-21T09:47:40.431 INFO:tasks.workunit.client.0.smithi022.stdout:183 2024-09-21T09:47:40.432 INFO:tasks.workunit.client.0.smithi022.stdout:184 2024-09-21T09:47:40.433 INFO:tasks.workunit.client.0.smithi022.stdout:185 2024-09-21T09:47:40.434 INFO:tasks.workunit.client.0.smithi022.stdout:186 2024-09-21T09:47:40.435 INFO:tasks.workunit.client.0.smithi022.stdout:187 2024-09-21T09:47:40.436 INFO:tasks.workunit.client.0.smithi022.stdout:188 2024-09-21T09:47:40.437 INFO:tasks.workunit.client.0.smithi022.stdout:189 2024-09-21T09:47:40.438 INFO:tasks.workunit.client.0.smithi022.stdout:190 2024-09-21T09:47:40.439 INFO:tasks.workunit.client.0.smithi022.stdout:191 2024-09-21T09:47:40.440 INFO:tasks.workunit.client.0.smithi022.stdout:192 2024-09-21T09:47:40.441 INFO:tasks.workunit.client.0.smithi022.stdout:193 2024-09-21T09:47:40.441 INFO:tasks.workunit.client.0.smithi022.stdout:194 2024-09-21T09:47:40.442 INFO:tasks.workunit.client.0.smithi022.stdout:195 2024-09-21T09:47:40.443 INFO:tasks.workunit.client.0.smithi022.stdout:196 2024-09-21T09:47:40.444 INFO:tasks.workunit.client.0.smithi022.stdout:197 2024-09-21T09:47:40.445 INFO:tasks.workunit.client.0.smithi022.stdout:198 2024-09-21T09:47:40.446 INFO:tasks.workunit.client.0.smithi022.stdout:199 2024-09-21T09:47:40.447 INFO:tasks.workunit.client.0.smithi022.stdout:200 2024-09-21T09:47:40.448 INFO:tasks.workunit.client.0.smithi022.stdout:201 2024-09-21T09:47:40.449 INFO:tasks.workunit.client.0.smithi022.stdout:202 2024-09-21T09:47:40.450 INFO:tasks.workunit.client.0.smithi022.stdout:203 2024-09-21T09:47:40.451 INFO:tasks.workunit.client.0.smithi022.stdout:204 2024-09-21T09:47:40.452 INFO:tasks.workunit.client.0.smithi022.stdout:205 2024-09-21T09:47:40.453 INFO:tasks.workunit.client.0.smithi022.stdout:206 2024-09-21T09:47:40.454 INFO:tasks.workunit.client.0.smithi022.stdout:207 2024-09-21T09:47:40.455 INFO:tasks.workunit.client.0.smithi022.stdout:208 2024-09-21T09:47:40.456 INFO:tasks.workunit.client.0.smithi022.stdout:209 2024-09-21T09:47:40.457 INFO:tasks.workunit.client.0.smithi022.stdout:210 2024-09-21T09:47:40.458 INFO:tasks.workunit.client.0.smithi022.stdout:211 2024-09-21T09:47:40.459 INFO:tasks.workunit.client.0.smithi022.stdout:212 2024-09-21T09:47:40.459 INFO:tasks.workunit.client.0.smithi022.stdout:213 2024-09-21T09:47:40.460 INFO:tasks.workunit.client.0.smithi022.stdout:214 2024-09-21T09:47:40.461 INFO:tasks.workunit.client.0.smithi022.stdout:215 2024-09-21T09:47:40.462 INFO:tasks.workunit.client.0.smithi022.stdout:216 2024-09-21T09:47:40.463 INFO:tasks.workunit.client.0.smithi022.stdout:217 2024-09-21T09:47:40.464 INFO:tasks.workunit.client.0.smithi022.stdout:218 2024-09-21T09:47:40.465 INFO:tasks.workunit.client.0.smithi022.stdout:219 2024-09-21T09:47:40.466 INFO:tasks.workunit.client.0.smithi022.stdout:220 2024-09-21T09:47:40.467 INFO:tasks.workunit.client.0.smithi022.stdout:221 2024-09-21T09:47:40.468 INFO:tasks.workunit.client.0.smithi022.stdout:222 2024-09-21T09:47:40.469 INFO:tasks.workunit.client.0.smithi022.stdout:223 2024-09-21T09:47:40.470 INFO:tasks.workunit.client.0.smithi022.stdout:224 2024-09-21T09:47:40.471 INFO:tasks.workunit.client.0.smithi022.stdout:225 2024-09-21T09:47:40.472 INFO:tasks.workunit.client.0.smithi022.stdout:226 2024-09-21T09:47:40.473 INFO:tasks.workunit.client.0.smithi022.stdout:227 2024-09-21T09:47:40.474 INFO:tasks.workunit.client.0.smithi022.stdout:228 2024-09-21T09:47:40.475 INFO:tasks.workunit.client.0.smithi022.stdout:229 2024-09-21T09:47:40.476 INFO:tasks.workunit.client.0.smithi022.stdout:230 2024-09-21T09:47:40.477 INFO:tasks.workunit.client.0.smithi022.stdout:231 2024-09-21T09:47:40.478 INFO:tasks.workunit.client.0.smithi022.stdout:232 2024-09-21T09:47:40.479 INFO:tasks.workunit.client.0.smithi022.stdout:233 2024-09-21T09:47:40.480 INFO:tasks.workunit.client.0.smithi022.stdout:234 2024-09-21T09:47:40.481 INFO:tasks.workunit.client.0.smithi022.stdout:235 2024-09-21T09:47:40.482 INFO:tasks.workunit.client.0.smithi022.stdout:236 2024-09-21T09:47:40.482 INFO:tasks.workunit.client.0.smithi022.stdout:237 2024-09-21T09:47:40.483 INFO:tasks.workunit.client.0.smithi022.stdout:238 2024-09-21T09:47:40.484 INFO:tasks.workunit.client.0.smithi022.stdout:239 2024-09-21T09:47:40.485 INFO:tasks.workunit.client.0.smithi022.stdout:240 2024-09-21T09:47:40.486 INFO:tasks.workunit.client.0.smithi022.stdout:241 2024-09-21T09:47:40.487 INFO:tasks.workunit.client.0.smithi022.stdout:242 2024-09-21T09:47:40.488 INFO:tasks.workunit.client.0.smithi022.stdout:243 2024-09-21T09:47:40.489 INFO:tasks.workunit.client.0.smithi022.stdout:244 2024-09-21T09:47:40.490 INFO:tasks.workunit.client.0.smithi022.stdout:245 2024-09-21T09:47:40.491 INFO:tasks.workunit.client.0.smithi022.stdout:246 2024-09-21T09:47:40.492 INFO:tasks.workunit.client.0.smithi022.stdout:247 2024-09-21T09:47:40.493 INFO:tasks.workunit.client.0.smithi022.stdout:248 2024-09-21T09:47:40.494 INFO:tasks.workunit.client.0.smithi022.stdout:249 2024-09-21T09:47:40.495 INFO:tasks.workunit.client.0.smithi022.stdout:250 2024-09-21T09:47:40.496 INFO:tasks.workunit.client.0.smithi022.stdout:251 2024-09-21T09:47:40.496 INFO:tasks.workunit.client.0.smithi022.stdout:252 2024-09-21T09:47:40.497 INFO:tasks.workunit.client.0.smithi022.stdout:253 2024-09-21T09:47:40.498 INFO:tasks.workunit.client.0.smithi022.stdout:254 2024-09-21T09:47:40.499 INFO:tasks.workunit.client.0.smithi022.stdout:255 2024-09-21T09:47:40.500 INFO:tasks.workunit.client.0.smithi022.stdout:256 2024-09-21T09:47:40.501 INFO:tasks.workunit.client.0.smithi022.stdout:257 2024-09-21T09:47:40.502 INFO:tasks.workunit.client.0.smithi022.stdout:258 2024-09-21T09:47:40.503 INFO:tasks.workunit.client.0.smithi022.stdout:259 2024-09-21T09:47:40.504 INFO:tasks.workunit.client.0.smithi022.stdout:260 2024-09-21T09:47:40.505 INFO:tasks.workunit.client.0.smithi022.stdout:261 2024-09-21T09:47:40.506 INFO:tasks.workunit.client.0.smithi022.stdout:262 2024-09-21T09:47:40.507 INFO:tasks.workunit.client.0.smithi022.stdout:263 2024-09-21T09:47:40.508 INFO:tasks.workunit.client.0.smithi022.stdout:264 2024-09-21T09:47:40.508 INFO:tasks.workunit.client.0.smithi022.stdout:265 2024-09-21T09:47:40.509 INFO:tasks.workunit.client.0.smithi022.stdout:266 2024-09-21T09:47:40.510 INFO:tasks.workunit.client.0.smithi022.stdout:267 2024-09-21T09:47:40.511 INFO:tasks.workunit.client.0.smithi022.stdout:268 2024-09-21T09:47:40.512 INFO:tasks.workunit.client.0.smithi022.stdout:269 2024-09-21T09:47:40.513 INFO:tasks.workunit.client.0.smithi022.stdout:270 2024-09-21T09:47:40.514 INFO:tasks.workunit.client.0.smithi022.stdout:271 2024-09-21T09:47:40.515 INFO:tasks.workunit.client.0.smithi022.stdout:272 2024-09-21T09:47:40.516 INFO:tasks.workunit.client.0.smithi022.stdout:273 2024-09-21T09:47:40.517 INFO:tasks.workunit.client.0.smithi022.stdout:274 2024-09-21T09:47:40.518 INFO:tasks.workunit.client.0.smithi022.stdout:275 2024-09-21T09:47:40.519 INFO:tasks.workunit.client.0.smithi022.stdout:276 2024-09-21T09:47:40.520 INFO:tasks.workunit.client.0.smithi022.stdout:277 2024-09-21T09:47:40.521 INFO:tasks.workunit.client.0.smithi022.stdout:278 2024-09-21T09:47:40.522 INFO:tasks.workunit.client.0.smithi022.stdout:279 2024-09-21T09:47:40.523 INFO:tasks.workunit.client.0.smithi022.stdout:280 2024-09-21T09:47:40.523 INFO:tasks.workunit.client.0.smithi022.stdout:281 2024-09-21T09:47:40.525 INFO:tasks.workunit.client.0.smithi022.stdout:282 2024-09-21T09:47:40.525 INFO:tasks.workunit.client.0.smithi022.stdout:283 2024-09-21T09:47:40.526 INFO:tasks.workunit.client.0.smithi022.stdout:284 2024-09-21T09:47:40.527 INFO:tasks.workunit.client.0.smithi022.stdout:285 2024-09-21T09:47:40.528 INFO:tasks.workunit.client.0.smithi022.stdout:286 2024-09-21T09:47:40.529 INFO:tasks.workunit.client.0.smithi022.stdout:287 2024-09-21T09:47:40.530 INFO:tasks.workunit.client.0.smithi022.stdout:288 2024-09-21T09:47:40.531 INFO:tasks.workunit.client.0.smithi022.stdout:289 2024-09-21T09:47:40.532 INFO:tasks.workunit.client.0.smithi022.stdout:290 2024-09-21T09:47:40.533 INFO:tasks.workunit.client.0.smithi022.stdout:291 2024-09-21T09:47:40.534 INFO:tasks.workunit.client.0.smithi022.stdout:292 2024-09-21T09:47:40.535 INFO:tasks.workunit.client.0.smithi022.stdout:293 2024-09-21T09:47:40.536 INFO:tasks.workunit.client.0.smithi022.stdout:294 2024-09-21T09:47:40.537 INFO:tasks.workunit.client.0.smithi022.stdout:295 2024-09-21T09:47:40.538 INFO:tasks.workunit.client.0.smithi022.stdout:296 2024-09-21T09:47:40.539 INFO:tasks.workunit.client.0.smithi022.stdout:297 2024-09-21T09:47:40.540 INFO:tasks.workunit.client.0.smithi022.stdout:298 2024-09-21T09:47:40.541 INFO:tasks.workunit.client.0.smithi022.stdout:299 2024-09-21T09:47:40.542 INFO:tasks.workunit.client.0.smithi022.stdout:300 2024-09-21T09:47:40.543 INFO:tasks.workunit.client.0.smithi022.stdout:301 2024-09-21T09:47:40.543 INFO:tasks.workunit.client.0.smithi022.stdout:302 2024-09-21T09:47:40.544 INFO:tasks.workunit.client.0.smithi022.stdout:303 2024-09-21T09:47:40.545 INFO:tasks.workunit.client.0.smithi022.stdout:304 2024-09-21T09:47:40.546 INFO:tasks.workunit.client.0.smithi022.stdout:305 2024-09-21T09:47:40.547 INFO:tasks.workunit.client.0.smithi022.stdout:306 2024-09-21T09:47:40.548 INFO:tasks.workunit.client.0.smithi022.stdout:307 2024-09-21T09:47:40.549 INFO:tasks.workunit.client.0.smithi022.stdout:308 2024-09-21T09:47:40.550 INFO:tasks.workunit.client.0.smithi022.stdout:309 2024-09-21T09:47:40.551 INFO:tasks.workunit.client.0.smithi022.stdout:310 2024-09-21T09:47:40.552 INFO:tasks.workunit.client.0.smithi022.stdout:311 2024-09-21T09:47:40.554 INFO:tasks.workunit.client.0.smithi022.stdout:312 2024-09-21T09:47:40.554 INFO:tasks.workunit.client.0.smithi022.stdout:313 2024-09-21T09:47:40.556 INFO:tasks.workunit.client.0.smithi022.stdout:314 2024-09-21T09:47:40.557 INFO:tasks.workunit.client.0.smithi022.stdout:315 2024-09-21T09:47:40.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:40 smithi022 ceph-mon[98615]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:47:40.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:40 smithi022 ceph-mon[98615]: Cluster is now healthy 2024-09-21T09:47:40.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:40 smithi022 ceph-mon[98615]: osdmap e967: 8 total, 8 up, 8 in 2024-09-21T09:47:40.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:40 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/4054658720' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129492-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:47:40.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:40 smithi022 ceph-mon[96736]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:47:40.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:40 smithi022 ceph-mon[96736]: Cluster is now healthy 2024-09-21T09:47:40.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:40 smithi022 ceph-mon[96736]: osdmap e967: 8 total, 8 up, 8 in 2024-09-21T09:47:40.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:40 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/4054658720' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129492-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:47:40.559 INFO:tasks.workunit.client.0.smithi022.stdout:316 2024-09-21T09:47:40.559 INFO:tasks.workunit.client.0.smithi022.stdout:317 2024-09-21T09:47:40.560 INFO:tasks.workunit.client.0.smithi022.stdout:318 2024-09-21T09:47:40.561 INFO:tasks.workunit.client.0.smithi022.stdout:319 2024-09-21T09:47:40.562 INFO:tasks.workunit.client.0.smithi022.stdout:320 2024-09-21T09:47:40.563 INFO:tasks.workunit.client.0.smithi022.stdout:321 2024-09-21T09:47:40.564 INFO:tasks.workunit.client.0.smithi022.stdout:322 2024-09-21T09:47:40.565 INFO:tasks.workunit.client.0.smithi022.stdout:323 2024-09-21T09:47:40.566 INFO:tasks.workunit.client.0.smithi022.stdout:324 2024-09-21T09:47:40.567 INFO:tasks.workunit.client.0.smithi022.stdout:325 2024-09-21T09:47:40.568 INFO:tasks.workunit.client.0.smithi022.stdout:326 2024-09-21T09:47:40.569 INFO:tasks.workunit.client.0.smithi022.stdout:327 2024-09-21T09:47:40.570 INFO:tasks.workunit.client.0.smithi022.stdout:328 2024-09-21T09:47:40.573 INFO:tasks.workunit.client.0.smithi022.stdout:329 2024-09-21T09:47:40.573 INFO:tasks.workunit.client.0.smithi022.stdout:330 2024-09-21T09:47:40.573 INFO:tasks.workunit.client.0.smithi022.stdout:331 2024-09-21T09:47:40.574 INFO:tasks.workunit.client.0.smithi022.stdout:332 2024-09-21T09:47:40.575 INFO:tasks.workunit.client.0.smithi022.stdout:333 2024-09-21T09:47:40.576 INFO:tasks.workunit.client.0.smithi022.stdout:334 2024-09-21T09:47:40.577 INFO:tasks.workunit.client.0.smithi022.stdout:335 2024-09-21T09:47:40.578 INFO:tasks.workunit.client.0.smithi022.stdout:336 2024-09-21T09:47:40.579 INFO:tasks.workunit.client.0.smithi022.stdout:337 2024-09-21T09:47:40.580 INFO:tasks.workunit.client.0.smithi022.stdout:338 2024-09-21T09:47:40.581 INFO:tasks.workunit.client.0.smithi022.stdout:339 2024-09-21T09:47:40.582 INFO:tasks.workunit.client.0.smithi022.stdout:340 2024-09-21T09:47:40.583 INFO:tasks.workunit.client.0.smithi022.stdout:341 2024-09-21T09:47:40.584 INFO:tasks.workunit.client.0.smithi022.stdout:342 2024-09-21T09:47:40.585 INFO:tasks.workunit.client.0.smithi022.stdout:343 2024-09-21T09:47:40.586 INFO:tasks.workunit.client.0.smithi022.stdout:344 2024-09-21T09:47:40.587 INFO:tasks.workunit.client.0.smithi022.stdout:345 2024-09-21T09:47:40.588 INFO:tasks.workunit.client.0.smithi022.stdout:346 2024-09-21T09:47:40.589 INFO:tasks.workunit.client.0.smithi022.stdout:347 2024-09-21T09:47:40.590 INFO:tasks.workunit.client.0.smithi022.stdout:348 2024-09-21T09:47:40.591 INFO:tasks.workunit.client.0.smithi022.stdout:349 2024-09-21T09:47:40.592 INFO:tasks.workunit.client.0.smithi022.stdout:350 2024-09-21T09:47:40.593 INFO:tasks.workunit.client.0.smithi022.stdout:351 2024-09-21T09:47:40.594 INFO:tasks.workunit.client.0.smithi022.stdout:352 2024-09-21T09:47:40.595 INFO:tasks.workunit.client.0.smithi022.stdout:353 2024-09-21T09:47:40.596 INFO:tasks.workunit.client.0.smithi022.stdout:354 2024-09-21T09:47:40.597 INFO:tasks.workunit.client.0.smithi022.stdout:355 2024-09-21T09:47:40.598 INFO:tasks.workunit.client.0.smithi022.stdout:356 2024-09-21T09:47:40.599 INFO:tasks.workunit.client.0.smithi022.stdout:357 2024-09-21T09:47:40.600 INFO:tasks.workunit.client.0.smithi022.stdout:358 2024-09-21T09:47:40.601 INFO:tasks.workunit.client.0.smithi022.stdout:359 2024-09-21T09:47:40.602 INFO:tasks.workunit.client.0.smithi022.stdout:360 2024-09-21T09:47:40.603 INFO:tasks.workunit.client.0.smithi022.stdout:361 2024-09-21T09:47:40.604 INFO:tasks.workunit.client.0.smithi022.stdout:362 2024-09-21T09:47:40.605 INFO:tasks.workunit.client.0.smithi022.stdout:363 2024-09-21T09:47:40.605 INFO:tasks.workunit.client.0.smithi022.stdout:364 2024-09-21T09:47:40.607 INFO:tasks.workunit.client.0.smithi022.stdout:365 2024-09-21T09:47:40.607 INFO:tasks.workunit.client.0.smithi022.stdout:366 2024-09-21T09:47:40.608 INFO:tasks.workunit.client.0.smithi022.stdout:367 2024-09-21T09:47:40.609 INFO:tasks.workunit.client.0.smithi022.stdout:368 2024-09-21T09:47:40.610 INFO:tasks.workunit.client.0.smithi022.stdout:369 2024-09-21T09:47:40.611 INFO:tasks.workunit.client.0.smithi022.stdout:370 2024-09-21T09:47:40.612 INFO:tasks.workunit.client.0.smithi022.stdout:371 2024-09-21T09:47:40.613 INFO:tasks.workunit.client.0.smithi022.stdout:372 2024-09-21T09:47:40.614 INFO:tasks.workunit.client.0.smithi022.stdout:373 2024-09-21T09:47:40.615 INFO:tasks.workunit.client.0.smithi022.stdout:374 2024-09-21T09:47:40.616 INFO:tasks.workunit.client.0.smithi022.stdout:375 2024-09-21T09:47:40.617 INFO:tasks.workunit.client.0.smithi022.stdout:376 2024-09-21T09:47:40.618 INFO:tasks.workunit.client.0.smithi022.stdout:377 2024-09-21T09:47:40.619 INFO:tasks.workunit.client.0.smithi022.stdout:378 2024-09-21T09:47:40.620 INFO:tasks.workunit.client.0.smithi022.stdout:379 2024-09-21T09:47:40.621 INFO:tasks.workunit.client.0.smithi022.stdout:380 2024-09-21T09:47:40.622 INFO:tasks.workunit.client.0.smithi022.stdout:381 2024-09-21T09:47:40.623 INFO:tasks.workunit.client.0.smithi022.stdout:382 2024-09-21T09:47:40.623 INFO:tasks.workunit.client.0.smithi022.stdout:383 2024-09-21T09:47:40.624 INFO:tasks.workunit.client.0.smithi022.stdout:384 2024-09-21T09:47:40.625 INFO:tasks.workunit.client.0.smithi022.stdout:385 2024-09-21T09:47:40.626 INFO:tasks.workunit.client.0.smithi022.stdout:386 2024-09-21T09:47:40.627 INFO:tasks.workunit.client.0.smithi022.stdout:387 2024-09-21T09:47:40.628 INFO:tasks.workunit.client.0.smithi022.stdout:388 2024-09-21T09:47:40.629 INFO:tasks.workunit.client.0.smithi022.stdout:389 2024-09-21T09:47:40.630 INFO:tasks.workunit.client.0.smithi022.stdout:390 2024-09-21T09:47:40.631 INFO:tasks.workunit.client.0.smithi022.stdout:391 2024-09-21T09:47:40.632 INFO:tasks.workunit.client.0.smithi022.stdout:392 2024-09-21T09:47:40.633 INFO:tasks.workunit.client.0.smithi022.stdout:393 2024-09-21T09:47:40.634 INFO:tasks.workunit.client.0.smithi022.stdout:394 2024-09-21T09:47:40.635 INFO:tasks.workunit.client.0.smithi022.stdout:395 2024-09-21T09:47:40.636 INFO:tasks.workunit.client.0.smithi022.stdout:396 2024-09-21T09:47:40.637 INFO:tasks.workunit.client.0.smithi022.stdout:397 2024-09-21T09:47:40.637 INFO:tasks.workunit.client.0.smithi022.stdout:398 2024-09-21T09:47:40.638 INFO:tasks.workunit.client.0.smithi022.stdout:399 2024-09-21T09:47:40.639 INFO:tasks.workunit.client.0.smithi022.stdout:400 2024-09-21T09:47:40.640 INFO:tasks.workunit.client.0.smithi022.stdout:401 2024-09-21T09:47:40.641 INFO:tasks.workunit.client.0.smithi022.stdout:402 2024-09-21T09:47:40.642 INFO:tasks.workunit.client.0.smithi022.stdout:403 2024-09-21T09:47:40.643 INFO:tasks.workunit.client.0.smithi022.stdout:404 2024-09-21T09:47:40.644 INFO:tasks.workunit.client.0.smithi022.stdout:405 2024-09-21T09:47:40.645 INFO:tasks.workunit.client.0.smithi022.stdout:406 2024-09-21T09:47:40.646 INFO:tasks.workunit.client.0.smithi022.stdout:407 2024-09-21T09:47:40.647 INFO:tasks.workunit.client.0.smithi022.stdout:408 2024-09-21T09:47:40.648 INFO:tasks.workunit.client.0.smithi022.stdout:409 2024-09-21T09:47:40.649 INFO:tasks.workunit.client.0.smithi022.stdout:410 2024-09-21T09:47:40.650 INFO:tasks.workunit.client.0.smithi022.stdout:411 2024-09-21T09:47:40.651 INFO:tasks.workunit.client.0.smithi022.stdout:412 2024-09-21T09:47:40.652 INFO:tasks.workunit.client.0.smithi022.stdout:413 2024-09-21T09:47:40.653 INFO:tasks.workunit.client.0.smithi022.stdout:414 2024-09-21T09:47:40.654 INFO:tasks.workunit.client.0.smithi022.stdout:415 2024-09-21T09:47:40.655 INFO:tasks.workunit.client.0.smithi022.stdout:416 2024-09-21T09:47:40.656 INFO:tasks.workunit.client.0.smithi022.stdout:417 2024-09-21T09:47:40.657 INFO:tasks.workunit.client.0.smithi022.stdout:418 2024-09-21T09:47:40.658 INFO:tasks.workunit.client.0.smithi022.stdout:419 2024-09-21T09:47:40.659 INFO:tasks.workunit.client.0.smithi022.stdout:420 2024-09-21T09:47:40.660 INFO:tasks.workunit.client.0.smithi022.stdout:421 2024-09-21T09:47:40.661 INFO:tasks.workunit.client.0.smithi022.stdout:422 2024-09-21T09:47:40.662 INFO:tasks.workunit.client.0.smithi022.stdout:423 2024-09-21T09:47:40.663 INFO:tasks.workunit.client.0.smithi022.stdout:424 2024-09-21T09:47:40.664 INFO:tasks.workunit.client.0.smithi022.stdout:425 2024-09-21T09:47:40.665 INFO:tasks.workunit.client.0.smithi022.stdout:426 2024-09-21T09:47:40.665 INFO:tasks.workunit.client.0.smithi022.stdout:427 2024-09-21T09:47:40.667 INFO:tasks.workunit.client.0.smithi022.stdout:428 2024-09-21T09:47:40.667 INFO:tasks.workunit.client.0.smithi022.stdout:429 2024-09-21T09:47:40.668 INFO:tasks.workunit.client.0.smithi022.stdout:430 2024-09-21T09:47:40.669 INFO:tasks.workunit.client.0.smithi022.stdout:431 2024-09-21T09:47:40.670 INFO:tasks.workunit.client.0.smithi022.stdout:432 2024-09-21T09:47:40.671 INFO:tasks.workunit.client.0.smithi022.stdout:433 2024-09-21T09:47:40.672 INFO:tasks.workunit.client.0.smithi022.stdout:434 2024-09-21T09:47:40.673 INFO:tasks.workunit.client.0.smithi022.stdout:435 2024-09-21T09:47:40.674 INFO:tasks.workunit.client.0.smithi022.stdout:436 2024-09-21T09:47:40.675 INFO:tasks.workunit.client.0.smithi022.stdout:437 2024-09-21T09:47:40.676 INFO:tasks.workunit.client.0.smithi022.stdout:438 2024-09-21T09:47:40.677 INFO:tasks.workunit.client.0.smithi022.stdout:439 2024-09-21T09:47:40.678 INFO:tasks.workunit.client.0.smithi022.stdout:440 2024-09-21T09:47:40.679 INFO:tasks.workunit.client.0.smithi022.stdout:441 2024-09-21T09:47:40.680 INFO:tasks.workunit.client.0.smithi022.stdout:442 2024-09-21T09:47:40.681 INFO:tasks.workunit.client.0.smithi022.stdout:443 2024-09-21T09:47:40.682 INFO:tasks.workunit.client.0.smithi022.stdout:444 2024-09-21T09:47:40.683 INFO:tasks.workunit.client.0.smithi022.stdout:445 2024-09-21T09:47:40.684 INFO:tasks.workunit.client.0.smithi022.stdout:446 2024-09-21T09:47:40.685 INFO:tasks.workunit.client.0.smithi022.stdout:447 2024-09-21T09:47:40.686 INFO:tasks.workunit.client.0.smithi022.stdout:448 2024-09-21T09:47:40.687 INFO:tasks.workunit.client.0.smithi022.stdout:449 2024-09-21T09:47:40.688 INFO:tasks.workunit.client.0.smithi022.stdout:450 2024-09-21T09:47:40.689 INFO:tasks.workunit.client.0.smithi022.stdout:451 2024-09-21T09:47:40.690 INFO:tasks.workunit.client.0.smithi022.stdout:452 2024-09-21T09:47:40.691 INFO:tasks.workunit.client.0.smithi022.stdout:453 2024-09-21T09:47:40.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:40 smithi064 ceph-mon[86709]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:47:40.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:40 smithi064 ceph-mon[86709]: Cluster is now healthy 2024-09-21T09:47:40.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:40 smithi064 ceph-mon[86709]: osdmap e967: 8 total, 8 up, 8 in 2024-09-21T09:47:40.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:40 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/4054658720' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129492-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:47:40.692 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:47:40 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:47:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:47:40.692 INFO:tasks.workunit.client.0.smithi022.stdout:454 2024-09-21T09:47:40.693 INFO:tasks.workunit.client.0.smithi022.stdout:455 2024-09-21T09:47:40.694 INFO:tasks.workunit.client.0.smithi022.stdout:456 2024-09-21T09:47:40.695 INFO:tasks.workunit.client.0.smithi022.stdout:457 2024-09-21T09:47:40.696 INFO:tasks.workunit.client.0.smithi022.stdout:458 2024-09-21T09:47:40.697 INFO:tasks.workunit.client.0.smithi022.stdout:459 2024-09-21T09:47:40.698 INFO:tasks.workunit.client.0.smithi022.stdout:460 2024-09-21T09:47:40.699 INFO:tasks.workunit.client.0.smithi022.stdout:461 2024-09-21T09:47:40.700 INFO:tasks.workunit.client.0.smithi022.stdout:462 2024-09-21T09:47:40.701 INFO:tasks.workunit.client.0.smithi022.stdout:463 2024-09-21T09:47:40.702 INFO:tasks.workunit.client.0.smithi022.stdout:464 2024-09-21T09:47:40.703 INFO:tasks.workunit.client.0.smithi022.stdout:465 2024-09-21T09:47:40.704 INFO:tasks.workunit.client.0.smithi022.stdout:466 2024-09-21T09:47:40.705 INFO:tasks.workunit.client.0.smithi022.stdout:467 2024-09-21T09:47:40.706 INFO:tasks.workunit.client.0.smithi022.stdout:468 2024-09-21T09:47:40.707 INFO:tasks.workunit.client.0.smithi022.stdout:469 2024-09-21T09:47:40.708 INFO:tasks.workunit.client.0.smithi022.stdout:470 2024-09-21T09:47:40.709 INFO:tasks.workunit.client.0.smithi022.stdout:471 2024-09-21T09:47:40.710 INFO:tasks.workunit.client.0.smithi022.stdout:472 2024-09-21T09:47:40.711 INFO:tasks.workunit.client.0.smithi022.stdout:473 2024-09-21T09:47:40.712 INFO:tasks.workunit.client.0.smithi022.stdout:474 2024-09-21T09:47:40.713 INFO:tasks.workunit.client.0.smithi022.stdout:475 2024-09-21T09:47:40.714 INFO:tasks.workunit.client.0.smithi022.stdout:476 2024-09-21T09:47:40.715 INFO:tasks.workunit.client.0.smithi022.stdout:477 2024-09-21T09:47:40.716 INFO:tasks.workunit.client.0.smithi022.stdout:478 2024-09-21T09:47:40.717 INFO:tasks.workunit.client.0.smithi022.stdout:479 2024-09-21T09:47:40.718 INFO:tasks.workunit.client.0.smithi022.stdout:480 2024-09-21T09:47:40.719 INFO:tasks.workunit.client.0.smithi022.stdout:481 2024-09-21T09:47:40.720 INFO:tasks.workunit.client.0.smithi022.stdout:482 2024-09-21T09:47:40.721 INFO:tasks.workunit.client.0.smithi022.stdout:483 2024-09-21T09:47:40.722 INFO:tasks.workunit.client.0.smithi022.stdout:484 2024-09-21T09:47:40.723 INFO:tasks.workunit.client.0.smithi022.stdout:485 2024-09-21T09:47:40.724 INFO:tasks.workunit.client.0.smithi022.stdout:486 2024-09-21T09:47:40.725 INFO:tasks.workunit.client.0.smithi022.stdout:487 2024-09-21T09:47:40.726 INFO:tasks.workunit.client.0.smithi022.stdout:488 2024-09-21T09:47:40.727 INFO:tasks.workunit.client.0.smithi022.stdout:489 2024-09-21T09:47:40.728 INFO:tasks.workunit.client.0.smithi022.stdout:490 2024-09-21T09:47:40.729 INFO:tasks.workunit.client.0.smithi022.stdout:491 2024-09-21T09:47:40.730 INFO:tasks.workunit.client.0.smithi022.stdout:492 2024-09-21T09:47:40.731 INFO:tasks.workunit.client.0.smithi022.stdout:493 2024-09-21T09:47:40.732 INFO:tasks.workunit.client.0.smithi022.stdout:494 2024-09-21T09:47:40.733 INFO:tasks.workunit.client.0.smithi022.stdout:495 2024-09-21T09:47:40.734 INFO:tasks.workunit.client.0.smithi022.stdout:496 2024-09-21T09:47:40.735 INFO:tasks.workunit.client.0.smithi022.stdout:497 2024-09-21T09:47:40.736 INFO:tasks.workunit.client.0.smithi022.stdout:498 2024-09-21T09:47:40.737 INFO:tasks.workunit.client.0.smithi022.stdout:499 2024-09-21T09:47:40.738 INFO:tasks.workunit.client.0.smithi022.stdout:500 2024-09-21T09:47:40.739 INFO:tasks.workunit.client.0.smithi022.stdout:501 2024-09-21T09:47:40.740 INFO:tasks.workunit.client.0.smithi022.stdout:502 2024-09-21T09:47:40.741 INFO:tasks.workunit.client.0.smithi022.stdout:503 2024-09-21T09:47:40.742 INFO:tasks.workunit.client.0.smithi022.stdout:504 2024-09-21T09:47:40.743 INFO:tasks.workunit.client.0.smithi022.stdout:505 2024-09-21T09:47:40.744 INFO:tasks.workunit.client.0.smithi022.stdout:506 2024-09-21T09:47:40.745 INFO:tasks.workunit.client.0.smithi022.stdout:507 2024-09-21T09:47:40.746 INFO:tasks.workunit.client.0.smithi022.stdout:508 2024-09-21T09:47:40.747 INFO:tasks.workunit.client.0.smithi022.stdout:509 2024-09-21T09:47:40.748 INFO:tasks.workunit.client.0.smithi022.stdout:510 2024-09-21T09:47:40.749 INFO:tasks.workunit.client.0.smithi022.stdout:511 2024-09-21T09:47:40.750 INFO:tasks.workunit.client.0.smithi022.stdout:512 2024-09-21T09:47:40.751 INFO:tasks.workunit.client.0.smithi022.stdout:513 2024-09-21T09:47:40.752 INFO:tasks.workunit.client.0.smithi022.stdout:514 2024-09-21T09:47:40.753 INFO:tasks.workunit.client.0.smithi022.stdout:515 2024-09-21T09:47:40.755 INFO:tasks.workunit.client.0.smithi022.stdout:516 2024-09-21T09:47:40.755 INFO:tasks.workunit.client.0.smithi022.stdout:517 2024-09-21T09:47:40.757 INFO:tasks.workunit.client.0.smithi022.stdout:518 2024-09-21T09:47:40.757 INFO:tasks.workunit.client.0.smithi022.stdout:519 2024-09-21T09:47:40.759 INFO:tasks.workunit.client.0.smithi022.stdout:520 2024-09-21T09:47:40.760 INFO:tasks.workunit.client.0.smithi022.stdout:521 2024-09-21T09:47:40.760 INFO:tasks.workunit.client.0.smithi022.stdout:522 2024-09-21T09:47:40.761 INFO:tasks.workunit.client.0.smithi022.stdout:523 2024-09-21T09:47:40.762 INFO:tasks.workunit.client.0.smithi022.stdout:524 2024-09-21T09:47:40.763 INFO:tasks.workunit.client.0.smithi022.stdout:525 2024-09-21T09:47:40.764 INFO:tasks.workunit.client.0.smithi022.stdout:526 2024-09-21T09:47:40.765 INFO:tasks.workunit.client.0.smithi022.stdout:527 2024-09-21T09:47:40.766 INFO:tasks.workunit.client.0.smithi022.stdout:528 2024-09-21T09:47:40.767 INFO:tasks.workunit.client.0.smithi022.stdout:529 2024-09-21T09:47:40.769 INFO:tasks.workunit.client.0.smithi022.stdout:530 2024-09-21T09:47:40.770 INFO:tasks.workunit.client.0.smithi022.stdout:531 2024-09-21T09:47:40.771 INFO:tasks.workunit.client.0.smithi022.stdout:532 2024-09-21T09:47:40.772 INFO:tasks.workunit.client.0.smithi022.stdout:533 2024-09-21T09:47:40.773 INFO:tasks.workunit.client.0.smithi022.stdout:534 2024-09-21T09:47:40.774 INFO:tasks.workunit.client.0.smithi022.stdout:535 2024-09-21T09:47:40.775 INFO:tasks.workunit.client.0.smithi022.stdout:536 2024-09-21T09:47:40.776 INFO:tasks.workunit.client.0.smithi022.stdout:537 2024-09-21T09:47:40.777 INFO:tasks.workunit.client.0.smithi022.stdout:538 2024-09-21T09:47:40.778 INFO:tasks.workunit.client.0.smithi022.stdout:539 2024-09-21T09:47:40.779 INFO:tasks.workunit.client.0.smithi022.stdout:540 2024-09-21T09:47:40.780 INFO:tasks.workunit.client.0.smithi022.stdout:541 2024-09-21T09:47:40.781 INFO:tasks.workunit.client.0.smithi022.stdout:542 2024-09-21T09:47:40.783 INFO:tasks.workunit.client.0.smithi022.stdout:543 2024-09-21T09:47:40.784 INFO:tasks.workunit.client.0.smithi022.stdout:544 2024-09-21T09:47:40.785 INFO:tasks.workunit.client.0.smithi022.stdout:545 2024-09-21T09:47:40.786 INFO:tasks.workunit.client.0.smithi022.stdout:546 2024-09-21T09:47:40.786 INFO:tasks.workunit.client.0.smithi022.stdout:547 2024-09-21T09:47:40.787 INFO:tasks.workunit.client.0.smithi022.stdout:548 2024-09-21T09:47:40.788 INFO:tasks.workunit.client.0.smithi022.stdout:549 2024-09-21T09:47:40.789 INFO:tasks.workunit.client.0.smithi022.stdout:550 2024-09-21T09:47:40.790 INFO:tasks.workunit.client.0.smithi022.stdout:551 2024-09-21T09:47:40.792 INFO:tasks.workunit.client.0.smithi022.stdout:552 2024-09-21T09:47:40.793 INFO:tasks.workunit.client.0.smithi022.stdout:553 2024-09-21T09:47:40.794 INFO:tasks.workunit.client.0.smithi022.stdout:554 2024-09-21T09:47:40.795 INFO:tasks.workunit.client.0.smithi022.stdout:555 2024-09-21T09:47:40.796 INFO:tasks.workunit.client.0.smithi022.stdout:556 2024-09-21T09:47:40.797 INFO:tasks.workunit.client.0.smithi022.stdout:557 2024-09-21T09:47:40.798 INFO:tasks.workunit.client.0.smithi022.stdout:558 2024-09-21T09:47:40.799 INFO:tasks.workunit.client.0.smithi022.stdout:559 2024-09-21T09:47:40.800 INFO:tasks.workunit.client.0.smithi022.stdout:560 2024-09-21T09:47:40.801 INFO:tasks.workunit.client.0.smithi022.stdout:561 2024-09-21T09:47:40.802 INFO:tasks.workunit.client.0.smithi022.stdout:562 2024-09-21T09:47:40.803 INFO:tasks.workunit.client.0.smithi022.stdout:563 2024-09-21T09:47:40.804 INFO:tasks.workunit.client.0.smithi022.stdout:564 2024-09-21T09:47:40.805 INFO:tasks.workunit.client.0.smithi022.stdout:565 2024-09-21T09:47:40.806 INFO:tasks.workunit.client.0.smithi022.stdout:566 2024-09-21T09:47:40.807 INFO:tasks.workunit.client.0.smithi022.stdout:567 2024-09-21T09:47:40.808 INFO:tasks.workunit.client.0.smithi022.stdout:568 2024-09-21T09:47:40.809 INFO:tasks.workunit.client.0.smithi022.stdout:569 2024-09-21T09:47:40.810 INFO:tasks.workunit.client.0.smithi022.stdout:570 2024-09-21T09:47:40.811 INFO:tasks.workunit.client.0.smithi022.stdout:571 2024-09-21T09:47:40.812 INFO:tasks.workunit.client.0.smithi022.stdout:572 2024-09-21T09:47:40.813 INFO:tasks.workunit.client.0.smithi022.stdout:573 2024-09-21T09:47:40.814 INFO:tasks.workunit.client.0.smithi022.stdout:574 2024-09-21T09:47:40.815 INFO:tasks.workunit.client.0.smithi022.stdout:575 2024-09-21T09:47:40.816 INFO:tasks.workunit.client.0.smithi022.stdout:576 2024-09-21T09:47:40.817 INFO:tasks.workunit.client.0.smithi022.stdout:577 2024-09-21T09:47:40.818 INFO:tasks.workunit.client.0.smithi022.stdout:578 2024-09-21T09:47:40.819 INFO:tasks.workunit.client.0.smithi022.stdout:579 2024-09-21T09:47:40.820 INFO:tasks.workunit.client.0.smithi022.stdout:580 2024-09-21T09:47:40.821 INFO:tasks.workunit.client.0.smithi022.stdout:581 2024-09-21T09:47:40.822 INFO:tasks.workunit.client.0.smithi022.stdout:582 2024-09-21T09:47:40.823 INFO:tasks.workunit.client.0.smithi022.stdout:583 2024-09-21T09:47:40.824 INFO:tasks.workunit.client.0.smithi022.stdout:584 2024-09-21T09:47:40.825 INFO:tasks.workunit.client.0.smithi022.stdout:585 2024-09-21T09:47:40.826 INFO:tasks.workunit.client.0.smithi022.stdout:586 2024-09-21T09:47:40.827 INFO:tasks.workunit.client.0.smithi022.stdout:587 2024-09-21T09:47:40.828 INFO:tasks.workunit.client.0.smithi022.stdout:588 2024-09-21T09:47:40.829 INFO:tasks.workunit.client.0.smithi022.stdout:589 2024-09-21T09:47:40.830 INFO:tasks.workunit.client.0.smithi022.stdout:590 2024-09-21T09:47:40.831 INFO:tasks.workunit.client.0.smithi022.stdout:591 2024-09-21T09:47:40.832 INFO:tasks.workunit.client.0.smithi022.stdout:592 2024-09-21T09:47:40.833 INFO:tasks.workunit.client.0.smithi022.stdout:593 2024-09-21T09:47:40.834 INFO:tasks.workunit.client.0.smithi022.stdout:594 2024-09-21T09:47:40.835 INFO:tasks.workunit.client.0.smithi022.stdout:595 2024-09-21T09:47:40.836 INFO:tasks.workunit.client.0.smithi022.stdout:596 2024-09-21T09:47:40.837 INFO:tasks.workunit.client.0.smithi022.stdout:597 2024-09-21T09:47:40.838 INFO:tasks.workunit.client.0.smithi022.stdout:598 2024-09-21T09:47:40.840 INFO:tasks.workunit.client.0.smithi022.stdout:599 2024-09-21T09:47:40.841 INFO:tasks.workunit.client.0.smithi022.stdout:600 2024-09-21T09:47:40.842 INFO:tasks.workunit.client.0.smithi022.stdout:601 2024-09-21T09:47:40.843 INFO:tasks.workunit.client.0.smithi022.stdout:602 2024-09-21T09:47:40.844 INFO:tasks.workunit.client.0.smithi022.stdout:603 2024-09-21T09:47:40.845 INFO:tasks.workunit.client.0.smithi022.stdout:604 2024-09-21T09:47:40.846 INFO:tasks.workunit.client.0.smithi022.stdout:605 2024-09-21T09:47:40.847 INFO:tasks.workunit.client.0.smithi022.stdout:606 2024-09-21T09:47:40.848 INFO:tasks.workunit.client.0.smithi022.stdout:607 2024-09-21T09:47:40.849 INFO:tasks.workunit.client.0.smithi022.stdout:608 2024-09-21T09:47:40.850 INFO:tasks.workunit.client.0.smithi022.stdout:609 2024-09-21T09:47:40.851 INFO:tasks.workunit.client.0.smithi022.stdout:610 2024-09-21T09:47:40.853 INFO:tasks.workunit.client.0.smithi022.stdout:611 2024-09-21T09:47:40.854 INFO:tasks.workunit.client.0.smithi022.stdout:612 2024-09-21T09:47:40.855 INFO:tasks.workunit.client.0.smithi022.stdout:613 2024-09-21T09:47:40.856 INFO:tasks.workunit.client.0.smithi022.stdout:614 2024-09-21T09:47:40.857 INFO:tasks.workunit.client.0.smithi022.stdout:615 2024-09-21T09:47:40.858 INFO:tasks.workunit.client.0.smithi022.stdout:616 2024-09-21T09:47:40.859 INFO:tasks.workunit.client.0.smithi022.stdout:617 2024-09-21T09:47:40.860 INFO:tasks.workunit.client.0.smithi022.stdout:618 2024-09-21T09:47:40.861 INFO:tasks.workunit.client.0.smithi022.stdout:619 2024-09-21T09:47:40.862 INFO:tasks.workunit.client.0.smithi022.stdout:620 2024-09-21T09:47:40.863 INFO:tasks.workunit.client.0.smithi022.stdout:621 2024-09-21T09:47:40.864 INFO:tasks.workunit.client.0.smithi022.stdout:622 2024-09-21T09:47:40.866 INFO:tasks.workunit.client.0.smithi022.stdout:623 2024-09-21T09:47:40.867 INFO:tasks.workunit.client.0.smithi022.stdout:624 2024-09-21T09:47:40.868 INFO:tasks.workunit.client.0.smithi022.stdout:625 2024-09-21T09:47:40.869 INFO:tasks.workunit.client.0.smithi022.stdout:626 2024-09-21T09:47:40.870 INFO:tasks.workunit.client.0.smithi022.stdout:627 2024-09-21T09:47:40.871 INFO:tasks.workunit.client.0.smithi022.stdout:628 2024-09-21T09:47:40.873 INFO:tasks.workunit.client.0.smithi022.stdout:629 2024-09-21T09:47:40.874 INFO:tasks.workunit.client.0.smithi022.stdout:630 2024-09-21T09:47:40.875 INFO:tasks.workunit.client.0.smithi022.stdout:631 2024-09-21T09:47:40.876 INFO:tasks.workunit.client.0.smithi022.stdout:632 2024-09-21T09:47:40.877 INFO:tasks.workunit.client.0.smithi022.stdout:633 2024-09-21T09:47:40.878 INFO:tasks.workunit.client.0.smithi022.stdout:634 2024-09-21T09:47:40.879 INFO:tasks.workunit.client.0.smithi022.stdout:635 2024-09-21T09:47:40.880 INFO:tasks.workunit.client.0.smithi022.stdout:636 2024-09-21T09:47:40.881 INFO:tasks.workunit.client.0.smithi022.stdout:637 2024-09-21T09:47:40.882 INFO:tasks.workunit.client.0.smithi022.stdout:638 2024-09-21T09:47:40.883 INFO:tasks.workunit.client.0.smithi022.stdout:639 2024-09-21T09:47:40.884 INFO:tasks.workunit.client.0.smithi022.stdout:640 2024-09-21T09:47:40.885 INFO:tasks.workunit.client.0.smithi022.stdout:641 2024-09-21T09:47:40.886 INFO:tasks.workunit.client.0.smithi022.stdout:642 2024-09-21T09:47:40.888 INFO:tasks.workunit.client.0.smithi022.stdout:643 2024-09-21T09:47:40.889 INFO:tasks.workunit.client.0.smithi022.stdout:644 2024-09-21T09:47:40.890 INFO:tasks.workunit.client.0.smithi022.stdout:645 2024-09-21T09:47:40.891 INFO:tasks.workunit.client.0.smithi022.stdout:646 2024-09-21T09:47:40.892 INFO:tasks.workunit.client.0.smithi022.stdout:647 2024-09-21T09:47:40.893 INFO:tasks.workunit.client.0.smithi022.stdout:648 2024-09-21T09:47:40.894 INFO:tasks.workunit.client.0.smithi022.stdout:649 2024-09-21T09:47:40.895 INFO:tasks.workunit.client.0.smithi022.stdout:650 2024-09-21T09:47:40.896 INFO:tasks.workunit.client.0.smithi022.stdout:651 2024-09-21T09:47:40.897 INFO:tasks.workunit.client.0.smithi022.stdout:652 2024-09-21T09:47:40.898 INFO:tasks.workunit.client.0.smithi022.stdout:653 2024-09-21T09:47:40.899 INFO:tasks.workunit.client.0.smithi022.stdout:654 2024-09-21T09:47:40.900 INFO:tasks.workunit.client.0.smithi022.stdout:655 2024-09-21T09:47:40.901 INFO:tasks.workunit.client.0.smithi022.stdout:656 2024-09-21T09:47:40.902 INFO:tasks.workunit.client.0.smithi022.stdout:657 2024-09-21T09:47:40.903 INFO:tasks.workunit.client.0.smithi022.stdout:658 2024-09-21T09:47:40.904 INFO:tasks.workunit.client.0.smithi022.stdout:659 2024-09-21T09:47:40.905 INFO:tasks.workunit.client.0.smithi022.stdout:660 2024-09-21T09:47:40.906 INFO:tasks.workunit.client.0.smithi022.stdout:661 2024-09-21T09:47:40.908 INFO:tasks.workunit.client.0.smithi022.stdout:662 2024-09-21T09:47:40.909 INFO:tasks.workunit.client.0.smithi022.stdout:663 2024-09-21T09:47:40.910 INFO:tasks.workunit.client.0.smithi022.stdout:664 2024-09-21T09:47:40.911 INFO:tasks.workunit.client.0.smithi022.stdout:665 2024-09-21T09:47:40.912 INFO:tasks.workunit.client.0.smithi022.stdout:666 2024-09-21T09:47:40.913 INFO:tasks.workunit.client.0.smithi022.stdout:667 2024-09-21T09:47:40.914 INFO:tasks.workunit.client.0.smithi022.stdout:668 2024-09-21T09:47:40.915 INFO:tasks.workunit.client.0.smithi022.stdout:669 2024-09-21T09:47:40.916 INFO:tasks.workunit.client.0.smithi022.stdout:670 2024-09-21T09:47:40.917 INFO:tasks.workunit.client.0.smithi022.stdout:671 2024-09-21T09:47:40.918 INFO:tasks.workunit.client.0.smithi022.stdout:672 2024-09-21T09:47:40.920 INFO:tasks.workunit.client.0.smithi022.stdout:673 2024-09-21T09:47:40.921 INFO:tasks.workunit.client.0.smithi022.stdout:674 2024-09-21T09:47:40.922 INFO:tasks.workunit.client.0.smithi022.stdout:675 2024-09-21T09:47:40.923 INFO:tasks.workunit.client.0.smithi022.stdout:676 2024-09-21T09:47:40.924 INFO:tasks.workunit.client.0.smithi022.stdout:677 2024-09-21T09:47:40.925 INFO:tasks.workunit.client.0.smithi022.stdout:678 2024-09-21T09:47:40.926 INFO:tasks.workunit.client.0.smithi022.stdout:679 2024-09-21T09:47:40.927 INFO:tasks.workunit.client.0.smithi022.stdout:680 2024-09-21T09:47:40.928 INFO:tasks.workunit.client.0.smithi022.stdout:681 2024-09-21T09:47:40.929 INFO:tasks.workunit.client.0.smithi022.stdout:682 2024-09-21T09:47:40.930 INFO:tasks.workunit.client.0.smithi022.stdout:683 2024-09-21T09:47:40.931 INFO:tasks.workunit.client.0.smithi022.stdout:684 2024-09-21T09:47:40.932 INFO:tasks.workunit.client.0.smithi022.stdout:685 2024-09-21T09:47:40.933 INFO:tasks.workunit.client.0.smithi022.stdout:686 2024-09-21T09:47:40.935 INFO:tasks.workunit.client.0.smithi022.stdout:687 2024-09-21T09:47:40.936 INFO:tasks.workunit.client.0.smithi022.stdout:688 2024-09-21T09:47:40.937 INFO:tasks.workunit.client.0.smithi022.stdout:689 2024-09-21T09:47:40.938 INFO:tasks.workunit.client.0.smithi022.stdout:690 2024-09-21T09:47:40.939 INFO:tasks.workunit.client.0.smithi022.stdout:691 2024-09-21T09:47:40.940 INFO:tasks.workunit.client.0.smithi022.stdout:692 2024-09-21T09:47:40.941 INFO:tasks.workunit.client.0.smithi022.stdout:693 2024-09-21T09:47:40.942 INFO:tasks.workunit.client.0.smithi022.stdout:694 2024-09-21T09:47:40.943 INFO:tasks.workunit.client.0.smithi022.stdout:695 2024-09-21T09:47:40.944 INFO:tasks.workunit.client.0.smithi022.stdout:696 2024-09-21T09:47:40.945 INFO:tasks.workunit.client.0.smithi022.stdout:697 2024-09-21T09:47:40.946 INFO:tasks.workunit.client.0.smithi022.stdout:698 2024-09-21T09:47:40.947 INFO:tasks.workunit.client.0.smithi022.stdout:699 2024-09-21T09:47:40.948 INFO:tasks.workunit.client.0.smithi022.stdout:700 2024-09-21T09:47:40.949 INFO:tasks.workunit.client.0.smithi022.stdout:701 2024-09-21T09:47:40.950 INFO:tasks.workunit.client.0.smithi022.stdout:702 2024-09-21T09:47:40.951 INFO:tasks.workunit.client.0.smithi022.stdout:703 2024-09-21T09:47:40.953 INFO:tasks.workunit.client.0.smithi022.stdout:704 2024-09-21T09:47:40.954 INFO:tasks.workunit.client.0.smithi022.stdout:705 2024-09-21T09:47:40.955 INFO:tasks.workunit.client.0.smithi022.stdout:706 2024-09-21T09:47:40.956 INFO:tasks.workunit.client.0.smithi022.stdout:707 2024-09-21T09:47:40.957 INFO:tasks.workunit.client.0.smithi022.stdout:708 2024-09-21T09:47:40.958 INFO:tasks.workunit.client.0.smithi022.stdout:709 2024-09-21T09:47:40.959 INFO:tasks.workunit.client.0.smithi022.stdout:710 2024-09-21T09:47:40.960 INFO:tasks.workunit.client.0.smithi022.stdout:711 2024-09-21T09:47:40.961 INFO:tasks.workunit.client.0.smithi022.stdout:712 2024-09-21T09:47:40.962 INFO:tasks.workunit.client.0.smithi022.stdout:713 2024-09-21T09:47:40.963 INFO:tasks.workunit.client.0.smithi022.stdout:714 2024-09-21T09:47:40.964 INFO:tasks.workunit.client.0.smithi022.stdout:715 2024-09-21T09:47:40.965 INFO:tasks.workunit.client.0.smithi022.stdout:716 2024-09-21T09:47:40.966 INFO:tasks.workunit.client.0.smithi022.stdout:717 2024-09-21T09:47:40.967 INFO:tasks.workunit.client.0.smithi022.stdout:718 2024-09-21T09:47:40.968 INFO:tasks.workunit.client.0.smithi022.stdout:719 2024-09-21T09:47:40.969 INFO:tasks.workunit.client.0.smithi022.stdout:720 2024-09-21T09:47:40.970 INFO:tasks.workunit.client.0.smithi022.stdout:721 2024-09-21T09:47:40.972 INFO:tasks.workunit.client.0.smithi022.stdout:722 2024-09-21T09:47:40.973 INFO:tasks.workunit.client.0.smithi022.stdout:723 2024-09-21T09:47:40.974 INFO:tasks.workunit.client.0.smithi022.stdout:724 2024-09-21T09:47:40.975 INFO:tasks.workunit.client.0.smithi022.stdout:725 2024-09-21T09:47:40.976 INFO:tasks.workunit.client.0.smithi022.stdout:726 2024-09-21T09:47:40.977 INFO:tasks.workunit.client.0.smithi022.stdout:727 2024-09-21T09:47:40.978 INFO:tasks.workunit.client.0.smithi022.stdout:728 2024-09-21T09:47:40.979 INFO:tasks.workunit.client.0.smithi022.stdout:729 2024-09-21T09:47:40.980 INFO:tasks.workunit.client.0.smithi022.stdout:730 2024-09-21T09:47:40.982 INFO:tasks.workunit.client.0.smithi022.stdout:731 2024-09-21T09:47:40.983 INFO:tasks.workunit.client.0.smithi022.stdout:732 2024-09-21T09:47:40.984 INFO:tasks.workunit.client.0.smithi022.stdout:733 2024-09-21T09:47:40.985 INFO:tasks.workunit.client.0.smithi022.stdout:734 2024-09-21T09:47:40.986 INFO:tasks.workunit.client.0.smithi022.stdout:735 2024-09-21T09:47:40.987 INFO:tasks.workunit.client.0.smithi022.stdout:736 2024-09-21T09:47:40.988 INFO:tasks.workunit.client.0.smithi022.stdout:737 2024-09-21T09:47:40.990 INFO:tasks.workunit.client.0.smithi022.stdout:738 2024-09-21T09:47:40.990 INFO:tasks.workunit.client.0.smithi022.stdout:739 2024-09-21T09:47:40.992 INFO:tasks.workunit.client.0.smithi022.stdout:740 2024-09-21T09:47:40.993 INFO:tasks.workunit.client.0.smithi022.stdout:741 2024-09-21T09:47:40.994 INFO:tasks.workunit.client.0.smithi022.stdout:742 2024-09-21T09:47:40.995 INFO:tasks.workunit.client.0.smithi022.stdout:743 2024-09-21T09:47:40.996 INFO:tasks.workunit.client.0.smithi022.stdout:744 2024-09-21T09:47:40.997 INFO:tasks.workunit.client.0.smithi022.stdout:745 2024-09-21T09:47:40.998 INFO:tasks.workunit.client.0.smithi022.stdout:746 2024-09-21T09:47:40.999 INFO:tasks.workunit.client.0.smithi022.stdout:747 2024-09-21T09:47:41.000 INFO:tasks.workunit.client.0.smithi022.stdout:748 2024-09-21T09:47:41.001 INFO:tasks.workunit.client.0.smithi022.stdout:749 2024-09-21T09:47:41.002 INFO:tasks.workunit.client.0.smithi022.stdout:750 2024-09-21T09:47:41.003 INFO:tasks.workunit.client.0.smithi022.stdout:751 2024-09-21T09:47:41.004 INFO:tasks.workunit.client.0.smithi022.stdout:752 2024-09-21T09:47:41.005 INFO:tasks.workunit.client.0.smithi022.stdout:753 2024-09-21T09:47:41.007 INFO:tasks.workunit.client.0.smithi022.stdout:754 2024-09-21T09:47:41.008 INFO:tasks.workunit.client.0.smithi022.stdout:755 2024-09-21T09:47:41.009 INFO:tasks.workunit.client.0.smithi022.stdout:756 2024-09-21T09:47:41.010 INFO:tasks.workunit.client.0.smithi022.stdout:757 2024-09-21T09:47:41.011 INFO:tasks.workunit.client.0.smithi022.stdout:758 2024-09-21T09:47:41.012 INFO:tasks.workunit.client.0.smithi022.stdout:759 2024-09-21T09:47:41.013 INFO:tasks.workunit.client.0.smithi022.stdout:760 2024-09-21T09:47:41.014 INFO:tasks.workunit.client.0.smithi022.stdout:761 2024-09-21T09:47:41.015 INFO:tasks.workunit.client.0.smithi022.stdout:762 2024-09-21T09:47:41.016 INFO:tasks.workunit.client.0.smithi022.stdout:763 2024-09-21T09:47:41.017 INFO:tasks.workunit.client.0.smithi022.stdout:764 2024-09-21T09:47:41.018 INFO:tasks.workunit.client.0.smithi022.stdout:765 2024-09-21T09:47:41.019 INFO:tasks.workunit.client.0.smithi022.stdout:766 2024-09-21T09:47:41.020 INFO:tasks.workunit.client.0.smithi022.stdout:767 2024-09-21T09:47:41.021 INFO:tasks.workunit.client.0.smithi022.stdout:768 2024-09-21T09:47:41.022 INFO:tasks.workunit.client.0.smithi022.stdout:769 2024-09-21T09:47:41.023 INFO:tasks.workunit.client.0.smithi022.stdout:770 2024-09-21T09:47:41.024 INFO:tasks.workunit.client.0.smithi022.stdout:771 2024-09-21T09:47:41.025 INFO:tasks.workunit.client.0.smithi022.stdout:772 2024-09-21T09:47:41.026 INFO:tasks.workunit.client.0.smithi022.stdout:773 2024-09-21T09:47:41.028 INFO:tasks.workunit.client.0.smithi022.stdout:774 2024-09-21T09:47:41.029 INFO:tasks.workunit.client.0.smithi022.stdout:775 2024-09-21T09:47:41.030 INFO:tasks.workunit.client.0.smithi022.stdout:776 2024-09-21T09:47:41.031 INFO:tasks.workunit.client.0.smithi022.stdout:777 2024-09-21T09:47:41.033 INFO:tasks.workunit.client.0.smithi022.stdout:778 2024-09-21T09:47:41.034 INFO:tasks.workunit.client.0.smithi022.stdout:779 2024-09-21T09:47:41.035 INFO:tasks.workunit.client.0.smithi022.stdout:780 2024-09-21T09:47:41.036 INFO:tasks.workunit.client.0.smithi022.stdout:781 2024-09-21T09:47:41.038 INFO:tasks.workunit.client.0.smithi022.stdout:782 2024-09-21T09:47:41.039 INFO:tasks.workunit.client.0.smithi022.stdout:783 2024-09-21T09:47:41.040 INFO:tasks.workunit.client.0.smithi022.stdout:784 2024-09-21T09:47:41.042 INFO:tasks.workunit.client.0.smithi022.stdout:785 2024-09-21T09:47:41.043 INFO:tasks.workunit.client.0.smithi022.stdout:786 2024-09-21T09:47:41.044 INFO:tasks.workunit.client.0.smithi022.stdout:787 2024-09-21T09:47:41.045 INFO:tasks.workunit.client.0.smithi022.stdout:788 2024-09-21T09:47:41.046 INFO:tasks.workunit.client.0.smithi022.stdout:789 2024-09-21T09:47:41.048 INFO:tasks.workunit.client.0.smithi022.stdout:790 2024-09-21T09:47:41.049 INFO:tasks.workunit.client.0.smithi022.stdout:791 2024-09-21T09:47:41.050 INFO:tasks.workunit.client.0.smithi022.stdout:792 2024-09-21T09:47:41.051 INFO:tasks.workunit.client.0.smithi022.stdout:793 2024-09-21T09:47:41.052 INFO:tasks.workunit.client.0.smithi022.stdout:794 2024-09-21T09:47:41.053 INFO:tasks.workunit.client.0.smithi022.stdout:795 2024-09-21T09:47:41.054 INFO:tasks.workunit.client.0.smithi022.stdout:796 2024-09-21T09:47:41.055 INFO:tasks.workunit.client.0.smithi022.stdout:797 2024-09-21T09:47:41.056 INFO:tasks.workunit.client.0.smithi022.stdout:798 2024-09-21T09:47:41.058 INFO:tasks.workunit.client.0.smithi022.stdout:799 2024-09-21T09:47:41.059 INFO:tasks.workunit.client.0.smithi022.stdout:800 2024-09-21T09:47:41.060 INFO:tasks.workunit.client.0.smithi022.stdout:801 2024-09-21T09:47:41.061 INFO:tasks.workunit.client.0.smithi022.stdout:802 2024-09-21T09:47:41.062 INFO:tasks.workunit.client.0.smithi022.stdout:803 2024-09-21T09:47:41.063 INFO:tasks.workunit.client.0.smithi022.stdout:804 2024-09-21T09:47:41.064 INFO:tasks.workunit.client.0.smithi022.stdout:805 2024-09-21T09:47:41.065 INFO:tasks.workunit.client.0.smithi022.stdout:806 2024-09-21T09:47:41.066 INFO:tasks.workunit.client.0.smithi022.stdout:807 2024-09-21T09:47:41.067 INFO:tasks.workunit.client.0.smithi022.stdout:808 2024-09-21T09:47:41.069 INFO:tasks.workunit.client.0.smithi022.stdout:809 2024-09-21T09:47:41.070 INFO:tasks.workunit.client.0.smithi022.stdout:810 2024-09-21T09:47:41.071 INFO:tasks.workunit.client.0.smithi022.stdout:811 2024-09-21T09:47:41.072 INFO:tasks.workunit.client.0.smithi022.stdout:812 2024-09-21T09:47:41.073 INFO:tasks.workunit.client.0.smithi022.stdout:813 2024-09-21T09:47:41.074 INFO:tasks.workunit.client.0.smithi022.stdout:814 2024-09-21T09:47:41.075 INFO:tasks.workunit.client.0.smithi022.stdout:815 2024-09-21T09:47:41.076 INFO:tasks.workunit.client.0.smithi022.stdout:816 2024-09-21T09:47:41.077 INFO:tasks.workunit.client.0.smithi022.stdout:817 2024-09-21T09:47:41.078 INFO:tasks.workunit.client.0.smithi022.stdout:818 2024-09-21T09:47:41.080 INFO:tasks.workunit.client.0.smithi022.stdout:819 2024-09-21T09:47:41.081 INFO:tasks.workunit.client.0.smithi022.stdout:820 2024-09-21T09:47:41.082 INFO:tasks.workunit.client.0.smithi022.stdout:821 2024-09-21T09:47:41.083 INFO:tasks.workunit.client.0.smithi022.stdout:822 2024-09-21T09:47:41.084 INFO:tasks.workunit.client.0.smithi022.stdout:823 2024-09-21T09:47:41.085 INFO:tasks.workunit.client.0.smithi022.stdout:824 2024-09-21T09:47:41.086 INFO:tasks.workunit.client.0.smithi022.stdout:825 2024-09-21T09:47:41.087 INFO:tasks.workunit.client.0.smithi022.stdout:826 2024-09-21T09:47:41.089 INFO:tasks.workunit.client.0.smithi022.stdout:827 2024-09-21T09:47:41.090 INFO:tasks.workunit.client.0.smithi022.stdout:828 2024-09-21T09:47:41.091 INFO:tasks.workunit.client.0.smithi022.stdout:829 2024-09-21T09:47:41.092 INFO:tasks.workunit.client.0.smithi022.stdout:830 2024-09-21T09:47:41.093 INFO:tasks.workunit.client.0.smithi022.stdout:831 2024-09-21T09:47:41.094 INFO:tasks.workunit.client.0.smithi022.stdout:832 2024-09-21T09:47:41.095 INFO:tasks.workunit.client.0.smithi022.stdout:833 2024-09-21T09:47:41.096 INFO:tasks.workunit.client.0.smithi022.stdout:834 2024-09-21T09:47:41.097 INFO:tasks.workunit.client.0.smithi022.stdout:835 2024-09-21T09:47:41.099 INFO:tasks.workunit.client.0.smithi022.stdout:836 2024-09-21T09:47:41.100 INFO:tasks.workunit.client.0.smithi022.stdout:837 2024-09-21T09:47:41.101 INFO:tasks.workunit.client.0.smithi022.stdout:838 2024-09-21T09:47:41.102 INFO:tasks.workunit.client.0.smithi022.stdout:839 2024-09-21T09:47:41.103 INFO:tasks.workunit.client.0.smithi022.stdout:840 2024-09-21T09:47:41.104 INFO:tasks.workunit.client.0.smithi022.stdout:841 2024-09-21T09:47:41.105 INFO:tasks.workunit.client.0.smithi022.stdout:842 2024-09-21T09:47:41.107 INFO:tasks.workunit.client.0.smithi022.stdout:843 2024-09-21T09:47:41.108 INFO:tasks.workunit.client.0.smithi022.stdout:844 2024-09-21T09:47:41.109 INFO:tasks.workunit.client.0.smithi022.stdout:845 2024-09-21T09:47:41.110 INFO:tasks.workunit.client.0.smithi022.stdout:846 2024-09-21T09:47:41.111 INFO:tasks.workunit.client.0.smithi022.stdout:847 2024-09-21T09:47:41.112 INFO:tasks.workunit.client.0.smithi022.stdout:848 2024-09-21T09:47:41.113 INFO:tasks.workunit.client.0.smithi022.stdout:849 2024-09-21T09:47:41.114 INFO:tasks.workunit.client.0.smithi022.stdout:850 2024-09-21T09:47:41.115 INFO:tasks.workunit.client.0.smithi022.stdout:851 2024-09-21T09:47:41.117 INFO:tasks.workunit.client.0.smithi022.stdout:852 2024-09-21T09:47:41.118 INFO:tasks.workunit.client.0.smithi022.stdout:853 2024-09-21T09:47:41.119 INFO:tasks.workunit.client.0.smithi022.stdout:854 2024-09-21T09:47:41.120 INFO:tasks.workunit.client.0.smithi022.stdout:855 2024-09-21T09:47:41.121 INFO:tasks.workunit.client.0.smithi022.stdout:856 2024-09-21T09:47:41.122 INFO:tasks.workunit.client.0.smithi022.stdout:857 2024-09-21T09:47:41.123 INFO:tasks.workunit.client.0.smithi022.stdout:858 2024-09-21T09:47:41.124 INFO:tasks.workunit.client.0.smithi022.stdout:859 2024-09-21T09:47:41.126 INFO:tasks.workunit.client.0.smithi022.stdout:860 2024-09-21T09:47:41.127 INFO:tasks.workunit.client.0.smithi022.stdout:861 2024-09-21T09:47:41.128 INFO:tasks.workunit.client.0.smithi022.stdout:862 2024-09-21T09:47:41.129 INFO:tasks.workunit.client.0.smithi022.stdout:863 2024-09-21T09:47:41.130 INFO:tasks.workunit.client.0.smithi022.stdout:864 2024-09-21T09:47:41.131 INFO:tasks.workunit.client.0.smithi022.stdout:865 2024-09-21T09:47:41.132 INFO:tasks.workunit.client.0.smithi022.stdout:866 2024-09-21T09:47:41.133 INFO:tasks.workunit.client.0.smithi022.stdout:867 2024-09-21T09:47:41.135 INFO:tasks.workunit.client.0.smithi022.stdout:868 2024-09-21T09:47:41.136 INFO:tasks.workunit.client.0.smithi022.stdout:869 2024-09-21T09:47:41.137 INFO:tasks.workunit.client.0.smithi022.stdout:870 2024-09-21T09:47:41.138 INFO:tasks.workunit.client.0.smithi022.stdout:871 2024-09-21T09:47:41.139 INFO:tasks.workunit.client.0.smithi022.stdout:872 2024-09-21T09:47:41.140 INFO:tasks.workunit.client.0.smithi022.stdout:873 2024-09-21T09:47:41.141 INFO:tasks.workunit.client.0.smithi022.stdout:874 2024-09-21T09:47:41.142 INFO:tasks.workunit.client.0.smithi022.stdout:875 2024-09-21T09:47:41.143 INFO:tasks.workunit.client.0.smithi022.stdout:876 2024-09-21T09:47:41.144 INFO:tasks.workunit.client.0.smithi022.stdout:877 2024-09-21T09:47:41.145 INFO:tasks.workunit.client.0.smithi022.stdout:878 2024-09-21T09:47:41.146 INFO:tasks.workunit.client.0.smithi022.stdout:879 2024-09-21T09:47:41.147 INFO:tasks.workunit.client.0.smithi022.stdout:880 2024-09-21T09:47:41.148 INFO:tasks.workunit.client.0.smithi022.stdout:881 2024-09-21T09:47:41.150 INFO:tasks.workunit.client.0.smithi022.stdout:882 2024-09-21T09:47:41.151 INFO:tasks.workunit.client.0.smithi022.stdout:883 2024-09-21T09:47:41.152 INFO:tasks.workunit.client.0.smithi022.stdout:884 2024-09-21T09:47:41.153 INFO:tasks.workunit.client.0.smithi022.stdout:885 2024-09-21T09:47:41.154 INFO:tasks.workunit.client.0.smithi022.stdout:886 2024-09-21T09:47:41.155 INFO:tasks.workunit.client.0.smithi022.stdout:887 2024-09-21T09:47:41.156 INFO:tasks.workunit.client.0.smithi022.stdout:888 2024-09-21T09:47:41.157 INFO:tasks.workunit.client.0.smithi022.stdout:889 2024-09-21T09:47:41.158 INFO:tasks.workunit.client.0.smithi022.stdout:890 2024-09-21T09:47:41.159 INFO:tasks.workunit.client.0.smithi022.stdout:891 2024-09-21T09:47:41.160 INFO:tasks.workunit.client.0.smithi022.stdout:892 2024-09-21T09:47:41.161 INFO:tasks.workunit.client.0.smithi022.stdout:893 2024-09-21T09:47:41.162 INFO:tasks.workunit.client.0.smithi022.stdout:894 2024-09-21T09:47:41.163 INFO:tasks.workunit.client.0.smithi022.stdout:895 2024-09-21T09:47:41.165 INFO:tasks.workunit.client.0.smithi022.stdout:896 2024-09-21T09:47:41.166 INFO:tasks.workunit.client.0.smithi022.stdout:897 2024-09-21T09:47:41.167 INFO:tasks.workunit.client.0.smithi022.stdout:898 2024-09-21T09:47:41.168 INFO:tasks.workunit.client.0.smithi022.stdout:899 2024-09-21T09:47:41.169 INFO:tasks.workunit.client.0.smithi022.stdout:900 2024-09-21T09:47:41.170 INFO:tasks.workunit.client.0.smithi022.stdout:901 2024-09-21T09:47:41.171 INFO:tasks.workunit.client.0.smithi022.stdout:902 2024-09-21T09:47:41.173 INFO:tasks.workunit.client.0.smithi022.stdout:903 2024-09-21T09:47:41.174 INFO:tasks.workunit.client.0.smithi022.stdout:904 2024-09-21T09:47:41.175 INFO:tasks.workunit.client.0.smithi022.stdout:905 2024-09-21T09:47:41.176 INFO:tasks.workunit.client.0.smithi022.stdout:906 2024-09-21T09:47:41.178 INFO:tasks.workunit.client.0.smithi022.stdout:907 2024-09-21T09:47:41.179 INFO:tasks.workunit.client.0.smithi022.stdout:908 2024-09-21T09:47:41.180 INFO:tasks.workunit.client.0.smithi022.stdout:909 2024-09-21T09:47:41.181 INFO:tasks.workunit.client.0.smithi022.stdout:910 2024-09-21T09:47:41.182 INFO:tasks.workunit.client.0.smithi022.stdout:911 2024-09-21T09:47:41.183 INFO:tasks.workunit.client.0.smithi022.stdout:912 2024-09-21T09:47:41.185 INFO:tasks.workunit.client.0.smithi022.stdout:913 2024-09-21T09:47:41.186 INFO:tasks.workunit.client.0.smithi022.stdout:914 2024-09-21T09:47:41.187 INFO:tasks.workunit.client.0.smithi022.stdout:915 2024-09-21T09:47:41.189 INFO:tasks.workunit.client.0.smithi022.stdout:916 2024-09-21T09:47:41.190 INFO:tasks.workunit.client.0.smithi022.stdout:917 2024-09-21T09:47:41.192 INFO:tasks.workunit.client.0.smithi022.stdout:918 2024-09-21T09:47:41.193 INFO:tasks.workunit.client.0.smithi022.stdout:919 2024-09-21T09:47:41.194 INFO:tasks.workunit.client.0.smithi022.stdout:920 2024-09-21T09:47:41.195 INFO:tasks.workunit.client.0.smithi022.stdout:921 2024-09-21T09:47:41.196 INFO:tasks.workunit.client.0.smithi022.stdout:922 2024-09-21T09:47:41.197 INFO:tasks.workunit.client.0.smithi022.stdout:923 2024-09-21T09:47:41.198 INFO:tasks.workunit.client.0.smithi022.stdout:924 2024-09-21T09:47:41.199 INFO:tasks.workunit.client.0.smithi022.stdout:925 2024-09-21T09:47:41.200 INFO:tasks.workunit.client.0.smithi022.stdout:926 2024-09-21T09:47:41.201 INFO:tasks.workunit.client.0.smithi022.stdout:927 2024-09-21T09:47:41.203 INFO:tasks.workunit.client.0.smithi022.stdout:928 2024-09-21T09:47:41.204 INFO:tasks.workunit.client.0.smithi022.stdout:929 2024-09-21T09:47:41.205 INFO:tasks.workunit.client.0.smithi022.stdout:930 2024-09-21T09:47:41.206 INFO:tasks.workunit.client.0.smithi022.stdout:931 2024-09-21T09:47:41.207 INFO:tasks.workunit.client.0.smithi022.stdout:932 2024-09-21T09:47:41.208 INFO:tasks.workunit.client.0.smithi022.stdout:933 2024-09-21T09:47:41.209 INFO:tasks.workunit.client.0.smithi022.stdout:934 2024-09-21T09:47:41.210 INFO:tasks.workunit.client.0.smithi022.stdout:935 2024-09-21T09:47:41.211 INFO:tasks.workunit.client.0.smithi022.stdout:936 2024-09-21T09:47:41.212 INFO:tasks.workunit.client.0.smithi022.stdout:937 2024-09-21T09:47:41.213 INFO:tasks.workunit.client.0.smithi022.stdout:938 2024-09-21T09:47:41.215 INFO:tasks.workunit.client.0.smithi022.stdout:939 2024-09-21T09:47:41.216 INFO:tasks.workunit.client.0.smithi022.stdout:940 2024-09-21T09:47:41.217 INFO:tasks.workunit.client.0.smithi022.stdout:941 2024-09-21T09:47:41.218 INFO:tasks.workunit.client.0.smithi022.stdout:942 2024-09-21T09:47:41.219 INFO:tasks.workunit.client.0.smithi022.stdout:943 2024-09-21T09:47:41.220 INFO:tasks.workunit.client.0.smithi022.stdout:944 2024-09-21T09:47:41.221 INFO:tasks.workunit.client.0.smithi022.stdout:945 2024-09-21T09:47:41.223 INFO:tasks.workunit.client.0.smithi022.stdout:946 2024-09-21T09:47:41.224 INFO:tasks.workunit.client.0.smithi022.stdout:947 2024-09-21T09:47:41.225 INFO:tasks.workunit.client.0.smithi022.stdout:948 2024-09-21T09:47:41.226 INFO:tasks.workunit.client.0.smithi022.stdout:949 2024-09-21T09:47:41.227 INFO:tasks.workunit.client.0.smithi022.stdout:950 2024-09-21T09:47:41.228 INFO:tasks.workunit.client.0.smithi022.stdout:951 2024-09-21T09:47:41.229 INFO:tasks.workunit.client.0.smithi022.stdout:952 2024-09-21T09:47:41.231 INFO:tasks.workunit.client.0.smithi022.stdout:953 2024-09-21T09:47:41.232 INFO:tasks.workunit.client.0.smithi022.stdout:954 2024-09-21T09:47:41.233 INFO:tasks.workunit.client.0.smithi022.stdout:955 2024-09-21T09:47:41.234 INFO:tasks.workunit.client.0.smithi022.stdout:956 2024-09-21T09:47:41.236 INFO:tasks.workunit.client.0.smithi022.stdout:957 2024-09-21T09:47:41.237 INFO:tasks.workunit.client.0.smithi022.stdout:958 2024-09-21T09:47:41.239 INFO:tasks.workunit.client.0.smithi022.stdout:959 2024-09-21T09:47:41.242 INFO:tasks.workunit.client.0.smithi022.stdout:960 2024-09-21T09:47:41.243 INFO:tasks.workunit.client.0.smithi022.stdout:961 2024-09-21T09:47:41.245 INFO:tasks.workunit.client.0.smithi022.stdout:962 2024-09-21T09:47:41.246 INFO:tasks.workunit.client.0.smithi022.stdout:963 2024-09-21T09:47:41.247 INFO:tasks.workunit.client.0.smithi022.stdout:964 2024-09-21T09:47:41.248 INFO:tasks.workunit.client.0.smithi022.stdout:965 2024-09-21T09:47:41.249 INFO:tasks.workunit.client.0.smithi022.stdout:966 2024-09-21T09:47:41.250 INFO:tasks.workunit.client.0.smithi022.stdout:967 2024-09-21T09:47:41.252 INFO:tasks.workunit.client.0.smithi022.stdout:968 2024-09-21T09:47:41.253 INFO:tasks.workunit.client.0.smithi022.stdout:969 2024-09-21T09:47:41.254 INFO:tasks.workunit.client.0.smithi022.stdout:970 2024-09-21T09:47:41.255 INFO:tasks.workunit.client.0.smithi022.stdout:971 2024-09-21T09:47:41.256 INFO:tasks.workunit.client.0.smithi022.stdout:972 2024-09-21T09:47:41.257 INFO:tasks.workunit.client.0.smithi022.stdout:973 2024-09-21T09:47:41.259 INFO:tasks.workunit.client.0.smithi022.stdout:974 2024-09-21T09:47:41.260 INFO:tasks.workunit.client.0.smithi022.stdout:975 2024-09-21T09:47:41.261 INFO:tasks.workunit.client.0.smithi022.stdout:976 2024-09-21T09:47:41.262 INFO:tasks.workunit.client.0.smithi022.stdout:977 2024-09-21T09:47:41.263 INFO:tasks.workunit.client.0.smithi022.stdout:978 2024-09-21T09:47:41.265 INFO:tasks.workunit.client.0.smithi022.stdout:979 2024-09-21T09:47:41.266 INFO:tasks.workunit.client.0.smithi022.stdout:980 2024-09-21T09:47:41.267 INFO:tasks.workunit.client.0.smithi022.stdout:981 2024-09-21T09:47:41.268 INFO:tasks.workunit.client.0.smithi022.stdout:982 2024-09-21T09:47:41.269 INFO:tasks.workunit.client.0.smithi022.stdout:983 2024-09-21T09:47:41.270 INFO:tasks.workunit.client.0.smithi022.stdout:984 2024-09-21T09:47:41.272 INFO:tasks.workunit.client.0.smithi022.stdout:985 2024-09-21T09:47:41.273 INFO:tasks.workunit.client.0.smithi022.stdout:986 2024-09-21T09:47:41.274 INFO:tasks.workunit.client.0.smithi022.stdout:987 2024-09-21T09:47:41.275 INFO:tasks.workunit.client.0.smithi022.stdout:988 2024-09-21T09:47:41.276 INFO:tasks.workunit.client.0.smithi022.stdout:989 2024-09-21T09:47:41.277 INFO:tasks.workunit.client.0.smithi022.stdout:990 2024-09-21T09:47:41.279 INFO:tasks.workunit.client.0.smithi022.stdout:991 2024-09-21T09:47:41.280 INFO:tasks.workunit.client.0.smithi022.stdout:992 2024-09-21T09:47:41.281 INFO:tasks.workunit.client.0.smithi022.stdout:993 2024-09-21T09:47:41.282 INFO:tasks.workunit.client.0.smithi022.stdout:994 2024-09-21T09:47:41.283 INFO:tasks.workunit.client.0.smithi022.stdout:995 2024-09-21T09:47:41.284 INFO:tasks.workunit.client.0.smithi022.stdout:996 2024-09-21T09:47:41.285 INFO:tasks.workunit.client.0.smithi022.stdout:997 2024-09-21T09:47:41.287 INFO:tasks.workunit.client.0.smithi022.stdout:998 2024-09-21T09:47:41.288 INFO:tasks.workunit.client.0.smithi022.stdout:999 2024-09-21T09:47:41.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:41 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/4054658720' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129492-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:47:41.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:41 smithi022 ceph-mon[98615]: osdmap e968: 8 total, 8 up, 8 in 2024-09-21T09:47:41.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:41 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/4054658720' entity='client.admin' cmd={"prefix": "osd dump","format":"json"} : dispatch 2024-09-21T09:47:41.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:41 smithi022 ceph-mon[98615]: pgmap v1091: 137 pgs: 12 creating+peering, 20 unknown, 105 active+clean; 583 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:47:41.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:41 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/4054658720' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129492-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:47:41.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:41 smithi022 ceph-mon[96736]: osdmap e968: 8 total, 8 up, 8 in 2024-09-21T09:47:41.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:41 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/4054658720' entity='client.admin' cmd={"prefix": "osd dump","format":"json"} : dispatch 2024-09-21T09:47:41.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:41 smithi022 ceph-mon[96736]: pgmap v1091: 137 pgs: 12 creating+peering, 20 unknown, 105 active+clean; 583 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:47:41.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:41 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/4054658720' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129492-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:47:41.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:41 smithi064 ceph-mon[86709]: osdmap e968: 8 total, 8 up, 8 in 2024-09-21T09:47:41.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:41 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/4054658720' entity='client.admin' cmd={"prefix": "osd dump","format":"json"} : dispatch 2024-09-21T09:47:41.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:41 smithi064 ceph-mon[86709]: pgmap v1091: 137 pgs: 12 creating+peering, 20 unknown, 105 active+clean; 583 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:47:42.251 INFO:tasks.workunit.client.0.smithi022.stdout:00000000 79 6f 75 20 6d 69 67 68 74 20 73 65 65 20 74 68 |you might see th| 2024-09-21T09:47:42.252 INFO:tasks.workunit.client.0.smithi022.stdout:00000010 69 73 |is| 2024-09-21T09:47:42.252 INFO:tasks.workunit.client.0.smithi022.stdout:00000012 2024-09-21T09:47:42.252 INFO:tasks.workunit.client.0.smithi022.stdout:00000000 79 6f 75 20 6d 69 67 68 74 20 73 65 65 20 74 68 |you might see th| 2024-09-21T09:47:42.252 INFO:tasks.workunit.client.0.smithi022.stdout:00000010 69 73 |is| 2024-09-21T09:47:42.252 INFO:tasks.workunit.client.0.smithi022.stdout:00000012 2024-09-21T09:47:42.252 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] ClsHello.WriteReturnData (4021 ms) 2024-09-21T09:47:42.252 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] ClsHello.Loud 2024-09-21T09:47:42.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:42 smithi022 ceph-mon[96736]: osdmap e969: 8 total, 8 up, 8 in 2024-09-21T09:47:42.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:42 smithi022 ceph-mon[98615]: osdmap e969: 8 total, 8 up, 8 in 2024-09-21T09:47:42.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:42 smithi064 ceph-mon[86709]: osdmap e969: 8 total, 8 up, 8 in 2024-09-21T09:47:43.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:43 smithi022 ceph-mon[98615]: osdmap e970: 8 total, 8 up, 8 in 2024-09-21T09:47:43.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:43 smithi022 ceph-mon[98615]: pgmap v1094: 105 pgs: 105 active+clean; 583 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:47:43.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:43 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:47:43.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:43 smithi022 ceph-mon[96736]: osdmap e970: 8 total, 8 up, 8 in 2024-09-21T09:47:43.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:43 smithi022 ceph-mon[96736]: pgmap v1094: 105 pgs: 105 active+clean; 583 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:47:43.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:43 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:47:43.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:43 smithi064 ceph-mon[86709]: osdmap e970: 8 total, 8 up, 8 in 2024-09-21T09:47:43.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:43 smithi064 ceph-mon[86709]: pgmap v1094: 105 pgs: 105 active+clean; 583 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:47:43.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:43 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:47:44.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:44 smithi022 ceph-mon[98615]: osdmap e971: 8 total, 8 up, 8 in 2024-09-21T09:47:44.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:44 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/3984744383' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129492-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:47:44.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:44 smithi022 ceph-mon[96736]: osdmap e971: 8 total, 8 up, 8 in 2024-09-21T09:47:44.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:44 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/3984744383' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129492-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:47:44.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:44 smithi064 ceph-mon[86709]: osdmap e971: 8 total, 8 up, 8 in 2024-09-21T09:47:44.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:44 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/3984744383' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129492-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:47:45.271 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] ClsHello.Loud (3024 ms) 2024-09-21T09:47:45.271 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] ClsHello.BadMethods 2024-09-21T09:47:45.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:45 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/3984744383' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129492-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:47:45.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:45 smithi022 ceph-mon[98615]: osdmap e972: 8 total, 8 up, 8 in 2024-09-21T09:47:45.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:45 smithi022 ceph-mon[98615]: pgmap v1097: 137 pgs: 4 creating+peering, 28 unknown, 105 active+clean; 583 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:47:45.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:45 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/3984744383' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129492-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:47:45.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:45 smithi022 ceph-mon[96736]: osdmap e972: 8 total, 8 up, 8 in 2024-09-21T09:47:45.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:45 smithi022 ceph-mon[96736]: pgmap v1097: 137 pgs: 4 creating+peering, 28 unknown, 105 active+clean; 583 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:47:45.694 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:45 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/3984744383' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129492-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:47:45.694 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:45 smithi064 ceph-mon[86709]: osdmap e972: 8 total, 8 up, 8 in 2024-09-21T09:47:45.694 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:45 smithi064 ceph-mon[86709]: pgmap v1097: 137 pgs: 4 creating+peering, 28 unknown, 105 active+clean; 583 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:47:46.058 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:47:45 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:47:45] "GET /metrics HTTP/1.1" 200 34078 "" "Prometheus/2.51.0" 2024-09-21T09:47:46.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:46 smithi022 ceph-mon[96736]: osdmap e973: 8 total, 8 up, 8 in 2024-09-21T09:47:46.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:46 smithi022 ceph-mon[98615]: osdmap e973: 8 total, 8 up, 8 in 2024-09-21T09:47:46.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:46 smithi064 ceph-mon[86709]: osdmap e973: 8 total, 8 up, 8 in 2024-09-21T09:47:47.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:47 smithi022 ceph-mon[98615]: osdmap e974: 8 total, 8 up, 8 in 2024-09-21T09:47:47.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:47 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/2601768118' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129492-5","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:47:47.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:47 smithi022 ceph-mon[98615]: pgmap v1100: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:47:47.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:47 smithi022 ceph-mon[96736]: osdmap e974: 8 total, 8 up, 8 in 2024-09-21T09:47:47.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:47 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/2601768118' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129492-5","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:47:47.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:47 smithi022 ceph-mon[96736]: pgmap v1100: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:47:47.559 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 21 09:47:47 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-3[117988]: 2024-09-21T09:47:47.297+0000 7fbfbebdb640 -1 osd.3 pg_epoch: 975 pg[38.2( v 975'1 (0'0,975'1] local-lis/les=974/975 n=1 ec=974/974 lis/c=974/974 les/c/f=975/975/0 sis=974) [3,7,5] r=0 lpr=974 crt=975'1 lcod 0'0 mlcod 0'0 active+clean] method hello.bad_reader tried to read object but is not marked RD 2024-09-21T09:47:47.559 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 21 09:47:47 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-3[117988]: 2024-09-21T09:47:47.299+0000 7fbfbebdb640 -1 osd.3 pg_epoch: 975 pg[38.2( v 975'2 (0'0,975'2] local-lis/les=974/975 n=1 ec=974/974 lis/c=974/974 les/c/f=975/975/0 sis=974) [3,7,5] r=0 lpr=974 crt=975'1 lcod 975'1 mlcod 975'1 active+clean] method hello.bad_writer tried to update object but is not marked WR 2024-09-21T09:47:47.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:47 smithi064 ceph-mon[86709]: osdmap e974: 8 total, 8 up, 8 in 2024-09-21T09:47:47.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:47 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/2601768118' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129492-5","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:47:47.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:47 smithi064 ceph-mon[86709]: pgmap v1100: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:47:48.290 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] ClsHello.BadMethods (3019 ms) 2024-09-21T09:47:48.290 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] ClsHello.Filter 2024-09-21T09:47:48.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:48 smithi022 ceph-mon[98615]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:47:48.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:48 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/2601768118' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129492-5","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:47:48.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:48 smithi022 ceph-mon[98615]: osdmap e975: 8 total, 8 up, 8 in 2024-09-21T09:47:48.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:48 smithi022 ceph-mon[96736]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:47:48.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:48 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/2601768118' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129492-5","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:47:48.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:48 smithi022 ceph-mon[96736]: osdmap e975: 8 total, 8 up, 8 in 2024-09-21T09:47:48.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:48 smithi064 ceph-mon[86709]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:47:48.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:48 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/2601768118' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129492-5","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:47:48.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:48 smithi064 ceph-mon[86709]: osdmap e975: 8 total, 8 up, 8 in 2024-09-21T09:47:49.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:49 smithi022 ceph-mon[98615]: osdmap e976: 8 total, 8 up, 8 in 2024-09-21T09:47:49.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:49 smithi022 ceph-mon[98615]: pgmap v1103: 105 pgs: 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:47:49.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:49 smithi022 ceph-mon[96736]: osdmap e976: 8 total, 8 up, 8 in 2024-09-21T09:47:49.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:49 smithi022 ceph-mon[96736]: pgmap v1103: 105 pgs: 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:47:49.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:49 smithi064 ceph-mon[86709]: osdmap e976: 8 total, 8 up, 8 in 2024-09-21T09:47:49.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:49 smithi064 ceph-mon[86709]: pgmap v1103: 105 pgs: 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:47:50.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:50 smithi022 ceph-mon[98615]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:47:50.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:50 smithi022 ceph-mon[98615]: Cluster is now healthy 2024-09-21T09:47:50.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:50 smithi022 ceph-mon[98615]: osdmap e977: 8 total, 8 up, 8 in 2024-09-21T09:47:50.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:50 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/3671983914' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129492-6","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:47:50.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:50 smithi022 ceph-mon[96736]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:47:50.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:50 smithi022 ceph-mon[96736]: Cluster is now healthy 2024-09-21T09:47:50.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:50 smithi022 ceph-mon[96736]: osdmap e977: 8 total, 8 up, 8 in 2024-09-21T09:47:50.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:50 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/3671983914' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129492-6","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:47:50.691 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:47:50 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:47:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:47:50.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:50 smithi064 ceph-mon[86709]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:47:50.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:50 smithi064 ceph-mon[86709]: Cluster is now healthy 2024-09-21T09:47:50.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:50 smithi064 ceph-mon[86709]: osdmap e977: 8 total, 8 up, 8 in 2024-09-21T09:47:50.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:50 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/3671983914' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129492-6","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:47:51.310 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] ClsHello.Filter (3020 ms) 2024-09-21T09:47:51.310 INFO:tasks.workunit.client.0.smithi022.stdout:[----------] 6 tests from ClsHello (18973 ms total) 2024-09-21T09:47:51.310 INFO:tasks.workunit.client.0.smithi022.stdout: 2024-09-21T09:47:51.310 INFO:tasks.workunit.client.0.smithi022.stdout:[----------] Global test environment tear-down 2024-09-21T09:47:51.310 INFO:tasks.workunit.client.0.smithi022.stdout:[==========] 6 tests from 1 test suite ran. (18973 ms total) 2024-09-21T09:47:51.310 INFO:tasks.workunit.client.0.smithi022.stdout:[ PASSED ] 6 tests. 2024-09-21T09:47:51.312 INFO:teuthology.orchestra.run:Running command with timeout 3600 2024-09-21T09:47:51.313 DEBUG:teuthology.orchestra.run.smithi022:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2024-09-21T09:47:51.382 INFO:tasks.workunit:Running workunit cls/test_cls_journal.sh... 2024-09-21T09:47:51.383 DEBUG:teuthology.orchestra.run.smithi022:workunit test cls/test_cls_journal.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_journal.sh 2024-09-21T09:47:51.457 INFO:tasks.workunit.client.0.smithi022.stdout:Running main() from gmock_main.cc 2024-09-21T09:47:51.457 INFO:tasks.workunit.client.0.smithi022.stdout:[==========] Running 27 tests from 1 test suite. 2024-09-21T09:47:51.457 INFO:tasks.workunit.client.0.smithi022.stdout:[----------] Global test environment set-up. 2024-09-21T09:47:51.457 INFO:tasks.workunit.client.0.smithi022.stdout:[----------] 27 tests from TestClsJournal 2024-09-21T09:47:51.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:51 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/3671983914' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129492-6","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:47:51.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:51 smithi022 ceph-mon[98615]: osdmap e978: 8 total, 8 up, 8 in 2024-09-21T09:47:51.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:51 smithi022 ceph-mon[98615]: pgmap v1106: 137 pgs: 7 creating+peering, 25 unknown, 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:47:51.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:51 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/3671983914' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129492-6","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:47:51.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:51 smithi022 ceph-mon[96736]: osdmap e978: 8 total, 8 up, 8 in 2024-09-21T09:47:51.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:51 smithi022 ceph-mon[96736]: pgmap v1106: 137 pgs: 7 creating+peering, 25 unknown, 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:47:51.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:51 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/3671983914' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129492-6","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:47:51.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:51 smithi064 ceph-mon[86709]: osdmap e978: 8 total, 8 up, 8 in 2024-09-21T09:47:51.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:51 smithi064 ceph-mon[86709]: pgmap v1106: 137 pgs: 7 creating+peering, 25 unknown, 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:47:52.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:52 smithi022 ceph-mon[96736]: osdmap e979: 8 total, 8 up, 8 in 2024-09-21T09:47:52.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:52 smithi022 ceph-mon[98615]: osdmap e979: 8 total, 8 up, 8 in 2024-09-21T09:47:52.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:52 smithi064 ceph-mon[86709]: osdmap e979: 8 total, 8 up, 8 in 2024-09-21T09:47:53.320 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestClsJournal.Create 2024-09-21T09:47:53.337 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestClsJournal.Create (17 ms) 2024-09-21T09:47:53.337 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestClsJournal.MinimumSet 2024-09-21T09:47:53.341 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestClsJournal.MinimumSet (5 ms) 2024-09-21T09:47:53.342 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestClsJournal.MinimumSetStale 2024-09-21T09:47:53.347 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestClsJournal.MinimumSetStale (5 ms) 2024-09-21T09:47:53.347 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestClsJournal.MinimumSetOrderConstraint 2024-09-21T09:47:53.350 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestClsJournal.MinimumSetOrderConstraint (3 ms) 2024-09-21T09:47:53.350 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestClsJournal.ActiveSet 2024-09-21T09:47:53.353 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestClsJournal.ActiveSet (3 ms) 2024-09-21T09:47:53.353 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestClsJournal.ActiveSetStale 2024-09-21T09:47:53.356 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestClsJournal.ActiveSetStale (3 ms) 2024-09-21T09:47:53.356 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestClsJournal.CreateDuplicate 2024-09-21T09:47:53.359 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestClsJournal.CreateDuplicate (3 ms) 2024-09-21T09:47:53.359 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestClsJournal.GetClient 2024-09-21T09:47:53.361 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestClsJournal.GetClient (3 ms) 2024-09-21T09:47:53.362 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestClsJournal.ClientRegister 2024-09-21T09:47:53.364 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestClsJournal.ClientRegister (2 ms) 2024-09-21T09:47:53.364 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestClsJournal.ClientRegisterDuplicate 2024-09-21T09:47:53.368 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestClsJournal.ClientRegisterDuplicate (4 ms) 2024-09-21T09:47:53.368 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestClsJournal.ClientUpdateData 2024-09-21T09:47:53.372 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestClsJournal.ClientUpdateData (4 ms) 2024-09-21T09:47:53.372 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestClsJournal.ClientUpdateState 2024-09-21T09:47:53.377 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestClsJournal.ClientUpdateState (5 ms) 2024-09-21T09:47:53.377 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestClsJournal.ClientUnregister 2024-09-21T09:47:53.380 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestClsJournal.ClientUnregister (3 ms) 2024-09-21T09:47:53.380 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestClsJournal.ClientUnregisterDNE 2024-09-21T09:47:53.385 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestClsJournal.ClientUnregisterDNE (5 ms) 2024-09-21T09:47:53.385 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestClsJournal.ClientUnregisterPruneTags 2024-09-21T09:47:53.493 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestClsJournal.ClientUnregisterPruneTags (108 ms) 2024-09-21T09:47:53.493 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestClsJournal.ClientCommit 2024-09-21T09:47:53.496 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestClsJournal.ClientCommit (3 ms) 2024-09-21T09:47:53.496 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestClsJournal.ClientCommitInvalid 2024-09-21T09:47:53.499 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestClsJournal.ClientCommitInvalid (3 ms) 2024-09-21T09:47:53.499 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestClsJournal.ClientCommitDNE 2024-09-21T09:47:53.500 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestClsJournal.ClientCommitDNE (1 ms) 2024-09-21T09:47:53.500 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestClsJournal.ClientList 2024-09-21T09:47:53.537 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestClsJournal.ClientList (37 ms) 2024-09-21T09:47:53.538 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestClsJournal.GetNextTagTid 2024-09-21T09:47:53.541 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestClsJournal.GetNextTagTid (5 ms) 2024-09-21T09:47:53.541 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestClsJournal.TagCreate 2024-09-21T09:47:53.552 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestClsJournal.TagCreate (10 ms) 2024-09-21T09:47:53.552 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestClsJournal.TagCreatePrunesTags 2024-09-21T09:47:53.559 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestClsJournal.TagCreatePrunesTags (7 ms) 2024-09-21T09:47:53.559 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestClsJournal.TagList 2024-09-21T09:47:53.657 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestClsJournal.TagList (98 ms) 2024-09-21T09:47:53.657 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestClsJournal.GuardAppend 2024-09-21T09:47:53.659 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestClsJournal.GuardAppend (2 ms) 2024-09-21T09:47:53.659 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestClsJournal.GuardAppendDNE 2024-09-21T09:47:53.660 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestClsJournal.GuardAppendDNE (1 ms) 2024-09-21T09:47:53.660 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestClsJournal.GuardAppendOverflow 2024-09-21T09:47:53.662 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestClsJournal.GuardAppendOverflow (2 ms) 2024-09-21T09:47:53.662 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestClsJournal.Append 2024-09-21T09:47:53.670 INFO:tasks.workunit.client.0.smithi022.stdout:padding is not enabled 2024-09-21T09:47:53.670 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestClsJournal.Append (8 ms) 2024-09-21T09:47:53.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:53 smithi064 ceph-mon[86709]: osdmap e980: 8 total, 8 up, 8 in 2024-09-21T09:47:53.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:53 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/501293563' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129632-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:47:53.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:53 smithi064 ceph-mon[86709]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129632-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:47:53.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:53 smithi064 ceph-mon[86709]: pgmap v1109: 137 pgs: 6 creating+peering, 26 unknown, 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:47:53.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:53 smithi022 ceph-mon[96736]: osdmap e980: 8 total, 8 up, 8 in 2024-09-21T09:47:53.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:53 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/501293563' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129632-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:47:53.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:53 smithi022 ceph-mon[96736]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129632-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:47:53.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:53 smithi022 ceph-mon[96736]: pgmap v1109: 137 pgs: 6 creating+peering, 26 unknown, 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:47:53.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:53 smithi022 ceph-mon[98615]: osdmap e980: 8 total, 8 up, 8 in 2024-09-21T09:47:53.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:53 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/501293563' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129632-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:47:53.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:53 smithi022 ceph-mon[98615]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129632-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:47:53.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:53 smithi022 ceph-mon[98615]: pgmap v1109: 137 pgs: 6 creating+peering, 26 unknown, 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:47:54.325 INFO:tasks.workunit.client.0.smithi022.stdout:[----------] 27 tests from TestClsJournal (350 ms total) 2024-09-21T09:47:54.325 INFO:tasks.workunit.client.0.smithi022.stdout: 2024-09-21T09:47:54.326 INFO:tasks.workunit.client.0.smithi022.stdout:[----------] Global test environment tear-down 2024-09-21T09:47:54.326 INFO:tasks.workunit.client.0.smithi022.stdout:[==========] 27 tests from 1 test suite ran. (2868 ms total) 2024-09-21T09:47:54.326 INFO:tasks.workunit.client.0.smithi022.stdout:[ PASSED ] 27 tests. 2024-09-21T09:47:54.328 INFO:teuthology.orchestra.run:Running command with timeout 3600 2024-09-21T09:47:54.328 DEBUG:teuthology.orchestra.run.smithi022:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2024-09-21T09:47:54.397 INFO:tasks.workunit:Running workunit cls/test_cls_lock.sh... 2024-09-21T09:47:54.398 DEBUG:teuthology.orchestra.run.smithi022:workunit test cls/test_cls_lock.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_lock.sh 2024-09-21T09:47:54.467 INFO:tasks.workunit.client.0.smithi022.stdout:Running main() from gmock_main.cc 2024-09-21T09:47:54.467 INFO:tasks.workunit.client.0.smithi022.stdout:[==========] Running 11 tests from 1 test suite. 2024-09-21T09:47:54.467 INFO:tasks.workunit.client.0.smithi022.stdout:[----------] Global test environment set-up. 2024-09-21T09:47:54.467 INFO:tasks.workunit.client.0.smithi022.stdout:[----------] 11 tests from ClsLock 2024-09-21T09:47:54.467 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] ClsLock.TestMultiLocking 2024-09-21T09:47:54.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:54 smithi064 ceph-mon[86709]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:47:54.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:54 smithi064 ceph-mon[86709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129632-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:47:54.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:54 smithi064 ceph-mon[86709]: osdmap e981: 8 total, 8 up, 8 in 2024-09-21T09:47:54.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:54 smithi022 ceph-mon[96736]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:47:54.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:54 smithi022 ceph-mon[96736]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129632-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:47:54.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:54 smithi022 ceph-mon[96736]: osdmap e981: 8 total, 8 up, 8 in 2024-09-21T09:47:54.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:54 smithi022 ceph-mon[98615]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:47:54.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:54 smithi022 ceph-mon[98615]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129632-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:47:54.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:54 smithi022 ceph-mon[98615]: osdmap e981: 8 total, 8 up, 8 in 2024-09-21T09:47:55.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:55 smithi064 ceph-mon[86709]: osdmap e982: 8 total, 8 up, 8 in 2024-09-21T09:47:55.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:55 smithi064 ceph-mon[86709]: pgmap v1112: 105 pgs: 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:47:55.709 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:55 smithi022 ceph-mon[98615]: osdmap e982: 8 total, 8 up, 8 in 2024-09-21T09:47:55.709 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:55 smithi022 ceph-mon[98615]: pgmap v1112: 105 pgs: 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:47:55.710 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:55 smithi022 ceph-mon[96736]: osdmap e982: 8 total, 8 up, 8 in 2024-09-21T09:47:55.710 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:55 smithi022 ceph-mon[96736]: pgmap v1112: 105 pgs: 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:47:56.058 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:47:55 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:47:55] "GET /metrics HTTP/1.1" 200 34078 "" "Prometheus/2.51.0" 2024-09-21T09:47:56.370 INFO:tasks.workunit.client.0.smithi022.stdout:lock: mylock 2024-09-21T09:47:56.370 INFO:tasks.workunit.client.0.smithi022.stdout: lock_type: exclusive 2024-09-21T09:47:56.370 INFO:tasks.workunit.client.0.smithi022.stdout: tag: 2024-09-21T09:47:56.370 INFO:tasks.workunit.client.0.smithi022.stdout: lockers: 2024-09-21T09:47:56.370 INFO:tasks.workunit.client.0.smithi022.stdout: client.44741 expiration=2024-09-21T09:49:56.365397+0000 addr=v1:172.21.15.22:0/55300247 cookie= 2024-09-21T09:47:56.374 INFO:tasks.workunit.client.0.smithi022.stdout:lock: mylock 2024-09-21T09:47:56.374 INFO:tasks.workunit.client.0.smithi022.stdout: lock_type: shared 2024-09-21T09:47:56.374 INFO:tasks.workunit.client.0.smithi022.stdout: tag: 2024-09-21T09:47:56.375 INFO:tasks.workunit.client.0.smithi022.stdout: lockers: 2024-09-21T09:47:56.375 INFO:tasks.workunit.client.0.smithi022.stdout: client.44741 expiration=2024-09-21T09:49:56.372131+0000 addr=v1:172.21.15.22:0/55300247 cookie= 2024-09-21T09:47:56.375 INFO:tasks.workunit.client.0.smithi022.stdout: client.44747 expiration=0.000000 addr=v1:172.21.15.22:0/3541080382 cookie= 2024-09-21T09:47:56.376 INFO:tasks.workunit.client.0.smithi022.stdout:lock: mylock 2024-09-21T09:47:56.376 INFO:tasks.workunit.client.0.smithi022.stdout: lock_type: shared 2024-09-21T09:47:56.376 INFO:tasks.workunit.client.0.smithi022.stdout: tag: 2024-09-21T09:47:56.376 INFO:tasks.workunit.client.0.smithi022.stdout: lockers: 2024-09-21T09:47:56.376 INFO:tasks.workunit.client.0.smithi022.stdout: client.44747 expiration=0.000000 addr=v1:172.21.15.22:0/3541080382 cookie= 2024-09-21T09:47:56.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:56 smithi064 ceph-mon[86709]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:47:56.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:56 smithi064 ceph-mon[86709]: Cluster is now healthy 2024-09-21T09:47:56.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:56 smithi064 ceph-mon[86709]: osdmap e983: 8 total, 8 up, 8 in 2024-09-21T09:47:56.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:56 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/55300247' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129687-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:47:56.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:56 smithi064 ceph-mon[86709]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129687-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:47:56.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:56 smithi022 ceph-mon[98615]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:47:56.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:56 smithi022 ceph-mon[98615]: Cluster is now healthy 2024-09-21T09:47:56.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:56 smithi022 ceph-mon[98615]: osdmap e983: 8 total, 8 up, 8 in 2024-09-21T09:47:56.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:56 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/55300247' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129687-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:47:56.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:56 smithi022 ceph-mon[98615]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129687-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:47:56.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:56 smithi022 ceph-mon[96736]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:47:56.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:56 smithi022 ceph-mon[96736]: Cluster is now healthy 2024-09-21T09:47:56.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:56 smithi022 ceph-mon[96736]: osdmap e983: 8 total, 8 up, 8 in 2024-09-21T09:47:56.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:56 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/55300247' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129687-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:47:56.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:56 smithi022 ceph-mon[96736]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129687-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:47:57.352 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] ClsLock.TestMultiLocking (2885 ms) 2024-09-21T09:47:57.352 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] ClsLock.TestMeta 2024-09-21T09:47:57.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:57 smithi064 ceph-mon[86709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129687-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:47:57.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:57 smithi064 ceph-mon[86709]: osdmap e984: 8 total, 8 up, 8 in 2024-09-21T09:47:57.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:57 smithi064 ceph-mon[86709]: pgmap v1115: 137 pgs: 4 creating+peering, 28 unknown, 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:47:57.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:57 smithi022 ceph-mon[98615]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129687-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:47:57.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:57 smithi022 ceph-mon[98615]: osdmap e984: 8 total, 8 up, 8 in 2024-09-21T09:47:57.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:57 smithi022 ceph-mon[98615]: pgmap v1115: 137 pgs: 4 creating+peering, 28 unknown, 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:47:57.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:57 smithi022 ceph-mon[96736]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129687-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:47:57.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:57 smithi022 ceph-mon[96736]: osdmap e984: 8 total, 8 up, 8 in 2024-09-21T09:47:57.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:57 smithi022 ceph-mon[96736]: pgmap v1115: 137 pgs: 4 creating+peering, 28 unknown, 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:47:58.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:58 smithi064 ceph-mon[86709]: osdmap e985: 8 total, 8 up, 8 in 2024-09-21T09:47:58.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:58 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:47:58.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:58 smithi022 ceph-mon[98615]: osdmap e985: 8 total, 8 up, 8 in 2024-09-21T09:47:58.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:58 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:47:58.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:58 smithi022 ceph-mon[96736]: osdmap e985: 8 total, 8 up, 8 in 2024-09-21T09:47:58.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:58 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:47:59.396 INFO:tasks.workunit.client.0.smithi022.stdout:lock: mylock 2024-09-21T09:47:59.396 INFO:tasks.workunit.client.0.smithi022.stdout: lock_type: shared 2024-09-21T09:47:59.396 INFO:tasks.workunit.client.0.smithi022.stdout: tag: 2024-09-21T09:47:59.396 INFO:tasks.workunit.client.0.smithi022.stdout: lockers: 2024-09-21T09:47:59.396 INFO:tasks.workunit.client.0.smithi022.stdout: client.34830 expiration=0.000000 addr=v1:172.21.15.22:0/2386121969 cookie= 2024-09-21T09:47:59.398 INFO:tasks.workunit.client.0.smithi022.stdout:lock: mylock 2024-09-21T09:47:59.398 INFO:tasks.workunit.client.0.smithi022.stdout: lock_type: exclusive 2024-09-21T09:47:59.398 INFO:tasks.workunit.client.0.smithi022.stdout: tag: new_tag 2024-09-21T09:47:59.398 INFO:tasks.workunit.client.0.smithi022.stdout: lockers: 2024-09-21T09:47:59.398 INFO:tasks.workunit.client.0.smithi022.stdout: client.34824 expiration=0.000000 addr=v1:172.21.15.22:0/1358002192 cookie= 2024-09-21T09:47:59.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:59 smithi064 ceph-mon[86709]: osdmap e986: 8 total, 8 up, 8 in 2024-09-21T09:47:59.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:59 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/1358002192' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129687-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:47:59.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:47:59 smithi064 ceph-mon[86709]: pgmap v1118: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:47:59.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:59 smithi022 ceph-mon[98615]: osdmap e986: 8 total, 8 up, 8 in 2024-09-21T09:47:59.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:59 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/1358002192' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129687-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:47:59.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:47:59 smithi022 ceph-mon[98615]: pgmap v1118: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:47:59.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:59 smithi022 ceph-mon[96736]: osdmap e986: 8 total, 8 up, 8 in 2024-09-21T09:47:59.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:59 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/1358002192' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129687-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:47:59.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:47:59 smithi022 ceph-mon[96736]: pgmap v1118: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:48:00.367 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] ClsLock.TestMeta (3015 ms) 2024-09-21T09:48:00.368 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] ClsLock.TestCookie 2024-09-21T09:48:00.692 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:48:00 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:48:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:48:00.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:00 smithi064 ceph-mon[86709]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:48:00.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:00 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/1358002192' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129687-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:48:00.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:00 smithi064 ceph-mon[86709]: osdmap e987: 8 total, 8 up, 8 in 2024-09-21T09:48:00.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:00 smithi022 ceph-mon[98615]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:48:00.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:00 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/1358002192' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129687-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:48:00.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:00 smithi022 ceph-mon[98615]: osdmap e987: 8 total, 8 up, 8 in 2024-09-21T09:48:00.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:00 smithi022 ceph-mon[96736]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:48:00.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:00 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/1358002192' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129687-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:48:00.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:00 smithi022 ceph-mon[96736]: osdmap e987: 8 total, 8 up, 8 in 2024-09-21T09:48:01.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:01 smithi064 ceph-mon[86709]: osdmap e988: 8 total, 8 up, 8 in 2024-09-21T09:48:01.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:01 smithi064 ceph-mon[86709]: pgmap v1121: 105 pgs: 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:48:01.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:01 smithi022 ceph-mon[98615]: osdmap e988: 8 total, 8 up, 8 in 2024-09-21T09:48:01.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:01 smithi022 ceph-mon[98615]: pgmap v1121: 105 pgs: 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:48:01.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:01 smithi022 ceph-mon[96736]: osdmap e988: 8 total, 8 up, 8 in 2024-09-21T09:48:01.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:01 smithi022 ceph-mon[96736]: pgmap v1121: 105 pgs: 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:48:02.401 INFO:tasks.workunit.client.0.smithi022.stdout:lock: mylock 2024-09-21T09:48:02.401 INFO:tasks.workunit.client.0.smithi022.stdout: lock_type: exclusive 2024-09-21T09:48:02.401 INFO:tasks.workunit.client.0.smithi022.stdout: tag: 2024-09-21T09:48:02.402 INFO:tasks.workunit.client.0.smithi022.stdout: lockers: 2024-09-21T09:48:02.404 INFO:tasks.workunit.client.0.smithi022.stdout:lock: mylock 2024-09-21T09:48:02.404 INFO:tasks.workunit.client.0.smithi022.stdout: lock_type: shared 2024-09-21T09:48:02.404 INFO:tasks.workunit.client.0.smithi022.stdout: tag: 2024-09-21T09:48:02.404 INFO:tasks.workunit.client.0.smithi022.stdout: lockers: 2024-09-21T09:48:02.404 INFO:tasks.workunit.client.0.smithi022.stdout: client.34836 expiration=0.000000 addr=v1:172.21.15.22:0/1426964963 cookie= 2024-09-21T09:48:02.404 INFO:tasks.workunit.client.0.smithi022.stdout: client.34836 expiration=0.000000 addr=v1:172.21.15.22:0/1426964963 cookie=new cookie 2024-09-21T09:48:02.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:02 smithi064 ceph-mon[86709]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:48:02.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:02 smithi064 ceph-mon[86709]: Cluster is now healthy 2024-09-21T09:48:02.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:02 smithi064 ceph-mon[86709]: osdmap e989: 8 total, 8 up, 8 in 2024-09-21T09:48:02.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:02 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/1426964963' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129687-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:48:02.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:02 smithi022 ceph-mon[98615]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:48:02.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:02 smithi022 ceph-mon[98615]: Cluster is now healthy 2024-09-21T09:48:02.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:02 smithi022 ceph-mon[98615]: osdmap e989: 8 total, 8 up, 8 in 2024-09-21T09:48:02.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:02 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/1426964963' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129687-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:48:02.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:02 smithi022 ceph-mon[96736]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:48:02.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:02 smithi022 ceph-mon[96736]: Cluster is now healthy 2024-09-21T09:48:02.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:02 smithi022 ceph-mon[96736]: osdmap e989: 8 total, 8 up, 8 in 2024-09-21T09:48:02.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:02 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/1426964963' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129687-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:48:03.393 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] ClsLock.TestCookie (3026 ms) 2024-09-21T09:48:03.394 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] ClsLock.TestMultipleLocks 2024-09-21T09:48:03.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:03 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/1426964963' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129687-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:48:03.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:03 smithi064 ceph-mon[86709]: osdmap e990: 8 total, 8 up, 8 in 2024-09-21T09:48:03.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:03 smithi064 ceph-mon[86709]: pgmap v1124: 137 pgs: 1 active, 3 creating+activating, 27 unknown, 106 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:48:03.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:03 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/1426964963' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129687-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:48:03.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:03 smithi022 ceph-mon[98615]: osdmap e990: 8 total, 8 up, 8 in 2024-09-21T09:48:03.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:03 smithi022 ceph-mon[98615]: pgmap v1124: 137 pgs: 1 active, 3 creating+activating, 27 unknown, 106 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:48:03.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:03 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/1426964963' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129687-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:48:03.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:03 smithi022 ceph-mon[96736]: osdmap e990: 8 total, 8 up, 8 in 2024-09-21T09:48:03.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:03 smithi022 ceph-mon[96736]: pgmap v1124: 137 pgs: 1 active, 3 creating+activating, 27 unknown, 106 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:48:04.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:04 smithi064 ceph-mon[86709]: osdmap e991: 8 total, 8 up, 8 in 2024-09-21T09:48:04.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:04 smithi022 ceph-mon[98615]: osdmap e991: 8 total, 8 up, 8 in 2024-09-21T09:48:04.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:04 smithi022 ceph-mon[96736]: osdmap e991: 8 total, 8 up, 8 in 2024-09-21T09:48:05.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:05 smithi064 ceph-mon[86709]: osdmap e992: 8 total, 8 up, 8 in 2024-09-21T09:48:05.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:05 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/2410287592' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129687-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:48:05.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:05 smithi064 ceph-mon[86709]: pgmap v1127: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:48:05.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:05 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:48:05.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:05 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:48:05.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:05 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:48:05.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:05 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:48:05.717 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:05 smithi022 ceph-mon[96736]: osdmap e992: 8 total, 8 up, 8 in 2024-09-21T09:48:05.717 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:05 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/2410287592' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129687-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:48:05.717 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:05 smithi022 ceph-mon[96736]: pgmap v1127: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:48:05.717 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:05 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:48:05.717 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:05 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:48:05.717 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:05 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:48:05.717 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:05 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:48:05.718 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:05 smithi022 ceph-mon[98615]: osdmap e992: 8 total, 8 up, 8 in 2024-09-21T09:48:05.718 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:05 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/2410287592' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129687-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:48:05.718 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:05 smithi022 ceph-mon[98615]: pgmap v1127: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:48:05.718 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:05 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:48:05.718 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:05 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:48:05.718 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:05 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:48:05.718 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:05 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:48:06.058 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:48:05 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:48:05] "GET /metrics HTTP/1.1" 200 34077 "" "Prometheus/2.51.0" 2024-09-21T09:48:06.411 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] ClsLock.TestMultipleLocks (3018 ms) 2024-09-21T09:48:06.411 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] ClsLock.TestLockDuration 2024-09-21T09:48:06.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:06 smithi064 ceph-mon[86709]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:48:06.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:06 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/2410287592' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129687-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:48:06.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:06 smithi064 ceph-mon[86709]: osdmap e993: 8 total, 8 up, 8 in 2024-09-21T09:48:06.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:06 smithi022 ceph-mon[98615]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:48:06.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:06 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/2410287592' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129687-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:48:06.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:06 smithi022 ceph-mon[98615]: osdmap e993: 8 total, 8 up, 8 in 2024-09-21T09:48:06.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:06 smithi022 ceph-mon[96736]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:48:06.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:06 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/2410287592' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129687-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:48:06.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:06 smithi022 ceph-mon[96736]: osdmap e993: 8 total, 8 up, 8 in 2024-09-21T09:48:07.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:07 smithi064 ceph-mon[86709]: osdmap e994: 8 total, 8 up, 8 in 2024-09-21T09:48:07.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:07 smithi064 ceph-mon[86709]: pgmap v1130: 105 pgs: 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:48:07.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:07 smithi022 ceph-mon[98615]: osdmap e994: 8 total, 8 up, 8 in 2024-09-21T09:48:07.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:07 smithi022 ceph-mon[98615]: pgmap v1130: 105 pgs: 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:48:07.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:07 smithi022 ceph-mon[96736]: osdmap e994: 8 total, 8 up, 8 in 2024-09-21T09:48:07.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:07 smithi022 ceph-mon[96736]: pgmap v1130: 105 pgs: 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:48:08.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:08 smithi064 ceph-mon[86709]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:48:08.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:08 smithi064 ceph-mon[86709]: Cluster is now healthy 2024-09-21T09:48:08.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:08 smithi064 ceph-mon[86709]: osdmap e995: 8 total, 8 up, 8 in 2024-09-21T09:48:08.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:08 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/4216511509' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129687-5","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:48:08.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:08 smithi022 ceph-mon[98615]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:48:08.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:08 smithi022 ceph-mon[98615]: Cluster is now healthy 2024-09-21T09:48:08.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:08 smithi022 ceph-mon[98615]: osdmap e995: 8 total, 8 up, 8 in 2024-09-21T09:48:08.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:08 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/4216511509' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129687-5","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:48:08.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:08 smithi022 ceph-mon[96736]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:48:08.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:08 smithi022 ceph-mon[96736]: Cluster is now healthy 2024-09-21T09:48:08.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:08 smithi022 ceph-mon[96736]: osdmap e995: 8 total, 8 up, 8 in 2024-09-21T09:48:08.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:08 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/4216511509' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129687-5","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:48:09.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:09 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/4216511509' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129687-5","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:48:09.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:09 smithi064 ceph-mon[86709]: osdmap e996: 8 total, 8 up, 8 in 2024-09-21T09:48:09.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:09 smithi064 ceph-mon[86709]: pgmap v1133: 137 pgs: 11 creating+peering, 21 unknown, 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:48:09.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:09 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/4216511509' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129687-5","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:48:09.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:09 smithi022 ceph-mon[98615]: osdmap e996: 8 total, 8 up, 8 in 2024-09-21T09:48:09.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:09 smithi022 ceph-mon[98615]: pgmap v1133: 137 pgs: 11 creating+peering, 21 unknown, 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:48:09.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:09 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/4216511509' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129687-5","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:48:09.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:09 smithi022 ceph-mon[96736]: osdmap e996: 8 total, 8 up, 8 in 2024-09-21T09:48:09.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:09 smithi022 ceph-mon[96736]: pgmap v1133: 137 pgs: 11 creating+peering, 21 unknown, 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:48:10.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:10 smithi064 ceph-mon[86709]: osdmap e997: 8 total, 8 up, 8 in 2024-09-21T09:48:10.692 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:48:10 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:48:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:48:10.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:10 smithi022 ceph-mon[98615]: osdmap e997: 8 total, 8 up, 8 in 2024-09-21T09:48:10.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:10 smithi022 ceph-mon[96736]: osdmap e997: 8 total, 8 up, 8 in 2024-09-21T09:48:11.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:11 smithi064 ceph-mon[86709]: pgmap v1135: 137 pgs: 11 creating+peering, 8 unknown, 118 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:48:11.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:11 smithi022 ceph-mon[98615]: pgmap v1135: 137 pgs: 11 creating+peering, 8 unknown, 118 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:48:11.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:11 smithi022 ceph-mon[96736]: pgmap v1135: 137 pgs: 11 creating+peering, 8 unknown, 118 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:48:13.589 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] ClsLock.TestLockDuration (7178 ms) 2024-09-21T09:48:13.589 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] ClsLock.TestAssertLocked 2024-09-21T09:48:13.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:13 smithi064 ceph-mon[86709]: pgmap v1136: 137 pgs: 11 creating+peering, 126 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:48:13.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:13 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:48:14.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:13 smithi022 ceph-mon[98615]: pgmap v1136: 137 pgs: 11 creating+peering, 126 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:48:14.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:13 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:48:14.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:13 smithi022 ceph-mon[96736]: pgmap v1136: 137 pgs: 11 creating+peering, 126 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:48:14.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:13 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:48:14.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:14 smithi064 ceph-mon[86709]: osdmap e998: 8 total, 8 up, 8 in 2024-09-21T09:48:15.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:14 smithi022 ceph-mon[98615]: osdmap e998: 8 total, 8 up, 8 in 2024-09-21T09:48:15.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:14 smithi022 ceph-mon[96736]: osdmap e998: 8 total, 8 up, 8 in 2024-09-21T09:48:15.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:15 smithi064 ceph-mon[86709]: pgmap v1138: 105 pgs: 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:48:15.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:15 smithi064 ceph-mon[86709]: osdmap e999: 8 total, 8 up, 8 in 2024-09-21T09:48:15.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:15 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/3480972433' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129687-6","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:48:15.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:15 smithi064 ceph-mon[86709]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129687-6","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:48:16.059 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:48:15 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:48:15] "GET /metrics HTTP/1.1" 200 36446 "" "Prometheus/2.51.0" 2024-09-21T09:48:16.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:15 smithi022 ceph-mon[96736]: pgmap v1138: 105 pgs: 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:48:16.060 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:15 smithi022 ceph-mon[96736]: osdmap e999: 8 total, 8 up, 8 in 2024-09-21T09:48:16.060 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:15 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/3480972433' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129687-6","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:48:16.060 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:15 smithi022 ceph-mon[96736]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129687-6","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:48:16.060 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:15 smithi022 ceph-mon[98615]: pgmap v1138: 105 pgs: 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:48:16.060 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:15 smithi022 ceph-mon[98615]: osdmap e999: 8 total, 8 up, 8 in 2024-09-21T09:48:16.060 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:15 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/3480972433' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129687-6","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:48:16.060 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:15 smithi022 ceph-mon[98615]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129687-6","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:48:16.617 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] ClsLock.TestAssertLocked (3028 ms) 2024-09-21T09:48:16.618 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] ClsLock.TestSetCookie 2024-09-21T09:48:16.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:16 smithi064 ceph-mon[86709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129687-6","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:48:16.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:16 smithi064 ceph-mon[86709]: osdmap e1000: 8 total, 8 up, 8 in 2024-09-21T09:48:17.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:16 smithi022 ceph-mon[98615]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129687-6","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:48:17.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:16 smithi022 ceph-mon[98615]: osdmap e1000: 8 total, 8 up, 8 in 2024-09-21T09:48:17.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:16 smithi022 ceph-mon[96736]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129687-6","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:48:17.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:16 smithi022 ceph-mon[96736]: osdmap e1000: 8 total, 8 up, 8 in 2024-09-21T09:48:17.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:17 smithi064 ceph-mon[86709]: pgmap v1141: 137 pgs: 13 creating+peering, 11 unknown, 113 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:48:17.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:17 smithi064 ceph-mon[86709]: osdmap e1001: 8 total, 8 up, 8 in 2024-09-21T09:48:18.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:17 smithi022 ceph-mon[98615]: pgmap v1141: 137 pgs: 13 creating+peering, 11 unknown, 113 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:48:18.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:17 smithi022 ceph-mon[98615]: osdmap e1001: 8 total, 8 up, 8 in 2024-09-21T09:48:18.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:17 smithi022 ceph-mon[96736]: pgmap v1141: 137 pgs: 13 creating+peering, 11 unknown, 113 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:48:18.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:17 smithi022 ceph-mon[96736]: osdmap e1001: 8 total, 8 up, 8 in 2024-09-21T09:48:18.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:18 smithi064 ceph-mon[86709]: osdmap e1002: 8 total, 8 up, 8 in 2024-09-21T09:48:18.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:18 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/679526470' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129687-7","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:48:18.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:18 smithi064 ceph-mon[86709]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129687-7","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:48:19.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:18 smithi022 ceph-mon[98615]: osdmap e1002: 8 total, 8 up, 8 in 2024-09-21T09:48:19.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:18 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/679526470' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129687-7","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:48:19.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:18 smithi022 ceph-mon[98615]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129687-7","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:48:19.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:18 smithi022 ceph-mon[96736]: osdmap e1002: 8 total, 8 up, 8 in 2024-09-21T09:48:19.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:18 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/679526470' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129687-7","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:48:19.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:18 smithi022 ceph-mon[96736]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129687-7","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:48:19.641 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] ClsLock.TestSetCookie (3024 ms) 2024-09-21T09:48:19.641 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] ClsLock.TestRenew 2024-09-21T09:48:19.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:19 smithi064 ceph-mon[86709]: pgmap v1144: 137 pgs: 10 creating+peering, 22 unknown, 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:48:19.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:19 smithi064 ceph-mon[86709]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:48:19.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:19 smithi064 ceph-mon[86709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129687-7","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:48:19.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:19 smithi064 ceph-mon[86709]: osdmap e1003: 8 total, 8 up, 8 in 2024-09-21T09:48:20.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:19 smithi022 ceph-mon[96736]: pgmap v1144: 137 pgs: 10 creating+peering, 22 unknown, 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:48:20.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:19 smithi022 ceph-mon[96736]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:48:20.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:19 smithi022 ceph-mon[96736]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129687-7","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:48:20.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:19 smithi022 ceph-mon[96736]: osdmap e1003: 8 total, 8 up, 8 in 2024-09-21T09:48:20.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:19 smithi022 ceph-mon[98615]: pgmap v1144: 137 pgs: 10 creating+peering, 22 unknown, 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:48:20.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:19 smithi022 ceph-mon[98615]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:48:20.059 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:19 smithi022 ceph-mon[98615]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129687-7","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:48:20.059 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:19 smithi022 ceph-mon[98615]: osdmap e1003: 8 total, 8 up, 8 in 2024-09-21T09:48:20.646 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:48:20 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:48:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:48:20.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:20 smithi064 ceph-mon[86709]: osdmap e1004: 8 total, 8 up, 8 in 2024-09-21T09:48:21.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:20 smithi022 ceph-mon[98615]: osdmap e1004: 8 total, 8 up, 8 in 2024-09-21T09:48:21.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:20 smithi022 ceph-mon[96736]: osdmap e1004: 8 total, 8 up, 8 in 2024-09-21T09:48:21.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:21 smithi064 ceph-mon[86709]: pgmap v1147: 105 pgs: 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:48:21.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:21 smithi064 ceph-mon[86709]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:48:21.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:21 smithi064 ceph-mon[86709]: Cluster is now healthy 2024-09-21T09:48:21.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:21 smithi064 ceph-mon[86709]: osdmap e1005: 8 total, 8 up, 8 in 2024-09-21T09:48:21.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:21 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/3371329538' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129687-8","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:48:22.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:21 smithi022 ceph-mon[98615]: pgmap v1147: 105 pgs: 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:48:22.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:21 smithi022 ceph-mon[98615]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:48:22.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:21 smithi022 ceph-mon[98615]: Cluster is now healthy 2024-09-21T09:48:22.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:21 smithi022 ceph-mon[98615]: osdmap e1005: 8 total, 8 up, 8 in 2024-09-21T09:48:22.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:21 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/3371329538' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129687-8","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:48:22.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:21 smithi022 ceph-mon[96736]: pgmap v1147: 105 pgs: 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:48:22.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:21 smithi022 ceph-mon[96736]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:48:22.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:21 smithi022 ceph-mon[96736]: Cluster is now healthy 2024-09-21T09:48:22.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:21 smithi022 ceph-mon[96736]: osdmap e1005: 8 total, 8 up, 8 in 2024-09-21T09:48:22.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:21 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/3371329538' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129687-8","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:48:22.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:22 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/3371329538' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129687-8","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:48:22.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:22 smithi064 ceph-mon[86709]: osdmap e1006: 8 total, 8 up, 8 in 2024-09-21T09:48:23.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:22 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/3371329538' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129687-8","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:48:23.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:22 smithi022 ceph-mon[98615]: osdmap e1006: 8 total, 8 up, 8 in 2024-09-21T09:48:23.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:22 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/3371329538' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129687-8","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:48:23.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:22 smithi022 ceph-mon[96736]: osdmap e1006: 8 total, 8 up, 8 in 2024-09-21T09:48:23.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:23 smithi064 ceph-mon[86709]: pgmap v1150: 137 pgs: 8 creating+peering, 21 unknown, 108 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:48:23.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:23 smithi064 ceph-mon[86709]: osdmap e1007: 8 total, 8 up, 8 in 2024-09-21T09:48:24.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:23 smithi022 ceph-mon[98615]: pgmap v1150: 137 pgs: 8 creating+peering, 21 unknown, 108 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:48:24.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:23 smithi022 ceph-mon[98615]: osdmap e1007: 8 total, 8 up, 8 in 2024-09-21T09:48:24.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:23 smithi022 ceph-mon[96736]: pgmap v1150: 137 pgs: 8 creating+peering, 21 unknown, 108 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:48:24.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:23 smithi022 ceph-mon[96736]: osdmap e1007: 8 total, 8 up, 8 in 2024-09-21T09:48:25.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:25 smithi064 ceph-mon[86709]: pgmap v1152: 137 pgs: 8 creating+peering, 9 unknown, 120 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-21T09:48:26.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:25 smithi022 ceph-mon[98615]: pgmap v1152: 137 pgs: 8 creating+peering, 9 unknown, 120 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-21T09:48:26.058 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:48:25 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:48:25] "GET /metrics HTTP/1.1" 200 36446 "" "Prometheus/2.51.0" 2024-09-21T09:48:26.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:25 smithi022 ceph-mon[96736]: pgmap v1152: 137 pgs: 8 creating+peering, 9 unknown, 120 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-21T09:48:27.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:27 smithi064 ceph-mon[86709]: pgmap v1153: 137 pgs: 137 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-21T09:48:28.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:27 smithi022 ceph-mon[98615]: pgmap v1153: 137 pgs: 137 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-21T09:48:28.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:27 smithi022 ceph-mon[96736]: pgmap v1153: 137 pgs: 137 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-21T09:48:28.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:28 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:48:29.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:28 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:48:29.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:28 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:48:29.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:29 smithi064 ceph-mon[86709]: pgmap v1154: 137 pgs: 137 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 129 B/s rd, 0 B/s wr, 0 op/s 2024-09-21T09:48:30.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:29 smithi022 ceph-mon[98615]: pgmap v1154: 137 pgs: 137 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 129 B/s rd, 0 B/s wr, 0 op/s 2024-09-21T09:48:30.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:29 smithi022 ceph-mon[96736]: pgmap v1154: 137 pgs: 137 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 129 B/s rd, 0 B/s wr, 0 op/s 2024-09-21T09:48:30.691 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:48:30 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:48:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:48:31.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:31 smithi064 ceph-mon[86709]: pgmap v1155: 137 pgs: 137 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 115 B/s rd, 0 B/s wr, 0 op/s 2024-09-21T09:48:32.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:31 smithi022 ceph-mon[98615]: pgmap v1155: 137 pgs: 137 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 115 B/s rd, 0 B/s wr, 0 op/s 2024-09-21T09:48:32.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:31 smithi022 ceph-mon[96736]: pgmap v1155: 137 pgs: 137 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 115 B/s rd, 0 B/s wr, 0 op/s 2024-09-21T09:48:33.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:33 smithi064 ceph-mon[86709]: pgmap v1156: 137 pgs: 137 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 102 B/s rd, 0 B/s wr, 0 op/s 2024-09-21T09:48:34.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:33 smithi022 ceph-mon[96736]: pgmap v1156: 137 pgs: 137 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 102 B/s rd, 0 B/s wr, 0 op/s 2024-09-21T09:48:34.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:33 smithi022 ceph-mon[98615]: pgmap v1156: 137 pgs: 137 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 102 B/s rd, 0 B/s wr, 0 op/s 2024-09-21T09:48:35.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:35 smithi064 ceph-mon[86709]: pgmap v1157: 137 pgs: 137 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 345 B/s rd, 0 B/s wr, 1 op/s 2024-09-21T09:48:36.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:35 smithi022 ceph-mon[98615]: pgmap v1157: 137 pgs: 137 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 345 B/s rd, 0 B/s wr, 1 op/s 2024-09-21T09:48:36.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:35 smithi022 ceph-mon[96736]: pgmap v1157: 137 pgs: 137 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 345 B/s rd, 0 B/s wr, 1 op/s 2024-09-21T09:48:36.058 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:48:35 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:48:35] "GET /metrics HTTP/1.1" 200 36445 "" "Prometheus/2.51.0" 2024-09-21T09:48:37.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:37 smithi064 ceph-mon[86709]: pgmap v1158: 137 pgs: 137 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 B/s wr, 0 op/s 2024-09-21T09:48:38.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:37 smithi022 ceph-mon[98615]: pgmap v1158: 137 pgs: 137 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 B/s wr, 0 op/s 2024-09-21T09:48:38.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:37 smithi022 ceph-mon[96736]: pgmap v1158: 137 pgs: 137 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 B/s wr, 0 op/s 2024-09-21T09:48:39.943 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:39 smithi064 ceph-mon[86709]: pgmap v1159: 137 pgs: 137 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 B/s wr, 0 op/s 2024-09-21T09:48:40.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:39 smithi022 ceph-mon[98615]: pgmap v1159: 137 pgs: 137 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 B/s wr, 0 op/s 2024-09-21T09:48:40.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:39 smithi022 ceph-mon[96736]: pgmap v1159: 137 pgs: 137 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 B/s wr, 0 op/s 2024-09-21T09:48:40.691 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:48:40 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:48:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:48:40.698 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] ClsLock.TestRenew (21057 ms) 2024-09-21T09:48:40.698 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] ClsLock.TestExclusiveEphemeralBasic 2024-09-21T09:48:42.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:41 smithi022 ceph-mon[98615]: pgmap v1160: 137 pgs: 137 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 0 B/s wr, 0 op/s 2024-09-21T09:48:42.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:41 smithi022 ceph-mon[98615]: osdmap e1008: 8 total, 8 up, 8 in 2024-09-21T09:48:42.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:41 smithi022 ceph-mon[96736]: pgmap v1160: 137 pgs: 137 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 0 B/s wr, 0 op/s 2024-09-21T09:48:42.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:41 smithi022 ceph-mon[96736]: osdmap e1008: 8 total, 8 up, 8 in 2024-09-21T09:48:42.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:41 smithi064 ceph-mon[86709]: pgmap v1160: 137 pgs: 137 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 0 B/s wr, 0 op/s 2024-09-21T09:48:42.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:41 smithi064 ceph-mon[86709]: osdmap e1008: 8 total, 8 up, 8 in 2024-09-21T09:48:43.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:42 smithi022 ceph-mon[98615]: osdmap e1009: 8 total, 8 up, 8 in 2024-09-21T09:48:43.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:42 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/1776955084' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129687-9","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:48:43.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:42 smithi022 ceph-mon[98615]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129687-9","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:48:43.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:42 smithi022 ceph-mon[98615]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129687-9","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:48:43.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:42 smithi022 ceph-mon[98615]: osdmap e1010: 8 total, 8 up, 8 in 2024-09-21T09:48:43.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:42 smithi022 ceph-mon[96736]: osdmap e1009: 8 total, 8 up, 8 in 2024-09-21T09:48:43.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:42 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/1776955084' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129687-9","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:48:43.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:42 smithi022 ceph-mon[96736]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129687-9","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:48:43.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:42 smithi022 ceph-mon[96736]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129687-9","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:48:43.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:42 smithi022 ceph-mon[96736]: osdmap e1010: 8 total, 8 up, 8 in 2024-09-21T09:48:43.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:42 smithi064 ceph-mon[86709]: osdmap e1009: 8 total, 8 up, 8 in 2024-09-21T09:48:43.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:42 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/1776955084' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129687-9","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:48:43.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:42 smithi064 ceph-mon[86709]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129687-9","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:48:43.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:42 smithi064 ceph-mon[86709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129687-9","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:48:43.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:42 smithi064 ceph-mon[86709]: osdmap e1010: 8 total, 8 up, 8 in 2024-09-21T09:48:44.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:43 smithi022 ceph-mon[98615]: pgmap v1163: 137 pgs: 5 creating+peering, 27 unknown, 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:48:44.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:43 smithi022 ceph-mon[98615]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:48:44.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:43 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:48:44.059 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:43 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:48:44.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:43 smithi022 ceph-mon[96736]: pgmap v1163: 137 pgs: 5 creating+peering, 27 unknown, 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:48:44.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:43 smithi022 ceph-mon[96736]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:48:44.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:43 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:48:44.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:43 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:48:44.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:43 smithi064 ceph-mon[86709]: pgmap v1163: 137 pgs: 5 creating+peering, 27 unknown, 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:48:44.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:43 smithi064 ceph-mon[86709]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:48:44.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:43 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:48:44.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:43 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:48:45.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:44 smithi022 ceph-mon[98615]: osdmap e1011: 8 total, 8 up, 8 in 2024-09-21T09:48:45.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:44 smithi022 ceph-mon[96736]: osdmap e1011: 8 total, 8 up, 8 in 2024-09-21T09:48:45.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:44 smithi064 ceph-mon[86709]: osdmap e1011: 8 total, 8 up, 8 in 2024-09-21T09:48:46.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:45 smithi022 ceph-mon[96736]: pgmap v1166: 137 pgs: 5 creating+peering, 20 unknown, 112 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:48:46.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:45 smithi022 ceph-mon[96736]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:48:46.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:45 smithi022 ceph-mon[96736]: Cluster is now healthy 2024-09-21T09:48:46.058 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:48:45 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:48:45] "GET /metrics HTTP/1.1" 200 36444 "" "Prometheus/2.51.0" 2024-09-21T09:48:46.059 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:45 smithi022 ceph-mon[98615]: pgmap v1166: 137 pgs: 5 creating+peering, 20 unknown, 112 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:48:46.059 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:45 smithi022 ceph-mon[98615]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:48:46.059 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:45 smithi022 ceph-mon[98615]: Cluster is now healthy 2024-09-21T09:48:46.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:45 smithi064 ceph-mon[86709]: pgmap v1166: 137 pgs: 5 creating+peering, 20 unknown, 112 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:48:46.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:45 smithi064 ceph-mon[86709]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:48:46.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:45 smithi064 ceph-mon[86709]: Cluster is now healthy 2024-09-21T09:48:47.732 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] ClsLock.TestExclusiveEphemeralBasic (7034 ms) 2024-09-21T09:48:47.732 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] ClsLock.TestExclusiveEphemeralStealEphemeral 2024-09-21T09:48:48.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:47 smithi022 ceph-mon[98615]: pgmap v1167: 137 pgs: 5 creating+peering, 132 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-21T09:48:48.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:47 smithi022 ceph-mon[96736]: pgmap v1167: 137 pgs: 5 creating+peering, 132 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-21T09:48:48.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:47 smithi064 ceph-mon[86709]: pgmap v1167: 137 pgs: 5 creating+peering, 132 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-21T09:48:49.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:48 smithi022 ceph-mon[96736]: osdmap e1012: 8 total, 8 up, 8 in 2024-09-21T09:48:49.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:48 smithi022 ceph-mon[98615]: osdmap e1012: 8 total, 8 up, 8 in 2024-09-21T09:48:49.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:48 smithi064 ceph-mon[86709]: osdmap e1012: 8 total, 8 up, 8 in 2024-09-21T09:48:50.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:49 smithi022 ceph-mon[98615]: pgmap v1169: 105 pgs: 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:48:50.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:49 smithi022 ceph-mon[98615]: osdmap e1013: 8 total, 8 up, 8 in 2024-09-21T09:48:50.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:49 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/2394968801' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129687-10","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:48:50.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:49 smithi022 ceph-mon[98615]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129687-10","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:48:50.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:49 smithi022 ceph-mon[96736]: pgmap v1169: 105 pgs: 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:48:50.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:49 smithi022 ceph-mon[96736]: osdmap e1013: 8 total, 8 up, 8 in 2024-09-21T09:48:50.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:49 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/2394968801' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129687-10","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:48:50.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:49 smithi022 ceph-mon[96736]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129687-10","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:48:50.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:49 smithi064 ceph-mon[86709]: pgmap v1169: 105 pgs: 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:48:50.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:49 smithi064 ceph-mon[86709]: osdmap e1013: 8 total, 8 up, 8 in 2024-09-21T09:48:50.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:49 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/2394968801' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129687-10","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:48:50.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:49 smithi064 ceph-mon[86709]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129687-10","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:48:50.691 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:48:50 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:48:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:48:51.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:50 smithi022 ceph-mon[98615]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129687-10","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:48:51.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:50 smithi022 ceph-mon[98615]: osdmap e1014: 8 total, 8 up, 8 in 2024-09-21T09:48:51.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:50 smithi022 ceph-mon[96736]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129687-10","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:48:51.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:50 smithi022 ceph-mon[96736]: osdmap e1014: 8 total, 8 up, 8 in 2024-09-21T09:48:51.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:50 smithi064 ceph-mon[86709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129687-10","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:48:51.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:50 smithi064 ceph-mon[86709]: osdmap e1014: 8 total, 8 up, 8 in 2024-09-21T09:48:52.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:51 smithi022 ceph-mon[98615]: pgmap v1172: 137 pgs: 21 unknown, 116 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:48:52.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:51 smithi022 ceph-mon[98615]: osdmap e1015: 8 total, 8 up, 8 in 2024-09-21T09:48:52.058 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:51 smithi022 ceph-mon[96736]: pgmap v1172: 137 pgs: 21 unknown, 116 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:48:52.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:51 smithi022 ceph-mon[96736]: osdmap e1015: 8 total, 8 up, 8 in 2024-09-21T09:48:52.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:51 smithi064 ceph-mon[86709]: pgmap v1172: 137 pgs: 21 unknown, 116 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:48:52.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:51 smithi064 ceph-mon[86709]: osdmap e1015: 8 total, 8 up, 8 in 2024-09-21T09:48:54.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:53 smithi022 ceph-mon[98615]: pgmap v1174: 137 pgs: 11 unknown, 126 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:48:54.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:53 smithi022 ceph-mon[96736]: pgmap v1174: 137 pgs: 11 unknown, 126 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:48:54.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:53 smithi064 ceph-mon[86709]: pgmap v1174: 137 pgs: 11 unknown, 126 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:48:55.183 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] ClsLock.TestExclusiveEphemeralStealEphemeral (7451 ms) 2024-09-21T09:48:55.183 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] ClsLock.TestExclusiveEphemeralStealExclusive 2024-09-21T09:48:55.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:55 smithi064 ceph-mon[86709]: pgmap v1175: 137 pgs: 137 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-21T09:48:55.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:55 smithi022 ceph-mon[98615]: pgmap v1175: 137 pgs: 137 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-21T09:48:55.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:55 smithi022 ceph-mon[96736]: pgmap v1175: 137 pgs: 137 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-21T09:48:56.057 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:48:55 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:48:55] "GET /metrics HTTP/1.1" 200 36444 "" "Prometheus/2.51.0" 2024-09-21T09:48:56.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:56 smithi064 ceph-mon[86709]: osdmap e1016: 8 total, 8 up, 8 in 2024-09-21T09:48:56.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:56 smithi022 ceph-mon[98615]: osdmap e1016: 8 total, 8 up, 8 in 2024-09-21T09:48:56.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:56 smithi022 ceph-mon[96736]: osdmap e1016: 8 total, 8 up, 8 in 2024-09-21T09:48:57.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:57 smithi022 ceph-mon[98615]: osdmap e1017: 8 total, 8 up, 8 in 2024-09-21T09:48:57.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:57 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/252928352' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129687-11","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:48:57.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:57 smithi022 ceph-mon[98615]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129687-11","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:48:57.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:57 smithi022 ceph-mon[98615]: pgmap v1178: 137 pgs: 6 creating+peering, 26 unknown, 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:48:57.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:57 smithi022 ceph-mon[96736]: osdmap e1017: 8 total, 8 up, 8 in 2024-09-21T09:48:57.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:57 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/252928352' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129687-11","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:48:57.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:57 smithi022 ceph-mon[96736]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129687-11","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:48:57.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:57 smithi022 ceph-mon[96736]: pgmap v1178: 137 pgs: 6 creating+peering, 26 unknown, 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:48:57.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:57 smithi064 ceph-mon[86709]: osdmap e1017: 8 total, 8 up, 8 in 2024-09-21T09:48:57.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:57 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/252928352' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129687-11","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:48:57.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:57 smithi064 ceph-mon[86709]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129687-11","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:48:57.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:57 smithi064 ceph-mon[86709]: pgmap v1178: 137 pgs: 6 creating+peering, 26 unknown, 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:48:58.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:58 smithi022 ceph-mon[98615]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:48:58.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:58 smithi022 ceph-mon[98615]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129687-11","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:48:58.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:58 smithi022 ceph-mon[98615]: osdmap e1018: 8 total, 8 up, 8 in 2024-09-21T09:48:58.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:58 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:48:58.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:58 smithi022 ceph-mon[96736]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:48:58.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:58 smithi022 ceph-mon[96736]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129687-11","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:48:58.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:58 smithi022 ceph-mon[96736]: osdmap e1018: 8 total, 8 up, 8 in 2024-09-21T09:48:58.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:58 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:48:58.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:58 smithi064 ceph-mon[86709]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:48:58.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:58 smithi064 ceph-mon[86709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-129687-11","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:48:58.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:58 smithi064 ceph-mon[86709]: osdmap e1018: 8 total, 8 up, 8 in 2024-09-21T09:48:58.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:58 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:48:59.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:59 smithi022 ceph-mon[98615]: osdmap e1019: 8 total, 8 up, 8 in 2024-09-21T09:48:59.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:48:59 smithi022 ceph-mon[98615]: pgmap v1181: 137 pgs: 6 creating+peering, 6 unknown, 125 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:48:59.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:59 smithi022 ceph-mon[96736]: osdmap e1019: 8 total, 8 up, 8 in 2024-09-21T09:48:59.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:48:59 smithi022 ceph-mon[96736]: pgmap v1181: 137 pgs: 6 creating+peering, 6 unknown, 125 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:48:59.694 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:59 smithi064 ceph-mon[86709]: osdmap e1019: 8 total, 8 up, 8 in 2024-09-21T09:48:59.694 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:48:59 smithi064 ceph-mon[86709]: pgmap v1181: 137 pgs: 6 creating+peering, 6 unknown, 125 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:49:00.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:00 smithi022 ceph-mon[98615]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:49:00.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:00 smithi022 ceph-mon[98615]: Cluster is now healthy 2024-09-21T09:49:00.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:00 smithi022 ceph-mon[96736]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:49:00.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:00 smithi022 ceph-mon[96736]: Cluster is now healthy 2024-09-21T09:49:00.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:00 smithi064 ceph-mon[86709]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:49:00.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:00 smithi064 ceph-mon[86709]: Cluster is now healthy 2024-09-21T09:49:00.692 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:49:00 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:49:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:49:01.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:01 smithi022 ceph-mon[98615]: pgmap v1182: 137 pgs: 6 creating+peering, 3 unknown, 128 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-21T09:49:01.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:01 smithi022 ceph-mon[96736]: pgmap v1182: 137 pgs: 6 creating+peering, 3 unknown, 128 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-21T09:49:01.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:01 smithi064 ceph-mon[86709]: pgmap v1182: 137 pgs: 6 creating+peering, 3 unknown, 128 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-21T09:49:02.277 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] ClsLock.TestExclusiveEphemeralStealExclusive (7094 ms) 2024-09-21T09:49:02.277 INFO:tasks.workunit.client.0.smithi022.stdout:[----------] 11 tests from ClsLock (67810 ms total) 2024-09-21T09:49:02.277 INFO:tasks.workunit.client.0.smithi022.stdout: 2024-09-21T09:49:02.277 INFO:tasks.workunit.client.0.smithi022.stdout:[----------] Global test environment tear-down 2024-09-21T09:49:02.277 INFO:tasks.workunit.client.0.smithi022.stdout:[==========] 11 tests from 1 test suite ran. (67810 ms total) 2024-09-21T09:49:02.277 INFO:tasks.workunit.client.0.smithi022.stdout:[ PASSED ] 11 tests. 2024-09-21T09:49:02.280 INFO:teuthology.orchestra.run:Running command with timeout 3600 2024-09-21T09:49:02.280 DEBUG:teuthology.orchestra.run.smithi022:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2024-09-21T09:49:02.353 INFO:tasks.workunit:Running workunit cls/test_cls_log.sh... 2024-09-21T09:49:02.353 DEBUG:teuthology.orchestra.run.smithi022:workunit test cls/test_cls_log.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_log.sh 2024-09-21T09:49:02.423 INFO:tasks.workunit.client.0.smithi022.stdout:Running main() from gmock_main.cc 2024-09-21T09:49:02.423 INFO:tasks.workunit.client.0.smithi022.stdout:[==========] Running 4 tests from 1 test suite. 2024-09-21T09:49:02.423 INFO:tasks.workunit.client.0.smithi022.stdout:[----------] Global test environment set-up. 2024-09-21T09:49:02.424 INFO:tasks.workunit.client.0.smithi022.stdout:[----------] 4 tests from cls_log 2024-09-21T09:49:02.424 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] cls_log.test_log_add_same_time 2024-09-21T09:49:03.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:03 smithi022 ceph-mon[98615]: osdmap e1020: 8 total, 8 up, 8 in 2024-09-21T09:49:03.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:03 smithi022 ceph-mon[98615]: pgmap v1184: 105 pgs: 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:49:03.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:03 smithi022 ceph-mon[96736]: osdmap e1020: 8 total, 8 up, 8 in 2024-09-21T09:49:03.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:03 smithi022 ceph-mon[96736]: pgmap v1184: 105 pgs: 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:49:03.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:03 smithi064 ceph-mon[86709]: osdmap e1020: 8 total, 8 up, 8 in 2024-09-21T09:49:03.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:03 smithi064 ceph-mon[86709]: pgmap v1184: 105 pgs: 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:49:04.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:04 smithi022 ceph-mon[98615]: osdmap e1021: 8 total, 8 up, 8 in 2024-09-21T09:49:04.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:04 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/3232475741' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-130038-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:49:04.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:04 smithi022 ceph-mon[98615]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-130038-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:49:04.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:04 smithi022 ceph-mon[96736]: osdmap e1021: 8 total, 8 up, 8 in 2024-09-21T09:49:04.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:04 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/3232475741' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-130038-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:49:04.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:04 smithi022 ceph-mon[96736]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-130038-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:49:04.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:04 smithi064 ceph-mon[86709]: osdmap e1021: 8 total, 8 up, 8 in 2024-09-21T09:49:04.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:04 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/3232475741' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-130038-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:49:04.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:04 smithi064 ceph-mon[86709]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-130038-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:49:05.294 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] cls_log.test_log_add_same_time (2871 ms) 2024-09-21T09:49:05.294 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] cls_log.test_log_add_different_time 2024-09-21T09:49:05.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:05 smithi022 ceph-mon[98615]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-130038-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:49:05.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:05 smithi022 ceph-mon[98615]: osdmap e1022: 8 total, 8 up, 8 in 2024-09-21T09:49:05.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:05 smithi022 ceph-mon[98615]: pgmap v1187: 137 pgs: 6 creating+peering, 26 unknown, 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:49:05.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:05 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:49:05.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:05 smithi022 ceph-mon[96736]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-130038-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:49:05.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:05 smithi022 ceph-mon[96736]: osdmap e1022: 8 total, 8 up, 8 in 2024-09-21T09:49:05.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:05 smithi022 ceph-mon[96736]: pgmap v1187: 137 pgs: 6 creating+peering, 26 unknown, 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:49:05.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:05 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:49:05.593 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:05 smithi064 ceph-mon[86709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-130038-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:49:05.593 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:05 smithi064 ceph-mon[86709]: osdmap e1022: 8 total, 8 up, 8 in 2024-09-21T09:49:05.593 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:05 smithi064 ceph-mon[86709]: pgmap v1187: 137 pgs: 6 creating+peering, 26 unknown, 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:49:05.593 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:05 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:49:06.058 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:49:05 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:49:05] "GET /metrics HTTP/1.1" 200 36443 "" "Prometheus/2.51.0" 2024-09-21T09:49:06.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:06 smithi022 ceph-mon[98615]: osdmap e1023: 8 total, 8 up, 8 in 2024-09-21T09:49:06.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:06 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:49:06.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:06 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:49:06.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:06 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:49:06.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:06 smithi022 ceph-mon[96736]: osdmap e1023: 8 total, 8 up, 8 in 2024-09-21T09:49:06.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:06 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:49:06.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:06 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:49:06.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:06 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:49:06.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:06 smithi064 ceph-mon[86709]: osdmap e1023: 8 total, 8 up, 8 in 2024-09-21T09:49:06.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:06 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:49:06.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:06 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:49:06.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:06 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:49:07.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:07 smithi022 ceph-mon[98615]: osdmap e1024: 8 total, 8 up, 8 in 2024-09-21T09:49:07.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:07 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/3942434310' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-130038-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:49:07.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:07 smithi022 ceph-mon[98615]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-130038-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:49:07.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:07 smithi022 ceph-mon[98615]: pgmap v1190: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:49:07.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:07 smithi022 ceph-mon[96736]: osdmap e1024: 8 total, 8 up, 8 in 2024-09-21T09:49:07.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:07 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/3942434310' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-130038-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:49:07.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:07 smithi022 ceph-mon[96736]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-130038-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:49:07.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:07 smithi022 ceph-mon[96736]: pgmap v1190: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:49:07.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:07 smithi064 ceph-mon[86709]: osdmap e1024: 8 total, 8 up, 8 in 2024-09-21T09:49:07.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:07 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/3942434310' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-130038-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:49:07.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:07 smithi064 ceph-mon[86709]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-130038-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:49:07.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:07 smithi064 ceph-mon[86709]: pgmap v1190: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:49:08.318 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] cls_log.test_log_add_different_time (3022 ms) 2024-09-21T09:49:08.318 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] cls_log.trim_by_time 2024-09-21T09:49:08.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:08 smithi022 ceph-mon[96736]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:49:08.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:08 smithi022 ceph-mon[96736]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-130038-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:49:08.559 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:08 smithi022 ceph-mon[96736]: osdmap e1025: 8 total, 8 up, 8 in 2024-09-21T09:49:08.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:08 smithi022 ceph-mon[98615]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:49:08.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:08 smithi022 ceph-mon[98615]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-130038-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:49:08.559 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:08 smithi022 ceph-mon[98615]: osdmap e1025: 8 total, 8 up, 8 in 2024-09-21T09:49:08.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:08 smithi064 ceph-mon[86709]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:49:08.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:08 smithi064 ceph-mon[86709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-130038-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:49:08.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:08 smithi064 ceph-mon[86709]: osdmap e1025: 8 total, 8 up, 8 in 2024-09-21T09:49:09.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:09 smithi064 ceph-mon[86709]: osdmap e1026: 8 total, 8 up, 8 in 2024-09-21T09:49:09.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:09 smithi064 ceph-mon[86709]: pgmap v1193: 105 pgs: 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:49:09.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:09 smithi022 ceph-mon[98615]: osdmap e1026: 8 total, 8 up, 8 in 2024-09-21T09:49:09.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:09 smithi022 ceph-mon[98615]: pgmap v1193: 105 pgs: 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:49:09.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:09 smithi022 ceph-mon[96736]: osdmap e1026: 8 total, 8 up, 8 in 2024-09-21T09:49:09.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:09 smithi022 ceph-mon[96736]: pgmap v1193: 105 pgs: 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:49:10.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:10 smithi064 ceph-mon[86709]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:49:10.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:10 smithi064 ceph-mon[86709]: Cluster is now healthy 2024-09-21T09:49:10.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:10 smithi064 ceph-mon[86709]: osdmap e1027: 8 total, 8 up, 8 in 2024-09-21T09:49:10.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:10 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/1038780338' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-130038-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:49:10.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:10 smithi064 ceph-mon[86709]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-130038-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:49:10.692 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:49:10 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:49:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:49:10.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:10 smithi022 ceph-mon[98615]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:49:10.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:10 smithi022 ceph-mon[98615]: Cluster is now healthy 2024-09-21T09:49:10.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:10 smithi022 ceph-mon[98615]: osdmap e1027: 8 total, 8 up, 8 in 2024-09-21T09:49:10.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:10 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/1038780338' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-130038-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:49:10.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:10 smithi022 ceph-mon[98615]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-130038-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:49:10.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:10 smithi022 ceph-mon[96736]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:49:10.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:10 smithi022 ceph-mon[96736]: Cluster is now healthy 2024-09-21T09:49:10.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:10 smithi022 ceph-mon[96736]: osdmap e1027: 8 total, 8 up, 8 in 2024-09-21T09:49:10.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:10 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/1038780338' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-130038-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:49:10.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:10 smithi022 ceph-mon[96736]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-130038-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:49:11.341 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] cls_log.trim_by_time (3025 ms) 2024-09-21T09:49:11.341 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] cls_log.trim_by_marker 2024-09-21T09:49:11.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:11 smithi064 ceph-mon[86709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-130038-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:49:11.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:11 smithi064 ceph-mon[86709]: osdmap e1028: 8 total, 8 up, 8 in 2024-09-21T09:49:11.693 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:11 smithi064 ceph-mon[86709]: pgmap v1196: 137 pgs: 6 creating+peering, 26 unknown, 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:49:11.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:11 smithi022 ceph-mon[96736]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-130038-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:49:11.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:11 smithi022 ceph-mon[96736]: osdmap e1028: 8 total, 8 up, 8 in 2024-09-21T09:49:11.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:11 smithi022 ceph-mon[96736]: pgmap v1196: 137 pgs: 6 creating+peering, 26 unknown, 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:49:11.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:11 smithi022 ceph-mon[98615]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-130038-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:49:11.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:11 smithi022 ceph-mon[98615]: osdmap e1028: 8 total, 8 up, 8 in 2024-09-21T09:49:11.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:11 smithi022 ceph-mon[98615]: pgmap v1196: 137 pgs: 6 creating+peering, 26 unknown, 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:49:12.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:12 smithi064 ceph-mon[86709]: osdmap e1029: 8 total, 8 up, 8 in 2024-09-21T09:49:12.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:12 smithi022 ceph-mon[98615]: osdmap e1029: 8 total, 8 up, 8 in 2024-09-21T09:49:12.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:12 smithi022 ceph-mon[96736]: osdmap e1029: 8 total, 8 up, 8 in 2024-09-21T09:49:13.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:13 smithi064 ceph-mon[86709]: osdmap e1030: 8 total, 8 up, 8 in 2024-09-21T09:49:13.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:13 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/639329249' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-130038-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:49:13.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:13 smithi064 ceph-mon[86709]: pgmap v1199: 137 pgs: 2 creating+peering, 30 unknown, 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:49:13.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:13 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:49:13.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:13 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:49:13.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:13 smithi022 ceph-mon[96736]: osdmap e1030: 8 total, 8 up, 8 in 2024-09-21T09:49:13.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:13 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/639329249' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-130038-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:49:13.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:13 smithi022 ceph-mon[96736]: pgmap v1199: 137 pgs: 2 creating+peering, 30 unknown, 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:49:13.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:13 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:49:13.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:13 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:49:13.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:13 smithi022 ceph-mon[98615]: osdmap e1030: 8 total, 8 up, 8 in 2024-09-21T09:49:13.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:13 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/639329249' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-130038-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:49:13.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:13 smithi022 ceph-mon[98615]: pgmap v1199: 137 pgs: 2 creating+peering, 30 unknown, 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:49:13.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:13 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' 2024-09-21T09:49:13.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:13 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:49:14.356 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] cls_log.trim_by_marker (3015 ms) 2024-09-21T09:49:14.357 INFO:tasks.workunit.client.0.smithi022.stdout:[----------] 4 tests from cls_log (11933 ms total) 2024-09-21T09:49:14.357 INFO:tasks.workunit.client.0.smithi022.stdout: 2024-09-21T09:49:14.357 INFO:tasks.workunit.client.0.smithi022.stdout:[----------] Global test environment tear-down 2024-09-21T09:49:14.357 INFO:tasks.workunit.client.0.smithi022.stdout:[==========] 4 tests from 1 test suite ran. (11933 ms total) 2024-09-21T09:49:14.357 INFO:tasks.workunit.client.0.smithi022.stdout:[ PASSED ] 4 tests. 2024-09-21T09:49:14.359 INFO:teuthology.orchestra.run:Running command with timeout 3600 2024-09-21T09:49:14.360 DEBUG:teuthology.orchestra.run.smithi022:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2024-09-21T09:49:14.430 INFO:tasks.workunit:Running workunit cls/test_cls_numops.sh... 2024-09-21T09:49:14.430 DEBUG:teuthology.orchestra.run.smithi022:workunit test cls/test_cls_numops.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_numops.sh 2024-09-21T09:49:14.502 INFO:tasks.workunit.client.0.smithi022.stdout:Running main() from gmock_main.cc 2024-09-21T09:49:14.502 INFO:tasks.workunit.client.0.smithi022.stdout:[==========] Running 4 tests from 1 test suite. 2024-09-21T09:49:14.503 INFO:tasks.workunit.client.0.smithi022.stdout:[----------] Global test environment set-up. 2024-09-21T09:49:14.503 INFO:tasks.workunit.client.0.smithi022.stdout:[----------] 4 tests from ClsNumOps 2024-09-21T09:49:14.503 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] ClsNumOps.Add 2024-09-21T09:49:14.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:14 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/639329249' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-130038-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:49:14.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:14 smithi064 ceph-mon[86709]: osdmap e1031: 8 total, 8 up, 8 in 2024-09-21T09:49:14.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:14 smithi064 ceph-mon[86709]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:49:14.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:14 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/639329249' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-130038-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:49:14.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:14 smithi022 ceph-mon[98615]: osdmap e1031: 8 total, 8 up, 8 in 2024-09-21T09:49:14.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:14 smithi022 ceph-mon[98615]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:49:14.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:14 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/639329249' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-130038-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:49:14.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:14 smithi022 ceph-mon[96736]: osdmap e1031: 8 total, 8 up, 8 in 2024-09-21T09:49:14.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:14 smithi022 ceph-mon[96736]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:49:15.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:15 smithi064 ceph-mon[86709]: osdmap e1032: 8 total, 8 up, 8 in 2024-09-21T09:49:15.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:15 smithi064 ceph-mon[86709]: pgmap v1202: 105 pgs: 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:49:15.717 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:15 smithi022 ceph-mon[98615]: osdmap e1032: 8 total, 8 up, 8 in 2024-09-21T09:49:15.717 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:15 smithi022 ceph-mon[98615]: pgmap v1202: 105 pgs: 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:49:15.717 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:15 smithi022 ceph-mon[96736]: osdmap e1032: 8 total, 8 up, 8 in 2024-09-21T09:49:15.717 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:15 smithi022 ceph-mon[96736]: pgmap v1202: 105 pgs: 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:49:16.058 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:49:15 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:49:15] "GET /metrics HTTP/1.1" 200 36435 "" "Prometheus/2.51.0" 2024-09-21T09:49:16.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:16 smithi064 ceph-mon[86709]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:49:16.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:16 smithi064 ceph-mon[86709]: Cluster is now healthy 2024-09-21T09:49:16.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:16 smithi064 ceph-mon[86709]: osdmap e1033: 8 total, 8 up, 8 in 2024-09-21T09:49:16.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:16 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/3267358639' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-130225-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:49:16.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:16 smithi064 ceph-mon[86709]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-130225-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:49:16.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:16 smithi022 ceph-mon[98615]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:49:16.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:16 smithi022 ceph-mon[98615]: Cluster is now healthy 2024-09-21T09:49:16.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:16 smithi022 ceph-mon[98615]: osdmap e1033: 8 total, 8 up, 8 in 2024-09-21T09:49:16.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:16 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/3267358639' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-130225-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:49:16.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:16 smithi022 ceph-mon[98615]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-130225-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:49:16.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:16 smithi022 ceph-mon[96736]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:49:16.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:16 smithi022 ceph-mon[96736]: Cluster is now healthy 2024-09-21T09:49:16.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:16 smithi022 ceph-mon[96736]: osdmap e1033: 8 total, 8 up, 8 in 2024-09-21T09:49:16.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:16 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/3267358639' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-130225-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:49:16.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:16 smithi022 ceph-mon[96736]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-130225-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:49:17.387 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] ClsNumOps.Add (2885 ms) 2024-09-21T09:49:17.387 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] ClsNumOps.Sub 2024-09-21T09:49:17.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:17 smithi064 ceph-mon[86709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-130225-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:49:17.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:17 smithi064 ceph-mon[86709]: osdmap e1034: 8 total, 8 up, 8 in 2024-09-21T09:49:17.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:17 smithi064 ceph-mon[86709]: pgmap v1205: 137 pgs: 9 creating+peering, 23 unknown, 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:49:17.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:17 smithi022 ceph-mon[96736]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-130225-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:49:17.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:17 smithi022 ceph-mon[96736]: osdmap e1034: 8 total, 8 up, 8 in 2024-09-21T09:49:17.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:17 smithi022 ceph-mon[96736]: pgmap v1205: 137 pgs: 9 creating+peering, 23 unknown, 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:49:17.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:17 smithi022 ceph-mon[98615]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-130225-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:49:17.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:17 smithi022 ceph-mon[98615]: osdmap e1034: 8 total, 8 up, 8 in 2024-09-21T09:49:17.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:17 smithi022 ceph-mon[98615]: pgmap v1205: 137 pgs: 9 creating+peering, 23 unknown, 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:49:18.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:18 smithi064 ceph-mon[86709]: osdmap e1035: 8 total, 8 up, 8 in 2024-09-21T09:49:18.810 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:18 smithi022 ceph-mon[96736]: osdmap e1035: 8 total, 8 up, 8 in 2024-09-21T09:49:18.810 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:18 smithi022 ceph-mon[98615]: osdmap e1035: 8 total, 8 up, 8 in 2024-09-21T09:49:19.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:19 smithi064 ceph-mon[86709]: osdmap e1036: 8 total, 8 up, 8 in 2024-09-21T09:49:19.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:19 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/3764772695' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-130225-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:49:19.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:19 smithi064 ceph-mon[86709]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-130225-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:49:19.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:19 smithi064 ceph-mon[86709]: pgmap v1208: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:49:19.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:19 smithi022 ceph-mon[98615]: osdmap e1036: 8 total, 8 up, 8 in 2024-09-21T09:49:19.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:19 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/3764772695' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-130225-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:49:19.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:19 smithi022 ceph-mon[98615]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-130225-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:49:19.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:19 smithi022 ceph-mon[98615]: pgmap v1208: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:49:19.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:19 smithi022 ceph-mon[96736]: osdmap e1036: 8 total, 8 up, 8 in 2024-09-21T09:49:19.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:19 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/3764772695' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-130225-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:49:19.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:19 smithi022 ceph-mon[96736]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-130225-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:49:19.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:19 smithi022 ceph-mon[96736]: pgmap v1208: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:49:20.404 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] ClsNumOps.Sub (3017 ms) 2024-09-21T09:49:20.405 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] ClsNumOps.Mul 2024-09-21T09:49:20.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:20 smithi064 ceph-mon[86709]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:49:20.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:20 smithi064 ceph-mon[86709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-130225-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:49:20.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:20 smithi064 ceph-mon[86709]: osdmap e1037: 8 total, 8 up, 8 in 2024-09-21T09:49:20.692 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:49:20 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:49:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:49:20.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:20 smithi022 ceph-mon[98615]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:49:20.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:20 smithi022 ceph-mon[98615]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-130225-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:49:20.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:20 smithi022 ceph-mon[98615]: osdmap e1037: 8 total, 8 up, 8 in 2024-09-21T09:49:20.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:20 smithi022 ceph-mon[96736]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:49:20.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:20 smithi022 ceph-mon[96736]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-130225-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:49:20.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:20 smithi022 ceph-mon[96736]: osdmap e1037: 8 total, 8 up, 8 in 2024-09-21T09:49:21.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:21 smithi064 ceph-mon[86709]: osdmap e1038: 8 total, 8 up, 8 in 2024-09-21T09:49:21.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:21 smithi064 ceph-mon[86709]: pgmap v1211: 105 pgs: 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:49:21.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:21 smithi022 ceph-mon[98615]: osdmap e1038: 8 total, 8 up, 8 in 2024-09-21T09:49:21.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:21 smithi022 ceph-mon[98615]: pgmap v1211: 105 pgs: 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:49:21.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:21 smithi022 ceph-mon[96736]: osdmap e1038: 8 total, 8 up, 8 in 2024-09-21T09:49:21.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:21 smithi022 ceph-mon[96736]: pgmap v1211: 105 pgs: 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:49:22.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:22 smithi064 ceph-mon[86709]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:49:22.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:22 smithi064 ceph-mon[86709]: Cluster is now healthy 2024-09-21T09:49:22.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:22 smithi064 ceph-mon[86709]: osdmap e1039: 8 total, 8 up, 8 in 2024-09-21T09:49:22.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:22 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/1552682361' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-130225-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:49:22.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:22 smithi064 ceph-mon[86709]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-130225-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:49:22.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:22 smithi022 ceph-mon[98615]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:49:22.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:22 smithi022 ceph-mon[98615]: Cluster is now healthy 2024-09-21T09:49:22.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:22 smithi022 ceph-mon[98615]: osdmap e1039: 8 total, 8 up, 8 in 2024-09-21T09:49:22.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:22 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/1552682361' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-130225-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:49:22.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:22 smithi022 ceph-mon[98615]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-130225-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:49:22.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:22 smithi022 ceph-mon[96736]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:49:22.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:22 smithi022 ceph-mon[96736]: Cluster is now healthy 2024-09-21T09:49:22.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:22 smithi022 ceph-mon[96736]: osdmap e1039: 8 total, 8 up, 8 in 2024-09-21T09:49:22.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:22 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/1552682361' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-130225-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:49:22.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:22 smithi022 ceph-mon[96736]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-130225-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:49:23.423 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] ClsNumOps.Mul (3019 ms) 2024-09-21T09:49:23.423 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] ClsNumOps.Div 2024-09-21T09:49:23.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:23 smithi064 ceph-mon[86709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-130225-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:49:23.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:23 smithi064 ceph-mon[86709]: osdmap e1040: 8 total, 8 up, 8 in 2024-09-21T09:49:23.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:23 smithi064 ceph-mon[86709]: pgmap v1214: 137 pgs: 6 creating+peering, 26 unknown, 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:49:23.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:23 smithi022 ceph-mon[98615]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-130225-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:49:23.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:23 smithi022 ceph-mon[98615]: osdmap e1040: 8 total, 8 up, 8 in 2024-09-21T09:49:23.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:23 smithi022 ceph-mon[98615]: pgmap v1214: 137 pgs: 6 creating+peering, 26 unknown, 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:49:23.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:23 smithi022 ceph-mon[96736]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-130225-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:49:23.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:23 smithi022 ceph-mon[96736]: osdmap e1040: 8 total, 8 up, 8 in 2024-09-21T09:49:23.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:23 smithi022 ceph-mon[96736]: pgmap v1214: 137 pgs: 6 creating+peering, 26 unknown, 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:49:24.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:24 smithi064 ceph-mon[86709]: osdmap e1041: 8 total, 8 up, 8 in 2024-09-21T09:49:24.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:24 smithi022 ceph-mon[98615]: osdmap e1041: 8 total, 8 up, 8 in 2024-09-21T09:49:24.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:24 smithi022 ceph-mon[96736]: osdmap e1041: 8 total, 8 up, 8 in 2024-09-21T09:49:25.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:25 smithi064 ceph-mon[86709]: osdmap e1042: 8 total, 8 up, 8 in 2024-09-21T09:49:25.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:25 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/2375440000' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-130225-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:49:25.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:25 smithi064 ceph-mon[86709]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-130225-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:49:25.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:25 smithi064 ceph-mon[86709]: pgmap v1217: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:49:25.709 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:25 smithi022 ceph-mon[96736]: osdmap e1042: 8 total, 8 up, 8 in 2024-09-21T09:49:25.710 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:25 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/2375440000' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-130225-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:49:25.710 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:25 smithi022 ceph-mon[96736]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-130225-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:49:25.710 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:25 smithi022 ceph-mon[96736]: pgmap v1217: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:49:25.710 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:25 smithi022 ceph-mon[98615]: osdmap e1042: 8 total, 8 up, 8 in 2024-09-21T09:49:25.710 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:25 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/2375440000' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-130225-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:49:25.710 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:25 smithi022 ceph-mon[98615]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-130225-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:49:25.710 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:25 smithi022 ceph-mon[98615]: pgmap v1217: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:49:26.058 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:49:25 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:49:25] "GET /metrics HTTP/1.1" 200 36435 "" "Prometheus/2.51.0" 2024-09-21T09:49:26.441 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] ClsNumOps.Div (3018 ms) 2024-09-21T09:49:26.441 INFO:tasks.workunit.client.0.smithi022.stdout:[----------] 4 tests from ClsNumOps (11939 ms total) 2024-09-21T09:49:26.442 INFO:tasks.workunit.client.0.smithi022.stdout: 2024-09-21T09:49:26.442 INFO:tasks.workunit.client.0.smithi022.stdout:[----------] Global test environment tear-down 2024-09-21T09:49:26.442 INFO:tasks.workunit.client.0.smithi022.stdout:[==========] 4 tests from 1 test suite ran. (11939 ms total) 2024-09-21T09:49:26.442 INFO:tasks.workunit.client.0.smithi022.stdout:[ PASSED ] 4 tests. 2024-09-21T09:49:26.444 INFO:teuthology.orchestra.run:Running command with timeout 3600 2024-09-21T09:49:26.444 DEBUG:teuthology.orchestra.run.smithi022:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2024-09-21T09:49:26.515 INFO:tasks.workunit:Running workunit cls/test_cls_rbd.sh... 2024-09-21T09:49:26.515 DEBUG:teuthology.orchestra.run.smithi022:workunit test cls/test_cls_rbd.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh 2024-09-21T09:49:26.590 INFO:tasks.workunit.client.0.smithi022.stdout:Running main() from gmock_main.cc 2024-09-21T09:49:26.590 INFO:tasks.workunit.client.0.smithi022.stdout:[==========] Running 70 tests from 1 test suite. 2024-09-21T09:49:26.590 INFO:tasks.workunit.client.0.smithi022.stdout:[----------] Global test environment set-up. 2024-09-21T09:49:26.590 INFO:tasks.workunit.client.0.smithi022.stdout:[----------] 70 tests from TestClsRbd 2024-09-21T09:49:26.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:26 smithi064 ceph-mon[86709]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:49:26.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:26 smithi064 ceph-mon[86709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-130225-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:49:26.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:26 smithi064 ceph-mon[86709]: osdmap e1043: 8 total, 8 up, 8 in 2024-09-21T09:49:26.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:26 smithi022 ceph-mon[98615]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:49:26.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:26 smithi022 ceph-mon[98615]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-130225-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:49:26.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:26 smithi022 ceph-mon[98615]: osdmap e1043: 8 total, 8 up, 8 in 2024-09-21T09:49:26.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:26 smithi022 ceph-mon[96736]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:49:26.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:26 smithi022 ceph-mon[96736]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-130225-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:49:26.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:26 smithi022 ceph-mon[96736]: osdmap e1043: 8 total, 8 up, 8 in 2024-09-21T09:49:27.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:27 smithi064 ceph-mon[86709]: osdmap e1044: 8 total, 8 up, 8 in 2024-09-21T09:49:27.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:27 smithi064 ceph-mon[86709]: pgmap v1220: 105 pgs: 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:49:27.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:27 smithi022 ceph-mon[98615]: osdmap e1044: 8 total, 8 up, 8 in 2024-09-21T09:49:27.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:27 smithi022 ceph-mon[98615]: pgmap v1220: 105 pgs: 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:49:27.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:27 smithi022 ceph-mon[96736]: osdmap e1044: 8 total, 8 up, 8 in 2024-09-21T09:49:27.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:27 smithi022 ceph-mon[96736]: pgmap v1220: 105 pgs: 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:49:28.448 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestClsRbd.get_all_features 2024-09-21T09:49:28.467 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestClsRbd.get_all_features (19 ms) 2024-09-21T09:49:28.467 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestClsRbd.copyup 2024-09-21T09:49:28.651 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestClsRbd.copyup (184 ms) 2024-09-21T09:49:28.651 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestClsRbd.sparse_copyup 2024-09-21T09:49:28.666 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestClsRbd.sparse_copyup (15 ms) 2024-09-21T09:49:28.666 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestClsRbd.get_and_set_id 2024-09-21T09:49:28.673 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestClsRbd.get_and_set_id (7 ms) 2024-09-21T09:49:28.673 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestClsRbd.add_remove_child 2024-09-21T09:49:28.685 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestClsRbd.add_remove_child (12 ms) 2024-09-21T09:49:28.685 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestClsRbd.directory_methods 2024-09-21T09:49:28.711 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestClsRbd.directory_methods (26 ms) 2024-09-21T09:49:28.711 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestClsRbd.create 2024-09-21T09:49:28.725 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestClsRbd.create (14 ms) 2024-09-21T09:49:28.725 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestClsRbd.get_features 2024-09-21T09:49:28.726 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestClsRbd.get_features (1 ms) 2024-09-21T09:49:28.726 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestClsRbd.get_object_prefix 2024-09-21T09:49:28.728 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestClsRbd.get_object_prefix (2 ms) 2024-09-21T09:49:28.728 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestClsRbd.get_create_timestamp 2024-09-21T09:49:28.729 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestClsRbd.get_create_timestamp (2 ms) 2024-09-21T09:49:28.730 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestClsRbd.get_access_timestamp 2024-09-21T09:49:28.731 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestClsRbd.get_access_timestamp (1 ms) 2024-09-21T09:49:28.731 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestClsRbd.get_modify_timestamp 2024-09-21T09:49:28.733 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestClsRbd.get_modify_timestamp (2 ms) 2024-09-21T09:49:28.733 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestClsRbd.get_data_pool 2024-09-21T09:49:28.737 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestClsRbd.get_data_pool (4 ms) 2024-09-21T09:49:28.737 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestClsRbd.get_size 2024-09-21T09:49:28.741 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestClsRbd.get_size (4 ms) 2024-09-21T09:49:28.741 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestClsRbd.set_size 2024-09-21T09:49:28.745 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestClsRbd.set_size (4 ms) 2024-09-21T09:49:28.745 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestClsRbd.protection_status 2024-09-21T09:49:28.764 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestClsRbd.protection_status (19 ms) 2024-09-21T09:49:28.764 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestClsRbd.snapshot_limits 2024-09-21T09:49:28.772 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestClsRbd.snapshot_limits (8 ms) 2024-09-21T09:49:28.772 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestClsRbd.parents_v1 2024-09-21T09:49:28.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:28 smithi022 ceph-mon[98615]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:49:28.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:28 smithi022 ceph-mon[98615]: Cluster is now healthy 2024-09-21T09:49:28.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:28 smithi022 ceph-mon[98615]: osdmap e1045: 8 total, 8 up, 8 in 2024-09-21T09:49:28.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:28 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/557503317' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-130331-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:49:28.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:28 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:49:28.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:28 smithi022 ceph-mon[96736]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:49:28.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:28 smithi022 ceph-mon[96736]: Cluster is now healthy 2024-09-21T09:49:28.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:28 smithi022 ceph-mon[96736]: osdmap e1045: 8 total, 8 up, 8 in 2024-09-21T09:49:28.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:28 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/557503317' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-130331-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:49:28.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:28 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:49:28.812 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestClsRbd.parents_v1 (41 ms) 2024-09-21T09:49:28.813 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestClsRbd.parents_v2 2024-09-21T09:49:28.842 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestClsRbd.parents_v2 (29 ms) 2024-09-21T09:49:28.842 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestClsRbd.snapshots 2024-09-21T09:49:28.863 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestClsRbd.snapshots (21 ms) 2024-09-21T09:49:28.863 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestClsRbd.snapid_race 2024-09-21T09:49:28.867 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestClsRbd.snapid_race (5 ms) 2024-09-21T09:49:28.868 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestClsRbd.stripingv2 2024-09-21T09:49:28.877 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestClsRbd.stripingv2 (9 ms) 2024-09-21T09:49:28.877 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestClsRbd.object_map_save 2024-09-21T09:49:28.881 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestClsRbd.object_map_save (4 ms) 2024-09-21T09:49:28.881 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestClsRbd.object_map_resize 2024-09-21T09:49:28.889 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestClsRbd.object_map_resize (8 ms) 2024-09-21T09:49:28.889 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestClsRbd.object_map_update 2024-09-21T09:49:28.894 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestClsRbd.object_map_update (5 ms) 2024-09-21T09:49:28.894 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestClsRbd.object_map_load_enoent 2024-09-21T09:49:28.894 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestClsRbd.object_map_load_enoent (0 ms) 2024-09-21T09:49:28.894 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestClsRbd.object_map_snap_add 2024-09-21T09:49:28.901 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestClsRbd.object_map_snap_add (7 ms) 2024-09-21T09:49:28.901 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestClsRbd.object_map_snap_remove 2024-09-21T09:49:28.908 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestClsRbd.object_map_snap_remove (7 ms) 2024-09-21T09:49:28.908 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestClsRbd.flags 2024-09-21T09:49:28.913 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestClsRbd.flags (5 ms) 2024-09-21T09:49:28.913 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestClsRbd.metadata 2024-09-21T09:49:28.936 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestClsRbd.metadata (23 ms) 2024-09-21T09:49:28.936 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestClsRbd.set_features 2024-09-21T09:49:28.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:28 smithi064 ceph-mon[86709]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:49:28.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:28 smithi064 ceph-mon[86709]: Cluster is now healthy 2024-09-21T09:49:28.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:28 smithi064 ceph-mon[86709]: osdmap e1045: 8 total, 8 up, 8 in 2024-09-21T09:49:28.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:28 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/557503317' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-130331-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T09:49:28.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:28 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:49:28.942 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestClsRbd.set_features (7 ms) 2024-09-21T09:49:28.942 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestClsRbd.mirror 2024-09-21T09:49:28.976 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestClsRbd.mirror (33 ms) 2024-09-21T09:49:28.976 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestClsRbd.mirror_image 2024-09-21T09:49:28.991 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestClsRbd.mirror_image (15 ms) 2024-09-21T09:49:28.991 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestClsRbd.mirror_image_status 2024-09-21T09:49:29.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:29 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/557503317' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-130331-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:49:29.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:29 smithi022 ceph-mon[98615]: osdmap e1046: 8 total, 8 up, 8 in 2024-09-21T09:49:29.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:29 smithi022 ceph-mon[98615]: pgmap v1223: 137 pgs: 12 creating+peering, 20 unknown, 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:49:29.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:29 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/557503317' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-130331-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:49:29.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:29 smithi022 ceph-mon[96736]: osdmap e1046: 8 total, 8 up, 8 in 2024-09-21T09:49:29.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:29 smithi022 ceph-mon[96736]: pgmap v1223: 137 pgs: 12 creating+peering, 20 unknown, 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:49:29.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:29 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/557503317' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi022-130331-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T09:49:29.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:29 smithi064 ceph-mon[86709]: osdmap e1046: 8 total, 8 up, 8 in 2024-09-21T09:49:29.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:29 smithi064 ceph-mon[86709]: pgmap v1223: 137 pgs: 12 creating+peering, 20 unknown, 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:49:30.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:30 smithi064 ceph-mon[86709]: osdmap e1047: 8 total, 8 up, 8 in 2024-09-21T09:49:30.692 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:49:30 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:49:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:49:30.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:30 smithi022 ceph-mon[98615]: osdmap e1047: 8 total, 8 up, 8 in 2024-09-21T09:49:30.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:30 smithi022 ceph-mon[96736]: osdmap e1047: 8 total, 8 up, 8 in 2024-09-21T09:49:31.343 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestClsRbd.mirror_image_status (2352 ms) 2024-09-21T09:49:31.343 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestClsRbd.mirror_image_map 2024-09-21T09:49:31.417 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestClsRbd.mirror_image_map (74 ms) 2024-09-21T09:49:31.417 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestClsRbd.mirror_instances 2024-09-21T09:49:31.424 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestClsRbd.mirror_instances (8 ms) 2024-09-21T09:49:31.425 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestClsRbd.mirror_snapshot 2024-09-21T09:49:31.435 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestClsRbd.mirror_snapshot (11 ms) 2024-09-21T09:49:31.436 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestClsRbd.group_dir_list 2024-09-21T09:49:31.438 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestClsRbd.group_dir_list (2 ms) 2024-09-21T09:49:31.438 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestClsRbd.group_dir_add 2024-09-21T09:49:31.440 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestClsRbd.group_dir_add (2 ms) 2024-09-21T09:49:31.440 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestClsRbd.dir_add_already_existing 2024-09-21T09:49:31.443 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestClsRbd.dir_add_already_existing (3 ms) 2024-09-21T09:49:31.443 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestClsRbd.group_dir_rename 2024-09-21T09:49:31.448 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestClsRbd.group_dir_rename (6 ms) 2024-09-21T09:49:31.449 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestClsRbd.group_dir_remove 2024-09-21T09:49:31.452 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestClsRbd.group_dir_remove (3 ms) 2024-09-21T09:49:31.452 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestClsRbd.group_dir_remove_missing 2024-09-21T09:49:31.457 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestClsRbd.group_dir_remove_missing (5 ms) 2024-09-21T09:49:31.457 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestClsRbd.group_image_add 2024-09-21T09:49:31.460 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestClsRbd.group_image_add (3 ms) 2024-09-21T09:49:31.460 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestClsRbd.group_image_remove 2024-09-21T09:49:31.463 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestClsRbd.group_image_remove (3 ms) 2024-09-21T09:49:31.463 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestClsRbd.group_image_list 2024-09-21T09:49:31.466 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestClsRbd.group_image_list (4 ms) 2024-09-21T09:49:31.466 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestClsRbd.group_image_clean 2024-09-21T09:49:31.470 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestClsRbd.group_image_clean (4 ms) 2024-09-21T09:49:31.471 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestClsRbd.image_group_add 2024-09-21T09:49:31.473 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestClsRbd.image_group_add (2 ms) 2024-09-21T09:49:31.473 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestClsRbd.image_group_remove 2024-09-21T09:49:31.476 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestClsRbd.image_group_remove (3 ms) 2024-09-21T09:49:31.476 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestClsRbd.image_group_get 2024-09-21T09:49:31.478 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestClsRbd.image_group_get (2 ms) 2024-09-21T09:49:31.478 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_name 2024-09-21T09:49:31.480 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestClsRbd.group_snap_set_empty_name (2 ms) 2024-09-21T09:49:31.480 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_id 2024-09-21T09:49:31.482 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestClsRbd.group_snap_set_empty_id (2 ms) 2024-09-21T09:49:31.482 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_id 2024-09-21T09:49:31.484 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_id (3 ms) 2024-09-21T09:49:31.484 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_name 2024-09-21T09:49:31.487 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_name (2 ms) 2024-09-21T09:49:31.487 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestClsRbd.group_snap_set 2024-09-21T09:49:31.489 INFO:tasks.workunit.client.0.smithi022.stdout:/home/jenkins-build/build/workspace/ceph-dev-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos9/DIST/centos9/MACHINE_SIZE/gigantic/release/17.2.7-1649-gfbd1a6ab/rpm/el9/BUILD/ceph-17.2.7-1649-gfbd1a6ab/src/test/cls_rbd/test_cls_rbd.cc:2681: Failure 2024-09-21T09:49:31.489 INFO:tasks.workunit.client.0.smithi022.stdout:Expected equality of these values: 2024-09-21T09:49:31.489 INFO:tasks.workunit.client.0.smithi022.stdout: 1U 2024-09-21T09:49:31.489 INFO:tasks.workunit.client.0.smithi022.stdout: Which is: 1 2024-09-21T09:49:31.489 INFO:tasks.workunit.client.0.smithi022.stdout: keys.size() 2024-09-21T09:49:31.490 INFO:tasks.workunit.client.0.smithi022.stdout: Which is: 3 2024-09-21T09:49:31.490 INFO:tasks.workunit.client.0.smithi022.stdout:[ FAILED ] TestClsRbd.group_snap_set (2 ms) 2024-09-21T09:49:31.490 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestClsRbd.group_snap_list 2024-09-21T09:49:31.492 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestClsRbd.group_snap_list (3 ms) 2024-09-21T09:49:31.492 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestClsRbd.group_snap_list_max_return 2024-09-21T09:49:31.509 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestClsRbd.group_snap_list_max_return (17 ms) 2024-09-21T09:49:31.509 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestClsRbd.group_snap_list_max_read 2024-09-21T09:49:31.719 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestClsRbd.group_snap_list_max_read (210 ms) 2024-09-21T09:49:31.720 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestClsRbd.group_snap_remove 2024-09-21T09:49:31.722 INFO:tasks.workunit.client.0.smithi022.stdout:/home/jenkins-build/build/workspace/ceph-dev-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos9/DIST/centos9/MACHINE_SIZE/gigantic/release/17.2.7-1649-gfbd1a6ab/rpm/el9/BUILD/ceph-17.2.7-1649-gfbd1a6ab/src/test/cls_rbd/test_cls_rbd.cc:2792: Failure 2024-09-21T09:49:31.722 INFO:tasks.workunit.client.0.smithi022.stdout:Expected equality of these values: 2024-09-21T09:49:31.722 INFO:tasks.workunit.client.0.smithi022.stdout: 1U 2024-09-21T09:49:31.722 INFO:tasks.workunit.client.0.smithi022.stdout: Which is: 1 2024-09-21T09:49:31.722 INFO:tasks.workunit.client.0.smithi022.stdout: keys.size() 2024-09-21T09:49:31.722 INFO:tasks.workunit.client.0.smithi022.stdout: Which is: 3 2024-09-21T09:49:31.722 INFO:tasks.workunit.client.0.smithi022.stdout:[ FAILED ] TestClsRbd.group_snap_remove (3 ms) 2024-09-21T09:49:31.722 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestClsRbd.group_snap_get_by_id 2024-09-21T09:49:31.726 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestClsRbd.group_snap_get_by_id (2 ms) 2024-09-21T09:49:31.726 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestClsRbd.trash_methods 2024-09-21T09:49:31.731 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestClsRbd.trash_methods (7 ms) 2024-09-21T09:49:31.731 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestClsRbd.op_features 2024-09-21T09:49:31.739 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestClsRbd.op_features (8 ms) 2024-09-21T09:49:31.739 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestClsRbd.clone_parent 2024-09-21T09:49:31.760 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestClsRbd.clone_parent (21 ms) 2024-09-21T09:49:31.760 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestClsRbd.clone_parent_ns 2024-09-21T09:49:31.768 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestClsRbd.clone_parent_ns (8 ms) 2024-09-21T09:49:31.768 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestClsRbd.clone_child 2024-09-21T09:49:31.779 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestClsRbd.clone_child (11 ms) 2024-09-21T09:49:31.779 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestClsRbd.namespace_methods 2024-09-21T09:49:31.786 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestClsRbd.namespace_methods (7 ms) 2024-09-21T09:49:31.786 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestClsRbd.migration 2024-09-21T09:49:31.795 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestClsRbd.migration (9 ms) 2024-09-21T09:49:31.795 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestClsRbd.migration_v1 2024-09-21T09:49:31.805 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestClsRbd.migration_v1 (10 ms) 2024-09-21T09:49:31.805 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestClsRbd.assert_snapc_seq 2024-09-21T09:49:31.807 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:31 smithi022 ceph-mon[96736]: pgmap v1225: 137 pgs: 12 creating+peering, 13 unknown, 112 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 2.6 MiB/s rd, 1.4 MiB/s wr, 1.19k op/s 2024-09-21T09:49:31.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:31 smithi022 ceph-mon[98615]: pgmap v1225: 137 pgs: 12 creating+peering, 13 unknown, 112 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 2.6 MiB/s rd, 1.4 MiB/s wr, 1.19k op/s 2024-09-21T09:49:31.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:31 smithi064 ceph-mon[86709]: pgmap v1225: 137 pgs: 12 creating+peering, 13 unknown, 112 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 2.6 MiB/s rd, 1.4 MiB/s wr, 1.19k op/s 2024-09-21T09:49:33.474 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestClsRbd.assert_snapc_seq (1669 ms) 2024-09-21T09:49:33.474 INFO:tasks.workunit.client.0.smithi022.stdout:[ RUN ] TestClsRbd.sparsify 2024-09-21T09:49:33.519 INFO:tasks.workunit.client.0.smithi022.stdout:[ OK ] TestClsRbd.sparsify (45 ms) 2024-09-21T09:49:33.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:33 smithi022 ceph-mon[96736]: osdmap e1048: 8 total, 8 up, 8 in 2024-09-21T09:49:33.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:33 smithi022 ceph-mon[96736]: pgmap v1227: 137 pgs: 12 creating+peering, 125 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 2.1 MiB/s rd, 1.1 MiB/s wr, 994 op/s 2024-09-21T09:49:33.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:33 smithi022 ceph-mon[98615]: osdmap e1048: 8 total, 8 up, 8 in 2024-09-21T09:49:33.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:33 smithi022 ceph-mon[98615]: pgmap v1227: 137 pgs: 12 creating+peering, 125 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 2.1 MiB/s rd, 1.1 MiB/s wr, 994 op/s 2024-09-21T09:49:33.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:33 smithi064 ceph-mon[86709]: osdmap e1048: 8 total, 8 up, 8 in 2024-09-21T09:49:33.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:33 smithi064 ceph-mon[86709]: pgmap v1227: 137 pgs: 12 creating+peering, 125 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 2.1 MiB/s rd, 1.1 MiB/s wr, 994 op/s 2024-09-21T09:49:34.475 INFO:tasks.workunit.client.0.smithi022.stdout:[----------] 70 tests from TestClsRbd (5071 ms total) 2024-09-21T09:49:34.475 INFO:tasks.workunit.client.0.smithi022.stdout: 2024-09-21T09:49:34.476 INFO:tasks.workunit.client.0.smithi022.stdout:[----------] Global test environment tear-down 2024-09-21T09:49:34.476 INFO:tasks.workunit.client.0.smithi022.stdout:[==========] 70 tests from 1 test suite ran. (7886 ms total) 2024-09-21T09:49:34.476 INFO:tasks.workunit.client.0.smithi022.stdout:[ PASSED ] 68 tests. 2024-09-21T09:49:34.476 INFO:tasks.workunit.client.0.smithi022.stdout:[ FAILED ] 2 tests, listed below: 2024-09-21T09:49:34.476 INFO:tasks.workunit.client.0.smithi022.stdout:[ FAILED ] TestClsRbd.group_snap_set 2024-09-21T09:49:34.476 INFO:tasks.workunit.client.0.smithi022.stdout:[ FAILED ] TestClsRbd.group_snap_remove 2024-09-21T09:49:34.476 INFO:tasks.workunit.client.0.smithi022.stdout: 2024-09-21T09:49:34.476 INFO:tasks.workunit.client.0.smithi022.stdout: 2 FAILED TESTS 2024-09-21T09:49:34.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:49:34.479 INFO:tasks.workunit:Stopping ['cls'] on client.0... 2024-09-21T09:49:34.479 DEBUG:teuthology.orchestra.run.smithi022:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2024-09-21T09:49:34.748 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:34 smithi022 ceph-mon[98615]: osdmap e1049: 8 total, 8 up, 8 in 2024-09-21T09:49:34.748 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:34 smithi022 ceph-mon[96736]: osdmap e1049: 8 total, 8 up, 8 in 2024-09-21T09:49:34.749 ERROR:teuthology.run_tasks:Saw exception from tasks. Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/parallel.py", line 48, in task with parallel.parallel() as p: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/full_sequential.py", line 30, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_ceph_62da54b4b83a96ca9976b7e1d0c8272a564a2208/qa/tasks/workunit.py", line 126, in task with parallel() as p: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_ceph_62da54b4b83a96ca9976b7e1d0c8272a564a2208/qa/tasks/workunit.py", line 434, in _run_tests remote.run( File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/remote.py", line 523, in run r = self._runner(client=self.ssh, name=self.shortname, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 455, in run r.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed (workunit test cls/test_cls_rbd.sh) on smithi022 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh' 2024-09-21T09:49:34.945 ERROR:teuthology.util.sentry: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=7193baab3b0642829283906d11ea9e22 Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/parallel.py", line 48, in task with parallel.parallel() as p: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/full_sequential.py", line 30, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_ceph_62da54b4b83a96ca9976b7e1d0c8272a564a2208/qa/tasks/workunit.py", line 126, in task with parallel() as p: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_ceph_62da54b4b83a96ca9976b7e1d0c8272a564a2208/qa/tasks/workunit.py", line 434, in _run_tests remote.run( File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/remote.py", line 523, in run r = self._runner(client=self.ssh, name=self.shortname, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 455, in run r.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed (workunit test cls/test_cls_rbd.sh) on smithi022 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh' 2024-09-21T09:49:34.948 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-09-21T09:49:34.957 INFO:tasks.cephadm:Teardown begin 2024-09-21T09:49:34.957 INFO:tasks.cephfs.filesystem:Destroying file system cephfs and related pools 2024-09-21T09:49:34.957 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid d65e801e-77fa-11ef-baf6-efdc52797490 -- ceph fs dump --format=json 2024-09-21T09:49:34.962 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:34 smithi064 ceph-mon[86709]: osdmap e1049: 8 total, 8 up, 8 in 2024-09-21T09:49:35.214 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:49:35.618 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:35 smithi022 ceph-mon[98615]: osdmap e1050: 8 total, 8 up, 8 in 2024-09-21T09:49:35.618 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:35 smithi022 ceph-mon[98615]: pgmap v1230: 105 pgs: 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:49:35.618 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:35 smithi022 ceph-mon[96736]: osdmap e1050: 8 total, 8 up, 8 in 2024-09-21T09:49:35.618 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:35 smithi022 ceph-mon[96736]: pgmap v1230: 105 pgs: 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:49:35.843 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:35 smithi064 ceph-mon[86709]: osdmap e1050: 8 total, 8 up, 8 in 2024-09-21T09:49:35.843 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:35 smithi064 ceph-mon[86709]: pgmap v1230: 105 pgs: 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:49:36.058 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:49:35 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:49:35] "GET /metrics HTTP/1.1" 200 36445 "" "Prometheus/2.51.0" 2024-09-21T09:49:37.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:37 smithi022 ceph-mon[98615]: pgmap v1231: 105 pgs: 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:49:37.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:37 smithi022 ceph-mon[96736]: pgmap v1231: 105 pgs: 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:49:37.841 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-09-21T09:49:37.841 INFO:teuthology.orchestra.run.smithi022.stdout:{"epoch":16,"btime":"2024-09-21T09:40:31:083981+0000","default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":54409,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6833/1932960939","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6832","nonce":1932960939},{"type":"v1","addr":"172.21.15.64:6833","nonce":1932960939}]},"join_fscid":-1,"export_targets":[],"features":4540719139924082687,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":16}],"filesystems":[{"mdsmap":{"epoch":15,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-09-21T09:27:37.584397+0000","modified":"2024-09-21T09:40:25.999326+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":599,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"max_mds":1,"in":[0],"up":{"mds_0":44414},"failed":[],"damaged":[],"stopped":[],"info":{"gid_44414":{"gid":44414,"name":"a","rank":0,"incarnation":12,"state":"up:active","state_seq":6,"addr":"172.21.15.22:6835/4150258157","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6834","nonce":4150258157},{"type":"v1","addr":"172.21.15.22:6835","nonce":4150258157}]},"join_fscid":-1,"export_targets":[],"features":4540719139924082687,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":44414,"qdb_cluster":[44414]},"id":1}]} 2024-09-21T09:49:37.841 INFO:teuthology.orchestra.run.smithi022.stderr:dumped fsmap epoch 16 2024-09-21T09:49:37.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:37 smithi064 ceph-mon[86709]: pgmap v1231: 105 pgs: 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:49:38.661 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid d65e801e-77fa-11ef-baf6-efdc52797490 -- ceph fs dump --format=json 2024-09-21T09:49:38.924 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:49:38.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:38 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/971151733' entity='client.admin' cmd={"prefix": "fs dump", "format": "json"} : dispatch 2024-09-21T09:49:39.046 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:38 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/971151733' entity='client.admin' cmd={"prefix": "fs dump", "format": "json"} : dispatch 2024-09-21T09:49:39.046 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:38 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/971151733' entity='client.admin' cmd={"prefix": "fs dump", "format": "json"} : dispatch 2024-09-21T09:49:39.921 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:39 smithi022 ceph-mon[98615]: pgmap v1232: 105 pgs: 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:49:39.922 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:39 smithi022 ceph-mon[96736]: pgmap v1232: 105 pgs: 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:49:39.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:39 smithi064 ceph-mon[86709]: pgmap v1232: 105 pgs: 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:49:40.692 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:49:40 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:49:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:49:41.141 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-09-21T09:49:41.142 INFO:teuthology.orchestra.run.smithi022.stdout:{"epoch":16,"btime":"2024-09-21T09:40:31:083981+0000","default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":54409,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6833/1932960939","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6832","nonce":1932960939},{"type":"v1","addr":"172.21.15.64:6833","nonce":1932960939}]},"join_fscid":-1,"export_targets":[],"features":4540719139924082687,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":16}],"filesystems":[{"mdsmap":{"epoch":15,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-09-21T09:27:37.584397+0000","modified":"2024-09-21T09:40:25.999326+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":599,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"max_mds":1,"in":[0],"up":{"mds_0":44414},"failed":[],"damaged":[],"stopped":[],"info":{"gid_44414":{"gid":44414,"name":"a","rank":0,"incarnation":12,"state":"up:active","state_seq":6,"addr":"172.21.15.22:6835/4150258157","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6834","nonce":4150258157},{"type":"v1","addr":"172.21.15.22:6835","nonce":4150258157}]},"join_fscid":-1,"export_targets":[],"features":4540719139924082687,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":44414,"qdb_cluster":[44414]},"id":1}]} 2024-09-21T09:49:41.142 INFO:teuthology.orchestra.run.smithi022.stderr:dumped fsmap epoch 16 2024-09-21T09:49:41.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:41 smithi022 ceph-mon[98615]: pgmap v1233: 105 pgs: 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:49:41.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:41 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/103057114' entity='client.admin' cmd={"prefix": "fs dump", "format": "json"} : dispatch 2024-09-21T09:49:41.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:41 smithi022 ceph-mon[96736]: pgmap v1233: 105 pgs: 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:49:41.808 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:41 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/103057114' entity='client.admin' cmd={"prefix": "fs dump", "format": "json"} : dispatch 2024-09-21T09:49:41.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:41 smithi064 ceph-mon[86709]: pgmap v1233: 105 pgs: 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:49:41.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:41 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/103057114' entity='client.admin' cmd={"prefix": "fs dump", "format": "json"} : dispatch 2024-09-21T09:49:41.951 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid d65e801e-77fa-11ef-baf6-efdc52797490 -- ceph osd dump --format=json 2024-09-21T09:49:42.213 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:49:43.687 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:43 smithi022 ceph-mon[98615]: pgmap v1234: 105 pgs: 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:49:43.688 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:43 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:49:43.688 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:43 smithi022 ceph-mon[96736]: pgmap v1234: 105 pgs: 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:49:43.688 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:43 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:49:43.943 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:43 smithi064 ceph-mon[86709]: pgmap v1234: 105 pgs: 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:49:43.943 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:43 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:49:44.116 INFO:teuthology.orchestra.run.smithi022.stdout: 2024-09-21T09:49:44.116 INFO:teuthology.orchestra.run.smithi022.stdout:{"epoch":1050,"fsid":"d65e801e-77fa-11ef-baf6-efdc52797490","created":"2024-09-21T09:22:04.474917+0000","modified":"2024-09-21T09:49:34.464843+0000","last_up_change":"2024-09-21T09:40:02.098011+0000","last_in_change":"2024-09-21T09:26:49.376072+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":27,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":60,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-21T09:25:07.405775+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-09-21T09:27:31.151960+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"57","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-09-21T09:27:34.558875+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"57","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"rbd","create_time":"2024-09-21T09:28:27.564651+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"66","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":66,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":3,"score_stable":3,"optimal_score":1,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"a1fbb74d-ffae-4d5f-81ab-9677d258a95c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":329,"up_from":343,"up_thru":1045,"down_at":330,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6802","nonce":1264061432},{"type":"v1","addr":"172.21.15.22:6803","nonce":1264061432}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6804","nonce":1264061432},{"type":"v1","addr":"172.21.15.22:6805","nonce":1264061432}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6808","nonce":1264061432},{"type":"v1","addr":"172.21.15.22:6809","nonce":1264061432}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6806","nonce":1264061432},{"type":"v1","addr":"172.21.15.22:6807","nonce":1264061432}]},"public_addr":"172.21.15.22:6803/1264061432","cluster_addr":"172.21.15.22:6805/1264061432","heartbeat_back_addr":"172.21.15.22:6809/1264061432","heartbeat_front_addr":"172.21.15.22:6807/1264061432","state":["exists","up"]},{"osd":1,"uuid":"fa4c9385-ac93-4983-92b4-f23f0a3731db","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":363,"up_from":376,"up_thru":1045,"down_at":364,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6810","nonce":2947123190},{"type":"v1","addr":"172.21.15.22:6811","nonce":2947123190}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6812","nonce":2947123190},{"type":"v1","addr":"172.21.15.22:6813","nonce":2947123190}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6816","nonce":2947123190},{"type":"v1","addr":"172.21.15.22:6817","nonce":2947123190}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6814","nonce":2947123190},{"type":"v1","addr":"172.21.15.22:6815","nonce":2947123190}]},"public_addr":"172.21.15.22:6811/2947123190","cluster_addr":"172.21.15.22:6813/2947123190","heartbeat_back_addr":"172.21.15.22:6817/2947123190","heartbeat_front_addr":"172.21.15.22:6815/2947123190","state":["exists","up"]},{"osd":2,"uuid":"1c27d056-2218-47dd-8f13-bc1c13888483","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":397,"up_from":410,"up_thru":1045,"down_at":398,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6818","nonce":4029603729},{"type":"v1","addr":"172.21.15.22:6819","nonce":4029603729}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6820","nonce":4029603729},{"type":"v1","addr":"172.21.15.22:6821","nonce":4029603729}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6824","nonce":4029603729},{"type":"v1","addr":"172.21.15.22:6825","nonce":4029603729}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6822","nonce":4029603729},{"type":"v1","addr":"172.21.15.22:6823","nonce":4029603729}]},"public_addr":"172.21.15.22:6819/4029603729","cluster_addr":"172.21.15.22:6821/4029603729","heartbeat_back_addr":"172.21.15.22:6825/4029603729","heartbeat_front_addr":"172.21.15.22:6823/4029603729","state":["exists","up"]},{"osd":3,"uuid":"8668a232-df4a-46ba-b97d-3ed2f6ddeab2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":431,"up_from":443,"up_thru":1045,"down_at":432,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6826","nonce":4226372252},{"type":"v1","addr":"172.21.15.22:6827","nonce":4226372252}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6828","nonce":4226372252},{"type":"v1","addr":"172.21.15.22:6829","nonce":4226372252}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6832","nonce":4226372252},{"type":"v1","addr":"172.21.15.22:6833","nonce":4226372252}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.22:6830","nonce":4226372252},{"type":"v1","addr":"172.21.15.22:6831","nonce":4226372252}]},"public_addr":"172.21.15.22:6827/4226372252","cluster_addr":"172.21.15.22:6829/4226372252","heartbeat_back_addr":"172.21.15.22:6833/4226372252","heartbeat_front_addr":"172.21.15.22:6831/4226372252","state":["exists","up"]},{"osd":4,"uuid":"44dd8dcc-dbf7-41f7-9ec2-e66922e6c521","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":30,"last_clean_end":465,"up_from":477,"up_thru":1045,"down_at":466,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2985474018},{"type":"v1","addr":"172.21.15.64:6801","nonce":2985474018}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6802","nonce":2985474018},{"type":"v1","addr":"172.21.15.64:6803","nonce":2985474018}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6806","nonce":2985474018},{"type":"v1","addr":"172.21.15.64:6807","nonce":2985474018}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":2985474018},{"type":"v1","addr":"172.21.15.64:6805","nonce":2985474018}]},"public_addr":"172.21.15.64:6801/2985474018","cluster_addr":"172.21.15.64:6803/2985474018","heartbeat_back_addr":"172.21.15.64:6807/2985474018","heartbeat_front_addr":"172.21.15.64:6805/2985474018","state":["exists","up"]},{"osd":5,"uuid":"a920292e-6301-4b67-b2a1-30f4788f8fd3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":36,"last_clean_end":498,"up_from":512,"up_thru":1045,"down_at":499,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":2760953518},{"type":"v1","addr":"172.21.15.64:6809","nonce":2760953518}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2760953518},{"type":"v1","addr":"172.21.15.64:6811","nonce":2760953518}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2760953518},{"type":"v1","addr":"172.21.15.64:6815","nonce":2760953518}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6812","nonce":2760953518},{"type":"v1","addr":"172.21.15.64:6813","nonce":2760953518}]},"public_addr":"172.21.15.64:6809/2760953518","cluster_addr":"172.21.15.64:6811/2760953518","heartbeat_back_addr":"172.21.15.64:6815/2760953518","heartbeat_front_addr":"172.21.15.64:6813/2760953518","state":["exists","up"]},{"osd":6,"uuid":"ed01fe25-9b7d-48b9-b054-e054280e5826","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":42,"last_clean_end":532,"up_from":543,"up_thru":1045,"down_at":533,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1881689898},{"type":"v1","addr":"172.21.15.64:6817","nonce":1881689898}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1881689898},{"type":"v1","addr":"172.21.15.64:6819","nonce":1881689898}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1881689898},{"type":"v1","addr":"172.21.15.64:6823","nonce":1881689898}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1881689898},{"type":"v1","addr":"172.21.15.64:6821","nonce":1881689898}]},"public_addr":"172.21.15.64:6817/1881689898","cluster_addr":"172.21.15.64:6819/1881689898","heartbeat_back_addr":"172.21.15.64:6823/1881689898","heartbeat_front_addr":"172.21.15.64:6821/1881689898","state":["exists","up"]},{"osd":7,"uuid":"a9433be2-fcc7-46f5-91ee-13fbe71bf2f7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":48,"last_clean_end":565,"up_from":578,"up_thru":1045,"down_at":566,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":927270672},{"type":"v1","addr":"172.21.15.64:6825","nonce":927270672}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":927270672},{"type":"v1","addr":"172.21.15.64:6827","nonce":927270672}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":927270672},{"type":"v1","addr":"172.21.15.64:6831","nonce":927270672}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":927270672},{"type":"v1","addr":"172.21.15.64:6829","nonce":927270672}]},"public_addr":"172.21.15.64:6825/927270672","cluster_addr":"172.21.15.64:6827/927270672","heartbeat_back_addr":"172.21.15.64:6831/927270672","heartbeat_front_addr":"172.21.15.64:6829/927270672","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:24:15.825542+0000","dead_epoch":329},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:24:40.497779+0000","dead_epoch":363},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:25:04.918091+0000","dead_epoch":397},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:25:30.663725+0000","dead_epoch":431},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:25:53.521949+0000","dead_epoch":465},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:26:17.565302+0000","dead_epoch":498},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:26:40.876016+0000","dead_epoch":532},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:27:04.436407+0000","dead_epoch":565}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.3","mappings":[{"from":7,"to":4}]},{"pgid":"3.a","mappings":[{"from":4,"to":3}]},{"pgid":"3.b","mappings":[{"from":0,"to":2}]},{"pgid":"3.17","mappings":[{"from":0,"to":2}]},{"pgid":"3.22","mappings":[{"from":0,"to":1}]},{"pgid":"3.24","mappings":[{"from":4,"to":2}]},{"pgid":"3.37","mappings":[{"from":5,"to":1}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.22:6801/3012222320":"2024-09-22T09:34:42.310852+0000","172.21.15.22:0/2451188142":"2024-09-22T09:34:42.310852+0000","172.21.15.22:0/4095519345":"2024-09-22T09:34:42.310852+0000","172.21.15.22:6800/3012222320":"2024-09-22T09:34:42.310852+0000","172.21.15.22:0/1238499882":"2024-09-22T09:34:42.310852+0000","172.21.15.22:0/1065832205":"2024-09-22T09:34:42.310852+0000","172.21.15.22:0/1728251114":"2024-09-22T09:22:26.473640+0000","172.21.15.22:0/4075962746":"2024-09-22T09:32:55.001383+0000","172.21.15.22:6800/3088744553":"2024-09-22T09:32:55.001383+0000","172.21.15.64:6835/3987771184":"2024-09-22T09:34:00.103818+0000","172.21.15.22:0/1467628521":"2024-09-22T09:22:26.473640+0000","172.21.15.22:0/4254041618":"2024-09-22T09:28:26.777111+0000","172.21.15.22:0/4200462415":"2024-09-22T09:22:50.910682+0000","172.21.15.22:6801/3088744553":"2024-09-22T09:32:55.001383+0000","172.21.15.22:0/2059096138":"2024-09-22T09:34:42.310852+0000","172.21.15.64:6834/3987771184":"2024-09-22T09:34:00.103818+0000","172.21.15.22:6801/2706373818":"2024-09-22T09:28:26.777111+0000","172.21.15.64:0/40177038":"2024-09-22T09:34:00.103818+0000","172.21.15.22:0/1937461032":"2024-09-22T09:28:26.777111+0000","172.21.15.64:6832/3589447551":"2024-09-22T09:40:22.989105+0000","172.21.15.22:0/1998705954":"2024-09-22T09:28:26.777111+0000","172.21.15.22:0/2669480478":"2024-09-22T09:22:50.910682+0000","172.21.15.64:0/2254922149":"2024-09-22T09:34:00.103818+0000","172.21.15.64:0/3221881884":"2024-09-22T09:34:00.103818+0000","172.21.15.64:0/2724345611":"2024-09-22T09:34:00.103818+0000","172.21.15.22:6800/2983422368":"2024-09-22T09:22:26.473640+0000","172.21.15.22:0/3249188488":"2024-09-22T09:28:26.777111+0000","172.21.15.64:6833/3589447551":"2024-09-22T09:40:22.989105+0000","172.21.15.22:6800/2706373818":"2024-09-22T09:28:26.777111+0000","172.21.15.64:0/1545650826":"2024-09-22T09:34:00.103818+0000","172.21.15.22:6800/2821000087":"2024-09-22T09:22:50.910682+0000","172.21.15.22:0/2111825743":"2024-09-22T09:32:55.001383+0000","172.21.15.22:0/3766882712":"2024-09-22T09:22:50.910682+0000","172.21.15.22:6801/2821000087":"2024-09-22T09:22:50.910682+0000","172.21.15.22:0/438209926":"2024-09-22T09:32:55.001383+0000","172.21.15.22:0/932593602":"2024-09-22T09:32:55.001383+0000","172.21.15.22:0/2844144671":"2024-09-22T09:34:42.310852+0000","172.21.15.22:6801/2983422368":"2024-09-22T09:22:26.473640+0000","172.21.15.64:0/2593942244":"2024-09-22T09:34:00.103818+0000","172.21.15.22:0/1022854075":"2024-09-22T09:22:26.473640+0000","172.21.15.22:0/3644531628":"2024-09-22T09:32:55.001383+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"},"teuthologyprofile":{"crush-device-class":"","crush-failure-domain":"osd","crush-root":"default","jerasure-per-chunk-alignment":"false","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van","w":"8"}},"removed_snaps_queue":[{"pool":60,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-21T09:49:44.728 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:44 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/3395961202' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-21T09:49:44.728 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:44 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/3395961202' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-21T09:49:44.755 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid d65e801e-77fa-11ef-baf6-efdc52797490 -- ceph fs fail cephfs 2024-09-21T09:49:44.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:44 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/3395961202' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-21T09:49:45.013 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:49:45.718 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:45 smithi022 ceph-mon[98615]: pgmap v1235: 105 pgs: 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:49:45.718 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:45 smithi022 ceph-mon[96736]: pgmap v1235: 105 pgs: 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:49:45.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:45 smithi064 ceph-mon[86709]: pgmap v1235: 105 pgs: 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:49:46.058 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:49:45 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:49:45] "GET /metrics HTTP/1.1" 200 34078 "" "Prometheus/2.51.0" 2024-09-21T09:49:46.924 INFO:teuthology.orchestra.run.smithi022.stderr:cephfs marked not joinable; MDS cannot join the cluster. All MDS ranks marked failed. 2024-09-21T09:49:47.235 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:46 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mon-a[96691]: 2024-09-21T09:49:46.920+0000 7fe95466e640 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-21T09:49:47.235 INFO:journalctl@ceph.mds.a.smithi022.stdout:Sep 21 09:49:46 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mds-a[123008]: -4579> 2024-09-21T09:40:28.170+0000 7ff8ba1d6640 -1 mds.pinger is_rank_lagging: rank=0 was never sent ping request. 2024-09-21T09:49:47.235 INFO:journalctl@ceph.mds.a.smithi022.stdout:Sep 21 09:49:46 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mds-a[123008]: ignoring --setuser ceph since I am not root 2024-09-21T09:49:47.235 INFO:journalctl@ceph.mds.a.smithi022.stdout:Sep 21 09:49:46 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mds-a[123008]: ignoring --setgroup ceph since I am not root 2024-09-21T09:49:47.235 INFO:journalctl@ceph.mds.a.smithi022.stdout:Sep 21 09:49:46 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mds-a[123008]: starting mds.a at 2024-09-21T09:49:47.561 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid d65e801e-77fa-11ef-baf6-efdc52797490 -- ceph fs rm cephfs --yes-i-really-mean-it 2024-09-21T09:49:47.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:47 smithi022 ceph-mon[98615]: pgmap v1236: 105 pgs: 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:49:47.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:47 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/2358764653' entity='client.admin' cmd={"prefix": "fs fail", "fs_name": "cephfs"} : dispatch 2024-09-21T09:49:47.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:47 smithi022 ceph-mon[98615]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-21T09:49:47.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:47 smithi022 ceph-mon[98615]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-21T09:49:47.808 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:47 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/2358764653' entity='client.admin' cmd='[{"prefix": "fs fail", "fs_name": "cephfs"}]': finished 2024-09-21T09:49:47.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:47 smithi022 ceph-mon[98615]: osdmap e1051: 8 total, 8 up, 8 in 2024-09-21T09:49:47.809 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:47 smithi022 ceph-mon[98615]: fsmap cephfs:0/1 1 up:standby, 1 failed 2024-09-21T09:49:47.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:47 smithi022 ceph-mon[96736]: pgmap v1236: 105 pgs: 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:49:47.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:47 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/2358764653' entity='client.admin' cmd={"prefix": "fs fail", "fs_name": "cephfs"} : dispatch 2024-09-21T09:49:47.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:47 smithi022 ceph-mon[96736]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-21T09:49:47.809 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:47 smithi022 ceph-mon[96736]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-21T09:49:47.810 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:47 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/2358764653' entity='client.admin' cmd='[{"prefix": "fs fail", "fs_name": "cephfs"}]': finished 2024-09-21T09:49:47.810 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:47 smithi022 ceph-mon[96736]: osdmap e1051: 8 total, 8 up, 8 in 2024-09-21T09:49:47.810 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:47 smithi022 ceph-mon[96736]: fsmap cephfs:0/1 1 up:standby, 1 failed 2024-09-21T09:49:47.822 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:49:47.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:47 smithi064 ceph-mon[86709]: pgmap v1236: 105 pgs: 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:49:47.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:47 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/2358764653' entity='client.admin' cmd={"prefix": "fs fail", "fs_name": "cephfs"} : dispatch 2024-09-21T09:49:47.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:47 smithi064 ceph-mon[86709]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-21T09:49:47.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:47 smithi064 ceph-mon[86709]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-21T09:49:47.943 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:47 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/2358764653' entity='client.admin' cmd='[{"prefix": "fs fail", "fs_name": "cephfs"}]': finished 2024-09-21T09:49:47.943 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:47 smithi064 ceph-mon[86709]: osdmap e1051: 8 total, 8 up, 8 in 2024-09-21T09:49:47.943 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:47 smithi064 ceph-mon[86709]: fsmap cephfs:0/1 1 up:standby, 1 failed 2024-09-21T09:49:49.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:48 smithi022 ceph-mon[98615]: mds.? [v2:172.21.15.22:6834/2456065250,v1:172.21.15.22:6835/2456065250] up:boot 2024-09-21T09:49:49.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:48 smithi022 ceph-mon[98615]: fsmap cephfs:0/1 2 up:standby, 1 failed 2024-09-21T09:49:49.058 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:48 smithi022 ceph-mon[98615]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "a"} : dispatch 2024-09-21T09:49:49.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:48 smithi022 ceph-mon[96736]: mds.? [v2:172.21.15.22:6834/2456065250,v1:172.21.15.22:6835/2456065250] up:boot 2024-09-21T09:49:49.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:48 smithi022 ceph-mon[96736]: fsmap cephfs:0/1 2 up:standby, 1 failed 2024-09-21T09:49:49.059 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:48 smithi022 ceph-mon[96736]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "a"} : dispatch 2024-09-21T09:49:49.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:48 smithi064 ceph-mon[86709]: mds.? [v2:172.21.15.22:6834/2456065250,v1:172.21.15.22:6835/2456065250] up:boot 2024-09-21T09:49:49.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:48 smithi064 ceph-mon[86709]: fsmap cephfs:0/1 2 up:standby, 1 failed 2024-09-21T09:49:49.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:48 smithi064 ceph-mon[86709]: from='mgr.24949 172.21.15.22:0/1671158210' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "a"} : dispatch 2024-09-21T09:49:50.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:49 smithi064 ceph-mon[86709]: pgmap v1238: 105 pgs: 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:49:50.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:49 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/397571988' entity='client.admin' cmd={"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true} : dispatch 2024-09-21T09:49:50.227 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:49 smithi022 ceph-mon[98615]: pgmap v1238: 105 pgs: 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:49:50.227 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:49 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/397571988' entity='client.admin' cmd={"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true} : dispatch 2024-09-21T09:49:50.227 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:49 smithi022 ceph-mon[96736]: pgmap v1238: 105 pgs: 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:49:50.227 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:49 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/397571988' entity='client.admin' cmd={"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true} : dispatch 2024-09-21T09:49:50.547 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid d65e801e-77fa-11ef-baf6-efdc52797490 -- ceph osd pool rm cephfs_metadata cephfs_metadata --yes-i-really-really-mean-it 2024-09-21T09:49:50.692 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:49:50 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:49:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:49:50.809 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:49:51.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:50 smithi064 ceph-mon[86709]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-21T09:49:51.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:50 smithi064 ceph-mon[86709]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-21T09:49:51.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:50 smithi064 ceph-mon[86709]: Cluster is now healthy 2024-09-21T09:49:51.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:50 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/397571988' entity='client.admin' cmd='[{"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true}]': finished 2024-09-21T09:49:51.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:50 smithi064 ceph-mon[86709]: fsmap 2 up:standby 2024-09-21T09:49:51.196 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:50 smithi022 ceph-mon[98615]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-21T09:49:51.196 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:50 smithi022 ceph-mon[98615]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-21T09:49:51.196 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:50 smithi022 ceph-mon[98615]: Cluster is now healthy 2024-09-21T09:49:51.196 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:50 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/397571988' entity='client.admin' cmd='[{"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true}]': finished 2024-09-21T09:49:51.196 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:50 smithi022 ceph-mon[98615]: fsmap 2 up:standby 2024-09-21T09:49:51.197 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:50 smithi022 ceph-mon[96736]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-21T09:49:51.197 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:50 smithi022 ceph-mon[96736]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-21T09:49:51.197 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:50 smithi022 ceph-mon[96736]: Cluster is now healthy 2024-09-21T09:49:51.197 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:50 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/397571988' entity='client.admin' cmd='[{"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true}]': finished 2024-09-21T09:49:51.197 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:50 smithi022 ceph-mon[96736]: fsmap 2 up:standby 2024-09-21T09:49:52.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:51 smithi064 ceph-mon[86709]: pgmap v1239: 105 pgs: 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:49:52.227 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:51 smithi022 ceph-mon[98615]: pgmap v1239: 105 pgs: 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:49:52.227 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:51 smithi022 ceph-mon[96736]: pgmap v1239: 105 pgs: 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:49:52.947 INFO:teuthology.orchestra.run.smithi022.stderr:pool 'cephfs_metadata' removed 2024-09-21T09:49:53.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:52 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/2175038264' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true} : dispatch 2024-09-21T09:49:53.308 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:52 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/2175038264' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true} : dispatch 2024-09-21T09:49:53.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:52 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/2175038264' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true} : dispatch 2024-09-21T09:49:53.621 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid d65e801e-77fa-11ef-baf6-efdc52797490 -- ceph osd pool rm cephfs_data cephfs_data --yes-i-really-really-mean-it 2024-09-21T09:49:53.882 INFO:teuthology.orchestra.run.smithi022.stderr:Inferring config /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:49:54.278 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:53 smithi022 ceph-mon[98615]: pgmap v1240: 105 pgs: 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:49:54.278 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:53 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/2175038264' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true}]': finished 2024-09-21T09:49:54.278 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:53 smithi022 ceph-mon[98615]: osdmap e1052: 8 total, 8 up, 8 in 2024-09-21T09:49:54.279 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:53 smithi022 ceph-mon[96736]: pgmap v1240: 105 pgs: 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:49:54.279 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:53 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/2175038264' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true}]': finished 2024-09-21T09:49:54.279 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:53 smithi022 ceph-mon[96736]: osdmap e1052: 8 total, 8 up, 8 in 2024-09-21T09:49:54.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:53 smithi064 ceph-mon[86709]: pgmap v1240: 105 pgs: 105 active+clean; 583 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:49:54.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:53 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/2175038264' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true}]': finished 2024-09-21T09:49:54.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:53 smithi064 ceph-mon[86709]: osdmap e1052: 8 total, 8 up, 8 in 2024-09-21T09:49:55.955 INFO:teuthology.orchestra.run.smithi022.stderr:pool 'cephfs_data' removed 2024-09-21T09:49:55.972 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:55 smithi022 ceph-mon[98615]: pgmap v1242: 73 pgs: 73 active+clean; 577 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:49:55.973 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:55 smithi022 ceph-mon[98615]: from='client.? 172.21.15.22:0/661900103' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true} : dispatch 2024-09-21T09:49:55.973 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:55 smithi022 ceph-mon[98615]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true} : dispatch 2024-09-21T09:49:55.973 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:55 smithi022 ceph-mon[96736]: pgmap v1242: 73 pgs: 73 active+clean; 577 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:49:55.973 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:55 smithi022 ceph-mon[96736]: from='client.? 172.21.15.22:0/661900103' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true} : dispatch 2024-09-21T09:49:55.973 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:55 smithi022 ceph-mon[96736]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true} : dispatch 2024-09-21T09:49:55.974 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:49:55 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: ::ffff:172.21.15.64 - - [21/Sep/2024:09:49:55] "GET /metrics HTTP/1.1" 200 34078 "" "Prometheus/2.51.0" 2024-09-21T09:49:56.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:55 smithi064 ceph-mon[86709]: pgmap v1242: 73 pgs: 73 active+clean; 577 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:49:56.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:55 smithi064 ceph-mon[86709]: from='client.? 172.21.15.22:0/661900103' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true} : dispatch 2024-09-21T09:49:56.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:55 smithi064 ceph-mon[86709]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true} : dispatch 2024-09-21T09:49:56.670 DEBUG:teuthology.orchestra.run.smithi022:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-21T09:49:56.701 DEBUG:teuthology.orchestra.run.smithi064:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-21T09:49:56.736 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-09-21T09:49:56.736 DEBUG:teuthology.orchestra.run.smithi022:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-09-21T09:49:56.759 DEBUG:teuthology.orchestra.run.smithi064:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-09-21T09:49:56.795 INFO:tasks.cephadm:Stopping all daemons... 2024-09-21T09:49:56.795 INFO:tasks.cephadm.mon.a:Stopping mon.a... 2024-09-21T09:49:56.795 DEBUG:teuthology.orchestra.run.smithi022:> sudo systemctl stop ceph-d65e801e-77fa-11ef-baf6-efdc52797490@mon.a 2024-09-21T09:49:56.959 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:56 smithi022 systemd[1]: Stopping Ceph mon.a for d65e801e-77fa-11ef-baf6-efdc52797490... 2024-09-21T09:49:57.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:56 smithi064 ceph-mon[86709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true}]': finished 2024-09-21T09:49:57.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:56 smithi064 ceph-mon[86709]: osdmap e1053: 8 total, 8 up, 8 in 2024-09-21T09:49:57.223 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:49:57 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: [21/Sep/2024:09:49:57] ENGINE Bus STOPPING 2024-09-21T09:49:57.223 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:56 smithi022 ceph-mon[98615]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true}]': finished 2024-09-21T09:49:57.223 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:56 smithi022 ceph-mon[98615]: osdmap e1053: 8 total, 8 up, 8 in 2024-09-21T09:49:57.224 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:56 smithi022 ceph-mon[96736]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true}]': finished 2024-09-21T09:49:57.224 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:56 smithi022 ceph-mon[96736]: osdmap e1053: 8 total, 8 up, 8 in 2024-09-21T09:49:57.224 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:57 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mon-a[96691]: 2024-09-21T09:49:57.119+0000 7fe959e79640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T09:49:57.224 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:57 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mon-a[96691]: 2024-09-21T09:49:57.119+0000 7fe959e79640 -1 mon.a@0(leader) e4 *** Got Signal Terminated *** 2024-09-21T09:49:57.558 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:57 smithi022 podman[131918]: 2024-09-21 09:49:57.364032189 +0000 UTC m=+0.369433646 container died 0513395c784d2ca33f510b41dec5e34718168c717e222018230be292f24d09d1 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mon-a, RELEASE=main-62da54b, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS) 2024-09-21T09:49:57.558 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:49:57 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: [21/Sep/2024:09:49:57] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-21T09:49:57.558 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:49:57 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: [21/Sep/2024:09:49:57] ENGINE Bus STOPPED 2024-09-21T09:49:57.558 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:49:57 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: [21/Sep/2024:09:49:57] ENGINE Bus STARTING 2024-09-21T09:49:57.559 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:49:57 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: [21/Sep/2024:09:49:57] ENGINE Serving on http://:::9283 2024-09-21T09:49:57.559 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:49:57 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: [21/Sep/2024:09:49:57] ENGINE Bus STARTED 2024-09-21T09:49:57.992 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:57 smithi022 podman[131918]: 2024-09-21 09:49:57.767574269 +0000 UTC m=+0.772975720 container cleanup 0513395c784d2ca33f510b41dec5e34718168c717e222018230be292f24d09d1 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mon-a, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=main-62da54b, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, ceph=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T09:49:57.992 INFO:journalctl@ceph.mon.a.smithi022.stdout:Sep 21 09:49:57 smithi022 bash[131918]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mon-a 2024-09-21T09:49:58.197 DEBUG:teuthology.orchestra.run.smithi022:> sudo pkill -f 'journalctl -f -n 0 -u ceph-d65e801e-77fa-11ef-baf6-efdc52797490@mon.a.service' 2024-09-21T09:49:58.252 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T09:49:58.253 INFO:tasks.cephadm.mon.a:Stopped mon.a 2024-09-21T09:49:58.253 INFO:tasks.cephadm.mon.b:Stopping mon.c... 2024-09-21T09:49:58.253 DEBUG:teuthology.orchestra.run.smithi022:> sudo systemctl stop ceph-d65e801e-77fa-11ef-baf6-efdc52797490@mon.c 2024-09-21T09:49:58.558 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:58 smithi022 systemd[1]: Stopping Ceph mon.c for d65e801e-77fa-11ef-baf6-efdc52797490... 2024-09-21T09:49:58.829 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:49:58 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: [21/Sep/2024:09:49:58] ENGINE Bus STOPPING 2024-09-21T09:49:58.830 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:58 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mon-c[98592]: 2024-09-21T09:49:58.566+0000 7faafba25640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T09:49:58.830 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:58 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mon-c[98592]: 2024-09-21T09:49:58.566+0000 7faafba25640 -1 mon.c@2(peon) e4 *** Got Signal Terminated *** 2024-09-21T09:49:58.830 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:58 smithi022 podman[132072]: 2024-09-21 09:49:58.658446846 +0000 UTC m=+0.210152478 container died c74b58b67bd6f4da50b76ef30d2472b035ec784c9d751309124c186a279eca21 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mon-c, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, RELEASE=main-62da54b, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T09:49:59.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:59 smithi022 podman[132072]: 2024-09-21 09:49:59.006318376 +0000 UTC m=+0.558024002 container cleanup c74b58b67bd6f4da50b76ef30d2472b035ec784c9d751309124c186a279eca21 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mon-c, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, RELEASE=main-62da54b, ceph=True, io.buildah.version=1.37.2) 2024-09-21T09:49:59.308 INFO:journalctl@ceph.mon.c.smithi022.stdout:Sep 21 09:49:59 smithi022 bash[132072]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mon-c 2024-09-21T09:49:59.308 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:49:58 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: [21/Sep/2024:09:49:58] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-21T09:49:59.308 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:49:58 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: [21/Sep/2024:09:49:58] ENGINE Bus STOPPED 2024-09-21T09:49:59.309 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:49:58 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: [21/Sep/2024:09:49:58] ENGINE Bus STARTING 2024-09-21T09:49:59.309 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:49:58 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: [21/Sep/2024:09:49:58] ENGINE Serving on http://:::9283 2024-09-21T09:49:59.309 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:49:58 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: [21/Sep/2024:09:49:58] ENGINE Bus STARTED 2024-09-21T09:49:59.494 DEBUG:teuthology.orchestra.run.smithi022:> sudo pkill -f 'journalctl -f -n 0 -u ceph-d65e801e-77fa-11ef-baf6-efdc52797490@mon.c.service' 2024-09-21T09:49:59.530 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T09:49:59.530 INFO:tasks.cephadm.mon.b:Stopped mon.c 2024-09-21T09:49:59.530 INFO:tasks.cephadm.mon.b:Stopping mon.b... 2024-09-21T09:49:59.531 DEBUG:teuthology.orchestra.run.smithi064:> sudo systemctl stop ceph-d65e801e-77fa-11ef-baf6-efdc52797490@mon.b 2024-09-21T09:49:59.834 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:59 smithi064 systemd[1]: Stopping Ceph mon.b for d65e801e-77fa-11ef-baf6-efdc52797490... 2024-09-21T09:50:00.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:59 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mon-b[86686]: 2024-09-21T09:49:59.831+0000 7fbea0189640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T09:50:00.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:59 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mon-b[86686]: 2024-09-21T09:49:59.831+0000 7fbea0189640 -1 mon.b@1(peon) e4 *** Got Signal Terminated *** 2024-09-21T09:50:00.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:49:59 smithi064 podman[118855]: 2024-09-21 09:49:59.966115914 +0000 UTC m=+0.231282653 container died 9dab26b6a9e74bd6f1779d35c9af92a426b9e4512d0b6dfb28eb894dab7b1ff7 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mon-b, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-62da54b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.vendor=CentOS) 2024-09-21T09:50:00.574 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:50:00 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:50:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"d65e801e-77fa-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:50:00.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:50:00 smithi064 podman[118855]: 2024-09-21 09:50:00.573302411 +0000 UTC m=+0.838469145 container cleanup 9dab26b6a9e74bd6f1779d35c9af92a426b9e4512d0b6dfb28eb894dab7b1ff7 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mon-b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=main-62da54b, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=) 2024-09-21T09:50:00.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:50:00 smithi064 bash[118855]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mon-b 2024-09-21T09:50:00.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Sep 21 09:50:00 smithi064 podman[118872]: 2024-09-21 09:50:00.79969636 +0000 UTC m=+0.829116047 container remove 9dab26b6a9e74bd6f1779d35c9af92a426b9e4512d0b6dfb28eb894dab7b1ff7 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mon-b, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=main-62da54b, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True) 2024-09-21T09:50:00.966 DEBUG:teuthology.orchestra.run.smithi064:> sudo pkill -f 'journalctl -f -n 0 -u ceph-d65e801e-77fa-11ef-baf6-efdc52797490@mon.b.service' 2024-09-21T09:50:01.008 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T09:50:01.009 INFO:tasks.cephadm.mon.b:Stopped mon.b 2024-09-21T09:50:01.009 INFO:tasks.cephadm.mgr.y:Stopping mgr.y... 2024-09-21T09:50:01.009 DEBUG:teuthology.orchestra.run.smithi022:> sudo systemctl stop ceph-d65e801e-77fa-11ef-baf6-efdc52797490@mgr.y 2024-09-21T09:50:01.308 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:50:01 smithi022 systemd[1]: Stopping Ceph mgr.y for d65e801e-77fa-11ef-baf6-efdc52797490... 2024-09-21T09:50:01.630 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:50:01 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: 2024-09-21T09:50:01.312+0000 7fef1fdee640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mgr -n mgr.y -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T09:50:01.631 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:50:01 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y[88361]: 2024-09-21T09:50:01.312+0000 7fef1fdee640 -1 mgr handle_mgr_signal *** Got signal Terminated *** 2024-09-21T09:50:01.631 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:50:01 smithi022 podman[132216]: 2024-09-21 09:50:01.344632501 +0000 UTC m=+0.139840370 container died 6c5bee78f09b7a71fe7142f96cb10a468f15bf8e8a7f1a81a23e0a36feca7ca8 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, RELEASE=main-62da54b, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0) 2024-09-21T09:50:01.947 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:50:01 smithi022 podman[132216]: 2024-09-21 09:50:01.744406102 +0000 UTC m=+0.539613960 container cleanup 6c5bee78f09b7a71fe7142f96cb10a468f15bf8e8a7f1a81a23e0a36feca7ca8 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=main-62da54b, io.buildah.version=1.37.2, GIT_CLEAN=True) 2024-09-21T09:50:01.947 INFO:journalctl@ceph.mgr.y.smithi022.stdout:Sep 21 09:50:01 smithi022 bash[132216]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-y 2024-09-21T09:50:02.092 DEBUG:teuthology.orchestra.run.smithi022:> sudo pkill -f 'journalctl -f -n 0 -u ceph-d65e801e-77fa-11ef-baf6-efdc52797490@mgr.y.service' 2024-09-21T09:50:02.128 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T09:50:02.129 INFO:tasks.cephadm.mgr.y:Stopped mgr.y 2024-09-21T09:50:02.129 INFO:tasks.cephadm.mgr.x:Stopping mgr.x... 2024-09-21T09:50:02.129 DEBUG:teuthology.orchestra.run.smithi064:> sudo systemctl stop ceph-d65e801e-77fa-11ef-baf6-efdc52797490@mgr.x 2024-09-21T09:50:02.441 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:50:02 smithi064 systemd[1]: Stopping Ceph mgr.x for d65e801e-77fa-11ef-baf6-efdc52797490... 2024-09-21T09:50:02.755 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:50:02 smithi064 podman[118989]: 2024-09-21 09:50:02.469210067 +0000 UTC m=+0.140461854 container died 61abc61fea8d38822833e6734ef06390abf36052535b0473748ff5f90aae7d43 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, RELEASE=main-62da54b, ceph=True) 2024-09-21T09:50:03.191 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:50:02 smithi064 podman[118989]: 2024-09-21 09:50:02.86108185 +0000 UTC m=+0.532333634 container cleanup 61abc61fea8d38822833e6734ef06390abf36052535b0473748ff5f90aae7d43 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, RELEASE=main-62da54b, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, io.buildah.version=1.37.2) 2024-09-21T09:50:03.192 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Sep 21 09:50:02 smithi064 bash[118989]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mgr-x 2024-09-21T09:50:03.447 DEBUG:teuthology.orchestra.run.smithi064:> sudo pkill -f 'journalctl -f -n 0 -u ceph-d65e801e-77fa-11ef-baf6-efdc52797490@mgr.x.service' 2024-09-21T09:50:03.483 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T09:50:03.483 INFO:tasks.cephadm.mgr.x:Stopped mgr.x 2024-09-21T09:50:03.483 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-09-21T09:50:03.484 DEBUG:teuthology.orchestra.run.smithi022:> sudo systemctl stop ceph-d65e801e-77fa-11ef-baf6-efdc52797490@osd.0 2024-09-21T09:50:03.786 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 21 09:50:03 smithi022 systemd[1]: Stopping Ceph osd.0 for d65e801e-77fa-11ef-baf6-efdc52797490... 2024-09-21T09:50:04.058 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 21 09:50:03 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-0[103843]: 2024-09-21T09:50:03.784+0000 7f9f6157b640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T09:50:04.058 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 21 09:50:03 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-0[103843]: 2024-09-21T09:50:03.784+0000 7f9f6157b640 -1 osd.0 1053 *** Got signal Terminated *** 2024-09-21T09:50:04.058 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 21 09:50:03 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-0[103843]: 2024-09-21T09:50:03.784+0000 7f9f6157b640 -1 osd.0 1053 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T09:50:09.552 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 21 09:50:09 smithi022 podman[132350]: 2024-09-21 09:50:09.258603665 +0000 UTC m=+5.576062501 container died 40ce23a2b5692f7801d559f179ec2889d38d95b7dc65fc444d6d3438210e26e3 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-0, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-62da54b, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731) 2024-09-21T09:50:09.808 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 21 09:50:09 smithi022 podman[132350]: 2024-09-21 09:50:09.632870493 +0000 UTC m=+5.950329327 container cleanup 40ce23a2b5692f7801d559f179ec2889d38d95b7dc65fc444d6d3438210e26e3 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-0, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, ceph=True, RELEASE=main-62da54b, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2) 2024-09-21T09:50:09.808 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 21 09:50:09 smithi022 bash[132350]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-0 2024-09-21T09:50:10.260 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 21 09:50:09 smithi022 podman[132374]: 2024-09-21 09:50:09.901143923 +0000 UTC m=+0.638235426 container remove 40ce23a2b5692f7801d559f179ec2889d38d95b7dc65fc444d6d3438210e26e3 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-0, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, RELEASE=main-62da54b, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, ceph=True) 2024-09-21T09:50:10.522 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 21 09:50:10 smithi022 podman[132465]: 2024-09-21 09:50:10.161462163 +0000 UTC m=+0.016711006 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:50:10.523 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 21 09:50:10 smithi022 podman[132465]: 2024-09-21 09:50:10.285604878 +0000 UTC m=+0.140853721 container create 9710551fb5a4de8dd7ded62ed84e3379ee6f658a642ed939ef57825a6b67cc2c (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-0-deactivate, org.label-schema.schema-version=1.0, RELEASE=main-62da54b, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T09:50:10.808 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 21 09:50:10 smithi022 podman[132465]: 2024-09-21 09:50:10.521608697 +0000 UTC m=+0.376857547 container init 9710551fb5a4de8dd7ded62ed84e3379ee6f658a642ed939ef57825a6b67cc2c (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-0-deactivate, maintainer=Guillaume Abrioux , ceph=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=main-62da54b, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T09:50:10.809 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 21 09:50:10 smithi022 podman[132465]: 2024-09-21 09:50:10.526754914 +0000 UTC m=+0.382003758 container start 9710551fb5a4de8dd7ded62ed84e3379ee6f658a642ed939ef57825a6b67cc2c (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-0-deactivate, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, RELEASE=main-62da54b, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, ceph=True, org.label-schema.vendor=CentOS) 2024-09-21T09:50:10.809 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 21 09:50:10 smithi022 podman[132465]: 2024-09-21 09:50:10.613139167 +0000 UTC m=+0.468388009 container attach 9710551fb5a4de8dd7ded62ed84e3379ee6f658a642ed939ef57825a6b67cc2c (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-0-deactivate, RELEASE=main-62da54b, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, ceph=True, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731) 2024-09-21T09:50:10.809 INFO:journalctl@ceph.osd.0.smithi022.stdout:Sep 21 09:50:10 smithi022 podman[132465]: 2024-09-21 09:50:10.720473962 +0000 UTC m=+0.575722805 container died 9710551fb5a4de8dd7ded62ed84e3379ee6f658a642ed939ef57825a6b67cc2c (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-0-deactivate, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, RELEASE=main-62da54b, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731) 2024-09-21T09:50:11.547 DEBUG:teuthology.orchestra.run.smithi022:> sudo pkill -f 'journalctl -f -n 0 -u ceph-d65e801e-77fa-11ef-baf6-efdc52797490@osd.0.service' 2024-09-21T09:50:11.591 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T09:50:11.591 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-09-21T09:50:11.591 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-09-21T09:50:11.591 DEBUG:teuthology.orchestra.run.smithi022:> sudo systemctl stop ceph-d65e801e-77fa-11ef-baf6-efdc52797490@osd.1 2024-09-21T09:50:11.808 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 21 09:50:11 smithi022 systemd[1]: Stopping Ceph osd.1 for d65e801e-77fa-11ef-baf6-efdc52797490... 2024-09-21T09:50:12.308 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 21 09:50:11 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-1[108459]: 2024-09-21T09:50:11.989+0000 7f8cd3367640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T09:50:12.308 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 21 09:50:11 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-1[108459]: 2024-09-21T09:50:11.989+0000 7f8cd3367640 -1 osd.1 1053 *** Got signal Terminated *** 2024-09-21T09:50:12.308 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 21 09:50:11 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-1[108459]: 2024-09-21T09:50:11.989+0000 7f8cd3367640 -1 osd.1 1053 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T09:50:15.442 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:50:15 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:50:15.163Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=nvmeof msg="Unable to refresh target groups" err="Get \"http://172.21.15.22:8765/sd/prometheus/sd-config?service=nvmeof\": dial tcp 172.21.15.22:8765: connect: connection refused" 2024-09-21T09:50:15.442 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:50:15 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:50:15.163Z caller=refresh.go:90 level=error component="discovery manager notify" discovery=http config=config-0 msg="Unable to refresh target groups" err="Get \"http://172.21.15.22:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.22:8765: connect: connection refused" 2024-09-21T09:50:15.442 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:50:15 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:50:15.168Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=smb msg="Unable to refresh target groups" err="Get \"http://172.21.15.22:8765/sd/prometheus/sd-config?service=smb\": dial tcp 172.21.15.22:8765: connect: connection refused" 2024-09-21T09:50:15.442 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:50:15 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:50:15.168Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.22:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.22:8765: connect: connection refused" 2024-09-21T09:50:15.442 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:50:15 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:50:15.168Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=node msg="Unable to refresh target groups" err="Get \"http://172.21.15.22:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.22:8765: connect: connection refused" 2024-09-21T09:50:15.443 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:50:15 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:50:15.168Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=nfs msg="Unable to refresh target groups" err="Get \"http://172.21.15.22:8765/sd/prometheus/sd-config?service=nfs\": dial tcp 172.21.15.22:8765: connect: connection refused" 2024-09-21T09:50:15.443 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:50:15 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:50:15.170Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.22:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.22:8765: connect: connection refused" 2024-09-21T09:50:17.698 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 21 09:50:17 smithi022 podman[132594]: 2024-09-21 09:50:17.430989486 +0000 UTC m=+5.554256225 container died 7cd0c18d224c55427944767fc60453ec5d64c875b6a7ca6defa3978dd350bd30 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-1, GIT_BRANCH=HEAD, RELEASE=main-62da54b, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, io.buildah.version=1.37.2) 2024-09-21T09:50:18.058 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 21 09:50:17 smithi022 podman[132594]: 2024-09-21 09:50:17.870629018 +0000 UTC m=+5.993895757 container cleanup 7cd0c18d224c55427944767fc60453ec5d64c875b6a7ca6defa3978dd350bd30 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-1, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=main-62da54b, ceph=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-09-21T09:50:18.058 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 21 09:50:17 smithi022 bash[132594]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-1 2024-09-21T09:50:18.550 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 21 09:50:18 smithi022 podman[132616]: 2024-09-21 09:50:18.114220059 +0000 UTC m=+0.679618303 container remove 7cd0c18d224c55427944767fc60453ec5d64c875b6a7ca6defa3978dd350bd30 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-1, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-62da54b, ceph=True, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS) 2024-09-21T09:50:18.808 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 21 09:50:18 smithi022 podman[132712]: 2024-09-21 09:50:18.452373196 +0000 UTC m=+0.016467237 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:50:18.808 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 21 09:50:18 smithi022 podman[132712]: 2024-09-21 09:50:18.573217732 +0000 UTC m=+0.137311758 container create 6225265b852691112acc35c680459ec9c930b2eaf7231eb8ca781b2248bfb94f (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-1-deactivate, maintainer=Guillaume Abrioux , RELEASE=main-62da54b, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T09:50:18.808 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 21 09:50:18 smithi022 podman[132712]: 2024-09-21 09:50:18.775596419 +0000 UTC m=+0.339690455 container init 6225265b852691112acc35c680459ec9c930b2eaf7231eb8ca781b2248bfb94f (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-1-deactivate, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, RELEASE=main-62da54b) 2024-09-21T09:50:18.808 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 21 09:50:18 smithi022 podman[132712]: 2024-09-21 09:50:18.78064853 +0000 UTC m=+0.344742559 container start 6225265b852691112acc35c680459ec9c930b2eaf7231eb8ca781b2248bfb94f (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-1-deactivate, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240731, RELEASE=main-62da54b, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-09-21T09:50:19.272 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 21 09:50:18 smithi022 podman[132712]: 2024-09-21 09:50:18.883948885 +0000 UTC m=+0.448042926 container attach 6225265b852691112acc35c680459ec9c930b2eaf7231eb8ca781b2248bfb94f (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-1-deactivate, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=, ceph=True, maintainer=Guillaume Abrioux , RELEASE=main-62da54b, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2) 2024-09-21T09:50:19.273 INFO:journalctl@ceph.osd.1.smithi022.stdout:Sep 21 09:50:18 smithi022 podman[132712]: 2024-09-21 09:50:18.980326016 +0000 UTC m=+0.544420057 container died 6225265b852691112acc35c680459ec9c930b2eaf7231eb8ca781b2248bfb94f (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-1-deactivate, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, RELEASE=main-62da54b, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T09:50:19.611 DEBUG:teuthology.orchestra.run.smithi022:> sudo pkill -f 'journalctl -f -n 0 -u ceph-d65e801e-77fa-11ef-baf6-efdc52797490@osd.1.service' 2024-09-21T09:50:19.651 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T09:50:19.651 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-09-21T09:50:19.651 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-09-21T09:50:19.652 DEBUG:teuthology.orchestra.run.smithi022:> sudo systemctl stop ceph-d65e801e-77fa-11ef-baf6-efdc52797490@osd.2 2024-09-21T09:50:19.995 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 21 09:50:19 smithi022 systemd[1]: Stopping Ceph osd.2 for d65e801e-77fa-11ef-baf6-efdc52797490... 2024-09-21T09:50:20.308 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 21 09:50:19 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-2[113367]: 2024-09-21T09:50:19.991+0000 7f02fad98640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T09:50:20.308 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 21 09:50:19 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-2[113367]: 2024-09-21T09:50:19.991+0000 7f02fad98640 -1 osd.2 1053 *** Got signal Terminated *** 2024-09-21T09:50:20.308 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 21 09:50:19 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-2[113367]: 2024-09-21T09:50:19.991+0000 7f02fad98640 -1 osd.2 1053 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T09:50:25.808 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 21 09:50:25 smithi022 podman[132839]: 2024-09-21 09:50:25.539870927 +0000 UTC m=+5.663429764 container died b8c9443066987ecda42255c21a370c89df1058a3fac9c29a920df395bc6c26da (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, RELEASE=main-62da54b, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T09:50:26.210 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 21 09:50:25 smithi022 podman[132839]: 2024-09-21 09:50:25.907054525 +0000 UTC m=+6.030613344 container cleanup b8c9443066987ecda42255c21a370c89df1058a3fac9c29a920df395bc6c26da (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, RELEASE=main-62da54b, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T09:50:26.210 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 21 09:50:25 smithi022 bash[132839]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-2 2024-09-21T09:50:26.558 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 21 09:50:26 smithi022 podman[132859]: 2024-09-21 09:50:26.208767957 +0000 UTC m=+0.666039999 container remove b8c9443066987ecda42255c21a370c89df1058a3fac9c29a920df395bc6c26da (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-62da54b, ceph=True, GIT_CLEAN=True, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2) 2024-09-21T09:50:26.930 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 21 09:50:26 smithi022 podman[132963]: 2024-09-21 09:50:26.609711248 +0000 UTC m=+0.111284319 container create e51676fb036b2bb983f023597ea435d392f5cabc583a4ecf2d5c37f607cd176d (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-2-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-62da54b) 2024-09-21T09:50:26.930 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 21 09:50:26 smithi022 podman[132963]: 2024-09-21 09:50:26.516038495 +0000 UTC m=+0.017611573 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:50:26.930 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 21 09:50:26 smithi022 podman[132963]: 2024-09-21 09:50:26.845460989 +0000 UTC m=+0.347034064 container init e51676fb036b2bb983f023597ea435d392f5cabc583a4ecf2d5c37f607cd176d (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-2-deactivate, GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, RELEASE=main-62da54b, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2) 2024-09-21T09:50:26.930 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 21 09:50:26 smithi022 podman[132963]: 2024-09-21 09:50:26.850384988 +0000 UTC m=+0.351958055 container start e51676fb036b2bb983f023597ea435d392f5cabc583a4ecf2d5c37f607cd176d (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-2-deactivate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=main-62da54b, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-09-21T09:50:26.930 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 21 09:50:26 smithi022 podman[132963]: 2024-09-21 09:50:26.928714597 +0000 UTC m=+0.430287662 container attach e51676fb036b2bb983f023597ea435d392f5cabc583a4ecf2d5c37f607cd176d (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-2-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=main-62da54b, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-09-21T09:50:27.308 INFO:journalctl@ceph.osd.2.smithi022.stdout:Sep 21 09:50:27 smithi022 podman[132963]: 2024-09-21 09:50:27.05051929 +0000 UTC m=+0.552092377 container died e51676fb036b2bb983f023597ea435d392f5cabc583a4ecf2d5c37f607cd176d (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-2-deactivate, RELEASE=main-62da54b, ceph=True, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2) 2024-09-21T09:50:27.668 DEBUG:teuthology.orchestra.run.smithi022:> sudo pkill -f 'journalctl -f -n 0 -u ceph-d65e801e-77fa-11ef-baf6-efdc52797490@osd.2.service' 2024-09-21T09:50:27.709 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T09:50:27.709 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-09-21T09:50:27.709 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-09-21T09:50:27.709 DEBUG:teuthology.orchestra.run.smithi022:> sudo systemctl stop ceph-d65e801e-77fa-11ef-baf6-efdc52797490@osd.3 2024-09-21T09:50:28.039 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 21 09:50:27 smithi022 systemd[1]: Stopping Ceph osd.3 for d65e801e-77fa-11ef-baf6-efdc52797490... 2024-09-21T09:50:28.308 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 21 09:50:28 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-3[117988]: 2024-09-21T09:50:28.035+0000 7fbfda610640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T09:50:28.308 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 21 09:50:28 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-3[117988]: 2024-09-21T09:50:28.035+0000 7fbfda610640 -1 osd.3 1053 *** Got signal Terminated *** 2024-09-21T09:50:28.308 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 21 09:50:28 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-3[117988]: 2024-09-21T09:50:28.035+0000 7fbfda610640 -1 osd.3 1053 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T09:50:30.442 INFO:journalctl@ceph.osd.5.smithi064.stdout:Sep 21 09:50:30 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-5[96798]: 2024-09-21T09:50:30.146+0000 7fe95437c640 -1 osd.5 1053 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-21T09:50:05.764363+0000 front 2024-09-21T09:50:05.764450+0000 (oldest deadline 2024-09-21T09:50:29.264172+0000) 2024-09-21T09:50:30.558 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 21 09:50:30 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-3[117988]: 2024-09-21T09:50:30.167+0000 7fbfd2c1d640 -1 osd.3 1053 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-21T09:50:07.625967+0000 front 2024-09-21T09:50:07.626098+0000 (oldest deadline 2024-09-21T09:50:29.326105+0000) 2024-09-21T09:50:30.942 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:50:30 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7[106022]: 2024-09-21T09:50:30.446+0000 7f338d84e640 -1 osd.7 1053 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-21T09:50:04.903220+0000 front 2024-09-21T09:50:04.903325+0000 (oldest deadline 2024-09-21T09:50:29.603062+0000) 2024-09-21T09:50:31.420 INFO:journalctl@ceph.osd.5.smithi064.stdout:Sep 21 09:50:31 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-5[96798]: 2024-09-21T09:50:31.142+0000 7fe95437c640 -1 osd.5 1053 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-21T09:50:05.764363+0000 front 2024-09-21T09:50:05.764450+0000 (oldest deadline 2024-09-21T09:50:29.264172+0000) 2024-09-21T09:50:31.420 INFO:journalctl@ceph.osd.4.smithi064.stdout:Sep 21 09:50:31 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-4[92249]: 2024-09-21T09:50:31.382+0000 7ff32b832640 -1 osd.4 1053 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-21T09:50:07.855908+0000 front 2024-09-21T09:50:07.855934+0000 (oldest deadline 2024-09-21T09:50:31.355721+0000) 2024-09-21T09:50:31.421 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:50:31 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-6[101337]: 2024-09-21T09:50:31.322+0000 7f2af067a640 -1 osd.6 1053 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-21T09:50:06.172724+0000 front 2024-09-21T09:50:06.172984+0000 (oldest deadline 2024-09-21T09:50:30.872936+0000) 2024-09-21T09:50:31.558 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 21 09:50:31 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-3[117988]: 2024-09-21T09:50:31.142+0000 7fbfd2c1d640 -1 osd.3 1053 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-21T09:50:07.625967+0000 front 2024-09-21T09:50:07.626098+0000 (oldest deadline 2024-09-21T09:50:29.326105+0000) 2024-09-21T09:50:31.691 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:50:31 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7[106022]: 2024-09-21T09:50:31.418+0000 7f338d84e640 -1 osd.7 1053 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-21T09:50:04.903220+0000 front 2024-09-21T09:50:04.903325+0000 (oldest deadline 2024-09-21T09:50:29.603062+0000) 2024-09-21T09:50:32.442 INFO:journalctl@ceph.osd.5.smithi064.stdout:Sep 21 09:50:32 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-5[96798]: 2024-09-21T09:50:32.160+0000 7fe95437c640 -1 osd.5 1053 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-21T09:50:05.764363+0000 front 2024-09-21T09:50:05.764450+0000 (oldest deadline 2024-09-21T09:50:29.264172+0000) 2024-09-21T09:50:32.442 INFO:journalctl@ceph.osd.4.smithi064.stdout:Sep 21 09:50:32 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-4[92249]: 2024-09-21T09:50:32.400+0000 7ff32b832640 -1 osd.4 1053 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-21T09:50:07.855908+0000 front 2024-09-21T09:50:07.855934+0000 (oldest deadline 2024-09-21T09:50:31.355721+0000) 2024-09-21T09:50:32.442 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:50:32 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-6[101337]: 2024-09-21T09:50:32.294+0000 7f2af067a640 -1 osd.6 1053 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-21T09:50:06.172724+0000 front 2024-09-21T09:50:06.172984+0000 (oldest deadline 2024-09-21T09:50:30.872936+0000) 2024-09-21T09:50:32.558 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 21 09:50:32 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-3[117988]: 2024-09-21T09:50:32.149+0000 7fbfd2c1d640 -1 osd.3 1053 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-21T09:50:07.625967+0000 front 2024-09-21T09:50:07.626098+0000 (oldest deadline 2024-09-21T09:50:29.326105+0000) 2024-09-21T09:50:32.941 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:50:32 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7[106022]: 2024-09-21T09:50:32.441+0000 7f338d84e640 -1 osd.7 1053 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-21T09:50:04.903220+0000 front 2024-09-21T09:50:04.903325+0000 (oldest deadline 2024-09-21T09:50:29.603062+0000) 2024-09-21T09:50:33.374 INFO:journalctl@ceph.osd.5.smithi064.stdout:Sep 21 09:50:33 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-5[96798]: 2024-09-21T09:50:33.118+0000 7fe95437c640 -1 osd.5 1053 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-21T09:50:05.764363+0000 front 2024-09-21T09:50:05.764450+0000 (oldest deadline 2024-09-21T09:50:29.264172+0000) 2024-09-21T09:50:33.375 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:50:33 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-6[101337]: 2024-09-21T09:50:33.264+0000 7f2af067a640 -1 osd.6 1053 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-21T09:50:06.172724+0000 front 2024-09-21T09:50:06.172984+0000 (oldest deadline 2024-09-21T09:50:30.872936+0000) 2024-09-21T09:50:33.662 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 21 09:50:33 smithi022 podman[133094]: 2024-09-21 09:50:33.390000959 +0000 UTC m=+5.456980252 container died c90b27f61bb68eef229931b9a71c344cd61b309e989fc3f74ee43afab03eebe0 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-3, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, RELEASE=main-62da54b, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.license=GPLv2) 2024-09-21T09:50:33.691 INFO:journalctl@ceph.osd.4.smithi064.stdout:Sep 21 09:50:33 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-4[92249]: 2024-09-21T09:50:33.372+0000 7ff32b832640 -1 osd.4 1053 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-21T09:50:07.855908+0000 front 2024-09-21T09:50:07.855934+0000 (oldest deadline 2024-09-21T09:50:31.355721+0000) 2024-09-21T09:50:33.692 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:50:33 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7[106022]: 2024-09-21T09:50:33.421+0000 7f338d84e640 -1 osd.7 1053 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-21T09:50:04.903220+0000 front 2024-09-21T09:50:04.903325+0000 (oldest deadline 2024-09-21T09:50:29.603062+0000) 2024-09-21T09:50:34.029 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 21 09:50:33 smithi022 podman[133094]: 2024-09-21 09:50:33.743376985 +0000 UTC m=+5.810356316 container cleanup c90b27f61bb68eef229931b9a71c344cd61b309e989fc3f74ee43afab03eebe0 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-3, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, RELEASE=main-62da54b) 2024-09-21T09:50:34.029 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 21 09:50:33 smithi022 bash[133094]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-3 2024-09-21T09:50:34.308 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 21 09:50:34 smithi022 podman[133115]: 2024-09-21 09:50:34.027854759 +0000 UTC m=+0.638166996 container remove c90b27f61bb68eef229931b9a71c344cd61b309e989fc3f74ee43afab03eebe0 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-3, ceph=True, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-62da54b, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS) 2024-09-21T09:50:34.402 INFO:journalctl@ceph.osd.5.smithi064.stdout:Sep 21 09:50:34 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-5[96798]: 2024-09-21T09:50:34.103+0000 7fe95437c640 -1 osd.5 1053 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-21T09:50:05.764363+0000 front 2024-09-21T09:50:05.764450+0000 (oldest deadline 2024-09-21T09:50:29.264172+0000) 2024-09-21T09:50:34.403 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:50:34 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-6[101337]: 2024-09-21T09:50:34.312+0000 7f2af067a640 -1 osd.6 1053 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-21T09:50:06.172724+0000 front 2024-09-21T09:50:06.172984+0000 (oldest deadline 2024-09-21T09:50:30.872936+0000) 2024-09-21T09:50:34.674 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 21 09:50:34 smithi022 podman[133209]: 2024-09-21 09:50:34.353650967 +0000 UTC m=+0.109781039 container create f43bdea9bd2d98cbeb7e959dad64e6f77d77fbbf5115e1af7b006418b208ca43 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-3-deactivate, GIT_CLEAN=True, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, RELEASE=main-62da54b, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=) 2024-09-21T09:50:34.674 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 21 09:50:34 smithi022 podman[133209]: 2024-09-21 09:50:34.260788301 +0000 UTC m=+0.016918383 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:50:34.674 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 21 09:50:34 smithi022 podman[133209]: 2024-09-21 09:50:34.589517366 +0000 UTC m=+0.345647441 container init f43bdea9bd2d98cbeb7e959dad64e6f77d77fbbf5115e1af7b006418b208ca43 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-3-deactivate, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0, RELEASE=main-62da54b, org.label-schema.build-date=20240731, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux ) 2024-09-21T09:50:34.674 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 21 09:50:34 smithi022 podman[133209]: 2024-09-21 09:50:34.594548457 +0000 UTC m=+0.350678530 container start f43bdea9bd2d98cbeb7e959dad64e6f77d77fbbf5115e1af7b006418b208ca43 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-3-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, RELEASE=main-62da54b, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0) 2024-09-21T09:50:34.675 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 21 09:50:34 smithi022 podman[133209]: 2024-09-21 09:50:34.672677474 +0000 UTC m=+0.428807557 container attach f43bdea9bd2d98cbeb7e959dad64e6f77d77fbbf5115e1af7b006418b208ca43 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-3-deactivate, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=main-62da54b, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0) 2024-09-21T09:50:34.691 INFO:journalctl@ceph.osd.4.smithi064.stdout:Sep 21 09:50:34 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-4[92249]: 2024-09-21T09:50:34.401+0000 7ff32b832640 -1 osd.4 1053 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-21T09:50:07.855908+0000 front 2024-09-21T09:50:07.855934+0000 (oldest deadline 2024-09-21T09:50:31.355721+0000) 2024-09-21T09:50:34.692 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:50:34 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7[106022]: 2024-09-21T09:50:34.415+0000 7f338d84e640 -1 osd.7 1053 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-21T09:50:04.903220+0000 front 2024-09-21T09:50:04.903325+0000 (oldest deadline 2024-09-21T09:50:29.603062+0000) 2024-09-21T09:50:35.058 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 21 09:50:34 smithi022 podman[133209]: 2024-09-21 09:50:34.787868536 +0000 UTC m=+0.543998608 container died f43bdea9bd2d98cbeb7e959dad64e6f77d77fbbf5115e1af7b006418b208ca43 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-3-deactivate, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-62da54b, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2) 2024-09-21T09:50:35.328 INFO:journalctl@ceph.osd.3.smithi022.stdout:Sep 21 09:50:35 smithi022 podman[133253]: 2024-09-21 09:50:35.309762681 +0000 UTC m=+0.515304829 container remove f43bdea9bd2d98cbeb7e959dad64e6f77d77fbbf5115e1af7b006418b208ca43 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-3-deactivate, RELEASE=main-62da54b, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD) 2024-09-21T09:50:35.333 DEBUG:teuthology.orchestra.run.smithi022:> sudo pkill -f 'journalctl -f -n 0 -u ceph-d65e801e-77fa-11ef-baf6-efdc52797490@osd.3.service' 2024-09-21T09:50:35.373 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T09:50:35.373 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-09-21T09:50:35.373 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-09-21T09:50:35.373 DEBUG:teuthology.orchestra.run.smithi064:> sudo systemctl stop ceph-d65e801e-77fa-11ef-baf6-efdc52797490@osd.4 2024-09-21T09:50:35.375 INFO:journalctl@ceph.osd.5.smithi064.stdout:Sep 21 09:50:35 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-5[96798]: 2024-09-21T09:50:35.118+0000 7fe95437c640 -1 osd.5 1053 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-21T09:50:05.764363+0000 front 2024-09-21T09:50:05.764450+0000 (oldest deadline 2024-09-21T09:50:29.264172+0000) 2024-09-21T09:50:35.375 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:50:35 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-6[101337]: 2024-09-21T09:50:35.331+0000 7f2af067a640 -1 osd.6 1053 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-21T09:50:06.172724+0000 front 2024-09-21T09:50:06.172984+0000 (oldest deadline 2024-09-21T09:50:30.872936+0000) 2024-09-21T09:50:35.692 INFO:journalctl@ceph.osd.4.smithi064.stdout:Sep 21 09:50:35 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-4[92249]: 2024-09-21T09:50:35.373+0000 7ff32b832640 -1 osd.4 1053 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-21T09:50:07.855908+0000 front 2024-09-21T09:50:07.855934+0000 (oldest deadline 2024-09-21T09:50:31.355721+0000) 2024-09-21T09:50:35.692 INFO:journalctl@ceph.osd.4.smithi064.stdout:Sep 21 09:50:35 smithi064 systemd[1]: Stopping Ceph osd.4 for d65e801e-77fa-11ef-baf6-efdc52797490... 2024-09-21T09:50:35.692 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:50:35 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7[106022]: 2024-09-21T09:50:35.436+0000 7f338d84e640 -1 osd.7 1053 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-21T09:50:04.903220+0000 front 2024-09-21T09:50:04.903325+0000 (oldest deadline 2024-09-21T09:50:29.603062+0000) 2024-09-21T09:50:36.096 INFO:journalctl@ceph.osd.4.smithi064.stdout:Sep 21 09:50:35 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-4[92249]: 2024-09-21T09:50:35.699+0000 7ff333225640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T09:50:36.096 INFO:journalctl@ceph.osd.4.smithi064.stdout:Sep 21 09:50:35 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-4[92249]: 2024-09-21T09:50:35.699+0000 7ff333225640 -1 osd.4 1053 *** Got signal Terminated *** 2024-09-21T09:50:36.096 INFO:journalctl@ceph.osd.4.smithi064.stdout:Sep 21 09:50:35 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-4[92249]: 2024-09-21T09:50:35.699+0000 7ff333225640 -1 osd.4 1053 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T09:50:36.360 INFO:journalctl@ceph.osd.5.smithi064.stdout:Sep 21 09:50:36 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-5[96798]: 2024-09-21T09:50:36.094+0000 7fe95437c640 -1 osd.5 1053 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-21T09:50:05.764363+0000 front 2024-09-21T09:50:05.764450+0000 (oldest deadline 2024-09-21T09:50:29.264172+0000) 2024-09-21T09:50:36.361 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:50:36 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-6[101337]: 2024-09-21T09:50:36.327+0000 7f2af067a640 -1 osd.6 1053 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-21T09:50:06.172724+0000 front 2024-09-21T09:50:06.172984+0000 (oldest deadline 2024-09-21T09:50:30.872936+0000) 2024-09-21T09:50:36.691 INFO:journalctl@ceph.osd.4.smithi064.stdout:Sep 21 09:50:36 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-4[92249]: 2024-09-21T09:50:36.359+0000 7ff32b832640 -1 osd.4 1053 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-21T09:50:07.855908+0000 front 2024-09-21T09:50:07.855934+0000 (oldest deadline 2024-09-21T09:50:31.355721+0000) 2024-09-21T09:50:36.692 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:50:36 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7[106022]: 2024-09-21T09:50:36.485+0000 7f338d84e640 -1 osd.7 1053 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-21T09:50:04.903220+0000 front 2024-09-21T09:50:04.903325+0000 (oldest deadline 2024-09-21T09:50:29.603062+0000) 2024-09-21T09:50:37.441 INFO:journalctl@ceph.osd.5.smithi064.stdout:Sep 21 09:50:37 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-5[96798]: 2024-09-21T09:50:37.132+0000 7fe95437c640 -1 osd.5 1053 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-21T09:50:05.764363+0000 front 2024-09-21T09:50:05.764450+0000 (oldest deadline 2024-09-21T09:50:29.264172+0000) 2024-09-21T09:50:37.442 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:50:37 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-6[101337]: 2024-09-21T09:50:37.316+0000 7f2af067a640 -1 osd.6 1053 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-21T09:50:06.172724+0000 front 2024-09-21T09:50:06.172984+0000 (oldest deadline 2024-09-21T09:50:30.872936+0000) 2024-09-21T09:50:37.442 INFO:journalctl@ceph.osd.4.smithi064.stdout:Sep 21 09:50:37 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-4[92249]: 2024-09-21T09:50:37.377+0000 7ff32b832640 -1 osd.4 1053 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-21T09:50:07.855908+0000 front 2024-09-21T09:50:07.855934+0000 (oldest deadline 2024-09-21T09:50:31.355721+0000) 2024-09-21T09:50:37.941 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:50:37 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7[106022]: 2024-09-21T09:50:37.452+0000 7f338d84e640 -1 osd.7 1053 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-21T09:50:04.903220+0000 front 2024-09-21T09:50:04.903325+0000 (oldest deadline 2024-09-21T09:50:29.603062+0000) 2024-09-21T09:50:38.402 INFO:journalctl@ceph.osd.5.smithi064.stdout:Sep 21 09:50:38 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-5[96798]: 2024-09-21T09:50:38.131+0000 7fe95437c640 -1 osd.5 1053 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-21T09:50:05.764363+0000 front 2024-09-21T09:50:05.764450+0000 (oldest deadline 2024-09-21T09:50:29.264172+0000) 2024-09-21T09:50:38.403 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:50:38 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-6[101337]: 2024-09-21T09:50:38.316+0000 7f2af067a640 -1 osd.6 1053 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-21T09:50:06.172724+0000 front 2024-09-21T09:50:06.172984+0000 (oldest deadline 2024-09-21T09:50:30.872936+0000) 2024-09-21T09:50:38.403 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:50:38 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-6[101337]: 2024-09-21T09:50:38.316+0000 7f2af067a640 -1 osd.6 1053 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-21T09:50:13.773813+0000 front 2024-09-21T09:50:13.773822+0000 (oldest deadline 2024-09-21T09:50:37.873810+0000) 2024-09-21T09:50:38.692 INFO:journalctl@ceph.osd.4.smithi064.stdout:Sep 21 09:50:38 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-4[92249]: 2024-09-21T09:50:38.400+0000 7ff32b832640 -1 osd.4 1053 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-21T09:50:07.855908+0000 front 2024-09-21T09:50:07.855934+0000 (oldest deadline 2024-09-21T09:50:31.355721+0000) 2024-09-21T09:50:38.692 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:50:38 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7[106022]: 2024-09-21T09:50:38.455+0000 7f338d84e640 -1 osd.7 1053 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-21T09:50:04.903220+0000 front 2024-09-21T09:50:04.903325+0000 (oldest deadline 2024-09-21T09:50:29.603062+0000) 2024-09-21T09:50:38.692 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:50:38 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7[106022]: 2024-09-21T09:50:38.455+0000 7f338d84e640 -1 osd.7 1053 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-21T09:50:13.604550+0000 front 2024-09-21T09:50:13.604475+0000 (oldest deadline 2024-09-21T09:50:38.304479+0000) 2024-09-21T09:50:39.442 INFO:journalctl@ceph.osd.5.smithi064.stdout:Sep 21 09:50:39 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-5[96798]: 2024-09-21T09:50:39.143+0000 7fe95437c640 -1 osd.5 1053 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-21T09:50:05.764363+0000 front 2024-09-21T09:50:05.764450+0000 (oldest deadline 2024-09-21T09:50:29.264172+0000) 2024-09-21T09:50:39.442 INFO:journalctl@ceph.osd.4.smithi064.stdout:Sep 21 09:50:39 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-4[92249]: 2024-09-21T09:50:39.392+0000 7ff32b832640 -1 osd.4 1053 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-21T09:50:07.855908+0000 front 2024-09-21T09:50:07.855934+0000 (oldest deadline 2024-09-21T09:50:31.355721+0000) 2024-09-21T09:50:39.442 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:50:39 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-6[101337]: 2024-09-21T09:50:39.280+0000 7f2af067a640 -1 osd.6 1053 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-21T09:50:06.172724+0000 front 2024-09-21T09:50:06.172984+0000 (oldest deadline 2024-09-21T09:50:30.872936+0000) 2024-09-21T09:50:39.442 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:50:39 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-6[101337]: 2024-09-21T09:50:39.280+0000 7f2af067a640 -1 osd.6 1053 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-21T09:50:13.773813+0000 front 2024-09-21T09:50:13.773822+0000 (oldest deadline 2024-09-21T09:50:37.873810+0000) 2024-09-21T09:50:39.941 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:50:39 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7[106022]: 2024-09-21T09:50:39.468+0000 7f338d84e640 -1 osd.7 1053 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-21T09:50:04.903220+0000 front 2024-09-21T09:50:04.903325+0000 (oldest deadline 2024-09-21T09:50:29.603062+0000) 2024-09-21T09:50:39.942 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:50:39 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7[106022]: 2024-09-21T09:50:39.468+0000 7f338d84e640 -1 osd.7 1053 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-21T09:50:13.604550+0000 front 2024-09-21T09:50:13.604475+0000 (oldest deadline 2024-09-21T09:50:38.304479+0000) 2024-09-21T09:50:40.391 INFO:journalctl@ceph.osd.5.smithi064.stdout:Sep 21 09:50:40 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-5[96798]: 2024-09-21T09:50:40.096+0000 7fe95437c640 -1 osd.5 1053 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-21T09:50:05.764363+0000 front 2024-09-21T09:50:05.764450+0000 (oldest deadline 2024-09-21T09:50:29.264172+0000) 2024-09-21T09:50:40.392 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:50:40 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-6[101337]: 2024-09-21T09:50:40.309+0000 7f2af067a640 -1 osd.6 1053 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-21T09:50:06.172724+0000 front 2024-09-21T09:50:06.172984+0000 (oldest deadline 2024-09-21T09:50:30.872936+0000) 2024-09-21T09:50:40.392 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:50:40 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-6[101337]: 2024-09-21T09:50:40.309+0000 7f2af067a640 -1 osd.6 1053 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-21T09:50:13.773813+0000 front 2024-09-21T09:50:13.773822+0000 (oldest deadline 2024-09-21T09:50:37.873810+0000) 2024-09-21T09:50:40.692 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:50:40 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7[106022]: 2024-09-21T09:50:40.466+0000 7f338d84e640 -1 osd.7 1053 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-21T09:50:04.903220+0000 front 2024-09-21T09:50:04.903325+0000 (oldest deadline 2024-09-21T09:50:29.603062+0000) 2024-09-21T09:50:40.692 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:50:40 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7[106022]: 2024-09-21T09:50:40.466+0000 7f338d84e640 -1 osd.7 1053 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-21T09:50:13.604550+0000 front 2024-09-21T09:50:13.604475+0000 (oldest deadline 2024-09-21T09:50:38.304479+0000) 2024-09-21T09:50:40.692 INFO:journalctl@ceph.osd.4.smithi064.stdout:Sep 21 09:50:40 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-4[92249]: 2024-09-21T09:50:40.389+0000 7ff32b832640 -1 osd.4 1053 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-21T09:50:07.855908+0000 front 2024-09-21T09:50:07.855934+0000 (oldest deadline 2024-09-21T09:50:31.355721+0000) 2024-09-21T09:50:40.692 INFO:journalctl@ceph.osd.4.smithi064.stdout:Sep 21 09:50:40 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-4[92249]: 2024-09-21T09:50:40.389+0000 7ff32b832640 -1 osd.4 1053 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-21T09:50:15.957326+0000 front 2024-09-21T09:50:15.957375+0000 (oldest deadline 2024-09-21T09:50:40.057237+0000) 2024-09-21T09:50:41.359 INFO:journalctl@ceph.osd.4.smithi064.stdout:Sep 21 09:50:41 smithi064 podman[119137]: 2024-09-21 09:50:41.094272167 +0000 UTC m=+5.505910749 container died 936099dbca14a58261a7050c0775f8f44deb5e8910cd89fb2501fc53b38fdd54 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-4, GIT_BRANCH=HEAD, ceph=True, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, RELEASE=main-62da54b) 2024-09-21T09:50:41.359 INFO:journalctl@ceph.osd.5.smithi064.stdout:Sep 21 09:50:41 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-5[96798]: 2024-09-21T09:50:41.095+0000 7fe95437c640 -1 osd.5 1053 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-21T09:50:05.764363+0000 front 2024-09-21T09:50:05.764450+0000 (oldest deadline 2024-09-21T09:50:29.264172+0000) 2024-09-21T09:50:41.359 INFO:journalctl@ceph.osd.5.smithi064.stdout:Sep 21 09:50:41 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-5[96798]: 2024-09-21T09:50:41.095+0000 7fe95437c640 -1 osd.5 1053 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-21T09:50:14.565115+0000 front 2024-09-21T09:50:14.565274+0000 (oldest deadline 2024-09-21T09:50:40.465056+0000) 2024-09-21T09:50:41.691 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:50:41 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-6[101337]: 2024-09-21T09:50:41.357+0000 7f2af067a640 -1 osd.6 1053 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-21T09:50:06.172724+0000 front 2024-09-21T09:50:06.172984+0000 (oldest deadline 2024-09-21T09:50:30.872936+0000) 2024-09-21T09:50:41.692 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:50:41 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-6[101337]: 2024-09-21T09:50:41.357+0000 7f2af067a640 -1 osd.6 1053 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-21T09:50:13.773813+0000 front 2024-09-21T09:50:13.773822+0000 (oldest deadline 2024-09-21T09:50:37.873810+0000) 2024-09-21T09:50:41.692 INFO:journalctl@ceph.osd.4.smithi064.stdout:Sep 21 09:50:41 smithi064 podman[119137]: 2024-09-21 09:50:41.581124837 +0000 UTC m=+5.992763457 container cleanup 936099dbca14a58261a7050c0775f8f44deb5e8910cd89fb2501fc53b38fdd54 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-4, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-62da54b) 2024-09-21T09:50:41.692 INFO:journalctl@ceph.osd.4.smithi064.stdout:Sep 21 09:50:41 smithi064 bash[119137]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-4 2024-09-21T09:50:41.693 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:50:41 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7[106022]: 2024-09-21T09:50:41.460+0000 7f338d84e640 -1 osd.7 1053 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-21T09:50:04.903220+0000 front 2024-09-21T09:50:04.903325+0000 (oldest deadline 2024-09-21T09:50:29.603062+0000) 2024-09-21T09:50:41.693 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:50:41 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7[106022]: 2024-09-21T09:50:41.460+0000 7f338d84e640 -1 osd.7 1053 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-21T09:50:13.604550+0000 front 2024-09-21T09:50:13.604475+0000 (oldest deadline 2024-09-21T09:50:38.304479+0000) 2024-09-21T09:50:42.191 INFO:journalctl@ceph.osd.4.smithi064.stdout:Sep 21 09:50:41 smithi064 podman[119157]: 2024-09-21 09:50:41.832592173 +0000 UTC m=+0.741268277 container remove 936099dbca14a58261a7050c0775f8f44deb5e8910cd89fb2501fc53b38fdd54 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-4, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.license=GPLv2, RELEASE=main-62da54b, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T09:50:42.192 INFO:journalctl@ceph.osd.5.smithi064.stdout:Sep 21 09:50:42 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-5[96798]: 2024-09-21T09:50:42.049+0000 7fe95437c640 -1 osd.5 1053 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-21T09:50:05.764363+0000 front 2024-09-21T09:50:05.764450+0000 (oldest deadline 2024-09-21T09:50:29.264172+0000) 2024-09-21T09:50:42.192 INFO:journalctl@ceph.osd.5.smithi064.stdout:Sep 21 09:50:42 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-5[96798]: 2024-09-21T09:50:42.049+0000 7fe95437c640 -1 osd.5 1053 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-21T09:50:14.565115+0000 front 2024-09-21T09:50:14.565274+0000 (oldest deadline 2024-09-21T09:50:40.465056+0000) 2024-09-21T09:50:42.500 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:50:42 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-6[101337]: 2024-09-21T09:50:42.392+0000 7f2af067a640 -1 osd.6 1053 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-21T09:50:06.172724+0000 front 2024-09-21T09:50:06.172984+0000 (oldest deadline 2024-09-21T09:50:30.872936+0000) 2024-09-21T09:50:42.500 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:50:42 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-6[101337]: 2024-09-21T09:50:42.392+0000 7f2af067a640 -1 osd.6 1053 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-21T09:50:13.773813+0000 front 2024-09-21T09:50:13.773822+0000 (oldest deadline 2024-09-21T09:50:37.873810+0000) 2024-09-21T09:50:42.501 INFO:journalctl@ceph.osd.4.smithi064.stdout:Sep 21 09:50:42 smithi064 podman[119252]: 2024-09-21 09:50:42.100703137 +0000 UTC m=+0.016396957 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:50:42.501 INFO:journalctl@ceph.osd.4.smithi064.stdout:Sep 21 09:50:42 smithi064 podman[119252]: 2024-09-21 09:50:42.199825023 +0000 UTC m=+0.115518844 container create 4ac196243e078280883a4ebd478bfa58f13a84e3c9c1428dc7fa2706d4cfe669 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-4-deactivate, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, RELEASE=main-62da54b, io.buildah.version=1.37.2, ceph=True) 2024-09-21T09:50:42.501 INFO:journalctl@ceph.osd.4.smithi064.stdout:Sep 21 09:50:42 smithi064 podman[119252]: 2024-09-21 09:50:42.427458243 +0000 UTC m=+0.343152064 container init 4ac196243e078280883a4ebd478bfa58f13a84e3c9c1428dc7fa2706d4cfe669 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-4-deactivate, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, org.label-schema.schema-version=1.0, RELEASE=main-62da54b, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD) 2024-09-21T09:50:42.501 INFO:journalctl@ceph.osd.4.smithi064.stdout:Sep 21 09:50:42 smithi064 podman[119252]: 2024-09-21 09:50:42.432953243 +0000 UTC m=+0.348647066 container start 4ac196243e078280883a4ebd478bfa58f13a84e3c9c1428dc7fa2706d4cfe669 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-4-deactivate, RELEASE=main-62da54b, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2) 2024-09-21T09:50:42.932 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:50:42 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7[106022]: 2024-09-21T09:50:42.498+0000 7f338d84e640 -1 osd.7 1053 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-21T09:50:04.903220+0000 front 2024-09-21T09:50:04.903325+0000 (oldest deadline 2024-09-21T09:50:29.603062+0000) 2024-09-21T09:50:42.933 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:50:42 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7[106022]: 2024-09-21T09:50:42.498+0000 7f338d84e640 -1 osd.7 1053 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-21T09:50:13.604550+0000 front 2024-09-21T09:50:13.604475+0000 (oldest deadline 2024-09-21T09:50:38.304479+0000) 2024-09-21T09:50:42.933 INFO:journalctl@ceph.osd.4.smithi064.stdout:Sep 21 09:50:42 smithi064 podman[119252]: 2024-09-21 09:50:42.527275251 +0000 UTC m=+0.442969083 container attach 4ac196243e078280883a4ebd478bfa58f13a84e3c9c1428dc7fa2706d4cfe669 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-4-deactivate, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, ceph=True, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, RELEASE=main-62da54b, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS) 2024-09-21T09:50:42.934 INFO:journalctl@ceph.osd.4.smithi064.stdout:Sep 21 09:50:42 smithi064 podman[119252]: 2024-09-21 09:50:42.634829205 +0000 UTC m=+0.550523026 container died 4ac196243e078280883a4ebd478bfa58f13a84e3c9c1428dc7fa2706d4cfe669 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-4-deactivate, ceph=True, RELEASE=main-62da54b, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T09:50:43.191 INFO:journalctl@ceph.osd.5.smithi064.stdout:Sep 21 09:50:43 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-5[96798]: 2024-09-21T09:50:43.021+0000 7fe95437c640 -1 osd.5 1053 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-21T09:50:05.764363+0000 front 2024-09-21T09:50:05.764450+0000 (oldest deadline 2024-09-21T09:50:29.264172+0000) 2024-09-21T09:50:43.191 INFO:journalctl@ceph.osd.5.smithi064.stdout:Sep 21 09:50:43 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-5[96798]: 2024-09-21T09:50:43.021+0000 7fe95437c640 -1 osd.5 1053 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-21T09:50:14.565115+0000 front 2024-09-21T09:50:14.565274+0000 (oldest deadline 2024-09-21T09:50:40.465056+0000) 2024-09-21T09:50:43.252 DEBUG:teuthology.orchestra.run.smithi064:> sudo pkill -f 'journalctl -f -n 0 -u ceph-d65e801e-77fa-11ef-baf6-efdc52797490@osd.4.service' 2024-09-21T09:50:43.295 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T09:50:43.295 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-09-21T09:50:43.296 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-09-21T09:50:43.296 DEBUG:teuthology.orchestra.run.smithi064:> sudo systemctl stop ceph-d65e801e-77fa-11ef-baf6-efdc52797490@osd.5 2024-09-21T09:50:43.523 INFO:journalctl@ceph.osd.5.smithi064.stdout:Sep 21 09:50:43 smithi064 systemd[1]: Stopping Ceph osd.5 for d65e801e-77fa-11ef-baf6-efdc52797490... 2024-09-21T09:50:43.524 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:50:43 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-6[101337]: 2024-09-21T09:50:43.421+0000 7f2af067a640 -1 osd.6 1053 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-21T09:50:06.172724+0000 front 2024-09-21T09:50:06.172984+0000 (oldest deadline 2024-09-21T09:50:30.872936+0000) 2024-09-21T09:50:43.524 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:50:43 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-6[101337]: 2024-09-21T09:50:43.421+0000 7f2af067a640 -1 osd.6 1053 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-21T09:50:13.773813+0000 front 2024-09-21T09:50:13.773822+0000 (oldest deadline 2024-09-21T09:50:37.873810+0000) 2024-09-21T09:50:43.941 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:50:43 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7[106022]: 2024-09-21T09:50:43.547+0000 7f338d84e640 -1 osd.7 1053 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-21T09:50:04.903220+0000 front 2024-09-21T09:50:04.903325+0000 (oldest deadline 2024-09-21T09:50:29.603062+0000) 2024-09-21T09:50:43.942 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:50:43 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7[106022]: 2024-09-21T09:50:43.547+0000 7f338d84e640 -1 osd.7 1053 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-21T09:50:13.604550+0000 front 2024-09-21T09:50:13.604475+0000 (oldest deadline 2024-09-21T09:50:38.304479+0000) 2024-09-21T09:50:43.942 INFO:journalctl@ceph.osd.5.smithi064.stdout:Sep 21 09:50:43 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-5[96798]: 2024-09-21T09:50:43.635+0000 7fe957d67640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T09:50:43.942 INFO:journalctl@ceph.osd.5.smithi064.stdout:Sep 21 09:50:43 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-5[96798]: 2024-09-21T09:50:43.635+0000 7fe957d67640 -1 osd.5 1053 *** Got signal Terminated *** 2024-09-21T09:50:43.942 INFO:journalctl@ceph.osd.5.smithi064.stdout:Sep 21 09:50:43 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-5[96798]: 2024-09-21T09:50:43.635+0000 7fe957d67640 -1 osd.5 1053 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T09:50:44.441 INFO:journalctl@ceph.osd.5.smithi064.stdout:Sep 21 09:50:44 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-5[96798]: 2024-09-21T09:50:44.029+0000 7fe95437c640 -1 osd.5 1053 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-21T09:50:05.764363+0000 front 2024-09-21T09:50:05.764450+0000 (oldest deadline 2024-09-21T09:50:29.264172+0000) 2024-09-21T09:50:44.442 INFO:journalctl@ceph.osd.5.smithi064.stdout:Sep 21 09:50:44 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-5[96798]: 2024-09-21T09:50:44.029+0000 7fe95437c640 -1 osd.5 1053 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-21T09:50:14.565115+0000 front 2024-09-21T09:50:14.565274+0000 (oldest deadline 2024-09-21T09:50:40.465056+0000) 2024-09-21T09:50:44.942 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:50:44 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-6[101337]: 2024-09-21T09:50:44.465+0000 7f2af067a640 -1 osd.6 1053 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-21T09:50:06.172724+0000 front 2024-09-21T09:50:06.172984+0000 (oldest deadline 2024-09-21T09:50:30.872936+0000) 2024-09-21T09:50:44.942 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:50:44 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-6[101337]: 2024-09-21T09:50:44.465+0000 7f2af067a640 -1 osd.6 1053 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-21T09:50:13.773813+0000 front 2024-09-21T09:50:13.773822+0000 (oldest deadline 2024-09-21T09:50:37.873810+0000) 2024-09-21T09:50:44.942 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:50:44 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7[106022]: 2024-09-21T09:50:44.582+0000 7f338d84e640 -1 osd.7 1053 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-21T09:50:04.903220+0000 front 2024-09-21T09:50:04.903325+0000 (oldest deadline 2024-09-21T09:50:29.603062+0000) 2024-09-21T09:50:44.942 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:50:44 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7[106022]: 2024-09-21T09:50:44.582+0000 7f338d84e640 -1 osd.7 1053 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-21T09:50:13.604550+0000 front 2024-09-21T09:50:13.604475+0000 (oldest deadline 2024-09-21T09:50:38.304479+0000) 2024-09-21T09:50:45.445 INFO:journalctl@ceph.osd.5.smithi064.stdout:Sep 21 09:50:45 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-5[96798]: 2024-09-21T09:50:45.060+0000 7fe95437c640 -1 osd.5 1053 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-21T09:50:05.764363+0000 front 2024-09-21T09:50:05.764450+0000 (oldest deadline 2024-09-21T09:50:29.264172+0000) 2024-09-21T09:50:45.445 INFO:journalctl@ceph.osd.5.smithi064.stdout:Sep 21 09:50:45 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-5[96798]: 2024-09-21T09:50:45.060+0000 7fe95437c640 -1 osd.5 1053 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-21T09:50:14.565115+0000 front 2024-09-21T09:50:14.565274+0000 (oldest deadline 2024-09-21T09:50:40.465056+0000) 2024-09-21T09:50:45.941 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:50:45 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7[106022]: 2024-09-21T09:50:45.566+0000 7f338d84e640 -1 osd.7 1053 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-21T09:50:04.903220+0000 front 2024-09-21T09:50:04.903325+0000 (oldest deadline 2024-09-21T09:50:29.603062+0000) 2024-09-21T09:50:45.942 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:50:45 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7[106022]: 2024-09-21T09:50:45.566+0000 7f338d84e640 -1 osd.7 1053 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-21T09:50:13.604550+0000 front 2024-09-21T09:50:13.604475+0000 (oldest deadline 2024-09-21T09:50:38.304479+0000) 2024-09-21T09:50:45.942 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:50:45 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-6[101337]: 2024-09-21T09:50:45.503+0000 7f2af067a640 -1 osd.6 1053 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-21T09:50:06.172724+0000 front 2024-09-21T09:50:06.172984+0000 (oldest deadline 2024-09-21T09:50:30.872936+0000) 2024-09-21T09:50:45.942 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:50:45 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-6[101337]: 2024-09-21T09:50:45.503+0000 7f2af067a640 -1 osd.6 1053 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-21T09:50:13.773813+0000 front 2024-09-21T09:50:13.773822+0000 (oldest deadline 2024-09-21T09:50:37.873810+0000) 2024-09-21T09:50:46.441 INFO:journalctl@ceph.osd.5.smithi064.stdout:Sep 21 09:50:46 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-5[96798]: 2024-09-21T09:50:46.038+0000 7fe95437c640 -1 osd.5 1053 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-21T09:50:05.764363+0000 front 2024-09-21T09:50:05.764450+0000 (oldest deadline 2024-09-21T09:50:29.264172+0000) 2024-09-21T09:50:46.442 INFO:journalctl@ceph.osd.5.smithi064.stdout:Sep 21 09:50:46 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-5[96798]: 2024-09-21T09:50:46.038+0000 7fe95437c640 -1 osd.5 1053 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-21T09:50:14.565115+0000 front 2024-09-21T09:50:14.565274+0000 (oldest deadline 2024-09-21T09:50:40.465056+0000) 2024-09-21T09:50:46.941 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:50:46 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-6[101337]: 2024-09-21T09:50:46.545+0000 7f2af067a640 -1 osd.6 1053 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-21T09:50:06.172724+0000 front 2024-09-21T09:50:06.172984+0000 (oldest deadline 2024-09-21T09:50:30.872936+0000) 2024-09-21T09:50:46.942 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:50:46 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-6[101337]: 2024-09-21T09:50:46.545+0000 7f2af067a640 -1 osd.6 1053 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-21T09:50:13.773813+0000 front 2024-09-21T09:50:13.773822+0000 (oldest deadline 2024-09-21T09:50:37.873810+0000) 2024-09-21T09:50:46.942 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:50:46 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7[106022]: 2024-09-21T09:50:46.570+0000 7f338d84e640 -1 osd.7 1053 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-21T09:50:04.903220+0000 front 2024-09-21T09:50:04.903325+0000 (oldest deadline 2024-09-21T09:50:29.603062+0000) 2024-09-21T09:50:46.942 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:50:46 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7[106022]: 2024-09-21T09:50:46.570+0000 7f338d84e640 -1 osd.7 1053 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-21T09:50:13.604550+0000 front 2024-09-21T09:50:13.604475+0000 (oldest deadline 2024-09-21T09:50:38.304479+0000) 2024-09-21T09:50:47.442 INFO:journalctl@ceph.osd.5.smithi064.stdout:Sep 21 09:50:47 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-5[96798]: 2024-09-21T09:50:47.071+0000 7fe95437c640 -1 osd.5 1053 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-21T09:50:05.764363+0000 front 2024-09-21T09:50:05.764450+0000 (oldest deadline 2024-09-21T09:50:29.264172+0000) 2024-09-21T09:50:47.442 INFO:journalctl@ceph.osd.5.smithi064.stdout:Sep 21 09:50:47 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-5[96798]: 2024-09-21T09:50:47.071+0000 7fe95437c640 -1 osd.5 1053 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-21T09:50:14.565115+0000 front 2024-09-21T09:50:14.565274+0000 (oldest deadline 2024-09-21T09:50:40.465056+0000) 2024-09-21T09:50:47.442 INFO:journalctl@ceph.osd.5.smithi064.stdout:Sep 21 09:50:47 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-5[96798]: 2024-09-21T09:50:47.071+0000 7fe95437c640 -1 osd.5 1053 heartbeat_check: no reply from 172.21.15.22:6822 osd.2 since back 2024-09-21T09:50:23.266761+0000 front 2024-09-21T09:50:23.266700+0000 (oldest deadline 2024-09-21T09:50:46.166646+0000) 2024-09-21T09:50:47.941 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:50:47 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7[106022]: 2024-09-21T09:50:47.596+0000 7f338d84e640 -1 osd.7 1053 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-21T09:50:04.903220+0000 front 2024-09-21T09:50:04.903325+0000 (oldest deadline 2024-09-21T09:50:29.603062+0000) 2024-09-21T09:50:47.942 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:50:47 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7[106022]: 2024-09-21T09:50:47.596+0000 7f338d84e640 -1 osd.7 1053 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-21T09:50:13.604550+0000 front 2024-09-21T09:50:13.604475+0000 (oldest deadline 2024-09-21T09:50:38.304479+0000) 2024-09-21T09:50:47.942 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:50:47 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-6[101337]: 2024-09-21T09:50:47.558+0000 7f2af067a640 -1 osd.6 1053 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-21T09:50:06.172724+0000 front 2024-09-21T09:50:06.172984+0000 (oldest deadline 2024-09-21T09:50:30.872936+0000) 2024-09-21T09:50:47.942 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:50:47 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-6[101337]: 2024-09-21T09:50:47.558+0000 7f2af067a640 -1 osd.6 1053 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-21T09:50:13.773813+0000 front 2024-09-21T09:50:13.773822+0000 (oldest deadline 2024-09-21T09:50:37.873810+0000) 2024-09-21T09:50:47.942 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:50:47 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-6[101337]: 2024-09-21T09:50:47.558+0000 7f2af067a640 -1 osd.6 1053 heartbeat_check: no reply from 172.21.15.22:6822 osd.2 since back 2024-09-21T09:50:23.174867+0000 front 2024-09-21T09:50:23.174852+0000 (oldest deadline 2024-09-21T09:50:46.674681+0000) 2024-09-21T09:50:48.441 INFO:journalctl@ceph.osd.5.smithi064.stdout:Sep 21 09:50:48 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-5[96798]: 2024-09-21T09:50:48.058+0000 7fe95437c640 -1 osd.5 1053 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-21T09:50:05.764363+0000 front 2024-09-21T09:50:05.764450+0000 (oldest deadline 2024-09-21T09:50:29.264172+0000) 2024-09-21T09:50:48.442 INFO:journalctl@ceph.osd.5.smithi064.stdout:Sep 21 09:50:48 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-5[96798]: 2024-09-21T09:50:48.058+0000 7fe95437c640 -1 osd.5 1053 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-21T09:50:14.565115+0000 front 2024-09-21T09:50:14.565274+0000 (oldest deadline 2024-09-21T09:50:40.465056+0000) 2024-09-21T09:50:48.442 INFO:journalctl@ceph.osd.5.smithi064.stdout:Sep 21 09:50:48 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-5[96798]: 2024-09-21T09:50:48.058+0000 7fe95437c640 -1 osd.5 1053 heartbeat_check: no reply from 172.21.15.22:6822 osd.2 since back 2024-09-21T09:50:23.266761+0000 front 2024-09-21T09:50:23.266700+0000 (oldest deadline 2024-09-21T09:50:46.166646+0000) 2024-09-21T09:50:48.942 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:50:48 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-6[101337]: 2024-09-21T09:50:48.542+0000 7f2af067a640 -1 osd.6 1053 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-21T09:50:06.172724+0000 front 2024-09-21T09:50:06.172984+0000 (oldest deadline 2024-09-21T09:50:30.872936+0000) 2024-09-21T09:50:48.942 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:50:48 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-6[101337]: 2024-09-21T09:50:48.542+0000 7f2af067a640 -1 osd.6 1053 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-21T09:50:13.773813+0000 front 2024-09-21T09:50:13.773822+0000 (oldest deadline 2024-09-21T09:50:37.873810+0000) 2024-09-21T09:50:48.942 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:50:48 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-6[101337]: 2024-09-21T09:50:48.542+0000 7f2af067a640 -1 osd.6 1053 heartbeat_check: no reply from 172.21.15.22:6822 osd.2 since back 2024-09-21T09:50:23.174867+0000 front 2024-09-21T09:50:23.174852+0000 (oldest deadline 2024-09-21T09:50:46.674681+0000) 2024-09-21T09:50:48.942 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:50:48 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7[106022]: 2024-09-21T09:50:48.605+0000 7f338d84e640 -1 osd.7 1053 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-21T09:50:04.903220+0000 front 2024-09-21T09:50:04.903325+0000 (oldest deadline 2024-09-21T09:50:29.603062+0000) 2024-09-21T09:50:48.942 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:50:48 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7[106022]: 2024-09-21T09:50:48.605+0000 7f338d84e640 -1 osd.7 1053 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-21T09:50:13.604550+0000 front 2024-09-21T09:50:13.604475+0000 (oldest deadline 2024-09-21T09:50:38.304479+0000) 2024-09-21T09:50:48.943 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:50:48 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7[106022]: 2024-09-21T09:50:48.605+0000 7f338d84e640 -1 osd.7 1053 heartbeat_check: no reply from 172.21.15.22:6822 osd.2 since back 2024-09-21T09:50:23.506042+0000 front 2024-09-21T09:50:23.506144+0000 (oldest deadline 2024-09-21T09:50:48.205982+0000) 2024-09-21T09:50:49.370 INFO:journalctl@ceph.osd.5.smithi064.stdout:Sep 21 09:50:49 smithi064 podman[119383]: 2024-09-21 09:50:49.075915854 +0000 UTC m=+5.550136874 container died b3225b090dba337265d1ede30df4d2a244593ae6b599825f9b9418db3ffa823e (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-5, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=main-62da54b, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T09:50:49.658 INFO:journalctl@ceph.osd.5.smithi064.stdout:Sep 21 09:50:49 smithi064 podman[119383]: 2024-09-21 09:50:49.483888916 +0000 UTC m=+5.958109933 container cleanup b3225b090dba337265d1ede30df4d2a244593ae6b599825f9b9418db3ffa823e (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-5, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-62da54b, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731) 2024-09-21T09:50:49.658 INFO:journalctl@ceph.osd.5.smithi064.stdout:Sep 21 09:50:49 smithi064 bash[119383]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-5 2024-09-21T09:50:49.658 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:50:49 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-6[101337]: 2024-09-21T09:50:49.504+0000 7f2af067a640 -1 osd.6 1053 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-21T09:50:06.172724+0000 front 2024-09-21T09:50:06.172984+0000 (oldest deadline 2024-09-21T09:50:30.872936+0000) 2024-09-21T09:50:49.658 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:50:49 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-6[101337]: 2024-09-21T09:50:49.504+0000 7f2af067a640 -1 osd.6 1053 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-21T09:50:13.773813+0000 front 2024-09-21T09:50:13.773822+0000 (oldest deadline 2024-09-21T09:50:37.873810+0000) 2024-09-21T09:50:49.658 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:50:49 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-6[101337]: 2024-09-21T09:50:49.504+0000 7f2af067a640 -1 osd.6 1053 heartbeat_check: no reply from 172.21.15.22:6822 osd.2 since back 2024-09-21T09:50:23.174867+0000 front 2024-09-21T09:50:23.174852+0000 (oldest deadline 2024-09-21T09:50:46.674681+0000) 2024-09-21T09:50:49.941 INFO:journalctl@ceph.osd.5.smithi064.stdout:Sep 21 09:50:49 smithi064 podman[119414]: 2024-09-21 09:50:49.760489062 +0000 UTC m=+0.676714981 container remove b3225b090dba337265d1ede30df4d2a244593ae6b599825f9b9418db3ffa823e (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-5, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, RELEASE=main-62da54b, CEPH_POINT_RELEASE=) 2024-09-21T09:50:49.942 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:50:49 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7[106022]: 2024-09-21T09:50:49.654+0000 7f338d84e640 -1 osd.7 1053 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-21T09:50:04.903220+0000 front 2024-09-21T09:50:04.903325+0000 (oldest deadline 2024-09-21T09:50:29.603062+0000) 2024-09-21T09:50:49.942 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:50:49 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7[106022]: 2024-09-21T09:50:49.654+0000 7f338d84e640 -1 osd.7 1053 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-21T09:50:13.604550+0000 front 2024-09-21T09:50:13.604475+0000 (oldest deadline 2024-09-21T09:50:38.304479+0000) 2024-09-21T09:50:49.942 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:50:49 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7[106022]: 2024-09-21T09:50:49.654+0000 7f338d84e640 -1 osd.7 1053 heartbeat_check: no reply from 172.21.15.22:6822 osd.2 since back 2024-09-21T09:50:23.506042+0000 front 2024-09-21T09:50:23.506144+0000 (oldest deadline 2024-09-21T09:50:48.205982+0000) 2024-09-21T09:50:50.407 INFO:journalctl@ceph.osd.5.smithi064.stdout:Sep 21 09:50:50 smithi064 podman[119510]: 2024-09-21 09:50:50.017375205 +0000 UTC m=+0.016545646 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:50:50.407 INFO:journalctl@ceph.osd.5.smithi064.stdout:Sep 21 09:50:50 smithi064 podman[119510]: 2024-09-21 09:50:50.119662703 +0000 UTC m=+0.118833185 container create 8b86accbf3958a3d3c08bdc6f3979dfcaf53064d7b675d5a9b6c8a945bf7e861 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-5-deactivate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, RELEASE=main-62da54b, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T09:50:50.407 INFO:journalctl@ceph.osd.5.smithi064.stdout:Sep 21 09:50:50 smithi064 podman[119510]: 2024-09-21 09:50:50.313808082 +0000 UTC m=+0.312978528 container init 8b86accbf3958a3d3c08bdc6f3979dfcaf53064d7b675d5a9b6c8a945bf7e861 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-5-deactivate, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, RELEASE=main-62da54b, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_BRANCH=HEAD) 2024-09-21T09:50:50.407 INFO:journalctl@ceph.osd.5.smithi064.stdout:Sep 21 09:50:50 smithi064 podman[119510]: 2024-09-21 09:50:50.31879064 +0000 UTC m=+0.317961082 container start 8b86accbf3958a3d3c08bdc6f3979dfcaf53064d7b675d5a9b6c8a945bf7e861 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-5-deactivate, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, RELEASE=main-62da54b, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux ) 2024-09-21T09:50:50.407 INFO:journalctl@ceph.osd.5.smithi064.stdout:Sep 21 09:50:50 smithi064 podman[119510]: 2024-09-21 09:50:50.405183989 +0000 UTC m=+0.404354445 container attach 8b86accbf3958a3d3c08bdc6f3979dfcaf53064d7b675d5a9b6c8a945bf7e861 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-5-deactivate, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , RELEASE=main-62da54b, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, ceph=True) 2024-09-21T09:50:50.692 INFO:journalctl@ceph.osd.5.smithi064.stdout:Sep 21 09:50:50 smithi064 podman[119510]: 2024-09-21 09:50:50.513760708 +0000 UTC m=+0.512931149 container died 8b86accbf3958a3d3c08bdc6f3979dfcaf53064d7b675d5a9b6c8a945bf7e861 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-5-deactivate, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_CLEAN=True, ceph=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, RELEASE=main-62da54b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T09:50:50.693 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:50:50 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-6[101337]: 2024-09-21T09:50:50.489+0000 7f2af067a640 -1 osd.6 1053 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-21T09:50:06.172724+0000 front 2024-09-21T09:50:06.172984+0000 (oldest deadline 2024-09-21T09:50:30.872936+0000) 2024-09-21T09:50:50.693 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:50:50 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-6[101337]: 2024-09-21T09:50:50.489+0000 7f2af067a640 -1 osd.6 1053 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-21T09:50:13.773813+0000 front 2024-09-21T09:50:13.773822+0000 (oldest deadline 2024-09-21T09:50:37.873810+0000) 2024-09-21T09:50:50.693 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:50:50 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-6[101337]: 2024-09-21T09:50:50.489+0000 7f2af067a640 -1 osd.6 1053 heartbeat_check: no reply from 172.21.15.22:6822 osd.2 since back 2024-09-21T09:50:23.174867+0000 front 2024-09-21T09:50:23.174852+0000 (oldest deadline 2024-09-21T09:50:46.674681+0000) 2024-09-21T09:50:50.693 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:50:50 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7[106022]: 2024-09-21T09:50:50.609+0000 7f338d84e640 -1 osd.7 1053 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-21T09:50:04.903220+0000 front 2024-09-21T09:50:04.903325+0000 (oldest deadline 2024-09-21T09:50:29.603062+0000) 2024-09-21T09:50:50.694 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:50:50 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7[106022]: 2024-09-21T09:50:50.609+0000 7f338d84e640 -1 osd.7 1053 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-21T09:50:13.604550+0000 front 2024-09-21T09:50:13.604475+0000 (oldest deadline 2024-09-21T09:50:38.304479+0000) 2024-09-21T09:50:50.694 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:50:50 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7[106022]: 2024-09-21T09:50:50.609+0000 7f338d84e640 -1 osd.7 1053 heartbeat_check: no reply from 172.21.15.22:6822 osd.2 since back 2024-09-21T09:50:23.506042+0000 front 2024-09-21T09:50:23.506144+0000 (oldest deadline 2024-09-21T09:50:48.205982+0000) 2024-09-21T09:50:51.055 INFO:journalctl@ceph.osd.5.smithi064.stdout:Sep 21 09:50:51 smithi064 podman[119554]: 2024-09-21 09:50:51.049991832 +0000 UTC m=+0.530593684 container remove 8b86accbf3958a3d3c08bdc6f3979dfcaf53064d7b675d5a9b6c8a945bf7e861 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-5-deactivate, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , RELEASE=main-62da54b, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0) 2024-09-21T09:50:51.076 DEBUG:teuthology.orchestra.run.smithi064:> sudo pkill -f 'journalctl -f -n 0 -u ceph-d65e801e-77fa-11ef-baf6-efdc52797490@osd.5.service' 2024-09-21T09:50:51.120 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T09:50:51.122 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-09-21T09:50:51.122 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-09-21T09:50:51.122 DEBUG:teuthology.orchestra.run.smithi064:> sudo systemctl stop ceph-d65e801e-77fa-11ef-baf6-efdc52797490@osd.6 2024-09-21T09:50:51.442 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:50:51 smithi064 systemd[1]: Stopping Ceph osd.6 for d65e801e-77fa-11ef-baf6-efdc52797490... 2024-09-21T09:50:51.942 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:50:51 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7[106022]: 2024-09-21T09:50:51.604+0000 7f338d84e640 -1 osd.7 1053 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-21T09:50:04.903220+0000 front 2024-09-21T09:50:04.903325+0000 (oldest deadline 2024-09-21T09:50:29.603062+0000) 2024-09-21T09:50:51.942 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:50:51 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7[106022]: 2024-09-21T09:50:51.604+0000 7f338d84e640 -1 osd.7 1053 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-21T09:50:13.604550+0000 front 2024-09-21T09:50:13.604475+0000 (oldest deadline 2024-09-21T09:50:38.304479+0000) 2024-09-21T09:50:51.943 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:50:51 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7[106022]: 2024-09-21T09:50:51.604+0000 7f338d84e640 -1 osd.7 1053 heartbeat_check: no reply from 172.21.15.22:6822 osd.2 since back 2024-09-21T09:50:23.506042+0000 front 2024-09-21T09:50:23.506144+0000 (oldest deadline 2024-09-21T09:50:48.205982+0000) 2024-09-21T09:50:51.943 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:50:51 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-6[101337]: 2024-09-21T09:50:51.463+0000 7f2af806d640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T09:50:51.943 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:50:51 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-6[101337]: 2024-09-21T09:50:51.463+0000 7f2af806d640 -1 osd.6 1053 *** Got signal Terminated *** 2024-09-21T09:50:51.944 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:50:51 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-6[101337]: 2024-09-21T09:50:51.463+0000 7f2af806d640 -1 osd.6 1053 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T09:50:51.944 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:50:51 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-6[101337]: 2024-09-21T09:50:51.489+0000 7f2af067a640 -1 osd.6 1053 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-21T09:50:06.172724+0000 front 2024-09-21T09:50:06.172984+0000 (oldest deadline 2024-09-21T09:50:30.872936+0000) 2024-09-21T09:50:51.945 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:50:51 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-6[101337]: 2024-09-21T09:50:51.489+0000 7f2af067a640 -1 osd.6 1053 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-21T09:50:13.773813+0000 front 2024-09-21T09:50:13.773822+0000 (oldest deadline 2024-09-21T09:50:37.873810+0000) 2024-09-21T09:50:51.946 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:50:51 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-6[101337]: 2024-09-21T09:50:51.489+0000 7f2af067a640 -1 osd.6 1053 heartbeat_check: no reply from 172.21.15.22:6822 osd.2 since back 2024-09-21T09:50:23.174867+0000 front 2024-09-21T09:50:23.174852+0000 (oldest deadline 2024-09-21T09:50:46.674681+0000) 2024-09-21T09:50:52.942 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:50:52 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-6[101337]: 2024-09-21T09:50:52.445+0000 7f2af067a640 -1 osd.6 1053 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-21T09:50:06.172724+0000 front 2024-09-21T09:50:06.172984+0000 (oldest deadline 2024-09-21T09:50:30.872936+0000) 2024-09-21T09:50:52.942 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:50:52 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-6[101337]: 2024-09-21T09:50:52.445+0000 7f2af067a640 -1 osd.6 1053 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-21T09:50:13.773813+0000 front 2024-09-21T09:50:13.773822+0000 (oldest deadline 2024-09-21T09:50:37.873810+0000) 2024-09-21T09:50:52.942 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:50:52 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-6[101337]: 2024-09-21T09:50:52.445+0000 7f2af067a640 -1 osd.6 1053 heartbeat_check: no reply from 172.21.15.22:6822 osd.2 since back 2024-09-21T09:50:23.174867+0000 front 2024-09-21T09:50:23.174852+0000 (oldest deadline 2024-09-21T09:50:46.674681+0000) 2024-09-21T09:50:52.942 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:50:52 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7[106022]: 2024-09-21T09:50:52.589+0000 7f338d84e640 -1 osd.7 1053 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-21T09:50:04.903220+0000 front 2024-09-21T09:50:04.903325+0000 (oldest deadline 2024-09-21T09:50:29.603062+0000) 2024-09-21T09:50:52.943 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:50:52 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7[106022]: 2024-09-21T09:50:52.589+0000 7f338d84e640 -1 osd.7 1053 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-21T09:50:13.604550+0000 front 2024-09-21T09:50:13.604475+0000 (oldest deadline 2024-09-21T09:50:38.304479+0000) 2024-09-21T09:50:52.943 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:50:52 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7[106022]: 2024-09-21T09:50:52.589+0000 7f338d84e640 -1 osd.7 1053 heartbeat_check: no reply from 172.21.15.22:6822 osd.2 since back 2024-09-21T09:50:23.506042+0000 front 2024-09-21T09:50:23.506144+0000 (oldest deadline 2024-09-21T09:50:48.205982+0000) 2024-09-21T09:50:53.691 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:50:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-6[101337]: 2024-09-21T09:50:53.398+0000 7f2af067a640 -1 osd.6 1053 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-21T09:50:06.172724+0000 front 2024-09-21T09:50:06.172984+0000 (oldest deadline 2024-09-21T09:50:30.872936+0000) 2024-09-21T09:50:53.692 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:50:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-6[101337]: 2024-09-21T09:50:53.398+0000 7f2af067a640 -1 osd.6 1053 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-21T09:50:13.773813+0000 front 2024-09-21T09:50:13.773822+0000 (oldest deadline 2024-09-21T09:50:37.873810+0000) 2024-09-21T09:50:53.692 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:50:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-6[101337]: 2024-09-21T09:50:53.398+0000 7f2af067a640 -1 osd.6 1053 heartbeat_check: no reply from 172.21.15.22:6822 osd.2 since back 2024-09-21T09:50:23.174867+0000 front 2024-09-21T09:50:23.174852+0000 (oldest deadline 2024-09-21T09:50:46.674681+0000) 2024-09-21T09:50:53.692 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:50:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7[106022]: 2024-09-21T09:50:53.637+0000 7f338d84e640 -1 osd.7 1053 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-21T09:50:04.903220+0000 front 2024-09-21T09:50:04.903325+0000 (oldest deadline 2024-09-21T09:50:29.603062+0000) 2024-09-21T09:50:53.692 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:50:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7[106022]: 2024-09-21T09:50:53.637+0000 7f338d84e640 -1 osd.7 1053 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-21T09:50:13.604550+0000 front 2024-09-21T09:50:13.604475+0000 (oldest deadline 2024-09-21T09:50:38.304479+0000) 2024-09-21T09:50:53.693 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:50:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7[106022]: 2024-09-21T09:50:53.637+0000 7f338d84e640 -1 osd.7 1053 heartbeat_check: no reply from 172.21.15.22:6822 osd.2 since back 2024-09-21T09:50:23.506042+0000 front 2024-09-21T09:50:23.506144+0000 (oldest deadline 2024-09-21T09:50:48.205982+0000) 2024-09-21T09:50:53.693 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:50:53 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7[106022]: 2024-09-21T09:50:53.637+0000 7f338d84e640 -1 osd.7 1053 heartbeat_check: no reply from 172.21.15.22:6830 osd.3 since back 2024-09-21T09:50:28.206629+0000 front 2024-09-21T09:50:28.206642+0000 (oldest deadline 2024-09-21T09:50:53.506420+0000) 2024-09-21T09:50:54.440 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:50:54 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=cleanup t=2024-09-21T09:50:54.058786051Z level=info msg="Completed cleanup jobs" duration=2.658177ms 2024-09-21T09:50:54.440 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:50:54 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=sqlstore.transactions t=2024-09-21T09:50:54.065448286Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-09-21T09:50:54.440 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:50:54 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=plugins.update.checker t=2024-09-21T09:50:54.21070897Z level=info msg="Update check succeeded" duration=67.527814ms 2024-09-21T09:50:54.692 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:50:54 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7[106022]: 2024-09-21T09:50:54.604+0000 7f338d84e640 -1 osd.7 1053 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-21T09:50:04.903220+0000 front 2024-09-21T09:50:04.903325+0000 (oldest deadline 2024-09-21T09:50:29.603062+0000) 2024-09-21T09:50:54.692 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:50:54 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7[106022]: 2024-09-21T09:50:54.604+0000 7f338d84e640 -1 osd.7 1053 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-21T09:50:13.604550+0000 front 2024-09-21T09:50:13.604475+0000 (oldest deadline 2024-09-21T09:50:38.304479+0000) 2024-09-21T09:50:54.692 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:50:54 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7[106022]: 2024-09-21T09:50:54.604+0000 7f338d84e640 -1 osd.7 1053 heartbeat_check: no reply from 172.21.15.22:6822 osd.2 since back 2024-09-21T09:50:23.506042+0000 front 2024-09-21T09:50:23.506144+0000 (oldest deadline 2024-09-21T09:50:48.205982+0000) 2024-09-21T09:50:54.692 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:50:54 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7[106022]: 2024-09-21T09:50:54.604+0000 7f338d84e640 -1 osd.7 1053 heartbeat_check: no reply from 172.21.15.22:6830 osd.3 since back 2024-09-21T09:50:28.206629+0000 front 2024-09-21T09:50:28.206642+0000 (oldest deadline 2024-09-21T09:50:53.506420+0000) 2024-09-21T09:50:54.693 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:50:54 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-6[101337]: 2024-09-21T09:50:54.437+0000 7f2af067a640 -1 osd.6 1053 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-21T09:50:06.172724+0000 front 2024-09-21T09:50:06.172984+0000 (oldest deadline 2024-09-21T09:50:30.872936+0000) 2024-09-21T09:50:54.693 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:50:54 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-6[101337]: 2024-09-21T09:50:54.437+0000 7f2af067a640 -1 osd.6 1053 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-21T09:50:13.773813+0000 front 2024-09-21T09:50:13.773822+0000 (oldest deadline 2024-09-21T09:50:37.873810+0000) 2024-09-21T09:50:54.693 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:50:54 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-6[101337]: 2024-09-21T09:50:54.437+0000 7f2af067a640 -1 osd.6 1053 heartbeat_check: no reply from 172.21.15.22:6822 osd.2 since back 2024-09-21T09:50:23.174867+0000 front 2024-09-21T09:50:23.174852+0000 (oldest deadline 2024-09-21T09:50:46.674681+0000) 2024-09-21T09:50:55.691 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:50:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-6[101337]: 2024-09-21T09:50:55.397+0000 7f2af067a640 -1 osd.6 1053 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-21T09:50:06.172724+0000 front 2024-09-21T09:50:06.172984+0000 (oldest deadline 2024-09-21T09:50:30.872936+0000) 2024-09-21T09:50:55.692 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:50:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-6[101337]: 2024-09-21T09:50:55.397+0000 7f2af067a640 -1 osd.6 1053 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-21T09:50:13.773813+0000 front 2024-09-21T09:50:13.773822+0000 (oldest deadline 2024-09-21T09:50:37.873810+0000) 2024-09-21T09:50:55.692 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:50:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-6[101337]: 2024-09-21T09:50:55.397+0000 7f2af067a640 -1 osd.6 1053 heartbeat_check: no reply from 172.21.15.22:6822 osd.2 since back 2024-09-21T09:50:23.174867+0000 front 2024-09-21T09:50:23.174852+0000 (oldest deadline 2024-09-21T09:50:46.674681+0000) 2024-09-21T09:50:55.692 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:50:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7[106022]: 2024-09-21T09:50:55.563+0000 7f338d84e640 -1 osd.7 1053 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-21T09:50:04.903220+0000 front 2024-09-21T09:50:04.903325+0000 (oldest deadline 2024-09-21T09:50:29.603062+0000) 2024-09-21T09:50:55.692 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:50:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7[106022]: 2024-09-21T09:50:55.563+0000 7f338d84e640 -1 osd.7 1053 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-21T09:50:13.604550+0000 front 2024-09-21T09:50:13.604475+0000 (oldest deadline 2024-09-21T09:50:38.304479+0000) 2024-09-21T09:50:55.693 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:50:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7[106022]: 2024-09-21T09:50:55.563+0000 7f338d84e640 -1 osd.7 1053 heartbeat_check: no reply from 172.21.15.22:6822 osd.2 since back 2024-09-21T09:50:23.506042+0000 front 2024-09-21T09:50:23.506144+0000 (oldest deadline 2024-09-21T09:50:48.205982+0000) 2024-09-21T09:50:55.693 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:50:55 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7[106022]: 2024-09-21T09:50:55.563+0000 7f338d84e640 -1 osd.7 1053 heartbeat_check: no reply from 172.21.15.22:6830 osd.3 since back 2024-09-21T09:50:28.206629+0000 front 2024-09-21T09:50:28.206642+0000 (oldest deadline 2024-09-21T09:50:53.506420+0000) 2024-09-21T09:50:56.691 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:50:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-6[101337]: 2024-09-21T09:50:56.360+0000 7f2af067a640 -1 osd.6 1053 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-21T09:50:06.172724+0000 front 2024-09-21T09:50:06.172984+0000 (oldest deadline 2024-09-21T09:50:30.872936+0000) 2024-09-21T09:50:56.692 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:50:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-6[101337]: 2024-09-21T09:50:56.360+0000 7f2af067a640 -1 osd.6 1053 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-21T09:50:13.773813+0000 front 2024-09-21T09:50:13.773822+0000 (oldest deadline 2024-09-21T09:50:37.873810+0000) 2024-09-21T09:50:56.692 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:50:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-6[101337]: 2024-09-21T09:50:56.360+0000 7f2af067a640 -1 osd.6 1053 heartbeat_check: no reply from 172.21.15.22:6822 osd.2 since back 2024-09-21T09:50:23.174867+0000 front 2024-09-21T09:50:23.174852+0000 (oldest deadline 2024-09-21T09:50:46.674681+0000) 2024-09-21T09:50:56.692 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:50:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-6[101337]: 2024-09-21T09:50:56.360+0000 7f2af067a640 -1 osd.6 1053 heartbeat_check: no reply from 172.21.15.22:6830 osd.3 since back 2024-09-21T09:50:32.575632+0000 front 2024-09-21T09:50:32.575662+0000 (oldest deadline 2024-09-21T09:50:56.075464+0000) 2024-09-21T09:50:56.692 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:50:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7[106022]: 2024-09-21T09:50:56.526+0000 7f338d84e640 -1 osd.7 1053 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-21T09:50:04.903220+0000 front 2024-09-21T09:50:04.903325+0000 (oldest deadline 2024-09-21T09:50:29.603062+0000) 2024-09-21T09:50:56.692 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:50:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7[106022]: 2024-09-21T09:50:56.526+0000 7f338d84e640 -1 osd.7 1053 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-21T09:50:13.604550+0000 front 2024-09-21T09:50:13.604475+0000 (oldest deadline 2024-09-21T09:50:38.304479+0000) 2024-09-21T09:50:56.692 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:50:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7[106022]: 2024-09-21T09:50:56.526+0000 7f338d84e640 -1 osd.7 1053 heartbeat_check: no reply from 172.21.15.22:6822 osd.2 since back 2024-09-21T09:50:23.506042+0000 front 2024-09-21T09:50:23.506144+0000 (oldest deadline 2024-09-21T09:50:48.205982+0000) 2024-09-21T09:50:56.693 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:50:56 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7[106022]: 2024-09-21T09:50:56.526+0000 7f338d84e640 -1 osd.7 1053 heartbeat_check: no reply from 172.21.15.22:6830 osd.3 since back 2024-09-21T09:50:28.206629+0000 front 2024-09-21T09:50:28.206642+0000 (oldest deadline 2024-09-21T09:50:53.506420+0000) 2024-09-21T09:50:57.247 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:50:56 smithi064 podman[119639]: 2024-09-21 09:50:56.973952832 +0000 UTC m=+5.624568444 container died c6a9a09637e5fdf130945b06cb13437f6df561735be9d226c471a470e7009b59 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-6, io.buildah.version=1.37.2, RELEASE=main-62da54b, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731) 2024-09-21T09:50:57.597 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:50:57 smithi064 podman[119639]: 2024-09-21 09:50:57.3443558 +0000 UTC m=+5.994971411 container cleanup c6a9a09637e5fdf130945b06cb13437f6df561735be9d226c471a470e7009b59 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-6, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, RELEASE=main-62da54b, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731) 2024-09-21T09:50:57.597 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:50:57 smithi064 bash[119639]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-6 2024-09-21T09:50:57.597 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:50:57 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7[106022]: 2024-09-21T09:50:57.484+0000 7f338d84e640 -1 osd.7 1053 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-21T09:50:04.903220+0000 front 2024-09-21T09:50:04.903325+0000 (oldest deadline 2024-09-21T09:50:29.603062+0000) 2024-09-21T09:50:57.597 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:50:57 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7[106022]: 2024-09-21T09:50:57.484+0000 7f338d84e640 -1 osd.7 1053 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-21T09:50:13.604550+0000 front 2024-09-21T09:50:13.604475+0000 (oldest deadline 2024-09-21T09:50:38.304479+0000) 2024-09-21T09:50:57.598 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:50:57 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7[106022]: 2024-09-21T09:50:57.484+0000 7f338d84e640 -1 osd.7 1053 heartbeat_check: no reply from 172.21.15.22:6822 osd.2 since back 2024-09-21T09:50:23.506042+0000 front 2024-09-21T09:50:23.506144+0000 (oldest deadline 2024-09-21T09:50:48.205982+0000) 2024-09-21T09:50:57.598 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:50:57 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7[106022]: 2024-09-21T09:50:57.484+0000 7f338d84e640 -1 osd.7 1053 heartbeat_check: no reply from 172.21.15.22:6830 osd.3 since back 2024-09-21T09:50:28.206629+0000 front 2024-09-21T09:50:28.206642+0000 (oldest deadline 2024-09-21T09:50:53.506420+0000) 2024-09-21T09:50:57.914 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:50:57 smithi064 podman[119661]: 2024-09-21 09:50:57.59568385 +0000 UTC m=+0.614963166 container remove c6a9a09637e5fdf130945b06cb13437f6df561735be9d226c471a470e7009b59 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-6, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, RELEASE=main-62da54b, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux ) 2024-09-21T09:50:58.192 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:50:57 smithi064 podman[119755]: 2024-09-21 09:50:57.91327462 +0000 UTC m=+0.099702874 container create 04d058699347dcfd672a07f4af3d053adae5067635862e813b46b8e77d389fd6 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-6-deactivate, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_CLEAN=True, RELEASE=main-62da54b, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-09-21T09:50:58.192 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:50:57 smithi064 podman[119755]: 2024-09-21 09:50:57.82976316 +0000 UTC m=+0.016191434 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:50:58.192 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:50:58 smithi064 podman[119755]: 2024-09-21 09:50:58.123876712 +0000 UTC m=+0.310304979 container init 04d058699347dcfd672a07f4af3d053adae5067635862e813b46b8e77d389fd6 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-6-deactivate, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, RELEASE=main-62da54b, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0) 2024-09-21T09:50:58.192 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:50:58 smithi064 podman[119755]: 2024-09-21 09:50:58.128990519 +0000 UTC m=+0.315418776 container start 04d058699347dcfd672a07f4af3d053adae5067635862e813b46b8e77d389fd6 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-6-deactivate, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, RELEASE=main-62da54b, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T09:50:58.678 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:50:58 smithi064 podman[119755]: 2024-09-21 09:50:58.215225463 +0000 UTC m=+0.401653730 container attach 04d058699347dcfd672a07f4af3d053adae5067635862e813b46b8e77d389fd6 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-6-deactivate, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, RELEASE=main-62da54b, org.label-schema.schema-version=1.0) 2024-09-21T09:50:58.679 INFO:journalctl@ceph.osd.6.smithi064.stdout:Sep 21 09:50:58 smithi064 podman[119755]: 2024-09-21 09:50:58.329760197 +0000 UTC m=+0.516188461 container died 04d058699347dcfd672a07f4af3d053adae5067635862e813b46b8e77d389fd6 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-6-deactivate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.37.2, RELEASE=main-62da54b, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=) 2024-09-21T09:50:58.679 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:50:58 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7[106022]: 2024-09-21T09:50:58.453+0000 7f338d84e640 -1 osd.7 1053 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-21T09:50:04.903220+0000 front 2024-09-21T09:50:04.903325+0000 (oldest deadline 2024-09-21T09:50:29.603062+0000) 2024-09-21T09:50:58.679 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:50:58 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7[106022]: 2024-09-21T09:50:58.453+0000 7f338d84e640 -1 osd.7 1053 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-21T09:50:13.604550+0000 front 2024-09-21T09:50:13.604475+0000 (oldest deadline 2024-09-21T09:50:38.304479+0000) 2024-09-21T09:50:58.679 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:50:58 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7[106022]: 2024-09-21T09:50:58.453+0000 7f338d84e640 -1 osd.7 1053 heartbeat_check: no reply from 172.21.15.22:6822 osd.2 since back 2024-09-21T09:50:23.506042+0000 front 2024-09-21T09:50:23.506144+0000 (oldest deadline 2024-09-21T09:50:48.205982+0000) 2024-09-21T09:50:58.679 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:50:58 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7[106022]: 2024-09-21T09:50:58.453+0000 7f338d84e640 -1 osd.7 1053 heartbeat_check: no reply from 172.21.15.22:6830 osd.3 since back 2024-09-21T09:50:28.206629+0000 front 2024-09-21T09:50:28.206642+0000 (oldest deadline 2024-09-21T09:50:53.506420+0000) 2024-09-21T09:50:58.999 DEBUG:teuthology.orchestra.run.smithi064:> sudo pkill -f 'journalctl -f -n 0 -u ceph-d65e801e-77fa-11ef-baf6-efdc52797490@osd.6.service' 2024-09-21T09:50:59.038 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T09:50:59.038 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-09-21T09:50:59.039 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-09-21T09:50:59.039 DEBUG:teuthology.orchestra.run.smithi064:> sudo systemctl stop ceph-d65e801e-77fa-11ef-baf6-efdc52797490@osd.7 2024-09-21T09:50:59.375 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:50:59 smithi064 systemd[1]: Stopping Ceph osd.7 for d65e801e-77fa-11ef-baf6-efdc52797490... 2024-09-21T09:50:59.691 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:50:59 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7[106022]: 2024-09-21T09:50:59.372+0000 7f3395241640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T09:50:59.692 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:50:59 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7[106022]: 2024-09-21T09:50:59.372+0000 7f3395241640 -1 osd.7 1053 *** Got signal Terminated *** 2024-09-21T09:50:59.692 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:50:59 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7[106022]: 2024-09-21T09:50:59.372+0000 7f3395241640 -1 osd.7 1053 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T09:50:59.692 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:50:59 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7[106022]: 2024-09-21T09:50:59.413+0000 7f338d84e640 -1 osd.7 1053 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-21T09:50:04.903220+0000 front 2024-09-21T09:50:04.903325+0000 (oldest deadline 2024-09-21T09:50:29.603062+0000) 2024-09-21T09:50:59.692 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:50:59 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7[106022]: 2024-09-21T09:50:59.413+0000 7f338d84e640 -1 osd.7 1053 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-21T09:50:13.604550+0000 front 2024-09-21T09:50:13.604475+0000 (oldest deadline 2024-09-21T09:50:38.304479+0000) 2024-09-21T09:50:59.692 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:50:59 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7[106022]: 2024-09-21T09:50:59.413+0000 7f338d84e640 -1 osd.7 1053 heartbeat_check: no reply from 172.21.15.22:6822 osd.2 since back 2024-09-21T09:50:23.506042+0000 front 2024-09-21T09:50:23.506144+0000 (oldest deadline 2024-09-21T09:50:48.205982+0000) 2024-09-21T09:50:59.692 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:50:59 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7[106022]: 2024-09-21T09:50:59.413+0000 7f338d84e640 -1 osd.7 1053 heartbeat_check: no reply from 172.21.15.22:6830 osd.3 since back 2024-09-21T09:50:28.206629+0000 front 2024-09-21T09:50:28.206642+0000 (oldest deadline 2024-09-21T09:50:53.506420+0000) 2024-09-21T09:51:00.943 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:51:00 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7[106022]: 2024-09-21T09:51:00.455+0000 7f338d84e640 -1 osd.7 1053 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-21T09:50:04.903220+0000 front 2024-09-21T09:50:04.903325+0000 (oldest deadline 2024-09-21T09:50:29.603062+0000) 2024-09-21T09:51:00.943 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:51:00 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7[106022]: 2024-09-21T09:51:00.455+0000 7f338d84e640 -1 osd.7 1053 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-21T09:50:13.604550+0000 front 2024-09-21T09:50:13.604475+0000 (oldest deadline 2024-09-21T09:50:38.304479+0000) 2024-09-21T09:51:00.943 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:51:00 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7[106022]: 2024-09-21T09:51:00.455+0000 7f338d84e640 -1 osd.7 1053 heartbeat_check: no reply from 172.21.15.22:6822 osd.2 since back 2024-09-21T09:50:23.506042+0000 front 2024-09-21T09:50:23.506144+0000 (oldest deadline 2024-09-21T09:50:48.205982+0000) 2024-09-21T09:51:00.944 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:51:00 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7[106022]: 2024-09-21T09:51:00.455+0000 7f338d84e640 -1 osd.7 1053 heartbeat_check: no reply from 172.21.15.22:6830 osd.3 since back 2024-09-21T09:50:28.206629+0000 front 2024-09-21T09:50:28.206642+0000 (oldest deadline 2024-09-21T09:50:53.506420+0000) 2024-09-21T09:51:01.941 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:51:01 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7[106022]: 2024-09-21T09:51:01.457+0000 7f338d84e640 -1 osd.7 1053 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-21T09:50:04.903220+0000 front 2024-09-21T09:50:04.903325+0000 (oldest deadline 2024-09-21T09:50:29.603062+0000) 2024-09-21T09:51:01.942 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:51:01 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7[106022]: 2024-09-21T09:51:01.457+0000 7f338d84e640 -1 osd.7 1053 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-21T09:50:13.604550+0000 front 2024-09-21T09:50:13.604475+0000 (oldest deadline 2024-09-21T09:50:38.304479+0000) 2024-09-21T09:51:01.942 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:51:01 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7[106022]: 2024-09-21T09:51:01.457+0000 7f338d84e640 -1 osd.7 1053 heartbeat_check: no reply from 172.21.15.22:6822 osd.2 since back 2024-09-21T09:50:23.506042+0000 front 2024-09-21T09:50:23.506144+0000 (oldest deadline 2024-09-21T09:50:48.205982+0000) 2024-09-21T09:51:01.942 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:51:01 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7[106022]: 2024-09-21T09:51:01.457+0000 7f338d84e640 -1 osd.7 1053 heartbeat_check: no reply from 172.21.15.22:6830 osd.3 since back 2024-09-21T09:50:28.206629+0000 front 2024-09-21T09:50:28.206642+0000 (oldest deadline 2024-09-21T09:50:53.506420+0000) 2024-09-21T09:51:02.692 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:51:02 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7[106022]: 2024-09-21T09:51:02.407+0000 7f338d84e640 -1 osd.7 1053 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-21T09:50:04.903220+0000 front 2024-09-21T09:50:04.903325+0000 (oldest deadline 2024-09-21T09:50:29.603062+0000) 2024-09-21T09:51:02.692 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:51:02 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7[106022]: 2024-09-21T09:51:02.407+0000 7f338d84e640 -1 osd.7 1053 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-21T09:50:13.604550+0000 front 2024-09-21T09:50:13.604475+0000 (oldest deadline 2024-09-21T09:50:38.304479+0000) 2024-09-21T09:51:02.692 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:51:02 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7[106022]: 2024-09-21T09:51:02.407+0000 7f338d84e640 -1 osd.7 1053 heartbeat_check: no reply from 172.21.15.22:6822 osd.2 since back 2024-09-21T09:50:23.506042+0000 front 2024-09-21T09:50:23.506144+0000 (oldest deadline 2024-09-21T09:50:48.205982+0000) 2024-09-21T09:51:02.692 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:51:02 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7[106022]: 2024-09-21T09:51:02.407+0000 7f338d84e640 -1 osd.7 1053 heartbeat_check: no reply from 172.21.15.22:6830 osd.3 since back 2024-09-21T09:50:28.206629+0000 front 2024-09-21T09:50:28.206642+0000 (oldest deadline 2024-09-21T09:50:53.506420+0000) 2024-09-21T09:51:02.692 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:51:02 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7[106022]: 2024-09-21T09:51:02.407+0000 7f338d84e640 -1 osd.7 1053 heartbeat_check: no reply from 172.21.15.64:6804 osd.4 since back 2024-09-21T09:50:39.207862+0000 front 2024-09-21T09:50:39.207937+0000 (oldest deadline 2024-09-21T09:51:02.008331+0000) 2024-09-21T09:51:03.692 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:51:03 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7[106022]: 2024-09-21T09:51:03.414+0000 7f338d84e640 -1 osd.7 1053 heartbeat_check: no reply from 172.21.15.22:6806 osd.0 since back 2024-09-21T09:50:04.903220+0000 front 2024-09-21T09:50:04.903325+0000 (oldest deadline 2024-09-21T09:50:29.603062+0000) 2024-09-21T09:51:03.692 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:51:03 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7[106022]: 2024-09-21T09:51:03.414+0000 7f338d84e640 -1 osd.7 1053 heartbeat_check: no reply from 172.21.15.22:6814 osd.1 since back 2024-09-21T09:50:13.604550+0000 front 2024-09-21T09:50:13.604475+0000 (oldest deadline 2024-09-21T09:50:38.304479+0000) 2024-09-21T09:51:03.692 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:51:03 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7[106022]: 2024-09-21T09:51:03.414+0000 7f338d84e640 -1 osd.7 1053 heartbeat_check: no reply from 172.21.15.22:6822 osd.2 since back 2024-09-21T09:50:23.506042+0000 front 2024-09-21T09:50:23.506144+0000 (oldest deadline 2024-09-21T09:50:48.205982+0000) 2024-09-21T09:51:03.692 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:51:03 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7[106022]: 2024-09-21T09:51:03.414+0000 7f338d84e640 -1 osd.7 1053 heartbeat_check: no reply from 172.21.15.22:6830 osd.3 since back 2024-09-21T09:50:28.206629+0000 front 2024-09-21T09:50:28.206642+0000 (oldest deadline 2024-09-21T09:50:53.506420+0000) 2024-09-21T09:51:03.692 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:51:03 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7[106022]: 2024-09-21T09:51:03.414+0000 7f338d84e640 -1 osd.7 1053 heartbeat_check: no reply from 172.21.15.64:6804 osd.4 since back 2024-09-21T09:50:39.207862+0000 front 2024-09-21T09:50:39.207937+0000 (oldest deadline 2024-09-21T09:51:02.008331+0000) 2024-09-21T09:51:05.199 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:51:04 smithi064 podman[119884]: 2024-09-21 09:51:04.942819972 +0000 UTC m=+5.678659751 container died 6f82655605e3f8dd46c90697549f33f1b6367e048120204dddeb9a008bd0a65f (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, RELEASE=main-62da54b, org.label-schema.license=GPLv2) 2024-09-21T09:51:05.599 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:51:05 smithi064 podman[119884]: 2024-09-21 09:51:05.288115812 +0000 UTC m=+6.023955584 container cleanup 6f82655605e3f8dd46c90697549f33f1b6367e048120204dddeb9a008bd0a65f (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-62da54b) 2024-09-21T09:51:05.599 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:51:05 smithi064 bash[119884]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7 2024-09-21T09:51:05.924 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:51:05 smithi064 podman[119906]: 2024-09-21 09:51:05.59770217 +0000 UTC m=+0.653500493 container remove 6f82655605e3f8dd46c90697549f33f1b6367e048120204dddeb9a008bd0a65f (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, RELEASE=main-62da54b, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, ceph=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2) 2024-09-21T09:51:06.192 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:51:05 smithi064 podman[120001]: 2024-09-21 09:51:05.825805318 +0000 UTC m=+0.016393450 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:51:06.192 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:51:05 smithi064 podman[120001]: 2024-09-21 09:51:05.931743403 +0000 UTC m=+0.122331527 container create d461a7492425a6334054e8b2861b5fd8eaddced1a1d5e82b551ffdf59baea246 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7-deactivate, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , RELEASE=main-62da54b, org.label-schema.build-date=20240731, GIT_CLEAN=True, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2) 2024-09-21T09:51:06.192 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:51:06 smithi064 podman[120001]: 2024-09-21 09:51:06.133889057 +0000 UTC m=+0.324477190 container init d461a7492425a6334054e8b2861b5fd8eaddced1a1d5e82b551ffdf59baea246 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7-deactivate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=main-62da54b, ceph=True, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T09:51:06.192 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:51:06 smithi064 podman[120001]: 2024-09-21 09:51:06.138990695 +0000 UTC m=+0.329578816 container start d461a7492425a6334054e8b2861b5fd8eaddced1a1d5e82b551ffdf59baea246 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7-deactivate, io.buildah.version=1.37.2, RELEASE=main-62da54b, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS) 2024-09-21T09:51:06.630 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:51:06 smithi064 podman[120001]: 2024-09-21 09:51:06.225441178 +0000 UTC m=+0.416029305 container attach d461a7492425a6334054e8b2861b5fd8eaddced1a1d5e82b551ffdf59baea246 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7-deactivate, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, RELEASE=main-62da54b, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS) 2024-09-21T09:51:06.630 INFO:journalctl@ceph.osd.7.smithi064.stdout:Sep 21 09:51:06 smithi064 podman[120001]: 2024-09-21 09:51:06.333843345 +0000 UTC m=+0.524431473 container died d461a7492425a6334054e8b2861b5fd8eaddced1a1d5e82b551ffdf59baea246 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-osd-7-deactivate, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, RELEASE=main-62da54b, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T09:51:06.936 DEBUG:teuthology.orchestra.run.smithi064:> sudo pkill -f 'journalctl -f -n 0 -u ceph-d65e801e-77fa-11ef-baf6-efdc52797490@osd.7.service' 2024-09-21T09:51:06.978 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T09:51:06.978 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-09-21T09:51:06.978 INFO:tasks.cephadm.mds.a:Stopping mds.a... 2024-09-21T09:51:06.978 DEBUG:teuthology.orchestra.run.smithi022:> sudo systemctl stop ceph-d65e801e-77fa-11ef-baf6-efdc52797490@mds.a 2024-09-21T09:51:07.279 INFO:journalctl@ceph.mds.a.smithi022.stdout:Sep 21 09:51:07 smithi022 systemd[1]: Stopping Ceph mds.a for d65e801e-77fa-11ef-baf6-efdc52797490... 2024-09-21T09:51:07.545 INFO:journalctl@ceph.mds.a.smithi022.stdout:Sep 21 09:51:07 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mds-a[123008]: 2024-09-21T09:51:07.276+0000 7fc7d07b7640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mds -n mds.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T09:51:07.545 INFO:journalctl@ceph.mds.a.smithi022.stdout:Sep 21 09:51:07 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mds-a[123008]: 2024-09-21T09:51:07.277+0000 7fc7d07b7640 -1 mds.a *** got signal Terminated *** 2024-09-21T09:51:11.278 INFO:journalctl@ceph.mds.a.smithi022.stdout:Sep 21 09:51:10 smithi022 podman[133337]: 2024-09-21 09:51:10.982705049 +0000 UTC m=+3.809151747 container died 88fab662525b5563af835556fb30bfb0e941628136c8e9b276067425e5f7466f (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mds-a, CEPH_POINT_RELEASE=, RELEASE=main-62da54b, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T09:51:11.558 INFO:journalctl@ceph.mds.a.smithi022.stdout:Sep 21 09:51:11 smithi022 podman[133337]: 2024-09-21 09:51:11.382838034 +0000 UTC m=+4.209284733 container cleanup 88fab662525b5563af835556fb30bfb0e941628136c8e9b276067425e5f7466f (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mds-a, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , RELEASE=main-62da54b, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T09:51:11.558 INFO:journalctl@ceph.mds.a.smithi022.stdout:Sep 21 09:51:11 smithi022 bash[133337]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mds-a 2024-09-21T09:51:11.771 DEBUG:teuthology.orchestra.run.smithi022:> sudo pkill -f 'journalctl -f -n 0 -u ceph-d65e801e-77fa-11ef-baf6-efdc52797490@mds.a.service' 2024-09-21T09:51:11.807 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T09:51:11.807 INFO:tasks.cephadm.mds.a:Stopped mds.a 2024-09-21T09:51:11.808 INFO:tasks.cephadm.mds.b:Stopping mds.b... 2024-09-21T09:51:11.808 DEBUG:teuthology.orchestra.run.smithi064:> sudo systemctl stop ceph-d65e801e-77fa-11ef-baf6-efdc52797490@mds.b 2024-09-21T09:51:12.152 INFO:journalctl@ceph.mds.b.smithi064.stdout:Sep 21 09:51:11 smithi064 systemd[1]: Stopping Ceph mds.b for d65e801e-77fa-11ef-baf6-efdc52797490... 2024-09-21T09:51:12.419 INFO:journalctl@ceph.mds.b.smithi064.stdout:Sep 21 09:51:12 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mds-b[110160]: 2024-09-21T09:51:12.150+0000 7f193b99c640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mds -n mds.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T09:51:12.419 INFO:journalctl@ceph.mds.b.smithi064.stdout:Sep 21 09:51:12 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mds-b[110160]: 2024-09-21T09:51:12.150+0000 7f193b99c640 -1 mds.b *** got signal Terminated *** 2024-09-21T09:51:14.575 INFO:journalctl@ceph.mds.b.smithi064.stdout:Sep 21 09:51:14 smithi064 podman[120129]: 2024-09-21 09:51:14.287326415 +0000 UTC m=+2.275013963 container died e520bfb5f2948d1e31fc65348bbb963b39eb4ccf21df32b6be89c0ed902e3fd7 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mds-b, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-62da54b, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T09:51:14.916 INFO:journalctl@ceph.mds.b.smithi064.stdout:Sep 21 09:51:14 smithi064 podman[120129]: 2024-09-21 09:51:14.67275384 +0000 UTC m=+2.660441391 container cleanup e520bfb5f2948d1e31fc65348bbb963b39eb4ccf21df32b6be89c0ed902e3fd7 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mds-b, org.label-schema.license=GPLv2, ceph=True, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, RELEASE=main-62da54b, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_CLEAN=True, CEPH_POINT_RELEASE=) 2024-09-21T09:51:14.916 INFO:journalctl@ceph.mds.b.smithi064.stdout:Sep 21 09:51:14 smithi064 bash[120129]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490-mds-b 2024-09-21T09:51:15.063 DEBUG:teuthology.orchestra.run.smithi064:> sudo pkill -f 'journalctl -f -n 0 -u ceph-d65e801e-77fa-11ef-baf6-efdc52797490@mds.b.service' 2024-09-21T09:51:15.100 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T09:51:15.100 INFO:tasks.cephadm.mds.b:Stopped mds.b 2024-09-21T09:51:15.100 INFO:tasks.cephadm.prometheus.a:Stopping prometheus.a... 2024-09-21T09:51:15.100 DEBUG:teuthology.orchestra.run.smithi064:> sudo systemctl stop ceph-d65e801e-77fa-11ef-baf6-efdc52797490@prometheus.a 2024-09-21T09:51:15.169 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:51:15 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:51:15.162Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=nvmeof msg="Unable to refresh target groups" err="Get \"http://172.21.15.22:8765/sd/prometheus/sd-config?service=nvmeof\": dial tcp 172.21.15.22:8765: connect: connection refused" 2024-09-21T09:51:15.170 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:51:15 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:51:15.163Z caller=refresh.go:90 level=error component="discovery manager notify" discovery=http config=config-0 msg="Unable to refresh target groups" err="Get \"http://172.21.15.22:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.22:8765: connect: connection refused" 2024-09-21T09:51:15.170 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:51:15 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:51:15.168Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.22:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.22:8765: connect: connection refused" 2024-09-21T09:51:15.170 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:51:15 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:51:15.168Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=nfs msg="Unable to refresh target groups" err="Get \"http://172.21.15.22:8765/sd/prometheus/sd-config?service=nfs\": dial tcp 172.21.15.22:8765: connect: connection refused" 2024-09-21T09:51:15.170 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:51:15 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:51:15.168Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=smb msg="Unable to refresh target groups" err="Get \"http://172.21.15.22:8765/sd/prometheus/sd-config?service=smb\": dial tcp 172.21.15.22:8765: connect: connection refused" 2024-09-21T09:51:15.170 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:51:15 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:51:15.168Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=node msg="Unable to refresh target groups" err="Get \"http://172.21.15.22:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.22:8765: connect: connection refused" 2024-09-21T09:51:15.439 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:51:15 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:51:15.169Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.22:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.22:8765: connect: connection refused" 2024-09-21T09:51:15.439 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:51:15 smithi064 systemd[1]: Stopping Ceph prometheus.a for d65e801e-77fa-11ef-baf6-efdc52797490... 2024-09-21T09:51:15.439 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:51:15 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:51:15.437Z caller=main.go:964 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-09-21T09:51:15.439 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:51:15 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:51:15.437Z caller=main.go:988 level=info msg="Stopping scrape discovery manager..." 2024-09-21T09:51:15.439 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:51:15 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:51:15.437Z caller=main.go:1002 level=info msg="Stopping notify discovery manager..." 2024-09-21T09:51:15.692 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:51:15 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:51:15.437Z caller=manager.go:177 level=info component="rule manager" msg="Stopping rule manager..." 2024-09-21T09:51:15.692 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:51:15 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:51:15.437Z caller=main.go:984 level=info msg="Scrape discovery manager stopped" 2024-09-21T09:51:15.692 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:51:15 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:51:15.437Z caller=main.go:998 level=info msg="Notify discovery manager stopped" 2024-09-21T09:51:15.692 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:51:15 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:51:15.437Z caller=manager.go:187 level=info component="rule manager" msg="Rule manager stopped" 2024-09-21T09:51:15.692 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:51:15 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:51:15.437Z caller=main.go:1039 level=info msg="Stopping scrape manager..." 2024-09-21T09:51:15.692 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:51:15 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:51:15.438Z caller=main.go:1031 level=info msg="Scrape manager stopped" 2024-09-21T09:51:15.692 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:51:15 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:51:15.439Z caller=notifier.go:618 level=info component=notifier msg="Stopping notification manager..." 2024-09-21T09:51:15.693 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:51:15 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:51:15.439Z caller=main.go:1261 level=info msg="Notifier manager stopped" 2024-09-21T09:51:15.693 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:51:15 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a[78395]: ts=2024-09-21T09:51:15.439Z caller=main.go:1273 level=info msg="See you next time!" 2024-09-21T09:51:15.693 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:51:15 smithi064 podman[120272]: 2024-09-21 09:51:15.448120786 +0000 UTC m=+0.120512663 container died 80629ffa21794f8a19c6f57c79913a56e5e53cf9661806f1399205594ad64556 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:51:16.062 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:51:15 smithi064 podman[120272]: 2024-09-21 09:51:15.809521397 +0000 UTC m=+0.481913265 container cleanup 80629ffa21794f8a19c6f57c79913a56e5e53cf9661806f1399205594ad64556 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:51:16.062 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Sep 21 09:51:15 smithi064 bash[120272]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490-prometheus-a 2024-09-21T09:51:16.200 DEBUG:teuthology.orchestra.run.smithi064:> sudo pkill -f 'journalctl -f -n 0 -u ceph-d65e801e-77fa-11ef-baf6-efdc52797490@prometheus.a.service' 2024-09-21T09:51:16.273 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T09:51:16.274 INFO:tasks.cephadm.prometheus.a:Stopped prometheus.a 2024-09-21T09:51:16.274 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid d65e801e-77fa-11ef-baf6-efdc52797490 --force --keep-logs 2024-09-21T09:51:18.250 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 21 09:51:17 smithi022 systemd[1]: Stopping Ceph alertmanager.a for d65e801e-77fa-11ef-baf6-efdc52797490... 2024-09-21T09:51:18.558 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 21 09:51:18 smithi022 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-alertmanager-a[87373]: ts=2024-09-21T09:51:18.248Z caller=main.go:542 level=info msg="Received SIGTERM, exiting gracefully..." 2024-09-21T09:51:18.558 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 21 09:51:18 smithi022 podman[133801]: 2024-09-21 09:51:18.320233599 +0000 UTC m=+0.168310864 container died caced80ba5448751abdbbc168dbbbe80bfdfc40282230f296db2806bfee957b0 (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-21T09:51:18.940 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 21 09:51:18 smithi022 podman[133801]: 2024-09-21 09:51:18.688211748 +0000 UTC m=+0.536289012 container cleanup caced80ba5448751abdbbc168dbbbe80bfdfc40282230f296db2806bfee957b0 (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-21T09:51:18.941 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 21 09:51:18 smithi022 bash[133801]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490-alertmanager-a 2024-09-21T09:51:19.308 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 21 09:51:18 smithi022 podman[133819]: 2024-09-21 09:51:18.939338748 +0000 UTC m=+0.617610261 container remove caced80ba5448751abdbbc168dbbbe80bfdfc40282230f296db2806bfee957b0 (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-21T09:51:19.308 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 21 09:51:19 smithi022 podman[133819]: 2024-09-21 09:51:19.023338096 +0000 UTC m=+0.701609557 volume remove f6536fdbfc955265bcebe950983bc750c0fae9a39245c8ef86833e82215289ab 2024-09-21T09:51:19.308 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 21 09:51:19 smithi022 systemd[1]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490@alertmanager.a.service: Deactivated successfully. 2024-09-21T09:51:19.309 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 21 09:51:19 smithi022 systemd[1]: Stopped Ceph alertmanager.a for d65e801e-77fa-11ef-baf6-efdc52797490. 2024-09-21T09:51:19.309 INFO:journalctl@ceph.alertmanager.a.smithi022.stdout:Sep 21 09:51:19 smithi022 systemd[1]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490@alertmanager.a.service: Consumed 1.904s CPU time. 2024-09-21T09:51:19.615 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:51:19 smithi022 systemd[1]: Stopping Ceph node-exporter.a for d65e801e-77fa-11ef-baf6-efdc52797490... 2024-09-21T09:51:19.978 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:51:19 smithi022 podman[133933]: 2024-09-21 09:51:19.614156151 +0000 UTC m=+0.118668216 container died 593c6158f524345fa890f79e64eccc692672235f972c563806a4c9320d33195e (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-21T09:51:20.308 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:51:20 smithi022 podman[133933]: 2024-09-21 09:51:20.084352321 +0000 UTC m=+0.588864384 container cleanup 593c6158f524345fa890f79e64eccc692672235f972c563806a4c9320d33195e (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-21T09:51:20.308 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:51:20 smithi022 bash[133933]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a 2024-09-21T09:51:20.662 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:51:20 smithi022 podman[133951]: 2024-09-21 09:51:20.360696265 +0000 UTC m=+0.748757762 container remove 593c6158f524345fa890f79e64eccc692672235f972c563806a4c9320d33195e (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-21T09:51:20.662 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:51:20 smithi022 systemd[1]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-09-21T09:51:20.662 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:51:20 smithi022 systemd[1]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490@node-exporter.a.service: Failed with result 'exit-code'. 2024-09-21T09:51:20.662 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:51:20 smithi022 systemd[1]: Stopped Ceph node-exporter.a for d65e801e-77fa-11ef-baf6-efdc52797490. 2024-09-21T09:51:20.663 INFO:journalctl@ceph.node-exporter.a.smithi022.stdout:Sep 21 09:51:20 smithi022 systemd[1]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490@node-exporter.a.service: Consumed 3.851s CPU time. 2024-09-21T09:51:21.242 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid d65e801e-77fa-11ef-baf6-efdc52797490 --force --keep-logs 2024-09-21T09:51:23.192 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:51:22 smithi064 systemd[1]: Stopping Ceph grafana.a for d65e801e-77fa-11ef-baf6-efdc52797490... 2024-09-21T09:51:23.483 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:51:23 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=server t=2024-09-21T09:51:23.200192673Z level=info msg="Shutdown started" reason="System signal: terminated" 2024-09-21T09:51:23.484 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:51:23 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=tracing t=2024-09-21T09:51:23.200254547Z level=info msg="Closing tracing" 2024-09-21T09:51:23.484 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:51:23 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=ticker t=2024-09-21T09:51:23.200282833Z level=info msg=stopped last_tick=2024-09-21T09:51:20Z 2024-09-21T09:51:23.484 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:51:23 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=grafana-apiserver t=2024-09-21T09:51:23.20030352Z level=info msg="StorageObjectCountTracker pruner is exiting" 2024-09-21T09:51:23.484 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:51:23 smithi064 ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a[114457]: logger=sqlstore.transactions t=2024-09-21T09:51:23.211696676Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-09-21T09:51:23.484 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:51:23 smithi064 podman[120724]: 2024-09-21 09:51:23.220935543 +0000 UTC m=+0.121524368 container died 0f60e823c29b66897b00eb3b410e0d32a44b30f76265da993fa3a4de35425a5e (image=quay.io/ceph/grafana:10.4.8, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a, maintainer=Grafana Labs ) 2024-09-21T09:51:23.858 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:51:23 smithi064 podman[120724]: 2024-09-21 09:51:23.581222379 +0000 UTC m=+0.481811205 container cleanup 0f60e823c29b66897b00eb3b410e0d32a44b30f76265da993fa3a4de35425a5e (image=quay.io/ceph/grafana:10.4.8, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a, maintainer=Grafana Labs ) 2024-09-21T09:51:23.858 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:51:23 smithi064 bash[120724]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a 2024-09-21T09:51:24.191 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:51:23 smithi064 podman[120741]: 2024-09-21 09:51:23.85733032 +0000 UTC m=+0.634265884 container remove 0f60e823c29b66897b00eb3b410e0d32a44b30f76265da993fa3a4de35425a5e (image=quay.io/ceph/grafana:10.4.8, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-grafana-a, maintainer=Grafana Labs ) 2024-09-21T09:51:24.192 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:51:23 smithi064 systemd[1]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490@grafana.a.service: Deactivated successfully. 2024-09-21T09:51:24.192 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:51:23 smithi064 systemd[1]: Stopped Ceph grafana.a for d65e801e-77fa-11ef-baf6-efdc52797490. 2024-09-21T09:51:24.192 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Sep 21 09:51:23 smithi064 systemd[1]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490@grafana.a.service: Consumed 7.629s CPU time. 2024-09-21T09:51:24.449 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:51:24 smithi064 systemd[1]: Stopping Ceph node-exporter.b for d65e801e-77fa-11ef-baf6-efdc52797490... 2024-09-21T09:51:24.721 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:51:24 smithi064 podman[120851]: 2024-09-21 09:51:24.448061587 +0000 UTC m=+0.115604499 container died a533420b0139f835486bd93cc492d96ac0333027426f3c475d08bf722a1f6e61 (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-21T09:51:25.104 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:51:24 smithi064 podman[120851]: 2024-09-21 09:51:24.810288927 +0000 UTC m=+0.477831881 container cleanup a533420b0139f835486bd93cc492d96ac0333027426f3c475d08bf722a1f6e61 (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-21T09:51:25.104 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:51:24 smithi064 bash[120851]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b 2024-09-21T09:51:25.393 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:51:25 smithi064 podman[120868]: 2024-09-21 09:51:25.102965912 +0000 UTC m=+0.658648676 container remove a533420b0139f835486bd93cc492d96ac0333027426f3c475d08bf722a1f6e61 (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-d65e801e-77fa-11ef-baf6-efdc52797490-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-21T09:51:25.393 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:51:25 smithi064 systemd[1]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-09-21T09:51:25.393 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:51:25 smithi064 systemd[1]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490@node-exporter.b.service: Failed with result 'exit-code'. 2024-09-21T09:51:25.393 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:51:25 smithi064 systemd[1]: Stopped Ceph node-exporter.b for d65e801e-77fa-11ef-baf6-efdc52797490. 2024-09-21T09:51:25.394 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Sep 21 09:51:25 smithi064 systemd[1]: ceph-d65e801e-77fa-11ef-baf6-efdc52797490@node-exporter.b.service: Consumed 3.855s CPU time. 2024-09-21T09:51:25.866 DEBUG:teuthology.orchestra.run.smithi022:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-21T09:51:25.901 DEBUG:teuthology.orchestra.run.smithi064:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-21T09:51:25.929 INFO:tasks.cephadm:Archiving crash dumps... 2024-09-21T09:51:25.931 DEBUG:teuthology.misc:Transferring archived files from smithi022:/var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/crash to /home/teuthworker/archive/teuthology-2024-09-21_03:08:26-upgrade-main-distro-default-smithi/7913638/remote/smithi022/crash 2024-09-21T09:51:25.932 DEBUG:teuthology.orchestra.run.smithi022:> sudo tar c -f - -C /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/crash -- . 2024-09-21T09:51:25.987 INFO:teuthology.orchestra.run.smithi022.stderr:tar: /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/crash: Cannot open: No such file or directory 2024-09-21T09:51:25.988 INFO:teuthology.orchestra.run.smithi022.stderr:tar: Error is not recoverable: exiting now 2024-09-21T09:51:25.989 DEBUG:teuthology.misc:Transferring archived files from smithi064:/var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/crash to /home/teuthworker/archive/teuthology-2024-09-21_03:08:26-upgrade-main-distro-default-smithi/7913638/remote/smithi064/crash 2024-09-21T09:51:25.990 DEBUG:teuthology.orchestra.run.smithi064:> sudo tar c -f - -C /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/crash -- . 2024-09-21T09:51:26.027 INFO:teuthology.orchestra.run.smithi064.stderr:tar: /var/lib/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/crash: Cannot open: No such file or directory 2024-09-21T09:51:26.027 INFO:teuthology.orchestra.run.smithi064.stderr:tar: Error is not recoverable: exiting now 2024-09-21T09:51:26.028 INFO:tasks.cephadm:Checking cluster log for badness... 2024-09-21T09:51:26.028 DEBUG:teuthology.orchestra.run.smithi022:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v '\(POOL_APP_NOT_ENABLED\)' | egrep -v OSD_DOWN | egrep -v 'mons down' | egrep -v 'mon down' | egrep -v MON_DOWN | egrep -v 'out of quorum' | egrep -v PG_DEGRADED | egrep -v 'Reduced data availability' | egrep -v 'Degraded data redundancy' | egrep -v OSDMAP_FLAGS | egrep -v 'mons down' | egrep -v 'mon down' | egrep -v MON_DOWN | egrep -v 'out of quorum' | egrep -v PG_AVAILABILITY | head -n 1 2024-09-21T09:51:26.064 INFO:teuthology.orchestra.run.smithi022.stdout:2024-09-21T09:39:49.007831+0000 mon.a (mon.0) 851 : cluster [WRN] Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-09-21T09:51:26.065 WARNING:tasks.cephadm:Found errors (ERR|WRN|SEC) in cluster log 2024-09-21T09:51:26.065 INFO:tasks.cephadm:Compressing logs... 2024-09-21T09:51:26.065 DEBUG:teuthology.orchestra.run.smithi022:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-09-21T09:51:26.107 DEBUG:teuthology.orchestra.run.smithi064:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-09-21T09:51:26.131 INFO:teuthology.orchestra.run.smithi022.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/ceph-osd.0.log 2024-09-21T09:51:26.131 INFO:teuthology.orchestra.run.smithi022.stderr:: No such file or directory 2024-09-21T09:51:26.133 INFO:teuthology.orchestra.run.smithi022.stderr:gzip -5 --verbose -- /var/log/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/ceph-osd.1.log 2024-09-21T09:51:26.133 INFO:teuthology.orchestra.run.smithi022.stderr:/var/log/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/ceph-osd.0.log: gzip -5 --verbose -- /var/log/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/ceph-volume.log 2024-09-21T09:51:26.133 INFO:teuthology.orchestra.run.smithi022.stderr:/var/log/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/ceph-osd.1.log: gzip -5 --verbose -- /var/log/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/ceph.audit.log 2024-09-21T09:51:26.133 INFO:teuthology.orchestra.run.smithi022.stderr:/var/log/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/ceph-volume.log: gzip -5 --verbose -- /var/log/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/ceph-mon.c.log 2024-09-21T09:51:26.133 INFO:teuthology.orchestra.run.smithi022.stderr:/var/log/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/ceph-mds.a.log 2024-09-21T09:51:26.137 INFO:teuthology.orchestra.run.smithi022.stderr:/var/log/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/ceph-mon.c.log: gzip -5 --verbose -- /var/log/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/ceph.cephadm.log 2024-09-21T09:51:26.138 INFO:teuthology.orchestra.run.smithi022.stderr:/var/log/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/ceph-mds.a.log: gzip -5 --verbose -- /var/log/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/ceph-mon.a.log 2024-09-21T09:51:26.138 INFO:teuthology.orchestra.run.smithi022.stderr:/var/log/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/ceph-osd.2.log 2024-09-21T09:51:26.138 INFO:teuthology.orchestra.run.smithi022.stderr:/var/log/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/ceph-mon.a.log: 90.9% -- replaced with /var/log/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/ceph.cephadm.log.gz 2024-09-21T09:51:26.140 INFO:teuthology.orchestra.run.smithi064.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/ceph-osd.7.log 2024-09-21T09:51:26.140 INFO:teuthology.orchestra.run.smithi064.stderr:: No such file or directory 2024-09-21T09:51:26.140 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/ceph-volume.log 2024-09-21T09:51:26.141 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/ceph-mds.b.log 2024-09-21T09:51:26.141 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/ceph-osd.7.log: /var/log/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/ceph-volume.log: gzip -5 --verbose -- /var/log/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/ceph-osd.4.log 2024-09-21T09:51:26.141 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/ceph-mds.b.log: gzip -5 --verbose -- /var/log/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/ceph.audit.log 2024-09-21T09:51:26.141 INFO:teuthology.orchestra.run.smithi064.stderr: 83.6% -- replaced with /var/log/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/ceph-mds.b.log.gz 2024-09-21T09:51:26.141 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/ceph.cephadm.log 2024-09-21T09:51:26.141 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/ceph-osd.4.log: /var/log/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/ceph-osd.6.log 2024-09-21T09:51:26.141 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/ceph-osd.5.log 2024-09-21T09:51:26.141 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/ceph-osd.6.log: gzip -5 --verbose -- /var/log/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/ceph.log 2024-09-21T09:51:26.142 INFO:teuthology.orchestra.run.smithi064.stderr: 84.1% -- replaced with /var/log/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/ceph.cephadm.log.gz 2024-09-21T09:51:26.142 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/ceph-osd.5.log: gzip -5 --verbose -- /var/log/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/ceph-mgr.x.log 2024-09-21T09:51:26.142 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/ceph-mon.b.log 2024-09-21T09:51:26.142 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/ceph-mgr.x.log: /var/log/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/ceph.log: 90.4% -- replaced with /var/log/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/ceph.audit.log.gz 2024-09-21T09:51:26.142 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-09-21T09:51:26.143 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/ceph-mon.b.log: 85.8% -- replaced with /var/log/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/ceph.log.gz 2024-09-21T09:51:26.144 INFO:teuthology.orchestra.run.smithi022.stderr: 94.1% -- replaced with /var/log/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/ceph.audit.log.gz 2024-09-21T09:51:26.145 INFO:teuthology.orchestra.run.smithi022.stderr:gzip -5 --verbose -- /var/log/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/ceph-mgr.y.log 2024-09-21T09:51:26.145 INFO:teuthology.orchestra.run.smithi022.stderr: 91.8% -- replaced with /var/log/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/ceph-mds.a.log.gz 2024-09-21T09:51:26.145 INFO:teuthology.orchestra.run.smithi022.stderr:/var/log/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/ceph-osd.2.log: gzip -5 --verbose -- /var/log/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/ceph.log 2024-09-21T09:51:26.145 INFO:teuthology.orchestra.run.smithi022.stderr:/var/log/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/ceph-mgr.y.log: gzip -5 --verbose -- /var/log/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/ceph-osd.3.log 2024-09-21T09:51:26.152 INFO:teuthology.orchestra.run.smithi022.stderr:/var/log/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/ceph.log: 91.1% -- replaced with /var/log/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/ceph-volume.log.gz 2024-09-21T09:51:26.152 INFO:teuthology.orchestra.run.smithi064.stderr: 91.1% -- replaced with /var/log/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/ceph-volume.log.gz 2024-09-21T09:51:26.152 INFO:teuthology.orchestra.run.smithi022.stderr: 92.0% -- replaced with /var/log/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/ceph.log.gz 2024-09-21T09:51:26.157 INFO:teuthology.orchestra.run.smithi022.stderr:gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-09-21T09:51:26.162 INFO:teuthology.orchestra.run.smithi022.stderr:/var/log/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/ceph-osd.3.log: /var/log/ceph/cephadm.log: 92.2% -- replaced with /var/log/ceph/cephadm.log.gz 2024-09-21T09:51:26.175 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/cephadm.log: 92.0% -- replaced with /var/log/ceph/cephadm.log.gz 2024-09-21T09:51:26.204 INFO:teuthology.orchestra.run.smithi064.stderr: 89.8% -- replaced with /var/log/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/ceph-mgr.x.log.gz 2024-09-21T09:51:26.591 INFO:teuthology.orchestra.run.smithi022.stderr: 89.2% -- replaced with /var/log/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/ceph-mgr.y.log.gz 2024-09-21T09:51:26.916 INFO:teuthology.orchestra.run.smithi064.stderr: 92.1% -- replaced with /var/log/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/ceph-mon.b.log.gz 2024-09-21T09:51:27.022 INFO:teuthology.orchestra.run.smithi022.stderr: 91.8% -- replaced with /var/log/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/ceph-mon.c.log.gz 2024-09-21T09:51:28.226 INFO:teuthology.orchestra.run.smithi022.stderr: 90.7% -- replaced with /var/log/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/ceph-mon.a.log.gz 2024-09-21T09:51:31.179 INFO:teuthology.orchestra.run.smithi064.stderr: 94.3% -- replaced with /var/log/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/ceph-osd.6.log.gz 2024-09-21T09:51:31.276 INFO:teuthology.orchestra.run.smithi064.stderr: 94.1% -- replaced with /var/log/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/ceph-osd.4.log.gz 2024-09-21T09:51:31.842 INFO:teuthology.orchestra.run.smithi064.stderr: 94.1% -- replaced with /var/log/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/ceph-osd.5.log.gz 2024-09-21T09:51:31.890 INFO:teuthology.orchestra.run.smithi064.stderr: 94.3% -- replaced with /var/log/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/ceph-osd.7.log.gz 2024-09-21T09:51:31.892 INFO:teuthology.orchestra.run.smithi064.stderr: 2024-09-21T09:51:31.892 INFO:teuthology.orchestra.run.smithi064.stderr:real 0m5.769s 2024-09-21T09:51:31.892 INFO:teuthology.orchestra.run.smithi064.stderr:user 0m21.769s 2024-09-21T09:51:31.892 INFO:teuthology.orchestra.run.smithi064.stderr:sys 0m0.761s 2024-09-21T09:51:32.345 INFO:teuthology.orchestra.run.smithi022.stderr: 94.0% -- replaced with /var/log/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/ceph-osd.0.log.gz 2024-09-21T09:51:32.961 INFO:teuthology.orchestra.run.smithi022.stderr: 93.8% -- replaced with /var/log/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/ceph-osd.2.log.gz 2024-09-21T09:51:34.193 INFO:teuthology.orchestra.run.smithi022.stderr: 94.5% -- replaced with /var/log/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/ceph-osd.3.log.gz 2024-09-21T09:51:34.749 INFO:teuthology.orchestra.run.smithi022.stderr: 94.3% -- replaced with /var/log/ceph/d65e801e-77fa-11ef-baf6-efdc52797490/ceph-osd.1.log.gz 2024-09-21T09:51:34.751 INFO:teuthology.orchestra.run.smithi022.stderr: 2024-09-21T09:51:34.751 INFO:teuthology.orchestra.run.smithi022.stderr:real 0m8.630s 2024-09-21T09:51:34.751 INFO:teuthology.orchestra.run.smithi022.stderr:user 0m32.060s 2024-09-21T09:51:34.751 INFO:teuthology.orchestra.run.smithi022.stderr:sys 0m1.053s 2024-09-21T09:51:34.752 INFO:tasks.cephadm:Archiving logs... 2024-09-21T09:51:34.752 DEBUG:teuthology.misc:Transferring archived files from smithi022:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-09-21_03:08:26-upgrade-main-distro-default-smithi/7913638/remote/smithi022/log 2024-09-21T09:51:34.753 DEBUG:teuthology.orchestra.run.smithi022:> sudo tar c -f - -C /var/log/ceph -- . 2024-09-21T09:51:38.923 DEBUG:teuthology.misc:Transferring archived files from smithi064:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-09-21_03:08:26-upgrade-main-distro-default-smithi/7913638/remote/smithi064/log 2024-09-21T09:51:38.924 DEBUG:teuthology.orchestra.run.smithi064:> sudo tar c -f - -C /var/log/ceph -- . 2024-09-21T09:51:41.597 INFO:tasks.cephadm:Removing cluster... 2024-09-21T09:51:41.597 DEBUG:teuthology.orchestra.run.smithi022:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid d65e801e-77fa-11ef-baf6-efdc52797490 --force 2024-09-21T09:51:42.112 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid d65e801e-77fa-11ef-baf6-efdc52797490 --force 2024-09-21T09:51:42.643 INFO:tasks.cephadm:Removing cephadm ... 2024-09-21T09:51:42.644 DEBUG:teuthology.orchestra.run.smithi022:> rm -rf /home/ubuntu/cephtest/cephadm 2024-09-21T09:51:42.661 DEBUG:teuthology.orchestra.run.smithi064:> rm -rf /home/ubuntu/cephtest/cephadm 2024-09-21T09:51:42.677 INFO:tasks.cephadm:Teardown complete 2024-09-21T09:51:42.677 DEBUG:teuthology.run_tasks:Unwinding manager install 2024-09-21T09:51:42.687 INFO:teuthology.task.install.util:Removing shipped files: /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer... 2024-09-21T09:51:42.687 DEBUG:teuthology.orchestra.run.smithi022:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2024-09-21T09:51:42.704 DEBUG:teuthology.orchestra.run.smithi064:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2024-09-21T09:51:42.794 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-09-21T09:51:42.810 INFO:teuthology.task.clock:Checking final clock skew... 2024-09-21T09:51:42.810 DEBUG:teuthology.orchestra.run.smithi022:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-21T09:51:42.813 DEBUG:teuthology.orchestra.run.smithi064:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-21T09:51:42.827 INFO:teuthology.orchestra.run.smithi022.stderr:bash: line 1: ntpq: command not found 2024-09-21T09:51:42.827 INFO:teuthology.orchestra.run.smithi064.stderr:bash: line 1: ntpq: command not found 2024-09-21T09:51:42.831 INFO:teuthology.orchestra.run.smithi022.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-21T09:51:42.831 INFO:teuthology.orchestra.run.smithi022.stdout:=============================================================================== 2024-09-21T09:51:42.832 INFO:teuthology.orchestra.run.smithi022.stdout:^* hv01.front.sepia.ceph.com 3 6 377 43 -3037ns[+2506ns] +/- 42ms 2024-09-21T09:51:42.832 INFO:teuthology.orchestra.run.smithi022.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 48 +247us[ +253us] +/- 65ms 2024-09-21T09:51:42.832 INFO:teuthology.orchestra.run.smithi022.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 47 -317us[ -311us] +/- 86ms 2024-09-21T09:51:42.832 INFO:teuthology.orchestra.run.smithi022.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T09:51:42.833 INFO:teuthology.orchestra.run.smithi064.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-21T09:51:42.833 INFO:teuthology.orchestra.run.smithi064.stdout:=============================================================================== 2024-09-21T09:51:42.834 INFO:teuthology.orchestra.run.smithi064.stdout:^* hv01.front.sepia.ceph.com 3 6 377 40 +8905ns[ +11us] +/- 42ms 2024-09-21T09:51:42.834 INFO:teuthology.orchestra.run.smithi064.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 42 +219us[ +221us] +/- 66ms 2024-09-21T09:51:42.834 INFO:teuthology.orchestra.run.smithi064.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 42 -321us[ -319us] +/- 86ms 2024-09-21T09:51:42.834 INFO:teuthology.orchestra.run.smithi064.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T09:51:42.834 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-09-21T09:51:42.869 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-09-21T09:51:42.869 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-09-21T09:51:42.882 DEBUG:teuthology.orchestra.run.smithi022:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-09-21T09:51:42.980 DEBUG:teuthology.orchestra.run.smithi064:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-09-21T09:51:43.061 DEBUG:teuthology.orchestra.run.smithi022:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-21T09:51:43.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:51:43.094 DEBUG:teuthology.orchestra.run.smithi064:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-21T09:51:43.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:51:43.123 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-09-21T09:51:43.133 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-09-21T09:51:43.142 INFO:teuthology.task.internal:Duration was 2156.156051 seconds 2024-09-21T09:51:43.143 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-09-21T09:51:43.153 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-09-21T09:51:43.153 DEBUG:teuthology.orchestra.run.smithi022:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-09-21T09:51:43.156 DEBUG:teuthology.orchestra.run.smithi064:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-09-21T09:51:43.194 INFO:teuthology.orchestra.run.smithi022.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-21T09:51:43.205 INFO:teuthology.orchestra.run.smithi064.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-21T09:51:43.599 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-09-21T09:51:43.599 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi022.front.sepia.ceph.com 2024-09-21T09:51:43.599 DEBUG:teuthology.orchestra.run.smithi022:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-09-21T09:51:43.662 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi064.front.sepia.ceph.com 2024-09-21T09:51:43.662 DEBUG:teuthology.orchestra.run.smithi064:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-09-21T09:51:43.687 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-09-21T09:51:43.687 DEBUG:teuthology.orchestra.run.smithi022:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-09-21T09:51:43.704 DEBUG:teuthology.orchestra.run.smithi064:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-09-21T09:51:43.755 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-09-21T09:51:43.755 DEBUG:teuthology.orchestra.run.smithi022:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-09-21T09:51:43.773 DEBUG:teuthology.orchestra.run.smithi064:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-09-21T09:51:44.221 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-09-21T09:51:44.231 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-09-21T09:51:44.232 DEBUG:teuthology.orchestra.run.smithi022:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-09-21T09:51:44.257 DEBUG:teuthology.orchestra.run.smithi064:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-09-21T09:51:44.284 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-09-21T09:51:44.294 DEBUG:teuthology.orchestra.run.smithi022:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-09-21T09:51:44.300 DEBUG:teuthology.orchestra.run.smithi064:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-09-21T09:51:44.322 INFO:teuthology.orchestra.run.smithi022.stdout:kernel.core_pattern = core 2024-09-21T09:51:44.349 INFO:teuthology.orchestra.run.smithi064.stdout:kernel.core_pattern = core 2024-09-21T09:51:44.400 DEBUG:teuthology.orchestra.run.smithi022:> test -e /home/ubuntu/cephtest/archive/coredump 2024-09-21T09:51:44.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:51:44.416 DEBUG:teuthology.orchestra.run.smithi064:> test -e /home/ubuntu/cephtest/archive/coredump 2024-09-21T09:51:44.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:51:44.456 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-09-21T09:51:44.466 INFO:teuthology.task.internal:Transferring archived files... 2024-09-21T09:51:44.467 DEBUG:teuthology.misc:Transferring archived files from smithi022:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-09-21_03:08:26-upgrade-main-distro-default-smithi/7913638/remote/smithi022 2024-09-21T09:51:44.467 DEBUG:teuthology.orchestra.run.smithi022:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-09-21T09:51:44.511 DEBUG:teuthology.misc:Transferring archived files from smithi064:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-09-21_03:08:26-upgrade-main-distro-default-smithi/7913638/remote/smithi064 2024-09-21T09:51:44.511 DEBUG:teuthology.orchestra.run.smithi064:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-09-21T09:51:44.550 INFO:teuthology.task.internal:Removing archive directory... 2024-09-21T09:51:44.550 DEBUG:teuthology.orchestra.run.smithi022:> rm -rf -- /home/ubuntu/cephtest/archive 2024-09-21T09:51:44.552 DEBUG:teuthology.orchestra.run.smithi064:> rm -rf -- /home/ubuntu/cephtest/archive 2024-09-21T09:51:44.599 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-09-21T09:51:44.611 INFO:teuthology.task.internal:Not uploading archives. 2024-09-21T09:51:44.611 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-09-21T09:51:44.621 INFO:teuthology.task.internal:Tidying up after the test... 2024-09-21T09:51:44.622 DEBUG:teuthology.orchestra.run.smithi022:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-09-21T09:51:44.624 DEBUG:teuthology.orchestra.run.smithi064:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-09-21T09:51:44.637 INFO:teuthology.orchestra.run.smithi022.stdout: 262169 4 drwxr-xr-x 3 ubuntu ubuntu 4096 Sep 21 09:51 /home/ubuntu/cephtest 2024-09-21T09:51:44.638 INFO:teuthology.orchestra.run.smithi022.stdout: 539195 4 drwxr-xr-x 3 ubuntu ubuntu 4096 Sep 21 09:45 /home/ubuntu/cephtest/mnt.0 2024-09-21T09:51:44.638 INFO:teuthology.orchestra.run.smithi022.stdout: 539196 4 drwxr-xr-x 3 ubuntu ubuntu 4096 Sep 21 09:49 /home/ubuntu/cephtest/mnt.0/client.0 2024-09-21T09:51:44.638 INFO:teuthology.orchestra.run.smithi022.stdout: 655361 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Sep 21 09:49 /home/ubuntu/cephtest/mnt.0/client.0/tmp 2024-09-21T09:51:44.639 INFO:teuthology.orchestra.run.smithi022.stderr:rmdir: failed to remove '/home/ubuntu/cephtest': Directory not empty 2024-09-21T09:51:44.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:51:44.642 ERROR:teuthology.run_tasks:Manager failed: internal.base Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 160, in run_tasks suppress = manager.__exit__(*exc_info) File "/usr/lib/python3.10/contextlib.py", line 142, in __exit__ next(self.gen) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/internal/__init__.py", line 53, in base run.wait( File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 479, in wait proc.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed on smithi022 with status 1: 'find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest' 2024-09-21T09:51:44.648 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-09-21T09:51:44.659 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-09-21T09:51:44.669 DEBUG:teuthology.run_tasks:Exception was not quenched, exiting: CommandFailedError: Command failed on smithi022 with status 1: 'find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest' 2024-09-21T09:51:44.673 INFO:teuthology.run:Summary data: description: upgrade/quincy-x/parallel/{0-random-distro$/{centos_9.stream_runc} 0-start 1-tasks mon_election/classic upgrade-sequence workload/{ec-rados-default rados_api rados_loadgenbig rbd_import_export test_rbd_api test_rbd_python}} duration: 2156.1560509204865 failure_reason: 'Command failed (workunit test cls/test_cls_rbd.sh) on smithi022 with status 1: ''mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh''' flavor: default owner: scheduled_teuthology@teuthology sentry_event: https://sentry.ceph.com/organizations/ceph/?query=7193baab3b0642829283906d11ea9e22 status: fail success: false 2024-09-21T09:51:44.673 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-09-21T09:51:44.680 INFO:teuthology.orchestra.run.smithi064.stdout: 262169 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Sep 21 09:51 /home/ubuntu/cephtest 2024-09-21T09:51:44.783 INFO:teuthology.run:FAIL